From 6c074197976a113f98ebd05a35ffb0590e3702fe Mon Sep 17 00:00:00 2001 From: Minteck Date: Sun, 6 Mar 2022 15:31:09 +0100 Subject: Fix: resolves #17 --- update/ponies/dict.js | 0 update/ponies/index.js | 0 update/ponies/infobox.js | 0 update/ponies/listgen.js | 96 ++++++++-------- update/ponies/pages.js | 0 update/ponies/parse.js | 278 +++++++++++++++++++++++------------------------ 6 files changed, 187 insertions(+), 187 deletions(-) mode change 100644 => 100755 update/ponies/dict.js mode change 100644 => 100755 update/ponies/index.js mode change 100644 => 100755 update/ponies/infobox.js mode change 100644 => 100755 update/ponies/listgen.js mode change 100644 => 100755 update/ponies/pages.js mode change 100644 => 100755 update/ponies/parse.js (limited to 'update/ponies') diff --git a/update/ponies/dict.js b/update/ponies/dict.js old mode 100644 new mode 100755 diff --git a/update/ponies/index.js b/update/ponies/index.js old mode 100644 new mode 100755 diff --git a/update/ponies/infobox.js b/update/ponies/infobox.js old mode 100644 new mode 100755 diff --git a/update/ponies/listgen.js b/update/ponies/listgen.js old mode 100644 new mode 100755 index 4e07607..46c77f7 --- a/update/ponies/listgen.js +++ b/update/ponies/listgen.js @@ -1,49 +1,49 @@ -/* - * MIT License - * - * Copyright (c) 2022- Minteck - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - */ - -const axios = require("axios"); -const fs = require("fs"); - -if (fs.existsSync("./data")) fs.rmSync("./data", {recursive: true}); -fs.mkdirSync("./data"); - -(async () => { - async function getCategory(category) { - console.log("Category:" + category); - let cat = (await axios.get("https://mlp.fandom.com/api.php?action=query&generator=categorymembers&gcmtitle=Category:" + encodeURI(category) + "&prop=categories&cllimit=max&gcmlimit=max&format=json")).data; - - return Object.keys(cat.query.pages).map(k => cat.query.pages[k].title).filter(k => !k.startsWith("List") && !k.includes("EG") && !k.toLowerCase().includes("ponies") && !k.includes(" and ") && !k.includes("(") && !k.includes("family")); - } - - let list = [...new Set([ - ...(await getCategory("Pegasus ponies")), - ...(await getCategory("Alicorn ponies")), - ...(await getCategory("Earth ponies")), - ...(await getCategory("Unicorn ponies")), - ...(await getCategory("Main characters")), - ...(await getCategory("Dragons")), - ])]; - fs.writeFileSync("./data/list.json", JSON.stringify(list, null, 4)) +/* + * MIT License + * + * Copyright (c) 2022- Minteck + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + */ + +const axios = require("axios"); +const fs = require("fs"); + +if (fs.existsSync("./data")) fs.rmSync("./data", {recursive: true}); +fs.mkdirSync("./data"); + +(async () => { + async function getCategory(category) { + console.log("Category:" + category); + let cat = (await axios.get("https://mlp.fandom.com/api.php?action=query&generator=categorymembers&gcmtitle=Category:" + encodeURI(category) + "&prop=categories&cllimit=max&gcmlimit=max&format=json")).data; + + return Object.keys(cat.query.pages).map(k => cat.query.pages[k].title).filter(k => !k.startsWith("List") && !k.includes("EG") && !k.toLowerCase().includes("ponies") && !k.includes(" and ") && !k.includes("(") && !k.includes("family")); + } + + let list = [...new Set([ + ...(await getCategory("Pegasus ponies")), + ...(await getCategory("Alicorn ponies")), + ...(await getCategory("Earth ponies")), + ...(await getCategory("Unicorn ponies")), + ...(await getCategory("Main characters")), + ...(await getCategory("Dragons")), + ])]; + fs.writeFileSync("./data/list.json", JSON.stringify(list, null, 4)) })() \ No newline at end of file diff --git a/update/ponies/pages.js b/update/ponies/pages.js old mode 100644 new mode 100755 diff --git a/update/ponies/parse.js b/update/ponies/parse.js old mode 100644 new mode 100755 index 8cf797d..71ce007 --- a/update/ponies/parse.js +++ b/update/ponies/parse.js @@ -1,140 +1,140 @@ -/* - * MIT License - * - * Copyright (c) 2022- Minteck - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - */ - -const fs = require('fs'); -const axios = require('axios'); - -console.log("Parsing infobox data..."); - -let ponies = {}; - -(async () => { - for (let title in JSON.parse(fs.readFileSync("./data/boxes.json").toString())) { - console.log("Parsing " + title + "..."); - let box = JSON.parse(fs.readFileSync("./data/boxes.json").toString())[title]; - let data = { - names: [title], - extract: "", - extract_fr: "", - generation: -1, - color: "000000", - image: "https://example.com", - kind: "Pony", - sex: "Unknown", - occupation: ["Unknown"], - residence: ["Unknown"], - mark: "https://example.com" - } - - if (typeof box._gen !== "undefined") data.generation = box._gen; - - if (typeof box.name2 !== "undefined") data.names.push(box.name2.replace(//g, "")); - if (typeof box.name3 !== "undefined") data.names.push(box.name3.replace(//g, "")); - if (typeof box.name4 !== "undefined") data.names.push(box.name4.replace(//g, "")); - if (typeof box.name5 !== "undefined") data.names.push(box.name5.replace(//g, "")); - - if (typeof box.nicknames !== "undefined") { - box.nicknames.replace(//g, "").split(",").filter(e => !e.match(/[^a-zA-Z0-9-_ ]/gm)).forEach((e, i) => { - data.names.push(e.trim()); - }); - } - - if (typeof box.kind !== "undefined") { - kp = box.kind.replace(//g, "").replace(/[^a-zA-Z0-9-_ ]/gm, "").split(" ")[0]; - data.kind = kp.substring(kp.replace(/([A-Z])([a-z0-9]*)$/g, "").length); - } - if (typeof box.sex !== "undefined") data.sex = box.sex.replace(//g, "").trim().startsWith("F") ? "F" : "M"; - if (typeof box._extract !== "undefined") { - data.extract = box._extract; - } - if (typeof box.coat !== "undefined") data.color = box.coat.replace(//g, "").trim().replace(/\[([a-z.\/ \nA-Z0-9:]*)\/(.{6})\/ (.*)\]/gm, "$2").replace(/{{perbang\|([0-9A-Fa-f].{5})(.*)/g, "$1"); - try { - if (typeof box.main !== "undefined") data.image = (await axios.head("https://mlp.fandom.com/Special:FilePath/" + encodeURI(box.main.replace(//g, "").trim()).replaceAll("?", "%3F").replaceAll("&", "%26"))).request.res.responseUrl; - } catch (e) { - try { - if (typeof box.main !== "undefined") data.image = e.request.res.responseUrl; - } catch (e2) { - console.error(e2); - throw e; - } - } - try { - if (typeof box.main1 !== "undefined") data.image = (await axios.head("https://mlp.fandom.com/Special:FilePath/" + encodeURI(box.main1.replace(//g, "").trim()).replaceAll("?", "%3F").replaceAll("&", "%26"))).request.res.responseUrl; - } catch (e) { - try { - if (typeof box.main1 !== "undefined") data.image = e.request.res.responseUrl; - } catch (e2) { - console.error(e2); - throw e; - } - } - try { - if (typeof box.image !== "undefined") data.image = (await axios.head("https://mlp.fandom.com/Special:FilePath/" + encodeURI(box.image.replace(//g, "").trim()).replaceAll("?", "%3F").replaceAll("&", "%26"))).request.res.responseUrl; - } catch (e) { - try { - if (typeof box.image !== "undefined") data.image = e.request.res.responseUrl; - } catch (e2) { - console.error(e2); - throw e; - } - } - if (typeof box["cutie mark"] !== "undefined") { - try { - data.markimg = box["cutie mark"].replace(//g, "").trim().split("[[File:")[1].split("|")[0]; - } catch (e) { - data.markimg = box["cutie mark"].replace(//g, "").trim().split("[[File:")[0].split("|")[0]; - } - try { - data.mark = (await axios.head("https://mlp.fandom.com/Special:FilePath/" + encodeURI(data.markimg).replaceAll("?", "%3F").replaceAll("&", "%26"))).request.res.responseUrl; - } catch (e) { - try { - data.mark = e.request.res.responseUrl; - } catch (e2) { - console.error(e2); - throw e; - } - } - } - - if (typeof box.occupation !== "undefined") { - occupations = []; - box.occupation/*.replace(/\[\[(.*)_(.*)\]\]|\[\[(.*)\|(.*)\]\]|\[\[(.*)\]\]/gm, "$2$4$5")*/.replace(//g, "").trim().replace(/\|/gm, "_").replace(/<( ||(|| )\/)( ||(|| )\/)(b|B)(r|R)( ||(|| )\/)( ||(|| )\/)>/gm, "|").replace(/( \|| \| | \| )/gm, "|").split("|").forEach((e) => { - occupations.push(e.trim().replace(/[\[\]]/gm, "").replace(/<(.*)>/gm, "").replace(/([a-zA-Z0-9 \-_,.'"]*)_([a-zA-Z0-9 \-_,.'"]*)/gm, "$2")); - }) - data.occupation = occupations; - } - if (typeof box.residence !== "undefined") { - residences = []; - box.residence/*.replace(/\[\[(.*)_(.*)\]\]|\[\[(.*)\|(.*)\]\]|\[\[(.*)\]\]/gm, "$2$4$5")*/.replace(//g, "").trim().replace(/\|/gm, "_").replace(/<( ||(|| )\/)( ||(|| )\/)(b|B)(r|R)( ||(|| )\/)( ||(|| )\/)>/gm, "|").replace(/( \|| \| | \| )/gm, "|").split("|").forEach((e) => { - residences.push(e.trim().replace(/[\[\]]/gm, "").replace(/<(.*)>/gm, "").replace(/([a-zA-Z0-9 \-_,.'"]*)_([a-zA-Z0-9 \-_,.'"]*)/gm, "$2")); - }) - data.residence = residences.replaceAll("Locations#", ""); - } - - if ((typeof box.name2 !== "undefined" || typeof box.name3 !== "undefined" || typeof box.name4 !== "undefined" || typeof box.name5 !== "undefined" || typeof box.coat !== "undefined" || typeof box.occupation !== "undefined" || typeof box.residence !== "undefined") && typeof box.sex !== "undefined") ponies[title] = data; - } -})() - +/* + * MIT License + * + * Copyright (c) 2022- Minteck + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + */ + +const fs = require('fs'); +const axios = require('axios'); + +console.log("Parsing infobox data..."); + +let ponies = {}; + +(async () => { + for (let title in JSON.parse(fs.readFileSync("./data/boxes.json").toString())) { + console.log("Parsing " + title + "..."); + let box = JSON.parse(fs.readFileSync("./data/boxes.json").toString())[title]; + let data = { + names: [title], + extract: "", + extract_fr: "", + generation: -1, + color: "000000", + image: "https://example.com", + kind: "Pony", + sex: "Unknown", + occupation: ["Unknown"], + residence: ["Unknown"], + mark: "https://example.com" + } + + if (typeof box._gen !== "undefined") data.generation = box._gen; + + if (typeof box.name2 !== "undefined") data.names.push(box.name2.replace(//g, "")); + if (typeof box.name3 !== "undefined") data.names.push(box.name3.replace(//g, "")); + if (typeof box.name4 !== "undefined") data.names.push(box.name4.replace(//g, "")); + if (typeof box.name5 !== "undefined") data.names.push(box.name5.replace(//g, "")); + + if (typeof box.nicknames !== "undefined") { + box.nicknames.replace(//g, "").split(",").filter(e => !e.match(/[^a-zA-Z0-9-_ ]/gm)).forEach((e, i) => { + data.names.push(e.trim()); + }); + } + + if (typeof box.kind !== "undefined") { + kp = box.kind.replace(//g, "").replace(/[^a-zA-Z0-9-_ ]/gm, "").split(" ")[0]; + data.kind = kp.substring(kp.replace(/([A-Z])([a-z0-9]*)$/g, "").length); + } + if (typeof box.sex !== "undefined") data.sex = box.sex.replace(//g, "").trim().startsWith("F") ? "F" : "M"; + if (typeof box._extract !== "undefined") { + data.extract = box._extract; + } + if (typeof box.coat !== "undefined") data.color = box.coat.replace(//g, "").trim().replace(/\[([a-z.\/ \nA-Z0-9:]*)\/(.{6})\/ (.*)\]/gm, "$2").replace(/{{perbang\|([0-9A-Fa-f].{5})(.*)/g, "$1"); + try { + if (typeof box.main !== "undefined") data.image = (await axios.head("https://mlp.fandom.com/Special:FilePath/" + encodeURI(box.main.replace(//g, "").trim()).replaceAll("?", "%3F").replaceAll("&", "%26"))).request.res.responseUrl; + } catch (e) { + try { + if (typeof box.main !== "undefined") data.image = e.request.res.responseUrl; + } catch (e2) { + console.error(e2); + throw e; + } + } + try { + if (typeof box.main1 !== "undefined") data.image = (await axios.head("https://mlp.fandom.com/Special:FilePath/" + encodeURI(box.main1.replace(//g, "").trim()).replaceAll("?", "%3F").replaceAll("&", "%26"))).request.res.responseUrl; + } catch (e) { + try { + if (typeof box.main1 !== "undefined") data.image = e.request.res.responseUrl; + } catch (e2) { + console.error(e2); + throw e; + } + } + try { + if (typeof box.image !== "undefined") data.image = (await axios.head("https://mlp.fandom.com/Special:FilePath/" + encodeURI(box.image.replace(//g, "").trim()).replaceAll("?", "%3F").replaceAll("&", "%26"))).request.res.responseUrl; + } catch (e) { + try { + if (typeof box.image !== "undefined") data.image = e.request.res.responseUrl; + } catch (e2) { + console.error(e2); + throw e; + } + } + if (typeof box["cutie mark"] !== "undefined") { + try { + data.markimg = box["cutie mark"].replace(//g, "").trim().split("[[File:")[1].split("|")[0]; + } catch (e) { + data.markimg = box["cutie mark"].replace(//g, "").trim().split("[[File:")[0].split("|")[0]; + } + try { + data.mark = (await axios.head("https://mlp.fandom.com/Special:FilePath/" + encodeURI(data.markimg).replaceAll("?", "%3F").replaceAll("&", "%26"))).request.res.responseUrl; + } catch (e) { + try { + data.mark = e.request.res.responseUrl; + } catch (e2) { + console.error(e2); + throw e; + } + } + } + + if (typeof box.occupation !== "undefined") { + occupations = []; + box.occupation/*.replace(/\[\[(.*)_(.*)\]\]|\[\[(.*)\|(.*)\]\]|\[\[(.*)\]\]/gm, "$2$4$5")*/.replace(//g, "").trim().replace(/\|/gm, "_").replace(/<( ||(|| )\/)( ||(|| )\/)(b|B)(r|R)( ||(|| )\/)( ||(|| )\/)>/gm, "|").replace(/( \|| \| | \| )/gm, "|").split("|").forEach((e) => { + occupations.push(e.trim().replace(/[\[\]]/gm, "").replace(/<(.*)>/gm, "").replace(/([a-zA-Z0-9 \-_,.'"]*)_([a-zA-Z0-9 \-_,.'"]*)/gm, "$2")); + }) + data.occupation = occupations; + } + if (typeof box.residence !== "undefined") { + residences = []; + box.residence/*.replace(/\[\[(.*)_(.*)\]\]|\[\[(.*)\|(.*)\]\]|\[\[(.*)\]\]/gm, "$2$4$5")*/.replace(//g, "").trim().replace(/\|/gm, "_").replace(/<( ||(|| )\/)( ||(|| )\/)(b|B)(r|R)( ||(|| )\/)( ||(|| )\/)>/gm, "|").replace(/( \|| \| | \| )/gm, "|").split("|").forEach((e) => { + residences.push(e.trim().replace(/[\[\]]/gm, "").replace(/<(.*)>/gm, "").replace(/([a-zA-Z0-9 \-_,.'"]*)_([a-zA-Z0-9 \-_,.'"]*)/gm, "$2")); + }) + data.residence = residences.replaceAll("Locations#", ""); + } + + if ((typeof box.name2 !== "undefined" || typeof box.name3 !== "undefined" || typeof box.name4 !== "undefined" || typeof box.name5 !== "undefined" || typeof box.coat !== "undefined" || typeof box.occupation !== "undefined" || typeof box.residence !== "undefined") && typeof box.sex !== "undefined") ponies[title] = data; + } +})() + fs.writeFileSync("./data/data.json", JSON.stringify(ponies, null, 4)); \ No newline at end of file -- cgit