diff options
Diffstat (limited to 'update')
| -rwxr-xr-x[-rw-r--r--] | update/episodes/index.js | 336 | ||||
| -rwxr-xr-x[-rw-r--r--] | update/episodes/series.json | 252 | ||||
| -rwxr-xr-x[-rw-r--r--] | update/ponies/dict.js | 0 | ||||
| -rwxr-xr-x[-rw-r--r--] | update/ponies/index.js | 0 | ||||
| -rwxr-xr-x[-rw-r--r--] | update/ponies/infobox.js | 0 | ||||
| -rwxr-xr-x[-rw-r--r--] | update/ponies/listgen.js | 96 | ||||
| -rwxr-xr-x[-rw-r--r--] | update/ponies/pages.js | 0 | ||||
| -rwxr-xr-x[-rw-r--r--] | update/ponies/parse.js | 278 |
8 files changed, 481 insertions, 481 deletions
diff --git a/update/episodes/index.js b/update/episodes/index.js index 1ae0ec3..611b1c1 100644..100755 --- a/update/episodes/index.js +++ b/update/episodes/index.js @@ -1,169 +1,169 @@ -/* - * MIT License - * - * Copyright (c) 2022- Minteck - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - */ - -const config = require("./series.json"); -const axios = require('axios'); -const fs = require('fs'); -const WikiTextParser = require('parse-wikitext'); -const parser = new WikiTextParser("mlp.fandom.com"); - -(async () => { - let all = []; - - for (let series of config) { - let data = { - command: series.cmd, - title: series.title, - date: series.date, - link: series.page, - description: { - series: null, - plot: null, - }, - seasons: [] - }; - console.log(series.title + " (" + data.date + ")"); - - ddata = (await axios.get("https://mlp.fandom.com/api.php?format=json&action=query&prop=extracts&exlimit=max&explaintext&exintro&titles=" + encodeURI(series.page) + "&redirects=")).data; - data.description.series = ddata.query.pages[Object.keys(ddata.query.pages)[0]].extract.split("\n")[0]; - data.description.plot = ddata.query.pages[Object.keys(ddata.query.pages)[0]].extract.split("\n")[1]; - - for (let s of series.seasons) { - console.log(" Season " + s.id); - - let season = { - name: "Season " + s.id, - id: "s" + s.id, - date: null, - count: null, - episodes: [] - } - - let sdata = (await axios.get("https://mlp.fandom.com/api.php?action=query&prop=revisions&titles=" + s.table.page + "&rvslots=*&rvprop=content&formatversion=2&format=json")).data; - let sections = parser.pageToSectionObject(sdata.query.pages[0].revisions[0].slots.main.content); - let episodes = sections.Episodes[s.table.section].content - .filter(i => i.startsWith("|")) - .join("||") - .replace("|}", "") - .split("|-").map(i => { - return i.split("||").filter(i => i.trim() !== ""); - }) - .filter(i => i.length > 1) - .map(i => { - let index = 0; - return i.map(i => { - return i.replace(/^(\| |\|)(.*)/gm, "$2") - .replace(/^style="(.*)"( |)\|( |)(.*)/gm, "$4") - .replace(/\[\[((.*)\|(.*)|(.*))\]\]/gm, "$3$4"); - }).map(i => { - if (index === 0) { - index++; - return i.replace(/(.*) \((.*)\)/gm, "$1|$2"); - } else { - index++; - return i; - } - }) - }); - - index = 1; - for (let e of episodes) { - process.stdout.write(" S" + s.id + "E" + index); - - let episode = { - local: null, - global: null, - date: null, - name: null, - writer: null, - cover: null, - characters: [], - plot: null, - page: null - } - - episode.local = e[s.table.fields.number].replace(/(.*)\|(.*)/gm, "$1").replace(/''(.*)''/gm, "$1").replace(/{{(.*)\|(.*)}}/gm, "$2").replace(/(.*)\|(.*)/gm, "$2"); - episode.global = e[s.table.fields.number].replace(/(.*)\|(.*)/gm, "$2"); - episode.date_pre = e[s.table.fields.date].replace(/{{(.*)\|(.*)\|(.*)\|link=(no|yes)}}/gm, "$3").replace(/{{(.*)\|(.*)}}/gm, "$2").replace(/(.*)\|(.*)/gm, "$2"); - try { - episode.date = new Date(e[s.table.fields.date].replace(/{{(.*)\|(.*)\|(.*)\|link=(no|yes)}}/gm, "$3").replace(/{{(.*)\|(.*)}}/gm, "$2").replace(/(.*)\|(.*)/gm, "$2")).toISOString(); - } catch (e) { - episode.date = null; - } - episode.name = e[s.table.fields.title]; - episode.writer = e[s.table.fields.writer].replace(/{{(.*)\|(.*)\|(.*)\|link=(no|yes)}}/gm, "$3").replace(/(]]|\[\[)/gm, "").replace(/(.*)\|(.*)/gm, "$2"); - - process.stdout.write(": " + episode.name + "\n"); - - if (episode.local.length !== 2 || episode.global.length > 3 || episode.global.length < 2 || episode.date_pre.length !== 10) { - console.log("FAILURE: " + JSON.stringify(episode)); - } - - let edata = (await axios.get("https://mlp.fandom.com/api.php?action=query&prop=revisions&titles=" + encodeURI(episode.name).replaceAll("?", "%3F").replaceAll("&", "%26") + "&rvslots=*&rvprop=content&formatversion=2&format=json")).data; - let eext = (await axios.get("https://mlp.fandom.com/api.php?format=json&action=query&prop=extracts&exlimit=max&explaintext&exintro&titles=" + encodeURI(episode.name).replaceAll("?", "%3F").replaceAll("&", "%26") + "&redirects=")).data; - let efull = (await axios.get("https://mlp.fandom.com/api.php?format=json&action=query&prop=extracts&explaintext&titles=" + encodeURI(episode.name).replaceAll("?", "%3F").replaceAll("&", "%26") + "&redirects=")).data; - - episode.plot = eext.query.pages[Object.keys(eext.query.pages)[0]].extract.replace(/(.*)\n(.*)/, "$2").replace(/(.*)\n\n(.*)/gm, "$2").replace(/([.?!])\s*(?=[A-Z])/g, "$1|").split("|").join(" "); - episode.page = efull.query.pages[Object.keys(eext.query.pages)[0]].extract - - let infobox = null; - try { - sections = parser.pageToSectionObject(edata.query.pages[0].revisions[0].slots.main.content); - box = parser.parseInfoBox(sections["content"]); - if (box.template === "Infobox episode") { - infobox = parser.parseInfoBox(sections["content"]).values; - } - } catch (e) { - console.log(edata); - throw e; - } - - if (infobox !== null) { - try { - if (typeof infobox.image !== "undefined") episode.cover = (await axios.head("https://mlp.fandom.com/Special:FilePath/" + encodeURI(infobox.image.replace(/<!--[\s\S]*?-->/g, "").trim()).replaceAll("?", "%3F").replaceAll("&", "%26"))).request.res.responseUrl; - } catch (e) { - try { - if (typeof infobox.image !== "undefined") episode.cover = e.request.res.responseUrl; - } catch (e2) { - console.error(e2); - throw e; - } - } - if (typeof infobox.featured !== "undefined") episode.characters = infobox.featured.replace(/\[\[(([a-zA-Z0-9 .\-# _\\\/]*)\|([a-zA-Z0-9 .\-_\\\/]*)|([a-zA-Z0-9 .\-#_\\\/]*))\]\]/gm, "$3$4").replace(/(<(\/| |)(\/| |)br(\/| |)(\/| |)>)/gm, "|||").split("|||") - } - - season.episodes.push(episode); - index++; - } - - season.count = season.episodes.length; - season.date = season.episodes[0].date; - data.seasons.push(season); - } - all.push(data); - } - - fs.writeFileSync("./data/series.json", JSON.stringify(all, null, 4)); +/*
+ * MIT License
+ *
+ * Copyright (c) 2022- Minteck
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+const config = require("./series.json");
+const axios = require('axios');
+const fs = require('fs');
+const WikiTextParser = require('parse-wikitext');
+const parser = new WikiTextParser("mlp.fandom.com");
+
+(async () => {
+ let all = [];
+
+ for (let series of config) {
+ let data = {
+ command: series.cmd,
+ title: series.title,
+ date: series.date,
+ link: series.page,
+ description: {
+ series: null,
+ plot: null,
+ },
+ seasons: []
+ };
+ console.log(series.title + " (" + data.date + ")");
+
+ ddata = (await axios.get("https://mlp.fandom.com/api.php?format=json&action=query&prop=extracts&exlimit=max&explaintext&exintro&titles=" + encodeURI(series.page) + "&redirects=")).data;
+ data.description.series = ddata.query.pages[Object.keys(ddata.query.pages)[0]].extract.split("\n")[0];
+ data.description.plot = ddata.query.pages[Object.keys(ddata.query.pages)[0]].extract.split("\n")[1];
+
+ for (let s of series.seasons) {
+ console.log(" Season " + s.id);
+
+ let season = {
+ name: "Season " + s.id,
+ id: "s" + s.id,
+ date: null,
+ count: null,
+ episodes: []
+ }
+
+ let sdata = (await axios.get("https://mlp.fandom.com/api.php?action=query&prop=revisions&titles=" + s.table.page + "&rvslots=*&rvprop=content&formatversion=2&format=json")).data;
+ let sections = parser.pageToSectionObject(sdata.query.pages[0].revisions[0].slots.main.content);
+ let episodes = sections.Episodes[s.table.section].content
+ .filter(i => i.startsWith("|"))
+ .join("||")
+ .replace("|}", "")
+ .split("|-").map(i => {
+ return i.split("||").filter(i => i.trim() !== "");
+ })
+ .filter(i => i.length > 1)
+ .map(i => {
+ let index = 0;
+ return i.map(i => {
+ return i.replace(/^(\| |\|)(.*)/gm, "$2")
+ .replace(/^style="(.*)"( |)\|( |)(.*)/gm, "$4")
+ .replace(/\[\[((.*)\|(.*)|(.*))\]\]/gm, "$3$4");
+ }).map(i => {
+ if (index === 0) {
+ index++;
+ return i.replace(/(.*) \((.*)\)/gm, "$1|$2");
+ } else {
+ index++;
+ return i;
+ }
+ })
+ });
+
+ index = 1;
+ for (let e of episodes) {
+ process.stdout.write(" S" + s.id + "E" + index);
+
+ let episode = {
+ local: null,
+ global: null,
+ date: null,
+ name: null,
+ writer: null,
+ cover: null,
+ characters: [],
+ plot: null,
+ page: null
+ }
+
+ episode.local = e[s.table.fields.number].replace(/(.*)\|(.*)/gm, "$1").replace(/''(.*)''/gm, "$1").replace(/{{(.*)\|(.*)}}/gm, "$2").replace(/(.*)\|(.*)/gm, "$2");
+ episode.global = e[s.table.fields.number].replace(/(.*)\|(.*)/gm, "$2");
+ episode.date_pre = e[s.table.fields.date].replace(/{{(.*)\|(.*)\|(.*)\|link=(no|yes)}}/gm, "$3").replace(/{{(.*)\|(.*)}}/gm, "$2").replace(/(.*)\|(.*)/gm, "$2");
+ try {
+ episode.date = new Date(e[s.table.fields.date].replace(/{{(.*)\|(.*)\|(.*)\|link=(no|yes)}}/gm, "$3").replace(/{{(.*)\|(.*)}}/gm, "$2").replace(/(.*)\|(.*)/gm, "$2")).toISOString();
+ } catch (e) {
+ episode.date = null;
+ }
+ episode.name = e[s.table.fields.title];
+ episode.writer = e[s.table.fields.writer].replace(/{{(.*)\|(.*)\|(.*)\|link=(no|yes)}}/gm, "$3").replace(/(]]|\[\[)/gm, "").replace(/(.*)\|(.*)/gm, "$2");
+
+ process.stdout.write(": " + episode.name + "\n");
+
+ if (episode.local.length !== 2 || episode.global.length > 3 || episode.global.length < 2 || episode.date_pre.length !== 10) {
+ console.log("FAILURE: " + JSON.stringify(episode));
+ }
+
+ let edata = (await axios.get("https://mlp.fandom.com/api.php?action=query&prop=revisions&titles=" + encodeURI(episode.name).replaceAll("?", "%3F").replaceAll("&", "%26") + "&rvslots=*&rvprop=content&formatversion=2&format=json")).data;
+ let eext = (await axios.get("https://mlp.fandom.com/api.php?format=json&action=query&prop=extracts&exlimit=max&explaintext&exintro&titles=" + encodeURI(episode.name).replaceAll("?", "%3F").replaceAll("&", "%26") + "&redirects=")).data;
+ let efull = (await axios.get("https://mlp.fandom.com/api.php?format=json&action=query&prop=extracts&explaintext&titles=" + encodeURI(episode.name).replaceAll("?", "%3F").replaceAll("&", "%26") + "&redirects=")).data;
+
+ episode.plot = eext.query.pages[Object.keys(eext.query.pages)[0]].extract.replace(/(.*)\n(.*)/, "$2").replace(/(.*)\n\n(.*)/gm, "$2").replace(/([.?!])\s*(?=[A-Z])/g, "$1|").split("|").join(" ");
+ episode.page = efull.query.pages[Object.keys(eext.query.pages)[0]].extract
+
+ let infobox = null;
+ try {
+ sections = parser.pageToSectionObject(edata.query.pages[0].revisions[0].slots.main.content);
+ box = parser.parseInfoBox(sections["content"]);
+ if (box.template === "Infobox episode") {
+ infobox = parser.parseInfoBox(sections["content"]).values;
+ }
+ } catch (e) {
+ console.log(edata);
+ throw e;
+ }
+
+ if (infobox !== null) {
+ try {
+ if (typeof infobox.image !== "undefined") episode.cover = (await axios.head("https://mlp.fandom.com/Special:FilePath/" + encodeURI(infobox.image.replace(/<!--[\s\S]*?-->/g, "").trim()).replaceAll("?", "%3F").replaceAll("&", "%26"))).request.res.responseUrl;
+ } catch (e) {
+ try {
+ if (typeof infobox.image !== "undefined") episode.cover = e.request.res.responseUrl;
+ } catch (e2) {
+ console.error(e2);
+ throw e;
+ }
+ }
+ if (typeof infobox.featured !== "undefined") episode.characters = infobox.featured.replace(/\[\[(([a-zA-Z0-9 .\-# _\\\/]*)\|([a-zA-Z0-9 .\-_\\\/]*)|([a-zA-Z0-9 .\-#_\\\/]*))\]\]/gm, "$3$4").replace(/(<(\/| |)(\/| |)br(\/| |)(\/| |)>)/gm, "|||").split("|||")
+ }
+
+ season.episodes.push(episode);
+ index++;
+ }
+
+ season.count = season.episodes.length;
+ season.date = season.episodes[0].date;
+ data.seasons.push(season);
+ }
+ all.push(data);
+ }
+
+ fs.writeFileSync("./data/series.json", JSON.stringify(all, null, 4));
})();
\ No newline at end of file diff --git a/update/episodes/series.json b/update/episodes/series.json index 0935658..413ff33 100644..100755 --- a/update/episodes/series.json +++ b/update/episodes/series.json @@ -1,127 +1,127 @@ -[ - { - "title": "Friendship is Magic", - "cmd": "FIM", - "page": "My Little Pony Friendship is Magic", - "date": "2010-2019", - "seasons": [ - { - "id": 1, - "table": { - "page": "Friendship is Magic animated media", - "section": "Season one", - "fields": { - "number": 0, - "title": 1, - "writer": 2, - "date": 3 - } - } - }, - { - "id": 2, - "table": { - "page": "Friendship is Magic animated media", - "section": "Season two", - "fields": { - "number": 0, - "title": 1, - "writer": 2, - "date": 3 - } - } - }, - { - "id": 3, - "table": { - "page": "Friendship is Magic animated media", - "section": "Season three", - "fields": { - "number": 0, - "title": 1, - "writer": 2, - "date": 3 - } - } - }, - { - "id": 4, - "table": { - "page": "Friendship is Magic animated media", - "section": "Season four", - "fields": { - "number": 0, - "title": 1, - "writer": 2, - "date": 3 - } - } - }, - { - "id": 5, - "table": { - "page": "Friendship is Magic animated media", - "section": "Season five", - "fields": { - "number": 0, - "title": 1, - "writer": 2, - "date": 3 - } - } - }, - { - "id": 6, - "table": { - "page": "Friendship is Magic animated media", - "section": "Season six", - "fields": { - "number": 0, - "title": 1, - "writer": 2, - "date": 3 - } - } - }, - { - "id": 7, - "table": { - "page": "Friendship is Magic animated media", - "section": "Season seven", - "fields": { - "number": 0, - "title": 1, - "writer": 2, - "date": 3 - } - } - }, - { - "id": 8, - "table": { - "page": "Friendship is Magic animated media", - "section": "Season eight", - "fields": { - "number": 0, - "title": 1, - "writer": 2, - "date": 3 - } - } - }, - { - "id": 9, - "table": { - "page": "Friendship is Magic animated media", - "section": "Season nine", - "fields": { - "number": 0, - "title": 1, - "writer": 2, - "date": 3 - } - } - } - ] - } +[
+ {
+ "title": "Friendship is Magic",
+ "cmd": "FIM",
+ "page": "My Little Pony Friendship is Magic",
+ "date": "2010-2019",
+ "seasons": [
+ {
+ "id": 1,
+ "table": {
+ "page": "Friendship is Magic animated media",
+ "section": "Season one",
+ "fields": {
+ "number": 0,
+ "title": 1,
+ "writer": 2,
+ "date": 3
+ }
+ }
+ },
+ {
+ "id": 2,
+ "table": {
+ "page": "Friendship is Magic animated media",
+ "section": "Season two",
+ "fields": {
+ "number": 0,
+ "title": 1,
+ "writer": 2,
+ "date": 3
+ }
+ }
+ },
+ {
+ "id": 3,
+ "table": {
+ "page": "Friendship is Magic animated media",
+ "section": "Season three",
+ "fields": {
+ "number": 0,
+ "title": 1,
+ "writer": 2,
+ "date": 3
+ }
+ }
+ },
+ {
+ "id": 4,
+ "table": {
+ "page": "Friendship is Magic animated media",
+ "section": "Season four",
+ "fields": {
+ "number": 0,
+ "title": 1,
+ "writer": 2,
+ "date": 3
+ }
+ }
+ },
+ {
+ "id": 5,
+ "table": {
+ "page": "Friendship is Magic animated media",
+ "section": "Season five",
+ "fields": {
+ "number": 0,
+ "title": 1,
+ "writer": 2,
+ "date": 3
+ }
+ }
+ },
+ {
+ "id": 6,
+ "table": {
+ "page": "Friendship is Magic animated media",
+ "section": "Season six",
+ "fields": {
+ "number": 0,
+ "title": 1,
+ "writer": 2,
+ "date": 3
+ }
+ }
+ },
+ {
+ "id": 7,
+ "table": {
+ "page": "Friendship is Magic animated media",
+ "section": "Season seven",
+ "fields": {
+ "number": 0,
+ "title": 1,
+ "writer": 2,
+ "date": 3
+ }
+ }
+ },
+ {
+ "id": 8,
+ "table": {
+ "page": "Friendship is Magic animated media",
+ "section": "Season eight",
+ "fields": {
+ "number": 0,
+ "title": 1,
+ "writer": 2,
+ "date": 3
+ }
+ }
+ },
+ {
+ "id": 9,
+ "table": {
+ "page": "Friendship is Magic animated media",
+ "section": "Season nine",
+ "fields": {
+ "number": 0,
+ "title": 1,
+ "writer": 2,
+ "date": 3
+ }
+ }
+ }
+ ]
+ }
]
\ No newline at end of file diff --git a/update/ponies/dict.js b/update/ponies/dict.js index c72a19e..c72a19e 100644..100755 --- a/update/ponies/dict.js +++ b/update/ponies/dict.js diff --git a/update/ponies/index.js b/update/ponies/index.js index 0cc6af8..0cc6af8 100644..100755 --- a/update/ponies/index.js +++ b/update/ponies/index.js diff --git a/update/ponies/infobox.js b/update/ponies/infobox.js index d3ca174..d3ca174 100644..100755 --- a/update/ponies/infobox.js +++ b/update/ponies/infobox.js diff --git a/update/ponies/listgen.js b/update/ponies/listgen.js index 4e07607..46c77f7 100644..100755 --- a/update/ponies/listgen.js +++ b/update/ponies/listgen.js @@ -1,49 +1,49 @@ -/* - * MIT License - * - * Copyright (c) 2022- Minteck - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - */ - -const axios = require("axios"); -const fs = require("fs"); - -if (fs.existsSync("./data")) fs.rmSync("./data", {recursive: true}); -fs.mkdirSync("./data"); - -(async () => { - async function getCategory(category) { - console.log("Category:" + category); - let cat = (await axios.get("https://mlp.fandom.com/api.php?action=query&generator=categorymembers&gcmtitle=Category:" + encodeURI(category) + "&prop=categories&cllimit=max&gcmlimit=max&format=json")).data; - - return Object.keys(cat.query.pages).map(k => cat.query.pages[k].title).filter(k => !k.startsWith("List") && !k.includes("EG") && !k.toLowerCase().includes("ponies") && !k.includes(" and ") && !k.includes("(") && !k.includes("family")); - } - - let list = [...new Set([ - ...(await getCategory("Pegasus ponies")), - ...(await getCategory("Alicorn ponies")), - ...(await getCategory("Earth ponies")), - ...(await getCategory("Unicorn ponies")), - ...(await getCategory("Main characters")), - ...(await getCategory("Dragons")), - ])]; - fs.writeFileSync("./data/list.json", JSON.stringify(list, null, 4)) +/*
+ * MIT License
+ *
+ * Copyright (c) 2022- Minteck
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+const axios = require("axios");
+const fs = require("fs");
+
+if (fs.existsSync("./data")) fs.rmSync("./data", {recursive: true});
+fs.mkdirSync("./data");
+
+(async () => {
+ async function getCategory(category) {
+ console.log("Category:" + category);
+ let cat = (await axios.get("https://mlp.fandom.com/api.php?action=query&generator=categorymembers&gcmtitle=Category:" + encodeURI(category) + "&prop=categories&cllimit=max&gcmlimit=max&format=json")).data;
+
+ return Object.keys(cat.query.pages).map(k => cat.query.pages[k].title).filter(k => !k.startsWith("List") && !k.includes("EG") && !k.toLowerCase().includes("ponies") && !k.includes(" and ") && !k.includes("(") && !k.includes("family"));
+ }
+
+ let list = [...new Set([
+ ...(await getCategory("Pegasus ponies")),
+ ...(await getCategory("Alicorn ponies")),
+ ...(await getCategory("Earth ponies")),
+ ...(await getCategory("Unicorn ponies")),
+ ...(await getCategory("Main characters")),
+ ...(await getCategory("Dragons")),
+ ])];
+ fs.writeFileSync("./data/list.json", JSON.stringify(list, null, 4))
})()
\ No newline at end of file diff --git a/update/ponies/pages.js b/update/ponies/pages.js index af14362..af14362 100644..100755 --- a/update/ponies/pages.js +++ b/update/ponies/pages.js diff --git a/update/ponies/parse.js b/update/ponies/parse.js index 8cf797d..71ce007 100644..100755 --- a/update/ponies/parse.js +++ b/update/ponies/parse.js @@ -1,140 +1,140 @@ -/* - * MIT License - * - * Copyright (c) 2022- Minteck - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - */ - -const fs = require('fs'); -const axios = require('axios'); - -console.log("Parsing infobox data..."); - -let ponies = {}; - -(async () => { - for (let title in JSON.parse(fs.readFileSync("./data/boxes.json").toString())) { - console.log("Parsing " + title + "..."); - let box = JSON.parse(fs.readFileSync("./data/boxes.json").toString())[title]; - let data = { - names: [title], - extract: "", - extract_fr: "", - generation: -1, - color: "000000", - image: "https://example.com", - kind: "Pony", - sex: "Unknown", - occupation: ["Unknown"], - residence: ["Unknown"], - mark: "https://example.com" - } - - if (typeof box._gen !== "undefined") data.generation = box._gen; - - if (typeof box.name2 !== "undefined") data.names.push(box.name2.replace(/<!--[\s\S]*?-->/g, "")); - if (typeof box.name3 !== "undefined") data.names.push(box.name3.replace(/<!--[\s\S]*?-->/g, "")); - if (typeof box.name4 !== "undefined") data.names.push(box.name4.replace(/<!--[\s\S]*?-->/g, "")); - if (typeof box.name5 !== "undefined") data.names.push(box.name5.replace(/<!--[\s\S]*?-->/g, "")); - - if (typeof box.nicknames !== "undefined") { - box.nicknames.replace(/<!--[\s\S]*?-->/g, "").split(",").filter(e => !e.match(/[^a-zA-Z0-9-_ ]/gm)).forEach((e, i) => { - data.names.push(e.trim()); - }); - } - - if (typeof box.kind !== "undefined") { - kp = box.kind.replace(/<!--[\s\S]*?-->/g, "").replace(/[^a-zA-Z0-9-_ ]/gm, "").split(" ")[0]; - data.kind = kp.substring(kp.replace(/([A-Z])([a-z0-9]*)$/g, "").length); - } - if (typeof box.sex !== "undefined") data.sex = box.sex.replace(/<!--[\s\S]*?-->/g, "").trim().startsWith("F") ? "F" : "M"; - if (typeof box._extract !== "undefined") { - data.extract = box._extract; - } - if (typeof box.coat !== "undefined") data.color = box.coat.replace(/<!--[\s\S]*?-->/g, "").trim().replace(/\[([a-z.\/ \nA-Z0-9:]*)\/(.{6})\/ (.*)\]/gm, "$2").replace(/{{perbang\|([0-9A-Fa-f].{5})(.*)/g, "$1"); - try { - if (typeof box.main !== "undefined") data.image = (await axios.head("https://mlp.fandom.com/Special:FilePath/" + encodeURI(box.main.replace(/<!--[\s\S]*?-->/g, "").trim()).replaceAll("?", "%3F").replaceAll("&", "%26"))).request.res.responseUrl; - } catch (e) { - try { - if (typeof box.main !== "undefined") data.image = e.request.res.responseUrl; - } catch (e2) { - console.error(e2); - throw e; - } - } - try { - if (typeof box.main1 !== "undefined") data.image = (await axios.head("https://mlp.fandom.com/Special:FilePath/" + encodeURI(box.main1.replace(/<!--[\s\S]*?-->/g, "").trim()).replaceAll("?", "%3F").replaceAll("&", "%26"))).request.res.responseUrl; - } catch (e) { - try { - if (typeof box.main1 !== "undefined") data.image = e.request.res.responseUrl; - } catch (e2) { - console.error(e2); - throw e; - } - } - try { - if (typeof box.image !== "undefined") data.image = (await axios.head("https://mlp.fandom.com/Special:FilePath/" + encodeURI(box.image.replace(/<!--[\s\S]*?-->/g, "").trim()).replaceAll("?", "%3F").replaceAll("&", "%26"))).request.res.responseUrl; - } catch (e) { - try { - if (typeof box.image !== "undefined") data.image = e.request.res.responseUrl; - } catch (e2) { - console.error(e2); - throw e; - } - } - if (typeof box["cutie mark"] !== "undefined") { - try { - data.markimg = box["cutie mark"].replace(/<!--[\s\S]*?-->/g, "").trim().split("[[File:")[1].split("|")[0]; - } catch (e) { - data.markimg = box["cutie mark"].replace(/<!--[\s\S]*?-->/g, "").trim().split("[[File:")[0].split("|")[0]; - } - try { - data.mark = (await axios.head("https://mlp.fandom.com/Special:FilePath/" + encodeURI(data.markimg).replaceAll("?", "%3F").replaceAll("&", "%26"))).request.res.responseUrl; - } catch (e) { - try { - data.mark = e.request.res.responseUrl; - } catch (e2) { - console.error(e2); - throw e; - } - } - } - - if (typeof box.occupation !== "undefined") { - occupations = []; - box.occupation/*.replace(/\[\[(.*)_(.*)\]\]|\[\[(.*)\|(.*)\]\]|\[\[(.*)\]\]/gm, "$2$4$5")*/.replace(/<!--[\s\S]*?-->/g, "").trim().replace(/\|/gm, "_").replace(/<( ||(|| )\/)( ||(|| )\/)(b|B)(r|R)( ||(|| )\/)( ||(|| )\/)>/gm, "|").replace(/( \|| \| | \| )/gm, "|").split("|").forEach((e) => { - occupations.push(e.trim().replace(/[\[\]]/gm, "").replace(/<(.*)>/gm, "").replace(/([a-zA-Z0-9 \-_,.'"]*)_([a-zA-Z0-9 \-_,.'"]*)/gm, "$2")); - }) - data.occupation = occupations; - } - if (typeof box.residence !== "undefined") { - residences = []; - box.residence/*.replace(/\[\[(.*)_(.*)\]\]|\[\[(.*)\|(.*)\]\]|\[\[(.*)\]\]/gm, "$2$4$5")*/.replace(/<!--[\s\S]*?-->/g, "").trim().replace(/\|/gm, "_").replace(/<( ||(|| )\/)( ||(|| )\/)(b|B)(r|R)( ||(|| )\/)( ||(|| )\/)>/gm, "|").replace(/( \|| \| | \| )/gm, "|").split("|").forEach((e) => { - residences.push(e.trim().replace(/[\[\]]/gm, "").replace(/<(.*)>/gm, "").replace(/([a-zA-Z0-9 \-_,.'"]*)_([a-zA-Z0-9 \-_,.'"]*)/gm, "$2")); - }) - data.residence = residences.replaceAll("Locations#", ""); - } - - if ((typeof box.name2 !== "undefined" || typeof box.name3 !== "undefined" || typeof box.name4 !== "undefined" || typeof box.name5 !== "undefined" || typeof box.coat !== "undefined" || typeof box.occupation !== "undefined" || typeof box.residence !== "undefined") && typeof box.sex !== "undefined") ponies[title] = data; - } -})() - +/*
+ * MIT License
+ *
+ * Copyright (c) 2022- Minteck
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+const fs = require('fs');
+const axios = require('axios');
+
+console.log("Parsing infobox data...");
+
+let ponies = {};
+
+(async () => {
+ for (let title in JSON.parse(fs.readFileSync("./data/boxes.json").toString())) {
+ console.log("Parsing " + title + "...");
+ let box = JSON.parse(fs.readFileSync("./data/boxes.json").toString())[title];
+ let data = {
+ names: [title],
+ extract: "",
+ extract_fr: "",
+ generation: -1,
+ color: "000000",
+ image: "https://example.com",
+ kind: "Pony",
+ sex: "Unknown",
+ occupation: ["Unknown"],
+ residence: ["Unknown"],
+ mark: "https://example.com"
+ }
+
+ if (typeof box._gen !== "undefined") data.generation = box._gen;
+
+ if (typeof box.name2 !== "undefined") data.names.push(box.name2.replace(/<!--[\s\S]*?-->/g, ""));
+ if (typeof box.name3 !== "undefined") data.names.push(box.name3.replace(/<!--[\s\S]*?-->/g, ""));
+ if (typeof box.name4 !== "undefined") data.names.push(box.name4.replace(/<!--[\s\S]*?-->/g, ""));
+ if (typeof box.name5 !== "undefined") data.names.push(box.name5.replace(/<!--[\s\S]*?-->/g, ""));
+
+ if (typeof box.nicknames !== "undefined") {
+ box.nicknames.replace(/<!--[\s\S]*?-->/g, "").split(",").filter(e => !e.match(/[^a-zA-Z0-9-_ ]/gm)).forEach((e, i) => {
+ data.names.push(e.trim());
+ });
+ }
+
+ if (typeof box.kind !== "undefined") {
+ kp = box.kind.replace(/<!--[\s\S]*?-->/g, "").replace(/[^a-zA-Z0-9-_ ]/gm, "").split(" ")[0];
+ data.kind = kp.substring(kp.replace(/([A-Z])([a-z0-9]*)$/g, "").length);
+ }
+ if (typeof box.sex !== "undefined") data.sex = box.sex.replace(/<!--[\s\S]*?-->/g, "").trim().startsWith("F") ? "F" : "M";
+ if (typeof box._extract !== "undefined") {
+ data.extract = box._extract;
+ }
+ if (typeof box.coat !== "undefined") data.color = box.coat.replace(/<!--[\s\S]*?-->/g, "").trim().replace(/\[([a-z.\/ \nA-Z0-9:]*)\/(.{6})\/ (.*)\]/gm, "$2").replace(/{{perbang\|([0-9A-Fa-f].{5})(.*)/g, "$1");
+ try {
+ if (typeof box.main !== "undefined") data.image = (await axios.head("https://mlp.fandom.com/Special:FilePath/" + encodeURI(box.main.replace(/<!--[\s\S]*?-->/g, "").trim()).replaceAll("?", "%3F").replaceAll("&", "%26"))).request.res.responseUrl;
+ } catch (e) {
+ try {
+ if (typeof box.main !== "undefined") data.image = e.request.res.responseUrl;
+ } catch (e2) {
+ console.error(e2);
+ throw e;
+ }
+ }
+ try {
+ if (typeof box.main1 !== "undefined") data.image = (await axios.head("https://mlp.fandom.com/Special:FilePath/" + encodeURI(box.main1.replace(/<!--[\s\S]*?-->/g, "").trim()).replaceAll("?", "%3F").replaceAll("&", "%26"))).request.res.responseUrl;
+ } catch (e) {
+ try {
+ if (typeof box.main1 !== "undefined") data.image = e.request.res.responseUrl;
+ } catch (e2) {
+ console.error(e2);
+ throw e;
+ }
+ }
+ try {
+ if (typeof box.image !== "undefined") data.image = (await axios.head("https://mlp.fandom.com/Special:FilePath/" + encodeURI(box.image.replace(/<!--[\s\S]*?-->/g, "").trim()).replaceAll("?", "%3F").replaceAll("&", "%26"))).request.res.responseUrl;
+ } catch (e) {
+ try {
+ if (typeof box.image !== "undefined") data.image = e.request.res.responseUrl;
+ } catch (e2) {
+ console.error(e2);
+ throw e;
+ }
+ }
+ if (typeof box["cutie mark"] !== "undefined") {
+ try {
+ data.markimg = box["cutie mark"].replace(/<!--[\s\S]*?-->/g, "").trim().split("[[File:")[1].split("|")[0];
+ } catch (e) {
+ data.markimg = box["cutie mark"].replace(/<!--[\s\S]*?-->/g, "").trim().split("[[File:")[0].split("|")[0];
+ }
+ try {
+ data.mark = (await axios.head("https://mlp.fandom.com/Special:FilePath/" + encodeURI(data.markimg).replaceAll("?", "%3F").replaceAll("&", "%26"))).request.res.responseUrl;
+ } catch (e) {
+ try {
+ data.mark = e.request.res.responseUrl;
+ } catch (e2) {
+ console.error(e2);
+ throw e;
+ }
+ }
+ }
+
+ if (typeof box.occupation !== "undefined") {
+ occupations = [];
+ box.occupation/*.replace(/\[\[(.*)_(.*)\]\]|\[\[(.*)\|(.*)\]\]|\[\[(.*)\]\]/gm, "$2$4$5")*/.replace(/<!--[\s\S]*?-->/g, "").trim().replace(/\|/gm, "_").replace(/<( ||(|| )\/)( ||(|| )\/)(b|B)(r|R)( ||(|| )\/)( ||(|| )\/)>/gm, "|").replace(/( \|| \| | \| )/gm, "|").split("|").forEach((e) => {
+ occupations.push(e.trim().replace(/[\[\]]/gm, "").replace(/<(.*)>/gm, "").replace(/([a-zA-Z0-9 \-_,.'"]*)_([a-zA-Z0-9 \-_,.'"]*)/gm, "$2"));
+ })
+ data.occupation = occupations;
+ }
+ if (typeof box.residence !== "undefined") {
+ residences = [];
+ box.residence/*.replace(/\[\[(.*)_(.*)\]\]|\[\[(.*)\|(.*)\]\]|\[\[(.*)\]\]/gm, "$2$4$5")*/.replace(/<!--[\s\S]*?-->/g, "").trim().replace(/\|/gm, "_").replace(/<( ||(|| )\/)( ||(|| )\/)(b|B)(r|R)( ||(|| )\/)( ||(|| )\/)>/gm, "|").replace(/( \|| \| | \| )/gm, "|").split("|").forEach((e) => {
+ residences.push(e.trim().replace(/[\[\]]/gm, "").replace(/<(.*)>/gm, "").replace(/([a-zA-Z0-9 \-_,.'"]*)_([a-zA-Z0-9 \-_,.'"]*)/gm, "$2"));
+ })
+ data.residence = residences.replaceAll("Locations#", "");
+ }
+
+ if ((typeof box.name2 !== "undefined" || typeof box.name3 !== "undefined" || typeof box.name4 !== "undefined" || typeof box.name5 !== "undefined" || typeof box.coat !== "undefined" || typeof box.occupation !== "undefined" || typeof box.residence !== "undefined") && typeof box.sex !== "undefined") ponies[title] = data;
+ }
+})()
+
fs.writeFileSync("./data/data.json", JSON.stringify(ponies, null, 4));
\ No newline at end of file |