From 6c074197976a113f98ebd05a35ffb0590e3702fe Mon Sep 17 00:00:00 2001 From: Minteck Date: Sun, 6 Mar 2022 15:31:09 +0100 Subject: Fix: resolves #17 --- update/episodes/index.js | 336 ++++++++++++++++++++++---------------------- update/episodes/series.json | 252 ++++++++++++++++----------------- 2 files changed, 294 insertions(+), 294 deletions(-) mode change 100644 => 100755 update/episodes/index.js mode change 100644 => 100755 update/episodes/series.json (limited to 'update/episodes') diff --git a/update/episodes/index.js b/update/episodes/index.js old mode 100644 new mode 100755 index 1ae0ec3..611b1c1 --- a/update/episodes/index.js +++ b/update/episodes/index.js @@ -1,169 +1,169 @@ -/* - * MIT License - * - * Copyright (c) 2022- Minteck - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - */ - -const config = require("./series.json"); -const axios = require('axios'); -const fs = require('fs'); -const WikiTextParser = require('parse-wikitext'); -const parser = new WikiTextParser("mlp.fandom.com"); - -(async () => { - let all = []; - - for (let series of config) { - let data = { - command: series.cmd, - title: series.title, - date: series.date, - link: series.page, - description: { - series: null, - plot: null, - }, - seasons: [] - }; - console.log(series.title + " (" + data.date + ")"); - - ddata = (await axios.get("https://mlp.fandom.com/api.php?format=json&action=query&prop=extracts&exlimit=max&explaintext&exintro&titles=" + encodeURI(series.page) + "&redirects=")).data; - data.description.series = ddata.query.pages[Object.keys(ddata.query.pages)[0]].extract.split("\n")[0]; - data.description.plot = ddata.query.pages[Object.keys(ddata.query.pages)[0]].extract.split("\n")[1]; - - for (let s of series.seasons) { - console.log(" Season " + s.id); - - let season = { - name: "Season " + s.id, - id: "s" + s.id, - date: null, - count: null, - episodes: [] - } - - let sdata = (await axios.get("https://mlp.fandom.com/api.php?action=query&prop=revisions&titles=" + s.table.page + "&rvslots=*&rvprop=content&formatversion=2&format=json")).data; - let sections = parser.pageToSectionObject(sdata.query.pages[0].revisions[0].slots.main.content); - let episodes = sections.Episodes[s.table.section].content - .filter(i => i.startsWith("|")) - .join("||") - .replace("|}", "") - .split("|-").map(i => { - return i.split("||").filter(i => i.trim() !== ""); - }) - .filter(i => i.length > 1) - .map(i => { - let index = 0; - return i.map(i => { - return i.replace(/^(\| |\|)(.*)/gm, "$2") - .replace(/^style="(.*)"( |)\|( |)(.*)/gm, "$4") - .replace(/\[\[((.*)\|(.*)|(.*))\]\]/gm, "$3$4"); - }).map(i => { - if (index === 0) { - index++; - return i.replace(/(.*) \((.*)\)/gm, "$1|$2"); - } else { - index++; - return i; - } - }) - }); - - index = 1; - for (let e of episodes) { - process.stdout.write(" S" + s.id + "E" + index); - - let episode = { - local: null, - global: null, - date: null, - name: null, - writer: null, - cover: null, - characters: [], - plot: null, - page: null - } - - episode.local = e[s.table.fields.number].replace(/(.*)\|(.*)/gm, "$1").replace(/''(.*)''/gm, "$1").replace(/{{(.*)\|(.*)}}/gm, "$2").replace(/(.*)\|(.*)/gm, "$2"); - episode.global = e[s.table.fields.number].replace(/(.*)\|(.*)/gm, "$2"); - episode.date_pre = e[s.table.fields.date].replace(/{{(.*)\|(.*)\|(.*)\|link=(no|yes)}}/gm, "$3").replace(/{{(.*)\|(.*)}}/gm, "$2").replace(/(.*)\|(.*)/gm, "$2"); - try { - episode.date = new Date(e[s.table.fields.date].replace(/{{(.*)\|(.*)\|(.*)\|link=(no|yes)}}/gm, "$3").replace(/{{(.*)\|(.*)}}/gm, "$2").replace(/(.*)\|(.*)/gm, "$2")).toISOString(); - } catch (e) { - episode.date = null; - } - episode.name = e[s.table.fields.title]; - episode.writer = e[s.table.fields.writer].replace(/{{(.*)\|(.*)\|(.*)\|link=(no|yes)}}/gm, "$3").replace(/(]]|\[\[)/gm, "").replace(/(.*)\|(.*)/gm, "$2"); - - process.stdout.write(": " + episode.name + "\n"); - - if (episode.local.length !== 2 || episode.global.length > 3 || episode.global.length < 2 || episode.date_pre.length !== 10) { - console.log("FAILURE: " + JSON.stringify(episode)); - } - - let edata = (await axios.get("https://mlp.fandom.com/api.php?action=query&prop=revisions&titles=" + encodeURI(episode.name).replaceAll("?", "%3F").replaceAll("&", "%26") + "&rvslots=*&rvprop=content&formatversion=2&format=json")).data; - let eext = (await axios.get("https://mlp.fandom.com/api.php?format=json&action=query&prop=extracts&exlimit=max&explaintext&exintro&titles=" + encodeURI(episode.name).replaceAll("?", "%3F").replaceAll("&", "%26") + "&redirects=")).data; - let efull = (await axios.get("https://mlp.fandom.com/api.php?format=json&action=query&prop=extracts&explaintext&titles=" + encodeURI(episode.name).replaceAll("?", "%3F").replaceAll("&", "%26") + "&redirects=")).data; - - episode.plot = eext.query.pages[Object.keys(eext.query.pages)[0]].extract.replace(/(.*)\n(.*)/, "$2").replace(/(.*)\n\n(.*)/gm, "$2").replace(/([.?!])\s*(?=[A-Z])/g, "$1|").split("|").join(" "); - episode.page = efull.query.pages[Object.keys(eext.query.pages)[0]].extract - - let infobox = null; - try { - sections = parser.pageToSectionObject(edata.query.pages[0].revisions[0].slots.main.content); - box = parser.parseInfoBox(sections["content"]); - if (box.template === "Infobox episode") { - infobox = parser.parseInfoBox(sections["content"]).values; - } - } catch (e) { - console.log(edata); - throw e; - } - - if (infobox !== null) { - try { - if (typeof infobox.image !== "undefined") episode.cover = (await axios.head("https://mlp.fandom.com/Special:FilePath/" + encodeURI(infobox.image.replace(//g, "").trim()).replaceAll("?", "%3F").replaceAll("&", "%26"))).request.res.responseUrl; - } catch (e) { - try { - if (typeof infobox.image !== "undefined") episode.cover = e.request.res.responseUrl; - } catch (e2) { - console.error(e2); - throw e; - } - } - if (typeof infobox.featured !== "undefined") episode.characters = infobox.featured.replace(/\[\[(([a-zA-Z0-9 .\-# _\\\/]*)\|([a-zA-Z0-9 .\-_\\\/]*)|([a-zA-Z0-9 .\-#_\\\/]*))\]\]/gm, "$3$4").replace(/(<(\/| |)(\/| |)br(\/| |)(\/| |)>)/gm, "|||").split("|||") - } - - season.episodes.push(episode); - index++; - } - - season.count = season.episodes.length; - season.date = season.episodes[0].date; - data.seasons.push(season); - } - all.push(data); - } - - fs.writeFileSync("./data/series.json", JSON.stringify(all, null, 4)); +/* + * MIT License + * + * Copyright (c) 2022- Minteck + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + */ + +const config = require("./series.json"); +const axios = require('axios'); +const fs = require('fs'); +const WikiTextParser = require('parse-wikitext'); +const parser = new WikiTextParser("mlp.fandom.com"); + +(async () => { + let all = []; + + for (let series of config) { + let data = { + command: series.cmd, + title: series.title, + date: series.date, + link: series.page, + description: { + series: null, + plot: null, + }, + seasons: [] + }; + console.log(series.title + " (" + data.date + ")"); + + ddata = (await axios.get("https://mlp.fandom.com/api.php?format=json&action=query&prop=extracts&exlimit=max&explaintext&exintro&titles=" + encodeURI(series.page) + "&redirects=")).data; + data.description.series = ddata.query.pages[Object.keys(ddata.query.pages)[0]].extract.split("\n")[0]; + data.description.plot = ddata.query.pages[Object.keys(ddata.query.pages)[0]].extract.split("\n")[1]; + + for (let s of series.seasons) { + console.log(" Season " + s.id); + + let season = { + name: "Season " + s.id, + id: "s" + s.id, + date: null, + count: null, + episodes: [] + } + + let sdata = (await axios.get("https://mlp.fandom.com/api.php?action=query&prop=revisions&titles=" + s.table.page + "&rvslots=*&rvprop=content&formatversion=2&format=json")).data; + let sections = parser.pageToSectionObject(sdata.query.pages[0].revisions[0].slots.main.content); + let episodes = sections.Episodes[s.table.section].content + .filter(i => i.startsWith("|")) + .join("||") + .replace("|}", "") + .split("|-").map(i => { + return i.split("||").filter(i => i.trim() !== ""); + }) + .filter(i => i.length > 1) + .map(i => { + let index = 0; + return i.map(i => { + return i.replace(/^(\| |\|)(.*)/gm, "$2") + .replace(/^style="(.*)"( |)\|( |)(.*)/gm, "$4") + .replace(/\[\[((.*)\|(.*)|(.*))\]\]/gm, "$3$4"); + }).map(i => { + if (index === 0) { + index++; + return i.replace(/(.*) \((.*)\)/gm, "$1|$2"); + } else { + index++; + return i; + } + }) + }); + + index = 1; + for (let e of episodes) { + process.stdout.write(" S" + s.id + "E" + index); + + let episode = { + local: null, + global: null, + date: null, + name: null, + writer: null, + cover: null, + characters: [], + plot: null, + page: null + } + + episode.local = e[s.table.fields.number].replace(/(.*)\|(.*)/gm, "$1").replace(/''(.*)''/gm, "$1").replace(/{{(.*)\|(.*)}}/gm, "$2").replace(/(.*)\|(.*)/gm, "$2"); + episode.global = e[s.table.fields.number].replace(/(.*)\|(.*)/gm, "$2"); + episode.date_pre = e[s.table.fields.date].replace(/{{(.*)\|(.*)\|(.*)\|link=(no|yes)}}/gm, "$3").replace(/{{(.*)\|(.*)}}/gm, "$2").replace(/(.*)\|(.*)/gm, "$2"); + try { + episode.date = new Date(e[s.table.fields.date].replace(/{{(.*)\|(.*)\|(.*)\|link=(no|yes)}}/gm, "$3").replace(/{{(.*)\|(.*)}}/gm, "$2").replace(/(.*)\|(.*)/gm, "$2")).toISOString(); + } catch (e) { + episode.date = null; + } + episode.name = e[s.table.fields.title]; + episode.writer = e[s.table.fields.writer].replace(/{{(.*)\|(.*)\|(.*)\|link=(no|yes)}}/gm, "$3").replace(/(]]|\[\[)/gm, "").replace(/(.*)\|(.*)/gm, "$2"); + + process.stdout.write(": " + episode.name + "\n"); + + if (episode.local.length !== 2 || episode.global.length > 3 || episode.global.length < 2 || episode.date_pre.length !== 10) { + console.log("FAILURE: " + JSON.stringify(episode)); + } + + let edata = (await axios.get("https://mlp.fandom.com/api.php?action=query&prop=revisions&titles=" + encodeURI(episode.name).replaceAll("?", "%3F").replaceAll("&", "%26") + "&rvslots=*&rvprop=content&formatversion=2&format=json")).data; + let eext = (await axios.get("https://mlp.fandom.com/api.php?format=json&action=query&prop=extracts&exlimit=max&explaintext&exintro&titles=" + encodeURI(episode.name).replaceAll("?", "%3F").replaceAll("&", "%26") + "&redirects=")).data; + let efull = (await axios.get("https://mlp.fandom.com/api.php?format=json&action=query&prop=extracts&explaintext&titles=" + encodeURI(episode.name).replaceAll("?", "%3F").replaceAll("&", "%26") + "&redirects=")).data; + + episode.plot = eext.query.pages[Object.keys(eext.query.pages)[0]].extract.replace(/(.*)\n(.*)/, "$2").replace(/(.*)\n\n(.*)/gm, "$2").replace(/([.?!])\s*(?=[A-Z])/g, "$1|").split("|").join(" "); + episode.page = efull.query.pages[Object.keys(eext.query.pages)[0]].extract + + let infobox = null; + try { + sections = parser.pageToSectionObject(edata.query.pages[0].revisions[0].slots.main.content); + box = parser.parseInfoBox(sections["content"]); + if (box.template === "Infobox episode") { + infobox = parser.parseInfoBox(sections["content"]).values; + } + } catch (e) { + console.log(edata); + throw e; + } + + if (infobox !== null) { + try { + if (typeof infobox.image !== "undefined") episode.cover = (await axios.head("https://mlp.fandom.com/Special:FilePath/" + encodeURI(infobox.image.replace(//g, "").trim()).replaceAll("?", "%3F").replaceAll("&", "%26"))).request.res.responseUrl; + } catch (e) { + try { + if (typeof infobox.image !== "undefined") episode.cover = e.request.res.responseUrl; + } catch (e2) { + console.error(e2); + throw e; + } + } + if (typeof infobox.featured !== "undefined") episode.characters = infobox.featured.replace(/\[\[(([a-zA-Z0-9 .\-# _\\\/]*)\|([a-zA-Z0-9 .\-_\\\/]*)|([a-zA-Z0-9 .\-#_\\\/]*))\]\]/gm, "$3$4").replace(/(<(\/| |)(\/| |)br(\/| |)(\/| |)>)/gm, "|||").split("|||") + } + + season.episodes.push(episode); + index++; + } + + season.count = season.episodes.length; + season.date = season.episodes[0].date; + data.seasons.push(season); + } + all.push(data); + } + + fs.writeFileSync("./data/series.json", JSON.stringify(all, null, 4)); })(); \ No newline at end of file diff --git a/update/episodes/series.json b/update/episodes/series.json old mode 100644 new mode 100755 index 0935658..413ff33 --- a/update/episodes/series.json +++ b/update/episodes/series.json @@ -1,127 +1,127 @@ -[ - { - "title": "Friendship is Magic", - "cmd": "FIM", - "page": "My Little Pony Friendship is Magic", - "date": "2010-2019", - "seasons": [ - { - "id": 1, - "table": { - "page": "Friendship is Magic animated media", - "section": "Season one", - "fields": { - "number": 0, - "title": 1, - "writer": 2, - "date": 3 - } - } - }, - { - "id": 2, - "table": { - "page": "Friendship is Magic animated media", - "section": "Season two", - "fields": { - "number": 0, - "title": 1, - "writer": 2, - "date": 3 - } - } - }, - { - "id": 3, - "table": { - "page": "Friendship is Magic animated media", - "section": "Season three", - "fields": { - "number": 0, - "title": 1, - "writer": 2, - "date": 3 - } - } - }, - { - "id": 4, - "table": { - "page": "Friendship is Magic animated media", - "section": "Season four", - "fields": { - "number": 0, - "title": 1, - "writer": 2, - "date": 3 - } - } - }, - { - "id": 5, - "table": { - "page": "Friendship is Magic animated media", - "section": "Season five", - "fields": { - "number": 0, - "title": 1, - "writer": 2, - "date": 3 - } - } - }, - { - "id": 6, - "table": { - "page": "Friendship is Magic animated media", - "section": "Season six", - "fields": { - "number": 0, - "title": 1, - "writer": 2, - "date": 3 - } - } - }, - { - "id": 7, - "table": { - "page": "Friendship is Magic animated media", - "section": "Season seven", - "fields": { - "number": 0, - "title": 1, - "writer": 2, - "date": 3 - } - } - }, - { - "id": 8, - "table": { - "page": "Friendship is Magic animated media", - "section": "Season eight", - "fields": { - "number": 0, - "title": 1, - "writer": 2, - "date": 3 - } - } - }, - { - "id": 9, - "table": { - "page": "Friendship is Magic animated media", - "section": "Season nine", - "fields": { - "number": 0, - "title": 1, - "writer": 2, - "date": 3 - } - } - } - ] - } +[ + { + "title": "Friendship is Magic", + "cmd": "FIM", + "page": "My Little Pony Friendship is Magic", + "date": "2010-2019", + "seasons": [ + { + "id": 1, + "table": { + "page": "Friendship is Magic animated media", + "section": "Season one", + "fields": { + "number": 0, + "title": 1, + "writer": 2, + "date": 3 + } + } + }, + { + "id": 2, + "table": { + "page": "Friendship is Magic animated media", + "section": "Season two", + "fields": { + "number": 0, + "title": 1, + "writer": 2, + "date": 3 + } + } + }, + { + "id": 3, + "table": { + "page": "Friendship is Magic animated media", + "section": "Season three", + "fields": { + "number": 0, + "title": 1, + "writer": 2, + "date": 3 + } + } + }, + { + "id": 4, + "table": { + "page": "Friendship is Magic animated media", + "section": "Season four", + "fields": { + "number": 0, + "title": 1, + "writer": 2, + "date": 3 + } + } + }, + { + "id": 5, + "table": { + "page": "Friendship is Magic animated media", + "section": "Season five", + "fields": { + "number": 0, + "title": 1, + "writer": 2, + "date": 3 + } + } + }, + { + "id": 6, + "table": { + "page": "Friendship is Magic animated media", + "section": "Season six", + "fields": { + "number": 0, + "title": 1, + "writer": 2, + "date": 3 + } + } + }, + { + "id": 7, + "table": { + "page": "Friendship is Magic animated media", + "section": "Season seven", + "fields": { + "number": 0, + "title": 1, + "writer": 2, + "date": 3 + } + } + }, + { + "id": 8, + "table": { + "page": "Friendship is Magic animated media", + "section": "Season eight", + "fields": { + "number": 0, + "title": 1, + "writer": 2, + "date": 3 + } + } + }, + { + "id": 9, + "table": { + "page": "Friendship is Magic animated media", + "section": "Season nine", + "fields": { + "number": 0, + "title": 1, + "writer": 2, + "date": 3 + } + } + } + ] + } ] \ No newline at end of file -- cgit