/* * MIT License * * Copyright (c) 2022- Minteck * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in all * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. * */ const config = require("./series.json"); const axios = require('axios'); const fs = require('fs'); const WikiTextParser = require('parse-wikitext'); const parser = new WikiTextParser("mlp.fandom.com"); (async () => { let all = []; for (let series of config) { let data = { command: series.cmd, title: series.title, date: series.date, link: series.page, description: { series: null, plot: null, }, seasons: [] }; console.log(series.title + " (" + data.date + ")"); ddata = (await axios.get("https://mlp.fandom.com/api.php?format=json&action=query&prop=extracts&exlimit=max&explaintext&exintro&titles=" + encodeURI(series.page) + "&redirects=")).data; data.description.series = ddata.query.pages[Object.keys(ddata.query.pages)[0]].extract.split("\n")[0]; data.description.plot = ddata.query.pages[Object.keys(ddata.query.pages)[0]].extract.split("\n")[1]; for (let s of series.seasons) { console.log(" Season " + s.id); let season = { name: "Season " + s.id, id: "s" + s.id, date: null, count: null, episodes: [] } let sdata = (await axios.get("https://mlp.fandom.com/api.php?action=query&prop=revisions&titles=" + s.table.page + "&rvslots=*&rvprop=content&formatversion=2&format=json")).data; let sections = parser.pageToSectionObject(sdata.query.pages[0].revisions[0].slots.main.content); let episodes = sections.Episodes[s.table.section].content .filter(i => i.startsWith("|")) .join("||") .replace("|}", "") .split("|-").map(i => { return i.split("||").filter(i => i.trim() !== ""); }) .filter(i => i.length > 1) .map(i => { let index = 0; return i.map(i => { return i.replace(/^(\| |\|)(.*)/gm, "$2") .replace(/^style="(.*)"( |)\|( |)(.*)/gm, "$4") .replace(/\[\[((.*)\|(.*)|(.*))\]\]/gm, "$3$4"); }).map(i => { if (index === 0) { index++; return i.replace(/(.*) \((.*)\)/gm, "$1|$2"); } else { index++; return i; } }) }); index = 1; for (let e of episodes) { process.stdout.write(" S" + s.id + "E" + index); let episode = { local: null, global: null, date: null, name: null, writer: null, cover: null, characters: [], plot: null, page: null } episode.local = e[s.table.fields.number].replace(/(.*)\|(.*)/gm, "$1").replace(/''(.*)''/gm, "$1").replace(/{{(.*)\|(.*)}}/gm, "$2").replace(/(.*)\|(.*)/gm, "$2"); episode.global = e[s.table.fields.number].replace(/(.*)\|(.*)/gm, "$2"); episode.date_pre = e[s.table.fields.date].replace(/{{(.*)\|(.*)\|(.*)\|link=(no|yes)}}/gm, "$3").replace(/{{(.*)\|(.*)}}/gm, "$2").replace(/(.*)\|(.*)/gm, "$2"); try { episode.date = new Date(e[s.table.fields.date].replace(/{{(.*)\|(.*)\|(.*)\|link=(no|yes)}}/gm, "$3").replace(/{{(.*)\|(.*)}}/gm, "$2").replace(/(.*)\|(.*)/gm, "$2")).toISOString(); } catch (e) { episode.date = null; } episode.name = e[s.table.fields.title]; episode.writer = e[s.table.fields.writer].replace(/{{(.*)\|(.*)\|(.*)\|link=(no|yes)}}/gm, "$3").replace(/(]]|\[\[)/gm, "").replace(/(.*)\|(.*)/gm, "$2"); process.stdout.write(": " + episode.name + "\n"); if (episode.local.length !== 2 || episode.global.length > 3 || episode.global.length < 2 || episode.date_pre.length !== 10) { console.log("FAILURE: " + JSON.stringify(episode)); } let edata = (await axios.get("https://mlp.fandom.com/api.php?action=query&prop=revisions&titles=" + encodeURI(episode.name).replaceAll("?", "%3F").replaceAll("&", "%26") + "&rvslots=*&rvprop=content&formatversion=2&format=json")).data; let eext = (await axios.get("https://mlp.fandom.com/api.php?format=json&action=query&prop=extracts&exlimit=max&explaintext&exintro&titles=" + encodeURI(episode.name).replaceAll("?", "%3F").replaceAll("&", "%26") + "&redirects=")).data; let efull = (await axios.get("https://mlp.fandom.com/api.php?format=json&action=query&prop=extracts&explaintext&titles=" + encodeURI(episode.name).replaceAll("?", "%3F").replaceAll("&", "%26") + "&redirects=")).data; episode.plot = eext.query.pages[Object.keys(eext.query.pages)[0]].extract.replace(/(.*)\n(.*)/, "$2").replace(/(.*)\n\n(.*)/gm, "$2").replace(/([.?!])\s*(?=[A-Z])/g, "$1|").split("|").join(" "); episode.page = efull.query.pages[Object.keys(eext.query.pages)[0]].extract let infobox = null; try { sections = parser.pageToSectionObject(edata.query.pages[0].revisions[0].slots.main.content); box = parser.parseInfoBox(sections["content"]); if (box.template === "Infobox episode") { infobox = parser.parseInfoBox(sections["content"]).values; } } catch (e) { console.log(edata); throw e; } if (infobox !== null) { try { if (typeof infobox.image !== "undefined") episode.cover = (await axios.head("https://mlp.fandom.com/Special:FilePath/" + encodeURI(infobox.image.replace(//g, "").trim()).replaceAll("?", "%3F").replaceAll("&", "%26"))).request.res.responseUrl; } catch (e) { try { if (typeof infobox.image !== "undefined") episode.cover = e.request.res.responseUrl; } catch (e2) { console.error(e2); throw e; } } if (typeof infobox.featured !== "undefined") episode.characters = infobox.featured.replace(/\[\[(([a-zA-Z0-9 .\-# _\\\/]*)\|([a-zA-Z0-9 .\-_\\\/]*)|([a-zA-Z0-9 .\-#_\\\/]*))\]\]/gm, "$3$4").replace(/(<(\/| |)(\/| |)br(\/| |)(\/| |)>)/gm, "|||").split("|||") } season.episodes.push(episode); index++; } season.count = season.episodes.length; season.date = season.episodes[0].date; data.seasons.push(season); } all.push(data); } fs.writeFileSync("./data/series.json", JSON.stringify(all, null, 4)); })();