Feature: implements #8, voids #9

1 files changed, 141 insertions, 0 deletions

diff --git a/update/episodes/index.js b/update/episodes/index.js
new file mode 100644
index 0000000..f071707
--- /dev/null
+++ b/update/episodes/index.js
@@ -0,0 +1,141 @@
+const config = require("./series.json");
+const axios = require('axios');
+const fs = require('fs');
+const WikiTextParser = require('parse-wikitext');
+const parser = new WikiTextParser("mlp.fandom.com");
+
+(async () => {
+    let all = [];
+
+    for (let series of config) {
+        let data = {
+            command: series.cmd,
+            title: series.title,
+            date: series.date,
+            link: series.page,
+            description: {
+                series: null,
+                plot: null,
+            },
+            seasons: []
+        };
+        console.log(series.title + " (" + data.date + ")");
+
+        ddata = (await axios.get("https://mlp.fandom.com/api.php?format=json&action=query&prop=extracts&exlimit=max&explaintext&exintro&titles=" + encodeURI(series.page) + "&redirects=")).data;
+        data.description.series = ddata.query.pages[Object.keys(ddata.query.pages)[0]].extract.split("\n")[0];
+        data.description.plot = ddata.query.pages[Object.keys(ddata.query.pages)[0]].extract.split("\n")[1];
+
+        for (let s of series.seasons) {
+            console.log("    Season " + s.id);
+
+            let season = {
+                name: "Season " + s.id,
+                id: "s" + s.id,
+                date: null,
+                count: null,
+                episodes: []
+            }
+
+            let sdata = (await axios.get("https://mlp.fandom.com/api.php?action=query&prop=revisions&titles=" + s.table.page + "&rvslots=*&rvprop=content&formatversion=2&format=json")).data;
+            let sections = parser.pageToSectionObject(sdata.query.pages[0].revisions[0].slots.main.content);
+            let episodes = sections.Episodes[s.table.section].content
+                .filter(i => i.startsWith("|"))
+                .join("||")
+                .replace("|}", "")
+                .split("|-").map(i => {
+                    return i.split("||").filter(i => i.trim() !== "");
+                })
+                .filter(i => i.length > 1)
+                .map(i => {
+                    let index = 0;
+                    return i.map(i => {
+                        return i.replace(/^(\| |\|)(.*)/gm, "$2")
+                            .replace(/^style="(.*)"( |)\|( |)(.*)/gm, "$4")
+                            .replace(/\[\[((.*)\|(.*)|(.*))\]\]/gm, "$3$4");
+                    }).map(i => {
+                        if (index === 0) {
+                            index++;
+                            return i.replace(/(.*) \((.*)\)/gm, "$1|$2");
+                        } else {
+                            index++;
+                            return i;
+                        }
+                    })
+                });
+
+            index = 1;
+            for (let e of episodes) {
+                process.stdout.write("        S" + s.id + "E" + index);
+
+                let episode = {
+                    local: null,
+                    global: null,
+                    date: null,
+                    name: null,
+                    writer: null,
+                    cover: null,
+                    characters: [],
+                    plot: null
+                }
+
+                episode.local = e[s.table.fields.number].replace(/(.*)\|(.*)/gm, "$1").replace(/''(.*)''/gm, "$1").replace(/{{(.*)\|(.*)}}/gm, "$2").replace(/(.*)\|(.*)/gm, "$2");
+                episode.global = e[s.table.fields.number].replace(/(.*)\|(.*)/gm, "$2");
+                episode.date_pre = e[s.table.fields.date].replace(/{{(.*)\|(.*)\|(.*)\|link=(no|yes)}}/gm, "$3").replace(/{{(.*)\|(.*)}}/gm, "$2").replace(/(.*)\|(.*)/gm, "$2");
+                try {
+                    episode.date = new Date(e[s.table.fields.date].replace(/{{(.*)\|(.*)\|(.*)\|link=(no|yes)}}/gm, "$3").replace(/{{(.*)\|(.*)}}/gm, "$2").replace(/(.*)\|(.*)/gm, "$2")).toISOString();
+                } catch (e) {
+                    episode.date = null;
+                }
+                episode.name = e[s.table.fields.title];
+                episode.writer = e[s.table.fields.writer].replace(/{{(.*)\|(.*)\|(.*)\|link=(no|yes)}}/gm, "$3").replace(/(]]|\[\[)/gm, "").replace(/(.*)\|(.*)/gm, "$2");
+
+                process.stdout.write(": " + episode.name + "\n");
+
+                if (episode.local.length !== 2 || episode.global.length > 3 || episode.global.length < 2 || episode.date_pre.length !== 10) {
+                    console.log("FAILURE: " + JSON.stringify(episode));
+                }
+
+                let edata = (await axios.get("https://mlp.fandom.com/api.php?action=query&prop=revisions&titles=" + encodeURI(episode.name).replaceAll("?", "%3F").replaceAll("&", "%26") + "&rvslots=*&rvprop=content&formatversion=2&format=json")).data;
+                let eext = (await axios.get("https://mlp.fandom.com/api.php?format=json&action=query&prop=extracts&exlimit=max&explaintext&exintro&titles=" + encodeURI(episode.name).replaceAll("?", "%3F").replaceAll("&", "%26") + "&redirects=")).data;
+
+                episode.plot = eext.query.pages[Object.keys(eext.query.pages)[0]].extract.replace(/(.*)\n(.*)/, "$2").replace(/(.*)\n\n(.*)/gm, "$2").replace(/([.?!])\s*(?=[A-Z])/g, "$1|").split("|").join(" ");
+
+                let infobox = null;
+                try {
+                    sections = parser.pageToSectionObject(edata.query.pages[0].revisions[0].slots.main.content);
+                    box = parser.parseInfoBox(sections["content"]);
+                    if (box.template === "Infobox episode") {
+                        infobox = parser.parseInfoBox(sections["content"]).values;
+                    }
+                } catch (e) {
+                    console.log(edata);
+                    throw e;
+                }
+
+                if (infobox !== null) {
+                    try {
+                        if (typeof infobox.image !== "undefined") episode.cover = (await axios.head("https://mlp.fandom.com/Special:FilePath/" + encodeURI(infobox.image.replace(/<!--[\s\S]*?-->/g, "").trim()).replaceAll("?", "%3F").replaceAll("&", "%26"))).request.res.responseUrl;
+                    } catch (e) {
+                        try {
+                            if (typeof infobox.image !== "undefined") episode.cover = e.request.res.responseUrl;
+                        } catch (e2) {
+                            console.error(e2);
+                            throw e;
+                        }
+                    }
+                    if (typeof infobox.featured !== "undefined") episode.characters = infobox.featured.replace(/\[\[(([a-zA-Z0-9 .\-#   _\\\/]*)\|([a-zA-Z0-9 .\-_\\\/]*)|([a-zA-Z0-9 .\-#_\\\/]*))\]\]/gm, "$3$4").replace(/(<(\/| |)(\/| |)br(\/| |)(\/| |)>)/gm, "|||").split("|||")
+                }
+
+                season.episodes.push(episode);
+                index++;
+            }
+
+            season.count = season.episodes.length;
+            season.date = season.episodes[0].date;
+            data.seasons.push(season);
+        }
+        all.push(data);
+    }
+
+    fs.writeFileSync("./data/series.json", JSON.stringify(all, null, 4));
+})();
\ No newline at end of file