aboutsummaryrefslogtreecommitdiff
path: root/update
diff options
context:
space:
mode:
Diffstat (limited to 'update')
-rw-r--r--update/episodes/index.js141
-rw-r--r--update/episodes/series.json127
-rw-r--r--update/ponies/dict.js (renamed from update/dict.js)0
-rw-r--r--update/ponies/index.js (renamed from update/index.js)0
-rw-r--r--update/ponies/infobox.js (renamed from update/infobox.js)0
-rw-r--r--update/ponies/listgen.js (renamed from update/listgen.js)0
-rw-r--r--update/ponies/pages.js (renamed from update/pages.js)0
-rw-r--r--update/ponies/parse.js (renamed from update/parse.js)51
8 files changed, 309 insertions, 10 deletions
diff --git a/update/episodes/index.js b/update/episodes/index.js
new file mode 100644
index 0000000..f071707
--- /dev/null
+++ b/update/episodes/index.js
@@ -0,0 +1,141 @@
+const config = require("./series.json");
+const axios = require('axios');
+const fs = require('fs');
+const WikiTextParser = require('parse-wikitext');
+const parser = new WikiTextParser("mlp.fandom.com");
+
+(async () => {
+ let all = [];
+
+ for (let series of config) {
+ let data = {
+ command: series.cmd,
+ title: series.title,
+ date: series.date,
+ link: series.page,
+ description: {
+ series: null,
+ plot: null,
+ },
+ seasons: []
+ };
+ console.log(series.title + " (" + data.date + ")");
+
+ ddata = (await axios.get("https://mlp.fandom.com/api.php?format=json&action=query&prop=extracts&exlimit=max&explaintext&exintro&titles=" + encodeURI(series.page) + "&redirects=")).data;
+ data.description.series = ddata.query.pages[Object.keys(ddata.query.pages)[0]].extract.split("\n")[0];
+ data.description.plot = ddata.query.pages[Object.keys(ddata.query.pages)[0]].extract.split("\n")[1];
+
+ for (let s of series.seasons) {
+ console.log(" Season " + s.id);
+
+ let season = {
+ name: "Season " + s.id,
+ id: "s" + s.id,
+ date: null,
+ count: null,
+ episodes: []
+ }
+
+ let sdata = (await axios.get("https://mlp.fandom.com/api.php?action=query&prop=revisions&titles=" + s.table.page + "&rvslots=*&rvprop=content&formatversion=2&format=json")).data;
+ let sections = parser.pageToSectionObject(sdata.query.pages[0].revisions[0].slots.main.content);
+ let episodes = sections.Episodes[s.table.section].content
+ .filter(i => i.startsWith("|"))
+ .join("||")
+ .replace("|}", "")
+ .split("|-").map(i => {
+ return i.split("||").filter(i => i.trim() !== "");
+ })
+ .filter(i => i.length > 1)
+ .map(i => {
+ let index = 0;
+ return i.map(i => {
+ return i.replace(/^(\| |\|)(.*)/gm, "$2")
+ .replace(/^style="(.*)"( |)\|( |)(.*)/gm, "$4")
+ .replace(/\[\[((.*)\|(.*)|(.*))\]\]/gm, "$3$4");
+ }).map(i => {
+ if (index === 0) {
+ index++;
+ return i.replace(/(.*) \((.*)\)/gm, "$1|$2");
+ } else {
+ index++;
+ return i;
+ }
+ })
+ });
+
+ index = 1;
+ for (let e of episodes) {
+ process.stdout.write(" S" + s.id + "E" + index);
+
+ let episode = {
+ local: null,
+ global: null,
+ date: null,
+ name: null,
+ writer: null,
+ cover: null,
+ characters: [],
+ plot: null
+ }
+
+ episode.local = e[s.table.fields.number].replace(/(.*)\|(.*)/gm, "$1").replace(/''(.*)''/gm, "$1").replace(/{{(.*)\|(.*)}}/gm, "$2").replace(/(.*)\|(.*)/gm, "$2");
+ episode.global = e[s.table.fields.number].replace(/(.*)\|(.*)/gm, "$2");
+ episode.date_pre = e[s.table.fields.date].replace(/{{(.*)\|(.*)\|(.*)\|link=(no|yes)}}/gm, "$3").replace(/{{(.*)\|(.*)}}/gm, "$2").replace(/(.*)\|(.*)/gm, "$2");
+ try {
+ episode.date = new Date(e[s.table.fields.date].replace(/{{(.*)\|(.*)\|(.*)\|link=(no|yes)}}/gm, "$3").replace(/{{(.*)\|(.*)}}/gm, "$2").replace(/(.*)\|(.*)/gm, "$2")).toISOString();
+ } catch (e) {
+ episode.date = null;
+ }
+ episode.name = e[s.table.fields.title];
+ episode.writer = e[s.table.fields.writer].replace(/{{(.*)\|(.*)\|(.*)\|link=(no|yes)}}/gm, "$3").replace(/(]]|\[\[)/gm, "").replace(/(.*)\|(.*)/gm, "$2");
+
+ process.stdout.write(": " + episode.name + "\n");
+
+ if (episode.local.length !== 2 || episode.global.length > 3 || episode.global.length < 2 || episode.date_pre.length !== 10) {
+ console.log("FAILURE: " + JSON.stringify(episode));
+ }
+
+ let edata = (await axios.get("https://mlp.fandom.com/api.php?action=query&prop=revisions&titles=" + encodeURI(episode.name).replaceAll("?", "%3F").replaceAll("&", "%26") + "&rvslots=*&rvprop=content&formatversion=2&format=json")).data;
+ let eext = (await axios.get("https://mlp.fandom.com/api.php?format=json&action=query&prop=extracts&exlimit=max&explaintext&exintro&titles=" + encodeURI(episode.name).replaceAll("?", "%3F").replaceAll("&", "%26") + "&redirects=")).data;
+
+ episode.plot = eext.query.pages[Object.keys(eext.query.pages)[0]].extract.replace(/(.*)\n(.*)/, "$2").replace(/(.*)\n\n(.*)/gm, "$2").replace(/([.?!])\s*(?=[A-Z])/g, "$1|").split("|").join(" ");
+
+ let infobox = null;
+ try {
+ sections = parser.pageToSectionObject(edata.query.pages[0].revisions[0].slots.main.content);
+ box = parser.parseInfoBox(sections["content"]);
+ if (box.template === "Infobox episode") {
+ infobox = parser.parseInfoBox(sections["content"]).values;
+ }
+ } catch (e) {
+ console.log(edata);
+ throw e;
+ }
+
+ if (infobox !== null) {
+ try {
+ if (typeof infobox.image !== "undefined") episode.cover = (await axios.head("https://mlp.fandom.com/Special:FilePath/" + encodeURI(infobox.image.replace(/<!--[\s\S]*?-->/g, "").trim()).replaceAll("?", "%3F").replaceAll("&", "%26"))).request.res.responseUrl;
+ } catch (e) {
+ try {
+ if (typeof infobox.image !== "undefined") episode.cover = e.request.res.responseUrl;
+ } catch (e2) {
+ console.error(e2);
+ throw e;
+ }
+ }
+ if (typeof infobox.featured !== "undefined") episode.characters = infobox.featured.replace(/\[\[(([a-zA-Z0-9 .\-# _\\\/]*)\|([a-zA-Z0-9 .\-_\\\/]*)|([a-zA-Z0-9 .\-#_\\\/]*))\]\]/gm, "$3$4").replace(/(<(\/| |)(\/| |)br(\/| |)(\/| |)>)/gm, "|||").split("|||")
+ }
+
+ season.episodes.push(episode);
+ index++;
+ }
+
+ season.count = season.episodes.length;
+ season.date = season.episodes[0].date;
+ data.seasons.push(season);
+ }
+ all.push(data);
+ }
+
+ fs.writeFileSync("./data/series.json", JSON.stringify(all, null, 4));
+})(); \ No newline at end of file
diff --git a/update/episodes/series.json b/update/episodes/series.json
new file mode 100644
index 0000000..0935658
--- /dev/null
+++ b/update/episodes/series.json
@@ -0,0 +1,127 @@
+[
+ {
+ "title": "Friendship is Magic",
+ "cmd": "FIM",
+ "page": "My Little Pony Friendship is Magic",
+ "date": "2010-2019",
+ "seasons": [
+ {
+ "id": 1,
+ "table": {
+ "page": "Friendship is Magic animated media",
+ "section": "Season one",
+ "fields": {
+ "number": 0,
+ "title": 1,
+ "writer": 2,
+ "date": 3
+ }
+ }
+ },
+ {
+ "id": 2,
+ "table": {
+ "page": "Friendship is Magic animated media",
+ "section": "Season two",
+ "fields": {
+ "number": 0,
+ "title": 1,
+ "writer": 2,
+ "date": 3
+ }
+ }
+ },
+ {
+ "id": 3,
+ "table": {
+ "page": "Friendship is Magic animated media",
+ "section": "Season three",
+ "fields": {
+ "number": 0,
+ "title": 1,
+ "writer": 2,
+ "date": 3
+ }
+ }
+ },
+ {
+ "id": 4,
+ "table": {
+ "page": "Friendship is Magic animated media",
+ "section": "Season four",
+ "fields": {
+ "number": 0,
+ "title": 1,
+ "writer": 2,
+ "date": 3
+ }
+ }
+ },
+ {
+ "id": 5,
+ "table": {
+ "page": "Friendship is Magic animated media",
+ "section": "Season five",
+ "fields": {
+ "number": 0,
+ "title": 1,
+ "writer": 2,
+ "date": 3
+ }
+ }
+ },
+ {
+ "id": 6,
+ "table": {
+ "page": "Friendship is Magic animated media",
+ "section": "Season six",
+ "fields": {
+ "number": 0,
+ "title": 1,
+ "writer": 2,
+ "date": 3
+ }
+ }
+ },
+ {
+ "id": 7,
+ "table": {
+ "page": "Friendship is Magic animated media",
+ "section": "Season seven",
+ "fields": {
+ "number": 0,
+ "title": 1,
+ "writer": 2,
+ "date": 3
+ }
+ }
+ },
+ {
+ "id": 8,
+ "table": {
+ "page": "Friendship is Magic animated media",
+ "section": "Season eight",
+ "fields": {
+ "number": 0,
+ "title": 1,
+ "writer": 2,
+ "date": 3
+ }
+ }
+ },
+ {
+ "id": 9,
+ "table": {
+ "page": "Friendship is Magic animated media",
+ "section": "Season nine",
+ "fields": {
+ "number": 0,
+ "title": 1,
+ "writer": 2,
+ "date": 3
+ }
+ }
+ }
+ ]
+ }
+] \ No newline at end of file
diff --git a/update/dict.js b/update/ponies/dict.js
index 88ec568..88ec568 100644
--- a/update/dict.js
+++ b/update/ponies/dict.js
diff --git a/update/index.js b/update/ponies/index.js
index 53acf34..53acf34 100644
--- a/update/index.js
+++ b/update/ponies/index.js
diff --git a/update/infobox.js b/update/ponies/infobox.js
index f53be0e..f53be0e 100644
--- a/update/infobox.js
+++ b/update/ponies/infobox.js
diff --git a/update/listgen.js b/update/ponies/listgen.js
index d9401f8..d9401f8 100644
--- a/update/listgen.js
+++ b/update/ponies/listgen.js
diff --git a/update/pages.js b/update/ponies/pages.js
index 7f3a1c8..7f3a1c8 100644
--- a/update/pages.js
+++ b/update/ponies/pages.js
diff --git a/update/parse.js b/update/ponies/parse.js
index 0b2baa9..bd06081 100644
--- a/update/parse.js
+++ b/update/ponies/parse.js
@@ -38,28 +38,59 @@ let ponies = {};
if (typeof box.kind !== "undefined") {
kp = box.kind.replace(/<!--[\s\S]*?-->/g, "").replace(/[^a-zA-Z0-9-_ ]/gm, "").split(" ")[0];
- data.kind = kp.substr(kp.replace(/([A-Z])([a-z0-9]*)$/g, "").length);
+ data.kind = kp.substring(kp.replace(/([A-Z])([a-z0-9]*)$/g, "").length);
}
if (typeof box.sex !== "undefined") data.sex = box.sex.replace(/<!--[\s\S]*?-->/g, "").trim().startsWith("F") ? "F" : "M";
if (typeof box._extract !== "undefined") {
data.extract = box._extract;
}
- if (typeof box._extract_fr !== "undefined") {
- data.extract_fr = box._extract_fr.replace(/ma petite amitié de poney est la magie/gmi, "My Little Pony Friendship is Magic").replace(/Mon amitié avec mon petit poney est magique/gmi, "My Little Pony Friendship is Magic").replace(/Mon petit Poney/gmi, "My Little Pony").replace(/Mon Petit Poney Une Nouvelle Génération/gmi, "My Little Pony A New Generation").replace(/Mon petit poney : une nouvelle génération/gmi, "My Little Pony: A New Generation").replace(/pegasus/gmi, "pégase").replace(/alicorn[^e]/gmi, "alicorne").replace(/une poney/gmi, "une ponette").replace(/petite génération de poney 5/gmi, "Génération 5");
- } else {
- data.extract_fr = data.extract;
- }
if (typeof box.coat !== "undefined") data.color = box.coat.replace(/<!--[\s\S]*?-->/g, "").trim().replace(/\[([a-z.\/ \nA-Z0-9:]*)\/(.{6})\/ (.*)\]/gm, "$2").replace(/{{perbang\|([0-9A-Fa-f].{5})(.*)/g, "$1");
- if (typeof box.main !== "undefined") data.image = "https://mlp.fandom.com/Special:FilePath/" + encodeURI(box.main.replace(/<!--[\s\S]*?-->/g, "").trim());
- if (typeof box.main1 !== "undefined") data.image = "https://mlp.fandom.com/Special:FilePath/" + encodeURI(box.main1.replace(/<!--[\s\S]*?-->/g, "").trim());
- if (typeof box.image !== "undefined") data.image = "https://mlp.fandom.com/Special:FilePath/" + encodeURI(box.image.replace(/<!--[\s\S]*?-->/g, "").trim());
+ try {
+ if (typeof box.main !== "undefined") data.image = (await axios.head("https://mlp.fandom.com/Special:FilePath/" + encodeURI(box.main.replace(/<!--[\s\S]*?-->/g, "").trim()).replaceAll("?", "%3F").replaceAll("&", "%26"))).request.res.responseUrl;
+ } catch (e) {
+ try {
+ if (typeof box.main !== "undefined") data.image = e.request.res.responseUrl;
+ } catch (e2) {
+ console.error(e2);
+ throw e;
+ }
+ }
+ try {
+ if (typeof box.main1 !== "undefined") data.image = (await axios.head("https://mlp.fandom.com/Special:FilePath/" + encodeURI(box.main1.replace(/<!--[\s\S]*?-->/g, "").trim()).replaceAll("?", "%3F").replaceAll("&", "%26"))).request.res.responseUrl;
+ } catch (e) {
+ try {
+ if (typeof box.main1 !== "undefined") data.image = e.request.res.responseUrl;
+ } catch (e2) {
+ console.error(e2);
+ throw e;
+ }
+ }
+ try {
+ if (typeof box.image !== "undefined") data.image = (await axios.head("https://mlp.fandom.com/Special:FilePath/" + encodeURI(box.image.replace(/<!--[\s\S]*?-->/g, "").trim()).replaceAll("?", "%3F").replaceAll("&", "%26"))).request.res.responseUrl;
+ } catch (e) {
+ try {
+ if (typeof box.image !== "undefined") data.image = e.request.res.responseUrl;
+ } catch (e2) {
+ console.error(e2);
+ throw e;
+ }
+ }
if (typeof box["cutie mark"] !== "undefined") {
try {
data.markimg = box["cutie mark"].replace(/<!--[\s\S]*?-->/g, "").trim().split("[[File:")[1].split("|")[0];
} catch (e) {
data.markimg = box["cutie mark"].replace(/<!--[\s\S]*?-->/g, "").trim().split("[[File:")[0].split("|")[0];
}
- data.mark = "https://mlp.fandom.com/Special:Redirect/file/" + encodeURI(data.markimg) + "?width=128";
+ try {
+ data.mark = (await axios.head("https://mlp.fandom.com/Special:FilePath/" + encodeURI(data.markimg).replaceAll("?", "%3F").replaceAll("&", "%26"))).request.res.responseUrl;
+ } catch (e) {
+ try {
+ data.mark = e.request.res.responseUrl;
+ } catch (e2) {
+ console.error(e2);
+ throw e;
+ }
+ }
}
if (typeof box.occupation !== "undefined") {
© Floofi Systems, Some rights reserved. Powered by cgit.