1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
|
const config = require("./series.json");
const axios = require('axios');
const fs = require('fs');
const WikiTextParser = require('parse-wikitext');
const parser = new WikiTextParser("mlp.fandom.com");
(async () => {
let all = [];
for (let series of config) {
let data = {
command: series.cmd,
title: series.title,
date: series.date,
link: series.page,
description: {
series: null,
plot: null,
},
seasons: []
};
console.log(series.title + " (" + data.date + ")");
ddata = (await axios.get("https://mlp.fandom.com/api.php?format=json&action=query&prop=extracts&exlimit=max&explaintext&exintro&titles=" + encodeURI(series.page) + "&redirects=")).data;
data.description.series = ddata.query.pages[Object.keys(ddata.query.pages)[0]].extract.split("\n")[0];
data.description.plot = ddata.query.pages[Object.keys(ddata.query.pages)[0]].extract.split("\n")[1];
for (let s of series.seasons) {
console.log(" Season " + s.id);
let season = {
name: "Season " + s.id,
id: "s" + s.id,
date: null,
count: null,
episodes: []
}
let sdata = (await axios.get("https://mlp.fandom.com/api.php?action=query&prop=revisions&titles=" + s.table.page + "&rvslots=*&rvprop=content&formatversion=2&format=json")).data;
let sections = parser.pageToSectionObject(sdata.query.pages[0].revisions[0].slots.main.content);
let episodes = sections.Episodes[s.table.section].content
.filter(i => i.startsWith("|"))
.join("||")
.replace("|}", "")
.split("|-").map(i => {
return i.split("||").filter(i => i.trim() !== "");
})
.filter(i => i.length > 1)
.map(i => {
let index = 0;
return i.map(i => {
return i.replace(/^(\| |\|)(.*)/gm, "$2")
.replace(/^style="(.*)"( |)\|( |)(.*)/gm, "$4")
.replace(/\[\[((.*)\|(.*)|(.*))\]\]/gm, "$3$4");
}).map(i => {
if (index === 0) {
index++;
return i.replace(/(.*) \((.*)\)/gm, "$1|$2");
} else {
index++;
return i;
}
})
});
index = 1;
for (let e of episodes) {
process.stdout.write(" S" + s.id + "E" + index);
let episode = {
local: null,
global: null,
date: null,
name: null,
writer: null,
cover: null,
characters: [],
plot: null
}
episode.local = e[s.table.fields.number].replace(/(.*)\|(.*)/gm, "$1").replace(/''(.*)''/gm, "$1").replace(/{{(.*)\|(.*)}}/gm, "$2").replace(/(.*)\|(.*)/gm, "$2");
episode.global = e[s.table.fields.number].replace(/(.*)\|(.*)/gm, "$2");
episode.date_pre = e[s.table.fields.date].replace(/{{(.*)\|(.*)\|(.*)\|link=(no|yes)}}/gm, "$3").replace(/{{(.*)\|(.*)}}/gm, "$2").replace(/(.*)\|(.*)/gm, "$2");
try {
episode.date = new Date(e[s.table.fields.date].replace(/{{(.*)\|(.*)\|(.*)\|link=(no|yes)}}/gm, "$3").replace(/{{(.*)\|(.*)}}/gm, "$2").replace(/(.*)\|(.*)/gm, "$2")).toISOString();
} catch (e) {
episode.date = null;
}
episode.name = e[s.table.fields.title];
episode.writer = e[s.table.fields.writer].replace(/{{(.*)\|(.*)\|(.*)\|link=(no|yes)}}/gm, "$3").replace(/(]]|\[\[)/gm, "").replace(/(.*)\|(.*)/gm, "$2");
process.stdout.write(": " + episode.name + "\n");
if (episode.local.length !== 2 || episode.global.length > 3 || episode.global.length < 2 || episode.date_pre.length !== 10) {
console.log("FAILURE: " + JSON.stringify(episode));
}
let edata = (await axios.get("https://mlp.fandom.com/api.php?action=query&prop=revisions&titles=" + encodeURI(episode.name).replaceAll("?", "%3F").replaceAll("&", "%26") + "&rvslots=*&rvprop=content&formatversion=2&format=json")).data;
let eext = (await axios.get("https://mlp.fandom.com/api.php?format=json&action=query&prop=extracts&exlimit=max&explaintext&exintro&titles=" + encodeURI(episode.name).replaceAll("?", "%3F").replaceAll("&", "%26") + "&redirects=")).data;
episode.plot = eext.query.pages[Object.keys(eext.query.pages)[0]].extract.replace(/(.*)\n(.*)/, "$2").replace(/(.*)\n\n(.*)/gm, "$2").replace(/([.?!])\s*(?=[A-Z])/g, "$1|").split("|").join(" ");
let infobox = null;
try {
sections = parser.pageToSectionObject(edata.query.pages[0].revisions[0].slots.main.content);
box = parser.parseInfoBox(sections["content"]);
if (box.template === "Infobox episode") {
infobox = parser.parseInfoBox(sections["content"]).values;
}
} catch (e) {
console.log(edata);
throw e;
}
if (infobox !== null) {
try {
if (typeof infobox.image !== "undefined") episode.cover = (await axios.head("https://mlp.fandom.com/Special:FilePath/" + encodeURI(infobox.image.replace(/<!--[\s\S]*?-->/g, "").trim()).replaceAll("?", "%3F").replaceAll("&", "%26"))).request.res.responseUrl;
} catch (e) {
try {
if (typeof infobox.image !== "undefined") episode.cover = e.request.res.responseUrl;
} catch (e2) {
console.error(e2);
throw e;
}
}
if (typeof infobox.featured !== "undefined") episode.characters = infobox.featured.replace(/\[\[(([a-zA-Z0-9 .\-# _\\\/]*)\|([a-zA-Z0-9 .\-_\\\/]*)|([a-zA-Z0-9 .\-#_\\\/]*))\]\]/gm, "$3$4").replace(/(<(\/| |)(\/| |)br(\/| |)(\/| |)>)/gm, "|||").split("|||")
}
season.episodes.push(episode);
index++;
}
season.count = season.episodes.length;
season.date = season.episodes[0].date;
data.seasons.push(season);
}
all.push(data);
}
fs.writeFileSync("./data/series.json", JSON.stringify(all, null, 4));
})();
|
