aboutsummaryrefslogtreecommitdiff
path: root/update
diff options
context:
space:
mode:
Diffstat (limited to 'update')
-rw-r--r--update/dict.js35
-rw-r--r--update/index.js4
-rw-r--r--update/infobox.js30
-rw-r--r--update/pages.js31
-rw-r--r--update/parse.js68
5 files changed, 168 insertions, 0 deletions
diff --git a/update/dict.js b/update/dict.js
new file mode 100644
index 0000000..24eb400
--- /dev/null
+++ b/update/dict.js
@@ -0,0 +1,35 @@
+const fs = require('fs');
+
+console.log("Optimizing search engine...");
+
+let search = {
+ entries: null,
+ associations: []
+}
+
+for (let page of JSON.parse(fs.readFileSync("./data/pages.json").toString())) {
+ search.associations.push({
+ title: page.query.toLowerCase().replace(/[^a-z]/gm, " ").replace(/\s\s+/g, " ").trim(),
+ endpoint: page.name
+ });
+ search.associations.push({
+ title: page.name.toLowerCase().replace(/[^a-z]/gm, " ").replace(/\s\s+/g, " ").trim(),
+ endpoint: page.name
+ });
+
+ if (typeof JSON.parse(fs.readFileSync("./data/data.json").toString())[page.name] !== "undefined") {
+ for (let nick of JSON.parse(fs.readFileSync("./data/data.json").toString())[page.name].names) {
+ search.associations.push({
+ title: nick.toLowerCase().replace(/[^a-z]/gm, " ").replace(/\s\s+/g, " ").trim(),
+ endpoint: page.name
+ });
+ }
+ }
+}
+
+search.entries = [];
+for (let association of search.associations) {
+ search.entries.push(association.title);
+}
+search.entries = [...new Set(search.entries)];
+fs.writeFileSync("./data/search.json", JSON.stringify(search, null, 4)); \ No newline at end of file
diff --git a/update/index.js b/update/index.js
new file mode 100644
index 0000000..7121c2d
--- /dev/null
+++ b/update/index.js
@@ -0,0 +1,4 @@
+require('child_process').spawnSync("node", ["update/pages.js"], { cwd: __dirname + "/..", stdio: "inherit" })
+require('child_process').spawnSync("node", ["update/infobox.js"], { cwd: __dirname + "/..", stdio: "inherit" })
+require('child_process').spawnSync("node", ["update/parse.js"], { cwd: __dirname + "/..", stdio: "inherit" })
+require('child_process').spawnSync("node", ["update/dict.js"], { cwd: __dirname + "/..", stdio: "inherit" }) \ No newline at end of file
diff --git a/update/infobox.js b/update/infobox.js
new file mode 100644
index 0000000..26a989d
--- /dev/null
+++ b/update/infobox.js
@@ -0,0 +1,30 @@
+const fs = require('fs');
+const axios = require("axios");
+const WikiTextParser = require('parse-wikitext');
+const parser = new WikiTextParser("mlp.fandom.com");
+
+console.log("Gathering infobox for each page...");
+
+(async () => {
+ let infoboxes = {};
+ for (let page of JSON.parse(fs.readFileSync("./data/pages.json").toString())) {
+ console.log("Gathering infobox for '" + page.name + "'...");
+ try {
+ let data = (await axios.get("https://mlp.fandom.com/api.php?action=query&prop=revisions&titles=" + page.name + "&rvslots=*&rvprop=content&formatversion=2&format=json")).data;
+ if (data.query.pages.length > 0) {
+ console.log("Results found, adding name to database")
+ sections = parser.pageToSectionObject(data.query.pages[0].revisions[0].slots.main.content);
+ box = parser.parseInfoBox(sections["content"]);
+ if (box.template === "Infobox character") {
+ infoboxes[page.name] = parser.parseInfoBox(sections["content"]).values;
+ }
+ } else {
+ console.log("No results found, ignoring name");
+ }
+ } catch (e) {
+ console.error(e);
+ }
+ }
+
+ fs.writeFileSync("./data/boxes.json", JSON.stringify(infoboxes, null, 4))
+})() \ No newline at end of file
diff --git a/update/pages.js b/update/pages.js
new file mode 100644
index 0000000..8449ca5
--- /dev/null
+++ b/update/pages.js
@@ -0,0 +1,31 @@
+const fs = require('fs');
+const axios = require('axios');
+
+if (fs.existsSync("./data")) fs.rmSync("./data", { recursive: true });
+fs.mkdirSync("./data");
+
+(async () => {
+ console.log("Gathering pages list...");
+ let pages = [];
+ for (let page of JSON.parse(fs.readFileSync("list.json").toString())) {
+ console.log("Searching for '" + page + "'...");
+ try {
+ let data = (await axios.get("https://mlp.fandom.com/api.php?action=query&list=search&srsearch=" + encodeURI(page) + "&srlimit=1&srenablerewrites=true&format=json")).data;
+ if (data.query.search.length > 0) {
+ console.log("Results found, adding name to database")
+ pages.push({
+ query: page,
+ name: data.query.search[0].title,
+ mwid: data.query.search[0].pageid,
+ words: data.query.search[0].wordcount,
+ })
+ } else {
+ console.log("No results found, ignoring name");
+ }
+ } catch (e) {
+ console.error(e);
+ }
+ }
+
+ fs.writeFileSync("./data/pages.json", JSON.stringify(pages, null, 4))
+})() \ No newline at end of file
diff --git a/update/parse.js b/update/parse.js
new file mode 100644
index 0000000..570c3fa
--- /dev/null
+++ b/update/parse.js
@@ -0,0 +1,68 @@
+const fs = require('fs');
+
+console.log("Parsing infobox data...");
+
+let ponies = {};
+
+(async () => {
+ for (let title in JSON.parse(fs.readFileSync("./data/boxes.json").toString())) {
+ let box = JSON.parse(fs.readFileSync("./data/boxes.json").toString())[title];
+ let data = {
+ names: [title],
+ color: "000000",
+ image: "https://example.com",
+ kind: "Pony",
+ sex: "Unknown",
+ occupation: ["Unknown"],
+ residence: ["Unknown"],
+ mark: "https://example.com"
+ }
+
+ if (typeof box.name2 !== "undefined") data.names.push(box.name2);
+ if (typeof box.name3 !== "undefined") data.names.push(box.name3);
+ if (typeof box.name4 !== "undefined") data.names.push(box.name4);
+ if (typeof box.name5 !== "undefined") data.names.push(box.name5);
+
+ if (typeof box.nicknames !== "undefined") {
+ box.nicknames.split(",").filter(e => !e.match(/[^a-zA-Z0-9-_ ]/gm)).forEach((e, i) => {
+ data.names.push(e.trim());
+ });
+ }
+
+ if (typeof box.kind !== "undefined") {
+ kp = box.kind.replace(/[^a-zA-Z0-9-_ ]/gm, "").split(" ")[0];
+ data.kind = kp.substr(kp.replace(/([A-Z])([a-z0-9]*)$/g, "").length);
+ }
+ if (typeof box.sex !== "undefined") data.sex = box.sex.startsWith("F") ? "F" : "M";
+ if (typeof box.coat !== "undefined") data.color = box.coat.replace(/\[(.*)\/(.{6})\/ (.*)\]/gm, "$2").replace(/{{perbang\|([0-9A-Fa-f].{5})(.*)/g, "$1");
+ if (typeof box.main !== "undefined") data.image = "https://mlp.fandom.com/Special:FilePath/" + encodeURI(box.main);
+ if (typeof box.main1 !== "undefined") data.image = "https://mlp.fandom.com/Special:FilePath/" + encodeURI(box.main1);
+ if (typeof box["cutie mark"] !== "undefined") {
+ try {
+ data.markimg = box["cutie mark"].split("[[File:")[1].split("|")[0];
+ } catch (e) {
+ data.markimg = box["cutie mark"].split("[[File:")[0].split("|")[0];
+ }
+ data.mark = "https://mlp.fandom.com/Special:Redirect/file/" + encodeURI(data.markimg) + "?width=128";
+ }
+
+ if (typeof box.occupation !== "undefined") {
+ occupations = [];
+ box.occupation.replace(/\((.*)\)/gm, "").replace(/\[\[(.*)_(.*)\]\]|\[\[(.*)\|(.*)\]\]|\[\[(.*)\]\]/gm, "$2$4$5").replace(/\|/gm, "_").replace(/<( ||(|| )\/)( ||(|| )\/)(b|B)(r|R)( ||(|| )\/)( ||(|| )\/)>/gm, "|").replace(/( \|| \| | \| )/gm, "|").split("|").forEach((e) => {
+ occupations.push(e.trim().replace(/[\[\]]/gm, "").replace(/<(.*)>/gm, ""));
+ })
+ data.occupation = occupations;
+ }
+ if (typeof box.residence !== "undefined") {
+ residences = [];
+ box.residence.replace(/\((.*)\)/gm, "").replace(/\[\[(.*)_(.*)\]\]|\[\[(.*)\|(.*)\]\]|\[\[(.*)\]\]/gm, "$2$4$5").replace(/\|/gm, "_").replace(/<( ||(|| )\/)( ||(|| )\/)(b|B)(r|R)( ||(|| )\/)( ||(|| )\/)>/gm, "|").replace(/( \|| \| | \| )/gm, "|").split("|").forEach((e) => {
+ residences.push(e.trim().replace(/[\[\]]/gm, "").replace(/<(.*)>/gm, ""));
+ })
+ data.residence = residences;
+ }
+
+ ponies[title] = data;
+ }
+})()
+
+fs.writeFileSync("./data/data.json", JSON.stringify(ponies, null, 4)); \ No newline at end of file
© Floofi Systems, Some rights reserved. Powered by cgit.