From 327119b4d1c2248b8a075cad3cd05ab92560e75d Mon Sep 17 00:00:00 2001 From: Minteck Date: Sun, 13 Feb 2022 16:16:18 +0100 Subject: Feature: implements #8, voids #9 --- bot.js | 5 +- commands/episode.js | 17 ++++++ commands/info.js | 2 +- config/version.txt | 2 +- handler/autocomplete.js | 13 ++++ handler/button.js | 50 +++++++++++++++- handler/command.js | 5 ++ handler/menu.js | 2 +- modules/episodepublic.js | 16 +++++ modules/episodereply.js | 75 +++++++++++++++++++++++ modules/findepisode.js | 52 ++++++++++++++++ modules/registers.js | 44 +++++++------- package-lock.json | 2 +- update/dict.js | 41 ------------- update/episodes/index.js | 141 ++++++++++++++++++++++++++++++++++++++++++++ update/episodes/series.json | 127 +++++++++++++++++++++++++++++++++++++++ update/index.js | 5 -- update/infobox.js | 71 ---------------------- update/listgen.js | 24 -------- update/pages.js | 28 --------- update/parse.js | 84 -------------------------- update/ponies/dict.js | 41 +++++++++++++ update/ponies/index.js | 5 ++ update/ponies/infobox.js | 71 ++++++++++++++++++++++ update/ponies/listgen.js | 24 ++++++++ update/ponies/pages.js | 28 +++++++++ update/ponies/parse.js | 115 ++++++++++++++++++++++++++++++++++++ 27 files changed, 806 insertions(+), 284 deletions(-) create mode 100644 commands/episode.js create mode 100644 handler/autocomplete.js create mode 100644 modules/episodepublic.js create mode 100644 modules/episodereply.js create mode 100644 modules/findepisode.js delete mode 100644 update/dict.js create mode 100644 update/episodes/index.js create mode 100644 update/episodes/series.json delete mode 100644 update/index.js delete mode 100644 update/infobox.js delete mode 100644 update/listgen.js delete mode 100644 update/pages.js delete mode 100644 update/parse.js create mode 100644 update/ponies/dict.js create mode 100644 update/ponies/index.js create mode 100644 update/ponies/infobox.js create mode 100644 update/ponies/listgen.js create mode 100644 update/ponies/pages.js create mode 100644 update/ponies/parse.js diff --git a/bot.js b/bot.js index 6f993a7..2eee3da 100644 --- a/bot.js +++ b/bot.js @@ -33,6 +33,7 @@ const commandHandler = require('./handler/command'); const buttonHandler = require('./handler/button'); const menuHandler = require('./handler/menu'); const errorHandler = require('./handler/errors'); +const autocompleteHandler = require('./handler/autocomplete'); global.langs = JSON.parse(fs.readFileSync("./user/userdata.json").toString()); global.spoils = JSON.parse(fs.readFileSync("./user/spoilers.json").toString()); @@ -122,8 +123,10 @@ client.on('interactionCreate', async interaction => { await buttonHandler(interaction); } else if (interaction.isSelectMenu()) { await menuHandler(interaction); + } else if (interaction.isAutocomplete()) { + await autocompleteHandler(interaction); } else { - throw new Error("Interaction type not supported") + errorHandler(interaction, new Error("Interaction type not supported")) } } catch (e) { errorHandler(interaction, e) diff --git a/commands/episode.js b/commands/episode.js new file mode 100644 index 0000000..bf5b9f2 --- /dev/null +++ b/commands/episode.js @@ -0,0 +1,17 @@ +const fs = require('fs'); +const { MessageActionRow, MessageButton, MessageSelectMenu, MessageEmbed } = require('discord.js'); + +const getEpisodePublic = require("../modules/episodepublic"); +const episodeReply = require("../modules/episodereply"); + +const series = JSON.parse(fs.readFileSync("./data/series.json").toString()); + +module.exports = async (interaction) => { + let select = interaction.options.getSubcommand(); + let episode = interaction.options.getString('episode').toLowerCase(); + + let sdata = series.filter(i => i.command.toLowerCase() === select)[0]; + if (typeof episode === "string") { + await episodeReply(interaction, select, episode, sdata); + } +} \ No newline at end of file diff --git a/commands/info.js b/commands/info.js index 77082bf..47d290a 100644 --- a/commands/info.js +++ b/commands/info.js @@ -76,7 +76,7 @@ module.exports = async (interaction) => { } let fields = [ - { name: l("Software version", "Version du logiciel", interaction.user.id, interaction.guild ? interaction.guild.id : 0), value: "v" + fs.readFileSync("./config/version.txt").toString().trim() + "." + fs.readFileSync("./.git/refs/heads/trunk").toString().substr(0, 8) + suffix + " (#" + client.shard.count + ")", inline: false }, + { name: l("Software version", "Version du logiciel", interaction.user.id, interaction.guild ? interaction.guild.id : 0), value: "v" + fs.readFileSync("./config/version.txt").toString().trim() + "." + fs.readFileSync("./.git/refs/heads/trunk").toString().substring(0, 8) + suffix + " (#" + client.shard.count + ")", inline: false }, { name: l("Kernel version", "Version du noyau", interaction.user.id, interaction.guild ? interaction.guild.id : 0), value: process.version, inline: true }, { name: l("Experience channel", "Canal d'expériences", interaction.user.id, interaction.guild ? interaction.guild.id : 0), value: channel, inline: true }, { name: l("Known ponies", "Poneys connus", interaction.user.id, interaction.guild ? interaction.guild.id : 0), value: Object.keys(JSON.parse(fs.readFileSync("./data/data.json").toString())).length.toString(), inline: true }, diff --git a/config/version.txt b/config/version.txt index bd21590..0750769 100644 --- a/config/version.txt +++ b/config/version.txt @@ -1 +1 @@ -1.4.45 \ No newline at end of file +1.4.49 \ No newline at end of file diff --git a/handler/autocomplete.js b/handler/autocomplete.js new file mode 100644 index 0000000..5696ab7 --- /dev/null +++ b/handler/autocomplete.js @@ -0,0 +1,13 @@ +const getEpisodes = require('../modules/findepisode'); +const fs = require("fs"); +const episodes = JSON.parse(fs.readFileSync("./data/series.json").toString()); + +module.exports = async (interaction) => { + try { + let series = interaction.options.getSubcommand(); + + if (interaction.commandName === "episode") { + await interaction.respond(getEpisodes(episodes.filter(i => i.command.toLowerCase() === series)[0], interaction.options.getFocused())); + } + } catch (e) {} +} \ No newline at end of file diff --git a/handler/button.js b/handler/button.js index f9f283e..4a8a343 100644 --- a/handler/button.js +++ b/handler/button.js @@ -5,6 +5,8 @@ const { MessageActionRow, MessageButton, MessageSelectMenu, MessageEmbed, Messag const getPixel = require('../modules/pixel.js'); const getEmbed = require("../modules/embed"); const getPublic = require("../modules/public"); +const episodeReply = require("../modules/episodereply"); +const series = JSON.parse(fs.readFileSync("./data/series.json").toString()); module.exports = async (interaction) => { keys = interaction.customId.split("|") @@ -14,9 +16,51 @@ module.exports = async (interaction) => { } else { statsButtons[keys[0]] = 1; } - fs.writeFile("./stats/buttons.json", JSON.stringify(statsButtons), () => {}); + fs.writeFile("./stats/buttons.json", JSON.stringify(statsButtons), () => { + }); - if (keys[0] === "pony.display") { + if (keys[0] === "episode.public") { + if (typeof cooldowns[interaction.user.id] !== "undefined" && new Date() - cooldowns[interaction.user.id] < 30000) { + await interaction.reply({ + ephemeral: interaction.guild !== null, + embeds: [ + new MessageEmbed() + .setColor('#dc2828') + .setTitle(l("Calm down!", "Calmez-vous !", interaction.user.id, interaction.guild ? interaction.guild.id : 0)) + .setDescription(l("Please wait " + Math.ceil((30000 - (new Date() - cooldowns[interaction.user.id]))/1000) + " seconds before you can use this again.", "Patientez encore " + Math.ceil((30000 - (new Date() - cooldowns[interaction.user.id]))/1000) + " secondes avant de pouvoir réutiliser ça.", interaction.user.id, interaction.guild ? interaction.guild.id : 0)) + ] + }); + } else { + try { + let select = keys[1]; + let episode = keys[2].toLowerCase(); + + let sdata = series.filter(i => i.command.toLowerCase() === select)[0]; + await episodeReply(interaction, select, episode, sdata, true); + await interaction.reply({ + ephemeral: interaction.guild !== null, + embeds: [ + new MessageEmbed() + .setColor('#28dc46') + .setTitle(l("Show to everypony", "Afficher à tous", interaction.user.id, interaction.guild ? interaction.guild.id : 0)) + .setDescription(l("This episode has been sent publicly to this channel.", "Cet épisode a été envoyé publiquement dans ce salon.", interaction.user.id, interaction.guild ? interaction.guild.id : 0)) + ] + }); + cooldowns[interaction.user.id] = new Date(); + } catch (e) { + console.error(e); + await interaction.reply({ + ephemeral: interaction.guild !== null, + embeds: [ + new MessageEmbed() + .setColor('#dc2828') + .setTitle(l("Show to everypony", "Afficher à tous", interaction.user.id, interaction.guild ? interaction.guild.id : 0)) + .setDescription(l("We are unable to send a message to this channel, make sure the bot have sufficient permissions.", "Nous ne parvenons pas à envoyer un message dans ce salon, assurez-vous que le robot dispose de sufficient de permissions.", interaction.user.id, interaction.guild ? interaction.guild.id : 0)) + ] + }); + } + } + } else if (keys[0] === "pony.display") { let row; if (statsPonies[keys[1]] !== undefined) { statsPonies[keys[1]]++; @@ -123,7 +167,7 @@ module.exports = async (interaction) => { new MessageEmbed() .setColor('#dc2828') .setTitle(l("Show to everypony", "Afficher à tous", interaction.user.id, interaction.guild ? interaction.guild.id : 0)) - .setDescription(l("We are unable to send a message to this channel, make sure the bot have sufficient permissions.", "Nous ne parvenons pas à envoyer un message dans ce salon, assurez-vous que le robot dispose de sufficient de permissions.", interaction.user.id, interaction.guild ? interaction.guild.id : 0)) + .setDescription(l("We are unable to send a message to this channel, make sure the bot have sufficient permissions.", "Nous ne parvenons pas à envoyer un message dans ce salon, assurez-vous que le robot dispose de suffisamment de permissions.", interaction.user.id, interaction.guild ? interaction.guild.id : 0)) ] }); } diff --git a/handler/command.js b/handler/command.js index c1957ae..9733e6c 100644 --- a/handler/command.js +++ b/handler/command.js @@ -16,6 +16,7 @@ const commandInfo = require('../commands/info'); const commandEval = require('../commands/eval'); const commandConfig = require('../commands/config'); const commandPony = require('../commands/pony'); +const commandEpisode = require('../commands/episode'); const commandHelp = require('../commands/help'); module.exports = async (interaction) => { @@ -85,4 +86,8 @@ module.exports = async (interaction) => { if (interaction.commandName === 'pony') { await commandPony(interaction); } + + if (interaction.commandName === 'episode') { + await commandEpisode(interaction); + } } \ No newline at end of file diff --git a/handler/menu.js b/handler/menu.js index 1102b9e..f4cc3bc 100644 --- a/handler/menu.js +++ b/handler/menu.js @@ -12,7 +12,7 @@ module.exports = async (interaction) => { fs.writeFile("./stats/menu.json", JSON.stringify(statsMenu), () => {}); if (keys[0].startsWith("report.issue.")) { - item = keys[0].substr(13) + item = keys[0].substring(13) tid = "./reports/Telemetry-Pony-" + (new Date().toISOString().replace(/[^a-zA-Z0-9]/gm, "-")) + ".txt"; fs.writeFileSync(tid, "-------------------------\nPonyfind Telemetry Report\n-------------------------\n\nReport Type:\n Pony Issue Report\n\n-------------------------\n\nReporter:\n " + interaction.user.tag + " (" + interaction.user.id + ")\n\nServer:\n " + (interaction.guild ? interaction.guild.name : "[Direct Messages]") + " (" + (interaction.guild ? interaction.guild.id : 0) + ")\n\nChannel:\n " + (interaction.channel ? interaction.channel.name : "[Direct Messages]") + " (" + (interaction.channel ? interaction.channel.id : 0) + ")\n\nItem:\n " + keys[1] + "\n\nReport Type:\n " + item) await interaction.reply({ diff --git a/modules/episodepublic.js b/modules/episodepublic.js new file mode 100644 index 0000000..73318f5 --- /dev/null +++ b/modules/episodepublic.js @@ -0,0 +1,16 @@ +const { MessageButton } = require("discord.js"); + +module.exports = (interaction, series, episode) => { + if (typeof cooldowns[interaction.user.id] !== "undefined" && new Date() - cooldowns[interaction.user.id] < 30000) { + return new MessageButton() + .setCustomId("episode.public|" + series + "|" + episode) + .setLabel(l("Wait " + Math.ceil((30000 - (new Date() - cooldowns[interaction.user.id]))/1000) + " seconds", "Patientez " + Math.ceil((30000 - (new Date() - cooldowns[interaction.user.id]))/1000) + " secondes", interaction.user.id, interaction.guild ? interaction.guild.id : 0)) + .setStyle("PRIMARY") + .setDisabled(true); + } else { + return new MessageButton() + .setCustomId("episode.public|" + series + "|" + episode) + .setLabel(l("Show to everypony", "Afficher à tous", interaction.user.id, interaction.guild ? interaction.guild.id : 0)) + .setStyle("PRIMARY") + } +} \ No newline at end of file diff --git a/modules/episodereply.js b/modules/episodereply.js new file mode 100644 index 0000000..79a05c1 --- /dev/null +++ b/modules/episodereply.js @@ -0,0 +1,75 @@ +const { MessageActionRow, MessageButton, MessageEmbed } = require("discord.js"); +const getEpisodePublic = require("./episodepublic"); +const fs = require("fs"); + +module.exports = async (interaction, series, episode, sdata, isPublic) => { + let sid = episode.split("-")[0]; + let spt = sid.substring(1); + let eid = episode.split("-")[1]; + let edata = sdata.seasons.filter(i => i.id === sid)[0].episodes.filter(i => i.local - 1 + 1 === eid - 1 + 1)[0]; + + let fields = [ + { name: l("Airing Date", "Date de diffusion", interaction.user.id, interaction.guild ? interaction.guild.id : 0), value: edata.date_pre, inline: true }, + { name: l("Written by", "Écrit par", interaction.user.id, interaction.guild ? interaction.guild.id : 0), value: edata.writer, inline: true }, + { name: l("Overall no.", "N° global", interaction.user.id, interaction.guild ? interaction.guild.id : 0), value: edata.global + "/" + sdata.seasons.map(i => { return i.count }).reduce((a, b) => { return a + b; }), inline: true }, + { name: l("Season no.", "N° dans la saison", interaction.user.id, interaction.guild ? interaction.guild.id : 0), value: edata.local + "/" + sdata.seasons.filter(i => i.id === sid)[0].count, inline: true }, + ] + + if (edata.characters.length > 0) { + fields.push({ name: l("Characters", "Personnages", interaction.user.id, interaction.guild ? interaction.guild.id : 0), value: edata.characters.join("\n"), inline: true }); + } + + let row; + if (interaction.guild && !isPublic) { + row = new MessageActionRow() + .addComponents( + getEpisodePublic(interaction, series, episode), + new MessageButton() + .setLabel(l("Read More", "Lire plus", interaction.user.id, interaction.guild ? interaction.guild.id : 0)) + .setStyle("LINK") + .setURL("https://mlp.fandom.com/wiki/" + encodeURI(edata.name)), + new MessageButton() + .setCustomId("result.report|" + edata.name) + .setLabel(l("Report an issue", "Signaler un problème", interaction.user.id, interaction.guild ? interaction.guild.id : 0)) + .setStyle("DANGER") + ) + } else { + row = new MessageActionRow() + .addComponents( + new MessageButton() + .setLabel(l("Read More", "Lire plus", interaction.user.id, interaction.guild ? interaction.guild.id : 0)) + .setStyle("LINK") + .setURL("https://mlp.fandom.com/wiki/" + encodeURI(edata.name)), + new MessageButton() + .setCustomId("result.report|" + edata.name) + .setLabel(l("Report an issue", "Signaler un problème", interaction.user.id, interaction.guild ? interaction.guild.id : 0)) + .setStyle("DANGER") + ) + } + + let reply = { + embeds: [ + new MessageEmbed() + .setColor("DEFAULT") + .setAuthor({ name: l("Season", "Saison", interaction.user.id, interaction.guild ? interaction.guild.id : 0) + " " + spt + " " + l("Episode", "Épisode", interaction.user.id, interaction.guild ? interaction.guild.id : 0) + " " + eid }) + .setTitle(edata.name) + .setDescription(edata.plot) + .setImage(edata.cover) + .addFields(fields) + .setFooter({ text: l("Content provided without warranty, use at your own risk.", "Contenu fourni sans aucune garantie, utilisez à vos risques et périls", interaction.user.id, interaction.guild ? interaction.guild.id : 0) }) + ], + components: [ + row + ] + }; + + if (!isPublic) { + reply.ephemeral = interaction.guild !== null; + } + + if (isPublic) { + await interaction.channel.send(reply); + } else { + await interaction.reply(reply); + } +} \ No newline at end of file diff --git a/modules/findepisode.js b/modules/findepisode.js new file mode 100644 index 0000000..f37ed4f --- /dev/null +++ b/modules/findepisode.js @@ -0,0 +1,52 @@ +const Fuse = require('fuse.js'); + +module.exports = (series, query) => { + let episodes = []; + + for (let season of series.seasons) { + for (let episode of season.episodes) { + episodes.push({ + season: season.id, + episode: episode.local, + global: episode.global, + name: episode.name, + date: episode.date, + year: new Date(episode.date).getUTCFullYear(), + plot: episode.plot, + cast: episode.characters + }) + } + } + + const fuse = new Fuse(episodes, { + keys: [ 'season', 'episode', 'global', 'name', 'year', 'plot', 'cast' ] + }) + + let results = []; + for (let item of fuse.search(query)) { + results.push({ + name: item.item.season.toUpperCase() + "E" + item.item.episode + " - " + item.item.name + " (" + item.item.year + ")", + value: item.item.season + "-" + item.item.episode + }); + } + + let first = [ + null, + null, + null, + null, + null, + null, + null + ] + + findex = 0; + for (let result of results) { + if (!first.includes(result) && findex < 7) { + first[findex] = result + findex++ + } + } + + return first; +} \ No newline at end of file diff --git a/modules/registers.js b/modules/registers.js index 7bf2b68..59ed699 100644 --- a/modules/registers.js +++ b/modules/registers.js @@ -1,4 +1,22 @@ -const { SlashCommandBuilder } = require('@discordjs/builders'); +const { SlashCommandBuilder, SlashCommandSubcommandGroupBuilder, SlashCommandSubcommandBuilder } = require('@discordjs/builders'); +const fs = require('fs'); + +let episodesCmd = new SlashCommandBuilder() + .setName("episode") + .setDescription("Gets info about a series' episode") + +for (let series of JSON.parse(fs.readFileSync("./data/series.json").toString())) { + let command = new SlashCommandSubcommandBuilder() + .setName(series.command.toLowerCase()) + .setDescription(series.title + " (" + series.date + ")") + .addStringOption(s => + s.setName("episode") + .setDescription("The episode to search for") + .setAutocomplete(true) + ) + + episodesCmd.addSubcommand(command) +} module.exports = [ new SlashCommandBuilder() @@ -18,16 +36,7 @@ module.exports = [ new SlashCommandBuilder() .setName('random') .setDescription("Picks a random pony"), - // new SlashCommandBuilder() - // .setName('lang') - // .setDescription("Changes the bot's language") - // .addStringOption(option => - // option.setName('locale') - // .setDescription('The selected language') - // .setRequired(true) - // .addChoice('Français', 'fr') - // .addChoice('English', 'en') - // ), + episodesCmd, new SlashCommandBuilder() .setName('config') .setDescription("Configure all aspects of the bot") @@ -65,16 +74,5 @@ module.exports = [ .addChoice('English', 'en') .addChoice('User prefered language (default)', 'off') ) - ), - // new SlashCommandBuilder() - // .setName('serverlang') - // .setDescription("Changes the bot's language on this server") - // .addStringOption(option => - // option.setName('locale') - // .setDescription('The selected language') - // .setRequired(true) - // .addChoice('Français', 'fr') - // .addChoice('English', 'en') - // .addChoice('User prefered language (default)', 'off') - // ) + ) ] \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 51f1dc6..0947bef 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,5 +1,5 @@ { - "name": "PonyDB", + "name": "Ponyfind", "lockfileVersion": 2, "requires": true, "packages": { diff --git a/update/dict.js b/update/dict.js deleted file mode 100644 index 88ec568..0000000 --- a/update/dict.js +++ /dev/null @@ -1,41 +0,0 @@ -const fs = require('fs'); - -console.log("Optimizing search engine..."); - -let search = { - entries: null, - associations: [] -} - -global.knownAssociations = []; - -for (let page of JSON.parse(fs.readFileSync("./data/pages.json").toString())) { - if (!knownAssociations.includes(page.name.toLowerCase().replace(/[^a-z]/gm, " ").replace(/\s\s+/g, " ").trim())) { - knownAssociations.push(page.name.toLowerCase().replace(/[^a-z]/gm, " ").replace(/\s\s+/g, " ").trim()); - search.associations.push({ - title: page.name.toLowerCase().replace(/[^a-z]/gm, " ").replace(/\s\s+/g, " ").trim(), - endpoint: page.name - }); - } - - if (typeof JSON.parse(fs.readFileSync("./data/data.json").toString())[page.name] !== "undefined") { - for (let nick of JSON.parse(fs.readFileSync("./data/data.json").toString())[page.name].names) { - if (!knownAssociations.includes(nick.toLowerCase().replace(/[^a-z]/gm, " ").replace(/\s\s+/g, " ").trim())) { - knownAssociations.push(nick.toLowerCase().replace(/[^a-z]/gm, " ").replace(/\s\s+/g, " ").trim()); - search.associations.push({ - title: nick.toLowerCase().replace(/[^a-z]/gm, " ").replace(/\s\s+/g, " ").trim(), - endpoint: page.name - }); - } - } - } -} - -search.entries = []; -for (let association of search.associations) { - search.entries.push(association.title); -} -search.entries = [...new Set(search.entries)]; -fs.writeFileSync("./data/search.json", JSON.stringify(search, null, 4)); - -console.log(JSON.parse(fs.readFileSync("./data/pages.json").toString()).length + " known characters"); \ No newline at end of file diff --git a/update/episodes/index.js b/update/episodes/index.js new file mode 100644 index 0000000..f071707 --- /dev/null +++ b/update/episodes/index.js @@ -0,0 +1,141 @@ +const config = require("./series.json"); +const axios = require('axios'); +const fs = require('fs'); +const WikiTextParser = require('parse-wikitext'); +const parser = new WikiTextParser("mlp.fandom.com"); + +(async () => { + let all = []; + + for (let series of config) { + let data = { + command: series.cmd, + title: series.title, + date: series.date, + link: series.page, + description: { + series: null, + plot: null, + }, + seasons: [] + }; + console.log(series.title + " (" + data.date + ")"); + + ddata = (await axios.get("https://mlp.fandom.com/api.php?format=json&action=query&prop=extracts&exlimit=max&explaintext&exintro&titles=" + encodeURI(series.page) + "&redirects=")).data; + data.description.series = ddata.query.pages[Object.keys(ddata.query.pages)[0]].extract.split("\n")[0]; + data.description.plot = ddata.query.pages[Object.keys(ddata.query.pages)[0]].extract.split("\n")[1]; + + for (let s of series.seasons) { + console.log(" Season " + s.id); + + let season = { + name: "Season " + s.id, + id: "s" + s.id, + date: null, + count: null, + episodes: [] + } + + let sdata = (await axios.get("https://mlp.fandom.com/api.php?action=query&prop=revisions&titles=" + s.table.page + "&rvslots=*&rvprop=content&formatversion=2&format=json")).data; + let sections = parser.pageToSectionObject(sdata.query.pages[0].revisions[0].slots.main.content); + let episodes = sections.Episodes[s.table.section].content + .filter(i => i.startsWith("|")) + .join("||") + .replace("|}", "") + .split("|-").map(i => { + return i.split("||").filter(i => i.trim() !== ""); + }) + .filter(i => i.length > 1) + .map(i => { + let index = 0; + return i.map(i => { + return i.replace(/^(\| |\|)(.*)/gm, "$2") + .replace(/^style="(.*)"( |)\|( |)(.*)/gm, "$4") + .replace(/\[\[((.*)\|(.*)|(.*))\]\]/gm, "$3$4"); + }).map(i => { + if (index === 0) { + index++; + return i.replace(/(.*) \((.*)\)/gm, "$1|$2"); + } else { + index++; + return i; + } + }) + }); + + index = 1; + for (let e of episodes) { + process.stdout.write(" S" + s.id + "E" + index); + + let episode = { + local: null, + global: null, + date: null, + name: null, + writer: null, + cover: null, + characters: [], + plot: null + } + + episode.local = e[s.table.fields.number].replace(/(.*)\|(.*)/gm, "$1").replace(/''(.*)''/gm, "$1").replace(/{{(.*)\|(.*)}}/gm, "$2").replace(/(.*)\|(.*)/gm, "$2"); + episode.global = e[s.table.fields.number].replace(/(.*)\|(.*)/gm, "$2"); + episode.date_pre = e[s.table.fields.date].replace(/{{(.*)\|(.*)\|(.*)\|link=(no|yes)}}/gm, "$3").replace(/{{(.*)\|(.*)}}/gm, "$2").replace(/(.*)\|(.*)/gm, "$2"); + try { + episode.date = new Date(e[s.table.fields.date].replace(/{{(.*)\|(.*)\|(.*)\|link=(no|yes)}}/gm, "$3").replace(/{{(.*)\|(.*)}}/gm, "$2").replace(/(.*)\|(.*)/gm, "$2")).toISOString(); + } catch (e) { + episode.date = null; + } + episode.name = e[s.table.fields.title]; + episode.writer = e[s.table.fields.writer].replace(/{{(.*)\|(.*)\|(.*)\|link=(no|yes)}}/gm, "$3").replace(/(]]|\[\[)/gm, "").replace(/(.*)\|(.*)/gm, "$2"); + + process.stdout.write(": " + episode.name + "\n"); + + if (episode.local.length !== 2 || episode.global.length > 3 || episode.global.length < 2 || episode.date_pre.length !== 10) { + console.log("FAILURE: " + JSON.stringify(episode)); + } + + let edata = (await axios.get("https://mlp.fandom.com/api.php?action=query&prop=revisions&titles=" + encodeURI(episode.name).replaceAll("?", "%3F").replaceAll("&", "%26") + "&rvslots=*&rvprop=content&formatversion=2&format=json")).data; + let eext = (await axios.get("https://mlp.fandom.com/api.php?format=json&action=query&prop=extracts&exlimit=max&explaintext&exintro&titles=" + encodeURI(episode.name).replaceAll("?", "%3F").replaceAll("&", "%26") + "&redirects=")).data; + + episode.plot = eext.query.pages[Object.keys(eext.query.pages)[0]].extract.replace(/(.*)\n(.*)/, "$2").replace(/(.*)\n\n(.*)/gm, "$2").replace(/([.?!])\s*(?=[A-Z])/g, "$1|").split("|").join(" "); + + let infobox = null; + try { + sections = parser.pageToSectionObject(edata.query.pages[0].revisions[0].slots.main.content); + box = parser.parseInfoBox(sections["content"]); + if (box.template === "Infobox episode") { + infobox = parser.parseInfoBox(sections["content"]).values; + } + } catch (e) { + console.log(edata); + throw e; + } + + if (infobox !== null) { + try { + if (typeof infobox.image !== "undefined") episode.cover = (await axios.head("https://mlp.fandom.com/Special:FilePath/" + encodeURI(infobox.image.replace(//g, "").trim()).replaceAll("?", "%3F").replaceAll("&", "%26"))).request.res.responseUrl; + } catch (e) { + try { + if (typeof infobox.image !== "undefined") episode.cover = e.request.res.responseUrl; + } catch (e2) { + console.error(e2); + throw e; + } + } + if (typeof infobox.featured !== "undefined") episode.characters = infobox.featured.replace(/\[\[(([a-zA-Z0-9 .\-# _\\\/]*)\|([a-zA-Z0-9 .\-_\\\/]*)|([a-zA-Z0-9 .\-#_\\\/]*))\]\]/gm, "$3$4").replace(/(<(\/| |)(\/| |)br(\/| |)(\/| |)>)/gm, "|||").split("|||") + } + + season.episodes.push(episode); + index++; + } + + season.count = season.episodes.length; + season.date = season.episodes[0].date; + data.seasons.push(season); + } + all.push(data); + } + + fs.writeFileSync("./data/series.json", JSON.stringify(all, null, 4)); +})(); \ No newline at end of file diff --git a/update/episodes/series.json b/update/episodes/series.json new file mode 100644 index 0000000..0935658 --- /dev/null +++ b/update/episodes/series.json @@ -0,0 +1,127 @@ +[ + { + "title": "Friendship is Magic", + "cmd": "FIM", + "page": "My Little Pony Friendship is Magic", + "date": "2010-2019", + "seasons": [ + { + "id": 1, + "table": { + "page": "Friendship is Magic animated media", + "section": "Season one", + "fields": { + "number": 0, + "title": 1, + "writer": 2, + "date": 3 + } + } + }, + { + "id": 2, + "table": { + "page": "Friendship is Magic animated media", + "section": "Season two", + "fields": { + "number": 0, + "title": 1, + "writer": 2, + "date": 3 + } + } + }, + { + "id": 3, + "table": { + "page": "Friendship is Magic animated media", + "section": "Season three", + "fields": { + "number": 0, + "title": 1, + "writer": 2, + "date": 3 + } + } + }, + { + "id": 4, + "table": { + "page": "Friendship is Magic animated media", + "section": "Season four", + "fields": { + "number": 0, + "title": 1, + "writer": 2, + "date": 3 + } + } + }, + { + "id": 5, + "table": { + "page": "Friendship is Magic animated media", + "section": "Season five", + "fields": { + "number": 0, + "title": 1, + "writer": 2, + "date": 3 + } + } + }, + { + "id": 6, + "table": { + "page": "Friendship is Magic animated media", + "section": "Season six", + "fields": { + "number": 0, + "title": 1, + "writer": 2, + "date": 3 + } + } + }, + { + "id": 7, + "table": { + "page": "Friendship is Magic animated media", + "section": "Season seven", + "fields": { + "number": 0, + "title": 1, + "writer": 2, + "date": 3 + } + } + }, + { + "id": 8, + "table": { + "page": "Friendship is Magic animated media", + "section": "Season eight", + "fields": { + "number": 0, + "title": 1, + "writer": 2, + "date": 3 + } + } + }, + { + "id": 9, + "table": { + "page": "Friendship is Magic animated media", + "section": "Season nine", + "fields": { + "number": 0, + "title": 1, + "writer": 2, + "date": 3 + } + } + } + ] + } +] \ No newline at end of file diff --git a/update/index.js b/update/index.js deleted file mode 100644 index 53acf34..0000000 --- a/update/index.js +++ /dev/null @@ -1,5 +0,0 @@ -require('child_process').spawnSync("node", ["update/listgen.js"], { cwd: __dirname + "/..", stdio: "inherit" }) -require('child_process').spawnSync("node", ["update/pages.js"], { cwd: __dirname + "/..", stdio: "inherit" }) -require('child_process').spawnSync("node", ["update/infobox.js"], { cwd: __dirname + "/..", stdio: "inherit" }) -require('child_process').spawnSync("node", ["update/parse.js"], { cwd: __dirname + "/..", stdio: "inherit" }) -require('child_process').spawnSync("node", ["update/dict.js"], { cwd: __dirname + "/..", stdio: "inherit" }) \ No newline at end of file diff --git a/update/infobox.js b/update/infobox.js deleted file mode 100644 index f53be0e..0000000 --- a/update/infobox.js +++ /dev/null @@ -1,71 +0,0 @@ -const fs = require('fs'); -const axios = require("axios"); -const WikiTextParser = require('parse-wikitext'); -const parser = new WikiTextParser("mlp.fandom.com"); - -console.log("Gathering infobox for each page..."); - -(async () => { - let infoboxes = {}; - for (let page of JSON.parse(fs.readFileSync("./data/pages.json").toString())) { - console.log("Gathering infobox for '" + page.name + "'..."); - try { - let data = (await axios.get("https://mlp.fandom.com/api.php?action=query&prop=revisions&titles=" + page.name + "&rvslots=*&rvprop=content&formatversion=2&format=json")).data; - let mwextracts = (await axios.get("https://mlp.fandom.com/api.php?format=json&action=query&prop=extracts&exlimit=max&explaintext&exintro&titles=" + encodeURI(page.name) + "&redirects=")).data; - let mwtext = (await axios.get("https://mlp.fandom.com/api.php?format=json&action=query&prop=extracts&exlimit=max&explaintext&titles=" + encodeURI(page.name) + "&redirects=")).data; - let extracts = ""; - try { - sentences = mwextracts.query.pages[Object.keys(mwextracts.query.pages)[0]].extract.trim().replace(/(.*)\n(.*)/, "$2").replace(/(.*)\n\n(.*)/gm, "$2").replace(/([.?!])\s*(?=[A-Z])/g, "$1|").split("|"); - extracts = sentences[0]; - if (extracts.length < 150 && sentences.length > 1) { - extracts = sentences[0] + " " + sentences[1]; - if (extracts.length < 150 && sentences.length > 2) { - extracts = sentences[0] + " " + sentences[1] + " " + sentences[2]; - } - } - } catch (e) { - extracts = ""; - } - let extracts_fr = extracts; - if (fs.existsSync("./modules/translate.php")) { - try { - extracts_fr = require('child_process').spawnSync("php", [ "translate.php", extracts ], { cwd: "./modules" }).stdout.toString() - } catch (e) { - extracts_fr = extracts; - } - } else { - extracts_fr = extracts; - } - if (data.query.pages.length > 0) { - console.log("Results found, adding name to database") - sections = parser.pageToSectionObject(data.query.pages[0].revisions[0].slots.main.content); - box = parser.parseInfoBox(sections["content"]); - if (box.template === "Infobox character") { - infoboxes[page.name] = parser.parseInfoBox(sections["content"]).values; - infoboxes[page.name]["_extract"] = extracts; - infoboxes[page.name]["_extract_fr"] = extracts_fr; - } - } else { - console.log("No results found, ignoring name"); - } - try { - if (mwtext.query.pages[Object.keys(mwextracts.query.pages)[0]].extract.toLowerCase().includes("friendship is magic") - || mwtext.query.pages[Object.keys(mwextracts.query.pages)[0]].extract.toLowerCase().includes("fim") - ) { - infoboxes[page.name]["_gen"] = 4; - } else if (mwtext.query.pages[Object.keys(mwextracts.query.pages)[0]].extract.toLowerCase().includes("a new generation") - || mwtext.query.pages[Object.keys(mwextracts.query.pages)[0]].extract.replace(/[.,?!;()"'-]/g, " ").replace(/\s+/g, " ").toLowerCase().split(" ").includes("ang") - || mwtext.query.pages[Object.keys(mwextracts.query.pages)[0]].extract.replace(/[.,?!;()"'-]/g, " ").replace(/\s+/g, " ").toLowerCase().split(" ").includes("ang") - ) { - infoboxes[page.name]["_gen"] = 5; - } else { - infoboxes[page.name]["_gen"] = -1; - } - } catch(e) {} - } catch (e) { - console.error(e); - } - } - - fs.writeFileSync("./data/boxes.json", JSON.stringify(infoboxes, null, 4)) -})() \ No newline at end of file diff --git a/update/listgen.js b/update/listgen.js deleted file mode 100644 index d9401f8..0000000 --- a/update/listgen.js +++ /dev/null @@ -1,24 +0,0 @@ -const axios = require("axios"); -const fs = require("fs"); - -if (fs.existsSync("./data")) fs.rmSync("./data", { recursive: true }); -fs.mkdirSync("./data"); - -(async () => { - async function getCategory(category) { - console.log("Category:" + category); - let cat = (await axios.get("https://mlp.fandom.com/api.php?action=query&generator=categorymembers&gcmtitle=Category:" + encodeURI(category) + "&prop=categories&cllimit=max&gcmlimit=max&format=json")).data; - - return Object.keys(cat.query.pages).map(k => cat.query.pages[k].title).filter(k => !k.startsWith("List") && !k.includes("EG") && !k.toLowerCase().includes("ponies") && !k.includes(" and ") && !k.includes("(") && !k.includes("family")); - } - - let list = [...new Set([ - ...(await getCategory("Pegasus ponies")), - ...(await getCategory("Alicorn ponies")), - ...(await getCategory("Earth ponies")), - ...(await getCategory("Unicorn ponies")), - ...(await getCategory("Main characters")), - ...(await getCategory("Dragons")), - ])]; - fs.writeFileSync("./data/list.json", JSON.stringify(list, null, 4)) -})() \ No newline at end of file diff --git a/update/pages.js b/update/pages.js deleted file mode 100644 index 7f3a1c8..0000000 --- a/update/pages.js +++ /dev/null @@ -1,28 +0,0 @@ -const fs = require('fs'); -const axios = require('axios'); - -(async () => { - console.log("Gathering pages list..."); - let pages = []; - for (let page of JSON.parse(fs.readFileSync("./data/list.json").toString())) { - console.log("Searching for '" + page + "'..."); - try { - let data = (await axios.get("https://mlp.fandom.com/api.php?action=query&list=search&srsearch=" + encodeURI(page) + "&srlimit=1&srenablerewrites=true&format=json")).data; - if (data.query.search.length > 0) { - console.log("Results found, adding name to database") - pages.push({ - query: page, - name: data.query.search[0].title, - mwid: data.query.search[0].pageid, - words: data.query.search[0].wordcount, - }) - } else { - console.log("No results found, ignoring name"); - } - } catch (e) { - console.error(e); - } - } - - fs.writeFileSync("./data/pages.json", JSON.stringify(pages, null, 4)) -})() \ No newline at end of file diff --git a/update/parse.js b/update/parse.js deleted file mode 100644 index 0b2baa9..0000000 --- a/update/parse.js +++ /dev/null @@ -1,84 +0,0 @@ -const fs = require('fs'); -const axios = require('axios'); - -console.log("Parsing infobox data..."); - -let ponies = {}; - -(async () => { - for (let title in JSON.parse(fs.readFileSync("./data/boxes.json").toString())) { - console.log("Parsing " + title + "..."); - let box = JSON.parse(fs.readFileSync("./data/boxes.json").toString())[title]; - let data = { - names: [title], - extract: "", - extract_fr: "", - generation: -1, - color: "000000", - image: "https://example.com", - kind: "Pony", - sex: "Unknown", - occupation: ["Unknown"], - residence: ["Unknown"], - mark: "https://example.com" - } - - if (typeof box._gen !== "undefined") data.generation = box._gen; - - if (typeof box.name2 !== "undefined") data.names.push(box.name2.replace(//g, "")); - if (typeof box.name3 !== "undefined") data.names.push(box.name3.replace(//g, "")); - if (typeof box.name4 !== "undefined") data.names.push(box.name4.replace(//g, "")); - if (typeof box.name5 !== "undefined") data.names.push(box.name5.replace(//g, "")); - - if (typeof box.nicknames !== "undefined") { - box.nicknames.replace(//g, "").split(",").filter(e => !e.match(/[^a-zA-Z0-9-_ ]/gm)).forEach((e, i) => { - data.names.push(e.trim()); - }); - } - - if (typeof box.kind !== "undefined") { - kp = box.kind.replace(//g, "").replace(/[^a-zA-Z0-9-_ ]/gm, "").split(" ")[0]; - data.kind = kp.substr(kp.replace(/([A-Z])([a-z0-9]*)$/g, "").length); - } - if (typeof box.sex !== "undefined") data.sex = box.sex.replace(//g, "").trim().startsWith("F") ? "F" : "M"; - if (typeof box._extract !== "undefined") { - data.extract = box._extract; - } - if (typeof box._extract_fr !== "undefined") { - data.extract_fr = box._extract_fr.replace(/ma petite amitié de poney est la magie/gmi, "My Little Pony Friendship is Magic").replace(/Mon amitié avec mon petit poney est magique/gmi, "My Little Pony Friendship is Magic").replace(/Mon petit Poney/gmi, "My Little Pony").replace(/Mon Petit Poney Une Nouvelle Génération/gmi, "My Little Pony A New Generation").replace(/Mon petit poney : une nouvelle génération/gmi, "My Little Pony: A New Generation").replace(/pegasus/gmi, "pégase").replace(/alicorn[^e]/gmi, "alicorne").replace(/une poney/gmi, "une ponette").replace(/petite génération de poney 5/gmi, "Génération 5"); - } else { - data.extract_fr = data.extract; - } - if (typeof box.coat !== "undefined") data.color = box.coat.replace(//g, "").trim().replace(/\[([a-z.\/ \nA-Z0-9:]*)\/(.{6})\/ (.*)\]/gm, "$2").replace(/{{perbang\|([0-9A-Fa-f].{5})(.*)/g, "$1"); - if (typeof box.main !== "undefined") data.image = "https://mlp.fandom.com/Special:FilePath/" + encodeURI(box.main.replace(//g, "").trim()); - if (typeof box.main1 !== "undefined") data.image = "https://mlp.fandom.com/Special:FilePath/" + encodeURI(box.main1.replace(//g, "").trim()); - if (typeof box.image !== "undefined") data.image = "https://mlp.fandom.com/Special:FilePath/" + encodeURI(box.image.replace(//g, "").trim()); - if (typeof box["cutie mark"] !== "undefined") { - try { - data.markimg = box["cutie mark"].replace(//g, "").trim().split("[[File:")[1].split("|")[0]; - } catch (e) { - data.markimg = box["cutie mark"].replace(//g, "").trim().split("[[File:")[0].split("|")[0]; - } - data.mark = "https://mlp.fandom.com/Special:Redirect/file/" + encodeURI(data.markimg) + "?width=128"; - } - - if (typeof box.occupation !== "undefined") { - occupations = []; - box.occupation/*.replace(/\[\[(.*)_(.*)\]\]|\[\[(.*)\|(.*)\]\]|\[\[(.*)\]\]/gm, "$2$4$5")*/.replace(//g, "").trim().replace(/\|/gm, "_").replace(/<( ||(|| )\/)( ||(|| )\/)(b|B)(r|R)( ||(|| )\/)( ||(|| )\/)>/gm, "|").replace(/( \|| \| | \| )/gm, "|").split("|").forEach((e) => { - occupations.push(e.trim().replace(/[\[\]]/gm, "").replace(/<(.*)>/gm, "").replace(/([a-zA-Z0-9 \-_,.'"]*)_([a-zA-Z0-9 \-_,.'"]*)/gm, "$2")); - }) - data.occupation = occupations; - } - if (typeof box.residence !== "undefined") { - residences = []; - box.residence/*.replace(/\[\[(.*)_(.*)\]\]|\[\[(.*)\|(.*)\]\]|\[\[(.*)\]\]/gm, "$2$4$5")*/.replace(//g, "").trim().replace(/\|/gm, "_").replace(/<( ||(|| )\/)( ||(|| )\/)(b|B)(r|R)( ||(|| )\/)( ||(|| )\/)>/gm, "|").replace(/( \|| \| | \| )/gm, "|").split("|").forEach((e) => { - residences.push(e.trim().replace(/[\[\]]/gm, "").replace(/<(.*)>/gm, "").replace(/([a-zA-Z0-9 \-_,.'"]*)_([a-zA-Z0-9 \-_,.'"]*)/gm, "$2")); - }) - data.residence = residences.replaceAll("Locations#", ""); - } - - if ((typeof box.name2 !== "undefined" || typeof box.name3 !== "undefined" || typeof box.name4 !== "undefined" || typeof box.name5 !== "undefined" || typeof box.coat !== "undefined" || typeof box.occupation !== "undefined" || typeof box.residence !== "undefined") && typeof box.sex !== "undefined") ponies[title] = data; - } -})() - -fs.writeFileSync("./data/data.json", JSON.stringify(ponies, null, 4)); \ No newline at end of file diff --git a/update/ponies/dict.js b/update/ponies/dict.js new file mode 100644 index 0000000..88ec568 --- /dev/null +++ b/update/ponies/dict.js @@ -0,0 +1,41 @@ +const fs = require('fs'); + +console.log("Optimizing search engine..."); + +let search = { + entries: null, + associations: [] +} + +global.knownAssociations = []; + +for (let page of JSON.parse(fs.readFileSync("./data/pages.json").toString())) { + if (!knownAssociations.includes(page.name.toLowerCase().replace(/[^a-z]/gm, " ").replace(/\s\s+/g, " ").trim())) { + knownAssociations.push(page.name.toLowerCase().replace(/[^a-z]/gm, " ").replace(/\s\s+/g, " ").trim()); + search.associations.push({ + title: page.name.toLowerCase().replace(/[^a-z]/gm, " ").replace(/\s\s+/g, " ").trim(), + endpoint: page.name + }); + } + + if (typeof JSON.parse(fs.readFileSync("./data/data.json").toString())[page.name] !== "undefined") { + for (let nick of JSON.parse(fs.readFileSync("./data/data.json").toString())[page.name].names) { + if (!knownAssociations.includes(nick.toLowerCase().replace(/[^a-z]/gm, " ").replace(/\s\s+/g, " ").trim())) { + knownAssociations.push(nick.toLowerCase().replace(/[^a-z]/gm, " ").replace(/\s\s+/g, " ").trim()); + search.associations.push({ + title: nick.toLowerCase().replace(/[^a-z]/gm, " ").replace(/\s\s+/g, " ").trim(), + endpoint: page.name + }); + } + } + } +} + +search.entries = []; +for (let association of search.associations) { + search.entries.push(association.title); +} +search.entries = [...new Set(search.entries)]; +fs.writeFileSync("./data/search.json", JSON.stringify(search, null, 4)); + +console.log(JSON.parse(fs.readFileSync("./data/pages.json").toString()).length + " known characters"); \ No newline at end of file diff --git a/update/ponies/index.js b/update/ponies/index.js new file mode 100644 index 0000000..53acf34 --- /dev/null +++ b/update/ponies/index.js @@ -0,0 +1,5 @@ +require('child_process').spawnSync("node", ["update/listgen.js"], { cwd: __dirname + "/..", stdio: "inherit" }) +require('child_process').spawnSync("node", ["update/pages.js"], { cwd: __dirname + "/..", stdio: "inherit" }) +require('child_process').spawnSync("node", ["update/infobox.js"], { cwd: __dirname + "/..", stdio: "inherit" }) +require('child_process').spawnSync("node", ["update/parse.js"], { cwd: __dirname + "/..", stdio: "inherit" }) +require('child_process').spawnSync("node", ["update/dict.js"], { cwd: __dirname + "/..", stdio: "inherit" }) \ No newline at end of file diff --git a/update/ponies/infobox.js b/update/ponies/infobox.js new file mode 100644 index 0000000..f53be0e --- /dev/null +++ b/update/ponies/infobox.js @@ -0,0 +1,71 @@ +const fs = require('fs'); +const axios = require("axios"); +const WikiTextParser = require('parse-wikitext'); +const parser = new WikiTextParser("mlp.fandom.com"); + +console.log("Gathering infobox for each page..."); + +(async () => { + let infoboxes = {}; + for (let page of JSON.parse(fs.readFileSync("./data/pages.json").toString())) { + console.log("Gathering infobox for '" + page.name + "'..."); + try { + let data = (await axios.get("https://mlp.fandom.com/api.php?action=query&prop=revisions&titles=" + page.name + "&rvslots=*&rvprop=content&formatversion=2&format=json")).data; + let mwextracts = (await axios.get("https://mlp.fandom.com/api.php?format=json&action=query&prop=extracts&exlimit=max&explaintext&exintro&titles=" + encodeURI(page.name) + "&redirects=")).data; + let mwtext = (await axios.get("https://mlp.fandom.com/api.php?format=json&action=query&prop=extracts&exlimit=max&explaintext&titles=" + encodeURI(page.name) + "&redirects=")).data; + let extracts = ""; + try { + sentences = mwextracts.query.pages[Object.keys(mwextracts.query.pages)[0]].extract.trim().replace(/(.*)\n(.*)/, "$2").replace(/(.*)\n\n(.*)/gm, "$2").replace(/([.?!])\s*(?=[A-Z])/g, "$1|").split("|"); + extracts = sentences[0]; + if (extracts.length < 150 && sentences.length > 1) { + extracts = sentences[0] + " " + sentences[1]; + if (extracts.length < 150 && sentences.length > 2) { + extracts = sentences[0] + " " + sentences[1] + " " + sentences[2]; + } + } + } catch (e) { + extracts = ""; + } + let extracts_fr = extracts; + if (fs.existsSync("./modules/translate.php")) { + try { + extracts_fr = require('child_process').spawnSync("php", [ "translate.php", extracts ], { cwd: "./modules" }).stdout.toString() + } catch (e) { + extracts_fr = extracts; + } + } else { + extracts_fr = extracts; + } + if (data.query.pages.length > 0) { + console.log("Results found, adding name to database") + sections = parser.pageToSectionObject(data.query.pages[0].revisions[0].slots.main.content); + box = parser.parseInfoBox(sections["content"]); + if (box.template === "Infobox character") { + infoboxes[page.name] = parser.parseInfoBox(sections["content"]).values; + infoboxes[page.name]["_extract"] = extracts; + infoboxes[page.name]["_extract_fr"] = extracts_fr; + } + } else { + console.log("No results found, ignoring name"); + } + try { + if (mwtext.query.pages[Object.keys(mwextracts.query.pages)[0]].extract.toLowerCase().includes("friendship is magic") + || mwtext.query.pages[Object.keys(mwextracts.query.pages)[0]].extract.toLowerCase().includes("fim") + ) { + infoboxes[page.name]["_gen"] = 4; + } else if (mwtext.query.pages[Object.keys(mwextracts.query.pages)[0]].extract.toLowerCase().includes("a new generation") + || mwtext.query.pages[Object.keys(mwextracts.query.pages)[0]].extract.replace(/[.,?!;()"'-]/g, " ").replace(/\s+/g, " ").toLowerCase().split(" ").includes("ang") + || mwtext.query.pages[Object.keys(mwextracts.query.pages)[0]].extract.replace(/[.,?!;()"'-]/g, " ").replace(/\s+/g, " ").toLowerCase().split(" ").includes("ang") + ) { + infoboxes[page.name]["_gen"] = 5; + } else { + infoboxes[page.name]["_gen"] = -1; + } + } catch(e) {} + } catch (e) { + console.error(e); + } + } + + fs.writeFileSync("./data/boxes.json", JSON.stringify(infoboxes, null, 4)) +})() \ No newline at end of file diff --git a/update/ponies/listgen.js b/update/ponies/listgen.js new file mode 100644 index 0000000..d9401f8 --- /dev/null +++ b/update/ponies/listgen.js @@ -0,0 +1,24 @@ +const axios = require("axios"); +const fs = require("fs"); + +if (fs.existsSync("./data")) fs.rmSync("./data", { recursive: true }); +fs.mkdirSync("./data"); + +(async () => { + async function getCategory(category) { + console.log("Category:" + category); + let cat = (await axios.get("https://mlp.fandom.com/api.php?action=query&generator=categorymembers&gcmtitle=Category:" + encodeURI(category) + "&prop=categories&cllimit=max&gcmlimit=max&format=json")).data; + + return Object.keys(cat.query.pages).map(k => cat.query.pages[k].title).filter(k => !k.startsWith("List") && !k.includes("EG") && !k.toLowerCase().includes("ponies") && !k.includes(" and ") && !k.includes("(") && !k.includes("family")); + } + + let list = [...new Set([ + ...(await getCategory("Pegasus ponies")), + ...(await getCategory("Alicorn ponies")), + ...(await getCategory("Earth ponies")), + ...(await getCategory("Unicorn ponies")), + ...(await getCategory("Main characters")), + ...(await getCategory("Dragons")), + ])]; + fs.writeFileSync("./data/list.json", JSON.stringify(list, null, 4)) +})() \ No newline at end of file diff --git a/update/ponies/pages.js b/update/ponies/pages.js new file mode 100644 index 0000000..7f3a1c8 --- /dev/null +++ b/update/ponies/pages.js @@ -0,0 +1,28 @@ +const fs = require('fs'); +const axios = require('axios'); + +(async () => { + console.log("Gathering pages list..."); + let pages = []; + for (let page of JSON.parse(fs.readFileSync("./data/list.json").toString())) { + console.log("Searching for '" + page + "'..."); + try { + let data = (await axios.get("https://mlp.fandom.com/api.php?action=query&list=search&srsearch=" + encodeURI(page) + "&srlimit=1&srenablerewrites=true&format=json")).data; + if (data.query.search.length > 0) { + console.log("Results found, adding name to database") + pages.push({ + query: page, + name: data.query.search[0].title, + mwid: data.query.search[0].pageid, + words: data.query.search[0].wordcount, + }) + } else { + console.log("No results found, ignoring name"); + } + } catch (e) { + console.error(e); + } + } + + fs.writeFileSync("./data/pages.json", JSON.stringify(pages, null, 4)) +})() \ No newline at end of file diff --git a/update/ponies/parse.js b/update/ponies/parse.js new file mode 100644 index 0000000..bd06081 --- /dev/null +++ b/update/ponies/parse.js @@ -0,0 +1,115 @@ +const fs = require('fs'); +const axios = require('axios'); + +console.log("Parsing infobox data..."); + +let ponies = {}; + +(async () => { + for (let title in JSON.parse(fs.readFileSync("./data/boxes.json").toString())) { + console.log("Parsing " + title + "..."); + let box = JSON.parse(fs.readFileSync("./data/boxes.json").toString())[title]; + let data = { + names: [title], + extract: "", + extract_fr: "", + generation: -1, + color: "000000", + image: "https://example.com", + kind: "Pony", + sex: "Unknown", + occupation: ["Unknown"], + residence: ["Unknown"], + mark: "https://example.com" + } + + if (typeof box._gen !== "undefined") data.generation = box._gen; + + if (typeof box.name2 !== "undefined") data.names.push(box.name2.replace(//g, "")); + if (typeof box.name3 !== "undefined") data.names.push(box.name3.replace(//g, "")); + if (typeof box.name4 !== "undefined") data.names.push(box.name4.replace(//g, "")); + if (typeof box.name5 !== "undefined") data.names.push(box.name5.replace(//g, "")); + + if (typeof box.nicknames !== "undefined") { + box.nicknames.replace(//g, "").split(",").filter(e => !e.match(/[^a-zA-Z0-9-_ ]/gm)).forEach((e, i) => { + data.names.push(e.trim()); + }); + } + + if (typeof box.kind !== "undefined") { + kp = box.kind.replace(//g, "").replace(/[^a-zA-Z0-9-_ ]/gm, "").split(" ")[0]; + data.kind = kp.substring(kp.replace(/([A-Z])([a-z0-9]*)$/g, "").length); + } + if (typeof box.sex !== "undefined") data.sex = box.sex.replace(//g, "").trim().startsWith("F") ? "F" : "M"; + if (typeof box._extract !== "undefined") { + data.extract = box._extract; + } + if (typeof box.coat !== "undefined") data.color = box.coat.replace(//g, "").trim().replace(/\[([a-z.\/ \nA-Z0-9:]*)\/(.{6})\/ (.*)\]/gm, "$2").replace(/{{perbang\|([0-9A-Fa-f].{5})(.*)/g, "$1"); + try { + if (typeof box.main !== "undefined") data.image = (await axios.head("https://mlp.fandom.com/Special:FilePath/" + encodeURI(box.main.replace(//g, "").trim()).replaceAll("?", "%3F").replaceAll("&", "%26"))).request.res.responseUrl; + } catch (e) { + try { + if (typeof box.main !== "undefined") data.image = e.request.res.responseUrl; + } catch (e2) { + console.error(e2); + throw e; + } + } + try { + if (typeof box.main1 !== "undefined") data.image = (await axios.head("https://mlp.fandom.com/Special:FilePath/" + encodeURI(box.main1.replace(//g, "").trim()).replaceAll("?", "%3F").replaceAll("&", "%26"))).request.res.responseUrl; + } catch (e) { + try { + if (typeof box.main1 !== "undefined") data.image = e.request.res.responseUrl; + } catch (e2) { + console.error(e2); + throw e; + } + } + try { + if (typeof box.image !== "undefined") data.image = (await axios.head("https://mlp.fandom.com/Special:FilePath/" + encodeURI(box.image.replace(//g, "").trim()).replaceAll("?", "%3F").replaceAll("&", "%26"))).request.res.responseUrl; + } catch (e) { + try { + if (typeof box.image !== "undefined") data.image = e.request.res.responseUrl; + } catch (e2) { + console.error(e2); + throw e; + } + } + if (typeof box["cutie mark"] !== "undefined") { + try { + data.markimg = box["cutie mark"].replace(//g, "").trim().split("[[File:")[1].split("|")[0]; + } catch (e) { + data.markimg = box["cutie mark"].replace(//g, "").trim().split("[[File:")[0].split("|")[0]; + } + try { + data.mark = (await axios.head("https://mlp.fandom.com/Special:FilePath/" + encodeURI(data.markimg).replaceAll("?", "%3F").replaceAll("&", "%26"))).request.res.responseUrl; + } catch (e) { + try { + data.mark = e.request.res.responseUrl; + } catch (e2) { + console.error(e2); + throw e; + } + } + } + + if (typeof box.occupation !== "undefined") { + occupations = []; + box.occupation/*.replace(/\[\[(.*)_(.*)\]\]|\[\[(.*)\|(.*)\]\]|\[\[(.*)\]\]/gm, "$2$4$5")*/.replace(//g, "").trim().replace(/\|/gm, "_").replace(/<( ||(|| )\/)( ||(|| )\/)(b|B)(r|R)( ||(|| )\/)( ||(|| )\/)>/gm, "|").replace(/( \|| \| | \| )/gm, "|").split("|").forEach((e) => { + occupations.push(e.trim().replace(/[\[\]]/gm, "").replace(/<(.*)>/gm, "").replace(/([a-zA-Z0-9 \-_,.'"]*)_([a-zA-Z0-9 \-_,.'"]*)/gm, "$2")); + }) + data.occupation = occupations; + } + if (typeof box.residence !== "undefined") { + residences = []; + box.residence/*.replace(/\[\[(.*)_(.*)\]\]|\[\[(.*)\|(.*)\]\]|\[\[(.*)\]\]/gm, "$2$4$5")*/.replace(//g, "").trim().replace(/\|/gm, "_").replace(/<( ||(|| )\/)( ||(|| )\/)(b|B)(r|R)( ||(|| )\/)( ||(|| )\/)>/gm, "|").replace(/( \|| \| | \| )/gm, "|").split("|").forEach((e) => { + residences.push(e.trim().replace(/[\[\]]/gm, "").replace(/<(.*)>/gm, "").replace(/([a-zA-Z0-9 \-_,.'"]*)_([a-zA-Z0-9 \-_,.'"]*)/gm, "$2")); + }) + data.residence = residences.replaceAll("Locations#", ""); + } + + if ((typeof box.name2 !== "undefined" || typeof box.name3 !== "undefined" || typeof box.name4 !== "undefined" || typeof box.name5 !== "undefined" || typeof box.coat !== "undefined" || typeof box.occupation !== "undefined" || typeof box.residence !== "undefined") && typeof box.sex !== "undefined") ponies[title] = data; + } +})() + +fs.writeFileSync("./data/data.json", JSON.stringify(ponies, null, 4)); \ No newline at end of file -- cgit