From 914bcbb474f6f186c212b2da0d9d864b5e75d8e4 Mon Sep 17 00:00:00 2001 From: RaindropsSys Date: Sat, 29 Jun 2024 20:59:16 +0200 Subject: Rename to Faunerie --- src/FaunerieSettings.ts | 290 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 290 insertions(+) create mode 100755 src/FaunerieSettings.ts (limited to 'src/FaunerieSettings.ts') diff --git a/src/FaunerieSettings.ts b/src/FaunerieSettings.ts new file mode 100755 index 0000000..74919ae --- /dev/null +++ b/src/FaunerieSettings.ts @@ -0,0 +1,290 @@ +import {FaunerieApp} from "./FaunerieApp"; +import * as fs from "node:fs"; +import * as path from "node:path"; +import {FaunerieImageType, FaunerieListType} from "libfaunerie"; + +export class FaunerieSettings { + instance: FaunerieApp; + + constructor(instance: FaunerieApp) { + this.instance = instance; + } + + async getThumbnailFiles() { + let _dataStore = this.instance.dataStore; + let files = []; + let filesPre = await fs.promises.readdir(_dataStore.source + "/thumbnails"); + + for (let i of filesPre) { + if ((await fs.promises.lstat(_dataStore.source + "/thumbnails/" + i)).isDirectory()) { + let list = await fs.promises.readdir(_dataStore.source + "/thumbnails/" + i); + + for (let j of list) { + if ((await fs.promises.lstat(_dataStore.source + "/thumbnails/" + i + "/" + j)).isDirectory()) { + let list2 = await fs.promises.readdir(_dataStore.source + "/thumbnails/" + i + "/" + j); + + for (let k of list2) { + if ((await fs.promises.lstat(_dataStore.source + "/thumbnails/" + i + "/" + j + "/" + k)).isDirectory()) { + let list3 = await fs.promises.readdir(_dataStore.source + "/thumbnails/" + i + "/" + j + "/" + k); + + for (let l of list3) { + files.push(_dataStore.source + "/thumbnails/" + i + "/" + j + "/" + k + "/" + l); + } + } + } + } + } + } + } + + return files; + } + + async getImageFiles() { + let _dataStore = this.instance.dataStore; + let files = []; + let filesPre = await fs.promises.readdir(_dataStore.source + "/images"); + + for (let i of filesPre) { + if ((await fs.promises.lstat(_dataStore.source + "/images/" + i)).isDirectory()) { + let list = await fs.promises.readdir(_dataStore.source + "/images/" + i); + + for (let j of list) { + if ((await fs.promises.lstat(_dataStore.source + "/images/" + i + "/" + j)).isDirectory()) { + let list2 = await fs.promises.readdir(_dataStore.source + "/images/" + i + "/" + j); + + for (let k of list2) { + if ((await fs.promises.lstat(_dataStore.source + "/images/" + i + "/" + j + "/" + k)).isDirectory()) { + let list3 = await fs.promises.readdir(_dataStore.source + "/images/" + i + "/" + j + "/" + k); + + for (let l of list3) { + files.push(_dataStore.source + "/images/" + i + "/" + j + "/" + k + "/" + l); + } + } + } + } + } + } + } + + return files; + } + + async processList(files: string[]) { + let _dataStore = this.instance.dataStore; + + let total = files.length; + let index = 0; + let orphans = []; + + for (let file of files) { + document.getElementById("load").innerText = "Looking for orphan files... " + file; + + if (!(await _dataStore.database.frontend.getImage(path.basename(file, path.extname(file))))) { + orphans.push(file); + } + + index++; + document.getElementById("progress").style.width = ((index / total) * 100) + "%"; + } + + return orphans; + } + + async processOrphans(orphans: string[]): Promise { + let hadErrors = false; + + document.getElementById("load").innerText = "Removing orphan files..."; + document.getElementById("progress").classList.remove("progress-bar-striped"); + document.getElementById("progress").style.width = "0%"; + + let index = 0; + + for (let orphan of orphans) { + document.getElementById("load").innerText = "Removing orphan files... " + orphan; + document.getElementById("progress").classList.remove("progress-bar-striped"); + document.getElementById("progress").style.width = ((index / orphans.length) * 100) + "%"; + + try { + await fs.promises.unlink(orphan); + } catch (e) { + console.error(e); + this.instance.loadingError("Failed to remove " + orphan); + hadErrors = true; + } + + index++; + } + + return hadErrors; + } + + async removeOrphans() { + let _dataStore = this.instance.dataStore; + if (_dataStore.loadedFromCache) return; + + _dataStore.loader.show(); + _dataStore.hadErrorsLoading = false; + document.getElementById("progress").classList.remove("progress-bar-striped"); + document.getElementById("loader-errors-list").innerHTML = ""; + document.getElementById("loader-errors").style.display = "none"; + + document.getElementById("load").innerText = "Looking for orphan files..."; + document.getElementById("progress").classList.remove("progress-bar-striped"); + document.getElementById("progress").style.width = "0%"; + + let files = [ + ...(await this.getThumbnailFiles()), + ...(await this.getImageFiles()) + ]; + + let hadErrors = false; + let orphans = await this.processList(files); + + if (orphans.length > 0) { + hadErrors = await this.processOrphans(orphans); + } + + document.getElementById("load").innerText = hadErrors ? "This operation completed with some errors." : "This operation completed successfully."; + document.getElementById("progress").classList.remove("progress-bar-striped"); + document.getElementById("progress").style.width = "100%"; + document.getElementById("loading-btn").classList.remove("disabled"); + document.getElementById("loader-errors").style.display = ""; + } + + private protectedEncode(b: string | ArrayBuffer) { + return require('zlib').deflateRawSync(b, {level: 9}); + } + + checkImageForCorruptions(image: any): Promise { + let i = image; + let instance = this.instance; + let _dataStore = this.instance.dataStore; + + return new Promise(async (res) => { + function next() { + let imageIsCorrupted: Function; + let img = i['mime_type'].startsWith("image/") ? new Image() : document.createElement("video"); + img.onload = img.oncanplaythrough = () => { + img.remove(); + if ("srcObject" in img) img.srcObject = null; + // noinspection HttpUrlsUsage + if (img.src.startsWith("https://") || img.src.startsWith("http://")) imageIsCorrupted(); + res(false); + } + img.onerror = imageIsCorrupted = () => { + instance.loadingError("Image " + i.id + " is corrupted"); + res(true); + } + img.src = _dataStore.database.frontend.getImageFile(i, FaunerieImageType.ViewURL); + } + + let imageIsCorrupted: Function; + let img = new Image(); + img.onload = img.oncanplaythrough = () => { + img.remove(); + if ("srcObject" in img) img.srcObject = null; + // noinspection HttpUrlsUsage + if (img.src.startsWith("https://") || img.src.startsWith("http://")) imageIsCorrupted(); + next(); + } + img.onerror = imageIsCorrupted = () => { + instance.loadingError("Image " + i.id + " is corrupted"); + res(true); + } + img.src = _dataStore.database.frontend.getImageFile(i, FaunerieImageType.ThumbnailURL); + }); + } + + async repairCorruptedImage(i: any) { + let _dataStore = this.instance.dataStore; + + try { + await fs.promises.writeFile(_dataStore.source + "/images/" + (i['sha512_hash'] ?? i['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (i['sha512_hash'] ?? i['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + i.id + ".bin", this.protectedEncode(Buffer.from(await (await fetch(i['view_url'])).arrayBuffer()))); + await fs.promises.writeFile(_dataStore.source + "/thumbnails/" + (i['sha512_hash'] ?? i['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (i['sha512_hash'] ?? i['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + i.id + ".bin", this.protectedEncode(Buffer.from(await (await fetch(i['representations']['thumb'])).arrayBuffer()))); + return true; + } catch (e) { + console.error(e); + this.instance.loadingError("Failed to repair image " + i.id); + return false; + } + } + + finishRepairing(hadErrorsFixing: boolean) { + if (this.instance.dataStore.hadErrorsLoading) { + if (hadErrorsFixing) { + document.getElementById("load").innerText = "Corrupted files found and could not be repaired."; + } else { + document.getElementById("load").innerText = "Corrupted files found and repaired successfully."; + } + + document.getElementById("progress").classList.remove("progress-bar-striped"); + document.getElementById("progress").style.width = "100%"; + document.getElementById("loading-btn").classList.remove("disabled"); + } else { + document.getElementById("load").innerText = "No corrupted files found."; + document.getElementById("progress").classList.remove("progress-bar-striped"); + document.getElementById("progress").style.width = "100%"; + document.getElementById("loading-btn").classList.remove("disabled"); + document.getElementById("loader-errors").style.display = ""; + } + } + + async repairScanForCorruptions() { + let corrupted = []; + let _dataStore = this.instance.dataStore; + + _dataStore.loader.show(); + _dataStore.hadErrorsLoading = false; + document.getElementById("progress").classList.remove("progress-bar-striped"); + document.getElementById("loader-errors-list").innerHTML = ""; + document.getElementById("loader-errors").style.display = "none"; + document.getElementById("load").innerText = "Checking for corrupted images..."; + + let total = await _dataStore.database.frontend.countImages(); + let index = 0; + + document.getElementById("progress").style.width = "0%"; + + for (let image of await _dataStore.database.frontend.getAllImages(FaunerieListType.Array) as any[]) { + document.getElementById("load").innerText = "Checking for corrupted images... " + Math.round(((index / total) * 100)) + "% (" + image.id + ")"; + if (await this.checkImageForCorruptions(image)) { + corrupted.push(image); + } + + index++; + document.getElementById("progress").style.width = ((index / total) * 100) + "%"; + } + + document.getElementById("progress").style.width = "0%"; + document.getElementById("load").innerText = "Repairing corrupted images..."; + index = 0; + + return corrupted; + } + + async repairProcessCorruptions(corrupted: any[]) { + let hadErrorsFixing = false; + let index = 0; + + for (let file of corrupted) { + hadErrorsFixing = hadErrorsFixing && await this.repairCorruptedImage(file); + + document.getElementById("progress").style.width = ((index / corrupted.length) * 100) + "%"; + document.getElementById("load").innerText = "Repairing corrupted images... " + Math.round(((index / corrupted.length) * 100)) + "% (" + file.id + ")"; + index++; + } + + return hadErrorsFixing; + } + + async repairCorruptions() { + if (this.instance.dataStore.loadedFromCache) return; + + this.finishRepairing( + await this.repairProcessCorruptions( + await this.repairScanForCorruptions() + ) + ) + } +} -- cgit