From c2f5b8216d85c113a132d34dd9470806d1f97f9e Mon Sep 17 00:00:00 2001 From: RaindropsSys Date: Sat, 29 Jun 2024 19:29:48 +0200 Subject: Rename to Faunerie --- ._.DS_Store | Bin 4096 -> 0 bytes index.ts | 10 +- package-lock.json | 8 +- package.json | 4 +- src/Faunerie.ts | 152 +++++++++ src/FaunerieFrontend.ts | 404 ++++++++++++++++++++++++ src/FaunerieImageType.ts | 3 + src/FaunerieListType.ts | 3 + src/FauneriePropertyStore.ts | 58 ++++ src/FaunerieSearch.ts | 678 +++++++++++++++++++++++++++++++++++++++++ src/FaunerieUpdater.ts | 522 +++++++++++++++++++++++++++++++ src/IFaunerieImage.ts | 1 + src/IFaunerieOption.ts | 8 + src/IFaunerieTag.ts | 1 + src/IFaunerieUpdaterGeneric.ts | 1 + src/IPrisbeamImage.ts | 1 - src/IPrisbeamOption.ts | 8 - src/IPrisbeamTag.ts | 1 - src/IPrisbeamUpdaterGeneric.ts | 1 - src/Prisbeam.ts | 152 --------- src/PrisbeamFrontend.ts | 404 ------------------------ src/PrisbeamImageType.ts | 3 - src/PrisbeamListType.ts | 3 - src/PrisbeamPropertyStore.ts | 58 ---- src/PrisbeamSearch.ts | 678 ----------------------------------------- src/PrisbeamUpdater.ts | 522 ------------------------------- 26 files changed, 1842 insertions(+), 1842 deletions(-) delete mode 100755 ._.DS_Store create mode 100755 src/Faunerie.ts create mode 100755 src/FaunerieFrontend.ts create mode 100755 src/FaunerieImageType.ts create mode 100755 src/FaunerieListType.ts create mode 100755 src/FauneriePropertyStore.ts create mode 100755 src/FaunerieSearch.ts create mode 100755 src/FaunerieUpdater.ts create mode 100755 src/IFaunerieImage.ts create mode 100755 src/IFaunerieOption.ts create mode 100755 src/IFaunerieTag.ts create mode 100755 src/IFaunerieUpdaterGeneric.ts delete mode 100755 src/IPrisbeamImage.ts delete mode 100755 src/IPrisbeamOption.ts delete mode 100755 src/IPrisbeamTag.ts delete mode 100755 src/IPrisbeamUpdaterGeneric.ts delete mode 100755 src/Prisbeam.ts delete mode 100755 src/PrisbeamFrontend.ts delete mode 100755 src/PrisbeamImageType.ts delete mode 100755 src/PrisbeamListType.ts delete mode 100755 src/PrisbeamPropertyStore.ts delete mode 100755 src/PrisbeamSearch.ts delete mode 100755 src/PrisbeamUpdater.ts diff --git a/._.DS_Store b/._.DS_Store deleted file mode 100755 index 09cd8c8..0000000 Binary files a/._.DS_Store and /dev/null differ diff --git a/index.ts b/index.ts index 5fef37f..601072c 100755 --- a/index.ts +++ b/index.ts @@ -1,6 +1,6 @@ -export {Prisbeam} from "./src/Prisbeam"; -export {PrisbeamImageType} from "./src/PrisbeamImageType"; -export {PrisbeamListType} from "./src/PrisbeamListType"; -export {PrisbeamUpdater} from "./src/PrisbeamUpdater"; +export {Faunerie} from "./src/Faunerie"; +export {FaunerieImageType} from "./src/FaunerieImageType"; +export {FaunerieListType} from "./src/FaunerieListType"; +export {FaunerieUpdater} from "./src/FaunerieUpdater"; -export const VERSION: string = "2.2.0"; +export const VERSION: string = require('./package.json').version; diff --git a/package-lock.json b/package-lock.json index 41308f8..c8f3a55 100755 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { - "name": "libprisbeam", - "version": "2.3.3", + "name": "libfaunerie", + "version": "2.4.0", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "libprisbeam", - "version": "2.3.3", + "name": "libfaunerie", + "version": "2.4.0", "hasInstallScript": true, "dependencies": { "@types/sqlite3": "^3.1.11" diff --git a/package.json b/package.json index f7b83d1..474a6b2 100755 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { - "name": "libprisbeam", - "version": "2.3.4", + "name": "libfaunerie", + "version": "2.4.0", "main": "index.js", "scripts": { "install": "tsc", diff --git a/src/Faunerie.ts b/src/Faunerie.ts new file mode 100755 index 0000000..aab2ec6 --- /dev/null +++ b/src/Faunerie.ts @@ -0,0 +1,152 @@ +import {FaunerieFrontend} from "./FaunerieFrontend"; +import {IFaunerieOption} from "./IFaunerieOption"; +import fs from "fs"; +import path from "path"; +import {VERSION} from "../index"; +import {FauneriePropertyStore} from "./FauneriePropertyStore"; +import {Database} from "sqlite3"; +import {SQLiteInstance} from "./SQLiteInstance"; + +export class Faunerie { + // noinspection JSUnusedGlobalSymbols + public version: string = VERSION; + public readonly verbose: boolean; + public readonly path: string; + readonly sensitiveImageProtocol: boolean; + public frontend: FaunerieFrontend; + readonly cache?: string; + private sqlite: SQLiteInstance; + private database: Database; + private readonly readOnly: boolean; + public propertyStore: FauneriePropertyStore; + + constructor(options: IFaunerieOption) { + this.verbose = options.verbose ?? true; + this.sqlite = require(options.sqlitePath ?? "sqlite3").verbose(); + + if (!fs.existsSync(path.resolve(options.database))) throw new Error("Invalid database folder specified: " + path.resolve(options.database)); + if (!fs.existsSync(path.resolve(options.database) + "/instance.pbmk")) throw new Error("Not a valid Faunerie database: " + path.resolve(options.database)); + + this.path = path.resolve(options.database); + + if (options.cachePath) { + if (!fs.existsSync(path.resolve(options.cachePath))) throw new Error("Invalid cache folder specified: " + path.resolve(options.cachePath)); + this.cache = path.resolve(options.cachePath); + } + + this.readOnly = options.readOnly; + this.sensitiveImageProtocol = options.sensitiveImageProtocol ?? false; + } + + async clean() { + if (this.readOnly) throw new Error("The database is open is read-only mode."); + await this._sql("DROP TABLE IF EXISTS image_tags"); + await this._sql("DROP TABLE IF EXISTS image_intensities"); + await this._sql("DROP TABLE IF EXISTS image_representations"); + await this._sql("DROP TABLE IF EXISTS image_categories"); + await this._sql("DROP TABLE IF EXISTS images"); + await this._sql("DROP TABLE IF EXISTS uploaders"); + await this._sql("DROP TABLE IF EXISTS tags"); + await this._sql("DROP TABLE IF EXISTS tags_pre"); + await this._sql("DROP TABLE IF EXISTS compressed"); + await this._sql("CREATE TABLE images (id INT NOT NULL UNIQUE, source_id INT NOT NULL UNIQUE, source_name TEXT, source TEXT NOT NULL, animated BOOL, aspect_ratio FLOAT, comment_count INT, created_at TIMESTAMP, deletion_reason LONGTEXT, description LONGTEXT, downvotes INT, duplicate_of INT, duration FLOAT, faves INT, first_seen_at TIMESTAMP, format TEXT, height INT, hidden_from_users BOOL, mime_type TEXT, name LONGTEXT, orig_sha512_hash TEXT, processed BOOL, score INT, sha512_hash TEXT, size INT, source_url LONGTEXT, spoilered BOOL, tag_count INT, thumbnails_generated BOOL, updated_at TIMESTAMP, uploader INT, upvotes INT, width INT, wilson_score FLOAT, PRIMARY KEY (id), FOREIGN KEY (uploader) REFERENCES uploaders(id))"); + await this._sql("CREATE TABLE image_tags (image_id INT NOT NULL UNIQUE, tags LONGTEXT NOT NULL, PRIMARY KEY (image_id), FOREIGN KEY (image_id) REFERENCES images(id))"); + await this._sql("CREATE TABLE image_intensities (image_id INT NOT NULL UNIQUE, ne FLOAT NOT NULL, nw FLOAT NOT NULL, se FLOAT NOT NULL, sw FLOAT NOT NULL, PRIMARY KEY (image_id), FOREIGN KEY (image_id) REFERENCES images(id))"); + await this._sql("CREATE TABLE image_representations (image_id INT NOT NULL UNIQUE, view LONGTEXT NOT NULL, full TEXT, large TEXT, medium TEXT, small TEXT, tall TEXT, thumb TEXT, thumb_small TEXT, thumb_tiny TEXT, PRIMARY KEY (image_id), FOREIGN KEY (image_id) REFERENCES images(id))"); + await this._sql("CREATE TABLE tags (id INT NOT NULL UNIQUE, name TEXT NOT NULL UNIQUE, alias INT, implications LONGTEXT, category TEXT, description LONGTEXT, description_short LONGTEXT, slug TEXT UNIQUE, PRIMARY KEY (id))"); + await this._sql("CREATE TABLE uploaders (id INT NOT NULL UNIQUE, name TEXT, PRIMARY KEY (id))"); + await this._sql("CREATE TABLE IF NOT EXISTS metadata (key TEXT NOT NULL UNIQUE, value LONGTEXT NOT NULL, PRIMARY KEY (key))"); + } + + _sql(query: string) { + let verbose = this.verbose; + if (verbose) console.debug("=>", query); + + return new Promise((res, rej) => { + this.database.all(query, function (err: Error | null, data?: any) { + if (err) { + if (verbose) console.debug("<=", data); + rej(err); + } else { + if (verbose) console.debug("<=", data); + res(data); + } + }); + }); + } + + // noinspection JSUnusedGlobalSymbols + async initialize(restoreBackup: boolean) { + if (restoreBackup) { + let backups = (await fs.promises.readdir(this.path)).filter(i => i.endsWith(".pbdb") && i !== "current.pbdb"); + + if (backups.length > 0 && !isNaN(parseInt(backups[0].split(".")[0]))) { + await fs.promises.copyFile(this.path + "/" + backups[0], this.path + "/current.pbdb"); + await fs.promises.unlink(this.path + "/" + backups[0]); + } + } + + if (this.cache) { + await fs.promises.copyFile(this.path + "/current.pbdb", this.cache + "/work.pbdb"); + + if (this.readOnly) { + this.database = new this.sqlite.Database(this.cache + "/work.pbdb", this.sqlite.OPEN_READONLY); + } else { + this.database = new this.sqlite.Database(this.cache + "/work.pbdb"); + } + } else { + if (this.readOnly) { + this.database = new this.sqlite.Database(this.path + "/current.pbdb", this.sqlite.OPEN_READONLY); + } else { + this.database = new this.sqlite.Database(this.path + "/current.pbdb"); + } + } + + await new Promise((res) => { + this.database.serialize(() => { + res(); + }); + }); + + if (!this.readOnly) { + if ((await this._sql("SELECT COUNT(*) FROM metadata WHERE key='libfaunerie_timestamp'"))[0]["COUNT(*)"] === 0) { + await this._sql('INSERT INTO metadata(key, value) VALUES ("libfaunerie_timestamp", "' + new Date().toISOString() + '")'); + } else { + await this._sql('UPDATE metadata SET value="' + new Date().toISOString() + '" WHERE key="libfaunerie_timestamp"'); + } + } + + await this._sql("CREATE TABLE IF NOT EXISTS metadata (key TEXT NOT NULL UNIQUE, value LONGTEXT NOT NULL, PRIMARY KEY (key))"); + + this.frontend = new FaunerieFrontend(this); + this.propertyStore = new FauneriePropertyStore(this); + await this.propertyStore.initialize(); + + await this.frontend.initialize(); + await this.defragment(); + } + + async defragment() { + await this._sql("VACUUM"); + } + + // noinspection JSUnusedGlobalSymbols + async close() { + await new Promise((res) => { + // @ts-ignore + this.database.wait(() => { + res(); + }); + }); + + await new Promise((res) => { + this.database.close(() => { + res(); + }); + }); + + if (this.cache) { + await fs.promises.copyFile(this.cache + "/work.pbdb", this.path + "/current.pbdb"); + } + } +} diff --git a/src/FaunerieFrontend.ts b/src/FaunerieFrontend.ts new file mode 100755 index 0000000..30be154 --- /dev/null +++ b/src/FaunerieFrontend.ts @@ -0,0 +1,404 @@ +import {FaunerieSearch} from "./FaunerieSearch"; +import {FaunerieImageType} from "./FaunerieImageType"; +import fs from "fs"; +import zlib from "zlib"; +import {FaunerieListType} from "./FaunerieListType"; +import {Faunerie} from "../index"; +import {SearchError} from "./SearchError"; +import {IFaunerieImage} from "./IFaunerieImage"; +import {IFaunerieTag} from "./IFaunerieTag"; + +export class FaunerieFrontend { + public tags: IFaunerieTag[]; + public tagsHashed: object; + private readonly backend: Faunerie; + readonly searchEngine: FaunerieSearch; + private readonly sensitiveImageProtocol: boolean; + + constructor(backend: Faunerie) { + this.backend = backend; + this.sensitiveImageProtocol = backend.sensitiveImageProtocol; + this.searchEngine = new FaunerieSearch(this); + } + + async initialize() { + this.tags = []; + this.tagsHashed = {}; + + for (let entry of await this.backend._sql("SELECT id, name FROM tags")) { + this.tags.push([entry["id"], entry["name"]]); + this.tagsHashed[entry["id"]] = entry["name"]; + } + } + + // noinspection JSUnusedGlobalSymbols + async search(query: string, allowUnknownTags: boolean = false) { + try { + if (query !== "*") { + let sql = this.searchEngine.buildQueryV2(query, allowUnknownTags); + return await this.imageListResolver(await this.backend._sql("SELECT * FROM images JOIN image_tags ON images.id=image_tags.image_id JOIN image_intensities ON images.id=image_intensities.image_id JOIN image_representations ON images.id=image_representations.image_id WHERE " + sql)); + } else { + return await this.getAllImages(FaunerieListType.Array); + } + } catch (e) { + if (e.message.startsWith("SQLITE_ERROR: Expression tree is too large (maximum depth 1000)")) { + throw new SearchError("This search query leads to an internal query that is too large."); + } else { + throw e; + } + } + } + + // noinspection JSUnusedGlobalSymbols + async getImageFileFromId(id: number, type: FaunerieImageType) { + let image = (await this.imageListResolver(await this.backend._sql("SELECT * FROM images JOIN image_tags ON images.id=image_tags.image_id JOIN image_intensities ON images.id=image_intensities.image_id JOIN image_representations ON images.id=image_representations.image_id WHERE id=" + id)))[0]; + return this.getImageFile(image, type); + } + + // noinspection JSUnusedGlobalSymbols + async getImage(id: number | string): Promise { + return (await this.imageListResolver(await this.backend._sql("SELECT * FROM images JOIN image_tags ON images.id=image_tags.image_id JOIN image_intensities ON images.id=image_intensities.image_id JOIN image_representations ON images.id=image_representations.image_id WHERE id=" + id)))[0] ?? null; + } + + // noinspection JSUnusedGlobalSymbols + async countImages(): Promise { + return ((await this.imageListResolver(await this.backend._sql("SELECT COUNT(*) FROM images")))[0] ?? {})["COUNT(*)"] ?? 0; + } + + getImageFile(image: object, type: FaunerieImageType) { + function getPath(backend: Faunerie) { + const path = require('path'); + + if (type === FaunerieImageType.ViewFile || type === FaunerieImageType.ViewURL) { + try { + let l: fs.PathLike; + + try { + l = backend.path + "/images/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 3) + "/" + image['id'] + path.extname(image['view_url']); + fs.lstatSync(l); + } catch (e) { + l = backend.path + "/images/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + image['id'] + path.extname(image['view_url']); + fs.lstatSync(l); + } + + if (type === FaunerieImageType.ViewURL) { + return "file://" + encodeURI(l.replaceAll("\\", "/")); + } else { + return l; + } + } catch (e) { + try { + let l: fs.PathLike; + + try { + l = backend.path + "/images/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 3) + "/" + image['id'] + ".mp4"; + fs.lstatSync(l); + } catch (e) { + l = backend.path + "/images/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + image['id'] + ".mp4"; + fs.lstatSync(l); + } + + if (type === FaunerieImageType.ViewURL) { + return "file://" + encodeURI(l.replaceAll("\\", "/")); + } else { + return l; + } + } catch (e) { + try { + let l: fs.PathLike; + + try { + l = backend.path + "/images/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 3) + "/" + image['id'] + ".bin"; + fs.lstatSync(l); + } catch (e) { + l = backend.path + "/images/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + image['id'] + ".bin"; + fs.lstatSync(l); + } + + if (type === FaunerieImageType.ViewURL) { + return "file://" + encodeURI(l.replaceAll("\\", "/")); + } else { + return l; + } + } catch (e) { + try { + let l: fs.PathLike; + + try { + l = backend.path + "/thumbnails/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 3) + "/" + image['id'] + path.extname(image['representations']['thumb']); + fs.lstatSync(l); + } catch (e) { + l = backend.path + "/thumbnails/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + image['id'] + path.extname(image['representations']['thumb']); + fs.lstatSync(l); + } + + if (type === FaunerieImageType.ViewURL) { + return "file://" + encodeURI(l.replaceAll("\\", "/")); + } else { + return l; + } + } catch (e) { + try { + let l: fs.PathLike; + + try { + l = backend.path + "/thumbnails/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 3) + "/" + image['id'] + ".bin"; + fs.lstatSync(l); + } catch (e) { + l = backend.path + "/thumbnails/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + image['id'] + ".bin"; + fs.lstatSync(l); + } + + if (type === FaunerieImageType.ViewURL) { + return "file://" + encodeURI(l.replaceAll("\\", "/")); + } else { + return l; + } + } catch (e) { + if (type === FaunerieImageType.ViewFile) { + return null; + } else { + return image['representations']['thumb']; + } + } + } + } + } + } + } else if (type === FaunerieImageType.ThumbnailFile || type === FaunerieImageType.ThumbnailURL) { + try { + let l: fs.PathLike; + + try { + l = backend.path + "/thumbnails/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 3) + "/" + image['id'] + path.extname(image['representations']['thumb']); + fs.lstatSync(l); + } catch (e) { + l = backend.path + "/thumbnails/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + image['id'] + path.extname(image['representations']['thumb']); + fs.lstatSync(l); + } + + if (type === FaunerieImageType.ThumbnailURL) { + return "file://" + encodeURI(l.replaceAll("\\", "/")); + } else { + return l; + } + } catch (e) { + try { + let l: fs.PathLike; + + try { + l = backend.path + "/thumbnails/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 3) + "/" + image['id'] + ".bin"; + fs.lstatSync(l); + } catch (e) { + l = backend.path + "/thumbnails/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + image['id'] + ".bin"; + fs.lstatSync(l); + } + + if (type === FaunerieImageType.ThumbnailURL) { + return "file://" + encodeURI(l.replaceAll("\\", "/")); + } else { + return l; + } + } catch (e) { + try { + let l: fs.PathLike; + + try { + l = backend.path + "/images/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 3) + "/" + image['id'] + path.extname(image['view_url']); + fs.lstatSync(l); + } catch (e) { + l = backend.path + "/images/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + image['id'] + path.extname(image['view_url']); + fs.lstatSync(l); + } + + if (type === FaunerieImageType.ThumbnailURL) { + return "file://" + encodeURI(l.replaceAll("\\", "/")); + } else { + return l; + } + } catch (e) { + try { + let l: fs.PathLike; + + try { + l = backend.path + "/images/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 3) + "/" + image['id'] + ".mp4"; + fs.lstatSync(l); + } catch (e) { + l = backend.path + "/images/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + image['id'] + ".mp4"; + fs.lstatSync(l); + } + + if (type === FaunerieImageType.ThumbnailURL) { + return "file://" + encodeURI(l.replaceAll("\\", "/")); + } else { + return l; + } + } catch (e) { + try { + let l: fs.PathLike; + + try { + l = backend.path + "/images/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 3) + "/" + image['id'] + ".bin"; + fs.lstatSync(l); + } catch (e) { + l = backend.path + "/images/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + image['id'] + ".bin"; + fs.lstatSync(l); + } + + if (type === FaunerieImageType.ThumbnailURL) { + return "file://" + encodeURI(l.replaceAll("\\", "/")); + } else { + return l; + } + } catch (e) { + if (type === FaunerieImageType.ThumbnailFile) { + return null; + } else { + return image['representations']['thumb']; + } + } + } + } + } + } + } + } + + let path = getPath(this.backend); + + if (type === FaunerieImageType.ThumbnailURL && path.endsWith(".bin")) { + if (this.sensitiveImageProtocol) { + return path.replace("file://", "pbip://") + "?mime=" + encodeURIComponent(image['mime_type']); + } else { + return URL.createObjectURL(new Blob([zlib.inflateRawSync(fs.readFileSync(path.replace("file://", ""))).buffer], {type: image['mime_type'].startsWith("video/") ? "image/gif" : image['mime_type']})); + } + } else if (type === FaunerieImageType.ViewURL && path.endsWith(".bin")) { + if (this.sensitiveImageProtocol) { + return path.replace("file://", "pbip://") + "?mime=" + encodeURIComponent(image['mime_type']); + } else { + return URL.createObjectURL(new Blob([zlib.inflateRawSync(fs.readFileSync(path.replace("file://", ""))).buffer], {type: image['mime_type']})); + } + } else { + return path; + } + } + + deserialize(str: string) { + return (str ?? "").replaceAll("\\_", "_").replaceAll("\\%", "%").replaceAll("\\'", "'"); + } + + async imageListResolver(list: [object]) { + for (let image of list) { + delete image['image_id']; + + image['name'] = this.deserialize(image['name']); + image['source_url'] = this.deserialize(image['source_url']); + if (image['source']) image['source'] = this.deserialize(image['source']); + image['description'] = this.deserialize(image['description']); + image['_categories'] = ["faved"]; + + image['representations'] = { + full: this.deserialize(image['full']), + large: this.deserialize(image['large']), + medium: this.deserialize(image['medium']), + small: this.deserialize(image['small']), + tall: this.deserialize(image['tall']), + thumb: this.deserialize(image['thumb']), + thumb_small: this.deserialize(image['thumb_small']), + thumb_tiny: this.deserialize(image['thumb_tiny']), + }; + image['view_url'] = this.deserialize(image['view']); + delete image['view']; + delete image['thumb_tiny']; + delete image['thumb_small']; + delete image['thumb']; + delete image['tall']; + delete image['small']; + delete image['medium']; + delete image['large']; + delete image['full']; + + image['intensities'] = { + ne: image['ne'], nw: image['nw'], se: image['se'], sw: image['sw'], + }; + delete image['ne']; + delete image['nw']; + delete image['se']; + delete image['sw']; + + image['tag_ids'] = image['tags'] ? image['tags'].substring(1, image['tags'].length - 1).split(",").map((i: string) => parseInt(i)) : []; + image['tags'] = image['tag_ids'].map((i: number) => this.deserialize(this.tagsHashed[i])); + + image['animated'] = image['animated'] === 1; + image['hidden_from_users'] = image['hidden_from_users'] === 1; + image['processed'] = image['processed'] === 1; + image['spoilered'] = image['spoilered'] === 1; + image['thumbnails_generated'] = image['thumbnails_generated'] === 1; + + image['source_id'] = image['source_id']; + image['source_name'] = image['source_name']; + image['source'] = image['source']; + } + + return list; + } + + async getAllImages(type: FaunerieListType = FaunerieListType.Array): Promise<{} | IFaunerieImage[]> { + let query = "SELECT * FROM images JOIN image_tags ON images.id=image_tags.image_id JOIN image_intensities ON images.id=image_intensities.image_id JOIN image_representations ON images.id=image_representations.image_id"; + + if (type === FaunerieListType.Array) { + return await this.imageListResolver(await this.backend._sql(query)); + } else { + let _list = await this.imageListResolver(await this.backend._sql(query)); + let list = {}; + + for (let image of _list) { + list[image['id']] = image; + } + + return list; + } + } + + // noinspection JSUnusedGlobalSymbols + async getImpliedTagIdsFromName(tag: string): Promise { + return (await this.backend._sql("SELECT implications FROM tags WHERE name = " + sqlstr(tag)))[0]["implications"].split(",").filter((i: string) => i.trim() !== "").map((i: string) => parseInt(i)); + } + + // noinspection JSUnusedGlobalSymbols + async getImpliedTagIdsFromId(tag: number): Promise { + return (await this.backend._sql("SELECT implications FROM tags WHERE id = " + tag))[0]["implications"].split(",").filter((i: string) => i.trim() !== "").map((i: string) => parseInt(i)); + } + + // noinspection JSUnusedGlobalSymbols + async getImpliedTagNamesFromName(tag: string): Promise { + let r = []; + let data = (await this.backend._sql("SELECT implications FROM tags WHERE name = " + sqlstr(tag)))[0]["implications"].split(",").filter((i: string) => i.trim() !== "").map((i: string) => parseInt(i)); + + for (let id of data) { + r.push((await this.backend._sql("SELECT name FROM tags WHERE id = " + id))[0]["name"]); + } + + return r; + } + + // noinspection JSUnusedGlobalSymbols + async getImpliedTagNamesFromId(tag: number): Promise { + let r = []; + let data = (await this.backend._sql("SELECT implications FROM tags WHERE id = " + tag))[0]["implications"].split(",").filter((i: string) => i.trim() !== "").map((i: string) => parseInt(i)); + + for (let id of data) { + r.push((await this.backend._sql("SELECT name FROM tags WHERE id = " + id))[0]["name"]); + } + + return r; + } +} + +function sqlstr(str?: string) { + if (str === null) { + return "NULL"; + } else { + return "'" + str.replaceAll("'", "''") + "'"; + } +} diff --git a/src/FaunerieImageType.ts b/src/FaunerieImageType.ts new file mode 100755 index 0000000..4e2a22b --- /dev/null +++ b/src/FaunerieImageType.ts @@ -0,0 +1,3 @@ +export enum FaunerieImageType { + ThumbnailURL, ViewURL, ThumbnailFile, ViewFile +} diff --git a/src/FaunerieListType.ts b/src/FaunerieListType.ts new file mode 100755 index 0000000..80f2bd0 --- /dev/null +++ b/src/FaunerieListType.ts @@ -0,0 +1,3 @@ +export enum FaunerieListType { + Array, Object +} diff --git a/src/FauneriePropertyStore.ts b/src/FauneriePropertyStore.ts new file mode 100755 index 0000000..5644b96 --- /dev/null +++ b/src/FauneriePropertyStore.ts @@ -0,0 +1,58 @@ +import {Faunerie} from "./Faunerie"; + +export class FauneriePropertyStore { + backend: Faunerie; + length: number; + + constructor(backend: Faunerie) { + this.backend = backend; + } + + async initialize() { + this.length = (await this.backend._sql("SELECT COUNT(*) FROM metadata"))[0]["COUNT(*)"]; + } + + private sqlstr(str?: string) { + if (str === null) { + return "NULL"; + } else { + return "'" + str.replaceAll("'", "''") + "'"; + } + } + + // noinspection JSUnusedGlobalSymbols + async setItem(key: string, value: string) { + if ((await this.backend._sql("SELECT COUNT(*) FROM metadata WHERE key = " + this.sqlstr(key)))[0]["COUNT(*)"] === 0) { + await this.backend._sql('INSERT INTO metadata(key, value) VALUES (' + this.sqlstr(key) + ', ' + this.sqlstr(value) + ')'); + this.length = (await this.backend._sql("SELECT COUNT(*) FROM metadata"))[0]["COUNT(*)"]; + } else { + await this.backend._sql('UPDATE metadata SET value = ' + this.sqlstr(value) + ' WHERE key = ' + this.sqlstr(key)); + } + } + + // noinspection JSUnusedGlobalSymbols + async removeItem(key: string) { + if ((await this.backend._sql("SELECT COUNT(*) FROM metadata WHERE key = " + this.sqlstr(key)))[0]["COUNT(*)"] === 0) { + return; + } else { + await this.backend._sql('DELETE FROM metadata WHERE key = ' + this.sqlstr(key)); + this.length = (await this.backend._sql("SELECT COUNT(*) FROM metadata"))[0]["COUNT(*)"]; + } + } + + // noinspection JSUnusedGlobalSymbols + async getItem(key: string) { + if ((await this.backend._sql("SELECT COUNT(*) FROM metadata WHERE key = " + this.sqlstr(key)))[0]["COUNT(*)"] === 0) { + return null; + } else { + return (await this.backend._sql('SELECT value FROM metadata WHERE key = ' + this.sqlstr(key)))[0]["value"]; + } + } + + // noinspection JSUnusedGlobalSymbols + async clear() { + await this.backend._sql("DROP TABLE IF EXISTS metadata"); + await this.backend._sql("CREATE TABLE metadata (key TEXT NOT NULL UNIQUE, value LONGTEXT NOT NULL, PRIMARY KEY (key))"); + this.length = (await this.backend._sql("SELECT COUNT(*) FROM metadata"))[0]["COUNT(*)"]; + } +} diff --git a/src/FaunerieSearch.ts b/src/FaunerieSearch.ts new file mode 100755 index 0000000..ebe3e37 --- /dev/null +++ b/src/FaunerieSearch.ts @@ -0,0 +1,678 @@ +import {SearchError} from "./SearchError"; +import {FaunerieFrontend} from "./FaunerieFrontend"; + +export interface IFaunerieSearchToken { + type: FaunerieSearchTokenType, + data?: string +} + +export enum FaunerieSearchTokenType { + Subquery, + Not, + And, + Or, + Query +} + +export class FaunerieSearch { + private readonly frontend: FaunerieFrontend; + + constructor(frontend: FaunerieFrontend) { + this.frontend = frontend; + } + + fillDate(str: string) { + return str.trim() + new Date(0).toISOString().substring(str.trim().length) + } + + checkQuery(query: string, allowUnknownTags: boolean) { + let frontend = this.frontend; + let sql = ""; + query = query.trim(); + + if (query === "") return ""; + + if (query.includes("~")) throw new SearchError("Unsupported '~' (fuzzy search) operator"); + if (query.includes("\\")) throw new SearchError("Unsupported '\\' (escape) operator"); + if (query.includes("^")) throw new SearchError("Unsupported '^' (boosting) operator"); + if (query.includes("\"")) throw new SearchError("Unsupported '\"' (quotation) operator"); + + let namespace = null; + let quantifier = null; + let value = query.split(":")[0]; + + if (query.includes(":")) { + namespace = query.split(":")[0].split(".")[0].toLowerCase(); + value = query.split(":")[1]; + + if (query.split(":")[0].includes(".")) { + quantifier = query.split(":")[0].split(".")[1].toLowerCase(); + } + } + + if (quantifier) { + if (!["lt", "lte", "gt", "gte"].includes(quantifier)) throw new SearchError("Unrecognized numeric qualifier '" + quantifier + "'"); + } + + let number: number; + + switch (namespace) { + case "created_at": + let date = new Date(this.fillDate(value)); + if (isNaN(date.getTime())) throw new SearchError("Invalid date/time value for 'created_at'"); + number = date.getTime(); + + if (isNaN(number) || !isFinite(number)) throw new Error("Number from getDate is NaN but shouldn't be"); + + switch (quantifier) { + case "lt": + sql = "images.created_at<" + number; + break; + case "lte": + sql = "images.created_at<=" + number; + break; + case "gt": + sql = "images.created_at>" + number; + break; + case "gte": + sql = "images.created_at>=" + number; + break; + default: + sql = "images.created_at=" + number; + break; + } + + break; + + case "my": + if (value === "hidden") throw new SearchError("Unsupported value for 'my': 'hidden'"); + if (value !== "upvotes" && value !== "downvotes" && value !== "faves" && value !== "uploads" && value !== "watched") throw new SearchError("Invalid value for 'my'"); + if (quantifier) throw new SearchError("'my' does not accept a numeric qualifier"); + + break; + + case "uploader": + throw new SearchError("Unsupported use of 'uploader'"); + + case "faved_by": + throw new SearchError("Unsupported use of 'faved_by'"); + + case "aspect_ratio": + number = parseFloat(value); + if (isNaN(number) || !isFinite(number)) throw new SearchError("Invalid numeric value for 'aspect_ratio'"); + + switch (quantifier) { + case "lt": + sql = "images.aspect_ratio<" + number; + break; + case "lte": + sql = "images.aspect_ratio<=" + number; + break; + case "gt": + sql = "images.aspect_ratio>" + number; + break; + case "gte": + sql = "images.aspect_ratio>=" + number; + break; + default: + sql = "images.aspect_ratio=" + number; + break; + } + + break; + + case "comment_count": + number = parseFloat(value); + if (isNaN(number) || !isFinite(number)) throw new SearchError("Invalid numeric value for 'comment_count'"); + + switch (quantifier) { + case "lt": + sql = "images.comment_count<" + number; + break; + case "lte": + sql = "images.comment_count<=" + number; + break; + case "gt": + sql = "images.comment_count>" + number; + break; + case "gte": + sql = "images.comment_count>=" + number; + break; + default: + sql = "images.comment_count=" + number; + break; + } + + break; + + case "downvotes": + number = parseFloat(value); + if (isNaN(number) || !isFinite(number)) throw new SearchError("Invalid numeric value for 'downvotes'"); + + switch (quantifier) { + case "lt": + sql = "images.downvotes<" + number; + break; + case "lte": + sql = "images.downvotes<=" + number; + break; + case "gt": + sql = "images.downvotes>" + number; + break; + case "gte": + sql = "images.downvotes>=" + number; + break; + default: + sql = "images.downvotes=" + number; + break; + } + + break; + + case "faves": + number = parseFloat(value); + if (isNaN(number) || !isFinite(number)) throw new SearchError("Invalid numeric value for 'faves'"); + + switch (quantifier) { + case "lt": + sql = "images.faves<" + number; + break; + case "lte": + sql = "images.faves<=" + number; + break; + case "gt": + sql = "images.faves>" + number; + break; + case "gte": + sql = "images.faves>=" + number; + break; + default: + sql = "images.faves=" + number; + break; + } + + break; + + case "height": + number = parseFloat(value); + if (isNaN(number) || !isFinite(number)) throw new SearchError("Invalid numeric value for 'height'"); + + switch (quantifier) { + case "lt": + sql = "images.height<" + number; + break; + case "lte": + sql = "images.height<=" + number; + break; + case "gt": + sql = "images.height>" + number; + break; + case "gte": + sql = "images.height>=" + number; + break; + default: + sql = "images.height=" + number; + break; + } + + break; + + case "id": + number = parseFloat(value); + if (isNaN(number) || !isFinite(number)) throw new SearchError("Invalid numeric value for 'id'"); + + switch (quantifier) { + case "lt": + sql = "images.source_id<" + number; + break; + case "lte": + sql = "images.source_id<=" + number; + break; + case "gt": + sql = "images.source_id>" + number; + break; + case "gte": + sql = "images.source_id>=" + number; + break; + default: + sql = "images.source_id=" + number; + break; + } + + break; + + case "score": + number = parseFloat(value); + if (isNaN(number) || !isFinite(number)) throw new SearchError("Invalid numeric value for 'score'"); + + switch (quantifier) { + case "lt": + sql = "images.score<" + number; + break; + case "lte": + sql = "images.score<=" + number; + break; + case "gt": + sql = "images.score>" + number; + break; + case "gte": + sql = "images.score>=" + number; + break; + default: + sql = "images.score=" + number; + break; + } + + break; + + case "tag_count": + number = parseFloat(value); + if (isNaN(number) || !isFinite(number)) throw new SearchError("Invalid numeric value for 'tag_count'"); + + switch (quantifier) { + case "lt": + sql = "images.tag_count<" + number; + break; + case "lte": + sql = "images.tag_count<=" + number; + break; + case "gt": + sql = "images.tag_count>" + number; + break; + case "gte": + sql = "images.tag_count>=" + number; + break; + default: + sql = "images.tag_count=" + number; + break; + } + + break; + + case "upvotes": + number = parseFloat(value); + if (isNaN(number) || !isFinite(number)) throw new SearchError("Invalid numeric value for 'upvotes'"); + + switch (quantifier) { + case "lt": + sql = "images.upvotes<" + number; + break; + case "lte": + sql = "images.upvotes<=" + number; + break; + case "gt": + sql = "images.upvotes>" + number; + break; + case "gte": + sql = "images.upvotes>=" + number; + break; + default: + sql = "images.upvotes=" + number; + break; + } + + break; + + case "width": + number = parseFloat(value); + if (isNaN(number) || !isFinite(number)) throw new SearchError("Invalid numeric value for 'width'"); + + switch (quantifier) { + case "lt": + sql = "images.width<" + number; + break; + case "lte": + sql = "images.width<=" + number; + break; + case "gt": + sql = "images.width>" + number; + break; + case "gte": + sql = "images.width>=" + number; + break; + default: + sql = "images.width=" + number; + break; + } + + break; + + case "wilson_score": + number = parseFloat(value); + if (isNaN(number) || !isFinite(number)) throw new SearchError("Invalid numeric value for 'wilson_score'"); + + switch (quantifier) { + case "lt": + sql = "images.wilson_score<" + number; + break; + case "lte": + sql = "images.wilson_score<=" + number; + break; + case "gt": + sql = "images.wilson_score>" + number; + break; + case "gte": + sql = "images.wilson_score>=" + number; + break; + default: + sql = "images.wilson_score=" + number; + break; + } + + break; + + case "sha512_hash": + if (quantifier) throw new SearchError("'sha512_hash' does not accept a numeric qualifier"); + sql = "images.sha512_hash LIKE '" + value.replaceAll("'", "''").replaceAll("%", "\\%").replaceAll("_", "\\_").replaceAll("*", "%") + "'"; + break; + + case "orig_sha512_hash": + if (quantifier) throw new SearchError("'orig_sha512_hash' does not accept a numeric qualifier"); + sql = "images.orig_sha512_hash LIKE '" + value.replaceAll("'", "''").replaceAll("%", "\\%").replaceAll("_", "\\_").replaceAll("*", "%") + "'"; + break; + + case "source_url": + if (quantifier) throw new SearchError("'source_url' does not accept a numeric qualifier"); + sql = "images.source_url LIKE '" + value.replaceAll("'", "''").replaceAll("%", "\\%").replaceAll("_", "\\_").replaceAll("*", "%") + "'"; + break; + + case "source": + if (quantifier) throw new SearchError("'source' does not accept a numeric qualifier"); + sql = "images.source_name LIKE '" + value.replaceAll("'", "''").replaceAll("%", "\\%").replaceAll("_", "\\_").replaceAll("*", "%") + "'"; + break; + + case "description": + if (quantifier) throw new SearchError("'description' does not accept a numeric qualifier"); + sql = "images.description LIKE '" + value.replaceAll("'", "''").replaceAll("%", "\\%").replaceAll("_", "\\_").replaceAll("*", "%") + "'"; + break; + + case "mime_type": + if (quantifier) throw new SearchError("'mime_type' does not accept a numeric qualifier"); + sql = "images.mime_type LIKE '" + value.replaceAll("'", "''").replaceAll("%", "\\%").replaceAll("_", "\\_").replaceAll("*", "%") + "'"; + break; + + default: + value = query.trim(); + let tags = frontend.tags; + let regex = new RegExp('^' + value.replace(/[.+^${}()|[\]\\]/g, '\\$&').replace(/\?/g, '.').replace(/\*/g, '.*') + '$'); + let matching = tags.filter(item => regex.test(item[1])); + + if (matching.length > 0) { + for (let tag of matching) { + sql += " OR image_tags.tags LIKE '%," + tag[0] + ",%'"; + } + + sql = "(" + sql.substring(4).trim() + ")"; + } else if (allowUnknownTags) { + sql = "image_tags.tags LIKE 'INVALID_TAG'"; + } else { + throw new SearchError("No tags matching '" + value.trim() + "' could be found"); + } + + break; + } + + return sql; + } + + buildQueryInner(query: string, allowUnknownTags: boolean = false) { + query = query.trim(); + let pos = 0; + + let inParentheses = false; + let currentParenthesesInner: string = ""; + let tokens: IFaunerieSearchToken[] = []; + let currentTag = ""; + let subParentheses = 0; + + while (pos < query.length) { + if (inParentheses) { + if (query[pos] === "(") subParentheses++; + if (query[pos] === ")") subParentheses--; + + if (subParentheses < 0) { + inParentheses = false; + tokens.push({ + type: FaunerieSearchTokenType.Subquery, + data: this.buildQueryInner(currentParenthesesInner, allowUnknownTags) + }); + currentParenthesesInner = null; + subParentheses = 0; + pos++; + continue; + } + + currentParenthesesInner += query[pos]; + pos++; + } else { + switch (query[pos]) { + case "!": + case "-": + if (currentTag.trim().length === 0) { + tokens.push({ + type: FaunerieSearchTokenType.Not, + data: null + }); + } else { + currentTag += query[pos]; + } + + pos++; + break; + + case ",": + if (currentTag.trim().length > 0) { + tokens.push({ + type: FaunerieSearchTokenType.Query, + data: this.checkQuery(currentTag.trim(), allowUnknownTags) + }); + currentTag = ""; + } + + tokens.push({ + type: FaunerieSearchTokenType.And, + data: null + }); + pos++; + break; + + case "&": + if (query[pos + 1] === "&") { + if (currentTag.trim().length > 0) { + tokens.push({ + type: FaunerieSearchTokenType.Query, + data: this.checkQuery(currentTag.trim(), allowUnknownTags) + }); + currentTag = ""; + } + + tokens.push({ + type: FaunerieSearchTokenType.And, + data: null + }); + pos += 2; + } else { + currentTag += query[pos]; + pos++; + } + + break; + + case "|": + if (query[pos + 1] === "|") { + if (currentTag.trim().length > 0) { + tokens.push({ + type: FaunerieSearchTokenType.Query, + data: this.checkQuery(currentTag.trim(), allowUnknownTags) + }); + currentTag = ""; + } + + tokens.push({ + type: FaunerieSearchTokenType.Or, + data: null + }); + pos += 2; + } else { + currentTag += query[pos]; + pos++; + } + + break; + + case "O": + if (query[pos + 1] === "R") { + if (currentTag.trim().length > 0) { + tokens.push({ + type: FaunerieSearchTokenType.Query, + data: this.checkQuery(currentTag.trim(), allowUnknownTags) + }); + currentTag = ""; + } + + tokens.push({ + type: FaunerieSearchTokenType.Or, + data: null + }); + pos += 2; + } else { + currentTag += query[pos]; + pos++; + } + + break; + + case "A": + if (query[pos + 1] === "N" && query[pos + 2] === "D") { + if (currentTag.trim().length > 0) { + tokens.push({ + type: FaunerieSearchTokenType.Query, + data: this.checkQuery(currentTag.trim(), allowUnknownTags) + }); + currentTag = ""; + } + + tokens.push({ + type: FaunerieSearchTokenType.And, + data: null + }); + pos += 3; + } else { + currentTag += query[pos]; + pos++; + } + + break; + + case "N": + if (query[pos + 1] === "O" && query[pos + 2] === "T") { + if (currentTag.trim().length > 0) { + tokens.push({ + type: FaunerieSearchTokenType.Query, + data: this.checkQuery(currentTag.trim(), allowUnknownTags) + }); + currentTag = ""; + } + + tokens.push({ + type: FaunerieSearchTokenType.Not, + data: null + }); + pos += 3; + } else { + currentTag += query[pos]; + pos++; + } + + break; + + case " ": + currentTag += query[pos]; + pos++; + break; + + case "(": + if (currentTag.trim().length === 0) { + inParentheses = true; + currentParenthesesInner = ""; + pos++; + } else { + currentTag += query[pos]; + pos++; + } + + break; + + case ")": + if (currentTag.trim().length === 0) { + throw new SearchError("Unexpected closing parenthesis."); + } else { + currentTag += query[pos]; + pos++; + } + + break; + + default: + currentTag += query[pos]; + pos++; + break; + } + } + } + + if (currentTag.trim().length > 0) { + tokens.push({ + type: FaunerieSearchTokenType.Query, + data: this.checkQuery(currentTag.trim(), allowUnknownTags) + }); + } + + return this.queryTokensToString(tokens); + } + + queryTokensToString(tokens: IFaunerieSearchToken[]) { + let str = ""; + + for (let token of tokens) { + switch (token.type) { + case FaunerieSearchTokenType.And: + str += "AND"; + break; + + case FaunerieSearchTokenType.Query: + str += token.data; + break; + + case FaunerieSearchTokenType.Subquery: + str += "(" + token.data + ")"; + break; + + case FaunerieSearchTokenType.Not: + str += "NOT"; + break; + + case FaunerieSearchTokenType.Or: + str += "OR"; + break; + } + + str += " "; + } + + return str.trim(); + } + + buildQueryV2(query: string, allowUnknownTags: boolean, allowExceedLength: boolean = false) { + if (query.length >= 1024 && !allowExceedLength) throw new SearchError("A search query needs to be shorter than 1024 characters."); + query = query.trim(); + + query = this.buildQueryInner(query, allowUnknownTags); + + if (query.length > 0) { + return "(" + query.trim() + ")"; + } else { + return "TRUE"; + } + } +} diff --git a/src/FaunerieUpdater.ts b/src/FaunerieUpdater.ts new file mode 100755 index 0000000..a2007d6 --- /dev/null +++ b/src/FaunerieUpdater.ts @@ -0,0 +1,522 @@ +import {Faunerie} from "./Faunerie"; +import fs from "fs"; +import http from "http"; +import {IFaunerieUpdaterGeneric} from "./IFaunerieUpdaterGeneric"; +import {IncomingMessage, ServerResponse} from "node:http"; + +export class FaunerieUpdater { + private readonly database: Faunerie; + + constructor(instance: Faunerie) { + this.database = instance; + } + + // noinspection JSUnusedGlobalSymbols + updateFromPreprocessed(preprocessed: string, tags: string, statusUpdateHandler: Function) { + return new Promise((res) => { + let sqlGet: Function; + let sqlQuery: Function; + let protectedEncode: Function; + let database = this.database; + + const zlib = require('zlib'); + + sqlGet = sqlQuery = async (query: string) => { + return await database._sql(query); + } + + protectedEncode = (b: string) => { + return zlib.deflateRawSync(b, {level: 9}); + } + + function sqlstr(str?: string) { + if (str === null) { + return "NULL"; + } else { + return "'" + str.replaceAll("'", "''") + "'"; + } + } + + function sleep(ms: number) { + return new Promise((res) => { + setTimeout(res, ms); + }); + } + + async function addToDB(images: object[]) { + await sqlQuery(`PRAGMA foreign_keys = OFF`); + + let tags = {}; + + for (let image of images) { + let imageTags = image['tags'].map((i: number, j: object) => { + return [image['tag_ids'][j], i]; + }); + + for (let tag of imageTags) { + tags[tag[0]] = tag[1]; + } + } + + let uploaders = {}; + + for (let image of images) { + if (image['uploader_id']) { + uploaders[image['uploader_id']] = image['uploader']; + } + } + + let index = 0; + let u = Object.entries(uploaders); + + for (let i = 0; i < u.length; i += 50) { + const chunk = u.slice(i, i + 50); + + for (let uploader of chunk) { + if ((await sqlGet(`SELECT COUNT(*) FROM uploaders WHERE id=${uploader[0]}`))[0]["COUNT(*)"]) { + await sqlQuery(`DELETE FROM uploaders WHERE id=${uploader[0]}`); + } + } + + // @ts-ignore + await sqlQuery(`INSERT INTO uploaders(id, name) VALUES ${chunk.map(uploader => `(${uploader[0]}, '${(uploader[1] ?? "").replaceAll("'", "''")}')`).join(",")}`); + + index += 50; + } + + index = 0; + + let v = images; + + for (let i = 0; i < v.length; i += 50) { + const chunk = v.slice(i, i + 50); + + for (let image of chunk) { + if ((await sqlGet(`SELECT COUNT(*) FROM images WHERE id=${image['id']}`))[0]["COUNT(*)"]) { + await sqlQuery(`DELETE FROM images WHERE id=${image['id']}`); + await sqlQuery(`DELETE FROM image_tags WHERE image_id=${image['id']}`); + await sqlQuery(`DELETE FROM image_intensities WHERE image_id=${image['id']}`); + await sqlQuery(`DELETE FROM image_representations WHERE image_id=${image['id']}`); + } + } + + await sqlQuery(`INSERT INTO images(id, source_id, source_name, source, animated, hidden_from_users, processed, spoilered, thumbnails_generated, aspect_ratio, duration, wilson_score, created_at, first_seen_at, updated_at, comment_count, downvotes, duplicate_of, faves, height, score, size, tag_count, upvotes, width, deletion_reason, description, format, mime_type, name, orig_sha512_hash, sha512_hash, source_url, uploader) VALUES ${chunk.map(image => `(${image['id']}, ${image['source_id'] ?? image['id']}, ${sqlstr(image['source_name'] ?? null)}, ${sqlstr(image['source'] ?? null)}, ${image['animated'] ? 'TRUE' : 'FALSE'}, ${image['hidden_from_users'] ? 'TRUE' : 'FALSE'}, ${image['processed'] ? 'TRUE' : 'FALSE'}, ${image['spoilered'] ? 'TRUE' : 'FALSE'}, ${image['thumbnails_generated'] ? 'TRUE' : 'FALSE'}, ${image['aspect_ratio']}, ${image['duration']}, ${image['wilson_score']}, ${new Date(image['created_at']).getTime() / 1000}, ${new Date(image['first_seen_at']).getTime() / 1000}, ${new Date(image['updated_at']).getTime() / 1000}, ${image['comment_count']}, ${image['downvotes']}, ${image['duplicate_of']}, ${image['faves']}, ${image['height']}, ${image['score']}, ${image['size']}, ${image['tag_count']}, ${image['upvotes']}, ${image['width']}, ${sqlstr(image['deletion_reason'])}, ${sqlstr(image['description'])}, ${sqlstr(image['format'])}, ${sqlstr(image['mime_type'])}, ${sqlstr(image['name'])}, ${sqlstr(image['orig_sha512_hash'])}, ${sqlstr(image['sha512_hash'])}, ${sqlstr(image['source_url'])}, ${image['uploader_id']})`).join(",")}`); + await sqlQuery(`INSERT INTO image_tags(image_id, tags) VALUES ${chunk.map(image => `(${image['id']}, ",${image['tag_ids'].join(",")},")`).join(",")}`); + await sqlQuery(`INSERT INTO image_intensities(image_id, ne, nw, se, sw) VALUES ${chunk.map(image => `(${image['id']}, ${image['intensities']?.ne ?? null}, ${image['intensities']?.nw ?? null}, ${image['intensities']?.se ?? null}, ${image['intensities']?.sw ?? null})`).join(",")}`); + await sqlQuery(`INSERT INTO image_representations(image_id, view, full, large, medium, small, tall, thumb, thumb_small, thumb_tiny) VALUES ${chunk.map(image => `(${image['id']}, ${sqlstr(image['view_url'])}, ${sqlstr(image['representations'].full)}, ${sqlstr(image['representations'].large)}, ${sqlstr(image['representations'].medium)}, ${sqlstr(image['representations'].small)}, ${sqlstr(image['representations'].tall)}, ${sqlstr(image['representations'].thumb)}, ${sqlstr(image['representations'].thumb_small)}, ${sqlstr(image['representations'].thumb_tiny)})`).join(",")}`); + + index += 50; + } + + await sqlQuery(`PRAGMA foreign_keys = ON`); + } + + async function cleanDB() { + await database.clean(); + } + + function timeToString(time: number | string | Date) { + if (!isNaN(parseInt(time as string))) { + time = new Date(time).getTime(); + } + + let periods = ["second", "minute", "hour", "day", "week", "month", "year", "age"]; + + let lengths = [60, 60, 24, 7, 4.35, 12, 100]; + + let difference = (time as number) / 1000; + let period: string; + + let j: number; + + for (j = 0; difference >= lengths[j] && j < lengths.length - 1; j++) { + difference /= lengths[j]; + } + + difference = Math.round(difference); + + period = periods[j]; + + return `${difference} ${period}${difference > 1 ? "s" : ""}`; + } + + if (typeof window !== "undefined") { + global = window; + } + + async function consolidateDB() { + await sqlQuery("DROP TABLE IF EXISTS tags_pre"); + + statusUpdateHandler([{ + title: "Saving database...", progress: 0, indeterminate: false + }]); + + if (database.cache) { + await fs.promises.copyFile(database.cache + "/work.pbdb", database.path + "/current.pbdb"); + } + + fs.unlinkSync(global.oldDBFile ?? null); + res(); + } + + let update = async () => { + const fs = require('fs'); + const path = require('path'); + + process.chdir(this.database.path); + + statusUpdateHandler([{ + title: "Backup up database...", progress: 0, indeterminate: false + }]); + + global.oldDBFile = this.database.path + "/" + new Date().getTime() + ".pbdb"; + fs.copyFileSync(this.database.path + "/current.pbdb", global.oldDBFile); + + statusUpdateHandler([{ + title: "Cleaning up database...", progress: 0, indeterminate: false + }]); + await cleanDB(); + + statusUpdateHandler([{ + title: "Preparing update...", progress: 0, indeterminate: false + }]); + + let sqlPreGet = (sql: string) => { + return new Promise((res, rej) => { + global.preDatabase.all(sql, function (err: Error | null, data: IFaunerieUpdaterGeneric) { + if (err) { + rej(err); + } else { + res(data); + } + }); + }); + } + + let sqlTagGet = (sql: string) => { + return new Promise((res, rej) => { + global.tagsDatabase.all(sql, function (err: Error | null, data: IFaunerieUpdaterGeneric) { + if (err) { + rej(err); + } else { + res(data); + } + }); + }); + } + + const sqlite3 = require(process.platform === "darwin" ? "../../../sql/mac" : "../../../sql/win"); + + global.preDatabase = new sqlite3.Database(preprocessed); + global.tagsDatabase = new sqlite3.Database(tags); + + await new Promise((res) => { + global.preDatabase.serialize(() => { + res(); + }); + }); + + const host = "127.0.0.1"; + const port = 19842; + const preprocessedImageCount = parseInt((await sqlPreGet("SELECT COUNT(*) FROM images"))[0]["COUNT(*)"]); + + let preprocessedServer = http.createServer(async (req: IncomingMessage, res: ServerResponse) => { + let page = 1; + let requestPage = new URL(req.url, "https://nothing.invalid").searchParams.get('page'); + let out = { + images: [], interactions: [], total: preprocessedImageCount + } + + if (requestPage && !isNaN(parseInt(requestPage))) { + page = parseInt(requestPage); + } + + let everything = (await sqlPreGet("SELECT * FROM images LIMIT 50 OFFSET " + ((page - 1) * 50))) as IFaunerieUpdaterGeneric[]; + out.images = everything.map(i => JSON.parse(atob(i['json']))); + + res.writeHead(200); + res.end(JSON.stringify(out)); + }); + + preprocessedServer.listen(port, host, () => { + // noinspection HttpUrlsUsage + console.log(`Server is running on http://${host}:${port}`); + }); + + let updateTags = ((await sqlTagGet("SELECT * FROM tags")) as IFaunerieUpdaterGeneric[]).map(i => JSON.parse(atob(i['json']))); + let index = 0; + + for (let i = 0; i < updateTags.length; i += 50) { + const chunk = updateTags.slice(i, i + 50); + + let aliases = []; + let implications = []; + + for (let tag of chunk) { + // noinspection ES6MissingAwait + aliases.push(sqlstr(await sqlTagGet(`SELECT target FROM aliases WHERE source = ` + tag['id'])[0]?.target ?? null)); + implications.push(sqlstr("," + (await sqlTagGet(`SELECT target FROM implications WHERE source = ` + tag['id']) as IFaunerieUpdaterGeneric[]).map(i => i['target']).join(",") + ",")); + } + + await sqlQuery(`INSERT INTO tags(id, name, alias, implications, category, description, description_short, slug) VALUES ${chunk.map((tag, index) => `(${tag['id']}, ${sqlstr(tag['name'])}, ${aliases[index]}, ${implications[index]}, ${sqlstr(tag['category'])}, ${sqlstr(tag['description'])}, ${sqlstr(tag['short_description'])}, ${sqlstr(tag['slug'])})`).join(",")}`); + index += 50; + + statusUpdateHandler([{ + title: "Preparing update... " + Math.round((index / updateTags.length) * 100) + "%", + progress: ((index / updateTags.length) * 100), + indeterminate: false + }]); + } + + statusUpdateHandler([{ + title: "Preparing update...", progress: 0, indeterminate: false + }]); + + global.statusInfo = []; + + function prettySize(s: number) { + if (s < 1024) { + return s.toFixed(0) + " B"; + } else if (s < 1024 ** 2) { + return (s / 1024).toFixed(0) + " KiB"; + } else if (s < 1024 ** 3) { + return (s / 1024 ** 2).toFixed(1) + " MiB"; + } else if (s < 1024 ** 4) { + return (s / 1024 ** 3).toFixed(2) + " GiB"; + } else { + return (s / 1024 ** 4).toFixed(2) + " TiB"; + } + } + + function doImageFileDownload(url: string, image: object) { + global.statusInfo[2] = { + title: "Image: " + image['id'], progress: 0, indeterminate: false + } + + statusUpdateHandler(global.statusInfo); + + const axios = require('axios'); + + return new Promise(async (res) => { + // noinspection JSUnusedGlobalSymbols + const response = await axios({ + url, + validateStatus: (s: number) => (s >= 200 && s < 300) || (s === 404 || s === 403 || s === 401), + method: 'GET', + responseType: 'arraybuffer', + onDownloadProgress: (event: IFaunerieUpdaterGeneric) => { + global.statusInfo[2] = { + title: "Image: " + image['id'] + " (" + Math.round((event['loaded'] / event['total']) * 100) + "%)", + progress: ((event['loaded'] / event['total']) * 100), + indeterminate: false + } + + statusUpdateHandler(global.statusInfo); + } + } as object); + + global.statusInfo[2] = null; + res(response.data); + }); + } + + global.doneFetching = false; + + if (!fs.existsSync("./thumbnails")) fs.mkdirSync("./thumbnails"); + if (!fs.existsSync("./images")) fs.mkdirSync("./images"); + + let total1: number; + let pages1: number; + let types = []; + + async function doRequest() { + try { + return (await (await fetch("http://127.0.0.1:19842")).json())['total']; + } catch (e) { + console.error(e); + return doRequest(); + } + } + + total1 = await doRequest(); + pages1 = Math.ceil(total1 / 50); + + types.push({ + query: "my:faves", name: "faved", pages: pages1, total: total1 + }); + + let prelists = {}; + global.totalPrelist = []; + global.totalPrelistFull = []; + global.times = []; + global.times2 = []; + global.totalPages = types.map(i => i['pages']).reduce((a, b) => a + b); + global.totalImages = types.map(i => i['total']).reduce((a, b) => a + b); + global.totalPageNumber = 0; + global.images = 0; + global.updateCategories = {}; + + console.log(global.totalImages + " images to download from sources, part of " + global.totalPages + " pages"); + + for (let type of types) { + let prelist = []; + let pages = type.pages; + + for (let pageNumber = 1; pageNumber <= pages; pageNumber++) { + if (global.doneFetching || !preprocessedServer) break; + + let start = new Date().getTime(); + let tryFetch = true; + let page: object = {}; + + while (tryFetch) { + try { + page = await (await fetch("http://127.0.0.1:19842/?page=" + pageNumber)).json(); + + page['images'] = page['images'].map((image: object) => { + if (image['representations']['thumb'].endsWith(".mp4") || image['representations']['thumb'].endsWith(".webm")) { + image['representations']['thumb'] = image['representations']['thumb'].substring(0, image['representations']['thumb'].length - path.extname(image['representations']['thumb']).length) + ".gif"; + } + + return image; + }); + + tryFetch = false; + } catch (e) { + console.error(e); + } + } + + await addToDB(page['images']); + + for (let image of page['images']) { + if (!global.updateCategories[image.id]) global.updateCategories[image.id] = { + upvotes: false, downvotes: false, watched: false, faved: false, uploads: false + } + + global.updateCategories[image.id][type.name] = true; + } + + prelist.push(...page['images']); + global.totalPrelist.push(...page['images']); + global.totalPrelistFull.push(...page['images']); + global.times.push(new Date().getTime() - start); + + let fetchEta = (global.totalPages - global.totalPageNumber) * (global.times.reduce((a: number, b: number) => a + b, 0) / global.times.length); + + if (global.totalPageNumber === global.totalPages) { + global.doneFetching = true; + } + + global.statusInfo[0] = { + title: "Fetching: " + Math.round((global.totalPageNumber / global.totalPages) * 100) + "% (" + global.totalPageNumber + "/" + global.totalPages + ") complete" + (global.times.length > 10 ? ", " + timeToString(fetchEta) : "") + " (" + global.totalPrelistFull.length + ")", + progress: ((global.totalPageNumber / global.totalPages) * 100), + indeterminate: false + } + + statusUpdateHandler(global.statusInfo); + global.totalPageNumber++; + } + + prelists[type.name] = prelist; + } + + global.doneFetching = true; + global.totalFetchingSize = global.totalPrelistFull.map((i: object) => i['size']).reduce((a: number, b: number) => a + b); + + try { + preprocessedServer.closeAllConnections(); + preprocessedServer.close(); + preprocessedServer = null; + } catch (e) { + console.error(e); + } + + while (global.totalPrelist.length > 0) { + let image = global.totalPrelist.shift(); + global.lastImage = image; + let start = new Date().getTime(); + let downloaded = false; + + let path1 = (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1); + let path2 = (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2); + let path3 = (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 3); + + if (!fs.existsSync("./images/" + path1)) fs.mkdirSync("./images/" + path1); + if (!fs.existsSync("./images/" + path1 + "/" + path2)) fs.mkdirSync("./images/" + path1 + "/" + path2); + if (!fs.existsSync("./images/" + path1 + "/" + path2 + "/" + path3)) fs.mkdirSync("./images/" + path1 + "/" + path2 + "/" + path3); + if (!fs.existsSync("./thumbnails/" + path1)) fs.mkdirSync("./thumbnails/" + path1); + if (!fs.existsSync("./thumbnails/" + path1 + "/" + path2)) fs.mkdirSync("./thumbnails/" + path1 + "/" + path2); + if (!fs.existsSync("./thumbnails/" + path1 + "/" + path2 + "/" + path3)) fs.mkdirSync("./thumbnails/" + path1 + "/" + path2 + "/" + path3); + + let fileName = path1 + "/" + path2 + "/" + path3 + "/" + image['id'] + ".bin"; + let fileName2 = path1 + "/" + path2 + "/" + path3 + "/" + image['id'] + ".bin"; + + let fileNamePre = path1 + "/" + path2 + "/" + path3 + "/." + image['id'] + ".bin"; + let fileNamePre2 = path1 + "/" + path2 + "/" + path3 + "/." + image['id'] + ".bin"; + + if (!fs.existsSync("./images/" + fileName) || !fs.existsSync("./thumbnails/" + fileName2)) { + if (fs.existsSync("./images/" + path1 + "/" + path2 + "/" + path3 + "/." + image['id'] + path.extname(image['view_url']))) fs.unlinkSync("./images/" + path1 + "/" + path2 + "/" + path3 + "/." + image['id'] + path.extname(image['view_url'])); + if (fs.existsSync("./thumbnails/" + path1 + "/" + path2 + "/" + path3 + "/." + image['id'] + path.extname(image['representations']['thumb']))) fs.unlinkSync("./thumbnails/" + path1 + "/" + path2 + "/" + path3 + "/." + image['id'] + path.extname(image['representations']['thumb'])); + if (fs.existsSync("./images/" + path1 + "/" + path2 + "/" + path3 + "/." + image['id'] + ".bin")) fs.unlinkSync("./images/" + path1 + "/" + path2 + "/" + path3 + "/." + image['id'] + ".bin"); + if (fs.existsSync("./thumbnails/" + path1 + "/" + path2 + "/" + path3 + "/." + image['id'] + ".bin")) fs.unlinkSync("./thumbnails/" + path1 + "/" + path2 + "/" + path3 + "/." + image['id'] + ".bin"); + if (fs.existsSync("./images/" + path1 + "/" + path2 + "/." + image['id'] + path.extname(image['view_url']))) fs.unlinkSync("./images/" + path1 + "/" + path2 + "/" + path3 + "/." + image['id'] + path.extname(image['view_url'])); + if (fs.existsSync("./thumbnails/" + path1 + "/" + path2 + "/." + image['id'] + path.extname(image['representations']['thumb']))) fs.unlinkSync("./thumbnails/" + path1 + "/" + path2 + "/." + image['id'] + path.extname(image['representations']['thumb'])); + if (fs.existsSync("./images/" + path1 + "/" + path2 + "/." + image['id'] + ".bin")) fs.unlinkSync("./images/" + path1 + "/" + path2 + "/." + image['id'] + ".bin"); + if (fs.existsSync("./thumbnails/" + path1 + "/" + path2 + "/." + image['id'] + ".bin")) fs.unlinkSync("./thumbnails/" + path1 + "/" + path2 + "/." + image['id'] + ".bin"); + + let tryFetch = true; + + while (tryFetch) { + try { + fs.writeFileSync("./images/" + fileNamePre, protectedEncode(await doImageFileDownload(image['view_url'], image))); + tryFetch = false; + } catch (e) { + console.error(e); + await sleep(1500); + } + } + + tryFetch = true; + + while (tryFetch) { + try { + fs.writeFileSync("./thumbnails/" + fileNamePre2, protectedEncode(await doImageFileDownload(image['representations']['thumb'], image))); + tryFetch = false; + } catch (e) { + console.error(e); + await sleep(1500); + } + } + + fs.renameSync("./thumbnails/" + fileNamePre2, "./thumbnails/" + fileName2); + fs.renameSync("./images/" + fileNamePre, "./images/" + fileName); + downloaded = true; + } + + if (downloaded) global.times2.push([new Date().getTime() - start, image['size']]); + global.totalPrelist.shift(); + global.images++; + + let eta = (global.totalImages - global.times2.length) * (global.times2.map((i: number[]) => i[0]).reduce((a: number, b: number) => a + b, 0) / global.times2.length); + let times = global.times2.map((i: number[]) => i[0] / i[1]).slice(0, 20); + let averageBps = times.length > 0 ? (times.reduce((a: number, b: number) => a + b) / times.length) : 0; + let dataLeft = global.totalFetchingSize - global.times2.map((i: number[]) => i[1]).reduce((a: number, b: number) => a + b, 0); + eta = dataLeft * averageBps; + + let title = "Downloading: " + Math.round((global.images / global.totalImages) * 100) + "% (" + global.images + "/" + global.totalImages + ", " + prettySize(global.times2.map((i: number[]) => i[1]).reduce((a: number, b: number) => a + b, 0)) + (global.totalFetchingSize ? "/" + prettySize(global.totalFetchingSize) : "") + ") complete" + (global.times2.length > 10 && eta > 1000 ? ", " + timeToString(eta) : ""); + + global.statusInfo[0] = { + title, progress: ((global.images / global.totalImages) * 100), indeterminate: false + } + + statusUpdateHandler(global.statusInfo); + } + + await consolidateDB(); + } + + update().then(_ => {}); + }); + } +} diff --git a/src/IFaunerieImage.ts b/src/IFaunerieImage.ts new file mode 100755 index 0000000..bb2cc37 --- /dev/null +++ b/src/IFaunerieImage.ts @@ -0,0 +1 @@ +export interface IFaunerieImage {} diff --git a/src/IFaunerieOption.ts b/src/IFaunerieOption.ts new file mode 100755 index 0000000..5397bb0 --- /dev/null +++ b/src/IFaunerieOption.ts @@ -0,0 +1,8 @@ +export interface IFaunerieOption { + database: string, + sqlitePath?: string, + cachePath?: string, + readOnly: boolean, + sensitiveImageProtocol?: boolean, + verbose?: boolean +} diff --git a/src/IFaunerieTag.ts b/src/IFaunerieTag.ts new file mode 100755 index 0000000..ef43c18 --- /dev/null +++ b/src/IFaunerieTag.ts @@ -0,0 +1 @@ +export type IFaunerieTag = [number, string]; diff --git a/src/IFaunerieUpdaterGeneric.ts b/src/IFaunerieUpdaterGeneric.ts new file mode 100755 index 0000000..5de77d6 --- /dev/null +++ b/src/IFaunerieUpdaterGeneric.ts @@ -0,0 +1 @@ +export interface IFaunerieUpdaterGeneric {} diff --git a/src/IPrisbeamImage.ts b/src/IPrisbeamImage.ts deleted file mode 100755 index 763bc8f..0000000 --- a/src/IPrisbeamImage.ts +++ /dev/null @@ -1 +0,0 @@ -export interface IPrisbeamImage {} diff --git a/src/IPrisbeamOption.ts b/src/IPrisbeamOption.ts deleted file mode 100755 index 75a467f..0000000 --- a/src/IPrisbeamOption.ts +++ /dev/null @@ -1,8 +0,0 @@ -export interface IPrisbeamOption { - database: string, - sqlitePath?: string, - cachePath?: string, - readOnly: boolean, - sensitiveImageProtocol?: boolean, - verbose?: boolean -} diff --git a/src/IPrisbeamTag.ts b/src/IPrisbeamTag.ts deleted file mode 100755 index df83443..0000000 --- a/src/IPrisbeamTag.ts +++ /dev/null @@ -1 +0,0 @@ -export type IPrisbeamTag = [number, string]; diff --git a/src/IPrisbeamUpdaterGeneric.ts b/src/IPrisbeamUpdaterGeneric.ts deleted file mode 100755 index 5be465b..0000000 --- a/src/IPrisbeamUpdaterGeneric.ts +++ /dev/null @@ -1 +0,0 @@ -export interface IPrisbeamUpdaterGeneric {} diff --git a/src/Prisbeam.ts b/src/Prisbeam.ts deleted file mode 100755 index e20d909..0000000 --- a/src/Prisbeam.ts +++ /dev/null @@ -1,152 +0,0 @@ -import {PrisbeamFrontend} from "./PrisbeamFrontend"; -import {IPrisbeamOption} from "./IPrisbeamOption"; -import fs from "fs"; -import path from "path"; -import {VERSION} from "../index"; -import {PrisbeamPropertyStore} from "./PrisbeamPropertyStore"; -import {Database} from "sqlite3"; -import {SQLiteInstance} from "./SQLiteInstance"; - -export class Prisbeam { - // noinspection JSUnusedGlobalSymbols - public version: string = VERSION; - public readonly verbose: boolean; - public readonly path: string; - readonly sensitiveImageProtocol: boolean; - public frontend: PrisbeamFrontend; - readonly cache?: string; - private sqlite: SQLiteInstance; - private database: Database; - private readonly readOnly: boolean; - public propertyStore: PrisbeamPropertyStore; - - constructor(options: IPrisbeamOption) { - this.verbose = options.verbose ?? true; - this.sqlite = require(options.sqlitePath ?? "sqlite3").verbose(); - - if (!fs.existsSync(path.resolve(options.database))) throw new Error("Invalid database folder specified: " + path.resolve(options.database)); - if (!fs.existsSync(path.resolve(options.database) + "/instance.pbmk")) throw new Error("Not a valid Prisbeam database: " + path.resolve(options.database)); - - this.path = path.resolve(options.database); - - if (options.cachePath) { - if (!fs.existsSync(path.resolve(options.cachePath))) throw new Error("Invalid cache folder specified: " + path.resolve(options.cachePath)); - this.cache = path.resolve(options.cachePath); - } - - this.readOnly = options.readOnly; - this.sensitiveImageProtocol = options.sensitiveImageProtocol ?? false; - } - - async clean() { - if (this.readOnly) throw new Error("The database is open is read-only mode."); - await this._sql("DROP TABLE IF EXISTS image_tags"); - await this._sql("DROP TABLE IF EXISTS image_intensities"); - await this._sql("DROP TABLE IF EXISTS image_representations"); - await this._sql("DROP TABLE IF EXISTS image_categories"); - await this._sql("DROP TABLE IF EXISTS images"); - await this._sql("DROP TABLE IF EXISTS uploaders"); - await this._sql("DROP TABLE IF EXISTS tags"); - await this._sql("DROP TABLE IF EXISTS tags_pre"); - await this._sql("DROP TABLE IF EXISTS compressed"); - await this._sql("CREATE TABLE images (id INT NOT NULL UNIQUE, source_id INT NOT NULL UNIQUE, source_name TEXT, source TEXT NOT NULL, animated BOOL, aspect_ratio FLOAT, comment_count INT, created_at TIMESTAMP, deletion_reason LONGTEXT, description LONGTEXT, downvotes INT, duplicate_of INT, duration FLOAT, faves INT, first_seen_at TIMESTAMP, format TEXT, height INT, hidden_from_users BOOL, mime_type TEXT, name LONGTEXT, orig_sha512_hash TEXT, processed BOOL, score INT, sha512_hash TEXT, size INT, source_url LONGTEXT, spoilered BOOL, tag_count INT, thumbnails_generated BOOL, updated_at TIMESTAMP, uploader INT, upvotes INT, width INT, wilson_score FLOAT, PRIMARY KEY (id), FOREIGN KEY (uploader) REFERENCES uploaders(id))"); - await this._sql("CREATE TABLE image_tags (image_id INT NOT NULL UNIQUE, tags LONGTEXT NOT NULL, PRIMARY KEY (image_id), FOREIGN KEY (image_id) REFERENCES images(id))"); - await this._sql("CREATE TABLE image_intensities (image_id INT NOT NULL UNIQUE, ne FLOAT NOT NULL, nw FLOAT NOT NULL, se FLOAT NOT NULL, sw FLOAT NOT NULL, PRIMARY KEY (image_id), FOREIGN KEY (image_id) REFERENCES images(id))"); - await this._sql("CREATE TABLE image_representations (image_id INT NOT NULL UNIQUE, view LONGTEXT NOT NULL, full TEXT, large TEXT, medium TEXT, small TEXT, tall TEXT, thumb TEXT, thumb_small TEXT, thumb_tiny TEXT, PRIMARY KEY (image_id), FOREIGN KEY (image_id) REFERENCES images(id))"); - await this._sql("CREATE TABLE tags (id INT NOT NULL UNIQUE, name TEXT NOT NULL UNIQUE, alias INT, implications LONGTEXT, category TEXT, description LONGTEXT, description_short LONGTEXT, slug TEXT UNIQUE, PRIMARY KEY (id))"); - await this._sql("CREATE TABLE uploaders (id INT NOT NULL UNIQUE, name TEXT, PRIMARY KEY (id))"); - await this._sql("CREATE TABLE IF NOT EXISTS metadata (key TEXT NOT NULL UNIQUE, value LONGTEXT NOT NULL, PRIMARY KEY (key))"); - } - - _sql(query: string) { - let verbose = this.verbose; - if (verbose) console.debug("=>", query); - - return new Promise((res, rej) => { - this.database.all(query, function (err: Error | null, data?: any) { - if (err) { - if (verbose) console.debug("<=", data); - rej(err); - } else { - if (verbose) console.debug("<=", data); - res(data); - } - }); - }); - } - - // noinspection JSUnusedGlobalSymbols - async initialize(restoreBackup: boolean) { - if (restoreBackup) { - let backups = (await fs.promises.readdir(this.path)).filter(i => i.endsWith(".pbdb") && i !== "current.pbdb"); - - if (backups.length > 0 && !isNaN(parseInt(backups[0].split(".")[0]))) { - await fs.promises.copyFile(this.path + "/" + backups[0], this.path + "/current.pbdb"); - await fs.promises.unlink(this.path + "/" + backups[0]); - } - } - - if (this.cache) { - await fs.promises.copyFile(this.path + "/current.pbdb", this.cache + "/work.pbdb"); - - if (this.readOnly) { - this.database = new this.sqlite.Database(this.cache + "/work.pbdb", this.sqlite.OPEN_READONLY); - } else { - this.database = new this.sqlite.Database(this.cache + "/work.pbdb"); - } - } else { - if (this.readOnly) { - this.database = new this.sqlite.Database(this.path + "/current.pbdb", this.sqlite.OPEN_READONLY); - } else { - this.database = new this.sqlite.Database(this.path + "/current.pbdb"); - } - } - - await new Promise((res) => { - this.database.serialize(() => { - res(); - }); - }); - - if (!this.readOnly) { - if ((await this._sql("SELECT COUNT(*) FROM metadata WHERE key='libprisbeam_timestamp'"))[0]["COUNT(*)"] === 0) { - await this._sql('INSERT INTO metadata(key, value) VALUES ("libprisbeam_timestamp", "' + new Date().toISOString() + '")'); - } else { - await this._sql('UPDATE metadata SET value="' + new Date().toISOString() + '" WHERE key="libprisbeam_timestamp"'); - } - } - - await this._sql("CREATE TABLE IF NOT EXISTS metadata (key TEXT NOT NULL UNIQUE, value LONGTEXT NOT NULL, PRIMARY KEY (key))"); - - this.frontend = new PrisbeamFrontend(this); - this.propertyStore = new PrisbeamPropertyStore(this); - await this.propertyStore.initialize(); - - await this.frontend.initialize(); - await this.defragment(); - } - - async defragment() { - await this._sql("VACUUM"); - } - - // noinspection JSUnusedGlobalSymbols - async close() { - await new Promise((res) => { - // @ts-ignore - this.database.wait(() => { - res(); - }); - }); - - await new Promise((res) => { - this.database.close(() => { - res(); - }); - }); - - if (this.cache) { - await fs.promises.copyFile(this.cache + "/work.pbdb", this.path + "/current.pbdb"); - } - } -} diff --git a/src/PrisbeamFrontend.ts b/src/PrisbeamFrontend.ts deleted file mode 100755 index 47506a2..0000000 --- a/src/PrisbeamFrontend.ts +++ /dev/null @@ -1,404 +0,0 @@ -import {PrisbeamSearch} from "./PrisbeamSearch"; -import {PrisbeamImageType} from "./PrisbeamImageType"; -import fs from "fs"; -import zlib from "zlib"; -import {PrisbeamListType} from "./PrisbeamListType"; -import {Prisbeam} from "../index"; -import {SearchError} from "./SearchError"; -import {IPrisbeamImage} from "./IPrisbeamImage"; -import {IPrisbeamTag} from "./IPrisbeamTag"; - -export class PrisbeamFrontend { - public tags: IPrisbeamTag[]; - public tagsHashed: object; - private readonly backend: Prisbeam; - readonly searchEngine: PrisbeamSearch; - private readonly sensitiveImageProtocol: boolean; - - constructor(backend: Prisbeam) { - this.backend = backend; - this.sensitiveImageProtocol = backend.sensitiveImageProtocol; - this.searchEngine = new PrisbeamSearch(this); - } - - async initialize() { - this.tags = []; - this.tagsHashed = {}; - - for (let entry of await this.backend._sql("SELECT id, name FROM tags")) { - this.tags.push([entry["id"], entry["name"]]); - this.tagsHashed[entry["id"]] = entry["name"]; - } - } - - // noinspection JSUnusedGlobalSymbols - async search(query: string, allowUnknownTags: boolean = false) { - try { - if (query !== "*") { - let sql = this.searchEngine.buildQueryV2(query, allowUnknownTags); - return await this.imageListResolver(await this.backend._sql("SELECT * FROM images JOIN image_tags ON images.id=image_tags.image_id JOIN image_intensities ON images.id=image_intensities.image_id JOIN image_representations ON images.id=image_representations.image_id WHERE " + sql)); - } else { - return await this.getAllImages(PrisbeamListType.Array); - } - } catch (e) { - if (e.message.startsWith("SQLITE_ERROR: Expression tree is too large (maximum depth 1000)")) { - throw new SearchError("This search query leads to an internal query that is too large."); - } else { - throw e; - } - } - } - - // noinspection JSUnusedGlobalSymbols - async getImageFileFromId(id: number, type: PrisbeamImageType) { - let image = (await this.imageListResolver(await this.backend._sql("SELECT * FROM images JOIN image_tags ON images.id=image_tags.image_id JOIN image_intensities ON images.id=image_intensities.image_id JOIN image_representations ON images.id=image_representations.image_id WHERE id=" + id)))[0]; - return this.getImageFile(image, type); - } - - // noinspection JSUnusedGlobalSymbols - async getImage(id: number | string): Promise { - return (await this.imageListResolver(await this.backend._sql("SELECT * FROM images JOIN image_tags ON images.id=image_tags.image_id JOIN image_intensities ON images.id=image_intensities.image_id JOIN image_representations ON images.id=image_representations.image_id WHERE id=" + id)))[0] ?? null; - } - - // noinspection JSUnusedGlobalSymbols - async countImages(): Promise { - return ((await this.imageListResolver(await this.backend._sql("SELECT COUNT(*) FROM images")))[0] ?? {})["COUNT(*)"] ?? 0; - } - - getImageFile(image: object, type: PrisbeamImageType) { - function getPath(backend: Prisbeam) { - const path = require('path'); - - if (type === PrisbeamImageType.ViewFile || type === PrisbeamImageType.ViewURL) { - try { - let l: fs.PathLike; - - try { - l = backend.path + "/images/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 3) + "/" + image['id'] + path.extname(image['view_url']); - fs.lstatSync(l); - } catch (e) { - l = backend.path + "/images/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + image['id'] + path.extname(image['view_url']); - fs.lstatSync(l); - } - - if (type === PrisbeamImageType.ViewURL) { - return "file://" + encodeURI(l.replaceAll("\\", "/")); - } else { - return l; - } - } catch (e) { - try { - let l: fs.PathLike; - - try { - l = backend.path + "/images/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 3) + "/" + image['id'] + ".mp4"; - fs.lstatSync(l); - } catch (e) { - l = backend.path + "/images/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + image['id'] + ".mp4"; - fs.lstatSync(l); - } - - if (type === PrisbeamImageType.ViewURL) { - return "file://" + encodeURI(l.replaceAll("\\", "/")); - } else { - return l; - } - } catch (e) { - try { - let l: fs.PathLike; - - try { - l = backend.path + "/images/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 3) + "/" + image['id'] + ".bin"; - fs.lstatSync(l); - } catch (e) { - l = backend.path + "/images/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + image['id'] + ".bin"; - fs.lstatSync(l); - } - - if (type === PrisbeamImageType.ViewURL) { - return "file://" + encodeURI(l.replaceAll("\\", "/")); - } else { - return l; - } - } catch (e) { - try { - let l: fs.PathLike; - - try { - l = backend.path + "/thumbnails/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 3) + "/" + image['id'] + path.extname(image['representations']['thumb']); - fs.lstatSync(l); - } catch (e) { - l = backend.path + "/thumbnails/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + image['id'] + path.extname(image['representations']['thumb']); - fs.lstatSync(l); - } - - if (type === PrisbeamImageType.ViewURL) { - return "file://" + encodeURI(l.replaceAll("\\", "/")); - } else { - return l; - } - } catch (e) { - try { - let l: fs.PathLike; - - try { - l = backend.path + "/thumbnails/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 3) + "/" + image['id'] + ".bin"; - fs.lstatSync(l); - } catch (e) { - l = backend.path + "/thumbnails/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + image['id'] + ".bin"; - fs.lstatSync(l); - } - - if (type === PrisbeamImageType.ViewURL) { - return "file://" + encodeURI(l.replaceAll("\\", "/")); - } else { - return l; - } - } catch (e) { - if (type === PrisbeamImageType.ViewFile) { - return null; - } else { - return image['representations']['thumb']; - } - } - } - } - } - } - } else if (type === PrisbeamImageType.ThumbnailFile || type === PrisbeamImageType.ThumbnailURL) { - try { - let l: fs.PathLike; - - try { - l = backend.path + "/thumbnails/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 3) + "/" + image['id'] + path.extname(image['representations']['thumb']); - fs.lstatSync(l); - } catch (e) { - l = backend.path + "/thumbnails/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + image['id'] + path.extname(image['representations']['thumb']); - fs.lstatSync(l); - } - - if (type === PrisbeamImageType.ThumbnailURL) { - return "file://" + encodeURI(l.replaceAll("\\", "/")); - } else { - return l; - } - } catch (e) { - try { - let l: fs.PathLike; - - try { - l = backend.path + "/thumbnails/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 3) + "/" + image['id'] + ".bin"; - fs.lstatSync(l); - } catch (e) { - l = backend.path + "/thumbnails/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + image['id'] + ".bin"; - fs.lstatSync(l); - } - - if (type === PrisbeamImageType.ThumbnailURL) { - return "file://" + encodeURI(l.replaceAll("\\", "/")); - } else { - return l; - } - } catch (e) { - try { - let l: fs.PathLike; - - try { - l = backend.path + "/images/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 3) + "/" + image['id'] + path.extname(image['view_url']); - fs.lstatSync(l); - } catch (e) { - l = backend.path + "/images/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + image['id'] + path.extname(image['view_url']); - fs.lstatSync(l); - } - - if (type === PrisbeamImageType.ThumbnailURL) { - return "file://" + encodeURI(l.replaceAll("\\", "/")); - } else { - return l; - } - } catch (e) { - try { - let l: fs.PathLike; - - try { - l = backend.path + "/images/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 3) + "/" + image['id'] + ".mp4"; - fs.lstatSync(l); - } catch (e) { - l = backend.path + "/images/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + image['id'] + ".mp4"; - fs.lstatSync(l); - } - - if (type === PrisbeamImageType.ThumbnailURL) { - return "file://" + encodeURI(l.replaceAll("\\", "/")); - } else { - return l; - } - } catch (e) { - try { - let l: fs.PathLike; - - try { - l = backend.path + "/images/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 3) + "/" + image['id'] + ".bin"; - fs.lstatSync(l); - } catch (e) { - l = backend.path + "/images/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + image['id'] + ".bin"; - fs.lstatSync(l); - } - - if (type === PrisbeamImageType.ThumbnailURL) { - return "file://" + encodeURI(l.replaceAll("\\", "/")); - } else { - return l; - } - } catch (e) { - if (type === PrisbeamImageType.ThumbnailFile) { - return null; - } else { - return image['representations']['thumb']; - } - } - } - } - } - } - } - } - - let path = getPath(this.backend); - - if (type === PrisbeamImageType.ThumbnailURL && path.endsWith(".bin")) { - if (this.sensitiveImageProtocol) { - return path.replace("file://", "pbip://") + "?mime=" + encodeURIComponent(image['mime_type']); - } else { - return URL.createObjectURL(new Blob([zlib.inflateRawSync(fs.readFileSync(path.replace("file://", ""))).buffer], {type: image['mime_type'].startsWith("video/") ? "image/gif" : image['mime_type']})); - } - } else if (type === PrisbeamImageType.ViewURL && path.endsWith(".bin")) { - if (this.sensitiveImageProtocol) { - return path.replace("file://", "pbip://") + "?mime=" + encodeURIComponent(image['mime_type']); - } else { - return URL.createObjectURL(new Blob([zlib.inflateRawSync(fs.readFileSync(path.replace("file://", ""))).buffer], {type: image['mime_type']})); - } - } else { - return path; - } - } - - deserialize(str: string) { - return (str ?? "").replaceAll("\\_", "_").replaceAll("\\%", "%").replaceAll("\\'", "'"); - } - - async imageListResolver(list: [object]) { - for (let image of list) { - delete image['image_id']; - - image['name'] = this.deserialize(image['name']); - image['source_url'] = this.deserialize(image['source_url']); - if (image['source']) image['source'] = this.deserialize(image['source']); - image['description'] = this.deserialize(image['description']); - image['_categories'] = ["faved"]; - - image['representations'] = { - full: this.deserialize(image['full']), - large: this.deserialize(image['large']), - medium: this.deserialize(image['medium']), - small: this.deserialize(image['small']), - tall: this.deserialize(image['tall']), - thumb: this.deserialize(image['thumb']), - thumb_small: this.deserialize(image['thumb_small']), - thumb_tiny: this.deserialize(image['thumb_tiny']), - }; - image['view_url'] = this.deserialize(image['view']); - delete image['view']; - delete image['thumb_tiny']; - delete image['thumb_small']; - delete image['thumb']; - delete image['tall']; - delete image['small']; - delete image['medium']; - delete image['large']; - delete image['full']; - - image['intensities'] = { - ne: image['ne'], nw: image['nw'], se: image['se'], sw: image['sw'], - }; - delete image['ne']; - delete image['nw']; - delete image['se']; - delete image['sw']; - - image['tag_ids'] = image['tags'] ? image['tags'].substring(1, image['tags'].length - 1).split(",").map((i: string) => parseInt(i)) : []; - image['tags'] = image['tag_ids'].map((i: number) => this.deserialize(this.tagsHashed[i])); - - image['animated'] = image['animated'] === 1; - image['hidden_from_users'] = image['hidden_from_users'] === 1; - image['processed'] = image['processed'] === 1; - image['spoilered'] = image['spoilered'] === 1; - image['thumbnails_generated'] = image['thumbnails_generated'] === 1; - - image['source_id'] = image['source_id']; - image['source_name'] = image['source_name']; - image['source'] = image['source']; - } - - return list; - } - - async getAllImages(type: PrisbeamListType = PrisbeamListType.Array): Promise<{} | IPrisbeamImage[]> { - let query = "SELECT * FROM images JOIN image_tags ON images.id=image_tags.image_id JOIN image_intensities ON images.id=image_intensities.image_id JOIN image_representations ON images.id=image_representations.image_id"; - - if (type === PrisbeamListType.Array) { - return await this.imageListResolver(await this.backend._sql(query)); - } else { - let _list = await this.imageListResolver(await this.backend._sql(query)); - let list = {}; - - for (let image of _list) { - list[image['id']] = image; - } - - return list; - } - } - - // noinspection JSUnusedGlobalSymbols - async getImpliedTagIdsFromName(tag: string): Promise { - return (await this.backend._sql("SELECT implications FROM tags WHERE name = " + sqlstr(tag)))[0]["implications"].split(",").filter((i: string) => i.trim() !== "").map((i: string) => parseInt(i)); - } - - // noinspection JSUnusedGlobalSymbols - async getImpliedTagIdsFromId(tag: number): Promise { - return (await this.backend._sql("SELECT implications FROM tags WHERE id = " + tag))[0]["implications"].split(",").filter((i: string) => i.trim() !== "").map((i: string) => parseInt(i)); - } - - // noinspection JSUnusedGlobalSymbols - async getImpliedTagNamesFromName(tag: string): Promise { - let r = []; - let data = (await this.backend._sql("SELECT implications FROM tags WHERE name = " + sqlstr(tag)))[0]["implications"].split(",").filter((i: string) => i.trim() !== "").map((i: string) => parseInt(i)); - - for (let id of data) { - r.push((await this.backend._sql("SELECT name FROM tags WHERE id = " + id))[0]["name"]); - } - - return r; - } - - // noinspection JSUnusedGlobalSymbols - async getImpliedTagNamesFromId(tag: number): Promise { - let r = []; - let data = (await this.backend._sql("SELECT implications FROM tags WHERE id = " + tag))[0]["implications"].split(",").filter((i: string) => i.trim() !== "").map((i: string) => parseInt(i)); - - for (let id of data) { - r.push((await this.backend._sql("SELECT name FROM tags WHERE id = " + id))[0]["name"]); - } - - return r; - } -} - -function sqlstr(str?: string) { - if (str === null) { - return "NULL"; - } else { - return "'" + str.replaceAll("'", "''") + "'"; - } -} diff --git a/src/PrisbeamImageType.ts b/src/PrisbeamImageType.ts deleted file mode 100755 index b46b78f..0000000 --- a/src/PrisbeamImageType.ts +++ /dev/null @@ -1,3 +0,0 @@ -export enum PrisbeamImageType { - ThumbnailURL, ViewURL, ThumbnailFile, ViewFile -} diff --git a/src/PrisbeamListType.ts b/src/PrisbeamListType.ts deleted file mode 100755 index dcdad57..0000000 --- a/src/PrisbeamListType.ts +++ /dev/null @@ -1,3 +0,0 @@ -export enum PrisbeamListType { - Array, Object -} diff --git a/src/PrisbeamPropertyStore.ts b/src/PrisbeamPropertyStore.ts deleted file mode 100755 index 35a2673..0000000 --- a/src/PrisbeamPropertyStore.ts +++ /dev/null @@ -1,58 +0,0 @@ -import {Prisbeam} from "./Prisbeam"; - -export class PrisbeamPropertyStore { - backend: Prisbeam; - length: number; - - constructor(backend: Prisbeam) { - this.backend = backend; - } - - async initialize() { - this.length = (await this.backend._sql("SELECT COUNT(*) FROM metadata"))[0]["COUNT(*)"]; - } - - private sqlstr(str?: string) { - if (str === null) { - return "NULL"; - } else { - return "'" + str.replaceAll("'", "''") + "'"; - } - } - - // noinspection JSUnusedGlobalSymbols - async setItem(key: string, value: string) { - if ((await this.backend._sql("SELECT COUNT(*) FROM metadata WHERE key = " + this.sqlstr(key)))[0]["COUNT(*)"] === 0) { - await this.backend._sql('INSERT INTO metadata(key, value) VALUES (' + this.sqlstr(key) + ', ' + this.sqlstr(value) + ')'); - this.length = (await this.backend._sql("SELECT COUNT(*) FROM metadata"))[0]["COUNT(*)"]; - } else { - await this.backend._sql('UPDATE metadata SET value = ' + this.sqlstr(value) + ' WHERE key = ' + this.sqlstr(key)); - } - } - - // noinspection JSUnusedGlobalSymbols - async removeItem(key: string) { - if ((await this.backend._sql("SELECT COUNT(*) FROM metadata WHERE key = " + this.sqlstr(key)))[0]["COUNT(*)"] === 0) { - return; - } else { - await this.backend._sql('DELETE FROM metadata WHERE key = ' + this.sqlstr(key)); - this.length = (await this.backend._sql("SELECT COUNT(*) FROM metadata"))[0]["COUNT(*)"]; - } - } - - // noinspection JSUnusedGlobalSymbols - async getItem(key: string) { - if ((await this.backend._sql("SELECT COUNT(*) FROM metadata WHERE key = " + this.sqlstr(key)))[0]["COUNT(*)"] === 0) { - return null; - } else { - return (await this.backend._sql('SELECT value FROM metadata WHERE key = ' + this.sqlstr(key)))[0]["value"]; - } - } - - // noinspection JSUnusedGlobalSymbols - async clear() { - await this.backend._sql("DROP TABLE IF EXISTS metadata"); - await this.backend._sql("CREATE TABLE metadata (key TEXT NOT NULL UNIQUE, value LONGTEXT NOT NULL, PRIMARY KEY (key))"); - this.length = (await this.backend._sql("SELECT COUNT(*) FROM metadata"))[0]["COUNT(*)"]; - } -} diff --git a/src/PrisbeamSearch.ts b/src/PrisbeamSearch.ts deleted file mode 100755 index de1a45e..0000000 --- a/src/PrisbeamSearch.ts +++ /dev/null @@ -1,678 +0,0 @@ -import {SearchError} from "./SearchError"; -import {PrisbeamFrontend} from "./PrisbeamFrontend"; - -export interface IPrisbeamSearchToken { - type: PrisbeamSearchTokenType, - data?: string -} - -export enum PrisbeamSearchTokenType { - Subquery, - Not, - And, - Or, - Query -} - -export class PrisbeamSearch { - private readonly frontend: PrisbeamFrontend; - - constructor(frontend: PrisbeamFrontend) { - this.frontend = frontend; - } - - fillDate(str: string) { - return str.trim() + new Date(0).toISOString().substring(str.trim().length) - } - - checkQuery(query: string, allowUnknownTags: boolean) { - let frontend = this.frontend; - let sql = ""; - query = query.trim(); - - if (query === "") return ""; - - if (query.includes("~")) throw new SearchError("Unsupported '~' (fuzzy search) operator"); - if (query.includes("\\")) throw new SearchError("Unsupported '\\' (escape) operator"); - if (query.includes("^")) throw new SearchError("Unsupported '^' (boosting) operator"); - if (query.includes("\"")) throw new SearchError("Unsupported '\"' (quotation) operator"); - - let namespace = null; - let quantifier = null; - let value = query.split(":")[0]; - - if (query.includes(":")) { - namespace = query.split(":")[0].split(".")[0].toLowerCase(); - value = query.split(":")[1]; - - if (query.split(":")[0].includes(".")) { - quantifier = query.split(":")[0].split(".")[1].toLowerCase(); - } - } - - if (quantifier) { - if (!["lt", "lte", "gt", "gte"].includes(quantifier)) throw new SearchError("Unrecognized numeric qualifier '" + quantifier + "'"); - } - - let number: number; - - switch (namespace) { - case "created_at": - let date = new Date(this.fillDate(value)); - if (isNaN(date.getTime())) throw new SearchError("Invalid date/time value for 'created_at'"); - number = date.getTime(); - - if (isNaN(number) || !isFinite(number)) throw new Error("Number from getDate is NaN but shouldn't be"); - - switch (quantifier) { - case "lt": - sql = "images.created_at<" + number; - break; - case "lte": - sql = "images.created_at<=" + number; - break; - case "gt": - sql = "images.created_at>" + number; - break; - case "gte": - sql = "images.created_at>=" + number; - break; - default: - sql = "images.created_at=" + number; - break; - } - - break; - - case "my": - if (value === "hidden") throw new SearchError("Unsupported value for 'my': 'hidden'"); - if (value !== "upvotes" && value !== "downvotes" && value !== "faves" && value !== "uploads" && value !== "watched") throw new SearchError("Invalid value for 'my'"); - if (quantifier) throw new SearchError("'my' does not accept a numeric qualifier"); - - break; - - case "uploader": - throw new SearchError("Unsupported use of 'uploader'"); - - case "faved_by": - throw new SearchError("Unsupported use of 'faved_by'"); - - case "aspect_ratio": - number = parseFloat(value); - if (isNaN(number) || !isFinite(number)) throw new SearchError("Invalid numeric value for 'aspect_ratio'"); - - switch (quantifier) { - case "lt": - sql = "images.aspect_ratio<" + number; - break; - case "lte": - sql = "images.aspect_ratio<=" + number; - break; - case "gt": - sql = "images.aspect_ratio>" + number; - break; - case "gte": - sql = "images.aspect_ratio>=" + number; - break; - default: - sql = "images.aspect_ratio=" + number; - break; - } - - break; - - case "comment_count": - number = parseFloat(value); - if (isNaN(number) || !isFinite(number)) throw new SearchError("Invalid numeric value for 'comment_count'"); - - switch (quantifier) { - case "lt": - sql = "images.comment_count<" + number; - break; - case "lte": - sql = "images.comment_count<=" + number; - break; - case "gt": - sql = "images.comment_count>" + number; - break; - case "gte": - sql = "images.comment_count>=" + number; - break; - default: - sql = "images.comment_count=" + number; - break; - } - - break; - - case "downvotes": - number = parseFloat(value); - if (isNaN(number) || !isFinite(number)) throw new SearchError("Invalid numeric value for 'downvotes'"); - - switch (quantifier) { - case "lt": - sql = "images.downvotes<" + number; - break; - case "lte": - sql = "images.downvotes<=" + number; - break; - case "gt": - sql = "images.downvotes>" + number; - break; - case "gte": - sql = "images.downvotes>=" + number; - break; - default: - sql = "images.downvotes=" + number; - break; - } - - break; - - case "faves": - number = parseFloat(value); - if (isNaN(number) || !isFinite(number)) throw new SearchError("Invalid numeric value for 'faves'"); - - switch (quantifier) { - case "lt": - sql = "images.faves<" + number; - break; - case "lte": - sql = "images.faves<=" + number; - break; - case "gt": - sql = "images.faves>" + number; - break; - case "gte": - sql = "images.faves>=" + number; - break; - default: - sql = "images.faves=" + number; - break; - } - - break; - - case "height": - number = parseFloat(value); - if (isNaN(number) || !isFinite(number)) throw new SearchError("Invalid numeric value for 'height'"); - - switch (quantifier) { - case "lt": - sql = "images.height<" + number; - break; - case "lte": - sql = "images.height<=" + number; - break; - case "gt": - sql = "images.height>" + number; - break; - case "gte": - sql = "images.height>=" + number; - break; - default: - sql = "images.height=" + number; - break; - } - - break; - - case "id": - number = parseFloat(value); - if (isNaN(number) || !isFinite(number)) throw new SearchError("Invalid numeric value for 'id'"); - - switch (quantifier) { - case "lt": - sql = "images.source_id<" + number; - break; - case "lte": - sql = "images.source_id<=" + number; - break; - case "gt": - sql = "images.source_id>" + number; - break; - case "gte": - sql = "images.source_id>=" + number; - break; - default: - sql = "images.source_id=" + number; - break; - } - - break; - - case "score": - number = parseFloat(value); - if (isNaN(number) || !isFinite(number)) throw new SearchError("Invalid numeric value for 'score'"); - - switch (quantifier) { - case "lt": - sql = "images.score<" + number; - break; - case "lte": - sql = "images.score<=" + number; - break; - case "gt": - sql = "images.score>" + number; - break; - case "gte": - sql = "images.score>=" + number; - break; - default: - sql = "images.score=" + number; - break; - } - - break; - - case "tag_count": - number = parseFloat(value); - if (isNaN(number) || !isFinite(number)) throw new SearchError("Invalid numeric value for 'tag_count'"); - - switch (quantifier) { - case "lt": - sql = "images.tag_count<" + number; - break; - case "lte": - sql = "images.tag_count<=" + number; - break; - case "gt": - sql = "images.tag_count>" + number; - break; - case "gte": - sql = "images.tag_count>=" + number; - break; - default: - sql = "images.tag_count=" + number; - break; - } - - break; - - case "upvotes": - number = parseFloat(value); - if (isNaN(number) || !isFinite(number)) throw new SearchError("Invalid numeric value for 'upvotes'"); - - switch (quantifier) { - case "lt": - sql = "images.upvotes<" + number; - break; - case "lte": - sql = "images.upvotes<=" + number; - break; - case "gt": - sql = "images.upvotes>" + number; - break; - case "gte": - sql = "images.upvotes>=" + number; - break; - default: - sql = "images.upvotes=" + number; - break; - } - - break; - - case "width": - number = parseFloat(value); - if (isNaN(number) || !isFinite(number)) throw new SearchError("Invalid numeric value for 'width'"); - - switch (quantifier) { - case "lt": - sql = "images.width<" + number; - break; - case "lte": - sql = "images.width<=" + number; - break; - case "gt": - sql = "images.width>" + number; - break; - case "gte": - sql = "images.width>=" + number; - break; - default: - sql = "images.width=" + number; - break; - } - - break; - - case "wilson_score": - number = parseFloat(value); - if (isNaN(number) || !isFinite(number)) throw new SearchError("Invalid numeric value for 'wilson_score'"); - - switch (quantifier) { - case "lt": - sql = "images.wilson_score<" + number; - break; - case "lte": - sql = "images.wilson_score<=" + number; - break; - case "gt": - sql = "images.wilson_score>" + number; - break; - case "gte": - sql = "images.wilson_score>=" + number; - break; - default: - sql = "images.wilson_score=" + number; - break; - } - - break; - - case "sha512_hash": - if (quantifier) throw new SearchError("'sha512_hash' does not accept a numeric qualifier"); - sql = "images.sha512_hash LIKE '" + value.replaceAll("'", "''").replaceAll("%", "\\%").replaceAll("_", "\\_").replaceAll("*", "%") + "'"; - break; - - case "orig_sha512_hash": - if (quantifier) throw new SearchError("'orig_sha512_hash' does not accept a numeric qualifier"); - sql = "images.orig_sha512_hash LIKE '" + value.replaceAll("'", "''").replaceAll("%", "\\%").replaceAll("_", "\\_").replaceAll("*", "%") + "'"; - break; - - case "source_url": - if (quantifier) throw new SearchError("'source_url' does not accept a numeric qualifier"); - sql = "images.source_url LIKE '" + value.replaceAll("'", "''").replaceAll("%", "\\%").replaceAll("_", "\\_").replaceAll("*", "%") + "'"; - break; - - case "source": - if (quantifier) throw new SearchError("'source' does not accept a numeric qualifier"); - sql = "images.source_name LIKE '" + value.replaceAll("'", "''").replaceAll("%", "\\%").replaceAll("_", "\\_").replaceAll("*", "%") + "'"; - break; - - case "description": - if (quantifier) throw new SearchError("'description' does not accept a numeric qualifier"); - sql = "images.description LIKE '" + value.replaceAll("'", "''").replaceAll("%", "\\%").replaceAll("_", "\\_").replaceAll("*", "%") + "'"; - break; - - case "mime_type": - if (quantifier) throw new SearchError("'mime_type' does not accept a numeric qualifier"); - sql = "images.mime_type LIKE '" + value.replaceAll("'", "''").replaceAll("%", "\\%").replaceAll("_", "\\_").replaceAll("*", "%") + "'"; - break; - - default: - value = query.trim(); - let tags = frontend.tags; - let regex = new RegExp('^' + value.replace(/[.+^${}()|[\]\\]/g, '\\$&').replace(/\?/g, '.').replace(/\*/g, '.*') + '$'); - let matching = tags.filter(item => regex.test(item[1])); - - if (matching.length > 0) { - for (let tag of matching) { - sql += " OR image_tags.tags LIKE '%," + tag[0] + ",%'"; - } - - sql = "(" + sql.substring(4).trim() + ")"; - } else if (allowUnknownTags) { - sql = "image_tags.tags LIKE 'INVALID_TAG'"; - } else { - throw new SearchError("No tags matching '" + value.trim() + "' could be found"); - } - - break; - } - - return sql; - } - - buildQueryInner(query: string, allowUnknownTags: boolean = false) { - query = query.trim(); - let pos = 0; - - let inParentheses = false; - let currentParenthesesInner: string = ""; - let tokens: IPrisbeamSearchToken[] = []; - let currentTag = ""; - let subParentheses = 0; - - while (pos < query.length) { - if (inParentheses) { - if (query[pos] === "(") subParentheses++; - if (query[pos] === ")") subParentheses--; - - if (subParentheses < 0) { - inParentheses = false; - tokens.push({ - type: PrisbeamSearchTokenType.Subquery, - data: this.buildQueryInner(currentParenthesesInner, allowUnknownTags) - }); - currentParenthesesInner = null; - subParentheses = 0; - pos++; - continue; - } - - currentParenthesesInner += query[pos]; - pos++; - } else { - switch (query[pos]) { - case "!": - case "-": - if (currentTag.trim().length === 0) { - tokens.push({ - type: PrisbeamSearchTokenType.Not, - data: null - }); - } else { - currentTag += query[pos]; - } - - pos++; - break; - - case ",": - if (currentTag.trim().length > 0) { - tokens.push({ - type: PrisbeamSearchTokenType.Query, - data: this.checkQuery(currentTag.trim(), allowUnknownTags) - }); - currentTag = ""; - } - - tokens.push({ - type: PrisbeamSearchTokenType.And, - data: null - }); - pos++; - break; - - case "&": - if (query[pos + 1] === "&") { - if (currentTag.trim().length > 0) { - tokens.push({ - type: PrisbeamSearchTokenType.Query, - data: this.checkQuery(currentTag.trim(), allowUnknownTags) - }); - currentTag = ""; - } - - tokens.push({ - type: PrisbeamSearchTokenType.And, - data: null - }); - pos += 2; - } else { - currentTag += query[pos]; - pos++; - } - - break; - - case "|": - if (query[pos + 1] === "|") { - if (currentTag.trim().length > 0) { - tokens.push({ - type: PrisbeamSearchTokenType.Query, - data: this.checkQuery(currentTag.trim(), allowUnknownTags) - }); - currentTag = ""; - } - - tokens.push({ - type: PrisbeamSearchTokenType.Or, - data: null - }); - pos += 2; - } else { - currentTag += query[pos]; - pos++; - } - - break; - - case "O": - if (query[pos + 1] === "R") { - if (currentTag.trim().length > 0) { - tokens.push({ - type: PrisbeamSearchTokenType.Query, - data: this.checkQuery(currentTag.trim(), allowUnknownTags) - }); - currentTag = ""; - } - - tokens.push({ - type: PrisbeamSearchTokenType.Or, - data: null - }); - pos += 2; - } else { - currentTag += query[pos]; - pos++; - } - - break; - - case "A": - if (query[pos + 1] === "N" && query[pos + 2] === "D") { - if (currentTag.trim().length > 0) { - tokens.push({ - type: PrisbeamSearchTokenType.Query, - data: this.checkQuery(currentTag.trim(), allowUnknownTags) - }); - currentTag = ""; - } - - tokens.push({ - type: PrisbeamSearchTokenType.And, - data: null - }); - pos += 3; - } else { - currentTag += query[pos]; - pos++; - } - - break; - - case "N": - if (query[pos + 1] === "O" && query[pos + 2] === "T") { - if (currentTag.trim().length > 0) { - tokens.push({ - type: PrisbeamSearchTokenType.Query, - data: this.checkQuery(currentTag.trim(), allowUnknownTags) - }); - currentTag = ""; - } - - tokens.push({ - type: PrisbeamSearchTokenType.Not, - data: null - }); - pos += 3; - } else { - currentTag += query[pos]; - pos++; - } - - break; - - case " ": - currentTag += query[pos]; - pos++; - break; - - case "(": - if (currentTag.trim().length === 0) { - inParentheses = true; - currentParenthesesInner = ""; - pos++; - } else { - currentTag += query[pos]; - pos++; - } - - break; - - case ")": - if (currentTag.trim().length === 0) { - throw new SearchError("Unexpected closing parenthesis."); - } else { - currentTag += query[pos]; - pos++; - } - - break; - - default: - currentTag += query[pos]; - pos++; - break; - } - } - } - - if (currentTag.trim().length > 0) { - tokens.push({ - type: PrisbeamSearchTokenType.Query, - data: this.checkQuery(currentTag.trim(), allowUnknownTags) - }); - } - - return this.queryTokensToString(tokens); - } - - queryTokensToString(tokens: IPrisbeamSearchToken[]) { - let str = ""; - - for (let token of tokens) { - switch (token.type) { - case PrisbeamSearchTokenType.And: - str += "AND"; - break; - - case PrisbeamSearchTokenType.Query: - str += token.data; - break; - - case PrisbeamSearchTokenType.Subquery: - str += "(" + token.data + ")"; - break; - - case PrisbeamSearchTokenType.Not: - str += "NOT"; - break; - - case PrisbeamSearchTokenType.Or: - str += "OR"; - break; - } - - str += " "; - } - - return str.trim(); - } - - buildQueryV2(query: string, allowUnknownTags: boolean, allowExceedLength: boolean = false) { - if (query.length >= 1024 && !allowExceedLength) throw new SearchError("A search query needs to be shorter than 1024 characters."); - query = query.trim(); - - query = this.buildQueryInner(query, allowUnknownTags); - - if (query.length > 0) { - return "(" + query.trim() + ")"; - } else { - return "TRUE"; - } - } -} diff --git a/src/PrisbeamUpdater.ts b/src/PrisbeamUpdater.ts deleted file mode 100755 index 0ca4b0d..0000000 --- a/src/PrisbeamUpdater.ts +++ /dev/null @@ -1,522 +0,0 @@ -import {Prisbeam} from "./Prisbeam"; -import fs from "fs"; -import http from "http"; -import {IPrisbeamUpdaterGeneric} from "./IPrisbeamUpdaterGeneric"; -import {IncomingMessage, ServerResponse} from "node:http"; - -export class PrisbeamUpdater { - private readonly database: Prisbeam; - - constructor(instance: Prisbeam) { - this.database = instance; - } - - // noinspection JSUnusedGlobalSymbols - updateFromPreprocessed(preprocessed: string, tags: string, statusUpdateHandler: Function) { - return new Promise((res) => { - let sqlGet: Function; - let sqlQuery: Function; - let protectedEncode: Function; - let database = this.database; - - const zlib = require('zlib'); - - sqlGet = sqlQuery = async (query: string) => { - return await database._sql(query); - } - - protectedEncode = (b: string) => { - return zlib.deflateRawSync(b, {level: 9}); - } - - function sqlstr(str?: string) { - if (str === null) { - return "NULL"; - } else { - return "'" + str.replaceAll("'", "''") + "'"; - } - } - - function sleep(ms: number) { - return new Promise((res) => { - setTimeout(res, ms); - }); - } - - async function addToDB(images: object[]) { - await sqlQuery(`PRAGMA foreign_keys = OFF`); - - let tags = {}; - - for (let image of images) { - let imageTags = image['tags'].map((i: number, j: object) => { - return [image['tag_ids'][j], i]; - }); - - for (let tag of imageTags) { - tags[tag[0]] = tag[1]; - } - } - - let uploaders = {}; - - for (let image of images) { - if (image['uploader_id']) { - uploaders[image['uploader_id']] = image['uploader']; - } - } - - let index = 0; - let u = Object.entries(uploaders); - - for (let i = 0; i < u.length; i += 50) { - const chunk = u.slice(i, i + 50); - - for (let uploader of chunk) { - if ((await sqlGet(`SELECT COUNT(*) FROM uploaders WHERE id=${uploader[0]}`))[0]["COUNT(*)"]) { - await sqlQuery(`DELETE FROM uploaders WHERE id=${uploader[0]}`); - } - } - - // @ts-ignore - await sqlQuery(`INSERT INTO uploaders(id, name) VALUES ${chunk.map(uploader => `(${uploader[0]}, '${(uploader[1] ?? "").replaceAll("'", "''")}')`).join(",")}`); - - index += 50; - } - - index = 0; - - let v = images; - - for (let i = 0; i < v.length; i += 50) { - const chunk = v.slice(i, i + 50); - - for (let image of chunk) { - if ((await sqlGet(`SELECT COUNT(*) FROM images WHERE id=${image['id']}`))[0]["COUNT(*)"]) { - await sqlQuery(`DELETE FROM images WHERE id=${image['id']}`); - await sqlQuery(`DELETE FROM image_tags WHERE image_id=${image['id']}`); - await sqlQuery(`DELETE FROM image_intensities WHERE image_id=${image['id']}`); - await sqlQuery(`DELETE FROM image_representations WHERE image_id=${image['id']}`); - } - } - - await sqlQuery(`INSERT INTO images(id, source_id, source_name, source, animated, hidden_from_users, processed, spoilered, thumbnails_generated, aspect_ratio, duration, wilson_score, created_at, first_seen_at, updated_at, comment_count, downvotes, duplicate_of, faves, height, score, size, tag_count, upvotes, width, deletion_reason, description, format, mime_type, name, orig_sha512_hash, sha512_hash, source_url, uploader) VALUES ${chunk.map(image => `(${image['id']}, ${image['source_id'] ?? image['id']}, ${sqlstr(image['source_name'] ?? null)}, ${sqlstr(image['source'] ?? null)}, ${image['animated'] ? 'TRUE' : 'FALSE'}, ${image['hidden_from_users'] ? 'TRUE' : 'FALSE'}, ${image['processed'] ? 'TRUE' : 'FALSE'}, ${image['spoilered'] ? 'TRUE' : 'FALSE'}, ${image['thumbnails_generated'] ? 'TRUE' : 'FALSE'}, ${image['aspect_ratio']}, ${image['duration']}, ${image['wilson_score']}, ${new Date(image['created_at']).getTime() / 1000}, ${new Date(image['first_seen_at']).getTime() / 1000}, ${new Date(image['updated_at']).getTime() / 1000}, ${image['comment_count']}, ${image['downvotes']}, ${image['duplicate_of']}, ${image['faves']}, ${image['height']}, ${image['score']}, ${image['size']}, ${image['tag_count']}, ${image['upvotes']}, ${image['width']}, ${sqlstr(image['deletion_reason'])}, ${sqlstr(image['description'])}, ${sqlstr(image['format'])}, ${sqlstr(image['mime_type'])}, ${sqlstr(image['name'])}, ${sqlstr(image['orig_sha512_hash'])}, ${sqlstr(image['sha512_hash'])}, ${sqlstr(image['source_url'])}, ${image['uploader_id']})`).join(",")}`); - await sqlQuery(`INSERT INTO image_tags(image_id, tags) VALUES ${chunk.map(image => `(${image['id']}, ",${image['tag_ids'].join(",")},")`).join(",")}`); - await sqlQuery(`INSERT INTO image_intensities(image_id, ne, nw, se, sw) VALUES ${chunk.map(image => `(${image['id']}, ${image['intensities']?.ne ?? null}, ${image['intensities']?.nw ?? null}, ${image['intensities']?.se ?? null}, ${image['intensities']?.sw ?? null})`).join(",")}`); - await sqlQuery(`INSERT INTO image_representations(image_id, view, full, large, medium, small, tall, thumb, thumb_small, thumb_tiny) VALUES ${chunk.map(image => `(${image['id']}, ${sqlstr(image['view_url'])}, ${sqlstr(image['representations'].full)}, ${sqlstr(image['representations'].large)}, ${sqlstr(image['representations'].medium)}, ${sqlstr(image['representations'].small)}, ${sqlstr(image['representations'].tall)}, ${sqlstr(image['representations'].thumb)}, ${sqlstr(image['representations'].thumb_small)}, ${sqlstr(image['representations'].thumb_tiny)})`).join(",")}`); - - index += 50; - } - - await sqlQuery(`PRAGMA foreign_keys = ON`); - } - - async function cleanDB() { - await database.clean(); - } - - function timeToString(time: number | string | Date) { - if (!isNaN(parseInt(time as string))) { - time = new Date(time).getTime(); - } - - let periods = ["second", "minute", "hour", "day", "week", "month", "year", "age"]; - - let lengths = [60, 60, 24, 7, 4.35, 12, 100]; - - let difference = (time as number) / 1000; - let period: string; - - let j: number; - - for (j = 0; difference >= lengths[j] && j < lengths.length - 1; j++) { - difference /= lengths[j]; - } - - difference = Math.round(difference); - - period = periods[j]; - - return `${difference} ${period}${difference > 1 ? "s" : ""}`; - } - - if (typeof window !== "undefined") { - global = window; - } - - async function consolidateDB() { - await sqlQuery("DROP TABLE IF EXISTS tags_pre"); - - statusUpdateHandler([{ - title: "Saving database...", progress: 0, indeterminate: false - }]); - - if (database.cache) { - await fs.promises.copyFile(database.cache + "/work.pbdb", database.path + "/current.pbdb"); - } - - fs.unlinkSync(global.oldDBFile ?? null); - res(); - } - - let update = async () => { - const fs = require('fs'); - const path = require('path'); - - process.chdir(this.database.path); - - statusUpdateHandler([{ - title: "Backup up database...", progress: 0, indeterminate: false - }]); - - global.oldDBFile = this.database.path + "/" + new Date().getTime() + ".pbdb"; - fs.copyFileSync(this.database.path + "/current.pbdb", global.oldDBFile); - - statusUpdateHandler([{ - title: "Cleaning up database...", progress: 0, indeterminate: false - }]); - await cleanDB(); - - statusUpdateHandler([{ - title: "Preparing update...", progress: 0, indeterminate: false - }]); - - let sqlPreGet = (sql: string) => { - return new Promise((res, rej) => { - global.preDatabase.all(sql, function (err: Error | null, data: IPrisbeamUpdaterGeneric) { - if (err) { - rej(err); - } else { - res(data); - } - }); - }); - } - - let sqlTagGet = (sql: string) => { - return new Promise((res, rej) => { - global.tagsDatabase.all(sql, function (err: Error | null, data: IPrisbeamUpdaterGeneric) { - if (err) { - rej(err); - } else { - res(data); - } - }); - }); - } - - const sqlite3 = require(process.platform === "darwin" ? "../../../sql/mac" : "../../../sql/win"); - - global.preDatabase = new sqlite3.Database(preprocessed); - global.tagsDatabase = new sqlite3.Database(tags); - - await new Promise((res) => { - global.preDatabase.serialize(() => { - res(); - }); - }); - - const host = "127.0.0.1"; - const port = 19842; - const preprocessedImageCount = parseInt((await sqlPreGet("SELECT COUNT(*) FROM images"))[0]["COUNT(*)"]); - - let preprocessedServer = http.createServer(async (req: IncomingMessage, res: ServerResponse) => { - let page = 1; - let requestPage = new URL(req.url, "https://nothing.invalid").searchParams.get('page'); - let out = { - images: [], interactions: [], total: preprocessedImageCount - } - - if (requestPage && !isNaN(parseInt(requestPage))) { - page = parseInt(requestPage); - } - - let everything = (await sqlPreGet("SELECT * FROM images LIMIT 50 OFFSET " + ((page - 1) * 50))) as IPrisbeamUpdaterGeneric[]; - out.images = everything.map(i => JSON.parse(atob(i['json']))); - - res.writeHead(200); - res.end(JSON.stringify(out)); - }); - - preprocessedServer.listen(port, host, () => { - // noinspection HttpUrlsUsage - console.log(`Server is running on http://${host}:${port}`); - }); - - let updateTags = ((await sqlTagGet("SELECT * FROM tags")) as IPrisbeamUpdaterGeneric[]).map(i => JSON.parse(atob(i['json']))); - let index = 0; - - for (let i = 0; i < updateTags.length; i += 50) { - const chunk = updateTags.slice(i, i + 50); - - let aliases = []; - let implications = []; - - for (let tag of chunk) { - // noinspection ES6MissingAwait - aliases.push(sqlstr(await sqlTagGet(`SELECT target FROM aliases WHERE source = ` + tag['id'])[0]?.target ?? null)); - implications.push(sqlstr("," + (await sqlTagGet(`SELECT target FROM implications WHERE source = ` + tag['id']) as IPrisbeamUpdaterGeneric[]).map(i => i['target']).join(",") + ",")); - } - - await sqlQuery(`INSERT INTO tags(id, name, alias, implications, category, description, description_short, slug) VALUES ${chunk.map((tag, index) => `(${tag['id']}, ${sqlstr(tag['name'])}, ${aliases[index]}, ${implications[index]}, ${sqlstr(tag['category'])}, ${sqlstr(tag['description'])}, ${sqlstr(tag['short_description'])}, ${sqlstr(tag['slug'])})`).join(",")}`); - index += 50; - - statusUpdateHandler([{ - title: "Preparing update... " + Math.round((index / updateTags.length) * 100) + "%", - progress: ((index / updateTags.length) * 100), - indeterminate: false - }]); - } - - statusUpdateHandler([{ - title: "Preparing update...", progress: 0, indeterminate: false - }]); - - global.statusInfo = []; - - function prettySize(s: number) { - if (s < 1024) { - return s.toFixed(0) + " B"; - } else if (s < 1024 ** 2) { - return (s / 1024).toFixed(0) + " KiB"; - } else if (s < 1024 ** 3) { - return (s / 1024 ** 2).toFixed(1) + " MiB"; - } else if (s < 1024 ** 4) { - return (s / 1024 ** 3).toFixed(2) + " GiB"; - } else { - return (s / 1024 ** 4).toFixed(2) + " TiB"; - } - } - - function doImageFileDownload(url: string, image: object) { - global.statusInfo[2] = { - title: "Image: " + image['id'], progress: 0, indeterminate: false - } - - statusUpdateHandler(global.statusInfo); - - const axios = require('axios'); - - return new Promise(async (res) => { - // noinspection JSUnusedGlobalSymbols - const response = await axios({ - url, - validateStatus: (s: number) => (s >= 200 && s < 300) || (s === 404 || s === 403 || s === 401), - method: 'GET', - responseType: 'arraybuffer', - onDownloadProgress: (event: IPrisbeamUpdaterGeneric) => { - global.statusInfo[2] = { - title: "Image: " + image['id'] + " (" + Math.round((event['loaded'] / event['total']) * 100) + "%)", - progress: ((event['loaded'] / event['total']) * 100), - indeterminate: false - } - - statusUpdateHandler(global.statusInfo); - } - } as object); - - global.statusInfo[2] = null; - res(response.data); - }); - } - - global.doneFetching = false; - - if (!fs.existsSync("./thumbnails")) fs.mkdirSync("./thumbnails"); - if (!fs.existsSync("./images")) fs.mkdirSync("./images"); - - let total1: number; - let pages1: number; - let types = []; - - async function doRequest() { - try { - return (await (await fetch("http://127.0.0.1:19842")).json())['total']; - } catch (e) { - console.error(e); - return doRequest(); - } - } - - total1 = await doRequest(); - pages1 = Math.ceil(total1 / 50); - - types.push({ - query: "my:faves", name: "faved", pages: pages1, total: total1 - }); - - let prelists = {}; - global.totalPrelist = []; - global.totalPrelistFull = []; - global.times = []; - global.times2 = []; - global.totalPages = types.map(i => i['pages']).reduce((a, b) => a + b); - global.totalImages = types.map(i => i['total']).reduce((a, b) => a + b); - global.totalPageNumber = 0; - global.images = 0; - global.updateCategories = {}; - - console.log(global.totalImages + " images to download from sources, part of " + global.totalPages + " pages"); - - for (let type of types) { - let prelist = []; - let pages = type.pages; - - for (let pageNumber = 1; pageNumber <= pages; pageNumber++) { - if (global.doneFetching || !preprocessedServer) break; - - let start = new Date().getTime(); - let tryFetch = true; - let page: object = {}; - - while (tryFetch) { - try { - page = await (await fetch("http://127.0.0.1:19842/?page=" + pageNumber)).json(); - - page['images'] = page['images'].map((image: object) => { - if (image['representations']['thumb'].endsWith(".mp4") || image['representations']['thumb'].endsWith(".webm")) { - image['representations']['thumb'] = image['representations']['thumb'].substring(0, image['representations']['thumb'].length - path.extname(image['representations']['thumb']).length) + ".gif"; - } - - return image; - }); - - tryFetch = false; - } catch (e) { - console.error(e); - } - } - - await addToDB(page['images']); - - for (let image of page['images']) { - if (!global.updateCategories[image.id]) global.updateCategories[image.id] = { - upvotes: false, downvotes: false, watched: false, faved: false, uploads: false - } - - global.updateCategories[image.id][type.name] = true; - } - - prelist.push(...page['images']); - global.totalPrelist.push(...page['images']); - global.totalPrelistFull.push(...page['images']); - global.times.push(new Date().getTime() - start); - - let fetchEta = (global.totalPages - global.totalPageNumber) * (global.times.reduce((a: number, b: number) => a + b, 0) / global.times.length); - - if (global.totalPageNumber === global.totalPages) { - global.doneFetching = true; - } - - global.statusInfo[0] = { - title: "Fetching: " + Math.round((global.totalPageNumber / global.totalPages) * 100) + "% (" + global.totalPageNumber + "/" + global.totalPages + ") complete" + (global.times.length > 10 ? ", " + timeToString(fetchEta) : "") + " (" + global.totalPrelistFull.length + ")", - progress: ((global.totalPageNumber / global.totalPages) * 100), - indeterminate: false - } - - statusUpdateHandler(global.statusInfo); - global.totalPageNumber++; - } - - prelists[type.name] = prelist; - } - - global.doneFetching = true; - global.totalFetchingSize = global.totalPrelistFull.map((i: object) => i['size']).reduce((a: number, b: number) => a + b); - - try { - preprocessedServer.closeAllConnections(); - preprocessedServer.close(); - preprocessedServer = null; - } catch (e) { - console.error(e); - } - - while (global.totalPrelist.length > 0) { - let image = global.totalPrelist.shift(); - global.lastImage = image; - let start = new Date().getTime(); - let downloaded = false; - - let path1 = (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1); - let path2 = (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2); - let path3 = (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 3); - - if (!fs.existsSync("./images/" + path1)) fs.mkdirSync("./images/" + path1); - if (!fs.existsSync("./images/" + path1 + "/" + path2)) fs.mkdirSync("./images/" + path1 + "/" + path2); - if (!fs.existsSync("./images/" + path1 + "/" + path2 + "/" + path3)) fs.mkdirSync("./images/" + path1 + "/" + path2 + "/" + path3); - if (!fs.existsSync("./thumbnails/" + path1)) fs.mkdirSync("./thumbnails/" + path1); - if (!fs.existsSync("./thumbnails/" + path1 + "/" + path2)) fs.mkdirSync("./thumbnails/" + path1 + "/" + path2); - if (!fs.existsSync("./thumbnails/" + path1 + "/" + path2 + "/" + path3)) fs.mkdirSync("./thumbnails/" + path1 + "/" + path2 + "/" + path3); - - let fileName = path1 + "/" + path2 + "/" + path3 + "/" + image['id'] + ".bin"; - let fileName2 = path1 + "/" + path2 + "/" + path3 + "/" + image['id'] + ".bin"; - - let fileNamePre = path1 + "/" + path2 + "/" + path3 + "/." + image['id'] + ".bin"; - let fileNamePre2 = path1 + "/" + path2 + "/" + path3 + "/." + image['id'] + ".bin"; - - if (!fs.existsSync("./images/" + fileName) || !fs.existsSync("./thumbnails/" + fileName2)) { - if (fs.existsSync("./images/" + path1 + "/" + path2 + "/" + path3 + "/." + image['id'] + path.extname(image['view_url']))) fs.unlinkSync("./images/" + path1 + "/" + path2 + "/" + path3 + "/." + image['id'] + path.extname(image['view_url'])); - if (fs.existsSync("./thumbnails/" + path1 + "/" + path2 + "/" + path3 + "/." + image['id'] + path.extname(image['representations']['thumb']))) fs.unlinkSync("./thumbnails/" + path1 + "/" + path2 + "/" + path3 + "/." + image['id'] + path.extname(image['representations']['thumb'])); - if (fs.existsSync("./images/" + path1 + "/" + path2 + "/" + path3 + "/." + image['id'] + ".bin")) fs.unlinkSync("./images/" + path1 + "/" + path2 + "/" + path3 + "/." + image['id'] + ".bin"); - if (fs.existsSync("./thumbnails/" + path1 + "/" + path2 + "/" + path3 + "/." + image['id'] + ".bin")) fs.unlinkSync("./thumbnails/" + path1 + "/" + path2 + "/" + path3 + "/." + image['id'] + ".bin"); - if (fs.existsSync("./images/" + path1 + "/" + path2 + "/." + image['id'] + path.extname(image['view_url']))) fs.unlinkSync("./images/" + path1 + "/" + path2 + "/" + path3 + "/." + image['id'] + path.extname(image['view_url'])); - if (fs.existsSync("./thumbnails/" + path1 + "/" + path2 + "/." + image['id'] + path.extname(image['representations']['thumb']))) fs.unlinkSync("./thumbnails/" + path1 + "/" + path2 + "/." + image['id'] + path.extname(image['representations']['thumb'])); - if (fs.existsSync("./images/" + path1 + "/" + path2 + "/." + image['id'] + ".bin")) fs.unlinkSync("./images/" + path1 + "/" + path2 + "/." + image['id'] + ".bin"); - if (fs.existsSync("./thumbnails/" + path1 + "/" + path2 + "/." + image['id'] + ".bin")) fs.unlinkSync("./thumbnails/" + path1 + "/" + path2 + "/." + image['id'] + ".bin"); - - let tryFetch = true; - - while (tryFetch) { - try { - fs.writeFileSync("./images/" + fileNamePre, protectedEncode(await doImageFileDownload(image['view_url'], image))); - tryFetch = false; - } catch (e) { - console.error(e); - await sleep(1500); - } - } - - tryFetch = true; - - while (tryFetch) { - try { - fs.writeFileSync("./thumbnails/" + fileNamePre2, protectedEncode(await doImageFileDownload(image['representations']['thumb'], image))); - tryFetch = false; - } catch (e) { - console.error(e); - await sleep(1500); - } - } - - fs.renameSync("./thumbnails/" + fileNamePre2, "./thumbnails/" + fileName2); - fs.renameSync("./images/" + fileNamePre, "./images/" + fileName); - downloaded = true; - } - - if (downloaded) global.times2.push([new Date().getTime() - start, image['size']]); - global.totalPrelist.shift(); - global.images++; - - let eta = (global.totalImages - global.times2.length) * (global.times2.map((i: number[]) => i[0]).reduce((a: number, b: number) => a + b, 0) / global.times2.length); - let times = global.times2.map((i: number[]) => i[0] / i[1]).slice(0, 20); - let averageBps = times.length > 0 ? (times.reduce((a: number, b: number) => a + b) / times.length) : 0; - let dataLeft = global.totalFetchingSize - global.times2.map((i: number[]) => i[1]).reduce((a: number, b: number) => a + b, 0); - eta = dataLeft * averageBps; - - let title = "Downloading: " + Math.round((global.images / global.totalImages) * 100) + "% (" + global.images + "/" + global.totalImages + ", " + prettySize(global.times2.map((i: number[]) => i[1]).reduce((a: number, b: number) => a + b, 0)) + (global.totalFetchingSize ? "/" + prettySize(global.totalFetchingSize) : "") + ") complete" + (global.times2.length > 10 && eta > 1000 ? ", " + timeToString(eta) : ""); - - global.statusInfo[0] = { - title, progress: ((global.images / global.totalImages) * 100), indeterminate: false - } - - statusUpdateHandler(global.statusInfo); - } - - await consolidateDB(); - } - - update().then(_ => {}); - }); - } -} -- cgit