From a8f6c1dc2e19747544e7fe11ae4cc50b65705da4 Mon Sep 17 00:00:00 2001 From: RaindropsSys Date: Sat, 4 May 2024 18:04:25 +0200 Subject: Migrating libprisbeam out of the Prisbeam repo --- .gitignore | 4 + index.ts | 6 + package.json | 3 + src/Prisbeam.ts | 150 ++++++++++ src/PrisbeamFrontend.ts | 397 +++++++++++++++++++++++++ src/PrisbeamImageType.ts | 3 + src/PrisbeamListType.ts | 3 + src/PrisbeamOptions.ts | 8 + src/PrisbeamPropertyStore.ts | 54 ++++ src/PrisbeamSearch.ts | 685 +++++++++++++++++++++++++++++++++++++++++++ src/PrisbeamUpdater.ts | 518 ++++++++++++++++++++++++++++++++ src/SearchError.ts | 6 + tsconfig.json | 11 + 13 files changed, 1848 insertions(+) create mode 100644 .gitignore create mode 100644 index.ts create mode 100644 package.json create mode 100644 src/Prisbeam.ts create mode 100644 src/PrisbeamFrontend.ts create mode 100644 src/PrisbeamImageType.ts create mode 100644 src/PrisbeamListType.ts create mode 100644 src/PrisbeamOptions.ts create mode 100644 src/PrisbeamPropertyStore.ts create mode 100644 src/PrisbeamSearch.ts create mode 100644 src/PrisbeamUpdater.ts create mode 100644 src/SearchError.ts create mode 100644 tsconfig.json diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..08d3d00 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +src/*.js +*.js +src/*.js.map +*.js.map diff --git a/index.ts b/index.ts new file mode 100644 index 0000000..5fef37f --- /dev/null +++ b/index.ts @@ -0,0 +1,6 @@ +export {Prisbeam} from "./src/Prisbeam"; +export {PrisbeamImageType} from "./src/PrisbeamImageType"; +export {PrisbeamListType} from "./src/PrisbeamListType"; +export {PrisbeamUpdater} from "./src/PrisbeamUpdater"; + +export const VERSION: string = "2.2.0"; diff --git a/package.json b/package.json new file mode 100644 index 0000000..f2c75cf --- /dev/null +++ b/package.json @@ -0,0 +1,3 @@ +{ + "main": "index.js" +} \ No newline at end of file diff --git a/src/Prisbeam.ts b/src/Prisbeam.ts new file mode 100644 index 0000000..c380b6f --- /dev/null +++ b/src/Prisbeam.ts @@ -0,0 +1,150 @@ +import {PrisbeamFrontend} from "./PrisbeamFrontend"; +import {PrisbeamOptions} from "./PrisbeamOptions"; +import fs from "fs"; +import path from "path"; +import {PrisbeamUpdater, VERSION} from "../index"; +import {PrisbeamPropertyStore} from "./PrisbeamPropertyStore"; + +export class Prisbeam { + public version: string = VERSION; + public readonly verbose: boolean; + public readonly path: string; + readonly sensitiveImageProtocol: boolean; + public frontend: PrisbeamFrontend; + readonly cache?: string; + private sqlite: any; + private database: any; + private readonly readOnly: boolean; + public propertyStore: PrisbeamPropertyStore; + + constructor(options: PrisbeamOptions) { + this.verbose = options.verbose ?? true; + this.sqlite = require(options.sqlitePath ?? "sqlite3").verbose(); + + if (!fs.existsSync(path.resolve(options.database))) throw new Error("Invalid database folder specified: " + path.resolve(options.database)); + if (!fs.existsSync(path.resolve(options.database) + "/instance.pbmk")) throw new Error("Not a valid Prisbeam database: " + path.resolve(options.database)); + + this.path = path.resolve(options.database); + + if (options.cachePath) { + if (!fs.existsSync(path.resolve(options.cachePath))) throw new Error("Invalid cache folder specified: " + path.resolve(options.cachePath)); + this.cache = path.resolve(options.cachePath); + } + + this.readOnly = options.readOnly; + this.sensitiveImageProtocol = options.sensitiveImageProtocol ?? false; + } + + async clean() { + if (this.readOnly) throw new Error("The database is open is read-only mode."); + await this._sql("DROP TABLE IF EXISTS image_tags"); + await this._sql("DROP TABLE IF EXISTS image_intensities"); + await this._sql("DROP TABLE IF EXISTS image_representations"); + await this._sql("DROP TABLE IF EXISTS image_categories"); + await this._sql("DROP TABLE IF EXISTS images"); + await this._sql("DROP TABLE IF EXISTS uploaders"); + await this._sql("DROP TABLE IF EXISTS tags"); + await this._sql("DROP TABLE IF EXISTS tags_pre"); + await this._sql("DROP TABLE IF EXISTS compressed"); + await this._sql("CREATE TABLE images (id INT NOT NULL UNIQUE, source_id INT NOT NULL UNIQUE, source_name TEXT, source TEXT NOT NULL, animated BOOL, aspect_ratio FLOAT, comment_count INT, created_at TIMESTAMP, deletion_reason LONGTEXT, description LONGTEXT, downvotes INT, duplicate_of INT, duration FLOAT, faves INT, first_seen_at TIMESTAMP, format TEXT, height INT, hidden_from_users BOOL, mime_type TEXT, name LONGTEXT, orig_sha512_hash TEXT, processed BOOL, score INT, sha512_hash TEXT, size INT, source_url LONGTEXT, spoilered BOOL, tag_count INT, thumbnails_generated BOOL, updated_at TIMESTAMP, uploader INT, upvotes INT, width INT, wilson_score FLOAT, PRIMARY KEY (id), FOREIGN KEY (uploader) REFERENCES uploaders(id))"); + await this._sql("CREATE TABLE image_tags (image_id INT NOT NULL UNIQUE, tags LONGTEXT NOT NULL, PRIMARY KEY (image_id), FOREIGN KEY (image_id) REFERENCES images(id))"); + await this._sql("CREATE TABLE image_intensities (image_id INT NOT NULL UNIQUE, ne FLOAT NOT NULL, nw FLOAT NOT NULL, se FLOAT NOT NULL, sw FLOAT NOT NULL, PRIMARY KEY (image_id), FOREIGN KEY (image_id) REFERENCES images(id))"); + await this._sql("CREATE TABLE image_representations (image_id INT NOT NULL UNIQUE, view LONGTEXT NOT NULL, full TEXT, large TEXT, medium TEXT, small TEXT, tall TEXT, thumb TEXT, thumb_small TEXT, thumb_tiny TEXT, PRIMARY KEY (image_id), FOREIGN KEY (image_id) REFERENCES images(id))"); + await this._sql("CREATE TABLE tags (id INT NOT NULL UNIQUE, name TEXT NOT NULL UNIQUE, alias INT, implications LONGTEXT, category TEXT, description LONGTEXT, description_short LONGTEXT, slug TEXT UNIQUE, PRIMARY KEY (id))"); + await this._sql("CREATE TABLE uploaders (id INT NOT NULL UNIQUE, name TEXT, PRIMARY KEY (id))"); + await this._sql("CREATE TABLE IF NOT EXISTS metadata (key TEXT NOT NULL UNIQUE, value LONGTEXT NOT NULL, PRIMARY KEY (key))"); + } + + _sql(query: string) { + let verbose = this.verbose; + if (verbose) console.debug("=>", query); + + return new Promise((res, rej) => { + this.database.all(query, function (err: Error | null, data?: any) { + if (err) { + if (verbose) console.debug("<=", data); + rej(err); + } else { + if (verbose) console.debug("<=", data); + res(data); + } + }); + }); + } + + async initialize(restoreBackup: boolean) { + if (restoreBackup) { + let backups = (await fs.promises.readdir(this.path)).filter(i => i.endsWith(".pbdb") && i !== "current.pbdb"); + + if (backups.length > 0 && !isNaN(parseInt(backups[0].split(".")[0]))) { + await fs.promises.copyFile(this.path + "/" + backups[0], this.path + "/current.pbdb"); + await fs.promises.unlink(this.path + "/" + backups[0]); + } + } + + if (this.cache) { + await fs.promises.copyFile(this.path + "/current.pbdb", this.cache + "/work.pbdb"); + + if (this.readOnly) { + this.database = new this.sqlite.Database(this.cache + "/work.pbdb", this.sqlite.OPEN_READONLY); + } else { + this.database = new this.sqlite.Database(this.cache + "/work.pbdb"); + } + } else { + if (this.readOnly) { + this.database = new this.sqlite.Database(this.path + "/current.pbdb", this.sqlite.OPEN_READONLY); + } else { + this.database = new this.sqlite.Database(this.path + "/current.pbdb"); + } + } + + await new Promise((res) => { + this.database.serialize(() => { + res(); + }); + }); + + if (!this.readOnly) { + if ((await this._sql("SELECT COUNT(*) FROM metadata WHERE key='libprisbeam_timestamp'"))[0]["COUNT(*)"] === 0) { + await this._sql('INSERT INTO metadata(key, value) VALUES ("libprisbeam_timestamp", "' + new Date().toISOString() + '")'); + } else { + await this._sql('UPDATE metadata SET value="' + new Date().toISOString() + '" WHERE key="libprisbeam_timestamp"'); + } + } + + await this._sql("CREATE TABLE IF NOT EXISTS metadata (key TEXT NOT NULL UNIQUE, value LONGTEXT NOT NULL, PRIMARY KEY (key))"); + + this.frontend = new PrisbeamFrontend(this); + this.propertyStore = new PrisbeamPropertyStore(this); + this.propertyStore.initialize(); + + this.frontend.initialize(); + await this.defragment(); + } + + async defragment() { + await this._sql("VACUUM"); + } + + async close() { + await new Promise((res) => { + this.database.wait(() => { + res(); + }); + }); + + this.database.close(); + + if (this.cache) { + await fs.promises.copyFile(this.cache + "/work.pbdb", this.path + "/current.pbdb"); + } + } + + /** + * @deprecated Use PrisbeamUpdater instead + */ + async updateFromPreprocessed(preprocessed: string, tags: string, statusUpdateHandler: Function) { + let updater = new PrisbeamUpdater(this); + await updater.updateFromPreprocessed(preprocessed, tags, statusUpdateHandler); + } +} diff --git a/src/PrisbeamFrontend.ts b/src/PrisbeamFrontend.ts new file mode 100644 index 0000000..4108331 --- /dev/null +++ b/src/PrisbeamFrontend.ts @@ -0,0 +1,397 @@ +import {PrisbeamSearch} from "./PrisbeamSearch"; +import {PrisbeamImageType} from "./PrisbeamImageType"; +import fs from "fs"; +import zlib from "zlib"; +import {PrisbeamListType} from "./PrisbeamListType"; +import {Prisbeam} from "../index"; +import {SearchError} from "./SearchError"; + +export class PrisbeamFrontend { + public tags: any[][]; + public tagsHashed: object; + private readonly backend: Prisbeam; + readonly searchEngine: PrisbeamSearch; + private readonly sensitiveImageProtocol: boolean; + + constructor(backend: Prisbeam) { + this.backend = backend; + this.sensitiveImageProtocol = backend.sensitiveImageProtocol; + this.searchEngine = new PrisbeamSearch(this); + } + + async initialize() { + this.tags = []; + this.tagsHashed = {}; + + for (let entry of await this.backend._sql("SELECT id, name FROM tags")) { + this.tags.push([entry["id"], entry["name"]]); + this.tagsHashed[entry["id"]] = entry["name"]; + } + } + + async search(query: string, allowUnknownTags: boolean = false) { + try { + if (query !== "*") { + let sql = this.searchEngine.buildQueryV2(query, allowUnknownTags); + return await this.imageListResolver(await this.backend._sql("SELECT * FROM images JOIN image_tags ON images.id=image_tags.image_id JOIN image_intensities ON images.id=image_intensities.image_id JOIN image_representations ON images.id=image_representations.image_id WHERE " + sql)); + } else { + return await this.getAllImages(PrisbeamListType.Array); + } + } catch (e) { + if (e.message.startsWith("SQLITE_ERROR: Expression tree is too large (maximum depth 1000)")) { + throw new SearchError("This search query leads to an internal query that is too large."); + } else { + throw e; + } + } + } + + // noinspection JSUnusedGlobalSymbols + async getImageFileFromId(id: number, type: PrisbeamImageType) { + let image = (await this.imageListResolver(await this.backend._sql("SELECT * FROM images JOIN image_tags ON images.id=image_tags.image_id JOIN image_intensities ON images.id=image_intensities.image_id JOIN image_representations ON images.id=image_representations.image_id WHERE id=" + id)))[0]; + return this.getImageFile(image, type); + } + + async getImage(id: number | string): Promise { + return (await this.imageListResolver(await this.backend._sql("SELECT * FROM images JOIN image_tags ON images.id=image_tags.image_id JOIN image_intensities ON images.id=image_intensities.image_id JOIN image_representations ON images.id=image_representations.image_id WHERE id=" + id)))[0] ?? null; + } + + async countImages(): Promise { + return ((await this.imageListResolver(await this.backend._sql("SELECT COUNT(*) FROM images")))[0] ?? {})["COUNT(*)"] ?? 0; + } + + getImageFile(image: object, type: PrisbeamImageType) { + function getPath(backend: Prisbeam) { + const path = require('path'); + + if (type === PrisbeamImageType.ViewFile || type === PrisbeamImageType.ViewURL) { + try { + let l: fs.PathLike; + + try { + l = backend.path + "/images/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 3) + "/" + image['id'] + path.extname(image['view_url']); + fs.lstatSync(l); + } catch (e) { + l = backend.path + "/images/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + image['id'] + path.extname(image['view_url']); + fs.lstatSync(l); + } + + if (type === PrisbeamImageType.ViewURL) { + return "file://" + encodeURI(l.replaceAll("\\", "/")); + } else { + return l; + } + } catch (e) { + try { + let l: fs.PathLike; + + try { + l = backend.path + "/images/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 3) + "/" + image['id'] + ".mp4"; + fs.lstatSync(l); + } catch (e) { + l = backend.path + "/images/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + image['id'] + ".mp4"; + fs.lstatSync(l); + } + + if (type === PrisbeamImageType.ViewURL) { + return "file://" + encodeURI(l.replaceAll("\\", "/")); + } else { + return l; + } + } catch (e) { + try { + let l: fs.PathLike; + + try { + l = backend.path + "/images/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 3) + "/" + image['id'] + ".bin"; + fs.lstatSync(l); + } catch (e) { + l = backend.path + "/images/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + image['id'] + ".bin"; + fs.lstatSync(l); + } + + if (type === PrisbeamImageType.ViewURL) { + return "file://" + encodeURI(l.replaceAll("\\", "/")); + } else { + return l; + } + } catch (e) { + try { + let l: fs.PathLike; + + try { + l = backend.path + "/thumbnails/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 3) + "/" + image['id'] + path.extname(image['representations']['thumb']); + fs.lstatSync(l); + } catch (e) { + l = backend.path + "/thumbnails/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + image['id'] + path.extname(image['representations']['thumb']); + fs.lstatSync(l); + } + + if (type === PrisbeamImageType.ViewURL) { + return "file://" + encodeURI(l.replaceAll("\\", "/")); + } else { + return l; + } + } catch (e) { + try { + let l: fs.PathLike; + + try { + l = backend.path + "/thumbnails/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 3) + "/" + image['id'] + ".bin"; + fs.lstatSync(l); + } catch (e) { + l = backend.path + "/thumbnails/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + image['id'] + ".bin"; + fs.lstatSync(l); + } + + if (type === PrisbeamImageType.ViewURL) { + return "file://" + encodeURI(l.replaceAll("\\", "/")); + } else { + return l; + } + } catch (e) { + if (type === PrisbeamImageType.ViewFile) { + return null; + } else { + return image['representations']['thumb']; + } + } + } + } + } + } + } else if (type === PrisbeamImageType.ThumbnailFile || type === PrisbeamImageType.ThumbnailURL) { + try { + let l: fs.PathLike; + + try { + l = backend.path + "/thumbnails/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 3) + "/" + image['id'] + path.extname(image['representations']['thumb']); + fs.lstatSync(l); + } catch (e) { + l = backend.path + "/thumbnails/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + image['id'] + path.extname(image['representations']['thumb']); + fs.lstatSync(l); + } + + if (type === PrisbeamImageType.ThumbnailURL) { + return "file://" + encodeURI(l.replaceAll("\\", "/")); + } else { + return l; + } + } catch (e) { + try { + let l: fs.PathLike; + + try { + l = backend.path + "/thumbnails/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 3) + "/" + image['id'] + ".bin"; + fs.lstatSync(l); + } catch (e) { + l = backend.path + "/thumbnails/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + image['id'] + ".bin"; + fs.lstatSync(l); + } + + if (type === PrisbeamImageType.ThumbnailURL) { + return "file://" + encodeURI(l.replaceAll("\\", "/")); + } else { + return l; + } + } catch (e) { + try { + let l: fs.PathLike; + + try { + l = backend.path + "/images/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 3) + "/" + image['id'] + path.extname(image['view_url']); + fs.lstatSync(l); + } catch (e) { + l = backend.path + "/images/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + image['id'] + path.extname(image['view_url']); + fs.lstatSync(l); + } + + if (type === PrisbeamImageType.ThumbnailURL) { + return "file://" + encodeURI(l.replaceAll("\\", "/")); + } else { + return l; + } + } catch (e) { + try { + let l: fs.PathLike; + + try { + l = backend.path + "/images/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 3) + "/" + image['id'] + ".mp4"; + fs.lstatSync(l); + } catch (e) { + l = backend.path + "/images/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + image['id'] + ".mp4"; + fs.lstatSync(l); + } + + if (type === PrisbeamImageType.ThumbnailURL) { + return "file://" + encodeURI(l.replaceAll("\\", "/")); + } else { + return l; + } + } catch (e) { + try { + let l: fs.PathLike; + + try { + l = backend.path + "/images/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 3) + "/" + image['id'] + ".bin"; + fs.lstatSync(l); + } catch (e) { + l = backend.path + "/images/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1) + "/" + (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2) + "/" + image['id'] + ".bin"; + fs.lstatSync(l); + } + + if (type === PrisbeamImageType.ThumbnailURL) { + return "file://" + encodeURI(l.replaceAll("\\", "/")); + } else { + return l; + } + } catch (e) { + if (type === PrisbeamImageType.ThumbnailFile) { + return null; + } else { + return image['representations']['thumb']; + } + } + } + } + } + } + } + } + + let path = getPath(this.backend); + + if (type === PrisbeamImageType.ThumbnailURL && path.endsWith(".bin")) { + if (this.sensitiveImageProtocol) { + return path.replace("file://", "pbip://") + "?mime=" + encodeURIComponent(image['mime_type']); + } else { + return URL.createObjectURL(new Blob([zlib.inflateRawSync(fs.readFileSync(path.replace("file://", ""))).buffer], {type: image['mime_type'].startsWith("video/") ? "image/gif" : image['mime_type']})); + } + } else if (type === PrisbeamImageType.ViewURL && path.endsWith(".bin")) { + if (this.sensitiveImageProtocol) { + return path.replace("file://", "pbip://") + "?mime=" + encodeURIComponent(image['mime_type']); + } else { + return URL.createObjectURL(new Blob([zlib.inflateRawSync(fs.readFileSync(path.replace("file://", ""))).buffer], {type: image['mime_type']})); + } + } else { + return path; + } + } + + deserialize(str: string) { + return (str ?? "").replaceAll("\\_", "_").replaceAll("\\%", "%").replaceAll("\\'", "'"); + } + + async imageListResolver(list: [object]) { + for (let image of list) { + delete image['image_id']; + + image['name'] = this.deserialize(image['name']); + image['source_url'] = this.deserialize(image['source_url']); + if (image['source']) image['source'] = this.deserialize(image['source']); + image['description'] = this.deserialize(image['description']); + image['_categories'] = ["faved"]; + + image['representations'] = { + full: this.deserialize(image['full']), + large: this.deserialize(image['large']), + medium: this.deserialize(image['medium']), + small: this.deserialize(image['small']), + tall: this.deserialize(image['tall']), + thumb: this.deserialize(image['thumb']), + thumb_small: this.deserialize(image['thumb_small']), + thumb_tiny: this.deserialize(image['thumb_tiny']), + }; + image['view_url'] = this.deserialize(image['view']); + delete image['view']; + delete image['thumb_tiny']; + delete image['thumb_small']; + delete image['thumb']; + delete image['tall']; + delete image['small']; + delete image['medium']; + delete image['large']; + delete image['full']; + + image['intensities'] = { + ne: image['ne'], nw: image['nw'], se: image['se'], sw: image['sw'], + }; + delete image['ne']; + delete image['nw']; + delete image['se']; + delete image['sw']; + + image['tag_ids'] = image['tags'].substring(1, image['tags'].length - 1).split(",").map((i: string) => parseInt(i)); + image['tags'] = image['tag_ids'].map((i: number) => this.deserialize(this.tagsHashed[i])); + + image['animated'] = image['animated'] === 1; + image['hidden_from_users'] = image['hidden_from_users'] === 1; + image['processed'] = image['processed'] === 1; + image['spoilered'] = image['spoilered'] === 1; + image['thumbnails_generated'] = image['thumbnails_generated'] === 1; + + image['source_id'] = image['source_id']; + image['source_name'] = image['source_name']; + image['source'] = image['source']; + } + + return list; + } + + async getAllImages(type: PrisbeamListType = PrisbeamListType.Array): Promise<{} | any[]> { + let query = "SELECT * FROM images JOIN image_tags ON images.id=image_tags.image_id JOIN image_intensities ON images.id=image_intensities.image_id JOIN image_representations ON images.id=image_representations.image_id"; + + if (type === PrisbeamListType.Array) { + return await this.imageListResolver(await this.backend._sql(query)); + } else { + let _list = await this.imageListResolver(await this.backend._sql(query)); + let list = {}; + + for (let image of _list) { + list[image['id']] = image; + } + + return list; + } + } + + // noinspection JSUnusedGlobalSymbols + async getImpliedTagIdsFromName(tag: string): Promise { + return (await this.backend._sql("SELECT implications FROM tags WHERE name = " + sqlstr(tag)))[0]["implications"].split(",").filter((i: string) => i.trim() !== "").map((i: string) => parseInt(i)); + } + + async getImpliedTagIdsFromId(tag: number): Promise { + return (await this.backend._sql("SELECT implications FROM tags WHERE id = " + tag))[0]["implications"].split(",").filter((i: string) => i.trim() !== "").map((i: string) => parseInt(i)); + } + + // noinspection JSUnusedGlobalSymbols + async getImpliedTagNamesFromName(tag: string): Promise { + let r = []; + let data = (await this.backend._sql("SELECT implications FROM tags WHERE name = " + sqlstr(tag)))[0]["implications"].split(",").filter((i: string) => i.trim() !== "").map((i: string) => parseInt(i)); + + for (let id of data) { + r.push((await this.backend._sql("SELECT name FROM tags WHERE id = " + id))[0]["name"]); + } + + return r; + } + + async getImpliedTagNamesFromId(tag: number): Promise { + let r = []; + let data = (await this.backend._sql("SELECT implications FROM tags WHERE id = " + tag))[0]["implications"].split(",").filter((i: string) => i.trim() !== "").map((i: string) => parseInt(i)); + + for (let id of data) { + r.push((await this.backend._sql("SELECT name FROM tags WHERE id = " + id))[0]["name"]); + } + + return r; + } +} + +function sqlstr(str?: string) { + if (str === null) { + return "NULL"; + } else { + return "'" + str.replaceAll("'", "''") + "'"; + } +} diff --git a/src/PrisbeamImageType.ts b/src/PrisbeamImageType.ts new file mode 100644 index 0000000..b46b78f --- /dev/null +++ b/src/PrisbeamImageType.ts @@ -0,0 +1,3 @@ +export enum PrisbeamImageType { + ThumbnailURL, ViewURL, ThumbnailFile, ViewFile +} diff --git a/src/PrisbeamListType.ts b/src/PrisbeamListType.ts new file mode 100644 index 0000000..dcdad57 --- /dev/null +++ b/src/PrisbeamListType.ts @@ -0,0 +1,3 @@ +export enum PrisbeamListType { + Array, Object +} diff --git a/src/PrisbeamOptions.ts b/src/PrisbeamOptions.ts new file mode 100644 index 0000000..c2fb64b --- /dev/null +++ b/src/PrisbeamOptions.ts @@ -0,0 +1,8 @@ +export interface PrisbeamOptions { + database: string, + sqlitePath?: string, + cachePath?: string, + readOnly: boolean, + sensitiveImageProtocol?: boolean, + verbose?: boolean +} diff --git a/src/PrisbeamPropertyStore.ts b/src/PrisbeamPropertyStore.ts new file mode 100644 index 0000000..a7c2e22 --- /dev/null +++ b/src/PrisbeamPropertyStore.ts @@ -0,0 +1,54 @@ +import {Prisbeam} from "./Prisbeam"; + +export class PrisbeamPropertyStore { + backend: Prisbeam; + length: number; + + constructor(backend: Prisbeam) { + this.backend = backend; + } + + async initialize() { + this.length = (await this.backend._sql("SELECT COUNT(*) FROM metadata"))[0]["COUNT(*)"]; + } + + private sqlstr(str?: string) { + if (str === null) { + return "NULL"; + } else { + return "'" + str.replaceAll("'", "''") + "'"; + } + } + + async setItem(key: string, value: string) { + if ((await this.backend._sql("SELECT COUNT(*) FROM metadata WHERE key = " + this.sqlstr(key)))[0]["COUNT(*)"] === 0) { + await this.backend._sql('INSERT INTO metadata(key, value) VALUES (' + this.sqlstr(key) + ', ' + this.sqlstr(value) + ')'); + this.length = (await this.backend._sql("SELECT COUNT(*) FROM metadata"))[0]["COUNT(*)"]; + } else { + await this.backend._sql('UPDATE metadata SET value = ' + this.sqlstr(value) + ' WHERE key = ' + this.sqlstr(key)); + } + } + + async removeItem(key: string) { + if ((await this.backend._sql("SELECT COUNT(*) FROM metadata WHERE key = " + this.sqlstr(key)))[0]["COUNT(*)"] === 0) { + return; + } else { + await this.backend._sql('DELETE FROM metadata WHERE key = ' + this.sqlstr(key)); + this.length = (await this.backend._sql("SELECT COUNT(*) FROM metadata"))[0]["COUNT(*)"]; + } + } + + async getItem(key: string) { + if ((await this.backend._sql("SELECT COUNT(*) FROM metadata WHERE key = " + this.sqlstr(key)))[0]["COUNT(*)"] === 0) { + return null; + } else { + return (await this.backend._sql('SELECT value FROM metadata WHERE key = ' + this.sqlstr(key)))[0]["value"]; + } + } + + async clear() { + await this.backend._sql("DROP TABLE IF EXISTS metadata"); + await this.backend._sql("CREATE TABLE metadata (key TEXT NOT NULL UNIQUE, value LONGTEXT NOT NULL, PRIMARY KEY (key))"); + this.length = (await this.backend._sql("SELECT COUNT(*) FROM metadata"))[0]["COUNT(*)"]; + } +} diff --git a/src/PrisbeamSearch.ts b/src/PrisbeamSearch.ts new file mode 100644 index 0000000..395f454 --- /dev/null +++ b/src/PrisbeamSearch.ts @@ -0,0 +1,685 @@ +import {SearchError} from "./SearchError"; +import {PrisbeamFrontend} from "./PrisbeamFrontend"; + +export interface PrisbeamSearchToken { + type: PrisbeamSearchTokenType, + data?: any +} + +export enum PrisbeamSearchTokenType { + Subquery, + Not, + And, + Or, + Query +} + +export class PrisbeamSearch { + private readonly frontend: PrisbeamFrontend; + + constructor(frontend: PrisbeamFrontend) { + this.frontend = frontend; + } + + fillDate(str: string) { + return str.trim() + new Date(0).toISOString().substring(str.trim().length) + } + + checkQuery(query: string, allowUnknownTags: boolean) { + let frontend = this.frontend; + let sql = ""; + query = query.trim(); + + if (query === "") return ""; + + if (query.includes("~")) throw new SearchError("Unsupported '~' (fuzzy search) operator"); + if (query.includes("\\")) throw new SearchError("Unsupported '\\' (escape) operator"); + if (query.includes("^")) throw new SearchError("Unsupported '^' (boosting) operator"); + if (query.includes("\"")) throw new SearchError("Unsupported '\"' (quotation) operator"); + + let namespace = null; + let quantifier = null; + let value = query.split(":")[0]; + + if (query.includes(":")) { + namespace = query.split(":")[0].split(".")[0].toLowerCase(); + value = query.split(":")[1]; + + if (query.split(":")[0].includes(".")) { + quantifier = query.split(":")[0].split(".")[1].toLowerCase(); + } + } + + if (quantifier) { + if (!["lt", "lte", "gt", "gte"].includes(quantifier)) throw new SearchError("Unrecognized numeric qualifier '" + quantifier + "'"); + } + + let number: number; + + switch (namespace) { + case "created_at": + let date = new Date(this.fillDate(value)); + if (isNaN(date.getTime())) throw new SearchError("Invalid date/time value for 'created_at'"); + number = date.getTime(); + + if (isNaN(number) || !isFinite(number)) throw new Error("Number from getDate is NaN but shouldn't be"); + + switch (quantifier) { + case "lt": + sql = "images.created_at<" + number; + break; + case "lte": + sql = "images.created_at<=" + number; + break; + case "gt": + sql = "images.created_at>" + number; + break; + case "gte": + sql = "images.created_at>=" + number; + break; + default: + sql = "images.created_at=" + number; + break; + } + + break; + + case "my": + if (value === "hidden") throw new SearchError("Unsupported value for 'my': 'hidden'"); + if (value !== "upvotes" && value !== "downvotes" && value !== "faves" && value !== "uploads" && value !== "watched") throw new SearchError("Invalid value for 'my'"); + if (quantifier) throw new SearchError("'my' does not accept a numeric qualifier"); + + break; + + case "uploader": + throw new SearchError("Unsupported use of 'uploader'"); + + case "faved_by": + throw new SearchError("Unsupported use of 'faved_by'"); + + case "aspect_ratio": + number = parseFloat(value); + if (isNaN(number) || !isFinite(number)) throw new SearchError("Invalid numeric value for 'aspect_ratio'"); + + switch (quantifier) { + case "lt": + sql = "images.aspect_ratio<" + number; + break; + case "lte": + sql = "images.aspect_ratio<=" + number; + break; + case "gt": + sql = "images.aspect_ratio>" + number; + break; + case "gte": + sql = "images.aspect_ratio>=" + number; + break; + default: + sql = "images.aspect_ratio=" + number; + break; + } + + break; + + case "comment_count": + number = parseFloat(value); + if (isNaN(number) || !isFinite(number)) throw new SearchError("Invalid numeric value for 'comment_count'"); + + switch (quantifier) { + case "lt": + sql = "images.comment_count<" + number; + break; + case "lte": + sql = "images.comment_count<=" + number; + break; + case "gt": + sql = "images.comment_count>" + number; + break; + case "gte": + sql = "images.comment_count>=" + number; + break; + default: + sql = "images.comment_count=" + number; + break; + } + + break; + + case "downvotes": + number = parseFloat(value); + if (isNaN(number) || !isFinite(number)) throw new SearchError("Invalid numeric value for 'downvotes'"); + + switch (quantifier) { + case "lt": + sql = "images.downvotes<" + number; + break; + case "lte": + sql = "images.downvotes<=" + number; + break; + case "gt": + sql = "images.downvotes>" + number; + break; + case "gte": + sql = "images.downvotes>=" + number; + break; + default: + sql = "images.downvotes=" + number; + break; + } + + break; + + case "faves": + number = parseFloat(value); + if (isNaN(number) || !isFinite(number)) throw new SearchError("Invalid numeric value for 'faves'"); + + switch (quantifier) { + case "lt": + sql = "images.faves<" + number; + break; + case "lte": + sql = "images.faves<=" + number; + break; + case "gt": + sql = "images.faves>" + number; + break; + case "gte": + sql = "images.faves>=" + number; + break; + default: + sql = "images.faves=" + number; + break; + } + + break; + + case "height": + number = parseFloat(value); + if (isNaN(number) || !isFinite(number)) throw new SearchError("Invalid numeric value for 'height'"); + + switch (quantifier) { + case "lt": + sql = "images.height<" + number; + break; + case "lte": + sql = "images.height<=" + number; + break; + case "gt": + sql = "images.height>" + number; + break; + case "gte": + sql = "images.height>=" + number; + break; + default: + sql = "images.height=" + number; + break; + } + + break; + + case "id": + number = parseFloat(value); + if (isNaN(number) || !isFinite(number)) throw new SearchError("Invalid numeric value for 'id'"); + + switch (quantifier) { + case "lt": + sql = "images.source_id<" + number; + break; + case "lte": + sql = "images.source_id<=" + number; + break; + case "gt": + sql = "images.source_id>" + number; + break; + case "gte": + sql = "images.source_id>=" + number; + break; + default: + sql = "images.source_id=" + number; + break; + } + + break; + + case "score": + number = parseFloat(value); + if (isNaN(number) || !isFinite(number)) throw new SearchError("Invalid numeric value for 'score'"); + + switch (quantifier) { + case "lt": + sql = "images.score<" + number; + break; + case "lte": + sql = "images.score<=" + number; + break; + case "gt": + sql = "images.score>" + number; + break; + case "gte": + sql = "images.score>=" + number; + break; + default: + sql = "images.score=" + number; + break; + } + + break; + + case "tag_count": + number = parseFloat(value); + if (isNaN(number) || !isFinite(number)) throw new SearchError("Invalid numeric value for 'tag_count'"); + + switch (quantifier) { + case "lt": + sql = "images.tag_count<" + number; + break; + case "lte": + sql = "images.tag_count<=" + number; + break; + case "gt": + sql = "images.tag_count>" + number; + break; + case "gte": + sql = "images.tag_count>=" + number; + break; + default: + sql = "images.tag_count=" + number; + break; + } + + break; + + case "upvotes": + number = parseFloat(value); + if (isNaN(number) || !isFinite(number)) throw new SearchError("Invalid numeric value for 'upvotes'"); + + switch (quantifier) { + case "lt": + sql = "images.upvotes<" + number; + break; + case "lte": + sql = "images.upvotes<=" + number; + break; + case "gt": + sql = "images.upvotes>" + number; + break; + case "gte": + sql = "images.upvotes>=" + number; + break; + default: + sql = "images.upvotes=" + number; + break; + } + + break; + + case "width": + number = parseFloat(value); + if (isNaN(number) || !isFinite(number)) throw new SearchError("Invalid numeric value for 'width'"); + + switch (quantifier) { + case "lt": + sql = "images.width<" + number; + break; + case "lte": + sql = "images.width<=" + number; + break; + case "gt": + sql = "images.width>" + number; + break; + case "gte": + sql = "images.width>=" + number; + break; + default: + sql = "images.width=" + number; + break; + } + + break; + + case "wilson_score": + number = parseFloat(value); + if (isNaN(number) || !isFinite(number)) throw new SearchError("Invalid numeric value for 'wilson_score'"); + + switch (quantifier) { + case "lt": + sql = "images.wilson_score<" + number; + break; + case "lte": + sql = "images.wilson_score<=" + number; + break; + case "gt": + sql = "images.wilson_score>" + number; + break; + case "gte": + sql = "images.wilson_score>=" + number; + break; + default: + sql = "images.wilson_score=" + number; + break; + } + + break; + + case "sha512_hash": + if (quantifier) throw new SearchError("'sha512_hash' does not accept a numeric qualifier"); + sql = "images.sha512_hash LIKE '" + value.replaceAll("'", "''").replaceAll("%", "\\%").replaceAll("_", "\\_").replaceAll("*", "%") + "'"; + break; + + case "orig_sha512_hash": + if (quantifier) throw new SearchError("'orig_sha512_hash' does not accept a numeric qualifier"); + sql = "images.orig_sha512_hash LIKE '" + value.replaceAll("'", "''").replaceAll("%", "\\%").replaceAll("_", "\\_").replaceAll("*", "%") + "'"; + break; + + case "source_url": + if (quantifier) throw new SearchError("'source_url' does not accept a numeric qualifier"); + sql = "images.source_url LIKE '" + value.replaceAll("'", "''").replaceAll("%", "\\%").replaceAll("_", "\\_").replaceAll("*", "%") + "'"; + break; + + case "source": + if (quantifier) throw new SearchError("'source' does not accept a numeric qualifier"); + sql = "images.source_name LIKE '" + value.replaceAll("'", "''").replaceAll("%", "\\%").replaceAll("_", "\\_").replaceAll("*", "%") + "'"; + break; + + case "description": + if (quantifier) throw new SearchError("'description' does not accept a numeric qualifier"); + sql = "images.description LIKE '" + value.replaceAll("'", "''").replaceAll("%", "\\%").replaceAll("_", "\\_").replaceAll("*", "%") + "'"; + break; + + case "mime_type": + if (quantifier) throw new SearchError("'mime_type' does not accept a numeric qualifier"); + sql = "images.mime_type LIKE '" + value.replaceAll("'", "''").replaceAll("%", "\\%").replaceAll("_", "\\_").replaceAll("*", "%") + "'"; + break; + + default: + value = query.trim(); + let tags = frontend.tags; + let regex = new RegExp('^' + value.replace(/[.+^${}()|[\]\\]/g, '\\$&').replace(/\?/g, '.').replace(/\*/g, '.*') + '$'); + let matching = tags.filter(item => regex.test(item[1])); + + if (matching.length > 0) { + for (let tag of matching) { + sql += " OR image_tags.tags LIKE '%," + tag[0] + ",%'"; + } + + sql = "(" + sql.substring(4).trim() + ")"; + } else if (allowUnknownTags) { + sql = "image_tags.tags LIKE 'INVALID_TAG'"; + } else { + throw new SearchError("No tags matching '" + value.trim() + "' could be found"); + } + + break; + } + + return sql; + } + + buildQueryInner(query: string, allowUnknownTags: boolean = false) { + query = query.trim(); + let pos = 0; + + let inParentheses = false; + let currentParenthesesInner: string; + let tokens: PrisbeamSearchToken[] = []; + let currentTag = ""; + let subParentheses = 0; + + while (pos < query.length) { + if (inParentheses) { + if (query[pos] === "(") subParentheses++; + if (query[pos] === ")") subParentheses--; + + if (subParentheses < 0) { + inParentheses = false; + tokens.push({ + type: PrisbeamSearchTokenType.Subquery, + data: this.buildQueryInner(currentParenthesesInner, allowUnknownTags) + }); + currentParenthesesInner = null; + subParentheses = 0; + pos++; + continue; + } + + currentParenthesesInner += query[pos]; + pos++; + } else { + switch (query[pos]) { + case "!": + case "-": + if (currentTag.trim().length === 0) { + tokens.push({ + type: PrisbeamSearchTokenType.Not, + data: null + }); + } else { + currentTag += query[pos]; + } + + pos++; + break; + + case ",": + if (currentTag.trim().length > 0) { + tokens.push({ + type: PrisbeamSearchTokenType.Query, + data: this.checkQuery(currentTag.trim(), allowUnknownTags) + }); + currentTag = ""; + } + + tokens.push({ + type: PrisbeamSearchTokenType.And, + data: null + }); + pos++; + break; + + case "&": + if (query[pos + 1] === "&") { + if (currentTag.trim().length > 0) { + tokens.push({ + type: PrisbeamSearchTokenType.Query, + data: this.checkQuery(currentTag.trim(), allowUnknownTags) + }); + currentTag = ""; + } + + tokens.push({ + type: PrisbeamSearchTokenType.And, + data: null + }); + pos += 2; + } else { + currentTag += query[pos]; + pos++; + } + + break; + + case "|": + if (query[pos + 1] === "|") { + if (currentTag.trim().length > 0) { + tokens.push({ + type: PrisbeamSearchTokenType.Query, + data: this.checkQuery(currentTag.trim(), allowUnknownTags) + }); + currentTag = ""; + } + + tokens.push({ + type: PrisbeamSearchTokenType.Or, + data: null + }); + pos += 2; + } else { + currentTag += query[pos]; + pos++; + } + + break; + + case "O": + if (query[pos + 1] === "R") { + if (currentTag.trim().length > 0) { + tokens.push({ + type: PrisbeamSearchTokenType.Query, + data: this.checkQuery(currentTag.trim(), allowUnknownTags) + }); + currentTag = ""; + } + + tokens.push({ + type: PrisbeamSearchTokenType.Or, + data: null + }); + pos += 2; + } else { + currentTag += query[pos]; + pos++; + } + + break; + + case "A": + if (query[pos + 1] === "N" && query[pos + 2] === "D") { + if (currentTag.trim().length > 0) { + tokens.push({ + type: PrisbeamSearchTokenType.Query, + data: this.checkQuery(currentTag.trim(), allowUnknownTags) + }); + currentTag = ""; + } + + tokens.push({ + type: PrisbeamSearchTokenType.And, + data: null + }); + pos += 3; + } else { + currentTag += query[pos]; + pos++; + } + + break; + + case "N": + if (query[pos + 1] === "O" && query[pos + 2] === "T") { + if (currentTag.trim().length > 0) { + tokens.push({ + type: PrisbeamSearchTokenType.Query, + data: this.checkQuery(currentTag.trim(), allowUnknownTags) + }); + currentTag = ""; + } + + tokens.push({ + type: PrisbeamSearchTokenType.Not, + data: null + }); + pos += 3; + } else { + currentTag += query[pos]; + pos++; + } + + break; + + case " ": + currentTag += query[pos]; + pos++; + break; + + case "(": + if (currentTag.trim().length === 0) { + inParentheses = true; + currentParenthesesInner = ""; + pos++; + } else { + currentTag += query[pos]; + pos++; + } + + break; + + case ")": + if (currentTag.trim().length === 0) { + throw new SearchError("Unexpected closing parenthesis."); + } else { + currentTag += query[pos]; + pos++; + } + + break; + + default: + currentTag += query[pos]; + pos++; + break; + } + } + } + + if (currentTag.trim().length > 0) { + tokens.push({ + type: PrisbeamSearchTokenType.Query, + data: this.checkQuery(currentTag.trim(), allowUnknownTags) + }); + } + + return this.queryTokensToString(tokens); + } + + queryTokensToString(tokens: PrisbeamSearchToken[]) { + let str = ""; + + for (let token of tokens) { + switch (token.type) { + case PrisbeamSearchTokenType.And: + str += "AND"; + break; + + case PrisbeamSearchTokenType.Query: + str += token.data; + break; + + case PrisbeamSearchTokenType.Subquery: + str += "(" + token.data + ")"; + break; + + case PrisbeamSearchTokenType.Not: + str += "NOT"; + break; + + case PrisbeamSearchTokenType.Or: + str += "OR"; + break; + } + + str += " "; + } + + return str.trim(); + } + + buildQueryV2(query: string, allowUnknownTags: boolean, allowExceedLength: boolean = false) { + if (query.length >= 1024 && !allowExceedLength) throw new SearchError("A search query needs to be shorter than 1024 characters."); + query = query.trim(); + + query = this.buildQueryInner(query, allowUnknownTags); + + if (query.length > 0) { + return "(" + query.trim() + ")"; + } else { + return "TRUE"; + } + } + + /** + * @deprecated + */ + buildQuery(_1: string, _2: boolean, _3: boolean = false): never { + throw new SearchError("The Prisbeam search engine version 1 has been removed from this version of libprisbeam and cannot be used anymore. Please switch to 'buildQueryV2' instead of 'buildQuery'."); + } +} diff --git a/src/PrisbeamUpdater.ts b/src/PrisbeamUpdater.ts new file mode 100644 index 0000000..56e33f8 --- /dev/null +++ b/src/PrisbeamUpdater.ts @@ -0,0 +1,518 @@ +import {Prisbeam} from "./Prisbeam"; +import fs from "fs"; +import http from "http"; + +export class PrisbeamUpdater { + private readonly database: Prisbeam; + + constructor(instance: Prisbeam) { + this.database = instance; + } + + updateFromPreprocessed(preprocessed: string, tags: string, statusUpdateHandler: Function) { + return new Promise((res) => { + let sqlGet: Function; + let sqlQuery: Function; + let protectedEncode: Function; + let database = this.database; + + const zlib = require('zlib'); + + sqlGet = sqlQuery = async (query: string) => { + return await database._sql(query); + } + + protectedEncode = (b: string) => { + return zlib.deflateRawSync(b, {level: 9}); + } + + function sqlstr(str?: string) { + if (str === null) { + return "NULL"; + } else { + return "'" + str.replaceAll("'", "''") + "'"; + } + } + + function sleep(ms: number) { + return new Promise((res) => { + setTimeout(res, ms); + }); + } + + async function addToDB(images: object[]) { + await sqlQuery(`PRAGMA foreign_keys = OFF`); + + let tags = {}; + + for (let image of images) { + let imageTags = image['tags'].map((i: number, j: object) => { + return [image['tag_ids'][j], i]; + }); + + for (let tag of imageTags) { + tags[tag[0]] = tag[1]; + } + } + + let uploaders = {}; + + for (let image of images) { + if (image['uploader_id']) { + uploaders[image['uploader_id']] = image['uploader']; + } + } + + let index = 0; + let u = Object.entries(uploaders); + + for (let i = 0; i < u.length; i += 50) { + const chunk = u.slice(i, i + 50); + + for (let uploader of chunk) { + if ((await sqlGet(`SELECT COUNT(*) FROM uploaders WHERE id=${uploader[0]}`))[0]["COUNT(*)"]) { + await sqlQuery(`DELETE FROM uploaders WHERE id=${uploader[0]}`); + } + } + + // @ts-ignore + await sqlQuery(`INSERT INTO uploaders(id, name) VALUES ${chunk.map(uploader => `(${uploader[0]}, '${(uploader[1] ?? "").replaceAll("'", "''")}')`).join(",")}`); + + index += 50; + } + + index = 0; + + let v = images; + + for (let i = 0; i < v.length; i += 50) { + const chunk = v.slice(i, i + 50); + + for (let image of chunk) { + if ((await sqlGet(`SELECT COUNT(*) FROM images WHERE id=${image['id']}`))[0]["COUNT(*)"]) { + await sqlQuery(`DELETE FROM images WHERE id=${image['id']}`); + await sqlQuery(`DELETE FROM image_tags WHERE image_id=${image['id']}`); + await sqlQuery(`DELETE FROM image_intensities WHERE image_id=${image['id']}`); + await sqlQuery(`DELETE FROM image_representations WHERE image_id=${image['id']}`); + } + } + + await sqlQuery(`INSERT INTO images(id, source_id, source_name, source, animated, hidden_from_users, processed, spoilered, thumbnails_generated, aspect_ratio, duration, wilson_score, created_at, first_seen_at, updated_at, comment_count, downvotes, duplicate_of, faves, height, score, size, tag_count, upvotes, width, deletion_reason, description, format, mime_type, name, orig_sha512_hash, sha512_hash, source_url, uploader) VALUES ${chunk.map(image => `(${image['id']}, ${image['source_id'] ?? image['id']}, ${sqlstr(image['source_name'] ?? null)}, ${sqlstr(image['source'] ?? null)}, ${image['animated'] ? 'TRUE' : 'FALSE'}, ${image['hidden_from_users'] ? 'TRUE' : 'FALSE'}, ${image['processed'] ? 'TRUE' : 'FALSE'}, ${image['spoilered'] ? 'TRUE' : 'FALSE'}, ${image['thumbnails_generated'] ? 'TRUE' : 'FALSE'}, ${image['aspect_ratio']}, ${image['duration']}, ${image['wilson_score']}, ${new Date(image['created_at']).getTime() / 1000}, ${new Date(image['first_seen_at']).getTime() / 1000}, ${new Date(image['updated_at']).getTime() / 1000}, ${image['comment_count']}, ${image['downvotes']}, ${image['duplicate_of']}, ${image['faves']}, ${image['height']}, ${image['score']}, ${image['size']}, ${image['tag_count']}, ${image['upvotes']}, ${image['width']}, ${sqlstr(image['deletion_reason'])}, ${sqlstr(image['description'])}, ${sqlstr(image['format'])}, ${sqlstr(image['mime_type'])}, ${sqlstr(image['name'])}, ${sqlstr(image['orig_sha512_hash'])}, ${sqlstr(image['sha512_hash'])}, ${sqlstr(image['source_url'])}, ${image['uploader_id']})`).join(",")}`); + await sqlQuery(`INSERT INTO image_tags(image_id, tags) VALUES ${chunk.map(image => `(${image['id']}, ",${image['tag_ids'].join(",")},")`).join(",")}`); + await sqlQuery(`INSERT INTO image_intensities(image_id, ne, nw, se, sw) VALUES ${chunk.map(image => `(${image['id']}, ${image['intensities']?.ne ?? null}, ${image['intensities']?.nw ?? null}, ${image['intensities']?.se ?? null}, ${image['intensities']?.sw ?? null})`).join(",")}`); + await sqlQuery(`INSERT INTO image_representations(image_id, view, full, large, medium, small, tall, thumb, thumb_small, thumb_tiny) VALUES ${chunk.map(image => `(${image['id']}, ${sqlstr(image['view_url'])}, ${sqlstr(image['representations'].full)}, ${sqlstr(image['representations'].large)}, ${sqlstr(image['representations'].medium)}, ${sqlstr(image['representations'].small)}, ${sqlstr(image['representations'].tall)}, ${sqlstr(image['representations'].thumb)}, ${sqlstr(image['representations'].thumb_small)}, ${sqlstr(image['representations'].thumb_tiny)})`).join(",")}`); + + index += 50; + } + + await sqlQuery(`PRAGMA foreign_keys = ON`); + } + + async function cleanDB() { + await database.clean(); + } + + function timeToString(time: number | string | Date) { + if (!isNaN(parseInt(time as string))) { + time = new Date(time).getTime(); + } + + let periods = ["second", "minute", "hour", "day", "week", "month", "year", "age"]; + + let lengths = [60, 60, 24, 7, 4.35, 12, 100]; + + let difference = (time as number) / 1000; + let period: string; + + let j: number; + + for (j = 0; difference >= lengths[j] && j < lengths.length - 1; j++) { + difference /= lengths[j]; + } + + difference = Math.round(difference); + + period = periods[j]; + + return `${difference} ${period}${difference > 1 ? "s" : ""}`; + } + + if (typeof window !== "undefined") { + global = window; + } + + async function consolidateDB() { + await sqlQuery("DROP TABLE IF EXISTS tags_pre"); + + statusUpdateHandler([{ + title: "Saving database...", progress: 0, indeterminate: false + }]); + + if (database.cache) { + await fs.promises.copyFile(database.cache + "/work.pbdb", database.path + "/current.pbdb"); + } + + fs.unlinkSync(global.oldDBFile ?? null); + res(); + } + + let update = async () => { + const fs = require('fs'); + const path = require('path'); + + process.chdir(this.database.path); + + statusUpdateHandler([{ + title: "Backup up database...", progress: 0, indeterminate: false + }]); + + global.oldDBFile = this.database.path + "/" + new Date().getTime() + ".pbdb"; + fs.copyFileSync(this.database.path + "/current.pbdb", global.oldDBFile); + + statusUpdateHandler([{ + title: "Cleaning up database...", progress: 0, indeterminate: false + }]); + await cleanDB(); + + statusUpdateHandler([{ + title: "Preparing update...", progress: 0, indeterminate: false + }]); + + let sqlPreGet = (sql: string) => { + return new Promise((res, rej) => { + global.preDatabase.all(sql, function (err: Error | null, data: any) { + if (err) { + rej(err); + } else { + res(data); + } + }); + }); + } + + let sqlTagGet = (sql: string) => { + return new Promise((res, rej) => { + global.tagsDatabase.all(sql, function (err: Error | null, data: any) { + if (err) { + rej(err); + } else { + res(data); + } + }); + }); + } + + const sqlite3 = require(process.platform === "darwin" ? "../../sql/mac" : "../../sql/win"); + + global.preDatabase = new sqlite3.Database(preprocessed); + global.tagsDatabase = new sqlite3.Database(tags); + + await new Promise((res) => { + global.preDatabase.serialize(() => { + res(); + }); + }); + + const host = "127.0.0.1"; + const port = 19842; + const preprocessedImageCount = parseInt((await sqlPreGet("SELECT COUNT(*) FROM images"))[0]["COUNT(*)"]); + + let preprocessedServer = http.createServer(async (req: any, res: any) => { + let page = 1; + let requestPage = new URL(req.url, "https://nothing.invalid").searchParams.get('page'); + let out = { + images: [], interactions: [], total: preprocessedImageCount + } + + if (requestPage && !isNaN(parseInt(requestPage))) { + page = parseInt(requestPage); + } + + let everything = (await sqlPreGet("SELECT * FROM images LIMIT 50 OFFSET " + ((page - 1) * 50))) as any[]; + out.images = everything.map(i => JSON.parse(atob(i.json))); + + res.writeHead(200); + res.end(JSON.stringify(out)); + }); + + preprocessedServer.listen(port, host, () => { + // noinspection HttpUrlsUsage + console.log(`Server is running on http://${host}:${port}`); + }); + + let updateTags = ((await sqlTagGet("SELECT * FROM tags")) as any[]).map(i => JSON.parse(atob(i['json']))); + let index = 0; + + for (let i = 0; i < updateTags.length; i += 50) { + const chunk = updateTags.slice(i, i + 50); + + let aliases = []; + let implications = []; + + for (let tag of chunk) { + aliases.push(sqlstr(await sqlTagGet(`SELECT target FROM aliases WHERE source = ` + tag['id'])[0]?.target ?? null)); + implications.push(sqlstr("," + (await sqlTagGet(`SELECT target FROM implications WHERE source = ` + tag['id']) as any[]).map(i => i.target).join(",") + ",")); + } + + await sqlQuery(`INSERT INTO tags(id, name, alias, implications, category, description, description_short, slug) VALUES ${chunk.map((tag, index) => `(${tag['id']}, ${sqlstr(tag['name'])}, ${aliases[index]}, ${implications[index]}, ${sqlstr(tag['category'])}, ${sqlstr(tag['description'])}, ${sqlstr(tag['short_description'])}, ${sqlstr(tag['slug'])})`).join(",")}`); + index += 50; + + statusUpdateHandler([{ + title: "Preparing update... " + Math.round((index / updateTags.length) * 100) + "%", + progress: ((index / updateTags.length) * 100), + indeterminate: false + }]); + } + + statusUpdateHandler([{ + title: "Preparing update...", progress: 0, indeterminate: false + }]); + + global.statusInfo = []; + + function prettySize(s: number) { + if (s < 1024) { + return s.toFixed(0) + " B"; + } else if (s < 1024 ** 2) { + return (s / 1024).toFixed(0) + " KiB"; + } else if (s < 1024 ** 3) { + return (s / 1024 ** 2).toFixed(1) + " MiB"; + } else if (s < 1024 ** 4) { + return (s / 1024 ** 3).toFixed(2) + " GiB"; + } else { + return (s / 1024 ** 4).toFixed(2) + " TiB"; + } + } + + function doImageFileDownload(url: string, image: object) { + global.statusInfo[2] = { + title: "Image: " + image['id'], progress: 0, indeterminate: false + } + + statusUpdateHandler(global.statusInfo); + + const axios = require('axios'); + + return new Promise(async (res) => { + // noinspection JSUnusedGlobalSymbols + const response = await axios({ + url, + validateStatus: (s: number) => (s >= 200 && s < 300) || (s === 404 || s === 403 || s === 401), + method: 'GET', + responseType: 'arraybuffer', + onDownloadProgress: (event: any) => { + global.statusInfo[2] = { + title: "Image: " + image['id'] + " (" + Math.round((event.loaded / event.total) * 100) + "%)", + progress: ((event.loaded / event.total) * 100), + indeterminate: false + } + + statusUpdateHandler(global.statusInfo); + } + } as object); + + global.statusInfo[2] = null; + res(response.data); + }); + } + + global.doneFetching = false; + + if (!fs.existsSync("./thumbnails")) fs.mkdirSync("./thumbnails"); + if (!fs.existsSync("./images")) fs.mkdirSync("./images"); + + let total1: number; + let pages1: number; + let types = []; + + async function doRequest() { + try { + return (await (await fetch("http://127.0.0.1:19842")).json())['total']; + } catch (e) { + console.error(e); + return doRequest(); + } + } + + total1 = await doRequest(); + pages1 = Math.ceil(total1 / 50); + + types.push({ + query: "my:faves", name: "faved", pages: pages1, total: total1 + }); + + let prelists = {}; + global.totalPrelist = []; + global.totalPrelistFull = []; + global.times = []; + global.times2 = []; + global.totalPages = types.map(i => i['pages']).reduce((a, b) => a + b); + global.totalImages = types.map(i => i['total']).reduce((a, b) => a + b); + global.totalPageNumber = 0; + global.images = 0; + global.updateCategories = {}; + + console.log(global.totalImages + " images to download from sources, part of " + global.totalPages + " pages"); + + for (let type of types) { + let prelist = []; + let pages = type.pages; + + for (let pageNumber = 1; pageNumber <= pages; pageNumber++) { + if (global.doneFetching || !preprocessedServer) break; + + let start = new Date().getTime(); + let tryFetch = true; + let page: object; + + while (tryFetch) { + try { + page = await (await fetch("http://127.0.0.1:19842/?page=" + pageNumber)).json(); + + page['images'] = page['images'].map((image: object) => { + if (image['representations']['thumb'].endsWith(".mp4") || image['representations']['thumb'].endsWith(".webm")) { + image['representations']['thumb'] = image['representations']['thumb'].substring(0, image['representations']['thumb'].length - path.extname(image['representations']['thumb']).length) + ".gif"; + } + + return image; + }); + + tryFetch = false; + } catch (e) { + console.error(e); + } + } + + await addToDB(page['images']); + + for (let image of page['images']) { + if (!global.updateCategories[image.id]) global.updateCategories[image.id] = { + upvotes: false, downvotes: false, watched: false, faved: false, uploads: false + } + + global.updateCategories[image.id][type.name] = true; + } + + prelist.push(...page['images']); + global.totalPrelist.push(...page['images']); + global.totalPrelistFull.push(...page['images']); + global.times.push(new Date().getTime() - start); + + let fetchEta = (global.totalPages - global.totalPageNumber) * (global.times.reduce((a: number, b: number) => a + b, 0) / global.times.length); + + if (global.totalPageNumber === global.totalPages) { + global.doneFetching = true; + } + + global.statusInfo[0] = { + title: "Fetching: " + Math.round((global.totalPageNumber / global.totalPages) * 100) + "% (" + global.totalPageNumber + "/" + global.totalPages + ") complete" + (global.times.length > 10 ? ", " + timeToString(fetchEta) : "") + " (" + global.totalPrelistFull.length + ")", + progress: ((global.totalPageNumber / global.totalPages) * 100), + indeterminate: false + } + + statusUpdateHandler(global.statusInfo); + global.totalPageNumber++; + } + + prelists[type.name] = prelist; + } + + global.doneFetching = true; + global.totalFetchingSize = global.totalPrelistFull.map((i: object) => i['size']).reduce((a: number, b: number) => a + b); + + try { + preprocessedServer.closeAllConnections(); + preprocessedServer.close(); + preprocessedServer = null; + } catch (e) { + console.error(e); + } + + while (global.totalPrelist.length > 0) { + let image = global.totalPrelist.shift(); + global.lastImage = image; + let start = new Date().getTime(); + let downloaded = false; + + let path1 = (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 1); + let path2 = (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 2); + let path3 = (image['sha512_hash'] ?? image['orig_sha512_hash'] ?? "0000000").substring(0, 3); + + if (!fs.existsSync("./images/" + path1)) fs.mkdirSync("./images/" + path1); + if (!fs.existsSync("./images/" + path1 + "/" + path2)) fs.mkdirSync("./images/" + path1 + "/" + path2); + if (!fs.existsSync("./images/" + path1 + "/" + path2 + "/" + path3)) fs.mkdirSync("./images/" + path1 + "/" + path2 + "/" + path3); + if (!fs.existsSync("./thumbnails/" + path1)) fs.mkdirSync("./thumbnails/" + path1); + if (!fs.existsSync("./thumbnails/" + path1 + "/" + path2)) fs.mkdirSync("./thumbnails/" + path1 + "/" + path2); + if (!fs.existsSync("./thumbnails/" + path1 + "/" + path2 + "/" + path3)) fs.mkdirSync("./thumbnails/" + path1 + "/" + path2 + "/" + path3); + + let fileName = path1 + "/" + path2 + "/" + path3 + "/" + image['id'] + ".bin"; + let fileName2 = path1 + "/" + path2 + "/" + path3 + "/" + image['id'] + ".bin"; + + let fileNamePre = path1 + "/" + path2 + "/" + path3 + "/." + image['id'] + ".bin"; + let fileNamePre2 = path1 + "/" + path2 + "/" + path3 + "/." + image['id'] + ".bin"; + + if (!fs.existsSync("./images/" + fileName) || !fs.existsSync("./thumbnails/" + fileName2)) { + if (fs.existsSync("./images/" + path1 + "/" + path2 + "/" + path3 + "/." + image['id'] + path.extname(image['view_url']))) fs.unlinkSync("./images/" + path1 + "/" + path2 + "/" + path3 + "/." + image['id'] + path.extname(image['view_url'])); + if (fs.existsSync("./thumbnails/" + path1 + "/" + path2 + "/" + path3 + "/." + image['id'] + path.extname(image['representations']['thumb']))) fs.unlinkSync("./thumbnails/" + path1 + "/" + path2 + "/" + path3 + "/." + image['id'] + path.extname(image['representations']['thumb'])); + if (fs.existsSync("./images/" + path1 + "/" + path2 + "/" + path3 + "/." + image['id'] + ".bin")) fs.unlinkSync("./images/" + path1 + "/" + path2 + "/" + path3 + "/." + image['id'] + ".bin"); + if (fs.existsSync("./thumbnails/" + path1 + "/" + path2 + "/" + path3 + "/." + image['id'] + ".bin")) fs.unlinkSync("./thumbnails/" + path1 + "/" + path2 + "/" + path3 + "/." + image['id'] + ".bin"); + if (fs.existsSync("./images/" + path1 + "/" + path2 + "/." + image['id'] + path.extname(image['view_url']))) fs.unlinkSync("./images/" + path1 + "/" + path2 + "/" + path3 + "/." + image['id'] + path.extname(image['view_url'])); + if (fs.existsSync("./thumbnails/" + path1 + "/" + path2 + "/." + image['id'] + path.extname(image['representations']['thumb']))) fs.unlinkSync("./thumbnails/" + path1 + "/" + path2 + "/." + image['id'] + path.extname(image['representations']['thumb'])); + if (fs.existsSync("./images/" + path1 + "/" + path2 + "/." + image['id'] + ".bin")) fs.unlinkSync("./images/" + path1 + "/" + path2 + "/." + image['id'] + ".bin"); + if (fs.existsSync("./thumbnails/" + path1 + "/" + path2 + "/." + image['id'] + ".bin")) fs.unlinkSync("./thumbnails/" + path1 + "/" + path2 + "/." + image['id'] + ".bin"); + + let tryFetch = true; + + while (tryFetch) { + try { + fs.writeFileSync("./images/" + fileNamePre, protectedEncode(await doImageFileDownload(image['view_url'], image))); + tryFetch = false; + } catch (e) { + console.error(e); + await sleep(1500); + } + } + + tryFetch = true; + + while (tryFetch) { + try { + fs.writeFileSync("./thumbnails/" + fileNamePre2, protectedEncode(await doImageFileDownload(image['representations']['thumb'], image))); + tryFetch = false; + } catch (e) { + console.error(e); + await sleep(1500); + } + } + + fs.renameSync("./thumbnails/" + fileNamePre2, "./thumbnails/" + fileName2); + fs.renameSync("./images/" + fileNamePre, "./images/" + fileName); + downloaded = true; + } + + if (downloaded) global.times2.push([new Date().getTime() - start, image['size']]); + global.totalPrelist.shift(); + global.images++; + + let eta = (global.totalImages - global.times2.length) * (global.times2.map((i: number[]) => i[0]).reduce((a: number, b: number) => a + b, 0) / global.times2.length); + let times = global.times2.map((i: number[]) => i[0] / i[1]).slice(0, 20); + let averageBps = times.length > 0 ? (times.reduce((a: number, b: number) => a + b) / times.length) : 0; + let dataLeft = global.totalFetchingSize - global.times2.map((i: number[]) => i[1]).reduce((a: number, b: number) => a + b, 0); + eta = dataLeft * averageBps; + + let title = "Downloading: " + Math.round((global.images / global.totalImages) * 100) + "% (" + global.images + "/" + global.totalImages + ", " + prettySize(global.times2.map((i: number[]) => i[1]).reduce((a: number, b: number) => a + b, 0)) + (global.totalFetchingSize ? "/" + prettySize(global.totalFetchingSize) : "") + ") complete" + (global.times2.length > 10 && eta > 1000 ? ", " + timeToString(eta) : ""); + + global.statusInfo[0] = { + title, progress: ((global.images / global.totalImages) * 100), indeterminate: false + } + + statusUpdateHandler(global.statusInfo); + } + + await consolidateDB(); + } + + update(); + }); + } +} diff --git a/src/SearchError.ts b/src/SearchError.ts new file mode 100644 index 0000000..3ee2831 --- /dev/null +++ b/src/SearchError.ts @@ -0,0 +1,6 @@ +export class SearchError extends Error { + constructor(message: string) { + super(message); + this.name = "SearchError"; + } +} diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..99ccdd6 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,11 @@ +{ + "compilerOptions": { + "module": "commonjs", + "target": "es2021", + "sourceMap": true, + "esModuleInterop": true + }, + "exclude": [ + "node_modules" + ], +} \ No newline at end of file -- cgit