1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
|
const PATH = require('fs').readFileSync(require('os').homedir() + "/.faunerie_path").toString().trim();
const sqlite3 = require('sqlite3').verbose();
const db = new sqlite3.Database(PATH, sqlite3.OPEN_READONLY);
const tags = require('./parsed_tags.json');
const fs = require('fs');
db.serialize(async () => {
function query(q) {
return new Promise((res, rej) => {
db.all(q, function (err, data) {
if (err) {
rej(err);
} else {
res(data);
}
});
});
}
function sqlstr(str) {
if (str === null) {
return "NULL";
} else {
return "'" + str.replaceAll("'", "''") + "'";
}
}
let i = 0;
let success = 0;
let total = 0;
let covered = 0;
let successPerCategory = [0, 0, null, 0, 0, 0, 0, 0, 0];
let tagsList = Object.entries(tags);
let done = 0;
let coverage = 0;
let matched = 0;
for (let _tag of tagsList) {
let name = _tag[0];
let tag = _tag[1];
let condition = tag.allowedNames.slice(0, 100).map(i => [i.replaceAll("_", " "), i.replaceAll("_", "+")]).reduce((a, b) => [...a, ...b]).map(i => "name = " + sqlstr(i) + " OR slug = " + sqlstr(i)).join(" OR ");
let matches = await query("SELECT * FROM tags WHERE name = " + sqlstr(name.replaceAll("_", " ")) + " OR slug = " + sqlstr(name.replaceAll("_", "+")));
let matchesAlias = await query("SELECT * FROM tags WHERE " + condition);
let match = null;
if (matches.length > 0) {
match = matches[0];
} else if (matchesAlias.length > 0) {
match = matchesAlias[0];
}
i++;
total += tag.usage;
done = (i / tagsList.length) * 100;
coverage = (covered / total) * 100;
matched = (success / i) * 100;
let txt = done.toFixed(2) + "% done - " + coverage.toFixed(2) + "% coverage - " + matched.toFixed(2) + "% matched (" + successPerCategory.map(j => ((j / i) * 100).toFixed(1) + "%").join(", ") + ") - Current: ";
if (match) {
tag.derpibooruMatch = [parseInt(match.id.toString().substring(2)), match.name]
txt += tag.id + " -> " + tag.derpibooruMatch;
success++;
successPerCategory[tag.category]++;
covered += tag.usage;
} else {
txt += tag.id + " -> ???";
}
txt = txt.substring(0, process.stdout.columns - 1);
process.stdout.write(txt + " ".repeat(process.stdout.columns - 1 - txt.length));
process.stdout.cursorTo(0);
}
process.stdout.clearLine(null);
process.stdout.write("Saving to disk...");
fs.writeFileSync("parsed_tags.json", JSON.stringify(tags, null, 2));
process.stdout.cursorTo(0);
console.log("Matching operation completed.");
console.log("Here is a breakdown:");
console.log(" * " + success + " tags out of " + i + " (" + matched.toFixed(3) + "%) could be matched successfully.")
console.log(successPerCategory.map((j, k) => " * " + ((j / i) * 100).toFixed(3) + "% from category " + k).join("\n"));
console.log(" * This means that the matched tags cover " + coverage.toFixed(3) + "% of posts on e621, roughly 1 in " + (100 / coverage).toFixed(1) + ".")
});
|
