summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorVolpeon <git@volpeon.ink>2021-10-19 18:14:00 +0200
committerVolpeon <git@volpeon.ink>2021-10-19 18:14:00 +0200
commita1de58f364ef04c97ecad380427e6181f3bf707d (patch)
treeba3c8df7a209081b858b423674e5c42ffec37ebe
parentCode improvements (diff)
downloadferalbot-a1de58f364ef04c97ecad380427e6181f3bf707d.tar.gz
feralbot-a1de58f364ef04c97ecad380427e6181f3bf707d.tar.bz2
feralbot-a1de58f364ef04c97ecad380427e6181f3bf707d.zip
Code improvements
-rw-r--r--package.json2
-rw-r--r--src/api/e621/index.ts41
-rw-r--r--src/index.ts2
-rw-r--r--src/services/dedupe.ts65
-rw-r--r--src/services/jobs.ts (renamed from src/jobs.ts)6
-rw-r--r--tsconfig.json1
-rw-r--r--yarn.lock10
7 files changed, 85 insertions, 42 deletions
diff --git a/package.json b/package.json
index b6632bb..8cf504a 100644
--- a/package.json
+++ b/package.json
@@ -12,7 +12,9 @@
12 "@types/sharp": "^0.29.2", 12 "@types/sharp": "^0.29.2",
13 "file-type": "^16.5.3", 13 "file-type": "^16.5.3",
14 "form-data": "^4.0.0", 14 "form-data": "^4.0.0",
15 "fp-ts": "^2.11.5",
15 "got": "^11.8.2", 16 "got": "^11.8.2",
17 "io-ts": "^2.2.16",
16 "nanoid": "^3.1.30", 18 "nanoid": "^3.1.30",
17 "sharp": "^0.29.1", 19 "sharp": "^0.29.1",
18 "ts-command-line-args": "^2.1.0", 20 "ts-command-line-args": "^2.1.0",
diff --git a/src/api/e621/index.ts b/src/api/e621/index.ts
index 5c5fd2a..a8abbcf 100644
--- a/src/api/e621/index.ts
+++ b/src/api/e621/index.ts
@@ -1,8 +1,7 @@
1import got from "got"; 1import got from "got";
2import config from "../../config"; 2import config from "../../config";
3import fs from "fs/promises";
4import path from "path";
5import delay from "../../util/delay"; 3import delay from "../../util/delay";
4import dedupe from "../../services/dedupe";
6 5
7export interface GetPostQuery { 6export interface GetPostQuery {
8 tags: readonly string[]; 7 tags: readonly string[];
@@ -34,35 +33,6 @@ export const client = got.extend({
34 }, 33 },
35}); 34});
36 35
37const dedupePath = path.join(__dirname, "e926dedupe.json");
38const dedupeMax = 50;
39let dedupeDb: number[] | undefined;
40
41async function loadDedupeDb() {
42 if (dedupeDb) {
43 return;
44 }
45
46 try {
47 await fs.stat(dedupePath);
48 } catch {
49 await saveDedupeDb();
50 }
51
52 const d = await fs.readFile(dedupePath, "utf8");
53 const dd = JSON.parse(d);
54
55 if (dd instanceof Array) {
56 dedupeDb = dd.slice(-1 * dedupeMax);
57 } else {
58 dedupeDb = [];
59 }
60}
61
62async function saveDedupeDb() {
63 await fs.writeFile(dedupePath, JSON.stringify(dedupeDb ?? []), "utf8");
64}
65
66export async function getPostById(id: number) { 36export async function getPostById(id: number) {
67 const response = await client 37 const response = await client
68 .get("https://e926.net/posts.json", { 38 .get("https://e926.net/posts.json", {
@@ -80,8 +50,6 @@ export async function getPostById(id: number) {
80} 50}
81 51
82export async function getRandomPost(query: GetPostQuery) { 52export async function getRandomPost(query: GetPostQuery) {
83 await loadDedupeDb();
84
85 const page = Math.floor(Math.random() * (query.maxPage - 1)) + 1; 53 const page = Math.floor(Math.random() * (query.maxPage - 1)) + 1;
86 54
87 const response = await client 55 const response = await client
@@ -101,13 +69,12 @@ export async function getRandomPost(query: GetPostQuery) {
101 const postIndex = Math.floor(Math.random() * response.posts.length); 69 const postIndex = Math.floor(Math.random() * response.posts.length);
102 const post = response.posts[postIndex]; 70 const post = response.posts[postIndex];
103 71
104 if (dedupeDb.includes(post.id)) { 72 const isDupe = await dedupe.check({ provider: "e926", id: post.id });
73
74 if (isDupe) {
105 await delay(1000); 75 await delay(1000);
106 return getRandomPost(query); 76 return getRandomPost(query);
107 } 77 }
108 78
109 dedupeDb.push(post.id);
110 await saveDedupeDb();
111
112 return post; 79 return post;
113} 80}
diff --git a/src/index.ts b/src/index.ts
index 6e9ad2e..5558540 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -1,5 +1,5 @@
1import config from "./config"; 1import config from "./config";
2import * as jobs from "./jobs"; 2import * as jobs from "./services/jobs";
3import * as cliArgs from "ts-command-line-args"; 3import * as cliArgs from "ts-command-line-args";
4 4
5const args = cliArgs.parse<{ 5const args = cliArgs.parse<{
diff --git a/src/services/dedupe.ts b/src/services/dedupe.ts
new file mode 100644
index 0000000..2eeb5ee
--- /dev/null
+++ b/src/services/dedupe.ts
@@ -0,0 +1,65 @@
1import fs from "fs/promises";
2import path from "path";
3import * as f from "fp-ts";
4import * as t from "io-ts";
5
6export const E926DedupeEntryC = t.type({
7 provider: t.literal("e926"),
8 id: t.number,
9});
10
11export const DedupeEntryC = E926DedupeEntryC;
12
13export type E926DedupeEntry = t.TypeOf<typeof E926DedupeEntryC>;
14
15export type DedupeEntry = t.TypeOf<typeof DedupeEntryC>;
16
17export class Dedupe {
18 private entries: DedupeEntry[] = [];
19
20 private readonly filePath: string;
21
22 private isLoaded = false;
23
24 constructor(private max: number, filename: string) {
25 this.filePath = path.join(process.cwd(), filename);
26 }
27
28 private async load() {
29 if (this.isLoaded) {
30 return;
31 }
32
33 try {
34 await fs.stat(this.filePath);
35 } catch {
36 await this.save();
37 }
38
39 const fileContent = await fs.readFile(this.filePath, "utf8");
40 const entries = t.array(DedupeEntryC).decode(fileContent);
41
42 if (f.either.isRight(entries)) {
43 this.entries = entries.right;
44 }
45 }
46
47 private async save() {
48 await fs.writeFile(this.filePath, JSON.stringify(this.entries ?? []), "utf8");
49 }
50
51 async check(entry: DedupeEntry) {
52 await this.load();
53
54 const has = !!this.entries.find((e) => e.provider === entry.provider && e.id === entry.id);
55
56 if (!has) {
57 this.entries.push(entry);
58 await this.save();
59 }
60
61 return has;
62 }
63}
64
65export default new Dedupe(50, "dedupe.json");
diff --git a/src/jobs.ts b/src/services/jobs.ts
index d77104f..cf3b894 100644
--- a/src/jobs.ts
+++ b/src/services/jobs.ts
@@ -1,6 +1,6 @@
1import * as e621 from "./api/e621"; 1import * as e621 from "../api/e621";
2import * as mastodon from "./api/mastodon"; 2import * as mastodon from "../api/mastodon";
3import config from "./config"; 3import config from "../config";
4import Sharp from "sharp"; 4import Sharp from "sharp";
5 5
6export async function postRandomPicture() { 6export async function postRandomPicture() {
diff --git a/tsconfig.json b/tsconfig.json
index ad89216..10b9c28 100644
--- a/tsconfig.json
+++ b/tsconfig.json
@@ -1,7 +1,6 @@
1{ 1{
2 "compilerOptions": { 2 "compilerOptions": {
3 "lib": ["es2020"], 3 "lib": ["es2020"],
4 "module": "ESNext",
5 "moduleResolution": "node", 4 "moduleResolution": "node",
6 "target": "ES2020", 5 "target": "ES2020",
7 "esModuleInterop": true, 6 "esModuleInterop": true,
diff --git a/yarn.lock b/yarn.lock
index 17b1093..9ea15d5 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -406,6 +406,11 @@ form-data@^4.0.0:
406 combined-stream "^1.0.8" 406 combined-stream "^1.0.8"
407 mime-types "^2.1.12" 407 mime-types "^2.1.12"
408 408
409fp-ts@^2.11.5:
410 version "2.11.5"
411 resolved "https://registry.yarnpkg.com/fp-ts/-/fp-ts-2.11.5.tgz#97cceb26655b1452d7088d6fb0864f84cceffbe4"
412 integrity sha512-OqlwJq1BdpB83BZXTqI+dNcA6uYk6qk4u9Cgnt64Y+XS7dwdbp/mobx8S2KXf2AXH+scNmA/UVK3SEFHR3vHZA==
413
409fs-constants@^1.0.0: 414fs-constants@^1.0.0:
410 version "1.0.0" 415 version "1.0.0"
411 resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad" 416 resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad"
@@ -497,6 +502,11 @@ ini@~1.3.0:
497 resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.8.tgz#a29da425b48806f34767a4efce397269af28432c" 502 resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.8.tgz#a29da425b48806f34767a4efce397269af28432c"
498 integrity sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew== 503 integrity sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==
499 504
505io-ts@^2.2.16:
506 version "2.2.16"
507 resolved "https://registry.yarnpkg.com/io-ts/-/io-ts-2.2.16.tgz#597dffa03db1913fc318c9c6df6931cb4ed808b2"
508 integrity sha512-y5TTSa6VP6le0hhmIyN0dqEXkrZeJLeC5KApJq6VLci3UEKF80lZ+KuoUs02RhBxNWlrqSNxzfI7otLX1Euv8Q==
509
500is-arrayish@^0.3.1: 510is-arrayish@^0.3.1:
501 version "0.3.2" 511 version "0.3.2"
502 resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.3.2.tgz#4574a2ae56f7ab206896fb431eaeed066fdf8f03" 512 resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.3.2.tgz#4574a2ae56f7ab206896fb431eaeed066fdf8f03"