4 files changed, 73 insertions, 41 deletions

diff --git a/src/api/e621/index.ts b/src/api/e621/index.ts
index 5c5fd2a..a8abbcf 100644
--- a/src/api/e621/index.ts
+++ b/src/api/e621/index.ts
@@ -1,8 +1,7 @@
 import got from "got";
 import config from "../../config";
-import fs from "fs/promises";
-import path from "path";
 import delay from "../../util/delay";
+import dedupe from "../../services/dedupe";
 
 export interface GetPostQuery {
     tags: readonly string[];
@@ -34,35 +33,6 @@ export const client = got.extend({
     },
 });
 
-const dedupePath = path.join(__dirname, "e926dedupe.json");
-const dedupeMax = 50;
-let dedupeDb: number[] | undefined;
-
-async function loadDedupeDb() {
-    if (dedupeDb) {
-        return;
-    }
-
-    try {
-        await fs.stat(dedupePath);
-    } catch {
-        await saveDedupeDb();
-    }
-
-    const d = await fs.readFile(dedupePath, "utf8");
-    const dd = JSON.parse(d);
-
-    if (dd instanceof Array) {
-        dedupeDb = dd.slice(-1 * dedupeMax);
-    } else {
-        dedupeDb = [];
-    }
-}
-
-async function saveDedupeDb() {
-    await fs.writeFile(dedupePath, JSON.stringify(dedupeDb ?? []), "utf8");
-}
-
 export async function getPostById(id: number) {
     const response = await client
         .get("https://e926.net/posts.json", {
@@ -80,8 +50,6 @@ export async function getPostById(id: number) {
 }
 
 export async function getRandomPost(query: GetPostQuery) {
-    await loadDedupeDb();
-
     const page = Math.floor(Math.random() * (query.maxPage - 1)) + 1;
 
     const response = await client
@@ -101,13 +69,12 @@ export async function getRandomPost(query: GetPostQuery) {
     const postIndex = Math.floor(Math.random() * response.posts.length);
     const post = response.posts[postIndex];
 
-    if (dedupeDb.includes(post.id)) {
+    const isDupe = await dedupe.check({ provider: "e926", id: post.id });
+
+    if (isDupe) {
         await delay(1000);
         return getRandomPost(query);
     }
 
-    dedupeDb.push(post.id);
-    await saveDedupeDb();
-
     return post;
 }
diff --git a/src/index.ts b/src/index.ts
index 6e9ad2e..5558540 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -1,5 +1,5 @@
 import config from "./config";
-import * as jobs from "./jobs";
+import * as jobs from "./services/jobs";
 import * as cliArgs from "ts-command-line-args";
 
 const args = cliArgs.parse<{
diff --git a/src/services/dedupe.ts b/src/services/dedupe.ts
new file mode 100644
index 0000000..2eeb5ee
--- /dev/null
+++ b/src/services/dedupe.ts
@@ -0,0 +1,65 @@
+import fs from "fs/promises";
+import path from "path";
+import * as f from "fp-ts";
+import * as t from "io-ts";
+
+export const E926DedupeEntryC = t.type({
+    provider: t.literal("e926"),
+    id: t.number,
+});
+
+export const DedupeEntryC = E926DedupeEntryC;
+
+export type E926DedupeEntry = t.TypeOf<typeof E926DedupeEntryC>;
+
+export type DedupeEntry = t.TypeOf<typeof DedupeEntryC>;
+
+export class Dedupe {
+    private entries: DedupeEntry[] = [];
+
+    private readonly filePath: string;
+
+    private isLoaded = false;
+
+    constructor(private max: number, filename: string) {
+        this.filePath = path.join(process.cwd(), filename);
+    }
+
+    private async load() {
+        if (this.isLoaded) {
+            return;
+        }
+
+        try {
+            await fs.stat(this.filePath);
+        } catch {
+            await this.save();
+        }
+
+        const fileContent = await fs.readFile(this.filePath, "utf8");
+        const entries = t.array(DedupeEntryC).decode(fileContent);
+
+        if (f.either.isRight(entries)) {
+            this.entries = entries.right;
+        }
+    }
+
+    private async save() {
+        await fs.writeFile(this.filePath, JSON.stringify(this.entries ?? []), "utf8");
+    }
+
+    async check(entry: DedupeEntry) {
+        await this.load();
+
+        const has = !!this.entries.find((e) => e.provider === entry.provider && e.id === entry.id);
+
+        if (!has) {
+            this.entries.push(entry);
+            await this.save();
+        }
+
+        return has;
+    }
+}
+
+export default new Dedupe(50, "dedupe.json");
diff --git a/src/jobs.ts b/src/services/jobs.ts
index d77104f..cf3b894 100644
--- a/src/jobs.ts
+++ b/src/services/jobs.ts
@@ -1,6 +1,6 @@
-import * as e621 from "./api/e621";
-import * as mastodon from "./api/mastodon";
-import config from "./config";
+import * as e621 from "../api/e621";
+import * as mastodon from "../api/mastodon";
+import config from "../config";
 import Sharp from "sharp";
 
 export async function postRandomPicture() {