7 files changed, 85 insertions, 42 deletions

diff --git a/package.json b/package.json
index b6632bb..8cf504a 100644
--- a/package.json
+++ b/package.json
@@ -12,7 +12,9 @@
     "@types/sharp": "^0.29.2",
     "file-type": "^16.5.3",
     "form-data": "^4.0.0",
+    "fp-ts": "^2.11.5",
     "got": "^11.8.2",
+    "io-ts": "^2.2.16",
     "nanoid": "^3.1.30",
     "sharp": "^0.29.1",
     "ts-command-line-args": "^2.1.0",
diff --git a/src/api/e621/index.ts b/src/api/e621/index.ts
index 5c5fd2a..a8abbcf 100644
--- a/src/api/e621/index.ts
+++ b/src/api/e621/index.ts
@@ -1,8 +1,7 @@
 import got from "got";
 import config from "../../config";
-import fs from "fs/promises";
-import path from "path";
 import delay from "../../util/delay";
+import dedupe from "../../services/dedupe";
 
 export interface GetPostQuery {
     tags: readonly string[];
@@ -34,35 +33,6 @@ export const client = got.extend({
     },
 });
 
-const dedupePath = path.join(__dirname, "e926dedupe.json");
-const dedupeMax = 50;
-let dedupeDb: number[] | undefined;
-
-async function loadDedupeDb() {
-    if (dedupeDb) {
-        return;
-    }
-
-    try {
-        await fs.stat(dedupePath);
-    } catch {
-        await saveDedupeDb();
-    }
-
-    const d = await fs.readFile(dedupePath, "utf8");
-    const dd = JSON.parse(d);
-
-    if (dd instanceof Array) {
-        dedupeDb = dd.slice(-1 * dedupeMax);
-    } else {
-        dedupeDb = [];
-    }
-}
-
-async function saveDedupeDb() {
-    await fs.writeFile(dedupePath, JSON.stringify(dedupeDb ?? []), "utf8");
-}
-
 export async function getPostById(id: number) {
     const response = await client
         .get("https://e926.net/posts.json", {
@@ -80,8 +50,6 @@ export async function getPostById(id: number) {
 }
 
 export async function getRandomPost(query: GetPostQuery) {
-    await loadDedupeDb();
-
     const page = Math.floor(Math.random() * (query.maxPage - 1)) + 1;
 
     const response = await client
@@ -101,13 +69,12 @@ export async function getRandomPost(query: GetPostQuery) {
     const postIndex = Math.floor(Math.random() * response.posts.length);
     const post = response.posts[postIndex];
 
-    if (dedupeDb.includes(post.id)) {
+    const isDupe = await dedupe.check({ provider: "e926", id: post.id });
+
+    if (isDupe) {
         await delay(1000);
         return getRandomPost(query);
     }
 
-    dedupeDb.push(post.id);
-    await saveDedupeDb();
-
     return post;
 }
diff --git a/src/index.ts b/src/index.ts
index 6e9ad2e..5558540 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -1,5 +1,5 @@
 import config from "./config";
-import * as jobs from "./jobs";
+import * as jobs from "./services/jobs";
 import * as cliArgs from "ts-command-line-args";
 
 const args = cliArgs.parse<{
diff --git a/src/services/dedupe.ts b/src/services/dedupe.ts
new file mode 100644
index 0000000..2eeb5ee
--- /dev/null
+++ b/src/services/dedupe.ts
@@ -0,0 +1,65 @@
+import fs from "fs/promises";
+import path from "path";
+import * as f from "fp-ts";
+import * as t from "io-ts";
+
+export const E926DedupeEntryC = t.type({
+    provider: t.literal("e926"),
+    id: t.number,
+});
+
+export const DedupeEntryC = E926DedupeEntryC;
+
+export type E926DedupeEntry = t.TypeOf<typeof E926DedupeEntryC>;
+
+export type DedupeEntry = t.TypeOf<typeof DedupeEntryC>;
+
+export class Dedupe {
+    private entries: DedupeEntry[] = [];
+
+    private readonly filePath: string;
+
+    private isLoaded = false;
+
+    constructor(private max: number, filename: string) {
+        this.filePath = path.join(process.cwd(), filename);
+    }
+
+    private async load() {
+        if (this.isLoaded) {
+            return;
+        }
+
+        try {
+            await fs.stat(this.filePath);
+        } catch {
+            await this.save();
+        }
+
+        const fileContent = await fs.readFile(this.filePath, "utf8");
+        const entries = t.array(DedupeEntryC).decode(fileContent);
+
+        if (f.either.isRight(entries)) {
+            this.entries = entries.right;
+        }
+    }
+
+    private async save() {
+        await fs.writeFile(this.filePath, JSON.stringify(this.entries ?? []), "utf8");
+    }
+
+    async check(entry: DedupeEntry) {
+        await this.load();
+
+        const has = !!this.entries.find((e) => e.provider === entry.provider && e.id === entry.id);
+
+        if (!has) {
+            this.entries.push(entry);
+            await this.save();
+        }
+
+        return has;
+    }
+}
+
+export default new Dedupe(50, "dedupe.json");
diff --git a/src/jobs.ts b/src/services/jobs.ts
index d77104f..cf3b894 100644
--- a/src/jobs.ts
+++ b/src/services/jobs.ts
@@ -1,6 +1,6 @@
-import * as e621 from "./api/e621";
-import * as mastodon from "./api/mastodon";
-import config from "./config";
+import * as e621 from "../api/e621";
+import * as mastodon from "../api/mastodon";
+import config from "../config";
 import Sharp from "sharp";
 
 export async function postRandomPicture() {
diff --git a/tsconfig.json b/tsconfig.json
index ad89216..10b9c28 100644
--- a/tsconfig.json
+++ b/tsconfig.json
@@ -1,7 +1,6 @@
 {
   "compilerOptions": {
     "lib": ["es2020"],
-    "module": "ESNext",
     "moduleResolution": "node",
     "target": "ES2020",
     "esModuleInterop": true,
diff --git a/yarn.lock b/yarn.lock
index 17b1093..9ea15d5 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -406,6 +406,11 @@ form-data@^4.0.0:
     combined-stream "^1.0.8"
     mime-types "^2.1.12"
 
+fp-ts@^2.11.5:
+  version "2.11.5"
+  resolved "https://registry.yarnpkg.com/fp-ts/-/fp-ts-2.11.5.tgz#97cceb26655b1452d7088d6fb0864f84cceffbe4"
+  integrity sha512-OqlwJq1BdpB83BZXTqI+dNcA6uYk6qk4u9Cgnt64Y+XS7dwdbp/mobx8S2KXf2AXH+scNmA/UVK3SEFHR3vHZA==
+
 fs-constants@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad"
@@ -497,6 +502,11 @@ ini@~1.3.0:
   resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.8.tgz#a29da425b48806f34767a4efce397269af28432c"
   integrity sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==
 
+io-ts@^2.2.16:
+  version "2.2.16"
+  resolved "https://registry.yarnpkg.com/io-ts/-/io-ts-2.2.16.tgz#597dffa03db1913fc318c9c6df6931cb4ed808b2"
+  integrity sha512-y5TTSa6VP6le0hhmIyN0dqEXkrZeJLeC5KApJq6VLci3UEKF80lZ+KuoUs02RhBxNWlrqSNxzfI7otLX1Euv8Q==
+
 is-arrayish@^0.3.1:
   version "0.3.2"
   resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.3.2.tgz#4574a2ae56f7ab206896fb431eaeed066fdf8f03"