1 files changed, 65 insertions, 0 deletions

diff --git a/src/services/dedupe.ts b/src/services/dedupe.ts
new file mode 100644
index 0000000..2eeb5ee
--- /dev/null
+++ b/src/services/dedupe.ts
@@ -0,0 +1,65 @@
+import fs from "fs/promises";
+import path from "path";
+import * as f from "fp-ts";
+import * as t from "io-ts";
+
+export const E926DedupeEntryC = t.type({
+    provider: t.literal("e926"),
+    id: t.number,
+});
+
+export const DedupeEntryC = E926DedupeEntryC;
+
+export type E926DedupeEntry = t.TypeOf<typeof E926DedupeEntryC>;
+
+export type DedupeEntry = t.TypeOf<typeof DedupeEntryC>;
+
+export class Dedupe {
+    private entries: DedupeEntry[] = [];
+
+    private readonly filePath: string;
+
+    private isLoaded = false;
+
+    constructor(private max: number, filename: string) {
+        this.filePath = path.join(process.cwd(), filename);
+    }
+
+    private async load() {
+        if (this.isLoaded) {
+            return;
+        }
+
+        try {
+            await fs.stat(this.filePath);
+        } catch {
+            await this.save();
+        }
+
+        const fileContent = await fs.readFile(this.filePath, "utf8");
+        const entries = t.array(DedupeEntryC).decode(fileContent);
+
+        if (f.either.isRight(entries)) {
+            this.entries = entries.right;
+        }
+    }
+
+    private async save() {
+        await fs.writeFile(this.filePath, JSON.stringify(this.entries ?? []), "utf8");
+    }
+
+    async check(entry: DedupeEntry) {
+        await this.load();
+
+        const has = !!this.entries.find((e) => e.provider === entry.provider && e.id === entry.id);
+
+        if (!has) {
+            this.entries.push(entry);
+            await this.save();
+        }
+
+        return has;
+    }
+}
+
+export default new Dedupe(50, "dedupe.json");