Code improvements, support a manual post queue

6 files changed, 129 insertions, 83 deletions

diff --git a/src/api/e621/index.ts b/src/api/e926/index.ts
index a8abbcf..43a3179 100644
--- a/src/api/e621/index.ts
+++ b/src/api/e926/index.ts
@@ -1,7 +1,9 @@
 import got from "got";
 import config from "../../config";
 import delay from "../../util/delay";
-import dedupe from "../../services/dedupe";
+import PostDatabase from "../../services/postDatabase";
+
+export const dedupeDb = new PostDatabase("e926dedupe.json", 50);
 
 export interface GetPostQuery {
     tags: readonly string[];
@@ -49,7 +51,7 @@ export async function getPostById(id: number) {
     return response.posts[0];
 }
 
-export async function getRandomPost(query: GetPostQuery) {
+export async function getRandomPost(query: GetPostQuery): Promise<Post> {
     const page = Math.floor(Math.random() * (query.maxPage - 1)) + 1;
 
     const response = await client
@@ -69,7 +71,7 @@ export async function getRandomPost(query: GetPostQuery) {
     const postIndex = Math.floor(Math.random() * response.posts.length);
     const post = response.posts[postIndex];
 
-    const isDupe = await dedupe.check({ provider: "e926", id: post.id });
+    const isDupe = await dedupeDb.insertIfNotExists({ provider: "e926", id: post.id });
 
     if (isDupe) {
         await delay(1000);
diff --git a/src/config.ts b/src/config.ts
index 2cbcda0..1483d18 100644
--- a/src/config.ts
+++ b/src/config.ts
@@ -1,4 +1,4 @@
-import { GetPostQuery } from "./api/e621";
+import { GetPostQuery } from "./api/e926";
 
 const mainQuery: GetPostQuery = {
     tags: [
@@ -27,10 +27,10 @@ const mainQuery: GetPostQuery = {
         "-photography_(artwork)",
         "-3d_(artwork)",
         "status:active",
-        "score:>=15",
+        "score:>=20",
         "inpool:false",
     ],
-    maxPage: 175,
+    maxPage: 117,
 };
 
 export default {
diff --git a/src/index.ts b/src/index.ts
index 5558540..86d55ed 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -1,13 +1,20 @@
 import config from "./config";
 import * as jobs from "./services/jobs";
 import * as cliArgs from "ts-command-line-args";
+import PostDatabase from "./services/postDatabase";
+
+export const queueDb = new PostDatabase("queue.json", 50);
 
 const args = cliArgs.parse<{
     id?: number;
+    enqueue?: number;
+    dequeue?: number;
     help?: boolean;
 }>(
     {
         id: { type: Number, optional: true },
+        enqueue: { type: Number, optional: true },
+        dequeue: { type: Number, optional: true },
         help: { type: Boolean, optional: true, alias: "h" },
     },
     {
@@ -23,7 +30,23 @@ const args = cliArgs.parse<{
 
     if (args.id) {
         await jobs.postSpecificPicture(args.id);
+    } else if (args.enqueue) {
+        console.log(`Enqueueing post ${args.enqueue}...`);
+
+        await queueDb.insertIfNotExists({ provider: "e926", id: args.enqueue });
+    } else if (args.dequeue) {
+        console.log(`Dequeueing post ${args.dequeue}...`);
+
+        await queueDb.remove({ provider: "e926", id: args.dequeue });
     } else {
-        await jobs.postRandomPicture();
+        console.log("Reading queue...");
+
+        const queued = await queueDb.takeFirst();
+
+        if (queued) {
+            await jobs.postSpecificPicture(queued.id);
+        } else {
+            await jobs.postRandomPicture();
+        }
     }
 })();
diff --git a/src/services/dedupe.ts b/src/services/dedupe.ts
deleted file mode 100644
index 2eeb5ee..0000000
--- a/src/services/dedupe.ts
+++ /dev/null
@@ -1,65 +0,0 @@
-import fs from "fs/promises";
-import path from "path";
-import * as f from "fp-ts";
-import * as t from "io-ts";
-
-export const E926DedupeEntryC = t.type({
-    provider: t.literal("e926"),
-    id: t.number,
-});
-
-export const DedupeEntryC = E926DedupeEntryC;
-
-export type E926DedupeEntry = t.TypeOf<typeof E926DedupeEntryC>;
-
-export type DedupeEntry = t.TypeOf<typeof DedupeEntryC>;
-
-export class Dedupe {
-    private entries: DedupeEntry[] = [];
-
-    private readonly filePath: string;
-
-    private isLoaded = false;
-
-    constructor(private max: number, filename: string) {
-        this.filePath = path.join(process.cwd(), filename);
-    }
-
-    private async load() {
-        if (this.isLoaded) {
-            return;
-        }
-
-        try {
-            await fs.stat(this.filePath);
-        } catch {
-            await this.save();
-        }
-
-        const fileContent = await fs.readFile(this.filePath, "utf8");
-        const entries = t.array(DedupeEntryC).decode(fileContent);
-
-        if (f.either.isRight(entries)) {
-            this.entries = entries.right;
-        }
-    }
-
-    private async save() {
-        await fs.writeFile(this.filePath, JSON.stringify(this.entries ?? []), "utf8");
-    }
-
-    async check(entry: DedupeEntry) {
-        await this.load();
-
-        const has = !!this.entries.find((e) => e.provider === entry.provider && e.id === entry.id);
-
-        if (!has) {
-            this.entries.push(entry);
-            await this.save();
-        }
-
-        return has;
-    }
-}
-
-export default new Dedupe(50, "dedupe.json");
diff --git a/src/services/jobs.ts b/src/services/jobs.ts
index cf3b894..354c66d 100644
--- a/src/services/jobs.ts
+++ b/src/services/jobs.ts
@@ -1,8 +1,18 @@
-import * as e621 from "../api/e621";
+import * as e621 from "../api/e926";
 import * as mastodon from "../api/mastodon";
 import config from "../config";
 import Sharp from "sharp";
 
+export async function postSpecificPicture(id: number) {
+    console.log(`Fetching post ${id}...`);
+
+    const post = await e621.getPostById(id);
+
+    console.log(`Got ${post.id}`);
+
+    await handlePost(post);
+}
+
 export async function postRandomPicture() {
   console.log("Fetching random post...");
 
@@ -15,16 +25,6 @@ export async function postRandomPicture() {
   await handlePost(post);
 }
 
-export async function postSpecificPicture(id: number) {
-    console.log(`Fetching post ${id}...`);
-
-    const post = await e621.getPostById(id);
-
-    console.log(`Got ${post.id}`);
-
-    await handlePost(post);
-}
-
 async function handlePost(post: e621.Post) {
     const source = post.sources.length ? post.sources[0] : undefined;
     const cws = config.cw.filter((w) => post.tags.general.includes(w));
diff --git a/src/services/postDatabase.ts b/src/services/postDatabase.ts
new file mode 100644
index 0000000..e3be7bb
--- /dev/null
+++ b/src/services/postDatabase.ts
@@ -0,0 +1,86 @@
+import fs from "fs/promises";
+import path from "path";
+import * as f from "fp-ts";
+import * as t from "io-ts";
+
+export const PostDatabaseEntryC = t.type({
+    provider: t.literal("e926"),
+    id: t.number,
+});
+
+export type PostDatabaseEntry = t.TypeOf<typeof PostDatabaseEntryC>;
+
+export class PostDatabase {
+    private entries: PostDatabaseEntry[] = [];
+
+    private readonly filePath: string;
+
+    private isLoaded = false;
+
+    constructor(filename: string, private max?: number) {
+        this.filePath = path.join(process.cwd(), filename);
+    }
+
+    private async load() {
+        if (this.isLoaded) {
+            return;
+        }
+
+        try {
+            await fs.stat(this.filePath);
+        } catch {
+            await this.save();
+        }
+
+        const fileContent = await fs.readFile(this.filePath, "utf8");
+        const entries = t.array(PostDatabaseEntryC).decode(fileContent);
+
+        if (f.either.isRight(entries)) {
+            this.entries = this.max ? entries.right.slice(0, -1 * this.max) : entries.right;
+        }
+    }
+
+    private async save() {
+        await fs.writeFile(this.filePath, JSON.stringify(this.entries), "utf8");
+    }
+
+    async insertIfNotExists(entry: PostDatabaseEntry): Promise<boolean> {
+        await this.load();
+
+        const has = !!this.entries.find((e) => e.provider === entry.provider && e.id === entry.id);
+
+        if (!has) {
+            this.entries.push(entry);
+            await this.save();
+        }
+
+        return has;
+    }
+
+    async takeFirst(): Promise<PostDatabaseEntry | undefined> {
+        await this.load();
+        const entry = this.entries.shift();
+        await this.save();
+        return entry;
+    }
+
+    async remove(entry: PostDatabaseEntry): Promise<boolean> {
+        await this.load();
+        const i = this.entries.findIndex((e) => e.provider === entry.provider && e.id === entry.id);
+
+        if (i === -1) {
+            return false;
+        }
+
+        this.entries.splice(i, 1);
+        await this.save();
+        return true;
+    }
+
+    async getCount() {
+        await this.load();
+        return this.entries.length;
+    }
+}
+
+export default PostDatabase;