From 163d8119c109c42e64ab37b01dec131f2cb5bf61 Mon Sep 17 00:00:00 2001
From: Volpeon <git@volpeon.ink>
Date: Tue, 19 Oct 2021 19:25:03 +0200
Subject: Code improvements, support a manual post queue
---
src/api/e621/index.ts | 80 -----------------------------------------
src/api/e926/index.ts | 82 ++++++++++++++++++++++++++++++++++++++++++
src/config.ts | 6 ++--
src/index.ts | 25 ++++++++++++-
src/services/dedupe.ts | 65 ---------------------------------
src/services/jobs.ts | 22 ++++++------
src/services/postDatabase.ts | 86 ++++++++++++++++++++++++++++++++++++++++++++
7 files changed, 206 insertions(+), 160 deletions(-)
delete mode 100644 src/api/e621/index.ts
create mode 100644 src/api/e926/index.ts
delete mode 100644 src/services/dedupe.ts
create mode 100644 src/services/postDatabase.ts
(limited to 'src')
diff --git a/src/api/e621/index.ts b/src/api/e621/index.ts
deleted file mode 100644
index a8abbcf..0000000
--- a/src/api/e621/index.ts
+++ /dev/null
@@ -1,80 +0,0 @@
-import got from "got";
-import config from "../../config";
-import delay from "../../util/delay";
-import dedupe from "../../services/dedupe";
-
-export interface GetPostQuery {
- tags: readonly string[];
- maxPage: number;
-}
-
-export interface Post {
- id: number;
- file: {
- url: string;
- };
- sources: readonly string[];
-
- tags: {
- general: readonly string[];
- species: readonly string[];
- character: readonly string[];
- copyright: readonly string[];
- artist: readonly string[];
- invalid: readonly string[];
- lore: readonly string[];
- meta: readonly string[];
- };
-}
-
-export const client = got.extend({
- headers: {
- "User-Agent": config.e621.userAgent,
- },
-});
-
-export async function getPostById(id: number) {
- const response = await client
- .get("https://e926.net/posts.json", {
- searchParams: {
- tags: `id:${id}`,
- },
- })
- .json<{ posts: readonly Post[] }>();
-
- if (!response.posts.length) {
- throw new Error("No posts received");
- }
-
- return response.posts[0];
-}
-
-export async function getRandomPost(query: GetPostQuery) {
- const page = Math.floor(Math.random() * (query.maxPage - 1)) + 1;
-
- const response = await client
- .get("https://e926.net/posts.json", {
- searchParams: {
- limit: 75,
- page,
- tags: query.tags.join(" "),
- },
- })
- .json<{ posts: readonly Post[] }>();
-
- if (!response.posts.length) {
- throw new Error("No posts received");
- }
-
- const postIndex = Math.floor(Math.random() * response.posts.length);
- const post = response.posts[postIndex];
-
- const isDupe = await dedupe.check({ provider: "e926", id: post.id });
-
- if (isDupe) {
- await delay(1000);
- return getRandomPost(query);
- }
-
- return post;
-}
diff --git a/src/api/e926/index.ts b/src/api/e926/index.ts
new file mode 100644
index 0000000..43a3179
--- /dev/null
+++ b/src/api/e926/index.ts
@@ -0,0 +1,82 @@
+import got from "got";
+import config from "../../config";
+import delay from "../../util/delay";
+import PostDatabase from "../../services/postDatabase";
+
+export const dedupeDb = new PostDatabase("e926dedupe.json", 50);
+
+export interface GetPostQuery {
+ tags: readonly string[];
+ maxPage: number;
+}
+
+export interface Post {
+ id: number;
+ file: {
+ url: string;
+ };
+ sources: readonly string[];
+
+ tags: {
+ general: readonly string[];
+ species: readonly string[];
+ character: readonly string[];
+ copyright: readonly string[];
+ artist: readonly string[];
+ invalid: readonly string[];
+ lore: readonly string[];
+ meta: readonly string[];
+ };
+}
+
+export const client = got.extend({
+ headers: {
+ "User-Agent": config.e621.userAgent,
+ },
+});
+
+export async function getPostById(id: number) {
+ const response = await client
+ .get("https://e926.net/posts.json", {
+ searchParams: {
+ tags: `id:${id}`,
+ },
+ })
+ .json<{ posts: readonly Post[] }>();
+
+ if (!response.posts.length) {
+ throw new Error("No posts received");
+ }
+
+ return response.posts[0];
+}
+
+export async function getRandomPost(query: GetPostQuery): Promise<Post> {
+ const page = Math.floor(Math.random() * (query.maxPage - 1)) + 1;
+
+ const response = await client
+ .get("https://e926.net/posts.json", {
+ searchParams: {
+ limit: 75,
+ page,
+ tags: query.tags.join(" "),
+ },
+ })
+ .json<{ posts: readonly Post[] }>();
+
+ if (!response.posts.length) {
+ throw new Error("No posts received");
+ }
+
+ const postIndex = Math.floor(Math.random() * response.posts.length);
+ const post = response.posts[postIndex];
+
+ const isDupe = await dedupeDb.insertIfNotExists({ provider: "e926", id: post.id });
+
+ if (isDupe) {
+ await delay(1000);
+ return getRandomPost(query);
+ }
+
+ return post;
+}
diff --git a/src/config.ts b/src/config.ts
index 2cbcda0..1483d18 100644
--- a/src/config.ts
+++ b/src/config.ts
@@ -1,4 +1,4 @@
-import { GetPostQuery } from "./api/e621";
+import { GetPostQuery } from "./api/e926";
const mainQuery: GetPostQuery = {
tags: [
@@ -27,10 +27,10 @@ const mainQuery: GetPostQuery = {
"-photography_(artwork)",
"-3d_(artwork)",
"status:active",
- "score:>=15",
+ "score:>=20",
"inpool:false",
],
- maxPage: 175,
+ maxPage: 117,
};
export default {
diff --git a/src/index.ts b/src/index.ts
index 5558540..86d55ed 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -1,13 +1,20 @@
import config from "./config";
import * as jobs from "./services/jobs";
import * as cliArgs from "ts-command-line-args";
+import PostDatabase from "./services/postDatabase";
+
+export const queueDb = new PostDatabase("queue.json", 50);
const args = cliArgs.parse<{
id?: number;
+ enqueue?: number;
+ dequeue?: number;
help?: boolean;
}>(
{
id: { type: Number, optional: true },
+ enqueue: { type: Number, optional: true },
+ dequeue: { type: Number, optional: true },
help: { type: Boolean, optional: true, alias: "h" },
},
{
@@ -23,7 +30,23 @@ const args = cliArgs.parse<{
if (args.id) {
await jobs.postSpecificPicture(args.id);
+ } else if (args.enqueue) {
+ console.log(`Enqueueing post ${args.enqueue}...`);
+
+ await queueDb.insertIfNotExists({ provider: "e926", id: args.enqueue });
+ } else if (args.dequeue) {
+ console.log(`Dequeueing post ${args.dequeue}...`);
+
+ await queueDb.remove({ provider: "e926", id: args.dequeue });
} else {
- await jobs.postRandomPicture();
+ console.log("Reading queue...");
+
+ const queued = await queueDb.takeFirst();
+
+ if (queued) {
+ await jobs.postSpecificPicture(queued.id);
+ } else {
+ await jobs.postRandomPicture();
+ }
}
})();
diff --git a/src/services/dedupe.ts b/src/services/dedupe.ts
deleted file mode 100644
index 2eeb5ee..0000000
--- a/src/services/dedupe.ts
+++ /dev/null
@@ -1,65 +0,0 @@
-import fs from "fs/promises";
-import path from "path";
-import * as f from "fp-ts";
-import * as t from "io-ts";
-
-export const E926DedupeEntryC = t.type({
- provider: t.literal("e926"),
- id: t.number,
-});
-
-export const DedupeEntryC = E926DedupeEntryC;
-
-export type E926DedupeEntry = t.TypeOf<typeof E926DedupeEntryC>;
-
-export type DedupeEntry = t.TypeOf<typeof DedupeEntryC>;
-
-export class Dedupe {
- private entries: DedupeEntry[] = [];
-
- private readonly filePath: string;
-
- private isLoaded = false;
-
- constructor(private max: number, filename: string) {
- this.filePath = path.join(process.cwd(), filename);
- }
-
- private async load() {
- if (this.isLoaded) {
- return;
- }
-
- try {
- await fs.stat(this.filePath);
- } catch {
- await this.save();
- }
-
- const fileContent = await fs.readFile(this.filePath, "utf8");
- const entries = t.array(DedupeEntryC).decode(fileContent);
-
- if (f.either.isRight(entries)) {
- this.entries = entries.right;
- }
- }
-
- private async save() {
- await fs.writeFile(this.filePath, JSON.stringify(this.entries ?? []), "utf8");
- }
-
- async check(entry: DedupeEntry) {
- await this.load();
-
- const has = !!this.entries.find((e) => e.provider === entry.provider && e.id === entry.id);
-
- if (!has) {
- this.entries.push(entry);
- await this.save();
- }
-
- return has;
- }
-}
-
-export default new Dedupe(50, "dedupe.json");
diff --git a/src/services/jobs.ts b/src/services/jobs.ts
index cf3b894..354c66d 100644
--- a/src/services/jobs.ts
+++ b/src/services/jobs.ts
@@ -1,8 +1,18 @@
-import * as e621 from "../api/e621";
+import * as e621 from "../api/e926";
import * as mastodon from "../api/mastodon";
import config from "../config";
import Sharp from "sharp";
+export async function postSpecificPicture(id: number) {
+ console.log(`Fetching post ${id}...`);
+
+ const post = await e621.getPostById(id);
+
+ console.log(`Got ${post.id}`);
+
+ await handlePost(post);
+}
+
export async function postRandomPicture() {
console.log("Fetching random post...");
@@ -15,16 +25,6 @@ export async function postRandomPicture() {
await handlePost(post);
}
-export async function postSpecificPicture(id: number) {
- console.log(`Fetching post ${id}...`);
-
- const post = await e621.getPostById(id);
-
- console.log(`Got ${post.id}`);
-
- await handlePost(post);
-}
-
async function handlePost(post: e621.Post) {
const source = post.sources.length ? post.sources[0] : undefined;
const cws = config.cw.filter((w) => post.tags.general.includes(w));
diff --git a/src/services/postDatabase.ts b/src/services/postDatabase.ts
new file mode 100644
index 0000000..e3be7bb
--- /dev/null
+++ b/src/services/postDatabase.ts
@@ -0,0 +1,86 @@
+import fs from "fs/promises";
+import path from "path";
+import * as f from "fp-ts";
+import * as t from "io-ts";
+
+export const PostDatabaseEntryC = t.type({
+ provider: t.literal("e926"),
+ id: t.number,
+});
+
+export type PostDatabaseEntry = t.TypeOf<typeof PostDatabaseEntryC>;
+
+export class PostDatabase {
+ private entries: PostDatabaseEntry[] = [];
+
+ private readonly filePath: string;
+
+ private isLoaded = false;
+
+ constructor(filename: string, private max?: number) {
+ this.filePath = path.join(process.cwd(), filename);
+ }
+
+ private async load() {
+ if (this.isLoaded) {
+ return;
+ }
+
+ try {
+ await fs.stat(this.filePath);
+ } catch {
+ await this.save();
+ }
+
+ const fileContent = await fs.readFile(this.filePath, "utf8");
+ const entries = t.array(PostDatabaseEntryC).decode(fileContent);
+
+ if (f.either.isRight(entries)) {
+ this.entries = this.max ? entries.right.slice(0, -1 * this.max) : entries.right;
+ }
+ }
+
+ private async save() {
+ await fs.writeFile(this.filePath, JSON.stringify(this.entries), "utf8");
+ }
+
+ async insertIfNotExists(entry: PostDatabaseEntry): Promise<boolean> {
+ await this.load();
+
+ const has = !!this.entries.find((e) => e.provider === entry.provider && e.id === entry.id);
+
+ if (!has) {
+ this.entries.push(entry);
+ await this.save();
+ }
+
+ return has;
+ }
+
+ async takeFirst(): Promise<PostDatabaseEntry | undefined> {
+ await this.load();
+ const entry = this.entries.shift();
+ await this.save();
+ return entry;
+ }
+
+ async remove(entry: PostDatabaseEntry): Promise<boolean> {
+ await this.load();
+ const i = this.entries.findIndex((e) => e.provider === entry.provider && e.id === entry.id);
+
+ if (i === -1) {
+ return false;
+ }
+
+ this.entries.splice(i, 1);
+ await this.save();
+ return true;
+ }
+
+ async getCount() {
+ await this.load();
+ return this.entries.length;
+ }
+}
+
+export default PostDatabase;
--
cgit v1.2.3-70-g09d2