From 163d8119c109c42e64ab37b01dec131f2cb5bf61 Mon Sep 17 00:00:00 2001 From: Volpeon Date: Tue, 19 Oct 2021 19:25:03 +0200 Subject: Code improvements, support a manual post queue --- src/api/e621/index.ts | 80 ----------------------------------------- src/api/e926/index.ts | 82 ++++++++++++++++++++++++++++++++++++++++++ src/config.ts | 6 ++-- src/index.ts | 25 ++++++++++++- src/services/dedupe.ts | 65 --------------------------------- src/services/jobs.ts | 22 ++++++------ src/services/postDatabase.ts | 86 ++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 206 insertions(+), 160 deletions(-) delete mode 100644 src/api/e621/index.ts create mode 100644 src/api/e926/index.ts delete mode 100644 src/services/dedupe.ts create mode 100644 src/services/postDatabase.ts (limited to 'src') diff --git a/src/api/e621/index.ts b/src/api/e621/index.ts deleted file mode 100644 index a8abbcf..0000000 --- a/src/api/e621/index.ts +++ /dev/null @@ -1,80 +0,0 @@ -import got from "got"; -import config from "../../config"; -import delay from "../../util/delay"; -import dedupe from "../../services/dedupe"; - -export interface GetPostQuery { - tags: readonly string[]; - maxPage: number; -} - -export interface Post { - id: number; - file: { - url: string; - }; - sources: readonly string[]; - - tags: { - general: readonly string[]; - species: readonly string[]; - character: readonly string[]; - copyright: readonly string[]; - artist: readonly string[]; - invalid: readonly string[]; - lore: readonly string[]; - meta: readonly string[]; - }; -} - -export const client = got.extend({ - headers: { - "User-Agent": config.e621.userAgent, - }, -}); - -export async function getPostById(id: number) { - const response = await client - .get("https://e926.net/posts.json", { - searchParams: { - tags: `id:${id}`, - }, - }) - .json<{ posts: readonly Post[] }>(); - - if (!response.posts.length) { - throw new Error("No posts received"); - } - - return response.posts[0]; -} - -export async function getRandomPost(query: GetPostQuery) { - const page = Math.floor(Math.random() * (query.maxPage - 1)) + 1; - - const response = await client - .get("https://e926.net/posts.json", { - searchParams: { - limit: 75, - page, - tags: query.tags.join(" "), - }, - }) - .json<{ posts: readonly Post[] }>(); - - if (!response.posts.length) { - throw new Error("No posts received"); - } - - const postIndex = Math.floor(Math.random() * response.posts.length); - const post = response.posts[postIndex]; - - const isDupe = await dedupe.check({ provider: "e926", id: post.id }); - - if (isDupe) { - await delay(1000); - return getRandomPost(query); - } - - return post; -} diff --git a/src/api/e926/index.ts b/src/api/e926/index.ts new file mode 100644 index 0000000..43a3179 --- /dev/null +++ b/src/api/e926/index.ts @@ -0,0 +1,82 @@ +import got from "got"; +import config from "../../config"; +import delay from "../../util/delay"; +import PostDatabase from "../../services/postDatabase"; + +export const dedupeDb = new PostDatabase("e926dedupe.json", 50); + +export interface GetPostQuery { + tags: readonly string[]; + maxPage: number; +} + +export interface Post { + id: number; + file: { + url: string; + }; + sources: readonly string[]; + + tags: { + general: readonly string[]; + species: readonly string[]; + character: readonly string[]; + copyright: readonly string[]; + artist: readonly string[]; + invalid: readonly string[]; + lore: readonly string[]; + meta: readonly string[]; + }; +} + +export const client = got.extend({ + headers: { + "User-Agent": config.e621.userAgent, + }, +}); + +export async function getPostById(id: number) { + const response = await client + .get("https://e926.net/posts.json", { + searchParams: { + tags: `id:${id}`, + }, + }) + .json<{ posts: readonly Post[] }>(); + + if (!response.posts.length) { + throw new Error("No posts received"); + } + + return response.posts[0]; +} + +export async function getRandomPost(query: GetPostQuery): Promise { + const page = Math.floor(Math.random() * (query.maxPage - 1)) + 1; + + const response = await client + .get("https://e926.net/posts.json", { + searchParams: { + limit: 75, + page, + tags: query.tags.join(" "), + }, + }) + .json<{ posts: readonly Post[] }>(); + + if (!response.posts.length) { + throw new Error("No posts received"); + } + + const postIndex = Math.floor(Math.random() * response.posts.length); + const post = response.posts[postIndex]; + + const isDupe = await dedupeDb.insertIfNotExists({ provider: "e926", id: post.id }); + + if (isDupe) { + await delay(1000); + return getRandomPost(query); + } + + return post; +} diff --git a/src/config.ts b/src/config.ts index 2cbcda0..1483d18 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,4 +1,4 @@ -import { GetPostQuery } from "./api/e621"; +import { GetPostQuery } from "./api/e926"; const mainQuery: GetPostQuery = { tags: [ @@ -27,10 +27,10 @@ const mainQuery: GetPostQuery = { "-photography_(artwork)", "-3d_(artwork)", "status:active", - "score:>=15", + "score:>=20", "inpool:false", ], - maxPage: 175, + maxPage: 117, }; export default { diff --git a/src/index.ts b/src/index.ts index 5558540..86d55ed 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,13 +1,20 @@ import config from "./config"; import * as jobs from "./services/jobs"; import * as cliArgs from "ts-command-line-args"; +import PostDatabase from "./services/postDatabase"; + +export const queueDb = new PostDatabase("queue.json", 50); const args = cliArgs.parse<{ id?: number; + enqueue?: number; + dequeue?: number; help?: boolean; }>( { id: { type: Number, optional: true }, + enqueue: { type: Number, optional: true }, + dequeue: { type: Number, optional: true }, help: { type: Boolean, optional: true, alias: "h" }, }, { @@ -23,7 +30,23 @@ const args = cliArgs.parse<{ if (args.id) { await jobs.postSpecificPicture(args.id); + } else if (args.enqueue) { + console.log(`Enqueueing post ${args.enqueue}...`); + + await queueDb.insertIfNotExists({ provider: "e926", id: args.enqueue }); + } else if (args.dequeue) { + console.log(`Dequeueing post ${args.dequeue}...`); + + await queueDb.remove({ provider: "e926", id: args.dequeue }); } else { - await jobs.postRandomPicture(); + console.log("Reading queue..."); + + const queued = await queueDb.takeFirst(); + + if (queued) { + await jobs.postSpecificPicture(queued.id); + } else { + await jobs.postRandomPicture(); + } } })(); diff --git a/src/services/dedupe.ts b/src/services/dedupe.ts deleted file mode 100644 index 2eeb5ee..0000000 --- a/src/services/dedupe.ts +++ /dev/null @@ -1,65 +0,0 @@ -import fs from "fs/promises"; -import path from "path"; -import * as f from "fp-ts"; -import * as t from "io-ts"; - -export const E926DedupeEntryC = t.type({ - provider: t.literal("e926"), - id: t.number, -}); - -export const DedupeEntryC = E926DedupeEntryC; - -export type E926DedupeEntry = t.TypeOf; - -export type DedupeEntry = t.TypeOf; - -export class Dedupe { - private entries: DedupeEntry[] = []; - - private readonly filePath: string; - - private isLoaded = false; - - constructor(private max: number, filename: string) { - this.filePath = path.join(process.cwd(), filename); - } - - private async load() { - if (this.isLoaded) { - return; - } - - try { - await fs.stat(this.filePath); - } catch { - await this.save(); - } - - const fileContent = await fs.readFile(this.filePath, "utf8"); - const entries = t.array(DedupeEntryC).decode(fileContent); - - if (f.either.isRight(entries)) { - this.entries = entries.right; - } - } - - private async save() { - await fs.writeFile(this.filePath, JSON.stringify(this.entries ?? []), "utf8"); - } - - async check(entry: DedupeEntry) { - await this.load(); - - const has = !!this.entries.find((e) => e.provider === entry.provider && e.id === entry.id); - - if (!has) { - this.entries.push(entry); - await this.save(); - } - - return has; - } -} - -export default new Dedupe(50, "dedupe.json"); diff --git a/src/services/jobs.ts b/src/services/jobs.ts index cf3b894..354c66d 100644 --- a/src/services/jobs.ts +++ b/src/services/jobs.ts @@ -1,8 +1,18 @@ -import * as e621 from "../api/e621"; +import * as e621 from "../api/e926"; import * as mastodon from "../api/mastodon"; import config from "../config"; import Sharp from "sharp"; +export async function postSpecificPicture(id: number) { + console.log(`Fetching post ${id}...`); + + const post = await e621.getPostById(id); + + console.log(`Got ${post.id}`); + + await handlePost(post); +} + export async function postRandomPicture() { console.log("Fetching random post..."); @@ -15,16 +25,6 @@ export async function postRandomPicture() { await handlePost(post); } -export async function postSpecificPicture(id: number) { - console.log(`Fetching post ${id}...`); - - const post = await e621.getPostById(id); - - console.log(`Got ${post.id}`); - - await handlePost(post); -} - async function handlePost(post: e621.Post) { const source = post.sources.length ? post.sources[0] : undefined; const cws = config.cw.filter((w) => post.tags.general.includes(w)); diff --git a/src/services/postDatabase.ts b/src/services/postDatabase.ts new file mode 100644 index 0000000..e3be7bb --- /dev/null +++ b/src/services/postDatabase.ts @@ -0,0 +1,86 @@ +import fs from "fs/promises"; +import path from "path"; +import * as f from "fp-ts"; +import * as t from "io-ts"; + +export const PostDatabaseEntryC = t.type({ + provider: t.literal("e926"), + id: t.number, +}); + +export type PostDatabaseEntry = t.TypeOf; + +export class PostDatabase { + private entries: PostDatabaseEntry[] = []; + + private readonly filePath: string; + + private isLoaded = false; + + constructor(filename: string, private max?: number) { + this.filePath = path.join(process.cwd(), filename); + } + + private async load() { + if (this.isLoaded) { + return; + } + + try { + await fs.stat(this.filePath); + } catch { + await this.save(); + } + + const fileContent = await fs.readFile(this.filePath, "utf8"); + const entries = t.array(PostDatabaseEntryC).decode(fileContent); + + if (f.either.isRight(entries)) { + this.entries = this.max ? entries.right.slice(0, -1 * this.max) : entries.right; + } + } + + private async save() { + await fs.writeFile(this.filePath, JSON.stringify(this.entries), "utf8"); + } + + async insertIfNotExists(entry: PostDatabaseEntry): Promise { + await this.load(); + + const has = !!this.entries.find((e) => e.provider === entry.provider && e.id === entry.id); + + if (!has) { + this.entries.push(entry); + await this.save(); + } + + return has; + } + + async takeFirst(): Promise { + await this.load(); + const entry = this.entries.shift(); + await this.save(); + return entry; + } + + async remove(entry: PostDatabaseEntry): Promise { + await this.load(); + const i = this.entries.findIndex((e) => e.provider === entry.provider && e.id === entry.id); + + if (i === -1) { + return false; + } + + this.entries.splice(i, 1); + await this.save(); + return true; + } + + async getCount() { + await this.load(); + return this.entries.length; + } +} + +export default PostDatabase; -- cgit v1.2.3-70-g09d2