From a1de58f364ef04c97ecad380427e6181f3bf707d Mon Sep 17 00:00:00 2001 From: Volpeon Date: Tue, 19 Oct 2021 18:14:00 +0200 Subject: Code improvements --- src/api/e621/index.ts | 41 ++++--------------------------- src/index.ts | 2 +- src/jobs.ts | 54 ----------------------------------------- src/services/dedupe.ts | 65 ++++++++++++++++++++++++++++++++++++++++++++++++++ src/services/jobs.ts | 54 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 124 insertions(+), 92 deletions(-) delete mode 100644 src/jobs.ts create mode 100644 src/services/dedupe.ts create mode 100644 src/services/jobs.ts (limited to 'src') diff --git a/src/api/e621/index.ts b/src/api/e621/index.ts index 5c5fd2a..a8abbcf 100644 --- a/src/api/e621/index.ts +++ b/src/api/e621/index.ts @@ -1,8 +1,7 @@ import got from "got"; import config from "../../config"; -import fs from "fs/promises"; -import path from "path"; import delay from "../../util/delay"; +import dedupe from "../../services/dedupe"; export interface GetPostQuery { tags: readonly string[]; @@ -34,35 +33,6 @@ export const client = got.extend({ }, }); -const dedupePath = path.join(__dirname, "e926dedupe.json"); -const dedupeMax = 50; -let dedupeDb: number[] | undefined; - -async function loadDedupeDb() { - if (dedupeDb) { - return; - } - - try { - await fs.stat(dedupePath); - } catch { - await saveDedupeDb(); - } - - const d = await fs.readFile(dedupePath, "utf8"); - const dd = JSON.parse(d); - - if (dd instanceof Array) { - dedupeDb = dd.slice(-1 * dedupeMax); - } else { - dedupeDb = []; - } -} - -async function saveDedupeDb() { - await fs.writeFile(dedupePath, JSON.stringify(dedupeDb ?? []), "utf8"); -} - export async function getPostById(id: number) { const response = await client .get("https://e926.net/posts.json", { @@ -80,8 +50,6 @@ export async function getPostById(id: number) { } export async function getRandomPost(query: GetPostQuery) { - await loadDedupeDb(); - const page = Math.floor(Math.random() * (query.maxPage - 1)) + 1; const response = await client @@ -101,13 +69,12 @@ export async function getRandomPost(query: GetPostQuery) { const postIndex = Math.floor(Math.random() * response.posts.length); const post = response.posts[postIndex]; - if (dedupeDb.includes(post.id)) { + const isDupe = await dedupe.check({ provider: "e926", id: post.id }); + + if (isDupe) { await delay(1000); return getRandomPost(query); } - dedupeDb.push(post.id); - await saveDedupeDb(); - return post; } diff --git a/src/index.ts b/src/index.ts index 6e9ad2e..5558540 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,5 +1,5 @@ import config from "./config"; -import * as jobs from "./jobs"; +import * as jobs from "./services/jobs"; import * as cliArgs from "ts-command-line-args"; const args = cliArgs.parse<{ diff --git a/src/jobs.ts b/src/jobs.ts deleted file mode 100644 index d77104f..0000000 --- a/src/jobs.ts +++ /dev/null @@ -1,54 +0,0 @@ -import * as e621 from "./api/e621"; -import * as mastodon from "./api/mastodon"; -import config from "./config"; -import Sharp from "sharp"; - -export async function postRandomPicture() { - console.log("Fetching random post..."); - - const queryIndex = Math.floor(Math.random() * config.e621.queries.length); - const query = config.e621.queries[queryIndex]; - const post = await e621.getRandomPost(query); - - console.log(`Got ${post.id} via query ${queryIndex}`); - - await handlePost(post); -} - -export async function postSpecificPicture(id: number) { - console.log(`Fetching post ${id}...`); - - const post = await e621.getPostById(id); - - console.log(`Got ${post.id}`); - - await handlePost(post); -} - -async function handlePost(post: e621.Post) { - const source = post.sources.length ? post.sources[0] : undefined; - const cws = config.cw.filter((w) => post.tags.general.includes(w)); - - console.log(`Downloading image...`); - - let file = await e621.client.get(post.file.url).buffer(); - - if (Buffer.byteLength(file) > 1024 * 1024 * 9) { - console.log(`Compressing...`); - - file = await Sharp(file) - .resize(1800, 1800, { fit: "inside", withoutEnlargement: true }) - .jpeg({ quality: 80, mozjpeg: true }) - .toBuffer(); - } - - console.log(`Uploading...`); - - const attachment = await mastodon.upload(file, post.id.toString(10)); - - console.log(`Posting status...`); - - const status = await mastodon.createStatus(`https://e926.net/posts/${post.id}`, source, cws, attachment.id); - - console.log(`Done! ${status.url}`); -} diff --git a/src/services/dedupe.ts b/src/services/dedupe.ts new file mode 100644 index 0000000..2eeb5ee --- /dev/null +++ b/src/services/dedupe.ts @@ -0,0 +1,65 @@ +import fs from "fs/promises"; +import path from "path"; +import * as f from "fp-ts"; +import * as t from "io-ts"; + +export const E926DedupeEntryC = t.type({ + provider: t.literal("e926"), + id: t.number, +}); + +export const DedupeEntryC = E926DedupeEntryC; + +export type E926DedupeEntry = t.TypeOf; + +export type DedupeEntry = t.TypeOf; + +export class Dedupe { + private entries: DedupeEntry[] = []; + + private readonly filePath: string; + + private isLoaded = false; + + constructor(private max: number, filename: string) { + this.filePath = path.join(process.cwd(), filename); + } + + private async load() { + if (this.isLoaded) { + return; + } + + try { + await fs.stat(this.filePath); + } catch { + await this.save(); + } + + const fileContent = await fs.readFile(this.filePath, "utf8"); + const entries = t.array(DedupeEntryC).decode(fileContent); + + if (f.either.isRight(entries)) { + this.entries = entries.right; + } + } + + private async save() { + await fs.writeFile(this.filePath, JSON.stringify(this.entries ?? []), "utf8"); + } + + async check(entry: DedupeEntry) { + await this.load(); + + const has = !!this.entries.find((e) => e.provider === entry.provider && e.id === entry.id); + + if (!has) { + this.entries.push(entry); + await this.save(); + } + + return has; + } +} + +export default new Dedupe(50, "dedupe.json"); diff --git a/src/services/jobs.ts b/src/services/jobs.ts new file mode 100644 index 0000000..cf3b894 --- /dev/null +++ b/src/services/jobs.ts @@ -0,0 +1,54 @@ +import * as e621 from "../api/e621"; +import * as mastodon from "../api/mastodon"; +import config from "../config"; +import Sharp from "sharp"; + +export async function postRandomPicture() { + console.log("Fetching random post..."); + + const queryIndex = Math.floor(Math.random() * config.e621.queries.length); + const query = config.e621.queries[queryIndex]; + const post = await e621.getRandomPost(query); + + console.log(`Got ${post.id} via query ${queryIndex}`); + + await handlePost(post); +} + +export async function postSpecificPicture(id: number) { + console.log(`Fetching post ${id}...`); + + const post = await e621.getPostById(id); + + console.log(`Got ${post.id}`); + + await handlePost(post); +} + +async function handlePost(post: e621.Post) { + const source = post.sources.length ? post.sources[0] : undefined; + const cws = config.cw.filter((w) => post.tags.general.includes(w)); + + console.log(`Downloading image...`); + + let file = await e621.client.get(post.file.url).buffer(); + + if (Buffer.byteLength(file) > 1024 * 1024 * 9) { + console.log(`Compressing...`); + + file = await Sharp(file) + .resize(1800, 1800, { fit: "inside", withoutEnlargement: true }) + .jpeg({ quality: 80, mozjpeg: true }) + .toBuffer(); + } + + console.log(`Uploading...`); + + const attachment = await mastodon.upload(file, post.id.toString(10)); + + console.log(`Posting status...`); + + const status = await mastodon.createStatus(`https://e926.net/posts/${post.id}`, source, cws, attachment.id); + + console.log(`Done! ${status.url}`); +} -- cgit v1.2.3-70-g09d2