From a1de58f364ef04c97ecad380427e6181f3bf707d Mon Sep 17 00:00:00 2001 From: Volpeon Date: Tue, 19 Oct 2021 18:14:00 +0200 Subject: Code improvements --- package.json | 2 ++ src/api/e621/index.ts | 41 ++++--------------------------- src/index.ts | 2 +- src/jobs.ts | 54 ----------------------------------------- src/services/dedupe.ts | 65 ++++++++++++++++++++++++++++++++++++++++++++++++++ src/services/jobs.ts | 54 +++++++++++++++++++++++++++++++++++++++++ tsconfig.json | 1 - yarn.lock | 10 ++++++++ 8 files changed, 136 insertions(+), 93 deletions(-) delete mode 100644 src/jobs.ts create mode 100644 src/services/dedupe.ts create mode 100644 src/services/jobs.ts diff --git a/package.json b/package.json index b6632bb..8cf504a 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,9 @@ "@types/sharp": "^0.29.2", "file-type": "^16.5.3", "form-data": "^4.0.0", + "fp-ts": "^2.11.5", "got": "^11.8.2", + "io-ts": "^2.2.16", "nanoid": "^3.1.30", "sharp": "^0.29.1", "ts-command-line-args": "^2.1.0", diff --git a/src/api/e621/index.ts b/src/api/e621/index.ts index 5c5fd2a..a8abbcf 100644 --- a/src/api/e621/index.ts +++ b/src/api/e621/index.ts @@ -1,8 +1,7 @@ import got from "got"; import config from "../../config"; -import fs from "fs/promises"; -import path from "path"; import delay from "../../util/delay"; +import dedupe from "../../services/dedupe"; export interface GetPostQuery { tags: readonly string[]; @@ -34,35 +33,6 @@ export const client = got.extend({ }, }); -const dedupePath = path.join(__dirname, "e926dedupe.json"); -const dedupeMax = 50; -let dedupeDb: number[] | undefined; - -async function loadDedupeDb() { - if (dedupeDb) { - return; - } - - try { - await fs.stat(dedupePath); - } catch { - await saveDedupeDb(); - } - - const d = await fs.readFile(dedupePath, "utf8"); - const dd = JSON.parse(d); - - if (dd instanceof Array) { - dedupeDb = dd.slice(-1 * dedupeMax); - } else { - dedupeDb = []; - } -} - -async function saveDedupeDb() { - await fs.writeFile(dedupePath, JSON.stringify(dedupeDb ?? []), "utf8"); -} - export async function getPostById(id: number) { const response = await client .get("https://e926.net/posts.json", { @@ -80,8 +50,6 @@ export async function getPostById(id: number) { } export async function getRandomPost(query: GetPostQuery) { - await loadDedupeDb(); - const page = Math.floor(Math.random() * (query.maxPage - 1)) + 1; const response = await client @@ -101,13 +69,12 @@ export async function getRandomPost(query: GetPostQuery) { const postIndex = Math.floor(Math.random() * response.posts.length); const post = response.posts[postIndex]; - if (dedupeDb.includes(post.id)) { + const isDupe = await dedupe.check({ provider: "e926", id: post.id }); + + if (isDupe) { await delay(1000); return getRandomPost(query); } - dedupeDb.push(post.id); - await saveDedupeDb(); - return post; } diff --git a/src/index.ts b/src/index.ts index 6e9ad2e..5558540 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,5 +1,5 @@ import config from "./config"; -import * as jobs from "./jobs"; +import * as jobs from "./services/jobs"; import * as cliArgs from "ts-command-line-args"; const args = cliArgs.parse<{ diff --git a/src/jobs.ts b/src/jobs.ts deleted file mode 100644 index d77104f..0000000 --- a/src/jobs.ts +++ /dev/null @@ -1,54 +0,0 @@ -import * as e621 from "./api/e621"; -import * as mastodon from "./api/mastodon"; -import config from "./config"; -import Sharp from "sharp"; - -export async function postRandomPicture() { - console.log("Fetching random post..."); - - const queryIndex = Math.floor(Math.random() * config.e621.queries.length); - const query = config.e621.queries[queryIndex]; - const post = await e621.getRandomPost(query); - - console.log(`Got ${post.id} via query ${queryIndex}`); - - await handlePost(post); -} - -export async function postSpecificPicture(id: number) { - console.log(`Fetching post ${id}...`); - - const post = await e621.getPostById(id); - - console.log(`Got ${post.id}`); - - await handlePost(post); -} - -async function handlePost(post: e621.Post) { - const source = post.sources.length ? post.sources[0] : undefined; - const cws = config.cw.filter((w) => post.tags.general.includes(w)); - - console.log(`Downloading image...`); - - let file = await e621.client.get(post.file.url).buffer(); - - if (Buffer.byteLength(file) > 1024 * 1024 * 9) { - console.log(`Compressing...`); - - file = await Sharp(file) - .resize(1800, 1800, { fit: "inside", withoutEnlargement: true }) - .jpeg({ quality: 80, mozjpeg: true }) - .toBuffer(); - } - - console.log(`Uploading...`); - - const attachment = await mastodon.upload(file, post.id.toString(10)); - - console.log(`Posting status...`); - - const status = await mastodon.createStatus(`https://e926.net/posts/${post.id}`, source, cws, attachment.id); - - console.log(`Done! ${status.url}`); -} diff --git a/src/services/dedupe.ts b/src/services/dedupe.ts new file mode 100644 index 0000000..2eeb5ee --- /dev/null +++ b/src/services/dedupe.ts @@ -0,0 +1,65 @@ +import fs from "fs/promises"; +import path from "path"; +import * as f from "fp-ts"; +import * as t from "io-ts"; + +export const E926DedupeEntryC = t.type({ + provider: t.literal("e926"), + id: t.number, +}); + +export const DedupeEntryC = E926DedupeEntryC; + +export type E926DedupeEntry = t.TypeOf; + +export type DedupeEntry = t.TypeOf; + +export class Dedupe { + private entries: DedupeEntry[] = []; + + private readonly filePath: string; + + private isLoaded = false; + + constructor(private max: number, filename: string) { + this.filePath = path.join(process.cwd(), filename); + } + + private async load() { + if (this.isLoaded) { + return; + } + + try { + await fs.stat(this.filePath); + } catch { + await this.save(); + } + + const fileContent = await fs.readFile(this.filePath, "utf8"); + const entries = t.array(DedupeEntryC).decode(fileContent); + + if (f.either.isRight(entries)) { + this.entries = entries.right; + } + } + + private async save() { + await fs.writeFile(this.filePath, JSON.stringify(this.entries ?? []), "utf8"); + } + + async check(entry: DedupeEntry) { + await this.load(); + + const has = !!this.entries.find((e) => e.provider === entry.provider && e.id === entry.id); + + if (!has) { + this.entries.push(entry); + await this.save(); + } + + return has; + } +} + +export default new Dedupe(50, "dedupe.json"); diff --git a/src/services/jobs.ts b/src/services/jobs.ts new file mode 100644 index 0000000..cf3b894 --- /dev/null +++ b/src/services/jobs.ts @@ -0,0 +1,54 @@ +import * as e621 from "../api/e621"; +import * as mastodon from "../api/mastodon"; +import config from "../config"; +import Sharp from "sharp"; + +export async function postRandomPicture() { + console.log("Fetching random post..."); + + const queryIndex = Math.floor(Math.random() * config.e621.queries.length); + const query = config.e621.queries[queryIndex]; + const post = await e621.getRandomPost(query); + + console.log(`Got ${post.id} via query ${queryIndex}`); + + await handlePost(post); +} + +export async function postSpecificPicture(id: number) { + console.log(`Fetching post ${id}...`); + + const post = await e621.getPostById(id); + + console.log(`Got ${post.id}`); + + await handlePost(post); +} + +async function handlePost(post: e621.Post) { + const source = post.sources.length ? post.sources[0] : undefined; + const cws = config.cw.filter((w) => post.tags.general.includes(w)); + + console.log(`Downloading image...`); + + let file = await e621.client.get(post.file.url).buffer(); + + if (Buffer.byteLength(file) > 1024 * 1024 * 9) { + console.log(`Compressing...`); + + file = await Sharp(file) + .resize(1800, 1800, { fit: "inside", withoutEnlargement: true }) + .jpeg({ quality: 80, mozjpeg: true }) + .toBuffer(); + } + + console.log(`Uploading...`); + + const attachment = await mastodon.upload(file, post.id.toString(10)); + + console.log(`Posting status...`); + + const status = await mastodon.createStatus(`https://e926.net/posts/${post.id}`, source, cws, attachment.id); + + console.log(`Done! ${status.url}`); +} diff --git a/tsconfig.json b/tsconfig.json index ad89216..10b9c28 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,7 +1,6 @@ { "compilerOptions": { "lib": ["es2020"], - "module": "ESNext", "moduleResolution": "node", "target": "ES2020", "esModuleInterop": true, diff --git a/yarn.lock b/yarn.lock index 17b1093..9ea15d5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -406,6 +406,11 @@ form-data@^4.0.0: combined-stream "^1.0.8" mime-types "^2.1.12" +fp-ts@^2.11.5: + version "2.11.5" + resolved "https://registry.yarnpkg.com/fp-ts/-/fp-ts-2.11.5.tgz#97cceb26655b1452d7088d6fb0864f84cceffbe4" + integrity sha512-OqlwJq1BdpB83BZXTqI+dNcA6uYk6qk4u9Cgnt64Y+XS7dwdbp/mobx8S2KXf2AXH+scNmA/UVK3SEFHR3vHZA== + fs-constants@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad" @@ -497,6 +502,11 @@ ini@~1.3.0: resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.8.tgz#a29da425b48806f34767a4efce397269af28432c" integrity sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew== +io-ts@^2.2.16: + version "2.2.16" + resolved "https://registry.yarnpkg.com/io-ts/-/io-ts-2.2.16.tgz#597dffa03db1913fc318c9c6df6931cb4ed808b2" + integrity sha512-y5TTSa6VP6le0hhmIyN0dqEXkrZeJLeC5KApJq6VLci3UEKF80lZ+KuoUs02RhBxNWlrqSNxzfI7otLX1Euv8Q== + is-arrayish@^0.3.1: version "0.3.2" resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.3.2.tgz#4574a2ae56f7ab206896fb431eaeed066fdf8f03" -- cgit v1.2.3-70-g09d2