diff options
author | Volpeon <git@volpeon.ink> | 2021-10-19 19:25:03 +0200 |
---|---|---|
committer | Volpeon <git@volpeon.ink> | 2021-10-19 19:25:03 +0200 |
commit | 163d8119c109c42e64ab37b01dec131f2cb5bf61 (patch) | |
tree | ee0abce3ce349482ca63b4064fcbebc0e5d2f4ff | |
parent | Code improvements (diff) | |
download | feralbot-163d8119c109c42e64ab37b01dec131f2cb5bf61.tar.gz feralbot-163d8119c109c42e64ab37b01dec131f2cb5bf61.tar.bz2 feralbot-163d8119c109c42e64ab37b01dec131f2cb5bf61.zip |
Code improvements, support a manual post queue
-rw-r--r-- | src/api/e926/index.ts (renamed from src/api/e621/index.ts) | 8 | ||||
-rw-r--r-- | src/config.ts | 6 | ||||
-rw-r--r-- | src/index.ts | 25 | ||||
-rw-r--r-- | src/services/dedupe.ts | 65 | ||||
-rw-r--r-- | src/services/jobs.ts | 22 | ||||
-rw-r--r-- | src/services/postDatabase.ts | 86 | ||||
-rw-r--r-- | tsconfig.json | 1 |
7 files changed, 130 insertions, 83 deletions
diff --git a/src/api/e621/index.ts b/src/api/e926/index.ts index a8abbcf..43a3179 100644 --- a/src/api/e621/index.ts +++ b/src/api/e926/index.ts | |||
@@ -1,7 +1,9 @@ | |||
1 | import got from "got"; | 1 | import got from "got"; |
2 | import config from "../../config"; | 2 | import config from "../../config"; |
3 | import delay from "../../util/delay"; | 3 | import delay from "../../util/delay"; |
4 | import dedupe from "../../services/dedupe"; | 4 | import PostDatabase from "../../services/postDatabase"; |
5 | |||
6 | export const dedupeDb = new PostDatabase("e926dedupe.json", 50); | ||
5 | 7 | ||
6 | export interface GetPostQuery { | 8 | export interface GetPostQuery { |
7 | tags: readonly string[]; | 9 | tags: readonly string[]; |
@@ -49,7 +51,7 @@ export async function getPostById(id: number) { | |||
49 | return response.posts[0]; | 51 | return response.posts[0]; |
50 | } | 52 | } |
51 | 53 | ||
52 | export async function getRandomPost(query: GetPostQuery) { | 54 | export async function getRandomPost(query: GetPostQuery): Promise<Post> { |
53 | const page = Math.floor(Math.random() * (query.maxPage - 1)) + 1; | 55 | const page = Math.floor(Math.random() * (query.maxPage - 1)) + 1; |
54 | 56 | ||
55 | const response = await client | 57 | const response = await client |
@@ -69,7 +71,7 @@ export async function getRandomPost(query: GetPostQuery) { | |||
69 | const postIndex = Math.floor(Math.random() * response.posts.length); | 71 | const postIndex = Math.floor(Math.random() * response.posts.length); |
70 | const post = response.posts[postIndex]; | 72 | const post = response.posts[postIndex]; |
71 | 73 | ||
72 | const isDupe = await dedupe.check({ provider: "e926", id: post.id }); | 74 | const isDupe = await dedupeDb.insertIfNotExists({ provider: "e926", id: post.id }); |
73 | 75 | ||
74 | if (isDupe) { | 76 | if (isDupe) { |
75 | await delay(1000); | 77 | await delay(1000); |
diff --git a/src/config.ts b/src/config.ts index 2cbcda0..1483d18 100644 --- a/src/config.ts +++ b/src/config.ts | |||
@@ -1,4 +1,4 @@ | |||
1 | import { GetPostQuery } from "./api/e621"; | 1 | import { GetPostQuery } from "./api/e926"; |
2 | 2 | ||
3 | const mainQuery: GetPostQuery = { | 3 | const mainQuery: GetPostQuery = { |
4 | tags: [ | 4 | tags: [ |
@@ -27,10 +27,10 @@ const mainQuery: GetPostQuery = { | |||
27 | "-photography_(artwork)", | 27 | "-photography_(artwork)", |
28 | "-3d_(artwork)", | 28 | "-3d_(artwork)", |
29 | "status:active", | 29 | "status:active", |
30 | "score:>=15", | 30 | "score:>=20", |
31 | "inpool:false", | 31 | "inpool:false", |
32 | ], | 32 | ], |
33 | maxPage: 175, | 33 | maxPage: 117, |
34 | }; | 34 | }; |
35 | 35 | ||
36 | export default { | 36 | export default { |
diff --git a/src/index.ts b/src/index.ts index 5558540..86d55ed 100644 --- a/src/index.ts +++ b/src/index.ts | |||
@@ -1,13 +1,20 @@ | |||
1 | import config from "./config"; | 1 | import config from "./config"; |
2 | import * as jobs from "./services/jobs"; | 2 | import * as jobs from "./services/jobs"; |
3 | import * as cliArgs from "ts-command-line-args"; | 3 | import * as cliArgs from "ts-command-line-args"; |
4 | import PostDatabase from "./services/postDatabase"; | ||
5 | |||
6 | export const queueDb = new PostDatabase("queue.json", 50); | ||
4 | 7 | ||
5 | const args = cliArgs.parse<{ | 8 | const args = cliArgs.parse<{ |
6 | id?: number; | 9 | id?: number; |
10 | enqueue?: number; | ||
11 | dequeue?: number; | ||
7 | help?: boolean; | 12 | help?: boolean; |
8 | }>( | 13 | }>( |
9 | { | 14 | { |
10 | id: { type: Number, optional: true }, | 15 | id: { type: Number, optional: true }, |
16 | enqueue: { type: Number, optional: true }, | ||
17 | dequeue: { type: Number, optional: true }, | ||
11 | help: { type: Boolean, optional: true, alias: "h" }, | 18 | help: { type: Boolean, optional: true, alias: "h" }, |
12 | }, | 19 | }, |
13 | { | 20 | { |
@@ -23,7 +30,23 @@ const args = cliArgs.parse<{ | |||
23 | 30 | ||
24 | if (args.id) { | 31 | if (args.id) { |
25 | await jobs.postSpecificPicture(args.id); | 32 | await jobs.postSpecificPicture(args.id); |
33 | } else if (args.enqueue) { | ||
34 | console.log(`Enqueueing post ${args.enqueue}...`); | ||
35 | |||
36 | await queueDb.insertIfNotExists({ provider: "e926", id: args.enqueue }); | ||
37 | } else if (args.dequeue) { | ||
38 | console.log(`Dequeueing post ${args.dequeue}...`); | ||
39 | |||
40 | await queueDb.remove({ provider: "e926", id: args.dequeue }); | ||
26 | } else { | 41 | } else { |
27 | await jobs.postRandomPicture(); | 42 | console.log("Reading queue..."); |
43 | |||
44 | const queued = await queueDb.takeFirst(); | ||
45 | |||
46 | if (queued) { | ||
47 | await jobs.postSpecificPicture(queued.id); | ||
48 | } else { | ||
49 | await jobs.postRandomPicture(); | ||
50 | } | ||
28 | } | 51 | } |
29 | })(); | 52 | })(); |
diff --git a/src/services/dedupe.ts b/src/services/dedupe.ts deleted file mode 100644 index 2eeb5ee..0000000 --- a/src/services/dedupe.ts +++ /dev/null | |||
@@ -1,65 +0,0 @@ | |||
1 | import fs from "fs/promises"; | ||
2 | import path from "path"; | ||
3 | import * as f from "fp-ts"; | ||
4 | import * as t from "io-ts"; | ||
5 | |||
6 | export const E926DedupeEntryC = t.type({ | ||
7 | provider: t.literal("e926"), | ||
8 | id: t.number, | ||
9 | }); | ||
10 | |||
11 | export const DedupeEntryC = E926DedupeEntryC; | ||
12 | |||
13 | export type E926DedupeEntry = t.TypeOf<typeof E926DedupeEntryC>; | ||
14 | |||
15 | export type DedupeEntry = t.TypeOf<typeof DedupeEntryC>; | ||
16 | |||
17 | export class Dedupe { | ||
18 | private entries: DedupeEntry[] = []; | ||
19 | |||
20 | private readonly filePath: string; | ||
21 | |||
22 | private isLoaded = false; | ||
23 | |||
24 | constructor(private max: number, filename: string) { | ||
25 | this.filePath = path.join(process.cwd(), filename); | ||
26 | } | ||
27 | |||
28 | private async load() { | ||
29 | if (this.isLoaded) { | ||
30 | return; | ||
31 | } | ||
32 | |||
33 | try { | ||
34 | await fs.stat(this.filePath); | ||
35 | } catch { | ||
36 | await this.save(); | ||
37 | } | ||
38 | |||
39 | const fileContent = await fs.readFile(this.filePath, "utf8"); | ||
40 | const entries = t.array(DedupeEntryC).decode(fileContent); | ||
41 | |||
42 | if (f.either.isRight(entries)) { | ||
43 | this.entries = entries.right; | ||
44 | } | ||
45 | } | ||
46 | |||
47 | private async save() { | ||
48 | await fs.writeFile(this.filePath, JSON.stringify(this.entries ?? []), "utf8"); | ||
49 | } | ||
50 | |||
51 | async check(entry: DedupeEntry) { | ||
52 | await this.load(); | ||
53 | |||
54 | const has = !!this.entries.find((e) => e.provider === entry.provider && e.id === entry.id); | ||
55 | |||
56 | if (!has) { | ||
57 | this.entries.push(entry); | ||
58 | await this.save(); | ||
59 | } | ||
60 | |||
61 | return has; | ||
62 | } | ||
63 | } | ||
64 | |||
65 | export default new Dedupe(50, "dedupe.json"); | ||
diff --git a/src/services/jobs.ts b/src/services/jobs.ts index cf3b894..354c66d 100644 --- a/src/services/jobs.ts +++ b/src/services/jobs.ts | |||
@@ -1,8 +1,18 @@ | |||
1 | import * as e621 from "../api/e621"; | 1 | import * as e621 from "../api/e926"; |
2 | import * as mastodon from "../api/mastodon"; | 2 | import * as mastodon from "../api/mastodon"; |
3 | import config from "../config"; | 3 | import config from "../config"; |
4 | import Sharp from "sharp"; | 4 | import Sharp from "sharp"; |
5 | 5 | ||
6 | export async function postSpecificPicture(id: number) { | ||
7 | console.log(`Fetching post ${id}...`); | ||
8 | |||
9 | const post = await e621.getPostById(id); | ||
10 | |||
11 | console.log(`Got ${post.id}`); | ||
12 | |||
13 | await handlePost(post); | ||
14 | } | ||
15 | |||
6 | export async function postRandomPicture() { | 16 | export async function postRandomPicture() { |
7 | console.log("Fetching random post..."); | 17 | console.log("Fetching random post..."); |
8 | 18 | ||
@@ -15,16 +25,6 @@ export async function postRandomPicture() { | |||
15 | await handlePost(post); | 25 | await handlePost(post); |
16 | } | 26 | } |
17 | 27 | ||
18 | export async function postSpecificPicture(id: number) { | ||
19 | console.log(`Fetching post ${id}...`); | ||
20 | |||
21 | const post = await e621.getPostById(id); | ||
22 | |||
23 | console.log(`Got ${post.id}`); | ||
24 | |||
25 | await handlePost(post); | ||
26 | } | ||
27 | |||
28 | async function handlePost(post: e621.Post) { | 28 | async function handlePost(post: e621.Post) { |
29 | const source = post.sources.length ? post.sources[0] : undefined; | 29 | const source = post.sources.length ? post.sources[0] : undefined; |
30 | const cws = config.cw.filter((w) => post.tags.general.includes(w)); | 30 | const cws = config.cw.filter((w) => post.tags.general.includes(w)); |
diff --git a/src/services/postDatabase.ts b/src/services/postDatabase.ts new file mode 100644 index 0000000..e3be7bb --- /dev/null +++ b/src/services/postDatabase.ts | |||
@@ -0,0 +1,86 @@ | |||
1 | import fs from "fs/promises"; | ||
2 | import path from "path"; | ||
3 | import * as f from "fp-ts"; | ||
4 | import * as t from "io-ts"; | ||
5 | |||
6 | export const PostDatabaseEntryC = t.type({ | ||
7 | provider: t.literal("e926"), | ||
8 | id: t.number, | ||
9 | }); | ||
10 | |||
11 | export type PostDatabaseEntry = t.TypeOf<typeof PostDatabaseEntryC>; | ||
12 | |||
13 | export class PostDatabase { | ||
14 | private entries: PostDatabaseEntry[] = []; | ||
15 | |||
16 | private readonly filePath: string; | ||
17 | |||
18 | private isLoaded = false; | ||
19 | |||
20 | constructor(filename: string, private max?: number) { | ||
21 | this.filePath = path.join(process.cwd(), filename); | ||
22 | } | ||
23 | |||
24 | private async load() { | ||
25 | if (this.isLoaded) { | ||
26 | return; | ||
27 | } | ||
28 | |||
29 | try { | ||
30 | await fs.stat(this.filePath); | ||
31 | } catch { | ||
32 | await this.save(); | ||
33 | } | ||
34 | |||
35 | const fileContent = await fs.readFile(this.filePath, "utf8"); | ||
36 | const entries = t.array(PostDatabaseEntryC).decode(fileContent); | ||
37 | |||
38 | if (f.either.isRight(entries)) { | ||
39 | this.entries = this.max ? entries.right.slice(0, -1 * this.max) : entries.right; | ||
40 | } | ||
41 | } | ||
42 | |||
43 | private async save() { | ||
44 | await fs.writeFile(this.filePath, JSON.stringify(this.entries), "utf8"); | ||
45 | } | ||
46 | |||
47 | async insertIfNotExists(entry: PostDatabaseEntry): Promise<boolean> { | ||
48 | await this.load(); | ||
49 | |||
50 | const has = !!this.entries.find((e) => e.provider === entry.provider && e.id === entry.id); | ||
51 | |||
52 | if (!has) { | ||
53 | this.entries.push(entry); | ||
54 | await this.save(); | ||
55 | } | ||
56 | |||
57 | return has; | ||
58 | } | ||
59 | |||
60 | async takeFirst(): Promise<PostDatabaseEntry | undefined> { | ||
61 | await this.load(); | ||
62 | const entry = this.entries.shift(); | ||
63 | await this.save(); | ||
64 | return entry; | ||
65 | } | ||
66 | |||
67 | async remove(entry: PostDatabaseEntry): Promise<boolean> { | ||
68 | await this.load(); | ||
69 | const i = this.entries.findIndex((e) => e.provider === entry.provider && e.id === entry.id); | ||
70 | |||
71 | if (i === -1) { | ||
72 | return false; | ||
73 | } | ||
74 | |||
75 | this.entries.splice(i, 1); | ||
76 | await this.save(); | ||
77 | return true; | ||
78 | } | ||
79 | |||
80 | async getCount() { | ||
81 | await this.load(); | ||
82 | return this.entries.length; | ||
83 | } | ||
84 | } | ||
85 | |||
86 | export default PostDatabase; | ||
diff --git a/tsconfig.json b/tsconfig.json index 10b9c28..c45ca4c 100644 --- a/tsconfig.json +++ b/tsconfig.json | |||
@@ -1,5 +1,6 @@ | |||
1 | { | 1 | { |
2 | "compilerOptions": { | 2 | "compilerOptions": { |
3 | "strict": true, | ||
3 | "lib": ["es2020"], | 4 | "lib": ["es2020"], |
4 | "moduleResolution": "node", | 5 | "moduleResolution": "node", |
5 | "target": "ES2020", | 6 | "target": "ES2020", |