summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--src/api/e926/index.ts (renamed from src/api/e621/index.ts)8
-rw-r--r--src/config.ts6
-rw-r--r--src/index.ts25
-rw-r--r--src/services/dedupe.ts65
-rw-r--r--src/services/jobs.ts22
-rw-r--r--src/services/postDatabase.ts86
-rw-r--r--tsconfig.json1
7 files changed, 130 insertions, 83 deletions
diff --git a/src/api/e621/index.ts b/src/api/e926/index.ts
index a8abbcf..43a3179 100644
--- a/src/api/e621/index.ts
+++ b/src/api/e926/index.ts
@@ -1,7 +1,9 @@
1import got from "got"; 1import got from "got";
2import config from "../../config"; 2import config from "../../config";
3import delay from "../../util/delay"; 3import delay from "../../util/delay";
4import dedupe from "../../services/dedupe"; 4import PostDatabase from "../../services/postDatabase";
5
6export const dedupeDb = new PostDatabase("e926dedupe.json", 50);
5 7
6export interface GetPostQuery { 8export interface GetPostQuery {
7 tags: readonly string[]; 9 tags: readonly string[];
@@ -49,7 +51,7 @@ export async function getPostById(id: number) {
49 return response.posts[0]; 51 return response.posts[0];
50} 52}
51 53
52export async function getRandomPost(query: GetPostQuery) { 54export async function getRandomPost(query: GetPostQuery): Promise<Post> {
53 const page = Math.floor(Math.random() * (query.maxPage - 1)) + 1; 55 const page = Math.floor(Math.random() * (query.maxPage - 1)) + 1;
54 56
55 const response = await client 57 const response = await client
@@ -69,7 +71,7 @@ export async function getRandomPost(query: GetPostQuery) {
69 const postIndex = Math.floor(Math.random() * response.posts.length); 71 const postIndex = Math.floor(Math.random() * response.posts.length);
70 const post = response.posts[postIndex]; 72 const post = response.posts[postIndex];
71 73
72 const isDupe = await dedupe.check({ provider: "e926", id: post.id }); 74 const isDupe = await dedupeDb.insertIfNotExists({ provider: "e926", id: post.id });
73 75
74 if (isDupe) { 76 if (isDupe) {
75 await delay(1000); 77 await delay(1000);
diff --git a/src/config.ts b/src/config.ts
index 2cbcda0..1483d18 100644
--- a/src/config.ts
+++ b/src/config.ts
@@ -1,4 +1,4 @@
1import { GetPostQuery } from "./api/e621"; 1import { GetPostQuery } from "./api/e926";
2 2
3const mainQuery: GetPostQuery = { 3const mainQuery: GetPostQuery = {
4 tags: [ 4 tags: [
@@ -27,10 +27,10 @@ const mainQuery: GetPostQuery = {
27 "-photography_(artwork)", 27 "-photography_(artwork)",
28 "-3d_(artwork)", 28 "-3d_(artwork)",
29 "status:active", 29 "status:active",
30 "score:>=15", 30 "score:>=20",
31 "inpool:false", 31 "inpool:false",
32 ], 32 ],
33 maxPage: 175, 33 maxPage: 117,
34}; 34};
35 35
36export default { 36export default {
diff --git a/src/index.ts b/src/index.ts
index 5558540..86d55ed 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -1,13 +1,20 @@
1import config from "./config"; 1import config from "./config";
2import * as jobs from "./services/jobs"; 2import * as jobs from "./services/jobs";
3import * as cliArgs from "ts-command-line-args"; 3import * as cliArgs from "ts-command-line-args";
4import PostDatabase from "./services/postDatabase";
5
6export const queueDb = new PostDatabase("queue.json", 50);
4 7
5const args = cliArgs.parse<{ 8const args = cliArgs.parse<{
6 id?: number; 9 id?: number;
10 enqueue?: number;
11 dequeue?: number;
7 help?: boolean; 12 help?: boolean;
8}>( 13}>(
9 { 14 {
10 id: { type: Number, optional: true }, 15 id: { type: Number, optional: true },
16 enqueue: { type: Number, optional: true },
17 dequeue: { type: Number, optional: true },
11 help: { type: Boolean, optional: true, alias: "h" }, 18 help: { type: Boolean, optional: true, alias: "h" },
12 }, 19 },
13 { 20 {
@@ -23,7 +30,23 @@ const args = cliArgs.parse<{
23 30
24 if (args.id) { 31 if (args.id) {
25 await jobs.postSpecificPicture(args.id); 32 await jobs.postSpecificPicture(args.id);
33 } else if (args.enqueue) {
34 console.log(`Enqueueing post ${args.enqueue}...`);
35
36 await queueDb.insertIfNotExists({ provider: "e926", id: args.enqueue });
37 } else if (args.dequeue) {
38 console.log(`Dequeueing post ${args.dequeue}...`);
39
40 await queueDb.remove({ provider: "e926", id: args.dequeue });
26 } else { 41 } else {
27 await jobs.postRandomPicture(); 42 console.log("Reading queue...");
43
44 const queued = await queueDb.takeFirst();
45
46 if (queued) {
47 await jobs.postSpecificPicture(queued.id);
48 } else {
49 await jobs.postRandomPicture();
50 }
28 } 51 }
29})(); 52})();
diff --git a/src/services/dedupe.ts b/src/services/dedupe.ts
deleted file mode 100644
index 2eeb5ee..0000000
--- a/src/services/dedupe.ts
+++ /dev/null
@@ -1,65 +0,0 @@
1import fs from "fs/promises";
2import path from "path";
3import * as f from "fp-ts";
4import * as t from "io-ts";
5
6export const E926DedupeEntryC = t.type({
7 provider: t.literal("e926"),
8 id: t.number,
9});
10
11export const DedupeEntryC = E926DedupeEntryC;
12
13export type E926DedupeEntry = t.TypeOf<typeof E926DedupeEntryC>;
14
15export type DedupeEntry = t.TypeOf<typeof DedupeEntryC>;
16
17export class Dedupe {
18 private entries: DedupeEntry[] = [];
19
20 private readonly filePath: string;
21
22 private isLoaded = false;
23
24 constructor(private max: number, filename: string) {
25 this.filePath = path.join(process.cwd(), filename);
26 }
27
28 private async load() {
29 if (this.isLoaded) {
30 return;
31 }
32
33 try {
34 await fs.stat(this.filePath);
35 } catch {
36 await this.save();
37 }
38
39 const fileContent = await fs.readFile(this.filePath, "utf8");
40 const entries = t.array(DedupeEntryC).decode(fileContent);
41
42 if (f.either.isRight(entries)) {
43 this.entries = entries.right;
44 }
45 }
46
47 private async save() {
48 await fs.writeFile(this.filePath, JSON.stringify(this.entries ?? []), "utf8");
49 }
50
51 async check(entry: DedupeEntry) {
52 await this.load();
53
54 const has = !!this.entries.find((e) => e.provider === entry.provider && e.id === entry.id);
55
56 if (!has) {
57 this.entries.push(entry);
58 await this.save();
59 }
60
61 return has;
62 }
63}
64
65export default new Dedupe(50, "dedupe.json");
diff --git a/src/services/jobs.ts b/src/services/jobs.ts
index cf3b894..354c66d 100644
--- a/src/services/jobs.ts
+++ b/src/services/jobs.ts
@@ -1,8 +1,18 @@
1import * as e621 from "../api/e621"; 1import * as e621 from "../api/e926";
2import * as mastodon from "../api/mastodon"; 2import * as mastodon from "../api/mastodon";
3import config from "../config"; 3import config from "../config";
4import Sharp from "sharp"; 4import Sharp from "sharp";
5 5
6export async function postSpecificPicture(id: number) {
7 console.log(`Fetching post ${id}...`);
8
9 const post = await e621.getPostById(id);
10
11 console.log(`Got ${post.id}`);
12
13 await handlePost(post);
14}
15
6export async function postRandomPicture() { 16export async function postRandomPicture() {
7 console.log("Fetching random post..."); 17 console.log("Fetching random post...");
8 18
@@ -15,16 +25,6 @@ export async function postRandomPicture() {
15 await handlePost(post); 25 await handlePost(post);
16} 26}
17 27
18export async function postSpecificPicture(id: number) {
19 console.log(`Fetching post ${id}...`);
20
21 const post = await e621.getPostById(id);
22
23 console.log(`Got ${post.id}`);
24
25 await handlePost(post);
26}
27
28async function handlePost(post: e621.Post) { 28async function handlePost(post: e621.Post) {
29 const source = post.sources.length ? post.sources[0] : undefined; 29 const source = post.sources.length ? post.sources[0] : undefined;
30 const cws = config.cw.filter((w) => post.tags.general.includes(w)); 30 const cws = config.cw.filter((w) => post.tags.general.includes(w));
diff --git a/src/services/postDatabase.ts b/src/services/postDatabase.ts
new file mode 100644
index 0000000..e3be7bb
--- /dev/null
+++ b/src/services/postDatabase.ts
@@ -0,0 +1,86 @@
1import fs from "fs/promises";
2import path from "path";
3import * as f from "fp-ts";
4import * as t from "io-ts";
5
6export const PostDatabaseEntryC = t.type({
7 provider: t.literal("e926"),
8 id: t.number,
9});
10
11export type PostDatabaseEntry = t.TypeOf<typeof PostDatabaseEntryC>;
12
13export class PostDatabase {
14 private entries: PostDatabaseEntry[] = [];
15
16 private readonly filePath: string;
17
18 private isLoaded = false;
19
20 constructor(filename: string, private max?: number) {
21 this.filePath = path.join(process.cwd(), filename);
22 }
23
24 private async load() {
25 if (this.isLoaded) {
26 return;
27 }
28
29 try {
30 await fs.stat(this.filePath);
31 } catch {
32 await this.save();
33 }
34
35 const fileContent = await fs.readFile(this.filePath, "utf8");
36 const entries = t.array(PostDatabaseEntryC).decode(fileContent);
37
38 if (f.either.isRight(entries)) {
39 this.entries = this.max ? entries.right.slice(0, -1 * this.max) : entries.right;
40 }
41 }
42
43 private async save() {
44 await fs.writeFile(this.filePath, JSON.stringify(this.entries), "utf8");
45 }
46
47 async insertIfNotExists(entry: PostDatabaseEntry): Promise<boolean> {
48 await this.load();
49
50 const has = !!this.entries.find((e) => e.provider === entry.provider && e.id === entry.id);
51
52 if (!has) {
53 this.entries.push(entry);
54 await this.save();
55 }
56
57 return has;
58 }
59
60 async takeFirst(): Promise<PostDatabaseEntry | undefined> {
61 await this.load();
62 const entry = this.entries.shift();
63 await this.save();
64 return entry;
65 }
66
67 async remove(entry: PostDatabaseEntry): Promise<boolean> {
68 await this.load();
69 const i = this.entries.findIndex((e) => e.provider === entry.provider && e.id === entry.id);
70
71 if (i === -1) {
72 return false;
73 }
74
75 this.entries.splice(i, 1);
76 await this.save();
77 return true;
78 }
79
80 async getCount() {
81 await this.load();
82 return this.entries.length;
83 }
84}
85
86export default PostDatabase;
diff --git a/tsconfig.json b/tsconfig.json
index 10b9c28..c45ca4c 100644
--- a/tsconfig.json
+++ b/tsconfig.json
@@ -1,5 +1,6 @@
1{ 1{
2 "compilerOptions": { 2 "compilerOptions": {
3 "strict": true,
3 "lib": ["es2020"], 4 "lib": ["es2020"],
4 "moduleResolution": "node", 5 "moduleResolution": "node",
5 "target": "ES2020", 6 "target": "ES2020",