feat(cli): init cli upload moe resources

packages/cli/src/cli.ts

@@ -49,7 +49,8 @@ cli
       const { insertDmhy } = await import('./commands/dmhy');
       await insertDmhy(database, meili, dir);
     } else if (platform === 'moe') {
-      throw new Error('unimplemented');
+      const { insertMoe } = await import('./commands/moe');
+      await insertMoe(database, meili, dir);
     }
 
     await connection.end();

packages/cli/src/commands/dmhy.ts

@@ -1,13 +1,14 @@
 import fs from 'fs-extra';
 import path from 'node:path';
 
-import type { FetchedResource } from 'animegarden';
 import type { Database, NewUser, NewTeam, MeiliSearch } from '@animegarden/database';
 
 import { fetchDmhyPage } from '@animegarden/scraper';
 import { insertDmhyResources, insertTeams, insertUsers } from '@animegarden/database';
 
-import { ufetch } from '../utils';
+import { splitChunks, ufetch } from '../utils';
+
+import { readResources } from './fs';
 
 export async function fetchDmhy(from: number, to: number | undefined, outDir: string) {
   await fs.mkdir(outDir, { recursive: true });
@@ -41,42 +42,8 @@ export async function fetchDmhy(from: number, to: number | undefined, outDir: st
   }
 }
 
-async function readDmhyResources(root: string) {
-  const map = new Map<string, FetchedResource>();
-  const traverse = async (folder: string): Promise<void> => {
-    const files = await fs.readdir(folder);
-    await Promise.all(
-      files.map(async (file) => {
-        const stat = await fs.stat(path.join(folder, file));
-        if (stat.isDirectory()) {
-          await traverse(path.join(folder, file));
-        } else if (stat.isFile() && file.endsWith('.json')) {
-          const p = path.join(folder, file);
-          const content = JSON.parse(await fs.readFile(p, 'utf-8')) as FetchedResource[];
-          for (const r of content) {
-            if (!map.has(r.href)) {
-              map.set(r.href, r);
-            }
-          }
-        }
-      })
-    );
-  };
-
-  await traverse(root);
-  return [...map.values()].sort((lhs, rhs) => rhs.createdAt.localeCompare(lhs.createdAt));
-}
-
-function splitChunks<T>(arr: T[], chunkSize = 1000): T[][] {
-  const chunkedArray = [];
-  for (let i = 0; i < arr.length; i += chunkSize) {
-    chunkedArray.push(arr.slice(i, i + chunkSize));
-  }
-  return chunkedArray;
-}
-
 export async function insertDmhy(database: Database, meili: MeiliSearch, dir: string) {
-  const all = await readDmhyResources(dir);
+  const all = await readResources(dir);
   console.log(`Read ${all.length} dmhy resources`);
 
   const users = new Map<string, NewUser>();

packages/cli/src/commands/fs.ts

@@ -0,0 +1,30 @@
+import fs from 'fs-extra';
+import path from 'node:path';
+
+import type { FetchedResource } from 'animegarden';
+
+export async function readResources(root: string) {
+  const map = new Map<string, FetchedResource>();
+  const traverse = async (folder: string): Promise<void> => {
+    const files = await fs.readdir(folder);
+    await Promise.all(
+      files.map(async (file) => {
+        const stat = await fs.stat(path.join(folder, file));
+        if (stat.isDirectory()) {
+          await traverse(path.join(folder, file));
+        } else if (stat.isFile() && file.endsWith('.json')) {
+          const p = path.join(folder, file);
+          const content = JSON.parse(await fs.readFile(p, 'utf-8')) as FetchedResource[];
+          for (const r of content) {
+            if (!map.has(r.href)) {
+              map.set(r.href, r);
+            }
+          }
+        }
+      })
+    );
+  };
+
+  await traverse(root);
+  return [...map.values()].sort((lhs, rhs) => rhs.createdAt.localeCompare(lhs.createdAt));
+}

packages/cli/src/commands/moe.ts

@@ -1,12 +1,21 @@
 import fs from 'fs-extra';
 import path from 'node:path';
 
-import type { FetchedResource } from 'animegarden';
-import type { Database, NewUser, NewTeam, MeiliSearch } from '@animegarden/database';
+import {
+  type Database,
+  type NewUser,
+  type NewTeam,
+  type MeiliSearch,
+  insertUsers,
+  insertTeams,
+  insertMoeResources
+} from '@animegarden/database';
 
 import { fetchMoePage } from '@animegarden/scraper';
 
-import { ufetch } from '../utils';
+import { splitChunks, ufetch } from '../utils';
+
+import { readResources } from './fs';
 
 export async function fetchMoe(from: number, to: number | undefined, outDir: string) {
   await fs.mkdir(outDir, { recursive: true });
@@ -39,3 +48,40 @@ export async function fetchMoe(from: number, to: number | undefined, outDir: str
     }
   }
 }
+
+export async function insertMoe(database: Database, meili: MeiliSearch, dir: string) {
+  const all = await readResources(dir);
+  console.log(`Read ${all.length} moe resources`);
+
+  const users = new Map<string, NewUser>();
+  const teams = new Map<string, NewTeam>();
+  for (const r of all) {
+    if (!users.has(r.publisher.id)) {
+      users.set(r.publisher.id, {
+        provider: 'moe',
+        providerId: r.publisher.id,
+        name: r.publisher.name,
+        avatar: r.publisher.avatar
+      });
+    }
+    if (r.fansub && !teams.has(r.fansub.id)) {
+      teams.set(r.fansub.id, {
+        provider: 'moe',
+        providerId: r.fansub.id,
+        name: r.fansub.name,
+        avatar: r.fansub.avatar
+      });
+    }
+  }
+
+  const usersResp = await insertUsers(database, [...users.values()]);
+  console.log(`Insert ${usersResp.length} users`);
+  const teamsResp = await insertTeams(database, [...teams.values()]);
+  console.log(`Insert ${teamsResp.length} teams`);
+
+  const chunks = splitChunks(all, 1000);
+  for (const resources of chunks) {
+    const resp = await insertMoeResources(database, meili, resources);
+    console.log(`Insert ${resp.length} dmhy resources`);
+  }
+}

packages/cli/src/utils/index.ts

@@ -25,3 +25,11 @@ export const ufetch = async (url: RequestInfo, init?: RequestInit): Promise<Resp
     return undefined;
   }
 };
+
+export function splitChunks<T>(arr: T[], chunkSize = 1000): T[][] {
+  const chunkedArray = [];
+  for (let i = 0; i < arr.length; i += chunkSize) {
+    chunkedArray.push(arr.slice(i, i + chunkSize));
+  }
+  return chunkedArray;
+}

packages/database/src/operations/moe.ts

@@ -16,4 +16,5 @@ export async function insertMoeResources(
   fetchedResources: FetchedResource[]
 ) {
   // TODO
+  return [];
 }