feat(database): sync meili search

packages/cli/src/cli.ts

@@ -44,12 +44,15 @@ cli
   .option('--username <string>', 'Postgres database username')
   .option('--password <string>', 'Postgres database password')
   .option('--database <string>', 'Postgres database name')
+  .option('--meili-url <url>', 'MeiliSearch URL')
+  .option('--meili-key <key>', 'MeiliSearch Key')
   .action(async (platform, dir, options) => {
     const { connection, database } = await connect(options);
+    const meili = await connectMeili({ url: options.meiliUrl, key: options.meiliKey });
 
     if (platform === 'dmhy') {
       const { insertDmhy } = await import('./commands/dmhy');
-      await insertDmhy(database, dir);
+      await insertDmhy(database, meili, dir);
     } else if (platform === 'moe') {
       throw new Error('unimplemented');
     }
@@ -94,10 +97,10 @@ async function connect(options: {
 
 cli
   .command('meili migrate')
-  .option('--url <url>', 'MeiliSearch URL')
-  .option('--key <key>', 'MeiliSearch Key')
+  .option('--meili-url <url>', 'MeiliSearch URL')
+  .option('--meili-key <key>', 'MeiliSearch Key')
   .action(async (options) => {
-    const meili = await connectMeili(options);
+    const meili = await connectMeili({ url: options.meiliUrl, key: options.meiliKey });
     const index = 'resources';
 
     // Create index

packages/cli/src/commands/dmhy.ts

@@ -2,7 +2,7 @@ import fs from 'fs-extra';
 import path from 'node:path';
 
 import type { FetchedResource } from 'animegarden';
-import type { Database, NewUser, NewTeam } from '@animegarden/database';
+import type { Database, NewUser, NewTeam, MeiliSearch } from '@animegarden/database';
 
 import { fetchDmhyPage } from '@animegarden/scraper';
 import { insertDmhyResources, insertTeams, insertUsers } from '@animegarden/database';
@@ -75,7 +75,7 @@ function splitChunks<T>(arr: T[], chunkSize = 1000): T[][] {
   return chunkedArray;
 }
 
-export async function insertDmhy(database: Database, dir: string) {
+export async function insertDmhy(database: Database, meili: MeiliSearch, dir: string) {
   const all = await readDmhyResources(dir);
   console.log(`Read ${all.length} dmhy resources`);
 
@@ -105,7 +105,7 @@ export async function insertDmhy(database: Database, dir: string) {
 
   const chunks = splitChunks(all, 1000);
   for (const resources of chunks) {
-    const resp = await insertDmhyResources(database, resources);
+    const resp = await insertDmhyResources(database, meili, resources);
     console.log(`Insert ${resp.length} dmhy resources`);
   }
 }

packages/database/src/meilisearch.ts

@@ -1,4 +1,5 @@
-import { Resource } from 'animegarden';
+import type { Resource } from './schema';
+
 import { MeiliSearch } from 'meilisearch';
 
 export { MeiliSearch };
@@ -10,7 +11,7 @@ export function connectMeiliSearch(host: string, key: string): MeiliSearch {
   });
 }
 
-export async function insertResource(client: MeiliSearch, resources: Resource[]) {
+export async function insertResourceDocuments(client: MeiliSearch, resources: Resource[]) {
   const resp = await client.index('resources').addDocuments(resources);
   return resp;
 }

packages/database/src/operations/index.ts

@@ -1 +1,3 @@
+export * from './sync';
+
 export * from './insert';

packages/database/src/operations/insert.ts

@@ -8,6 +8,7 @@ import type { NewUser, NewTeam, NewResource, Resource } from '../schema';
 import { users } from '../schema/user';
 import { teams } from '../schema/team';
 import { resources } from '../schema/resource';
+import { insertResourceDocuments } from '../meilisearch';
 
 export async function insertUsers(database: Database, newUsers: NewUser[]) {
   return await database
@@ -49,7 +50,7 @@ export async function insertDmhyResources(
       }
     })
     .filter(Boolean) as Resource[];
-  await meili.index('resources').addDocuments(docs);
+  await insertResourceDocuments(meili, docs);
 
   return data;
 }

packages/database/src/operations/sync.ts

@@ -0,0 +1,11 @@
+import type MeiliSearch from 'meilisearch';
+
+import type { Database } from '../connection';
+
+import { insertResourceDocuments } from '../meilisearch';
+
+export async function syncResourcesToMeili(database: Database, meili: MeiliSearch) {
+  const res = await database.query.resources.findMany({ offset: 0, limit: 1000 });
+  await insertResourceDocuments(meili, res);
+  return { count: res.length };
+}

packages/server/src/admin/dmhy.ts

@@ -12,6 +12,7 @@ import {
 
 import { storage } from '../storage';
 import { database } from '../database';
+import { meiliSearch } from '../meilisearch';
 import { logger as rootLogger } from '../logger';
 
 const logger = rootLogger.forkIntegrationLogger('dmhy');
@@ -96,7 +97,7 @@ export async function refreshDmhyResources() {
       }
     }
 
-    const result = await insertDmhyResources(database, res);
+    const result = await insertDmhyResources(database, meiliSearch, res);
     const count = result.length;
 
     if (count === 0) break;

packages/server/src/admin/index.ts

@@ -1,5 +1,6 @@
 import { registerApp } from '../app';
 
+import { syncDocuments } from './meili';
 import { refreshDmhyResources } from './dmhy';
 
 export function registerAdmin() {
@@ -8,5 +9,10 @@ export function registerAdmin() {
       const r = await refreshDmhyResources();
       return req.json(r);
     });
+
+    app.post(`/admin/resources/sync`, async (req) => {
+      const r = await syncDocuments();
+      return req.json(r);
+    });
   });
 }

packages/server/src/admin/meili.ts

@@ -0,0 +1,8 @@
+import { syncResourcesToMeili } from '@animegarden/database';
+
+import { database } from '../database';
+import { meiliSearch } from '../meilisearch';
+
+export async function syncDocuments() {
+  return await syncResourcesToMeili(database, meiliSearch);
+}