diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 7459b052..bd378494 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -44,12 +44,15 @@ cli .option('--username ', 'Postgres database username') .option('--password ', 'Postgres database password') .option('--database ', 'Postgres database name') + .option('--meili-url ', 'MeiliSearch URL') + .option('--meili-key ', 'MeiliSearch Key') .action(async (platform, dir, options) => { const { connection, database } = await connect(options); + const meili = await connectMeili({ url: options.meiliUrl, key: options.meiliKey }); if (platform === 'dmhy') { const { insertDmhy } = await import('./commands/dmhy'); - await insertDmhy(database, dir); + await insertDmhy(database, meili, dir); } else if (platform === 'moe') { throw new Error('unimplemented'); } @@ -94,10 +97,10 @@ async function connect(options: { cli .command('meili migrate') - .option('--url ', 'MeiliSearch URL') - .option('--key ', 'MeiliSearch Key') + .option('--meili-url ', 'MeiliSearch URL') + .option('--meili-key ', 'MeiliSearch Key') .action(async (options) => { - const meili = await connectMeili(options); + const meili = await connectMeili({ url: options.meiliUrl, key: options.meiliKey }); const index = 'resources'; // Create index diff --git a/packages/cli/src/commands/dmhy.ts b/packages/cli/src/commands/dmhy.ts index a597ec10..7a6b5caf 100644 --- a/packages/cli/src/commands/dmhy.ts +++ b/packages/cli/src/commands/dmhy.ts @@ -2,7 +2,7 @@ import fs from 'fs-extra'; import path from 'node:path'; import type { FetchedResource } from 'animegarden'; -import type { Database, NewUser, NewTeam } from '@animegarden/database'; +import type { Database, NewUser, NewTeam, MeiliSearch } from '@animegarden/database'; import { fetchDmhyPage } from '@animegarden/scraper'; import { insertDmhyResources, insertTeams, insertUsers } from '@animegarden/database'; @@ -75,7 +75,7 @@ function splitChunks(arr: T[], chunkSize = 1000): T[][] { return chunkedArray; } -export async function insertDmhy(database: Database, dir: string) { +export async function insertDmhy(database: Database, meili: MeiliSearch, dir: string) { const all = await readDmhyResources(dir); console.log(`Read ${all.length} dmhy resources`); @@ -105,7 +105,7 @@ export async function insertDmhy(database: Database, dir: string) { const chunks = splitChunks(all, 1000); for (const resources of chunks) { - const resp = await insertDmhyResources(database, resources); + const resp = await insertDmhyResources(database, meili, resources); console.log(`Insert ${resp.length} dmhy resources`); } } diff --git a/packages/database/src/meilisearch.ts b/packages/database/src/meilisearch.ts index 188111b0..2fecca2b 100644 --- a/packages/database/src/meilisearch.ts +++ b/packages/database/src/meilisearch.ts @@ -1,4 +1,5 @@ -import { Resource } from 'animegarden'; +import type { Resource } from './schema'; + import { MeiliSearch } from 'meilisearch'; export { MeiliSearch }; @@ -10,7 +11,7 @@ export function connectMeiliSearch(host: string, key: string): MeiliSearch { }); } -export async function insertResource(client: MeiliSearch, resources: Resource[]) { +export async function insertResourceDocuments(client: MeiliSearch, resources: Resource[]) { const resp = await client.index('resources').addDocuments(resources); return resp; } diff --git a/packages/database/src/operations/index.ts b/packages/database/src/operations/index.ts index 52268551..3f023b1e 100644 --- a/packages/database/src/operations/index.ts +++ b/packages/database/src/operations/index.ts @@ -1 +1,3 @@ +export * from './sync'; + export * from './insert'; diff --git a/packages/database/src/operations/insert.ts b/packages/database/src/operations/insert.ts index bda124b8..3ebeaac9 100644 --- a/packages/database/src/operations/insert.ts +++ b/packages/database/src/operations/insert.ts @@ -8,6 +8,7 @@ import type { NewUser, NewTeam, NewResource, Resource } from '../schema'; import { users } from '../schema/user'; import { teams } from '../schema/team'; import { resources } from '../schema/resource'; +import { insertResourceDocuments } from '../meilisearch'; export async function insertUsers(database: Database, newUsers: NewUser[]) { return await database @@ -49,7 +50,7 @@ export async function insertDmhyResources( } }) .filter(Boolean) as Resource[]; - await meili.index('resources').addDocuments(docs); + await insertResourceDocuments(meili, docs); return data; } diff --git a/packages/database/src/operations/sync.ts b/packages/database/src/operations/sync.ts new file mode 100644 index 00000000..2370b3a4 --- /dev/null +++ b/packages/database/src/operations/sync.ts @@ -0,0 +1,11 @@ +import type MeiliSearch from 'meilisearch'; + +import type { Database } from '../connection'; + +import { insertResourceDocuments } from '../meilisearch'; + +export async function syncResourcesToMeili(database: Database, meili: MeiliSearch) { + const res = await database.query.resources.findMany({ offset: 0, limit: 1000 }); + await insertResourceDocuments(meili, res); + return { count: res.length }; +} diff --git a/packages/server/src/admin/dmhy.ts b/packages/server/src/admin/dmhy.ts index 362570a5..fa92cc26 100644 --- a/packages/server/src/admin/dmhy.ts +++ b/packages/server/src/admin/dmhy.ts @@ -12,6 +12,7 @@ import { import { storage } from '../storage'; import { database } from '../database'; +import { meiliSearch } from '../meilisearch'; import { logger as rootLogger } from '../logger'; const logger = rootLogger.forkIntegrationLogger('dmhy'); @@ -96,7 +97,7 @@ export async function refreshDmhyResources() { } } - const result = await insertDmhyResources(database, res); + const result = await insertDmhyResources(database, meiliSearch, res); const count = result.length; if (count === 0) break; diff --git a/packages/server/src/admin/index.ts b/packages/server/src/admin/index.ts index 5ba65535..3095f54b 100644 --- a/packages/server/src/admin/index.ts +++ b/packages/server/src/admin/index.ts @@ -1,5 +1,6 @@ import { registerApp } from '../app'; +import { syncDocuments } from './meili'; import { refreshDmhyResources } from './dmhy'; export function registerAdmin() { @@ -8,5 +9,10 @@ export function registerAdmin() { const r = await refreshDmhyResources(); return req.json(r); }); + + app.post(`/admin/resources/sync`, async (req) => { + const r = await syncDocuments(); + return req.json(r); + }); }); } diff --git a/packages/server/src/admin/meili.ts b/packages/server/src/admin/meili.ts new file mode 100644 index 00000000..e07a452e --- /dev/null +++ b/packages/server/src/admin/meili.ts @@ -0,0 +1,8 @@ +import { syncResourcesToMeili } from '@animegarden/database'; + +import { database } from '../database'; +import { meiliSearch } from '../meilisearch'; + +export async function syncDocuments() { + return await syncResourcesToMeili(database, meiliSearch); +}