feat(server): list from database

yjl9903 · Jan 16, 2024 · 6763fa5 · 6763fa5
1 parent 02ff03a
commit 6763fa5
Show file tree

Hide file tree

Showing 2 changed files with 188 additions and 3 deletions.
diff --git a/packages/server/src/query/detail.ts b/packages/server/src/query/detail.ts
@@ -6,9 +6,9 @@ import { prefixStorage } from 'unstorage';
 import { resources } from '@animegarden/database';
 import { fetchDmhyDetail } from '@animegarden/scraper';
 
-import { logger as rootLogger } from '../logger';
 import { storage } from '../storage';
 import { database } from '../database';
+import { logger as rootLogger } from '../logger';
 
 const logger = rootLogger.forkIntegrationLogger('detail');
 

diff --git a/packages/server/src/query/resources.ts b/packages/server/src/query/resources.ts
@@ -1,7 +1,22 @@
 import type { Context } from 'hono';
-import type { ResolvedFilterOptions } from 'animegarden';
+import type { Resource as DbResource } from '@animegarden/database';
 
+import { hash } from 'ohash';
+import { memoAsync } from 'memofunc';
+import { prefixStorage } from 'unstorage';
+import { and, eq, gte, ilike, lte, notIlike, or, desc } from 'drizzle-orm';
+
+import { normalizeTitle, Resource, type ResolvedFilterOptions, ResourceType } from 'animegarden';
+import { getRefreshTimestamp, getTeam, getUser, resources } from '@animegarden/database';
+
+import { storage } from '../storage';
+import { database } from '../database';
 import { searchResources } from './search';
+import { logger as rootLogger } from '../logger';
+
+const logger = rootLogger.forkIntegrationLogger('detail');
+
+const resourcesStorage = prefixStorage(storage, 'resources');
 
 export async function queryResources(ctx: Context, filter: ResolvedFilterOptions) {
   if (filter.search) {
@@ -12,6 +27,176 @@ export async function queryResources(ctx: Context, filter: ResolvedFilterOptions
       complete: resp.complete
     };
   } else {
-    return { resources: [], complete: true };
+    const resp = await listResourcesFromDB(filter);
+    return {
+      timestamp: resp.timestamp,
+      resources: resp.resources,
+      complete: resp.complete
+    };
+  }
+}
+
+const listResourcesFromDB = memoAsync(
+  async (options: ResolvedFilterOptions) => {
+    const timestampPromise = getRefreshTimestamp(storage);
+
+    const {
+      page,
+      pageSize,
+      fansubId,
+      fansubName,
+      publisherId,
+      type,
+      before,
+      after,
+      include,
+      exclude
+    } = options;
+
+    const result = await database
+      .select()
+      .from(resources)
+      .where(
+        and(
+          eq(resources.isDeleted, false),
+          eq(resources.isDuplicated, options.duplicate),
+          type ? eq(resources.type, type) : undefined,
+          after ? gte(resources.createdAt, after) : undefined,
+          before ? lte(resources.createdAt, before) : undefined,
+          // TODO: fansub
+          // TODO: publisher
+          // Include
+          include && include?.length > 0
+            ? or(...include.map((t) => ilike(resources.titleAlt, `%${normalizeTitle(t)}%`)))
+            : undefined,
+          ...(exclude?.map((t) => notIlike(resources.titleAlt, `%${normalizeTitle(t)}%`)) ?? [])
+        )
+      )
+      .orderBy(desc(resources.createdAt), desc(resources.id))
+      .offset((page - 1) * pageSize)
+      .limit(pageSize + 1); // Used for determining whether there are rest resources
+
+    const timestamp = await timestampPromise;
+
+    return {
+      timestamp,
+      resources: await transformFromDb(result.slice(0, pageSize)),
+      complete: result.length <= pageSize
+    };
+  },
+  {
+    external: {
+      async get([params]) {
+        const key = hash(params);
+        const [resp, timestamp] = await Promise.all([
+          resourcesStorage.getItem<{ timestamp: Date; resources: Resource[]; complete: boolean }>(
+            key
+          ),
+          getRefreshTimestamp(storage)
+        ] as const);
+        if (resp) {
+          // Hack: due to the serialization issue, manually transform data
+          resp.timestamp = new Date(resp.timestamp);
+          if (resp.timestamp.getTime() === timestamp.getTime()) {
+            logger.info(`Resources list cache ${key} hit (now is ${resp.timestamp})`);
+            return resp;
+          }
+          // Cache stale
+          logger.info(
+            `Resources list cache ${key} stale (fetched at ${resp.timestamp}, now is ${timestamp})`
+          );
+          await resourcesStorage.removeItem(key);
+        }
+        // Cache miss
+        logger.info(`Resources list cache ${key} miss (now is ${timestamp})`);
+        return undefined;
+      },
+      async set([params], value) {
+        // Cache set
+        const key = hash(params);
+        logger.info(`Resources list cache ${key} has been set at ${value.timestamp}`);
+
+        return resourcesStorage.setItem(key, value, {
+          // Cached 1 hour
+          ttl: 60 * 60
+        });
+      },
+      async remove([params]) {
+        return resourcesStorage.removeItem(hash(params));
+      },
+      async clear() {}
+    }
+  }
+);
+
+async function transformFromDb(resources: DbResource[]) {
+  const result: Resource[] = [];
+  for (const r of resources) {
+    const fansub = r.fansubId ? await getTeam(database, r.provider, r.fansubId) : undefined;
+    const publisher = (await getUser(database, r.provider, r.publisherId))!;
+
+    const href = r.provider === 'dmhy' ? `https://share.dmhy.org/topics/view/${r.href}` : r.href;
+    const fansubHref =
+      r.provider === 'dmhy' && r.fansubId
+        ? `https://share.dmhy.org/topics/list/team_id/${r.fansubId}`
+        : undefined;
+    const publisherHref =
+      r.provider === 'dmhy'
+        ? `https://share.dmhy.org/topics/list/user_id/${r.publisherId}`
+        : undefined;
+
+    result.push({
+      id: r.id,
+      provider: r.provider,
+      providerId: r.providerId,
+      title: r.title,
+      href,
+      type: r.type as ResourceType,
+      magnet: r.magnet,
+      size: r.size,
+      // When reading this field from cache, it will be transfromed to string
+      createdAt: new Date(r.createdAt!).toISOString(),
+      fetchedAt: new Date(r.fetchedAt!).toISOString(),
+      fansub: fansub
+        ? {
+            id: fansub.providerId,
+            name: fansub.name,
+            // @ts-ignore
+            href: fansubHref
+          }
+        : undefined,
+      publisher: {
+        id: publisher.providerId,
+        name: publisher.name,
+        // @ts-ignore
+        href: publisherHref
+      }
+    });
   }
+  return result;
+  // return result.map((r) => ({
+  //   id: r.id,
+  //   title: r.title,
+  //   href: `https://share.dmhy.org/topics/view/${r.href}`,
+  //   type: r.type,
+  //   magnet: r.magnet,
+  //   size: r.size,
+  //   // When reading this field from cache, it will be transfromed to string
+  //   createdAt:
+  //     typeof r.createdAt === 'string' && /^\d+$/.test(r.createdAt)
+  //       ? new Date(Number(r.createdAt))
+  //       : new Date(r.createdAt),
+  //   fansub: r.fansubName
+  //     ? {
+  //         id: r.fansubId,
+  //         name: r.fansubName,
+  //         href: `https://share.dmhy.org/topics/list/team_id/${r.fansubId}`
+  //       }
+  //     : undefined,
+  //   publisher: {
+  //     id: r.publisherId,
+  //     name: r.publisherName!,
+  //     href: `https://share.dmhy.org/topics/list/user_id/${r.publisherId}`
+  //   }
+  // }));
 }