feat(animegarden)!: change filter api

yjl9903 · Jan 16, 2024 · d4ffb4c · d4ffb4c
1 parent c966c8a
commit d4ffb4c
Show file tree

Hide file tree

Showing 5 changed files with 60 additions and 77 deletions.
diff --git a/packages/animegarden/src/garden/filter.ts b/packages/animegarden/src/garden/filter.ts
@@ -11,6 +11,10 @@ export function makeResourcesFilter(
   const resolved = parseSearchURL(new URLSearchParams(), options);
   const chains: Array<(resource: Resource) => boolean> = [];
 
+  if (resolved.provider) {
+    const providers = resolved.provider;
+    chains.push((r) => providers.includes(r.provider));
+  }
   if (resolved.fansubId) {
     const fansubId = resolved.fansubId.map((id) => '' + id);
     chains.push((r) => (r.fansub ? fansubId.includes(r.fansub.id) : false));
@@ -36,24 +40,25 @@ export function makeResourcesFilter(
     chains.push((r) => new Date(r.createdAt).getTime() >= after);
   }
 
-  if (resolved.include || resolved.search) {
-    const include = resolved.include ?? [];
-    if (resolved.search) {
-      include.push(resolved.search);
-    }
-    for (const arr of include) {
-      arr.splice(0, arr.length, ...arr.map((k) => normalizeTitle(k).toLowerCase()));
-    }
+  if (resolved.search) {
+    // Match at least one search keyword
+    const search = resolved.search.map((k) => normalizeTitle(k).toLowerCase());
     chains.push((r) => {
       const titleAlt = normalizeTitle(r.title).toLowerCase();
-      return include.every((keys) => keys.some((key) => titleAlt.indexOf(key) !== -1));
+      return search.some((key) => titleAlt.indexOf(key) !== -1);
     });
-  }
-  if (resolved.exclude) {
-    const exclude = resolved.exclude.map((k) => normalizeTitle(k).toLowerCase());
+  } else if (resolved.include) {
+    // Match exact at least one of the keywords, and no excluded keywords
+    const include = resolved.include.map((k) => normalizeTitle(k).toLowerCase());
+    const exclude = (resolved.exclude ?? []).map((k) => normalizeTitle(k).toLowerCase());
     chains.push((r) => {
       const titleAlt = normalizeTitle(r.title).toLowerCase();
-      return exclude.every((key) => titleAlt.indexOf(key) === -1);
+      const matched = include.some((key) => titleAlt.indexOf(key) !== -1);
+      if (matched) {
+        return exclude.every((key) => titleAlt.indexOf(key) === -1);
+      } else {
+        return false;
+      }
     });
   }
 

diff --git a/packages/animegarden/src/garden/types.ts b/packages/animegarden/src/garden/types.ts
@@ -1,6 +1,11 @@
 import type { Resource } from '../types';
 
 export interface FilterOptions {
+  /**
+   * Only filter resources in the specific provider
+   */
+  provider?: string | string[];
+
   /**
    * Filter by the group id of fansub
    */
@@ -41,7 +46,7 @@ export interface FilterOptions {
   /**
    * Include keywords
    */
-  include?: (string | string[])[];
+  include?: string | string[];
 
   /**
    * Exclude keywords
@@ -60,6 +65,8 @@ export interface FilterOptions {
 }
 
 export interface ResolvedFilterOptions {
+  provider: string[];
+
   /**
    * Filter by the group id of fansub
    */
@@ -100,7 +107,7 @@ export interface ResolvedFilterOptions {
   /**
    * Include keywords
    */
-  include?: string[][];
+  include?: string[];
 
   /**
    * Exclude keywords

diff --git a/packages/animegarden/src/garden/url.ts b/packages/animegarden/src/garden/url.ts
@@ -1,7 +1,7 @@
 import { z } from 'zod';
 
 import { QueryType } from './constant';
-import { normalizeTitle } from './utils';
+// import { normalizeTitle } from './utils';
 import { FilterOptions, ResolvedFilterOptions } from './types';
 
 const dateLike = z
@@ -15,10 +15,10 @@ const stringArrayLike = z.coerce
   .catch(undefined)
   .pipe(stringArray)
   .optional();
-const stringArrayArray = z.union([
-  z.string().transform((s) => [[s]]),
-  z.array(stringArray).default([])
-]);
+// const stringArrayArray = z.union([
+//   z.string().transform((s) => [[s]]),
+//   z.array(stringArray).default([])
+// ]);
 
 const numberArray = z.union([z.array(z.coerce.number()), z.coerce.number().transform((n) => [n])]);
 const numberArrayLike = z.coerce
@@ -28,7 +28,10 @@ const numberArrayLike = z.coerce
   .pipe(numberArray)
   .optional();
 
+const providerEnum = z.array(z.enum(['dmhy', 'moe'])).default(['dmhy', 'moe']);
+
 export const FilterSchema = z.object({
+  provider: providerEnum,
   page: z
     .number()
     .default(1)
@@ -44,11 +47,12 @@ export const FilterSchema = z.object({
   before: dateLike,
   after: dateLike,
   search: stringArray.optional(),
-  include: stringArrayArray.optional(),
+  include: stringArray.optional(),
   exclude: stringArray.optional()
 });
 
 const parser = {
+  provider: providerEnum,
   page: z.coerce
     .number()
     .default(1)
@@ -64,12 +68,7 @@ const parser = {
   before: dateLike,
   after: dateLike,
   search: stringArrayLike,
-  include: z.coerce
-    .string()
-    .transform((t) => JSON.parse(t))
-    .catch(undefined)
-    .pipe(stringArrayArray)
-    .optional(),
+  include: stringArrayLike,
   exclude: stringArrayLike
 };
 
@@ -80,6 +79,7 @@ export function parseSearchURL(
   body?: FilterOptions
 ): ResolvedFilterOptions {
   const entries = [...Object.entries(parser)].map(([key, parser]) => {
+    // Try parsing body
     if (body && typeof body === 'object') {
       const content = body[key as ParserKey];
       if (content !== null && content !== undefined) {
@@ -90,6 +90,7 @@ export function parseSearchURL(
         }
       }
     }
+    // Try parsing params
     {
       const content = params.get(key);
       if (content !== null && content !== '') {
@@ -104,54 +105,6 @@ export function parseSearchURL(
 
   const filtered = Object.fromEntries(entries) as ResolvedFilterOptions;
 
-  if (filtered.search) {
-    const MIN_LEN = 4;
-
-    const newSearch: string[] = [];
-    const newInclude: string[][] = filtered.include ?? [];
-    const newExclude: string[] = filtered.exclude ?? [];
-
-    for (const text of filtered.search) {
-      const word = normalizeTitle(text)
-        .replace(/^(\+|-)?"([^"]*)"$/, '$1$2')
-        .replace(/%2b/g, '+');
-      if (word[0] === '+') {
-        if (word.length - 1 <= MIN_LEN) {
-          newInclude.push([word.slice(1)]);
-        } else {
-          newExclude.push(`+"${word.slice(1)}"`);
-        }
-      } else if (word[0] === '-') {
-        if (word.length - 1 <= MIN_LEN) {
-          newExclude.push(word.slice(1));
-        } else {
-          newSearch.push(`-"${word.slice(1)}"`);
-        }
-      } else {
-        if (word.length <= MIN_LEN) {
-          newInclude.push([word]);
-        } else {
-          newSearch.push(`"${word}"`);
-        }
-      }
-    }
-
-    filtered.search = newSearch;
-    if (newInclude.length > 0) {
-      filtered.include = newInclude;
-    }
-    if (newExclude.length > 0) {
-      filtered.exclude = newExclude;
-    }
-  } else {
-    if (filtered.include) {
-      filtered.include = filtered.include.map((arr) => arr.map((t) => normalizeTitle(t)));
-    }
-    if (filtered.exclude) {
-      filtered.exclude = filtered.exclude.map((t) => normalizeTitle(t));
-    }
-  }
-
   const isNaN = (d: unknown): boolean => d === undefined || d === null || Number.isNaN(d);
   if (isNaN(filtered.page)) {
     filtered.page = 1;
@@ -166,6 +119,9 @@ export function parseSearchURL(
 export function stringifySearchURL(baseURL: string, options: FilterOptions): URL {
   const url = new URL('resources', baseURL);
 
+  if (options.provider && options.provider.length > 0) {
+    url.searchParams.set('provider', JSON.stringify(options.provider));
+  }
   if (options.page) {
     url.searchParams.set('page', '' + options.page);
   }

diff --git a/packages/animegarden/src/types.ts b/packages/animegarden/src/types.ts
@@ -20,6 +20,10 @@ export type ResourceType =
   | '其他';
 
 export interface Resource {
+  provider: string;
+
+  providerId: string;
+
   title: string;
 
   href: string;
@@ -45,13 +49,15 @@ export interface Resource {
   createdAt: string;
 
   fetchedAt: string;
-
-  provider: string;
 }
 
 export type FetchedResource = Omit<Resource, 'fetchedAt'>;
 
 export interface ResourceDetail {
+  provider: string;
+
+  providerId: string;
+
   title: string;
 
   href: string;

diff --git a/packages/scraper/src/dmhy/index.ts b/packages/scraper/src/dmhy/index.ts
@@ -39,6 +39,7 @@ export async function fetchDmhyPage(
     throw new Error('dmhy server is down');
   }
 
+  const now = new Date();
   const res: FetchedResource[] = [];
   $('#topic_list tbody tr').each((_idx, row) => {
     const tds = $(row).find('td');
@@ -55,8 +56,14 @@ export async function fetchDmhyPage(
     const publisherName = publisher.text();
     const publisherId = publisher.attr('href')!.split('/').at(-1)!;
 
+    const lastHref = href.split('/').at(-1);
+    if (!lastHref) return;
+    const matchId = /^(\d+)/.exec(lastHref);
+    if (!matchId) return;
+
     res.push({
       provider: 'dmhy',
+      providerId: matchId[1],
       title,
       href,
       type,
@@ -155,6 +162,8 @@ export async function fetchDmhyDetail(
     });
 
   return {
+    provider: 'dmhy',
+    providerId: matchId[1],
     title,
     href: url.href,
     type,