feat(scraper): fetch moe resources

yjl9903 · Jul 25, 2024 · 2234b75 · 2234b75
1 parent 124ee0c
commit 2234b75
Show file tree

Hide file tree

Showing 5 changed files with 251 additions and 13 deletions.
diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts
@@ -22,7 +22,8 @@ cli
       const { fetchDmhy } = await import('./commands/dmhy');
       await fetchDmhy(+options.from, options.to ? +options.to : undefined, options.outDir);
     } else if (platform === 'moe') {
-      throw new Error('unimplemented');
+      const { fetchMoe } = await import('./commands/moe');
+      await fetchMoe(+options.from, options.to ? +options.to : undefined, options.outDir);
     }
   });
 

diff --git a/packages/cli/src/commands/moe.ts b/packages/cli/src/commands/moe.ts
@@ -0,0 +1,41 @@
+import fs from 'fs-extra';
+import path from 'node:path';
+
+import type { FetchedResource } from 'animegarden';
+import type { Database, NewUser, NewTeam, MeiliSearch } from '@animegarden/database';
+
+import { fetchMoePage } from '@animegarden/scraper';
+
+import { ufetch } from '../utils';
+
+export async function fetchMoe(from: number, to: number | undefined, outDir: string) {
+  await fs.mkdir(outDir, { recursive: true });
+  if ((await fs.readdir(outDir)).length > 0) {
+    throw new Error(`Out dir ${outDir} is not empty`);
+  }
+
+  let empty = 0;
+  for (let page = from; to === undefined || page <= to; page++) {
+    console.log(`Fetch moe page ${page}`);
+
+    const r = await fetchMoePage(ufetch, {
+      page,
+      retry: Number.MAX_SAFE_INTEGER
+    });
+
+    await fs.promises.writeFile(
+      path.join(outDir, `${page}.json`),
+      JSON.stringify(r, null, 2),
+      'utf-8'
+    );
+
+    if (r.length === 0) {
+      empty++;
+      if (empty >= 10) {
+        break;
+      }
+    } else {
+      empty = 0;
+    }
+  }
+}
diff --git a/packages/scraper/src/moe/index.ts b/packages/scraper/src/moe/index.ts
@@ -2,6 +2,9 @@ import type { FetchedResource, ResourceDetail } from 'animegarden';
 
 import { retryFn } from 'animegarden';
 
+import { getType } from './tag';
+import { fetchTeam, fetchUser } from './user';
+
 export interface FetchMoePageOptions {
   page?: number;
 
@@ -15,13 +18,20 @@ export interface FetchMoeDetailOptions {
 const TRACKER = `&tr=https%3A%2F%2Ftr.bangumi.moe%3A9696%2Fannounce&tr=http%3A%2F%2Ftr.bangumi.moe%3A6969%2Fannounce&tr=udp%3A%2F%2Ftr.bangumi.moe%3A6969%2Fannounce&tr=http%3A%2F%2Fopen.acgtracker.com%3A1096%2Fannounce&tr=http%3A%2F%2F208.67.16.113%3A8000%2Fannounce&tr=udp%3A%2F%2F208.67.16.113%3A8000%2Fannounce&tr=http%3A%2F%2Ftracker.ktxp.com%3A6868%2Fannounce&tr=http%3A%2F%2Ftracker.ktxp.com%3A7070%2Fannounce&tr=http%3A%2F%2Ft2.popgo.org%3A7456%2Fannonce&tr=http%3A%2F%2Fbt.sc-ol.com%3A2710%2Fannounce&tr=http%3A%2F%2Fshare.camoe.cn%3A8080%2Fannounce&tr=http%3A%2F%2F61.154.116.205%3A8000%2Fannounce&tr=http%3A%2F%2Fbt.rghost.net%3A80%2Fannounce&tr=http%3A%2F%2Ftracker.openbittorrent.com%3A80%2Fannounce&tr=http%3A%2F%2Ftracker.publicbt.com%3A80%2Fannounce&tr=http%3A%2F%2Ftracker.prq.to%2Fannounce&tr=http%3A%2F%2Fopen.nyaatorrents.info%3A6544%2Fannounce`;
 
 export async function fetchMoePage(
-  ofetch: (request: string) => Promise<Response>,
+  ofetch: (request: string, init?: RequestInit) => Promise<Response>,
   options: FetchMoePageOptions = {}
 ): Promise<FetchedResource[]> {
   const { page = 1, retry = 5 } = options;
 
   const resp = await retryFn(async () => {
-    const resp = await ofetch(`https://bangumi.moe/api/torrent/page/${page}`);
+    const resp = await ofetch(`https://bangumi.moe/api/torrent/page/${page}`, {
+      headers: new Headers([
+        [
+          'User-Agent',
+          'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 Edg/126.0.0.0'
+        ]
+      ])
+    });
     if (!resp.ok) {
       throw new Error(resp.statusText, { cause: resp });
     }
@@ -32,34 +42,91 @@ export async function fetchMoePage(
   }
   const data = await resp.json();
 
-  // TODO
   const result: FetchedResource[] = [];
   for (const torrent of data?.torrents ?? []) {
+    const user = await fetchUser(ofetch, torrent.uploader_id);
+    const team = torrent.team_id ? await fetchTeam(ofetch, torrent.team_id) : undefined;
+
     result.push({
       provider: 'moe',
       providerId: torrent._id,
       title: torrent.title,
-      href: `https://bangumi.moe/torrent/${torrent._id}`,
+      href: torrent._id,
       magnet: torrent.magnet,
       tracker: TRACKER,
-      type: '動畫',
+      type: getType(torrent.tag_ids),
       size: torrent.size,
-      fansub: undefined,
       publisher: {
-        id: '',
-        name: ''
+        id: torrent.uploader_id,
+        name: user.name,
+        avatar: team?.avatar
       },
+      fansub: team
+        ? {
+            id: torrent.team_id,
+            name: team.name,
+            avatar: team.avatar
+          }
+        : undefined,
       createdAt: torrent.publish_time
     });
   }
 
-  return [];
+  return result;
 }
 
 export async function fetchMoeDetail(
-  ofetch: (request: string) => Promise<Response>,
-  href: string,
+  ofetch: (request: string, init?: RequestInit) => Promise<Response>,
+  id: string,
   options: FetchMoeDetailOptions = {}
 ): Promise<ResourceDetail | undefined> {
-  return undefined;
+  const { retry = 5 } = options;
+
+  const resp = await retryFn(async () => {
+    const resp = await ofetch(`https://bangumi.moe/api/torrent/fetch`, {
+      method: 'POST',
+      body: JSON.stringify({ _id: id })
+    });
+    if (!resp.ok) {
+      throw new Error(resp.statusText, { cause: resp });
+    }
+    return resp;
+  }, retry);
+  if (!resp.ok) {
+    throw new Error('Failed connecting https://bangumi.moe/');
+  }
+
+  const torrent = await resp.json();
+
+  const user = await fetchUser(ofetch, torrent.uploader_id);
+  const team = await fetchTeam(ofetch, torrent.team_id);
+
+  return {
+    provider: 'moe',
+    providerId: torrent._id,
+    title: torrent.title,
+    href: torrent._id,
+    description: torrent.introduction,
+    magnet: {
+      user: '',
+      href: torrent.magnet + TRACKER,
+      href2: torrent.magnet,
+      ddplay: '',
+      files: torrent.content.map((t: [string, string]) => ({ name: t[0], size: t[1] })),
+      hasMoreFiles: false
+    },
+    type: getType(torrent.tag_ids),
+    size: torrent.size,
+    publisher: {
+      id: torrent.uploader_id,
+      name: user.name,
+      avatar: team.avatar
+    },
+    fansub: {
+      id: torrent.team_id,
+      name: team.name,
+      avatar: team.avatar
+    },
+    createdAt: torrent.publish_time
+  };
 }
diff --git a/packages/scraper/src/moe/tag.ts b/packages/scraper/src/moe/tag.ts
@@ -0,0 +1,38 @@
+import type { ResourceType } from 'animegarden';
+
+export const Anime = '549ef207fe682f7549f1ea90';
+
+export const Collection = '54967e14ff43b99e284d0bf7';
+
+export const Music = '549eef6ffe682f7549f1ea8b';
+
+export const Manga = '549eefebfe682f7549f1ea8c';
+
+export const TV = '549ff1db30bcfc225bf9e607';
+
+export const Other = '549ef250fe682f7549f1ea91';
+
+export const Game = '549ef015fe682f7549f1ea8d';
+
+export function getType(tags: string[]): ResourceType {
+  for (const tag of tags) {
+    switch (tag) {
+      case Anime:
+        return '動畫';
+      case Collection:
+        return '季度全集';
+      case Manga:
+        return '漫畫';
+      case Music:
+        return '音樂';
+      case TV:
+        return '日劇';
+      case Game:
+        return '遊戲';
+      case Other:
+        return '其他';
+      default:
+    }
+  }
+  return '其他';
+}
diff --git a/packages/scraper/src/moe/user.ts b/packages/scraper/src/moe/user.ts
@@ -0,0 +1,91 @@
+import { retryFn } from 'animegarden';
+
+const Users = new Map<string, { provider: 'moe'; providerId: string; name: string }>();
+
+const Teams = new Map<
+  string,
+  { provider: 'moe'; providerId: string; name: string; avatar: string }
+>();
+
+export async function fetchUser(
+  ofetch: (request: string, init?: RequestInit) => Promise<Response>,
+  id: string
+) {
+  if (Users.get(id)) {
+    return Users.get(id)!;
+  }
+
+  const resp = await retryFn(async () => {
+    const resp = await ofetch(`https://bangumi.moe/api/user/fetch`, {
+      method: 'POST',
+      body: JSON.stringify({ _ids: [id] }),
+      headers: new Headers([
+        [
+          'User-Agent',
+          'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 Edg/126.0.0.0'
+        ]
+      ])
+    });
+    if (!resp.ok) {
+      throw new Error(resp.statusText, { cause: resp });
+    }
+    return resp;
+  }, 10);
+  if (!resp.ok) {
+    throw new Error('Failed connecting https://bangumi.moe/');
+  }
+
+  const data = await resp.json();
+
+  const user = {
+    provider: 'moe',
+    providerId: id,
+    name: data[0].username
+  } as const;
+
+  Users.set(id, user);
+
+  return user;
+}
+
+export async function fetchTeam(
+  ofetch: (request: string, init?: RequestInit) => Promise<Response>,
+  id: string
+) {
+  if (Teams.get(id)) {
+    return Teams.get(id)!;
+  }
+
+  const resp = await retryFn(async () => {
+    const resp = await ofetch(`https://bangumi.moe/api/team/fetch`, {
+      method: 'POST',
+      body: JSON.stringify({ _ids: [id] }),
+      headers: new Headers([
+        [
+          'User-Agent',
+          'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 Edg/126.0.0.0'
+        ]
+      ])
+    });
+    if (!resp.ok) {
+      throw new Error(resp.statusText, { cause: resp });
+    }
+    return resp;
+  }, 10);
+  if (!resp.ok) {
+    throw new Error('Failed connecting https://bangumi.moe/');
+  }
+
+  const data = await resp.json();
+
+  const team = {
+    provider: 'moe',
+    providerId: id,
+    name: data[0].name,
+    avatar: data[0].icon
+  } as const;
+
+  Teams.set(id, team);
+
+  return team;
+}