From 110ac2ae56fb963fc93ca3f42a8dc12a0c83da8e Mon Sep 17 00:00:00 2001 From: tomolld Date: Wed, 17 Jul 2024 18:32:13 +0900 Subject: [PATCH 1/4] =?UTF-8?q?translate=E3=83=AB=E3=83=BC=E3=83=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker-compose.yml | 2 +- web/app/components/Header.tsx | 6 +- web/app/components/LoadingSpinner.tsx | 2 +- web/app/components/ui/badge.tsx | 54 ++-- web/app/components/ui/progress.tsx | 42 +-- web/app/components/ui/scroll-area.tsx | 76 +++--- .../extractNumberedElements.ts | 0 web/app/{utils => libs}/pageService.ts | 2 +- .../pageVersion.ts} | 2 +- web/app/libs/pageVersionTranslationInfo.ts | 22 ++ web/app/{utils => libs}/sourceTextService.ts | 2 +- web/app/libs/userAITranslationInfo.tsx | 54 ++++ web/app/{utils => libs}/userReadHistory.ts | 2 +- web/app/{utils => libs}/userService.ts | 4 +- .../GoogleSignInAndGeminiApiKeyForm.tsx | 140 +++++----- .../_index/components/URLTranslationForm.tsx | 9 +- .../components/UserAITranslationStatus.tsx | 101 +++++++ .../_index/components/UserReadHistoryList.tsx | 77 ------ .../components/UserTranslationStatus.tsx | 97 +++++++ web/app/routes/_index/constants.ts | 1 - web/app/routes/_index/libs/translation.ts | 101 +++++++ .../routes/_index/libs/translationUtils.ts | 175 ++++++++++++ .../libs/userTranslationqueueService.ts | 20 ++ web/app/routes/_index/route.tsx | 96 ++++--- web/app/routes/_index/types.ts | 32 --- .../_index/utils/extractTranslations.ts | 31 --- web/app/routes/_index/utils/translation.ts | 258 ------------------ web/app/routes/api.auth.callback.google.ts | 2 +- web/app/routes/auth.login.tsx | 106 +------ .../GoogleSignInAndGeminiApiKeyForm.tsx | 86 ++++++ .../components/URLTranslationForm.tsx | 60 ++++ .../components/UserAITranslationStatus.tsx | 101 +++++++ .../translate/components/translatedList.tsx | 22 ++ web/app/routes/translate/constants.ts | 3 + web/app/routes/translate/libs/translation.ts | 101 +++++++ .../routes/translate/libs/translationUtils.ts | 175 ++++++++++++ .../libs/userTranslationqueueService.ts | 20 ++ web/app/routes/translate/route.tsx | 126 +++++++++ web/app/routes/translate/types.ts | 43 +++ .../translate/utils/addNumbersToContent.ts | 77 ++++++ .../routes/translate/utils/extractArticle.ts | 17 ++ .../routes/translate/utils/fetchWithRetry.ts | 32 +++ web/app/routes/translate/utils/gemini.ts | 61 +++++ .../translate/utils/generateGeminiMessage.ts | 48 ++++ web/app/utils/auth.server.ts | 38 +-- web/app/utils/pageVersionTranslationInfo.ts | 59 ---- web/app/utils/signup.server.ts | 25 -- web/biome.json | 1 - web/package.json | 26 +- .../migrations/20240717054302_/migration.sql | 38 +++ .../migrations/20240717063343_/migration.sql | 10 + .../migrations/20240717065258_/migration.sql | 11 + web/prisma/schema.prisma | 125 +++++---- web/public/robots.txt | 0 web/test/translation.test.ts | 2 +- 55 files changed, 1931 insertions(+), 892 deletions(-) rename web/app/{utils => libs}/extractNumberedElements.ts (100%) rename web/app/{utils => libs}/pageService.ts (81%) rename web/app/{utils/pageVersionService.ts => libs/pageVersion.ts} (94%) create mode 100644 web/app/libs/pageVersionTranslationInfo.ts rename web/app/{utils => libs}/sourceTextService.ts (94%) create mode 100644 web/app/libs/userAITranslationInfo.tsx rename web/app/{utils => libs}/userReadHistory.ts (95%) rename web/app/{utils => libs}/userService.ts (63%) create mode 100644 web/app/routes/_index/components/UserAITranslationStatus.tsx delete mode 100644 web/app/routes/_index/components/UserReadHistoryList.tsx create mode 100644 web/app/routes/_index/components/UserTranslationStatus.tsx delete mode 100644 web/app/routes/_index/constants.ts create mode 100644 web/app/routes/_index/libs/translation.ts create mode 100644 web/app/routes/_index/libs/translationUtils.ts create mode 100644 web/app/routes/_index/libs/userTranslationqueueService.ts delete mode 100644 web/app/routes/_index/types.ts delete mode 100644 web/app/routes/_index/utils/extractTranslations.ts delete mode 100644 web/app/routes/_index/utils/translation.ts create mode 100644 web/app/routes/translate/components/GoogleSignInAndGeminiApiKeyForm.tsx create mode 100644 web/app/routes/translate/components/URLTranslationForm.tsx create mode 100644 web/app/routes/translate/components/UserAITranslationStatus.tsx create mode 100644 web/app/routes/translate/components/translatedList.tsx create mode 100644 web/app/routes/translate/constants.ts create mode 100644 web/app/routes/translate/libs/translation.ts create mode 100644 web/app/routes/translate/libs/translationUtils.ts create mode 100644 web/app/routes/translate/libs/userTranslationqueueService.ts create mode 100644 web/app/routes/translate/route.tsx create mode 100644 web/app/routes/translate/types.ts create mode 100644 web/app/routes/translate/utils/addNumbersToContent.ts create mode 100644 web/app/routes/translate/utils/extractArticle.ts create mode 100644 web/app/routes/translate/utils/fetchWithRetry.ts create mode 100644 web/app/routes/translate/utils/gemini.ts create mode 100644 web/app/routes/translate/utils/generateGeminiMessage.ts delete mode 100644 web/app/utils/pageVersionTranslationInfo.ts delete mode 100644 web/app/utils/signup.server.ts create mode 100644 web/prisma/migrations/20240717054302_/migration.sql create mode 100644 web/prisma/migrations/20240717063343_/migration.sql create mode 100644 web/prisma/migrations/20240717065258_/migration.sql create mode 100644 web/public/robots.txt diff --git a/docker-compose.yml b/docker-compose.yml index 4c58587..e04164b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,7 +17,7 @@ services: restart: always ports: - '6379:6379' - command: redis-server --save 20 1 --loglevel warning --requirepass eYVX7EwVmmxKPCDmwMtyKVge8oLd2t81 + command: redis-server --save 20 1 --loglevel warning volumes: - redis:/data diff --git a/web/app/components/Header.tsx b/web/app/components/Header.tsx index 8d3172c..4610980 100644 --- a/web/app/components/Header.tsx +++ b/web/app/components/Header.tsx @@ -35,11 +35,7 @@ export function Header({ safeUser, targetLanguage }: HeaderProps) { {safeUser ? ( <> - + diff --git a/web/app/components/LoadingSpinner.tsx b/web/app/components/LoadingSpinner.tsx index a946f22..5fb8d51 100644 --- a/web/app/components/LoadingSpinner.tsx +++ b/web/app/components/LoadingSpinner.tsx @@ -1,7 +1,7 @@ export function LoadingSpinner() { return ( <> -
+
); } diff --git a/web/app/components/ui/badge.tsx b/web/app/components/ui/badge.tsx index 83d5988..864106d 100644 --- a/web/app/components/ui/badge.tsx +++ b/web/app/components/ui/badge.tsx @@ -1,36 +1,36 @@ -import * as React from "react" -import { cva, type VariantProps } from "class-variance-authority" +import { type VariantProps, cva } from "class-variance-authority"; +import type * as React from "react"; -import { cn } from "~/utils/cn" +import { cn } from "~/utils/cn"; const badgeVariants = cva( - "inline-flex items-center rounded-md border px-2.5 py-0.5 text-xs font-semibold transition-colors focus:outline-none focus:ring-2 focus:ring-ring focus:ring-offset-2", - { - variants: { - variant: { - default: - "border-transparent bg-primary text-primary-foreground shadow hover:bg-primary/80", - secondary: - "border-transparent bg-secondary text-secondary-foreground hover:bg-secondary/80", - destructive: - "border-transparent bg-destructive text-destructive-foreground shadow hover:bg-destructive/80", - outline: "text-foreground", - }, - }, - defaultVariants: { - variant: "default", - }, - } -) + "inline-flex items-center rounded-md border px-2.5 py-0.5 text-xs font-semibold transition-colors focus:outline-none focus:ring-2 focus:ring-ring focus:ring-offset-2", + { + variants: { + variant: { + default: + "border-transparent bg-primary text-primary-foreground shadow hover:bg-primary/80", + secondary: + "border-transparent bg-secondary text-secondary-foreground hover:bg-secondary/80", + destructive: + "border-transparent bg-destructive text-destructive-foreground shadow hover:bg-destructive/80", + outline: "text-foreground", + }, + }, + defaultVariants: { + variant: "default", + }, + }, +); export interface BadgeProps - extends React.HTMLAttributes, - VariantProps {} + extends React.HTMLAttributes, + VariantProps {} function Badge({ className, variant, ...props }: BadgeProps) { - return ( -
- ) + return ( +
+ ); } -export { Badge, badgeVariants } +export { Badge, badgeVariants }; diff --git a/web/app/components/ui/progress.tsx b/web/app/components/ui/progress.tsx index 3c49bdd..54cac2c 100644 --- a/web/app/components/ui/progress.tsx +++ b/web/app/components/ui/progress.tsx @@ -1,26 +1,26 @@ -import * as React from "react" -import * as ProgressPrimitive from "@radix-ui/react-progress" +import * as ProgressPrimitive from "@radix-ui/react-progress"; +import * as React from "react"; -import { cn } from "~/utils/cn" +import { cn } from "~/utils/cn"; const Progress = React.forwardRef< - React.ElementRef, - React.ComponentPropsWithoutRef + React.ElementRef, + React.ComponentPropsWithoutRef >(({ className, value, ...props }, ref) => ( - - - -)) -Progress.displayName = ProgressPrimitive.Root.displayName + + + +)); +Progress.displayName = ProgressPrimitive.Root.displayName; -export { Progress } +export { Progress }; diff --git a/web/app/components/ui/scroll-area.tsx b/web/app/components/ui/scroll-area.tsx index c25685a..734eacb 100644 --- a/web/app/components/ui/scroll-area.tsx +++ b/web/app/components/ui/scroll-area.tsx @@ -1,46 +1,46 @@ -import * as React from "react" -import * as ScrollAreaPrimitive from "@radix-ui/react-scroll-area" +import * as ScrollAreaPrimitive from "@radix-ui/react-scroll-area"; +import * as React from "react"; -import { cn } from "~/utils/cn" +import { cn } from "~/utils/cn"; const ScrollArea = React.forwardRef< - React.ElementRef, - React.ComponentPropsWithoutRef + React.ElementRef, + React.ComponentPropsWithoutRef >(({ className, children, ...props }, ref) => ( - - - {children} - - - - -)) -ScrollArea.displayName = ScrollAreaPrimitive.Root.displayName + + + {children} + + + + +)); +ScrollArea.displayName = ScrollAreaPrimitive.Root.displayName; const ScrollBar = React.forwardRef< - React.ElementRef, - React.ComponentPropsWithoutRef + React.ElementRef, + React.ComponentPropsWithoutRef >(({ className, orientation = "vertical", ...props }, ref) => ( - - - -)) -ScrollBar.displayName = ScrollAreaPrimitive.ScrollAreaScrollbar.displayName + + + +)); +ScrollBar.displayName = ScrollAreaPrimitive.ScrollAreaScrollbar.displayName; -export { ScrollArea, ScrollBar } +export { ScrollArea, ScrollBar }; diff --git a/web/app/utils/extractNumberedElements.ts b/web/app/libs/extractNumberedElements.ts similarity index 100% rename from web/app/utils/extractNumberedElements.ts rename to web/app/libs/extractNumberedElements.ts diff --git a/web/app/utils/pageService.ts b/web/app/libs/pageService.ts similarity index 81% rename from web/app/utils/pageService.ts rename to web/app/libs/pageService.ts index 4b3e3ad..a9997af 100644 --- a/web/app/utils/pageService.ts +++ b/web/app/libs/pageService.ts @@ -1,4 +1,4 @@ -import { prisma } from "./prisma"; +import { prisma } from "../utils/prisma"; export async function getOrCreatePageId(url: string): Promise { const page = await prisma.page.upsert({ diff --git a/web/app/utils/pageVersionService.ts b/web/app/libs/pageVersion.ts similarity index 94% rename from web/app/utils/pageVersionService.ts rename to web/app/libs/pageVersion.ts index 4de8c16..f2f4ebb 100644 --- a/web/app/utils/pageVersionService.ts +++ b/web/app/libs/pageVersion.ts @@ -1,5 +1,5 @@ import { createHash } from "node:crypto"; -import { prisma } from "./prisma"; +import { prisma } from "../utils/prisma"; export async function getOrCreatePageVersionId( url: string, diff --git a/web/app/libs/pageVersionTranslationInfo.ts b/web/app/libs/pageVersionTranslationInfo.ts new file mode 100644 index 0000000..d88f93a --- /dev/null +++ b/web/app/libs/pageVersionTranslationInfo.ts @@ -0,0 +1,22 @@ +import { prisma } from "~/utils/prisma"; + +export async function getOrCreatePageVersionTranslationInfo( + pageVersionId: number, + targetLanguage: string, + translationTitle: string, +) { + return await prisma.pageVersionTranslationInfo.upsert({ + where: { + pageVersionId_targetLanguage: { + pageVersionId, + targetLanguage, + }, + }, + update: {}, // 既存のレコードがある場合は更新しない + create: { + pageVersionId, + targetLanguage, + translationTitle, + }, + }); +} diff --git a/web/app/utils/sourceTextService.ts b/web/app/libs/sourceTextService.ts similarity index 94% rename from web/app/utils/sourceTextService.ts rename to web/app/libs/sourceTextService.ts index fc79f7a..2517cca 100644 --- a/web/app/utils/sourceTextService.ts +++ b/web/app/libs/sourceTextService.ts @@ -1,5 +1,5 @@ import { createHash } from "node:crypto"; -import { prisma } from "./prisma"; +import { prisma } from "../utils/prisma"; export async function getOrCreateSourceTextId( text: string, diff --git a/web/app/libs/userAITranslationInfo.tsx b/web/app/libs/userAITranslationInfo.tsx new file mode 100644 index 0000000..43fd1b6 --- /dev/null +++ b/web/app/libs/userAITranslationInfo.tsx @@ -0,0 +1,54 @@ +import { prisma } from "~/utils/prisma"; + +export async function getOrCreateUserAITranslationInfo( + userId: number, + pageVersionId: number, + targetLanguage: string, +) { + try { + const userAITranslationInfo = await prisma.userAITranslationInfo.upsert({ + where: { + userId_pageVersionId_targetLanguage: { + userId, + pageVersionId, + targetLanguage, + }, + }, + update: {}, + create: { + userId, + pageVersionId, + targetLanguage, + aiTranslationStatus: "pending", + aiTranslationProgress: 0, + }, + }); + return userAITranslationInfo; + } catch (error) { + console.error("Error in getOrCreateUserAITranslationInfo:", error); + throw error; + } +} + +export async function updateUserAITranslationInfo( + userId: number, + pageVersionId: number, + targetLanguage: string, + status: string, + progress: number, +) { + return await prisma.userAITranslationInfo.update({ + where: { + userId_pageVersionId_targetLanguage: { + userId, + pageVersionId, + targetLanguage, + }, + }, + data: { + aiTranslationStatus: status, + aiTranslationProgress: progress, + lastTranslatedAt: new Date(), // 明示的に更新 + }, + }); +} diff --git a/web/app/utils/userReadHistory.ts b/web/app/libs/userReadHistory.ts similarity index 95% rename from web/app/utils/userReadHistory.ts rename to web/app/libs/userReadHistory.ts index 10641c9..db62027 100644 --- a/web/app/utils/userReadHistory.ts +++ b/web/app/libs/userReadHistory.ts @@ -1,4 +1,4 @@ -import { prisma } from "./prisma"; +import { prisma } from "../utils/prisma"; export async function updateUserReadHistory( userId: number, diff --git a/web/app/utils/userService.ts b/web/app/libs/userService.ts similarity index 63% rename from web/app/utils/userService.ts rename to web/app/libs/userService.ts index 4cd9d6d..ee38d20 100644 --- a/web/app/utils/userService.ts +++ b/web/app/libs/userService.ts @@ -1,10 +1,10 @@ -import { prisma } from "./prisma"; +import { prisma } from "../utils/prisma"; export async function getOrCreateAIUser(name: string): Promise { const user = await prisma.user.upsert({ where: { email: `${name}@ai.com` }, update: {}, - create: { name, email: `${name}@ai.com`, isAI: true }, + create: { name, email: `${name}@ai.com`, isAI: true, image: "" }, }); return user.id; diff --git a/web/app/routes/_index/components/GoogleSignInAndGeminiApiKeyForm.tsx b/web/app/routes/_index/components/GoogleSignInAndGeminiApiKeyForm.tsx index 60479db..7487e69 100644 --- a/web/app/routes/_index/components/GoogleSignInAndGeminiApiKeyForm.tsx +++ b/web/app/routes/_index/components/GoogleSignInAndGeminiApiKeyForm.tsx @@ -1,82 +1,86 @@ import { useForm } from "@conform-to/react"; import { getFormProps, getInputProps } from "@conform-to/react"; -import { parseWithZod, getZodConstraint } from "@conform-to/zod"; +import type { SubmissionResult } from "@conform-to/react"; +import { getZodConstraint, parseWithZod } from "@conform-to/zod"; import { Form } from "@remix-run/react"; +import { useActionData } from "@remix-run/react"; +import { Save } from "lucide-react"; import { GoogleForm } from "~/components/GoogleForm"; import { Button } from "~/components/ui/button"; import { Card, CardContent, CardHeader, CardTitle } from "~/components/ui/card"; import { Input } from "~/components/ui/input"; -import { useActionData } from "@remix-run/react"; -import type { SubmissionResult } from "@conform-to/react"; -import { Save } from "lucide-react"; -import { geminiApiKeySchema } from "../types"; +import { geminiApiKeySchema } from "../../translate/types"; interface GoogleSignInAndGeminiApiKeyFormProps { - isLoggedIn: boolean; - hasGeminiApiKey: boolean; - error?: string; + isLoggedIn: boolean; + hasGeminiApiKey: boolean; + error?: string; } export function GoogleSignInAndGeminiApiKeyForm({ - isLoggedIn, - hasGeminiApiKey, - error, + isLoggedIn, + hasGeminiApiKey, + error, }: GoogleSignInAndGeminiApiKeyFormProps) { - const lastResult = useActionData(); - const [form, { geminiApiKey }] = useForm({ - id: "gemini-api-key-form", - lastResult, - constraint: getZodConstraint(geminiApiKeySchema), - shouldValidate: "onBlur", - shouldRevalidate: "onInput", - onValidate({ formData }) { - return parseWithZod(formData, { schema: geminiApiKeySchema }); - }, - }); + const lastResult = useActionData(); + const [form, { geminiApiKey }] = useForm({ + id: "gemini-api-key-form", + lastResult, + constraint: getZodConstraint(geminiApiKeySchema), + shouldValidate: "onBlur", + shouldRevalidate: "onInput", + onValidate({ formData }) { + return parseWithZod(formData, { schema: geminiApiKeySchema }); + }, + }); - return ( - - - - {isLoggedIn - ? hasGeminiApiKey - ? "Update Gemini API Key" - : "Set Gemini API Key" - : "Sign in and Set Gemini API Key"} - - - - {!isLoggedIn && ( -
- -
- )} - {isLoggedIn && !hasGeminiApiKey && ( -
-
-
- -
- -
-
{geminiApiKey.errors}
- {form.errors &&

{form.errors}

} -
- )} - - - ); + return ( + + + + {isLoggedIn + ? hasGeminiApiKey + ? "Update Gemini API Key" + : "Set Gemini API Key" + : "Sign in and Set Gemini API Key"} + + + + {!isLoggedIn && ( +
+ +
+ )} + {isLoggedIn && !hasGeminiApiKey && ( +
+
+
+ +
+ +
+
+ {geminiApiKey.errors} +
+ {form.errors && ( +

{form.errors}

+ )} +
+ )} + + + ); } diff --git a/web/app/routes/_index/components/URLTranslationForm.tsx b/web/app/routes/_index/components/URLTranslationForm.tsx index ff07f1d..291678e 100644 --- a/web/app/routes/_index/components/URLTranslationForm.tsx +++ b/web/app/routes/_index/components/URLTranslationForm.tsx @@ -39,11 +39,16 @@ export function URLTranslationForm() { />
{fields.url.errors}
-
diff --git a/web/app/routes/_index/components/UserAITranslationStatus.tsx b/web/app/routes/_index/components/UserAITranslationStatus.tsx new file mode 100644 index 0000000..7cb4ac5 --- /dev/null +++ b/web/app/routes/_index/components/UserAITranslationStatus.tsx @@ -0,0 +1,101 @@ +import { Link } from "@remix-run/react"; +import { Badge } from "~/components/ui/badge"; +import { Card, CardContent, CardHeader, CardTitle } from "~/components/ui/card"; +import { Progress } from "~/components/ui/progress"; +import { ScrollArea } from "~/components/ui/scroll-area"; +import type { UserAITranslationInfoItem } from "../../translate/types"; + +type UserAITranslationStatusProps = { + userAITranslationInfo: UserAITranslationInfoItem[]; + targetLanguage: string; +}; + +export function UserAITranslationStatus({ + userAITranslationInfo = [], + targetLanguage, +}: UserAITranslationStatusProps) { + if (!userAITranslationInfo || userAITranslationInfo.length === 0) { + return ( + + + Translation Status ({targetLanguage}) + + +

No translation history available.

+ + + ); + } + + return ( + + + Translation Status ({targetLanguage}) + + + +
+ {userAITranslationInfo.map((item) => { + const translationInfo = + item.pageVersion.pageVersionTranslationInfo?.[0]; + return ( + + + + + {item.pageVersion.title} + {translationInfo?.translationTitle && ( + + {translationInfo.translationTitle} + + )} + + + +

+ {item.pageVersion.page.url} +

+ + {item.aiTranslationStatus} + + +

+ Last updated:{" "} + {new Date(item.lastTranslatedAt).toLocaleString()} +

+ + + + ); + })} +
+ + + + ); +} + +function getVariantForStatus( + status: string, +): "default" | "secondary" | "destructive" | "outline" { + switch (status) { + case "completed": + return "default"; + case "processing": + return "secondary"; + case "failed": + return "destructive"; + default: + return "outline"; + } +} diff --git a/web/app/routes/_index/components/UserReadHistoryList.tsx b/web/app/routes/_index/components/UserReadHistoryList.tsx deleted file mode 100644 index 1034953..0000000 --- a/web/app/routes/_index/components/UserReadHistoryList.tsx +++ /dev/null @@ -1,77 +0,0 @@ -import { Card, CardContent, CardHeader, CardTitle } from "~/components/ui/card"; -import { ScrollArea } from "~/components/ui/scroll-area"; -import type { UserReadHistoryItem } from "../types"; -import { Link } from "@remix-run/react"; -import { Badge } from "~/components/ui/badge"; -import { Progress } from "~/components/ui/progress"; - -type UserReadHistoryListProps = { - userReadHistory: UserReadHistoryItem[]; - targetLanguage: string; -}; - -export function UserReadHistoryList({ userReadHistory, targetLanguage }: UserReadHistoryListProps) { - return ( - - - Recently Read - - - -
- {userReadHistory.map((item) => { - const translationInfo = item.pageVersion.pageVersionTranslationInfo[0]; - return ( - - - - {item.pageVersion.title} - - -

- {item.pageVersion.page.url} -

-

- {new Date(item.readAt).toLocaleDateString()} -

- {translationInfo && ( - <> - - {translationInfo.translationStatus} - - - - )} - {!translationInfo && ( - - Not started - - )} - - - - ); - })} -
- - - - ); -}; - -function getVariantForStatus(status: string): "default" | "secondary" | "destructive" | "outline" { - switch (status) { - case "completed": - return "default"; - case "processing": - return "secondary"; - case "failed": - return "destructive"; - default: - return "outline"; - } -} \ No newline at end of file diff --git a/web/app/routes/_index/components/UserTranslationStatus.tsx b/web/app/routes/_index/components/UserTranslationStatus.tsx new file mode 100644 index 0000000..383b329 --- /dev/null +++ b/web/app/routes/_index/components/UserTranslationStatus.tsx @@ -0,0 +1,97 @@ +import { Link } from "@remix-run/react"; +import { Badge } from "~/components/ui/badge"; +import { Card, CardContent, CardHeader, CardTitle } from "~/components/ui/card"; +import { Progress } from "~/components/ui/progress"; +import { ScrollArea } from "~/components/ui/scroll-area"; +import type { UserAITranslationInfoItem } from "../../translate/types"; + +type UserAITranslationStatusProps = { + userAITranslationInfo: UserAITranslationInfoItem[]; + targetLanguage: string; +}; + +export function UserAITranslationStatus({ + userAITranslationInfo = [], + targetLanguage, +}: UserAITranslationStatusProps) { + if (!userAITranslationInfo || userAITranslationInfo.length === 0) { + return ( + + + Translation Status ({targetLanguage}) + + +

No translation history available.

+ + + ); + } + + return ( + + + Translation Status ({targetLanguage}) + + + +
+ {userAITranslationInfo.map((item) => { + const translationInfo = + item.pageVersion.pageVersionTranslationInfo?.[0]; + return ( + + + + + {translationInfo?.translationTitle || + item.pageVersion.title} + + + +

+ {item.pageVersion.page.url} +

+ + {item.aiTranslationStatus} + + +

+ Last updated:{" "} + {new Date(item.lastTranslatedAt).toLocaleString()} +

+ + + + ); + })} +
+ + + + ); +} + +function getVariantForStatus( + status: string, +): "default" | "secondary" | "destructive" | "outline" { + switch (status) { + case "completed": + return "default"; + case "processing": + return "secondary"; + case "failed": + return "destructive"; + default: + return "outline"; + } +} diff --git a/web/app/routes/_index/constants.ts b/web/app/routes/_index/constants.ts deleted file mode 100644 index b4daf73..0000000 --- a/web/app/routes/_index/constants.ts +++ /dev/null @@ -1 +0,0 @@ -export const MAX_CHUNK_SIZE = 20000; diff --git a/web/app/routes/_index/libs/translation.ts b/web/app/routes/_index/libs/translation.ts new file mode 100644 index 0000000..787cde6 --- /dev/null +++ b/web/app/routes/_index/libs/translation.ts @@ -0,0 +1,101 @@ +import type { Job } from "bull"; +import { getOrCreatePageId } from "../../../libs/pageService"; +import { getOrCreatePageVersionId } from "../../../libs/pageVersion"; +import { + getOrCreateUserAITranslationInfo, + updateUserAITranslationInfo, +} from "../../../libs/userAITranslationInfo"; +import { updateUserReadHistory } from "../../../libs/userReadHistory"; +import type { NumberedElement } from "../../translate/types"; +import { + getOrCreateTranslations, + splitNumberedElements, +} from "./translationUtils"; +import { setupUserQueue } from "./userTranslationqueueService"; + +export async function translate( + geminiApiKey: string, + userId: number, + targetLanguage: string, + title: string, + numberedContent: string, + numberedElements: NumberedElement[], + url: string, +): Promise { + const pageId = await getOrCreatePageId(url || ""); + const pageVersionId = await getOrCreatePageVersionId( + url, + title, + numberedContent, + pageId, + ); + + const userAITranslationHistory = await getOrCreateUserAITranslationInfo( + userId, + pageVersionId, + targetLanguage, + ); + await updateUserReadHistory(userId, pageVersionId, 0); + + if (userAITranslationHistory.aiTranslationStatus === "completed") { + return userAITranslationHistory.aiTranslationStatus; + } + + const userTranslationQueue = setupUserQueue(userId, geminiApiKey); + await userTranslationQueue.add({ + pageId, + pageVersionId, + targetLanguage, + title, + numberedElements, + }); + + return userAITranslationHistory.aiTranslationStatus; +} + +export async function processTranslationJob( + job: Job, + geminiApiKey: string, + userId: number, +) { + const { pageId, pageVersionId, targetLanguage, title, numberedElements } = + job.data; + try { + const chunks = splitNumberedElements(numberedElements); + const totalChunks = chunks.length; + for (let i = 0; i < chunks.length; i++) { + await getOrCreateTranslations( + geminiApiKey, + chunks[i], + targetLanguage, + pageId, + pageVersionId, + title, + ); + const progress = ((i + 1) / totalChunks) * 100; + await updateUserAITranslationInfo( + userId, + pageVersionId, + targetLanguage, + "in_progress", + progress, + ); + } + await updateUserAITranslationInfo( + userId, + pageVersionId, + targetLanguage, + "completed", + 100, + ); + } catch (error) { + console.error("Background translation job failed:", error); + await updateUserAITranslationInfo( + userId, + pageVersionId, + targetLanguage, + "failed", + 0, + ); + } +} diff --git a/web/app/routes/_index/libs/translationUtils.ts b/web/app/routes/_index/libs/translationUtils.ts new file mode 100644 index 0000000..cc4d483 --- /dev/null +++ b/web/app/routes/_index/libs/translationUtils.ts @@ -0,0 +1,175 @@ +import { getOrCreatePageVersionTranslationInfo } from "../../../libs/pageVersionTranslationInfo"; +import { getOrCreateSourceTextId } from "../../../libs/sourceTextService"; +import { getOrCreateAIUser } from "../../../libs/userService"; +import { prisma } from "../../../utils/prisma"; +import { AI_MODEL, MAX_CHUNK_SIZE } from "../../translate/constants"; +import type { NumberedElement } from "../../translate/types"; +import { getGeminiModelResponse } from "../utils/gemini"; + +export function splitNumberedElements( + elements: NumberedElement[], +): NumberedElement[][] { + const chunks: NumberedElement[][] = []; + let currentChunk: NumberedElement[] = []; + let currentSize = 0; + + for (const element of elements) { + if ( + currentSize + element.text.length > MAX_CHUNK_SIZE && + currentChunk.length > 0 + ) { + chunks.push(currentChunk); + currentChunk = []; + currentSize = 0; + } + currentChunk.push(element); + currentSize += element.text.length; + } + + if (currentChunk.length > 0) { + chunks.push(currentChunk); + } + return chunks; +} + +export function extractTranslations(jsonString: string): NumberedElement[] { + try { + const parsedData = JSON.parse(jsonString); + + if (Array.isArray(parsedData)) { + return parsedData.map((item) => ({ + number: Number(item.number), + text: String(item.text), + })); + } + console.error("Parsed data is not an array"); + return []; + } catch (error) { + console.error("Failed to parse JSON:", error); + return []; + } +} + +export async function getOrCreateTranslations( + geminiApiKey: string, + elements: NumberedElement[], + targetLanguage: string, + pageId: number, + pageVersionId: number, + title: string, +): Promise { + const translations: NumberedElement[] = []; + const untranslatedElements: NumberedElement[] = []; + const sourceTextsId = await Promise.all( + elements.map((element) => + getOrCreateSourceTextId( + element.text, + element.number, + pageId, + pageVersionId, + ), + ), + ); + + const existingTranslations = await prisma.translateText.findMany({ + where: { + sourceTextId: { in: sourceTextsId }, + targetLanguage, + }, + orderBy: [{ point: "desc" }, { createdAt: "desc" }], + }); + + const translationMap = new Map( + existingTranslations.map((t) => [t.sourceTextId, t]), + ); + + elements.forEach((element, index) => { + const sourceTextId = sourceTextsId[index]; + const existingTranslation = translationMap.get(sourceTextId); + + if (existingTranslation) { + translations.push({ + number: element.number, + text: existingTranslation.text, + }); + } else { + untranslatedElements.push(element); + } + }); + + if (untranslatedElements.length > 0) { + const newTranslations = await translateUntranslatedElements( + geminiApiKey, + untranslatedElements, + targetLanguage, + pageId, + pageVersionId, + title, + ); + translations.push(...newTranslations); + } + + return translations.sort((a, b) => a.number - b.number); +} + +async function translateUntranslatedElements( + geminiApiKey: string, + untranslatedElements: NumberedElement[], + targetLanguage: string, + pageId: number, + pageVersionId: number, + title: string, +): Promise { + const source_text = untranslatedElements + .map((el) => JSON.stringify(el)) + .join("\n"); + const translatedText = await getGeminiModelResponse( + geminiApiKey, + AI_MODEL, + title, + source_text, + targetLanguage, + ); + + const extractedTranslations = extractTranslations(translatedText); + await getOrCreatePageVersionTranslationInfo( + pageVersionId, + targetLanguage, + extractedTranslations[0].text, + ); + + const systemUserId = await getOrCreateAIUser(AI_MODEL); + + await Promise.all( + extractedTranslations.map(async (translation) => { + const sourceText = untranslatedElements.find( + (el) => el.number === translation.number, + )?.text; + + if (!sourceText) { + console.error( + `Source text not found for translation number ${translation.number}`, + ); + return; + } + + const sourceTextId = await getOrCreateSourceTextId( + sourceText, + translation.number, + pageId, + pageVersionId, + ); + await prisma.translateText.create({ + data: { + targetLanguage, + text: translation.text, + sourceTextId, + pageId, + userId: systemUserId, + }, + }); + }), + ); + + return extractedTranslations; +} diff --git a/web/app/routes/_index/libs/userTranslationqueueService.ts b/web/app/routes/_index/libs/userTranslationqueueService.ts new file mode 100644 index 0000000..541ddf7 --- /dev/null +++ b/web/app/routes/_index/libs/userTranslationqueueService.ts @@ -0,0 +1,20 @@ +import Queue, { type Queue as QueueType } from "bull"; +import { REDIS_URL } from "../../translate/constants"; +import { processTranslationJob } from "./translation"; + +const createUserTranslationQueue = (userId: number) => + new Queue(`translation-user-${userId}`, REDIS_URL); + +const userTranslationQueues: { [userId: number]: QueueType } = {}; + +export function setupUserQueue(userId: number, geminiApiKey: string) { + if (userTranslationQueues[userId]) { + return userTranslationQueues[userId]; + } + const userTranslationQueue = createUserTranslationQueue(userId); + userTranslationQueue.process(async (job) => { + await processTranslationJob(job, geminiApiKey, userId); + }); + userTranslationQueues[userId] = userTranslationQueue; + return userTranslationQueue; +} diff --git a/web/app/routes/_index/route.tsx b/web/app/routes/_index/route.tsx index 8314c6d..d24b9bd 100644 --- a/web/app/routes/_index/route.tsx +++ b/web/app/routes/_index/route.tsx @@ -4,29 +4,31 @@ import type { LoaderFunctionArgs, MetaFunction, } from "@remix-run/node"; -import { json } from "@remix-run/node"; import { redirect } from "@remix-run/node"; import { typedjson, useTypedLoaderData } from "remix-typedjson"; import { z } from "zod"; import { Header } from "~/components/Header"; import { authenticator } from "~/utils/auth.server"; import { getSession } from "~/utils/session.server"; -import { extractNumberedElements } from "../../utils/extractNumberedElements"; +import { extractNumberedElements } from "../../libs/extractNumberedElements"; import { prisma } from "../../utils/prisma"; +import { geminiApiKeySchema } from "../translate/types"; +import { + type UserAITranslationInfoItem, + UserAITranslationInfoSchema, +} from "../translate/types"; import { GoogleSignInAndGeminiApiKeyForm } from "./components/GoogleSignInAndGeminiApiKeyForm"; import { URLTranslationForm, urlTranslationSchema, } from "./components/URLTranslationForm"; +import { translate } from "./libs/translation"; import { addNumbersToContent } from "./utils/addNumbersToContent"; import { extractArticle } from "./utils/articleUtils"; import { fetchWithRetry } from "./utils/fetchWithRetry"; import { validateGeminiApiKey } from "./utils/gemini"; -import { translate } from "./utils/translation"; -import { geminiApiKeySchema } from "./types"; -import type { UserReadHistoryItem } from "./types"; -import { UserReadHistoryList } from "./components/UserReadHistoryList"; +import { UserAITranslationStatus } from "./components/UserAITranslationStatus"; export const meta: MetaFunction = () => { return [ @@ -39,41 +41,61 @@ export const meta: MetaFunction = () => { ]; }; - export async function loader({ request }: LoaderFunctionArgs) { const safeUser = await authenticator.isAuthenticated(request); const session = await getSession(request.headers.get("Cookie")); const targetLanguage = session.get("targetLanguage") || "ja"; let hasGeminiApiKey = false; - let userReadHistory: UserReadHistoryItem[] = []; + let userAITranslationInfo: UserAITranslationInfoItem[] = []; + if (safeUser) { const dbUser = await prisma.user.findUnique({ where: { id: safeUser.id } }); hasGeminiApiKey = !!dbUser?.geminiApiKey; - - userReadHistory = await prisma.userReadHistory.findMany({ - where: { userId: safeUser.id }, - include: { - pageVersion: { - include: { - page: true, - pageVersionTranslationInfo: { - where: { targetLanguage } - } - } - } - }, - orderBy: { readAt: 'desc' }, - take: 10 - }); - } + const rawTranslationInfo = await prisma.userAITranslationInfo.findMany({ + where: { + userId: safeUser.id, + targetLanguage, + }, + include: { + pageVersion: { + select: { + title: true, + page: { + select: { + url: true, + }, + }, + pageVersionTranslationInfo: { + where: { + targetLanguage, + }, + }, + }, + }, + }, + orderBy: { + lastTranslatedAt: "desc", + }, + take: 10, + }); + + // Validate and transform data + userAITranslationInfo = z + .array(UserAITranslationInfoSchema) + .parse(rawTranslationInfo); + } - return typedjson({ safeUser, targetLanguage, hasGeminiApiKey, userReadHistory }); + return typedjson({ + safeUser, + targetLanguage, + hasGeminiApiKey, + userAITranslationInfo, + }); } export async function action({ request }: ActionFunctionArgs) { - const formData = await request.clone().formData(); switch (formData.get("intent")) { case "SignInWithGoogle": @@ -116,7 +138,9 @@ export async function action({ request }: ActionFunctionArgs) { if (submission.status !== "success") { return submission.reply(); } - const dbUser = await prisma.user.findUnique({ where: { id: safeUser.id } }); + const dbUser = await prisma.user.findUnique({ + where: { id: safeUser.id }, + }); const geminiApiKey = dbUser?.geminiApiKey; if (!geminiApiKey) { return submission.reply({ @@ -144,9 +168,8 @@ export async function action({ request }: ActionFunctionArgs) { } } export default function Index() { - const { safeUser, targetLanguage, hasGeminiApiKey, userReadHistory } = - useTypedLoaderData(); - + const { safeUser, targetLanguage, hasGeminiApiKey, userAITranslationInfo } = + useTypedLoaderData(); return (
@@ -169,10 +192,13 @@ export default function Index() {
)} {safeUser && hasGeminiApiKey && ( -
- -
- )} +
+ +
+ )}
diff --git a/web/app/routes/_index/types.ts b/web/app/routes/_index/types.ts deleted file mode 100644 index d5287d3..0000000 --- a/web/app/routes/_index/types.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { z } from "zod"; - -export type TranslationStatus = "pending" | "in_progress" | "completed" | "failed"; - -export interface TranslationStatusRecord { - id: number; - pageVersionId: number; - language: string; - status: TranslationStatus; -} - -export const geminiApiKeySchema = z.object({ - geminiApiKey: z.string().min(1, "API key is required"), -}); - -export type UserReadHistoryItem = { - id: number; - readAt: Date; - pageVersion: { - id: number; - title: string; - page: { - url: string; - }; - pageVersionTranslationInfo: Array<{ - targetLanguage: string; - translationTitle: string; - translationStatus: string; - translationProgress: number; - }>; - }; -}; \ No newline at end of file diff --git a/web/app/routes/_index/utils/extractTranslations.ts b/web/app/routes/_index/utils/extractTranslations.ts deleted file mode 100644 index b2ef609..0000000 --- a/web/app/routes/_index/utils/extractTranslations.ts +++ /dev/null @@ -1,31 +0,0 @@ -export function extractTranslations( - text: string, -): { number: number; text: string }[] { - try { - // まず、文字列をJSONとしてパースしてみる - const parsed = JSON.parse(text); - if (Array.isArray(parsed)) { - // すでに配列の場合は、そのまま返す - return parsed; - } - } catch (error) { - // JSONとしてパースできない場合は、元の正規表現ベースの処理を行う - } - - const translations: { number: number; text: string }[] = []; - const regex = - /{\s*"number"\s*:\s*(\d+)\s*,\s*"text"\s*:\s*"((?:\\.|[^"\\])*)"\s*}/g; - let match: RegExpExecArray | null; - - while (true) { - match = regex.exec(text); - if (match === null) break; - - translations.push({ - number: Number.parseInt(match[1], 10), - text: match[2].replace(/\\"/g, '"').replace(/\\n/g, "\n"), - }); - } - - return translations; -} diff --git a/web/app/routes/_index/utils/translation.ts b/web/app/routes/_index/utils/translation.ts deleted file mode 100644 index 2e1f101..0000000 --- a/web/app/routes/_index/utils/translation.ts +++ /dev/null @@ -1,258 +0,0 @@ -import { getOrCreatePageId } from "../../../utils/pageService"; -import { getOrCreatePageVersionId } from "../../../utils/pageVersionService"; -import { prisma } from "../../../utils/prisma"; -import { getOrCreateSourceTextId } from "../../../utils/sourceTextService"; -import { getOrCreateAIUser } from "../../../utils/userService"; -import { updateUserReadHistory } from "./../../../utils/userReadHistory"; -import { getGeminiModelResponse } from "./gemini"; -import { updatePageVersionTranslationInfoTranslationStatusAndTranslationProgress, getOrCreatePageVersionTranslationInfo } from "./../../../utils/pageVersionTranslationInfo"; -import Queue, { type Queue as QueueType } from "bull"; -import { updatePageVersionTranslationInfoTitle } from "../../../utils/pageVersionTranslationInfo"; -const REDIS_URL = process.env.REDIS_URL || "redis://localhost:6379"; -const createUserTranslationQueue = (userId: number) => - new Queue(`translation-user-${userId}`, REDIS_URL); -const userTranslationQueues: { [userId: number]: QueueType } = {}; - -const MAX_CHUNK_SIZE = 30000; -export type NumberedElement = { - number: number; - text: string; -}; - -export async function translate( - geminiApiKey: string, - userId: number, - targetLanguage: string, - title: string, - numberedContent: string, - numberedElements: NumberedElement[], - url: string, -): Promise { - const pageId = await getOrCreatePageId(url || ""); - const pageVersionId = await getOrCreatePageVersionId( - url, - title, - numberedContent, - pageId, - ); - - const pageVersionTranslationInfo = await getOrCreatePageVersionTranslationInfo( - pageVersionId, - targetLanguage - ); - await updateUserReadHistory(userId, pageVersionId, 0); - - if (pageVersionTranslationInfo.translationStatus === "completed") { - return pageVersionTranslationInfo.translationStatus; - } - - const userTranslationQueue = setupUserQueue(userId, geminiApiKey); - await userTranslationQueue.add({ - pageId, - pageVersionId, - targetLanguage, - title, - numberedElements, - }); - - return pageVersionTranslationInfo.translationStatus; -} - -export function setupUserQueue(userId: number, geminiApiKey: string) { - if (userTranslationQueues[userId]) { - return userTranslationQueues[userId]; - } - const userTranslationQueue = createUserTranslationQueue(userId); - userTranslationQueue.process(async (job) => { - const { pageId, pageVersionId, targetLanguage, title, numberedElements } = - job.data; - try { - const chunks = splitNumberedElements(numberedElements); - const totalChunks = chunks.length; - for (let i = 0; i < chunks.length; i++) { - await getOrCreateTranslations( - geminiApiKey, - chunks[i], - targetLanguage, - pageId, - pageVersionId, - title, - ); - const progress = ((i + 1) / totalChunks) * 100; - await updatePageVersionTranslationInfoTranslationStatusAndTranslationProgress(pageVersionId, targetLanguage, "in_progress", progress); - } - await updatePageVersionTranslationInfoTranslationStatusAndTranslationProgress(pageVersionId, targetLanguage, "completed", 100); - } catch (error) { - console.error("Background translation job failed:", error); - await updatePageVersionTranslationInfoTranslationStatusAndTranslationProgress(pageVersionId, targetLanguage, "failed", 0); - } - }); - userTranslationQueues[userId] = userTranslationQueue; - return userTranslationQueue; -} - -function splitNumberedElements( - elements: NumberedElement[], -): NumberedElement[][] { - const chunks: NumberedElement[][] = []; - let currentChunk: NumberedElement[] = []; - let currentSize = 0; - - for (const element of elements) { - if ( - currentSize + element.text.length > MAX_CHUNK_SIZE && - currentChunk.length > 0 - ) { - chunks.push(currentChunk); - currentChunk = []; - currentSize = 0; - } - currentChunk.push(element); - currentSize += element.text.length; - } - - if (currentChunk.length > 0) { - chunks.push(currentChunk); - } - return chunks; -} - -export function extractTranslations( - text: string, -): { number: number; text: string }[] { - const translations: { number: number; text: string }[] = []; - const regex = - /{\s*"number"\s*:\s*(\d+)\s*,\s*"text"\s*:\s*"((?:\\.|[^"\\])*)"\s*}/g; - let match: RegExpExecArray | null; - - while (true) { - match = regex.exec(text); - if (match === null) break; - - translations.push({ - number: Number.parseInt(match[1], 10), - text: match[2].replace(/\\"/g, '"').replace(/\\n/g, "\n"), - }); - } - return translations; -} - -async function getOrCreateTranslations( - geminiApiKey: string, - elements: { number: number; text: string }[], - targetLanguage: string, - pageId: number, - pageVersionId: number, - title: string, -): Promise<{ number: number; text: string }[]> { - const translations: { number: number; text: string }[] = []; - const untranslatedElements: { number: number; text: string }[] = []; - const sourceTextsId = await Promise.all( - elements.map((element) => - getOrCreateSourceTextId( - element.text, - element.number, - pageId, - pageVersionId, - ), - ), - ); - - const existingTranslations = await prisma.translateText.findMany({ - where: { - sourceTextId: { in: sourceTextsId }, - targetLanguage, - }, - orderBy: [{ point: "desc" }, { createdAt: "desc" }], - }); - - const translationMap = new Map( - existingTranslations.map((t) => [t.sourceTextId, t]), - ); - - elements.forEach((element, index) => { - const sourceTextId = sourceTextsId[index]; - const existingTranslation = translationMap.get(sourceTextId); - - if (existingTranslation) { - translations.push({ - number: element.number, - text: existingTranslation.text, - }); - } else { - untranslatedElements.push(element); - } - }); - if (untranslatedElements.length > 0) { - const newTranslations = await translateUntranslatedElements( - geminiApiKey, - untranslatedElements, - targetLanguage, - pageId, - pageVersionId, - title, - ); - translations.push(...newTranslations); - } - - return translations.sort((a, b) => a.number - b.number); -} - -async function translateUntranslatedElements( - geminiApiKey: string, - untranslatedElements: { number: number; text: string }[], - targetLanguage: string, - pageId: number, - pageVersionId: number, - title: string, -): Promise<{ number: number; text: string }[]> { - const source_text = untranslatedElements - .map((el) => JSON.stringify(el)) - .join("\n"); - const model = "gemini-1.5-pro-latest"; - const translatedText = await getGeminiModelResponse( - geminiApiKey, - model, - title, - source_text, - targetLanguage, - ); - - const extractedTranslations = extractTranslations(translatedText); - await updatePageVersionTranslationInfoTitle(pageVersionId, targetLanguage, extractedTranslations[0].text); - - const systemUserId = await getOrCreateAIUser(model); - - await Promise.all( - extractedTranslations.map(async (translation) => { - const sourceText = untranslatedElements.find( - (el) => el.number === translation.number, - )?.text; - - if (!sourceText) { - console.error( - `Source text not found for translation number ${translation.number}`, - ); - return; - } - - const sourceTextId = await getOrCreateSourceTextId( - sourceText, - translation.number, - pageId, - pageVersionId, - ); - await prisma.translateText.create({ - data: { - targetLanguage, - text: translation.text, - sourceTextId, - pageId, - userId: systemUserId, - }, - }); - }), - ); - - return extractedTranslations; -} diff --git a/web/app/routes/api.auth.callback.google.ts b/web/app/routes/api.auth.callback.google.ts index 9e6fce7..897fedf 100644 --- a/web/app/routes/api.auth.callback.google.ts +++ b/web/app/routes/api.auth.callback.google.ts @@ -5,7 +5,7 @@ export const loader = ({ request }: LoaderFunctionArgs) => { try { return authenticator.authenticate("google", request, { successRedirect: "/", - failureRedirect: "/auth/login", + failureRedirect: "/", }); } catch (error) { console.error("Google authentication error:", error); diff --git a/web/app/routes/auth.login.tsx b/web/app/routes/auth.login.tsx index 00a8467..27db384 100644 --- a/web/app/routes/auth.login.tsx +++ b/web/app/routes/auth.login.tsx @@ -1,30 +1,9 @@ -import { getFormProps, getInputProps, useForm } from "@conform-to/react"; -import { parseWithZod } from "@conform-to/zod"; import type { ActionFunctionArgs, LoaderFunctionArgs } from "@remix-run/node"; import { json } from "@remix-run/node"; -import { Link } from "@remix-run/react"; -import { Form, useActionData } from "@remix-run/react"; -import { z } from "zod"; -import { Alert, AlertDescription } from "~/components/ui/alert"; -import { Button } from "~/components/ui/button"; -import { - Card, - CardContent, - CardFooter, - CardHeader, - CardTitle, -} from "~/components/ui/card"; -import { Input } from "~/components/ui/input"; -import { Label } from "~/components/ui/label"; -import { Separator } from "~/components/ui/separator"; +import { Card, CardContent, CardHeader, CardTitle } from "~/components/ui/card"; import { GoogleForm } from "../components/GoogleForm"; import { authenticator } from "../utils/auth.server"; -const loginSchema = z.object({ - email: z.string().email("有効なメールアドレスを入力してください"), - password: z.string().min(4, "パスワードは4文字以上である必要があります"), -}); - export const loader = async ({ request }: LoaderFunctionArgs) => { const safeUser = await authenticator.isAuthenticated(request, { successRedirect: "/", @@ -33,41 +12,13 @@ export const loader = async ({ request }: LoaderFunctionArgs) => { }; export const action = async ({ request }: ActionFunctionArgs) => { - const formData = await request.clone().formData(); - const action = String(formData.get("_action")); - const submission = parseWithZod(formData, { schema: loginSchema }); - try { - switch (action) { - case "SignIn": - if (submission.status !== "success") { - return json({ result: submission.reply() }); - } - return authenticator.authenticate("user-pass", request, { - successRedirect: "/", - failureRedirect: "/auth/login", - }); - case "SignInWithGoogle": - return authenticator.authenticate("google", request, { - successRedirect: "/", - failureRedirect: "/auth/login", - }); - default: - return json({ result: { status: "error", message: "Invalid action" } }); - } - } catch (error) { - return json({ result: { status: "error", message: error } }); - } + return authenticator.authenticate("google", request, { + successRedirect: "/", + failureRedirect: "/auth/login", + }); }; const LoginPage = () => { - const lastSubmission = useActionData(); - const [form, { email, password }] = useForm({ - id: "login-form", - onValidate({ formData }) { - return parseWithZod(formData, { schema: loginSchema }); - }, - }); - return (
@@ -75,53 +26,8 @@ const LoginPage = () => { Login -
- {lastSubmission?.result?.status === "error" && ( - - - {JSON.stringify(lastSubmission.result)} - - - )} -
-
- - - {email.errors && ( -

e{email.errors}

- )} -
-
- - - {password.errors && ( -

{password.errors}

- )} -
- -
-
- - - -
- Or continue with -
-

- Don't have an account?{" "} - - Sign Up - -

- +
); diff --git a/web/app/routes/translate/components/GoogleSignInAndGeminiApiKeyForm.tsx b/web/app/routes/translate/components/GoogleSignInAndGeminiApiKeyForm.tsx new file mode 100644 index 0000000..7ab0d51 --- /dev/null +++ b/web/app/routes/translate/components/GoogleSignInAndGeminiApiKeyForm.tsx @@ -0,0 +1,86 @@ +import { useForm } from "@conform-to/react"; +import { getFormProps, getInputProps } from "@conform-to/react"; +import type { SubmissionResult } from "@conform-to/react"; +import { getZodConstraint, parseWithZod } from "@conform-to/zod"; +import { Form } from "@remix-run/react"; +import { useActionData } from "@remix-run/react"; +import { Save } from "lucide-react"; +import { GoogleForm } from "~/components/GoogleForm"; +import { Button } from "~/components/ui/button"; +import { Card, CardContent, CardHeader, CardTitle } from "~/components/ui/card"; +import { Input } from "~/components/ui/input"; +import { geminiApiKeySchema } from "../types"; +interface GoogleSignInAndGeminiApiKeyFormProps { + isLoggedIn: boolean; + hasGeminiApiKey: boolean; + error?: string; +} + +export function GoogleSignInAndGeminiApiKeyForm({ + isLoggedIn, + hasGeminiApiKey, + error, +}: GoogleSignInAndGeminiApiKeyFormProps) { + const lastResult = useActionData(); + const [form, { geminiApiKey }] = useForm({ + id: "gemini-api-key-form", + lastResult, + constraint: getZodConstraint(geminiApiKeySchema), + shouldValidate: "onBlur", + shouldRevalidate: "onInput", + onValidate({ formData }) { + return parseWithZod(formData, { schema: geminiApiKeySchema }); + }, + }); + + return ( + + + + {isLoggedIn + ? hasGeminiApiKey + ? "Update Gemini API Key" + : "Set Gemini API Key" + : "Sign in and Set Gemini API Key"} + + + + {!isLoggedIn && ( +
+ +
+ )} + {isLoggedIn && !hasGeminiApiKey && ( +
+
+
+ +
+ +
+
+ {geminiApiKey.errors} +
+ {form.errors && ( +

{form.errors}

+ )} +
+ )} + + + ); +} diff --git a/web/app/routes/translate/components/URLTranslationForm.tsx b/web/app/routes/translate/components/URLTranslationForm.tsx new file mode 100644 index 0000000..291678e --- /dev/null +++ b/web/app/routes/translate/components/URLTranslationForm.tsx @@ -0,0 +1,60 @@ +import { getFormProps, getInputProps, useForm } from "@conform-to/react"; +import { getZodConstraint, parseWithZod } from "@conform-to/zod"; +import { Form, useNavigation } from "@remix-run/react"; +import { Languages } from "lucide-react"; +import { z } from "zod"; +import { LoadingSpinner } from "~/components/LoadingSpinner"; +import { Button } from "~/components/ui/button"; +import { Input } from "~/components/ui/input"; + +const urlTranslationSchema = z.object({ + url: z + .string() + .min(1, { message: "URLを入力してください" }) + .url("有効なURLを入力してください"), +}); + +export function URLTranslationForm() { + const navigation = useNavigation(); + + const [form, fields] = useForm({ + id: "url-translation-form", + constraint: getZodConstraint(urlTranslationSchema), + shouldValidate: "onBlur", + shouldRevalidate: "onInput", + onValidate({ formData }) { + return parseWithZod(formData, { schema: urlTranslationSchema }); + }, + }); + + return ( +
+
+
+
+ +
{fields.url.errors}
+
+ +
+
+
+ ); +} + +export { urlTranslationSchema }; diff --git a/web/app/routes/translate/components/UserAITranslationStatus.tsx b/web/app/routes/translate/components/UserAITranslationStatus.tsx new file mode 100644 index 0000000..1be42d4 --- /dev/null +++ b/web/app/routes/translate/components/UserAITranslationStatus.tsx @@ -0,0 +1,101 @@ +import { Link } from "@remix-run/react"; +import { Badge } from "~/components/ui/badge"; +import { Card, CardContent, CardHeader, CardTitle } from "~/components/ui/card"; +import { Progress } from "~/components/ui/progress"; +import { ScrollArea } from "~/components/ui/scroll-area"; +import type { UserAITranslationInfoItem } from "../types"; + +type UserAITranslationStatusProps = { + userAITranslationInfo: UserAITranslationInfoItem[]; + targetLanguage: string; +}; + +export function UserAITranslationStatus({ + userAITranslationInfo = [], + targetLanguage, +}: UserAITranslationStatusProps) { + if (!userAITranslationInfo || userAITranslationInfo.length === 0) { + return ( + + + Translation Status ({targetLanguage}) + + +

No translation history available.

+ + + ); + } + + return ( + + + Translation Status ({targetLanguage}) + + + +
+ {userAITranslationInfo.map((item) => { + const translationInfo = + item.pageVersion.pageVersionTranslationInfo?.[0]; + return ( + + + + + {item.pageVersion.title} + {translationInfo?.translationTitle && ( + + {translationInfo.translationTitle} + + )} + + + +

+ {item.pageVersion.page.url} +

+ + {item.aiTranslationStatus} + + +

+ Last updated:{" "} + {new Date(item.lastTranslatedAt).toLocaleString()} +

+ + + + ); + })} +
+ + + + ); +} + +function getVariantForStatus( + status: string, +): "default" | "secondary" | "destructive" | "outline" { + switch (status) { + case "completed": + return "default"; + case "processing": + return "secondary"; + case "failed": + return "destructive"; + default: + return "outline"; + } +} diff --git a/web/app/routes/translate/components/translatedList.tsx b/web/app/routes/translate/components/translatedList.tsx new file mode 100644 index 0000000..212ce64 --- /dev/null +++ b/web/app/routes/translate/components/translatedList.tsx @@ -0,0 +1,22 @@ +import type { PageVersion } from "@prisma/client"; +import { useLoaderData } from "@remix-run/react"; +interface LoaderData { + pageVersionList: PageVersion[]; +} + +export function TranslatedList(language: string) { + const { pageVersionList } = useLoaderData(); + return ( +
+

Translated List (Language: {language})

+
    + {pageVersionList.map((pageVersion) => ( +
  • + {/* ページのタイトルや他の情報を表示 */} + {pageVersion.title} - {pageVersion.url} +
    • + ))} +
+
+ ); +} diff --git a/web/app/routes/translate/constants.ts b/web/app/routes/translate/constants.ts new file mode 100644 index 0000000..eab1693 --- /dev/null +++ b/web/app/routes/translate/constants.ts @@ -0,0 +1,3 @@ +export const REDIS_URL = process.env.REDIS_URL || "redis://localhost:6379"; +export const MAX_CHUNK_SIZE = 30000; +export const AI_MODEL = "gemini-1.5-pro-latest"; diff --git a/web/app/routes/translate/libs/translation.ts b/web/app/routes/translate/libs/translation.ts new file mode 100644 index 0000000..18a4916 --- /dev/null +++ b/web/app/routes/translate/libs/translation.ts @@ -0,0 +1,101 @@ +import type { Job } from "bull"; +import { getOrCreatePageId } from "../../../libs/pageService"; +import { getOrCreatePageVersionId } from "../../../libs/pageVersion"; +import { + getOrCreateUserAITranslationInfo, + updateUserAITranslationInfo, +} from "../../../libs/userAITranslationInfo"; +import { updateUserReadHistory } from "../../../libs/userReadHistory"; +import type { NumberedElement } from "../types"; +import { + getOrCreateTranslations, + splitNumberedElements, +} from "./translationUtils"; +import { setupUserQueue } from "./userTranslationqueueService"; + +export async function translate( + geminiApiKey: string, + userId: number, + targetLanguage: string, + title: string, + numberedContent: string, + numberedElements: NumberedElement[], + url: string, +): Promise { + const pageId = await getOrCreatePageId(url || ""); + const pageVersionId = await getOrCreatePageVersionId( + url, + title, + numberedContent, + pageId, + ); + + const userAITranslationHistory = await getOrCreateUserAITranslationInfo( + userId, + pageVersionId, + targetLanguage, + ); + await updateUserReadHistory(userId, pageVersionId, 0); + + if (userAITranslationHistory.aiTranslationStatus === "completed") { + return userAITranslationHistory.aiTranslationStatus; + } + + const userTranslationQueue = setupUserQueue(userId, geminiApiKey); + await userTranslationQueue.add({ + pageId, + pageVersionId, + targetLanguage, + title, + numberedElements, + }); + + return userAITranslationHistory.aiTranslationStatus; +} + +export async function processTranslationJob( + job: Job, + geminiApiKey: string, + userId: number, +) { + const { pageId, pageVersionId, targetLanguage, title, numberedElements } = + job.data; + try { + const chunks = splitNumberedElements(numberedElements); + const totalChunks = chunks.length; + for (let i = 0; i < chunks.length; i++) { + await getOrCreateTranslations( + geminiApiKey, + chunks[i], + targetLanguage, + pageId, + pageVersionId, + title, + ); + const progress = ((i + 1) / totalChunks) * 100; + await updateUserAITranslationInfo( + userId, + pageVersionId, + targetLanguage, + "in_progress", + progress, + ); + } + await updateUserAITranslationInfo( + userId, + pageVersionId, + targetLanguage, + "completed", + 100, + ); + } catch (error) { + console.error("Background translation job failed:", error); + await updateUserAITranslationInfo( + userId, + pageVersionId, + targetLanguage, + "failed", + 0, + ); + } +} diff --git a/web/app/routes/translate/libs/translationUtils.ts b/web/app/routes/translate/libs/translationUtils.ts new file mode 100644 index 0000000..c83e420 --- /dev/null +++ b/web/app/routes/translate/libs/translationUtils.ts @@ -0,0 +1,175 @@ +import { getOrCreatePageVersionTranslationInfo } from "../../../libs/pageVersionTranslationInfo"; +import { getOrCreateSourceTextId } from "../../../libs/sourceTextService"; +import { getOrCreateAIUser } from "../../../libs/userService"; +import { prisma } from "../../../utils/prisma"; +import { AI_MODEL, MAX_CHUNK_SIZE } from "../constants"; +import type { NumberedElement } from "../types"; +import { getGeminiModelResponse } from "../utils/gemini"; + +export function splitNumberedElements( + elements: NumberedElement[], +): NumberedElement[][] { + const chunks: NumberedElement[][] = []; + let currentChunk: NumberedElement[] = []; + let currentSize = 0; + + for (const element of elements) { + if ( + currentSize + element.text.length > MAX_CHUNK_SIZE && + currentChunk.length > 0 + ) { + chunks.push(currentChunk); + currentChunk = []; + currentSize = 0; + } + currentChunk.push(element); + currentSize += element.text.length; + } + + if (currentChunk.length > 0) { + chunks.push(currentChunk); + } + return chunks; +} + +export function extractTranslations(jsonString: string): NumberedElement[] { + try { + const parsedData = JSON.parse(jsonString); + + if (Array.isArray(parsedData)) { + return parsedData.map((item) => ({ + number: Number(item.number), + text: String(item.text), + })); + } + console.error("Parsed data is not an array"); + return []; + } catch (error) { + console.error("Failed to parse JSON:", error); + return []; + } +} + +export async function getOrCreateTranslations( + geminiApiKey: string, + elements: NumberedElement[], + targetLanguage: string, + pageId: number, + pageVersionId: number, + title: string, +): Promise { + const translations: NumberedElement[] = []; + const untranslatedElements: NumberedElement[] = []; + const sourceTextsId = await Promise.all( + elements.map((element) => + getOrCreateSourceTextId( + element.text, + element.number, + pageId, + pageVersionId, + ), + ), + ); + + const existingTranslations = await prisma.translateText.findMany({ + where: { + sourceTextId: { in: sourceTextsId }, + targetLanguage, + }, + orderBy: [{ point: "desc" }, { createdAt: "desc" }], + }); + + const translationMap = new Map( + existingTranslations.map((t) => [t.sourceTextId, t]), + ); + + elements.forEach((element, index) => { + const sourceTextId = sourceTextsId[index]; + const existingTranslation = translationMap.get(sourceTextId); + + if (existingTranslation) { + translations.push({ + number: element.number, + text: existingTranslation.text, + }); + } else { + untranslatedElements.push(element); + } + }); + + if (untranslatedElements.length > 0) { + const newTranslations = await translateUntranslatedElements( + geminiApiKey, + untranslatedElements, + targetLanguage, + pageId, + pageVersionId, + title, + ); + translations.push(...newTranslations); + } + + return translations.sort((a, b) => a.number - b.number); +} + +async function translateUntranslatedElements( + geminiApiKey: string, + untranslatedElements: NumberedElement[], + targetLanguage: string, + pageId: number, + pageVersionId: number, + title: string, +): Promise { + const source_text = untranslatedElements + .map((el) => JSON.stringify(el)) + .join("\n"); + const translatedText = await getGeminiModelResponse( + geminiApiKey, + AI_MODEL, + title, + source_text, + targetLanguage, + ); + + const extractedTranslations = extractTranslations(translatedText); + await getOrCreatePageVersionTranslationInfo( + pageVersionId, + targetLanguage, + extractedTranslations[0].text, + ); + + const systemUserId = await getOrCreateAIUser(AI_MODEL); + + await Promise.all( + extractedTranslations.map(async (translation) => { + const sourceText = untranslatedElements.find( + (el) => el.number === translation.number, + )?.text; + + if (!sourceText) { + console.error( + `Source text not found for translation number ${translation.number}`, + ); + return; + } + + const sourceTextId = await getOrCreateSourceTextId( + sourceText, + translation.number, + pageId, + pageVersionId, + ); + await prisma.translateText.create({ + data: { + targetLanguage, + text: translation.text, + sourceTextId, + pageId, + userId: systemUserId, + }, + }); + }), + ); + + return extractedTranslations; +} diff --git a/web/app/routes/translate/libs/userTranslationqueueService.ts b/web/app/routes/translate/libs/userTranslationqueueService.ts new file mode 100644 index 0000000..7f50e83 --- /dev/null +++ b/web/app/routes/translate/libs/userTranslationqueueService.ts @@ -0,0 +1,20 @@ +import Queue, { type Queue as QueueType } from "bull"; +import { REDIS_URL } from "../constants"; +import { processTranslationJob } from "./translation"; + +const createUserTranslationQueue = (userId: number) => + new Queue(`translation-user-${userId}`, REDIS_URL); + +const userTranslationQueues: { [userId: number]: QueueType } = {}; + +export function setupUserQueue(userId: number, geminiApiKey: string) { + if (userTranslationQueues[userId]) { + return userTranslationQueues[userId]; + } + const userTranslationQueue = createUserTranslationQueue(userId); + userTranslationQueue.process(async (job) => { + await processTranslationJob(job, geminiApiKey, userId); + }); + userTranslationQueues[userId] = userTranslationQueue; + return userTranslationQueue; +} diff --git a/web/app/routes/translate/route.tsx b/web/app/routes/translate/route.tsx new file mode 100644 index 0000000..5886fe5 --- /dev/null +++ b/web/app/routes/translate/route.tsx @@ -0,0 +1,126 @@ +import { parseWithZod } from "@conform-to/zod"; +import type { ActionFunctionArgs, LoaderFunctionArgs } from "@remix-run/node"; +import { typedjson, useTypedLoaderData } from "remix-typedjson"; +import { authenticator } from "~/utils/auth.server"; +import { prisma } from "~/utils/prisma"; +import { getSession } from "~/utils/session.server"; +import { extractNumberedElements } from "./../../libs/extractNumberedElements"; +import { + URLTranslationForm, + urlTranslationSchema, +} from "./components/URLTranslationForm"; +import { UserAITranslationStatus } from "./components/UserAITranslationStatus"; +import { translate } from "./libs/translation"; +import { + type UserAITranslationInfoItem, + UserAITranslationInfoSchema, +} from "./types"; +import { addNumbersToContent } from "./utils/addNumbersToContent"; +import { extractArticle } from "./utils/extractArticle"; +import { fetchWithRetry } from "./utils/fetchWithRetry"; +import { z } from "zod"; +import { redirect } from "@remix-run/node"; + +export async function loader({ request }: LoaderFunctionArgs) { + const safeUser = await authenticator.isAuthenticated(request, { + failureRedirect: "/", + }); + + const dbUser = await prisma.user.findUnique({ where: { id: safeUser.id } }); + if (!dbUser?.geminiApiKey) { + return redirect("/settings"); + } + + const session = await getSession(request.headers.get("Cookie")); + const targetLanguage = session.get("targetLanguage") || "ja"; + let userAITranslationInfo: UserAITranslationInfoItem[] = []; + const rawTranslationInfo = await prisma.userAITranslationInfo.findMany({ + where: { + userId: safeUser.id, + targetLanguage, + }, + include: { + pageVersion: { + select: { + title: true, + page: { + select: { + url: true, + }, + }, + pageVersionTranslationInfo: { + where: { + targetLanguage, + }, + }, + }, + }, + }, + orderBy: { + lastTranslatedAt: "desc", + }, + take: 10, + }); + + // Validate and transform data + userAITranslationInfo = z + .array(UserAITranslationInfoSchema) + .parse(rawTranslationInfo); + + return typedjson({ + safeUser, + targetLanguage, + userAITranslationInfo, + }); +} +export async function action({ request }: ActionFunctionArgs) { + const user = await authenticator.isAuthenticated(request, { + failureRedirect: "/", + }); + const formData = await request.formData(); + const submission = parseWithZod(formData, { schema: urlTranslationSchema }); + + if (submission.status !== "success") { + return submission.reply(); + } + + const dbUser = await prisma.user.findUnique({ where: { id: user.id } }); + if (!dbUser?.geminiApiKey) { + return submission.reply({ formErrors: ["Gemini API key is not set"] }); + } + + const html = await fetchWithRetry(submission.value.url); + const { content, title } = extractArticle(html); + const numberedContent = addNumbersToContent(content); + const extractedNumberedElements = extractNumberedElements(numberedContent); + const session = await getSession(request.headers.get("Cookie")); + const targetLanguage = session.get("targetLanguage") || "ja"; + + await translate( + dbUser.geminiApiKey, + user.id, + targetLanguage, + title, + numberedContent, + extractedNumberedElements, + submission.value.url, + ); + + return submission.reply(); +} + +export default function TranslatePage() { + const { user, targetLanguage, userAITranslationInfo } = + useTypedLoaderData(); + + return ( +
+

Translate

+ + +
+ ); +} diff --git a/web/app/routes/translate/types.ts b/web/app/routes/translate/types.ts new file mode 100644 index 0000000..76225c7 --- /dev/null +++ b/web/app/routes/translate/types.ts @@ -0,0 +1,43 @@ +import { z } from "zod"; + +export const geminiApiKeySchema = z.object({ + geminiApiKey: z.string().min(1, "API key is required"), +}); + +export const PageVersionTranslationInfoSchema = z.object({ + id: z.number(), + pageVersionId: z.number(), + targetLanguage: z.string(), + translationTitle: z.string(), +}); + +export const UserAITranslationInfoSchema = z.object({ + id: z.number(), + userId: z.number(), + pageVersionId: z.number(), + targetLanguage: z.string(), + aiTranslationStatus: z.string(), + aiTranslationProgress: z.number(), + lastTranslatedAt: z.string().or(z.date()), + pageVersion: z.object({ + title: z.string(), + page: z.object({ + url: z.string(), + }), + pageVersionTranslationInfo: z + .array(PageVersionTranslationInfoSchema) + .optional(), + }), +}); + +export type UserAITranslationInfoItem = z.infer< + typeof UserAITranslationInfoSchema +>; +export type PageVersionTranslationInfoItem = z.infer< + typeof PageVersionTranslationInfoSchema +>; + +export type NumberedElement = { + number: number; + text: string; +}; diff --git a/web/app/routes/translate/utils/addNumbersToContent.ts b/web/app/routes/translate/utils/addNumbersToContent.ts new file mode 100644 index 0000000..af69ea2 --- /dev/null +++ b/web/app/routes/translate/utils/addNumbersToContent.ts @@ -0,0 +1,77 @@ +import DOMPurify from "isomorphic-dompurify"; +import { JSDOM } from "jsdom"; +const { Node } = new JSDOM().window; + +export function shouldProcessElement(element: Element): boolean { + const htmlElement = element as HTMLElement; + + const isTranslatable = htmlElement.getAttribute("translate") !== "no"; + const isNotExcludedClass = !htmlElement.classList.contains("notranslate"); + const isNotEditable = !htmlElement.isContentEditable; + const isNotAlreadyTranslated = + htmlElement.getAttribute("data-translated") !== "true"; + const isNotTooltip = !htmlElement.classList.contains( + "eveeve-source-text-tooltip", + ); + const isNotExcludedTag = ![ + "script", + "style", + "textarea", + "input", + "pre", + "noscript", + "iframe", + "source", + ].includes(element.nodeName.toLowerCase()); + const isVisible = htmlElement.style.display !== "none"; + const isVisiblyHidden = htmlElement.style.visibility !== "hidden"; + + return ( + isTranslatable && + isNotExcludedClass && + isNotEditable && + isNotAlreadyTranslated && + isNotTooltip && + isNotExcludedTag && + isVisible && + isVisiblyHidden + ); +} + +export function addNumbersToContent(content: string): string { + const sanitizedContent = DOMPurify.sanitize(content); + const doc = new JSDOM(sanitizedContent); + let currentNumber = 1; + + function processNode(node: Node) { + if ( + node.nodeType === Node.ELEMENT_NODE && + shouldProcessElement(node as Element) + ) { + const element = node as Element; + const hasDirectTextContent = Array.from(element.childNodes).some( + (childNode) => + childNode.nodeType === Node.TEXT_NODE && + childNode.textContent?.trim() !== "", + ); + + if (hasDirectTextContent) { + element.setAttribute("data-number", currentNumber.toString()); + currentNumber++; + return; + } + + element.childNodes.forEach(processNode); + } + } + + Array.from(doc.window.document.body.childNodes).forEach(processNode); + + return Array.from(doc.window.document.body.childNodes) + .map((node) => + node.nodeType === Node.ELEMENT_NODE + ? (node as Element).outerHTML + : node.textContent, + ) + .join(""); +} diff --git a/web/app/routes/translate/utils/extractArticle.ts b/web/app/routes/translate/utils/extractArticle.ts new file mode 100644 index 0000000..f64ea86 --- /dev/null +++ b/web/app/routes/translate/utils/extractArticle.ts @@ -0,0 +1,17 @@ +import { Readability } from "@mozilla/readability"; +import { JSDOM } from "jsdom"; + +export const extractArticle = ( + html: string, +): { content: string; title: string } => { + const dom = new JSDOM(html); + const document = dom.window.document; + const reader = new Readability(document); + const article = reader.parse(); + + if (!article) { + throw new Error("記事の抽出に失敗しました"); + } + + return { content: article.content, title: article.title }; +}; diff --git a/web/app/routes/translate/utils/fetchWithRetry.ts b/web/app/routes/translate/utils/fetchWithRetry.ts new file mode 100644 index 0000000..43e4528 --- /dev/null +++ b/web/app/routes/translate/utils/fetchWithRetry.ts @@ -0,0 +1,32 @@ +export async function fetchWithRetry( + url: string, + maxRetries = 3, +): Promise { + let lastError: Error | null = null; + + for (let retryCount = 0; retryCount < maxRetries; retryCount++) { + try { + const response = await fetch(url, {}); + + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); + } + + return await response.text(); + } catch (error: unknown) { + const typedError = error as Error; + console.error(`Attempt ${retryCount + 1} failed: ${typedError}`); + lastError = typedError; + + if (retryCount < maxRetries - 1) { + await new Promise((resolve) => + setTimeout(resolve, 1000 * (retryCount + 1)), + ); + } + } + } + + throw new Error( + `Failed to fetch ${url} after ${maxRetries} attempts. Last error: ${lastError?.message}`, + ); +} diff --git a/web/app/routes/translate/utils/gemini.ts b/web/app/routes/translate/utils/gemini.ts new file mode 100644 index 0000000..4f01482 --- /dev/null +++ b/web/app/routes/translate/utils/gemini.ts @@ -0,0 +1,61 @@ +import { + GoogleGenerativeAI, + HarmBlockThreshold, + HarmCategory, +} from "@google/generative-ai"; +import { generateSystemMessage } from "./generateGeminiMessage"; + +export async function getGeminiModelResponse( + geminiApiKey: string, + model: string, + title: string, + source_text: string, + target_language: string, +) { + const genAI = new GoogleGenerativeAI(geminiApiKey); + const safetySetting = [ + { + category: HarmCategory.HARM_CATEGORY_HARASSMENT, + threshold: HarmBlockThreshold.BLOCK_NONE, + }, + { + category: HarmCategory.HARM_CATEGORY_HATE_SPEECH, + threshold: HarmBlockThreshold.BLOCK_NONE, + }, + { + category: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT, + threshold: HarmBlockThreshold.BLOCK_NONE, + }, + { + category: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT, + threshold: HarmBlockThreshold.BLOCK_NONE, + }, + ]; + const modelConfig = genAI.getGenerativeModel({ + model: model, + safetySettings: safetySetting, + generationConfig: { + responseMimeType: "application/json", + }, + }); + const result = await modelConfig.generateContent( + generateSystemMessage(title, source_text, target_language), + ); + return result.response.text(); +} + +export async function validateGeminiApiKey(apiKey: string): Promise { + try { + const genAI = new GoogleGenerativeAI(apiKey); + const model = genAI.getGenerativeModel({ model: "gemini-pro" }); + + const result = await model.generateContent("Hello, World!"); + const response = await result.response; + const text = response.text(); + + return text.length > 0; + } catch (error) { + console.error("Gemini API key validation failed:", error); + return false; + } +} diff --git a/web/app/routes/translate/utils/generateGeminiMessage.ts b/web/app/routes/translate/utils/generateGeminiMessage.ts new file mode 100644 index 0000000..c735790 --- /dev/null +++ b/web/app/routes/translate/utils/generateGeminiMessage.ts @@ -0,0 +1,48 @@ +export function generateSystemMessage( + title: string, + source_text: string, + target_language: string, +): string { + return ` + You are a skilled translator. Your task is to accurately translate the given text into beautiful and natural sentences in the target language. Please follow these guidelines: + + 1. Maintain consistency: Use the same words for recurring terms with the same meaning to avoid confusing the reader. + 2. Preserve style: Keep a consistent writing style throughout to ensure the translation reads naturally as a single work. + 3. Context awareness: The provided sequence is a passage from one work. The number indicates the sentence's position within that work. + 4. Reader-friendly: While considering the document title, translate in a way that is easy for readers to understand and enjoy. + + Document title: ${title} + + Translate the following array of English texts into ${target_language}. + + Important instructions: + - Do not explain your process or self-reference. + - Present the translated result as a JavaScript array of objects. + - Each object should have 'number' and 'text' properties. + - Include the translated title as the first item with 'number: 0'. + - For the content, the 'number' should correspond to the original text's position. + - The 'text' should contain the translation. + - Always enclose the 'text' value in double quotes. + - Maintain the original array structure and order, with the title translation added as the first item. + - Output ONLY the translated array. No additional text or explanations. + + Input array: + ${source_text} + + Translate to ${target_language} and output in the following format: + [ + { + "number": 0, + "text": "Translated title" + }, + { + "number": 1, + "text": "Translated text for item 1" + }, + { + "number": 2, + "text": "Translated text for item 2" + }, + ... + ]`; +} diff --git a/web/app/utils/auth.server.ts b/web/app/utils/auth.server.ts index b84d27a..e07a504 100644 --- a/web/app/utils/auth.server.ts +++ b/web/app/utils/auth.server.ts @@ -1,6 +1,4 @@ -import bcrypt from "bcryptjs"; -import { Authenticator, AuthorizationError } from "remix-auth"; -import { FormStrategy } from "remix-auth-form"; +import { Authenticator } from "remix-auth"; import { GoogleStrategy } from "remix-auth-google"; import type { SafeUser } from "../types"; import { prisma } from "./prisma"; @@ -14,37 +12,6 @@ if (!SESSION_SECRET) { const authenticator = new Authenticator(sessionStorage); -const formStrategy = new FormStrategy(async ({ form }) => { - const email = form.get("email"); - const password = form.get("password"); - - if (!(email && password)) { - throw new Error("Invalid Request"); - } - - const user = await prisma.user.findUnique({ - where: { email: String(email) }, - }); - - if (!user) { - throw new AuthorizationError("User not found"); - } - - if (!user.password) { - throw new AuthorizationError("User has no password set."); - } - const passwordsMatch = await bcrypt.compare(String(password), user.password); - - if (!passwordsMatch) { - throw new AuthorizationError("Invalid password"); - } - - const { password: _, geminiApiKey: __, ...safeUser } = user; - return safeUser; -}); - -authenticator.use(formStrategy, "user-pass"); - const googleStrategy = new GoogleStrategy( { clientID: process.env.GOOGLE_CLIENT_ID || "", @@ -56,7 +23,7 @@ const googleStrategy = new GoogleStrategy( where: { email: profile.emails[0].value }, }); if (user) { - const { password, geminiApiKey, ...safeUser } = user; + const { geminiApiKey, ...safeUser } = user; return safeUser as SafeUser; } try { @@ -64,6 +31,7 @@ const googleStrategy = new GoogleStrategy( data: { email: profile.emails[0].value || "", name: profile.displayName, + image: profile.photos[0].value, provider: "google", }, }); diff --git a/web/app/utils/pageVersionTranslationInfo.ts b/web/app/utils/pageVersionTranslationInfo.ts deleted file mode 100644 index fd976d8..0000000 --- a/web/app/utils/pageVersionTranslationInfo.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { prisma } from "~/utils/prisma" - -export async function updatePageVersionTranslationInfoTranslationStatusAndTranslationProgress( - pageVersionId: number, - targetLanguage: string, - translationStatus: string, - translationProgress: number, -) { - await prisma.pageVersionTranslationInfo.update({ - where: { - pageVersionId_targetLanguage: { - pageVersionId, - targetLanguage, - }, - }, - data: { - translationStatus, - translationProgress, - }, - }); - return { - pageVersionId, - translationStatus, - translationProgress, - }; -} - -export async function updatePageVersionTranslationInfoTitle(pageVersionId: number, targetLanguage: string, translationTitle: string) { - await prisma.pageVersionTranslationInfo.update({ - where: { - pageVersionId_targetLanguage: { - pageVersionId, - targetLanguage, - }, - }, - data: { - translationTitle, - }, - }); -} - -export async function getOrCreatePageVersionTranslationInfo(pageVersionId: number, targetLanguage: string, translationTitle?: string) { - return await prisma.pageVersionTranslationInfo.upsert({ - where: { - pageVersionId_targetLanguage: { - pageVersionId, - targetLanguage, - }, - }, - update: {}, - create: { - pageVersionId, - targetLanguage, - translationTitle: translationTitle ?? '', - translationStatus: "in_progress", - translationProgress: 0, - }, - }); -} diff --git a/web/app/utils/signup.server.ts b/web/app/utils/signup.server.ts deleted file mode 100644 index d2a7c36..0000000 --- a/web/app/utils/signup.server.ts +++ /dev/null @@ -1,25 +0,0 @@ -import bcrypt from "bcryptjs"; -import { prisma } from "./prisma"; - -export const createUser = async ( - data: Record<"name" | "email" | "password", string>, -) => { - const { name, email, password } = data; - - if (!(name && email && password)) { - throw new Error("Invalid input"); - } - - const existingUser = await prisma.user.findUnique({ where: { email } }); - - if (existingUser) { - return { error: { message: "メールアドレスは既に登録済みです" } }; - } - - const hashedPassword = await bcrypt.hash(data.password, 12); - const newUser = await prisma.user.create({ - data: { name, email, password: hashedPassword }, - }); - - return { id: newUser.id, email: newUser.email, name: newUser.name }; -}; diff --git a/web/biome.json b/web/biome.json index b92f384..6286816 100644 --- a/web/biome.json +++ b/web/biome.json @@ -30,7 +30,6 @@ "parser": { "allowComments": true }, "formatter": { "enabled": true, - "indentStyle": "space", "indentWidth": 2, "lineWidth": 80 } diff --git a/web/package.json b/web/package.json index e1e79f1..2719f73 100644 --- a/web/package.json +++ b/web/package.json @@ -14,9 +14,9 @@ "dependencies": { "@conform-to/react": "^1.1.5", "@conform-to/zod": "^1.1.5", - "@google/generative-ai": "^0.14.1", + "@google/generative-ai": "^0.15.0", "@mozilla/readability": "^0.5.0", - "@prisma/client": "5.16.2", + "@prisma/client": "5.17.0", "@radix-ui/react-dropdown-menu": "^2.1.1", "@radix-ui/react-icons": "^1.3.0", "@radix-ui/react-label": "^2.1.0", @@ -25,9 +25,9 @@ "@radix-ui/react-select": "^2.1.1", "@radix-ui/react-separator": "^1.1.0", "@radix-ui/react-slot": "^1.1.0", - "@remix-run/react": "^2.10.2", - "@remix-run/serve": "^2.10.2", - "@supabase/supabase-js": "^2.44.3", + "@remix-run/react": "^2.10.3", + "@remix-run/serve": "^2.10.3", + "@supabase/supabase-js": "^2.44.4", "@tailwindcss/typography": "^0.5.13", "@types/bull": "^4.10.0", "bcryptjs": "^2.4.3", @@ -36,7 +36,7 @@ "clsx": "^2.1.1", "html-react-parser": "^5.1.10", "htmlparser2": "^9.1.0", - "isbot": "^4.4.0", + "isbot": "^5.1.13", "isomorphic-dompurify": "^2.13.0", "lucide-react": "^0.408.0", "next-themes": "^0.3.0", @@ -52,8 +52,8 @@ }, "devDependencies": { "@biomejs/biome": "^1.8.3", - "@remix-run/dev": "^2.10.2", - "@remix-run/testing": "^2.10.2", + "@remix-run/dev": "^2.10.3", + "@remix-run/testing": "^2.10.3", "@testing-library/jest-dom": "^6.4.6", "@testing-library/user-event": "^14.5.2", "@types/bcryptjs": "^2.4.6", @@ -61,14 +61,14 @@ "@types/react-dom": "^18.3.0", "autoprefixer": "^10.4.19", "postcss": "^8.4.39", - "prisma": "^5.16.2", - "tailwindcss": "^3.4.4", + "prisma": "^5.17.0", + "tailwindcss": "^3.4.6", "tsx": "^4.16.2", "typescript": "^5.5.3", - "vite": "^5.3.3", + "vite": "^5.3.4", "vite-tsconfig-paths": "^4.3.2", - "vitest": "^2.0.2", - "wrangler": "3.57.1" + "vitest": "^2.0.3", + "wrangler": "3.65.0" }, "engines": { "node": ">=20.0.0" diff --git a/web/prisma/migrations/20240717054302_/migration.sql b/web/prisma/migrations/20240717054302_/migration.sql new file mode 100644 index 0000000..453da2b --- /dev/null +++ b/web/prisma/migrations/20240717054302_/migration.sql @@ -0,0 +1,38 @@ +/* + Warnings: + + - You are about to drop the column `translation_progress` on the `page_version_translation_info` table. All the data in the column will be lost. + - You are about to drop the column `translation_status` on the `page_version_translation_info` table. All the data in the column will be lost. + +*/ +-- AlterTable +ALTER TABLE "page_version_translation_info" DROP COLUMN "translation_progress", +DROP COLUMN "translation_status"; + +-- CreateTable +CREATE TABLE "user_translation_history" ( + "id" SERIAL NOT NULL, + "user_id" INTEGER NOT NULL, + "page_version_id" INTEGER NOT NULL, + "target_language" TEXT NOT NULL, + "ai_translation_status" TEXT NOT NULL DEFAULT 'pending', + "ai_translation_progress" INTEGER NOT NULL DEFAULT 0, + "last_translated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "user_translation_history_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE INDEX "user_translation_history_user_id_idx" ON "user_translation_history"("user_id"); + +-- CreateIndex +CREATE INDEX "user_translation_history_page_version_id_idx" ON "user_translation_history"("page_version_id"); + +-- CreateIndex +CREATE UNIQUE INDEX "user_translation_history_user_id_page_version_id_target_lan_key" ON "user_translation_history"("user_id", "page_version_id", "target_language"); + +-- AddForeignKey +ALTER TABLE "user_translation_history" ADD CONSTRAINT "user_translation_history_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "users"("id") ON DELETE RESTRICT ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "user_translation_history" ADD CONSTRAINT "user_translation_history_page_version_id_fkey" FOREIGN KEY ("page_version_id") REFERENCES "page_versions"("id") ON DELETE RESTRICT ON UPDATE CASCADE; diff --git a/web/prisma/migrations/20240717063343_/migration.sql b/web/prisma/migrations/20240717063343_/migration.sql new file mode 100644 index 0000000..211693c --- /dev/null +++ b/web/prisma/migrations/20240717063343_/migration.sql @@ -0,0 +1,10 @@ +/* + Warnings: + + - You are about to drop the column `password` on the `users` table. All the data in the column will be lost. + - Added the required column `image` to the `users` table without a default value. This is not possible if the table is not empty. + +*/ +-- AlterTable +ALTER TABLE "users" DROP COLUMN "password", +ADD COLUMN "image" TEXT NOT NULL; diff --git a/web/prisma/migrations/20240717065258_/migration.sql b/web/prisma/migrations/20240717065258_/migration.sql new file mode 100644 index 0000000..3a27254 --- /dev/null +++ b/web/prisma/migrations/20240717065258_/migration.sql @@ -0,0 +1,11 @@ +/* + Warnings: + + - A unique constraint covering the columns `[url,content_hash]` on the table `page_versions` will be added. If there are existing duplicate values, this will fail. + +*/ +-- DropIndex +DROP INDEX "page_versions_url_created_at_key"; + +-- CreateIndex +CREATE UNIQUE INDEX "page_versions_url_content_hash_key" ON "page_versions"("url", "content_hash"); diff --git a/web/prisma/schema.prisma b/web/prisma/schema.prisma index 00e3fb6..c0d9550 100644 --- a/web/prisma/schema.prisma +++ b/web/prisma/schema.prisma @@ -8,32 +8,51 @@ datasource db { } model User { - id Int @id @default(autoincrement()) - email String @unique - password String? - name String - plan String @default("free") - totalPoints Int @default(0) @map("total_points") - isAI Boolean @default(false) @map("is_ai") - provider String @default("Credentials") - geminiApiKey String? @map("gemini_api_key") - createdAt DateTime @default(now()) @map("created_at") - updatedAt DateTime @updatedAt @map("updated_at") - userReadHistory UserReadHistory[] - apiUsages ApiUsage[] - translations TranslateText[] - votes Vote[] + id Int @id @default(autoincrement()) + email String @unique + name String + image String + plan String @default("free") + totalPoints Int @default(0) @map("total_points") + isAI Boolean @default(false) @map("is_ai") + provider String @default("Credentials") + geminiApiKey String? @map("gemini_api_key") + createdAt DateTime @default(now()) @map("created_at") + updatedAt DateTime @updatedAt @map("updated_at") + userReadHistory UserReadHistory[] + apiUsages ApiUsage[] + translations TranslateText[] + votes Vote[] + userAITranslationInfo UserAITranslationInfo[] @@map("users") } + +model UserAITranslationInfo { + id Int @id @default(autoincrement()) + userId Int @map("user_id") + pageVersionId Int @map("page_version_id") + targetLanguage String @map("target_language") + aiTranslationStatus String @default("pending") @map("ai_translation_status") + aiTranslationProgress Int @default(0) @map("ai_translation_progress") + lastTranslatedAt DateTime @default(now()) @map("last_translated_at") + user User @relation(fields: [userId], references: [id]) + pageVersion PageVersion @relation(fields: [pageVersionId], references: [id]) + + @@unique([userId, pageVersionId, targetLanguage]) + @@index([userId]) + @@index([pageVersionId]) + @@map("user_translation_history") +} + model UserReadHistory { - id Int @id @default(autoincrement()) - userId Int @map("user_id") - pageVersionId Int @map("page_version_id") - readAt DateTime @default(now()) @map("read_at") - lastReadDataNumber Int @default(0) @map("last_read_data_number") - user User @relation(fields: [userId], references: [id]) - pageVersion PageVersion @relation(fields: [pageVersionId], references: [id]) + id Int @id @default(autoincrement()) + userId Int @map("user_id") + pageVersionId Int @map("page_version_id") + readAt DateTime @default(now()) @map("read_at") + lastReadDataNumber Int @default(0) @map("last_read_data_number") + user User @relation(fields: [userId], references: [id]) + pageVersion PageVersion @relation(fields: [pageVersionId], references: [id]) @@unique([userId, pageVersionId]) @@index([userId]) @@ -41,7 +60,6 @@ model UserReadHistory { @@map("user_read_history") } - model Page { id Int @id @default(autoincrement()) url String @unique @db.VarChar(255) @@ -54,32 +72,31 @@ model Page { } model PageVersionTranslationInfo { - id Int @id @default(autoincrement()) - pageVersionId Int @map("page_version_id") - targetLanguage String @map("target_language") - translationTitle String @map("translation_title") - translationStatus String @default("pending") @map("translation_status") - translationProgress Int @default(0) @map("translation_progress") - pageVersion PageVersion @relation(fields: [pageVersionId], references: [id]) + id Int @id @default(autoincrement()) + pageVersionId Int @map("page_version_id") + targetLanguage String @map("target_language") + translationTitle String @map("translation_title") + pageVersion PageVersion @relation(fields: [pageVersionId], references: [id]) @@unique([pageVersionId, targetLanguage]) @@map("page_version_translation_info") } model PageVersion { - id Int @id @default(autoincrement()) - url String - title String - content String - contentHash Bytes @map("content_hash") - pageId Int @map("page_id") - createdAt DateTime @default(now()) @map("created_at") - page Page @relation(fields: [pageId], references: [id]) - sourceTexts SourceText[] + id Int @id @default(autoincrement()) + url String + title String + content String + contentHash Bytes @map("content_hash") + pageId Int @map("page_id") + createdAt DateTime @default(now()) @map("created_at") + page Page @relation(fields: [pageId], references: [id]) + sourceTexts SourceText[] pageVersionTranslationInfo PageVersionTranslationInfo[] - userReadHistory UserReadHistory[] + userReadHistory UserReadHistory[] + userAITranslationInfo UserAITranslationInfo[] - @@unique([url, createdAt]) + @@unique([url, contentHash]) @@map("page_versions") } @@ -100,19 +117,19 @@ model SourceText { } model TranslateText { - id Int @id @default(autoincrement()) - targetLanguage String - text String - sourceTextId Int @map("source_text_id") - pageId Int @map("page_id") - userId Int @map("user_id") - point Int @default(0) - editCount Int @default(0) @map("edit_count") - createdAt DateTime @default(now()) @map("created_at") - page Page @relation(fields: [pageId], references: [id]) - sourceText SourceText @relation(fields: [sourceTextId], references: [id]) - user User @relation(fields: [userId], references: [id]) - votes Vote[] + id Int @id @default(autoincrement()) + targetLanguage String + text String + sourceTextId Int @map("source_text_id") + pageId Int @map("page_id") + userId Int @map("user_id") + point Int @default(0) + editCount Int @default(0) @map("edit_count") + createdAt DateTime @default(now()) @map("created_at") + page Page @relation(fields: [pageId], references: [id]) + sourceText SourceText @relation(fields: [sourceTextId], references: [id]) + user User @relation(fields: [userId], references: [id]) + votes Vote[] @@map("translate_texts") } diff --git a/web/public/robots.txt b/web/public/robots.txt new file mode 100644 index 0000000..e69de29 diff --git a/web/test/translation.test.ts b/web/test/translation.test.ts index 5daac5d..c4c7033 100644 --- a/web/test/translation.test.ts +++ b/web/test/translation.test.ts @@ -1,6 +1,6 @@ import { createRemixStub } from "@remix-run/testing"; import { test, vi } from "vitest"; -import { action } from "../app/routes/_index/utils/translation"; +import { action } from "../app/routes/_index/libs/translation"; // モックの translateAndDisplayContent 関数 vi.mock("./translation", () => ({ From 7515492b8cae267e9abac3e0f0b389fd78115ade Mon Sep 17 00:00:00 2001 From: tomolld Date: Wed, 17 Jul 2024 21:10:12 +0900 Subject: [PATCH 2/4] =?UTF-8?q?=E3=83=AA=E3=83=95=E3=82=A1=E3=82=AF?= =?UTF-8?q?=E3=82=BF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- web/app/components/Header.tsx | 2 +- web/app/components/ui/tooltip.tsx | 28 +++ .../GoogleSignInAndGeminiApiKeyForm.tsx | 34 +++ .../_index/components/URLTranslationForm.tsx | 60 ------ .../components/UserAITranslationStatus.tsx | 101 --------- .../components/UserTranslationStatus.tsx | 97 --------- .../_index/components/translatedList.tsx | 22 -- web/app/routes/_index/libs/translation.ts | 101 --------- .../routes/_index/libs/translationUtils.ts | 175 --------------- .../libs/userTranslationqueueService.ts | 20 -- web/app/routes/_index/route.tsx | 177 +++------------ web/app/routes/_index/types.ts | 5 + .../_index/utils/addNumbersToContent.ts | 77 ------- web/app/routes/_index/utils/articleUtils.ts | 17 -- web/app/routes/_index/utils/fetchWithRetry.ts | 32 --- .../GoogleSignInAndGeminiApiKeyForm.tsx | 1 - .../translate/components/translatedList.tsx | 22 -- .../routes/translate/libs/translationUtils.ts | 2 +- web/app/routes/translate/route.tsx | 202 ++++++++++-------- web/app/routes/translate/utils/gemini.ts | 61 ------ .../translate/utils/generateGeminiMessage.ts | 48 ----- web/app/routes/types.ts | 43 ++++ web/app/{routes/_index => }/utils/gemini.ts | 33 ++- .../utils/generateGeminiMessage.ts | 0 web/biome.json | 70 +++--- web/components.json | 30 +-- web/package.json | 149 ++++++------- web/public/_routes.json | 6 +- web/tsconfig.json | 58 ++--- 29 files changed, 440 insertions(+), 1233 deletions(-) create mode 100644 web/app/components/ui/tooltip.tsx delete mode 100644 web/app/routes/_index/components/URLTranslationForm.tsx delete mode 100644 web/app/routes/_index/components/UserAITranslationStatus.tsx delete mode 100644 web/app/routes/_index/components/UserTranslationStatus.tsx delete mode 100644 web/app/routes/_index/components/translatedList.tsx delete mode 100644 web/app/routes/_index/libs/translation.ts delete mode 100644 web/app/routes/_index/libs/translationUtils.ts delete mode 100644 web/app/routes/_index/libs/userTranslationqueueService.ts create mode 100644 web/app/routes/_index/types.ts delete mode 100644 web/app/routes/_index/utils/addNumbersToContent.ts delete mode 100644 web/app/routes/_index/utils/articleUtils.ts delete mode 100644 web/app/routes/_index/utils/fetchWithRetry.ts delete mode 100644 web/app/routes/translate/components/translatedList.tsx delete mode 100644 web/app/routes/translate/utils/gemini.ts delete mode 100644 web/app/routes/translate/utils/generateGeminiMessage.ts create mode 100644 web/app/routes/types.ts rename web/app/{routes/_index => }/utils/gemini.ts (62%) rename web/app/{routes/_index => }/utils/generateGeminiMessage.ts (100%) diff --git a/web/app/components/Header.tsx b/web/app/components/Header.tsx index 4610980..5342bc9 100644 --- a/web/app/components/Header.tsx +++ b/web/app/components/Header.tsx @@ -1,6 +1,6 @@ import { Link } from "@remix-run/react"; import { Form, useSubmit } from "@remix-run/react"; -import { LogIn, LogOut } from "lucide-react"; // Lucide アイコンをインポート +import { LogIn, LogOut } from "lucide-react"; import { ModeToggle } from "~/components/dark-mode-toggle"; import type { SafeUser } from "../types"; import { TargetLanguageSelect } from "./TargetLanguageSelect"; diff --git a/web/app/components/ui/tooltip.tsx b/web/app/components/ui/tooltip.tsx new file mode 100644 index 0000000..b48902e --- /dev/null +++ b/web/app/components/ui/tooltip.tsx @@ -0,0 +1,28 @@ +import * as TooltipPrimitive from "@radix-ui/react-tooltip"; +import * as React from "react"; + +import { cn } from "~/utils/cn"; + +const TooltipProvider = TooltipPrimitive.Provider; + +const Tooltip = TooltipPrimitive.Root; + +const TooltipTrigger = TooltipPrimitive.Trigger; + +const TooltipContent = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, sideOffset = 4, ...props }, ref) => ( + +)); +TooltipContent.displayName = TooltipPrimitive.Content.displayName; + +export { Tooltip, TooltipTrigger, TooltipContent, TooltipProvider }; diff --git a/web/app/routes/_index/components/GoogleSignInAndGeminiApiKeyForm.tsx b/web/app/routes/_index/components/GoogleSignInAndGeminiApiKeyForm.tsx index 7487e69..6931beb 100644 --- a/web/app/routes/_index/components/GoogleSignInAndGeminiApiKeyForm.tsx +++ b/web/app/routes/_index/components/GoogleSignInAndGeminiApiKeyForm.tsx @@ -4,12 +4,21 @@ import type { SubmissionResult } from "@conform-to/react"; import { getZodConstraint, parseWithZod } from "@conform-to/zod"; import { Form } from "@remix-run/react"; import { useActionData } from "@remix-run/react"; +import { Link } from "@remix-run/react"; import { Save } from "lucide-react"; +import { ExternalLink, Key } from "lucide-react"; import { GoogleForm } from "~/components/GoogleForm"; import { Button } from "~/components/ui/button"; import { Card, CardContent, CardHeader, CardTitle } from "~/components/ui/card"; import { Input } from "~/components/ui/input"; +import { + Tooltip, + TooltipContent, + TooltipProvider, + TooltipTrigger, +} from "~/components/ui/tooltip"; import { geminiApiKeySchema } from "../../translate/types"; + interface GoogleSignInAndGeminiApiKeyFormProps { isLoggedIn: boolean; hasGeminiApiKey: boolean; @@ -52,6 +61,31 @@ export function GoogleSignInAndGeminiApiKeyForm({ )} {isLoggedIn && !hasGeminiApiKey && (
+
+ + + + + + + + + Create your Gemini API Key at Google AI Studio + + + +
- -
-
- -
{fields.url.errors}
-
- -
- -
- ); -} - -export { urlTranslationSchema }; diff --git a/web/app/routes/_index/components/UserAITranslationStatus.tsx b/web/app/routes/_index/components/UserAITranslationStatus.tsx deleted file mode 100644 index 7cb4ac5..0000000 --- a/web/app/routes/_index/components/UserAITranslationStatus.tsx +++ /dev/null @@ -1,101 +0,0 @@ -import { Link } from "@remix-run/react"; -import { Badge } from "~/components/ui/badge"; -import { Card, CardContent, CardHeader, CardTitle } from "~/components/ui/card"; -import { Progress } from "~/components/ui/progress"; -import { ScrollArea } from "~/components/ui/scroll-area"; -import type { UserAITranslationInfoItem } from "../../translate/types"; - -type UserAITranslationStatusProps = { - userAITranslationInfo: UserAITranslationInfoItem[]; - targetLanguage: string; -}; - -export function UserAITranslationStatus({ - userAITranslationInfo = [], - targetLanguage, -}: UserAITranslationStatusProps) { - if (!userAITranslationInfo || userAITranslationInfo.length === 0) { - return ( - - - Translation Status ({targetLanguage}) - - -

No translation history available.

- - - ); - } - - return ( - - - Translation Status ({targetLanguage}) - - - -
- {userAITranslationInfo.map((item) => { - const translationInfo = - item.pageVersion.pageVersionTranslationInfo?.[0]; - return ( - - - - - {item.pageVersion.title} - {translationInfo?.translationTitle && ( - - {translationInfo.translationTitle} - - )} - - - -

- {item.pageVersion.page.url} -

- - {item.aiTranslationStatus} - - -

- Last updated:{" "} - {new Date(item.lastTranslatedAt).toLocaleString()} -

- - - - ); - })} -
- - - - ); -} - -function getVariantForStatus( - status: string, -): "default" | "secondary" | "destructive" | "outline" { - switch (status) { - case "completed": - return "default"; - case "processing": - return "secondary"; - case "failed": - return "destructive"; - default: - return "outline"; - } -} diff --git a/web/app/routes/_index/components/UserTranslationStatus.tsx b/web/app/routes/_index/components/UserTranslationStatus.tsx deleted file mode 100644 index 383b329..0000000 --- a/web/app/routes/_index/components/UserTranslationStatus.tsx +++ /dev/null @@ -1,97 +0,0 @@ -import { Link } from "@remix-run/react"; -import { Badge } from "~/components/ui/badge"; -import { Card, CardContent, CardHeader, CardTitle } from "~/components/ui/card"; -import { Progress } from "~/components/ui/progress"; -import { ScrollArea } from "~/components/ui/scroll-area"; -import type { UserAITranslationInfoItem } from "../../translate/types"; - -type UserAITranslationStatusProps = { - userAITranslationInfo: UserAITranslationInfoItem[]; - targetLanguage: string; -}; - -export function UserAITranslationStatus({ - userAITranslationInfo = [], - targetLanguage, -}: UserAITranslationStatusProps) { - if (!userAITranslationInfo || userAITranslationInfo.length === 0) { - return ( - - - Translation Status ({targetLanguage}) - - -

No translation history available.

- - - ); - } - - return ( - - - Translation Status ({targetLanguage}) - - - -
- {userAITranslationInfo.map((item) => { - const translationInfo = - item.pageVersion.pageVersionTranslationInfo?.[0]; - return ( - - - - - {translationInfo?.translationTitle || - item.pageVersion.title} - - - -

- {item.pageVersion.page.url} -

- - {item.aiTranslationStatus} - - -

- Last updated:{" "} - {new Date(item.lastTranslatedAt).toLocaleString()} -

- - - - ); - })} -
- - - - ); -} - -function getVariantForStatus( - status: string, -): "default" | "secondary" | "destructive" | "outline" { - switch (status) { - case "completed": - return "default"; - case "processing": - return "secondary"; - case "failed": - return "destructive"; - default: - return "outline"; - } -} diff --git a/web/app/routes/_index/components/translatedList.tsx b/web/app/routes/_index/components/translatedList.tsx deleted file mode 100644 index 212ce64..0000000 --- a/web/app/routes/_index/components/translatedList.tsx +++ /dev/null @@ -1,22 +0,0 @@ -import type { PageVersion } from "@prisma/client"; -import { useLoaderData } from "@remix-run/react"; -interface LoaderData { - pageVersionList: PageVersion[]; -} - -export function TranslatedList(language: string) { - const { pageVersionList } = useLoaderData(); - return ( -
-

Translated List (Language: {language})

-
    - {pageVersionList.map((pageVersion) => ( -
  • - {/* ページのタイトルや他の情報を表示 */} - {pageVersion.title} - {pageVersion.url} -
    • - ))} -
-
- ); -} diff --git a/web/app/routes/_index/libs/translation.ts b/web/app/routes/_index/libs/translation.ts deleted file mode 100644 index 787cde6..0000000 --- a/web/app/routes/_index/libs/translation.ts +++ /dev/null @@ -1,101 +0,0 @@ -import type { Job } from "bull"; -import { getOrCreatePageId } from "../../../libs/pageService"; -import { getOrCreatePageVersionId } from "../../../libs/pageVersion"; -import { - getOrCreateUserAITranslationInfo, - updateUserAITranslationInfo, -} from "../../../libs/userAITranslationInfo"; -import { updateUserReadHistory } from "../../../libs/userReadHistory"; -import type { NumberedElement } from "../../translate/types"; -import { - getOrCreateTranslations, - splitNumberedElements, -} from "./translationUtils"; -import { setupUserQueue } from "./userTranslationqueueService"; - -export async function translate( - geminiApiKey: string, - userId: number, - targetLanguage: string, - title: string, - numberedContent: string, - numberedElements: NumberedElement[], - url: string, -): Promise { - const pageId = await getOrCreatePageId(url || ""); - const pageVersionId = await getOrCreatePageVersionId( - url, - title, - numberedContent, - pageId, - ); - - const userAITranslationHistory = await getOrCreateUserAITranslationInfo( - userId, - pageVersionId, - targetLanguage, - ); - await updateUserReadHistory(userId, pageVersionId, 0); - - if (userAITranslationHistory.aiTranslationStatus === "completed") { - return userAITranslationHistory.aiTranslationStatus; - } - - const userTranslationQueue = setupUserQueue(userId, geminiApiKey); - await userTranslationQueue.add({ - pageId, - pageVersionId, - targetLanguage, - title, - numberedElements, - }); - - return userAITranslationHistory.aiTranslationStatus; -} - -export async function processTranslationJob( - job: Job, - geminiApiKey: string, - userId: number, -) { - const { pageId, pageVersionId, targetLanguage, title, numberedElements } = - job.data; - try { - const chunks = splitNumberedElements(numberedElements); - const totalChunks = chunks.length; - for (let i = 0; i < chunks.length; i++) { - await getOrCreateTranslations( - geminiApiKey, - chunks[i], - targetLanguage, - pageId, - pageVersionId, - title, - ); - const progress = ((i + 1) / totalChunks) * 100; - await updateUserAITranslationInfo( - userId, - pageVersionId, - targetLanguage, - "in_progress", - progress, - ); - } - await updateUserAITranslationInfo( - userId, - pageVersionId, - targetLanguage, - "completed", - 100, - ); - } catch (error) { - console.error("Background translation job failed:", error); - await updateUserAITranslationInfo( - userId, - pageVersionId, - targetLanguage, - "failed", - 0, - ); - } -} diff --git a/web/app/routes/_index/libs/translationUtils.ts b/web/app/routes/_index/libs/translationUtils.ts deleted file mode 100644 index cc4d483..0000000 --- a/web/app/routes/_index/libs/translationUtils.ts +++ /dev/null @@ -1,175 +0,0 @@ -import { getOrCreatePageVersionTranslationInfo } from "../../../libs/pageVersionTranslationInfo"; -import { getOrCreateSourceTextId } from "../../../libs/sourceTextService"; -import { getOrCreateAIUser } from "../../../libs/userService"; -import { prisma } from "../../../utils/prisma"; -import { AI_MODEL, MAX_CHUNK_SIZE } from "../../translate/constants"; -import type { NumberedElement } from "../../translate/types"; -import { getGeminiModelResponse } from "../utils/gemini"; - -export function splitNumberedElements( - elements: NumberedElement[], -): NumberedElement[][] { - const chunks: NumberedElement[][] = []; - let currentChunk: NumberedElement[] = []; - let currentSize = 0; - - for (const element of elements) { - if ( - currentSize + element.text.length > MAX_CHUNK_SIZE && - currentChunk.length > 0 - ) { - chunks.push(currentChunk); - currentChunk = []; - currentSize = 0; - } - currentChunk.push(element); - currentSize += element.text.length; - } - - if (currentChunk.length > 0) { - chunks.push(currentChunk); - } - return chunks; -} - -export function extractTranslations(jsonString: string): NumberedElement[] { - try { - const parsedData = JSON.parse(jsonString); - - if (Array.isArray(parsedData)) { - return parsedData.map((item) => ({ - number: Number(item.number), - text: String(item.text), - })); - } - console.error("Parsed data is not an array"); - return []; - } catch (error) { - console.error("Failed to parse JSON:", error); - return []; - } -} - -export async function getOrCreateTranslations( - geminiApiKey: string, - elements: NumberedElement[], - targetLanguage: string, - pageId: number, - pageVersionId: number, - title: string, -): Promise { - const translations: NumberedElement[] = []; - const untranslatedElements: NumberedElement[] = []; - const sourceTextsId = await Promise.all( - elements.map((element) => - getOrCreateSourceTextId( - element.text, - element.number, - pageId, - pageVersionId, - ), - ), - ); - - const existingTranslations = await prisma.translateText.findMany({ - where: { - sourceTextId: { in: sourceTextsId }, - targetLanguage, - }, - orderBy: [{ point: "desc" }, { createdAt: "desc" }], - }); - - const translationMap = new Map( - existingTranslations.map((t) => [t.sourceTextId, t]), - ); - - elements.forEach((element, index) => { - const sourceTextId = sourceTextsId[index]; - const existingTranslation = translationMap.get(sourceTextId); - - if (existingTranslation) { - translations.push({ - number: element.number, - text: existingTranslation.text, - }); - } else { - untranslatedElements.push(element); - } - }); - - if (untranslatedElements.length > 0) { - const newTranslations = await translateUntranslatedElements( - geminiApiKey, - untranslatedElements, - targetLanguage, - pageId, - pageVersionId, - title, - ); - translations.push(...newTranslations); - } - - return translations.sort((a, b) => a.number - b.number); -} - -async function translateUntranslatedElements( - geminiApiKey: string, - untranslatedElements: NumberedElement[], - targetLanguage: string, - pageId: number, - pageVersionId: number, - title: string, -): Promise { - const source_text = untranslatedElements - .map((el) => JSON.stringify(el)) - .join("\n"); - const translatedText = await getGeminiModelResponse( - geminiApiKey, - AI_MODEL, - title, - source_text, - targetLanguage, - ); - - const extractedTranslations = extractTranslations(translatedText); - await getOrCreatePageVersionTranslationInfo( - pageVersionId, - targetLanguage, - extractedTranslations[0].text, - ); - - const systemUserId = await getOrCreateAIUser(AI_MODEL); - - await Promise.all( - extractedTranslations.map(async (translation) => { - const sourceText = untranslatedElements.find( - (el) => el.number === translation.number, - )?.text; - - if (!sourceText) { - console.error( - `Source text not found for translation number ${translation.number}`, - ); - return; - } - - const sourceTextId = await getOrCreateSourceTextId( - sourceText, - translation.number, - pageId, - pageVersionId, - ); - await prisma.translateText.create({ - data: { - targetLanguage, - text: translation.text, - sourceTextId, - pageId, - userId: systemUserId, - }, - }); - }), - ); - - return extractedTranslations; -} diff --git a/web/app/routes/_index/libs/userTranslationqueueService.ts b/web/app/routes/_index/libs/userTranslationqueueService.ts deleted file mode 100644 index 541ddf7..0000000 --- a/web/app/routes/_index/libs/userTranslationqueueService.ts +++ /dev/null @@ -1,20 +0,0 @@ -import Queue, { type Queue as QueueType } from "bull"; -import { REDIS_URL } from "../../translate/constants"; -import { processTranslationJob } from "./translation"; - -const createUserTranslationQueue = (userId: number) => - new Queue(`translation-user-${userId}`, REDIS_URL); - -const userTranslationQueues: { [userId: number]: QueueType } = {}; - -export function setupUserQueue(userId: number, geminiApiKey: string) { - if (userTranslationQueues[userId]) { - return userTranslationQueues[userId]; - } - const userTranslationQueue = createUserTranslationQueue(userId); - userTranslationQueue.process(async (job) => { - await processTranslationJob(job, geminiApiKey, userId); - }); - userTranslationQueues[userId] = userTranslationQueue; - return userTranslationQueue; -} diff --git a/web/app/routes/_index/route.tsx b/web/app/routes/_index/route.tsx index d24b9bd..bcd90e7 100644 --- a/web/app/routes/_index/route.tsx +++ b/web/app/routes/_index/route.tsx @@ -5,30 +5,13 @@ import type { MetaFunction, } from "@remix-run/node"; import { redirect } from "@remix-run/node"; -import { typedjson, useTypedLoaderData } from "remix-typedjson"; -import { z } from "zod"; -import { Header } from "~/components/Header"; +import { json } from "@remix-run/node"; +import { useLoaderData } from "@remix-run/react"; import { authenticator } from "~/utils/auth.server"; -import { getSession } from "~/utils/session.server"; -import { extractNumberedElements } from "../../libs/extractNumberedElements"; -import { prisma } from "../../utils/prisma"; -import { geminiApiKeySchema } from "../translate/types"; -import { - type UserAITranslationInfoItem, - UserAITranslationInfoSchema, -} from "../translate/types"; +import { validateGeminiApiKey } from "~/utils/gemini"; +import { prisma } from "~/utils/prisma"; import { GoogleSignInAndGeminiApiKeyForm } from "./components/GoogleSignInAndGeminiApiKeyForm"; -import { - URLTranslationForm, - urlTranslationSchema, -} from "./components/URLTranslationForm"; -import { translate } from "./libs/translation"; -import { addNumbersToContent } from "./utils/addNumbersToContent"; -import { extractArticle } from "./utils/articleUtils"; -import { fetchWithRetry } from "./utils/fetchWithRetry"; -import { validateGeminiApiKey } from "./utils/gemini"; - -import { UserAITranslationStatus } from "./components/UserAITranslationStatus"; +import { geminiApiKeySchema } from "./types"; export const meta: MetaFunction = () => { return [ @@ -43,163 +26,67 @@ export const meta: MetaFunction = () => { export async function loader({ request }: LoaderFunctionArgs) { const safeUser = await authenticator.isAuthenticated(request); - const session = await getSession(request.headers.get("Cookie")); - const targetLanguage = session.get("targetLanguage") || "ja"; - - let hasGeminiApiKey = false; - let userAITranslationInfo: UserAITranslationInfoItem[] = []; - if (safeUser) { const dbUser = await prisma.user.findUnique({ where: { id: safeUser.id } }); - hasGeminiApiKey = !!dbUser?.geminiApiKey; - - const rawTranslationInfo = await prisma.userAITranslationInfo.findMany({ - where: { - userId: safeUser.id, - targetLanguage, - }, - include: { - pageVersion: { - select: { - title: true, - page: { - select: { - url: true, - }, - }, - pageVersionTranslationInfo: { - where: { - targetLanguage, - }, - }, - }, - }, - }, - orderBy: { - lastTranslatedAt: "desc", - }, - take: 10, - }); - - // Validate and transform data - userAITranslationInfo = z - .array(UserAITranslationInfoSchema) - .parse(rawTranslationInfo); + if (dbUser?.geminiApiKey) { + return redirect("/translate"); + } } - - return typedjson({ - safeUser, - targetLanguage, - hasGeminiApiKey, - userAITranslationInfo, - }); + return json({ safeUser, hasGeminiApiKey: false }); } export async function action({ request }: ActionFunctionArgs) { - const formData = await request.clone().formData(); + const clone = request.clone(); + const formData = await clone.formData(); switch (formData.get("intent")) { case "SignInWithGoogle": return authenticator.authenticate("google", request, { successRedirect: "/", - failureRedirect: "/auth/login", + failureRedirect: "/", }); case "saveGeminiApiKey": { - const safeUser = await authenticator.isAuthenticated(request); - if (!safeUser) { - return redirect("/auth/login"); - } + const user = await authenticator.isAuthenticated(request, { + failureRedirect: "/", + }); const submission = parseWithZod(formData, { schema: geminiApiKeySchema }); if (submission.status !== "success") { return submission.reply(); } - const geminiApiKey = formData.get("geminiApiKey") as string; - const isValid = await validateGeminiApiKey(geminiApiKey); + const isValid = await validateGeminiApiKey(submission.value.geminiApiKey); if (!isValid) { return submission.reply({ formErrors: ["Gemini API key validation failed"], }); } await prisma.user.update({ - where: { id: safeUser.id }, + where: { id: user.id }, data: { geminiApiKey: submission.value.geminiApiKey }, }); - return submission.reply(); + return redirect("/translate"); } - case "translateUrl": { - const safeUser = await authenticator.isAuthenticated(request); - if (!safeUser) { - return redirect("/auth/login"); - } - const submission = parseWithZod(formData, { - schema: urlTranslationSchema, - }); - if (submission.status !== "success") { - return submission.reply(); - } - const dbUser = await prisma.user.findUnique({ - where: { id: safeUser.id }, - }); - const geminiApiKey = dbUser?.geminiApiKey; - if (!geminiApiKey) { - return submission.reply({ - formErrors: ["Gemini API key is not set"], - }); - } - const html = await fetchWithRetry(submission.value.url); - const { content, title } = extractArticle(html); - const numberedContent = addNumbersToContent(content); - const extractedNumberedElements = - extractNumberedElements(numberedContent); - const session = await getSession(request.headers.get("Cookie")); - const targetLanguage = session.get("targetLanguage") || "ja"; - await translate( - geminiApiKey, - safeUser.id, - targetLanguage, - title, - numberedContent, - extractedNumberedElements, - submission.value.url, - ); - return submission.reply(); - } + default: + return null; } } + export default function Index() { - const { safeUser, targetLanguage, hasGeminiApiKey, userAITranslationInfo } = - useTypedLoaderData(); + const { safeUser } = useLoaderData(); return (
-
-
-
- -
- {(!safeUser || !hasGeminiApiKey) && ( -
-
- -
-
- )} - {safeUser && hasGeminiApiKey && ( -
- -
- )} -
+

+ Everyone Translate Everything +

+

+ EveEveは、インターネット上のテキストを翻訳できるオープンソースプロジェクトです。 +

+
); diff --git a/web/app/routes/_index/types.ts b/web/app/routes/_index/types.ts new file mode 100644 index 0000000..f0996a2 --- /dev/null +++ b/web/app/routes/_index/types.ts @@ -0,0 +1,5 @@ +import { z } from "zod"; + +export const geminiApiKeySchema = z.object({ + geminiApiKey: z.string().min(1, "API key is required"), +}); diff --git a/web/app/routes/_index/utils/addNumbersToContent.ts b/web/app/routes/_index/utils/addNumbersToContent.ts deleted file mode 100644 index af69ea2..0000000 --- a/web/app/routes/_index/utils/addNumbersToContent.ts +++ /dev/null @@ -1,77 +0,0 @@ -import DOMPurify from "isomorphic-dompurify"; -import { JSDOM } from "jsdom"; -const { Node } = new JSDOM().window; - -export function shouldProcessElement(element: Element): boolean { - const htmlElement = element as HTMLElement; - - const isTranslatable = htmlElement.getAttribute("translate") !== "no"; - const isNotExcludedClass = !htmlElement.classList.contains("notranslate"); - const isNotEditable = !htmlElement.isContentEditable; - const isNotAlreadyTranslated = - htmlElement.getAttribute("data-translated") !== "true"; - const isNotTooltip = !htmlElement.classList.contains( - "eveeve-source-text-tooltip", - ); - const isNotExcludedTag = ![ - "script", - "style", - "textarea", - "input", - "pre", - "noscript", - "iframe", - "source", - ].includes(element.nodeName.toLowerCase()); - const isVisible = htmlElement.style.display !== "none"; - const isVisiblyHidden = htmlElement.style.visibility !== "hidden"; - - return ( - isTranslatable && - isNotExcludedClass && - isNotEditable && - isNotAlreadyTranslated && - isNotTooltip && - isNotExcludedTag && - isVisible && - isVisiblyHidden - ); -} - -export function addNumbersToContent(content: string): string { - const sanitizedContent = DOMPurify.sanitize(content); - const doc = new JSDOM(sanitizedContent); - let currentNumber = 1; - - function processNode(node: Node) { - if ( - node.nodeType === Node.ELEMENT_NODE && - shouldProcessElement(node as Element) - ) { - const element = node as Element; - const hasDirectTextContent = Array.from(element.childNodes).some( - (childNode) => - childNode.nodeType === Node.TEXT_NODE && - childNode.textContent?.trim() !== "", - ); - - if (hasDirectTextContent) { - element.setAttribute("data-number", currentNumber.toString()); - currentNumber++; - return; - } - - element.childNodes.forEach(processNode); - } - } - - Array.from(doc.window.document.body.childNodes).forEach(processNode); - - return Array.from(doc.window.document.body.childNodes) - .map((node) => - node.nodeType === Node.ELEMENT_NODE - ? (node as Element).outerHTML - : node.textContent, - ) - .join(""); -} diff --git a/web/app/routes/_index/utils/articleUtils.ts b/web/app/routes/_index/utils/articleUtils.ts deleted file mode 100644 index f64ea86..0000000 --- a/web/app/routes/_index/utils/articleUtils.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { Readability } from "@mozilla/readability"; -import { JSDOM } from "jsdom"; - -export const extractArticle = ( - html: string, -): { content: string; title: string } => { - const dom = new JSDOM(html); - const document = dom.window.document; - const reader = new Readability(document); - const article = reader.parse(); - - if (!article) { - throw new Error("記事の抽出に失敗しました"); - } - - return { content: article.content, title: article.title }; -}; diff --git a/web/app/routes/_index/utils/fetchWithRetry.ts b/web/app/routes/_index/utils/fetchWithRetry.ts deleted file mode 100644 index 43e4528..0000000 --- a/web/app/routes/_index/utils/fetchWithRetry.ts +++ /dev/null @@ -1,32 +0,0 @@ -export async function fetchWithRetry( - url: string, - maxRetries = 3, -): Promise { - let lastError: Error | null = null; - - for (let retryCount = 0; retryCount < maxRetries; retryCount++) { - try { - const response = await fetch(url, {}); - - if (!response.ok) { - throw new Error(`HTTP error! status: ${response.status}`); - } - - return await response.text(); - } catch (error: unknown) { - const typedError = error as Error; - console.error(`Attempt ${retryCount + 1} failed: ${typedError}`); - lastError = typedError; - - if (retryCount < maxRetries - 1) { - await new Promise((resolve) => - setTimeout(resolve, 1000 * (retryCount + 1)), - ); - } - } - } - - throw new Error( - `Failed to fetch ${url} after ${maxRetries} attempts. Last error: ${lastError?.message}`, - ); -} diff --git a/web/app/routes/translate/components/GoogleSignInAndGeminiApiKeyForm.tsx b/web/app/routes/translate/components/GoogleSignInAndGeminiApiKeyForm.tsx index 7ab0d51..4fa46ca 100644 --- a/web/app/routes/translate/components/GoogleSignInAndGeminiApiKeyForm.tsx +++ b/web/app/routes/translate/components/GoogleSignInAndGeminiApiKeyForm.tsx @@ -19,7 +19,6 @@ interface GoogleSignInAndGeminiApiKeyFormProps { export function GoogleSignInAndGeminiApiKeyForm({ isLoggedIn, hasGeminiApiKey, - error, }: GoogleSignInAndGeminiApiKeyFormProps) { const lastResult = useActionData(); const [form, { geminiApiKey }] = useForm({ diff --git a/web/app/routes/translate/components/translatedList.tsx b/web/app/routes/translate/components/translatedList.tsx deleted file mode 100644 index 212ce64..0000000 --- a/web/app/routes/translate/components/translatedList.tsx +++ /dev/null @@ -1,22 +0,0 @@ -import type { PageVersion } from "@prisma/client"; -import { useLoaderData } from "@remix-run/react"; -interface LoaderData { - pageVersionList: PageVersion[]; -} - -export function TranslatedList(language: string) { - const { pageVersionList } = useLoaderData(); - return ( -
-

Translated List (Language: {language})

-
    - {pageVersionList.map((pageVersion) => ( -
  • - {/* ページのタイトルや他の情報を表示 */} - {pageVersion.title} - {pageVersion.url} -
    • - ))} -
-
- ); -} diff --git a/web/app/routes/translate/libs/translationUtils.ts b/web/app/routes/translate/libs/translationUtils.ts index c83e420..4b0d636 100644 --- a/web/app/routes/translate/libs/translationUtils.ts +++ b/web/app/routes/translate/libs/translationUtils.ts @@ -1,10 +1,10 @@ import { getOrCreatePageVersionTranslationInfo } from "../../../libs/pageVersionTranslationInfo"; import { getOrCreateSourceTextId } from "../../../libs/sourceTextService"; import { getOrCreateAIUser } from "../../../libs/userService"; +import { getGeminiModelResponse } from "../../../utils/gemini"; import { prisma } from "../../../utils/prisma"; import { AI_MODEL, MAX_CHUNK_SIZE } from "../constants"; import type { NumberedElement } from "../types"; -import { getGeminiModelResponse } from "../utils/gemini"; export function splitNumberedElements( elements: NumberedElement[], diff --git a/web/app/routes/translate/route.tsx b/web/app/routes/translate/route.tsx index 5886fe5..e4c6881 100644 --- a/web/app/routes/translate/route.tsx +++ b/web/app/routes/translate/route.tsx @@ -1,126 +1,146 @@ import { parseWithZod } from "@conform-to/zod"; import type { ActionFunctionArgs, LoaderFunctionArgs } from "@remix-run/node"; +import { redirect } from "@remix-run/node"; +import { useFetcher } from "@remix-run/react"; +import { useEffect, useState } from "react"; import { typedjson, useTypedLoaderData } from "remix-typedjson"; +import { z } from "zod"; +import { Header } from "~/components/Header"; import { authenticator } from "~/utils/auth.server"; import { prisma } from "~/utils/prisma"; import { getSession } from "~/utils/session.server"; import { extractNumberedElements } from "./../../libs/extractNumberedElements"; import { - URLTranslationForm, - urlTranslationSchema, + URLTranslationForm, + urlTranslationSchema, } from "./components/URLTranslationForm"; import { UserAITranslationStatus } from "./components/UserAITranslationStatus"; import { translate } from "./libs/translation"; import { - type UserAITranslationInfoItem, - UserAITranslationInfoSchema, + type UserAITranslationInfoItem, + UserAITranslationInfoSchema, } from "./types"; import { addNumbersToContent } from "./utils/addNumbersToContent"; import { extractArticle } from "./utils/extractArticle"; import { fetchWithRetry } from "./utils/fetchWithRetry"; -import { z } from "zod"; -import { redirect } from "@remix-run/node"; +import { useRevalidator } from "@remix-run/react"; export async function loader({ request }: LoaderFunctionArgs) { - const safeUser = await authenticator.isAuthenticated(request, { - failureRedirect: "/", - }); + const safeUser = await authenticator.isAuthenticated(request, { + failureRedirect: "/", + }); - const dbUser = await prisma.user.findUnique({ where: { id: safeUser.id } }); - if (!dbUser?.geminiApiKey) { - return redirect("/settings"); - } + const dbUser = await prisma.user.findUnique({ where: { id: safeUser.id } }); + if (!dbUser?.geminiApiKey) { + redirect("/"); + } - const session = await getSession(request.headers.get("Cookie")); - const targetLanguage = session.get("targetLanguage") || "ja"; - let userAITranslationInfo: UserAITranslationInfoItem[] = []; - const rawTranslationInfo = await prisma.userAITranslationInfo.findMany({ - where: { - userId: safeUser.id, - targetLanguage, - }, - include: { - pageVersion: { - select: { - title: true, - page: { - select: { - url: true, - }, - }, - pageVersionTranslationInfo: { - where: { - targetLanguage, - }, - }, - }, - }, - }, - orderBy: { - lastTranslatedAt: "desc", - }, - take: 10, - }); + const session = await getSession(request.headers.get("Cookie")); + const targetLanguage = session.get("targetLanguage") || "ja"; + let userAITranslationInfo: UserAITranslationInfoItem[] = []; + const rawTranslationInfo = await prisma.userAITranslationInfo.findMany({ + where: { + userId: safeUser.id, + targetLanguage, + }, + include: { + pageVersion: { + select: { + title: true, + page: { + select: { + url: true, + }, + }, + pageVersionTranslationInfo: { + where: { + targetLanguage, + }, + }, + }, + }, + }, + orderBy: { + lastTranslatedAt: "desc", + }, + take: 10, + }); - // Validate and transform data - userAITranslationInfo = z - .array(UserAITranslationInfoSchema) - .parse(rawTranslationInfo); + // Validate and transform data + userAITranslationInfo = z + .array(UserAITranslationInfoSchema) + .parse(rawTranslationInfo); - return typedjson({ - safeUser, - targetLanguage, - userAITranslationInfo, - }); + return typedjson({ + safeUser, + targetLanguage, + userAITranslationInfo, + }); } + export async function action({ request }: ActionFunctionArgs) { - const user = await authenticator.isAuthenticated(request, { - failureRedirect: "/", - }); - const formData = await request.formData(); - const submission = parseWithZod(formData, { schema: urlTranslationSchema }); + const safeUser = await authenticator.isAuthenticated(request, { + failureRedirect: "/", + }); + const formData = await request.formData(); + const submission = parseWithZod(formData, { schema: urlTranslationSchema }); - if (submission.status !== "success") { - return submission.reply(); - } + if (submission.status !== "success") { + return submission.reply(); + } - const dbUser = await prisma.user.findUnique({ where: { id: user.id } }); - if (!dbUser?.geminiApiKey) { - return submission.reply({ formErrors: ["Gemini API key is not set"] }); - } + const dbUser = await prisma.user.findUnique({ where: { id: safeUser.id } }); + if (!dbUser?.geminiApiKey) { + return submission.reply({ formErrors: ["Gemini API key is not set"] }); + } - const html = await fetchWithRetry(submission.value.url); - const { content, title } = extractArticle(html); - const numberedContent = addNumbersToContent(content); - const extractedNumberedElements = extractNumberedElements(numberedContent); - const session = await getSession(request.headers.get("Cookie")); - const targetLanguage = session.get("targetLanguage") || "ja"; + const html = await fetchWithRetry(submission.value.url); + const { content, title } = extractArticle(html); + const numberedContent = addNumbersToContent(content); + const extractedNumberedElements = extractNumberedElements(numberedContent); + console.log("extractedNumberedElements", extractedNumberedElements); + const session = await getSession(request.headers.get("Cookie")); + const targetLanguage = session.get("targetLanguage") || "ja"; - await translate( - dbUser.geminiApiKey, - user.id, - targetLanguage, - title, - numberedContent, - extractedNumberedElements, - submission.value.url, - ); + await translate( + dbUser.geminiApiKey, + safeUser.id, + targetLanguage, + title, + numberedContent, + extractedNumberedElements, + submission.value.url, + ); - return submission.reply(); + return submission.reply(); } export default function TranslatePage() { - const { user, targetLanguage, userAITranslationInfo } = + const { safeUser, targetLanguage, userAITranslationInfo } = useTypedLoaderData(); + const revalidator = useRevalidator(); - return ( -
-

Translate

- - -
- ); + useEffect(() => { + const intervalId = setInterval(() => { + revalidator.revalidate(); + }, 5000); + + return () => clearInterval(intervalId); + }, [revalidator]); + return ( +
+
+
+
+ +
+
+ +
+
+
+ ); } diff --git a/web/app/routes/translate/utils/gemini.ts b/web/app/routes/translate/utils/gemini.ts deleted file mode 100644 index 4f01482..0000000 --- a/web/app/routes/translate/utils/gemini.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { - GoogleGenerativeAI, - HarmBlockThreshold, - HarmCategory, -} from "@google/generative-ai"; -import { generateSystemMessage } from "./generateGeminiMessage"; - -export async function getGeminiModelResponse( - geminiApiKey: string, - model: string, - title: string, - source_text: string, - target_language: string, -) { - const genAI = new GoogleGenerativeAI(geminiApiKey); - const safetySetting = [ - { - category: HarmCategory.HARM_CATEGORY_HARASSMENT, - threshold: HarmBlockThreshold.BLOCK_NONE, - }, - { - category: HarmCategory.HARM_CATEGORY_HATE_SPEECH, - threshold: HarmBlockThreshold.BLOCK_NONE, - }, - { - category: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT, - threshold: HarmBlockThreshold.BLOCK_NONE, - }, - { - category: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT, - threshold: HarmBlockThreshold.BLOCK_NONE, - }, - ]; - const modelConfig = genAI.getGenerativeModel({ - model: model, - safetySettings: safetySetting, - generationConfig: { - responseMimeType: "application/json", - }, - }); - const result = await modelConfig.generateContent( - generateSystemMessage(title, source_text, target_language), - ); - return result.response.text(); -} - -export async function validateGeminiApiKey(apiKey: string): Promise { - try { - const genAI = new GoogleGenerativeAI(apiKey); - const model = genAI.getGenerativeModel({ model: "gemini-pro" }); - - const result = await model.generateContent("Hello, World!"); - const response = await result.response; - const text = response.text(); - - return text.length > 0; - } catch (error) { - console.error("Gemini API key validation failed:", error); - return false; - } -} diff --git a/web/app/routes/translate/utils/generateGeminiMessage.ts b/web/app/routes/translate/utils/generateGeminiMessage.ts deleted file mode 100644 index c735790..0000000 --- a/web/app/routes/translate/utils/generateGeminiMessage.ts +++ /dev/null @@ -1,48 +0,0 @@ -export function generateSystemMessage( - title: string, - source_text: string, - target_language: string, -): string { - return ` - You are a skilled translator. Your task is to accurately translate the given text into beautiful and natural sentences in the target language. Please follow these guidelines: - - 1. Maintain consistency: Use the same words for recurring terms with the same meaning to avoid confusing the reader. - 2. Preserve style: Keep a consistent writing style throughout to ensure the translation reads naturally as a single work. - 3. Context awareness: The provided sequence is a passage from one work. The number indicates the sentence's position within that work. - 4. Reader-friendly: While considering the document title, translate in a way that is easy for readers to understand and enjoy. - - Document title: ${title} - - Translate the following array of English texts into ${target_language}. - - Important instructions: - - Do not explain your process or self-reference. - - Present the translated result as a JavaScript array of objects. - - Each object should have 'number' and 'text' properties. - - Include the translated title as the first item with 'number: 0'. - - For the content, the 'number' should correspond to the original text's position. - - The 'text' should contain the translation. - - Always enclose the 'text' value in double quotes. - - Maintain the original array structure and order, with the title translation added as the first item. - - Output ONLY the translated array. No additional text or explanations. - - Input array: - ${source_text} - - Translate to ${target_language} and output in the following format: - [ - { - "number": 0, - "text": "Translated title" - }, - { - "number": 1, - "text": "Translated text for item 1" - }, - { - "number": 2, - "text": "Translated text for item 2" - }, - ... - ]`; -} diff --git a/web/app/routes/types.ts b/web/app/routes/types.ts new file mode 100644 index 0000000..76225c7 --- /dev/null +++ b/web/app/routes/types.ts @@ -0,0 +1,43 @@ +import { z } from "zod"; + +export const geminiApiKeySchema = z.object({ + geminiApiKey: z.string().min(1, "API key is required"), +}); + +export const PageVersionTranslationInfoSchema = z.object({ + id: z.number(), + pageVersionId: z.number(), + targetLanguage: z.string(), + translationTitle: z.string(), +}); + +export const UserAITranslationInfoSchema = z.object({ + id: z.number(), + userId: z.number(), + pageVersionId: z.number(), + targetLanguage: z.string(), + aiTranslationStatus: z.string(), + aiTranslationProgress: z.number(), + lastTranslatedAt: z.string().or(z.date()), + pageVersion: z.object({ + title: z.string(), + page: z.object({ + url: z.string(), + }), + pageVersionTranslationInfo: z + .array(PageVersionTranslationInfoSchema) + .optional(), + }), +}); + +export type UserAITranslationInfoItem = z.infer< + typeof UserAITranslationInfoSchema +>; +export type PageVersionTranslationInfoItem = z.infer< + typeof PageVersionTranslationInfoSchema +>; + +export type NumberedElement = { + number: number; + text: string; +}; diff --git a/web/app/routes/_index/utils/gemini.ts b/web/app/utils/gemini.ts similarity index 62% rename from web/app/routes/_index/utils/gemini.ts rename to web/app/utils/gemini.ts index 4f01482..7f0bc5f 100644 --- a/web/app/routes/_index/utils/gemini.ts +++ b/web/app/utils/gemini.ts @@ -4,6 +4,8 @@ import { HarmCategory, } from "@google/generative-ai"; import { generateSystemMessage } from "./generateGeminiMessage"; +const MAX_RETRIES = 3; +const RETRY_DELAY = 5000; export async function getGeminiModelResponse( geminiApiKey: string, @@ -38,12 +40,33 @@ export async function getGeminiModelResponse( responseMimeType: "application/json", }, }); - const result = await modelConfig.generateContent( - generateSystemMessage(title, source_text, target_language), - ); - return result.response.text(); -} + let lastError: Error | null = null; + + for (let retryCount = 0; retryCount < MAX_RETRIES; retryCount++) { + try { + const result = await modelConfig.generateContent( + generateSystemMessage(title, source_text, target_language), + ); + console.log("result", result.response.text()); + return result.response.text(); + } catch (error: unknown) { + const typedError = error as Error; + console.error( + `Translation attempt ${retryCount + 1} failed:`, + typedError, + ); + lastError = typedError; + if (retryCount < MAX_RETRIES - 1) { + const delay = 1000 * (retryCount + 1); + console.log(`Retrying in ${delay / 1000} seconds...`); + await new Promise((resolve) => setTimeout(resolve, delay)); + } + } + } + console.error("Max retries reached. Translation failed."); + throw lastError || new Error("Translation failed after max retries"); +} export async function validateGeminiApiKey(apiKey: string): Promise { try { const genAI = new GoogleGenerativeAI(apiKey); diff --git a/web/app/routes/_index/utils/generateGeminiMessage.ts b/web/app/utils/generateGeminiMessage.ts similarity index 100% rename from web/app/routes/_index/utils/generateGeminiMessage.ts rename to web/app/utils/generateGeminiMessage.ts diff --git a/web/biome.json b/web/biome.json index 6286816..a8079cd 100644 --- a/web/biome.json +++ b/web/biome.json @@ -1,37 +1,37 @@ { - "$schema": "https://biomejs.dev/schemas/1.8.3/schema.json", - "organizeImports": { - "enabled": true - }, - "files": { - "ignore": ["bun.lockb", "build", ".vscode"] - }, - "linter": { - "enabled": true, - "rules": { - "recommended": true, - "correctness": { - "noUnusedImports": "error" - } - } - }, - "formatter": { - "enabled": true - }, - "javascript": { - "parser": { - "unsafeParameterDecoratorsEnabled": true - }, - "formatter": { - "enabled": true - } - }, - "json": { - "parser": { "allowComments": true }, - "formatter": { - "enabled": true, - "indentWidth": 2, - "lineWidth": 80 - } - } + "$schema": "https://biomejs.dev/schemas/1.8.3/schema.json", + "organizeImports": { + "enabled": true + }, + "files": { + "ignore": ["bun.lockb", "build", ".vscode"] + }, + "linter": { + "enabled": true, + "rules": { + "recommended": true, + "correctness": { + "noUnusedImports": "error" + } + } + }, + "formatter": { + "enabled": true + }, + "javascript": { + "parser": { + "unsafeParameterDecoratorsEnabled": true + }, + "formatter": { + "enabled": true + } + }, + "json": { + "parser": { "allowComments": true }, + "formatter": { + "enabled": true, + "indentWidth": 2, + "lineWidth": 80 + } + } } diff --git a/web/components.json b/web/components.json index 516e414..cf75b27 100644 --- a/web/components.json +++ b/web/components.json @@ -1,17 +1,17 @@ { - "$schema": "https://ui.shadcn.com/schema.json", - "style": "new-york", - "rsc": false, - "tsx": true, - "tailwind": { - "config": " tailwind.config.ts", - "css": "app/tailwind.css", - "baseColor": "slate", - "cssVariables": true, - "prefix": "" - }, - "aliases": { - "components": "~/components", - "utils": "~/utils/cn" - } + "$schema": "https://ui.shadcn.com/schema.json", + "style": "new-york", + "rsc": false, + "tsx": true, + "tailwind": { + "config": " tailwind.config.ts", + "css": "app/tailwind.css", + "baseColor": "slate", + "cssVariables": true, + "prefix": "" + }, + "aliases": { + "components": "~/components", + "utils": "~/utils/cn" + } } diff --git a/web/package.json b/web/package.json index 2719f73..d18c31c 100644 --- a/web/package.json +++ b/web/package.json @@ -1,76 +1,77 @@ { - "private": true, - "sideEffects": false, - "type": "module", - "scripts": { - "build": "remix vite:build", - "install-and-build": "bun install && bun run build", - "deploy": "bun run build && wrangler pages deploy", - "dev": "remix vite:dev", - "start": "remix-serve ./build/server/index.js", - "typecheck": "tsc", - "check": "bunx @biomejs/biome check --write ." - }, - "dependencies": { - "@conform-to/react": "^1.1.5", - "@conform-to/zod": "^1.1.5", - "@google/generative-ai": "^0.15.0", - "@mozilla/readability": "^0.5.0", - "@prisma/client": "5.17.0", - "@radix-ui/react-dropdown-menu": "^2.1.1", - "@radix-ui/react-icons": "^1.3.0", - "@radix-ui/react-label": "^2.1.0", - "@radix-ui/react-progress": "^1.1.0", - "@radix-ui/react-scroll-area": "^1.1.0", - "@radix-ui/react-select": "^2.1.1", - "@radix-ui/react-separator": "^1.1.0", - "@radix-ui/react-slot": "^1.1.0", - "@remix-run/react": "^2.10.3", - "@remix-run/serve": "^2.10.3", - "@supabase/supabase-js": "^2.44.4", - "@tailwindcss/typography": "^0.5.13", - "@types/bull": "^4.10.0", - "bcryptjs": "^2.4.3", - "bull": "^4.15.1", - "class-variance-authority": "^0.7.0", - "clsx": "^2.1.1", - "html-react-parser": "^5.1.10", - "htmlparser2": "^9.1.0", - "isbot": "^5.1.13", - "isomorphic-dompurify": "^2.13.0", - "lucide-react": "^0.408.0", - "next-themes": "^0.3.0", - "react": "^18.3.1", - "react-dom": "^18.3.1", - "react-icons": "^5.2.1", - "remix-auth": "^3.7.0", - "remix-auth-form": "^1.5.0", - "remix-auth-google": "^2.0.0", - "remix-typedjson": "^0.4.1", - "tailwind-merge": "^2.4.0", - "tailwindcss-animate": "^1.0.7" - }, - "devDependencies": { - "@biomejs/biome": "^1.8.3", - "@remix-run/dev": "^2.10.3", - "@remix-run/testing": "^2.10.3", - "@testing-library/jest-dom": "^6.4.6", - "@testing-library/user-event": "^14.5.2", - "@types/bcryptjs": "^2.4.6", - "@types/react": "^18.3.3", - "@types/react-dom": "^18.3.0", - "autoprefixer": "^10.4.19", - "postcss": "^8.4.39", - "prisma": "^5.17.0", - "tailwindcss": "^3.4.6", - "tsx": "^4.16.2", - "typescript": "^5.5.3", - "vite": "^5.3.4", - "vite-tsconfig-paths": "^4.3.2", - "vitest": "^2.0.3", - "wrangler": "3.65.0" - }, - "engines": { - "node": ">=20.0.0" - } + "private": true, + "sideEffects": false, + "type": "module", + "scripts": { + "build": "remix vite:build", + "install-and-build": "bun install && bun run build", + "deploy": "bun run build && wrangler pages deploy", + "dev": "remix vite:dev", + "start": "remix-serve ./build/server/index.js", + "typecheck": "tsc", + "check": "bunx @biomejs/biome check --write ." + }, + "dependencies": { + "@conform-to/react": "^1.1.5", + "@conform-to/zod": "^1.1.5", + "@google/generative-ai": "^0.15.0", + "@mozilla/readability": "^0.5.0", + "@prisma/client": "5.17.0", + "@radix-ui/react-dropdown-menu": "^2.1.1", + "@radix-ui/react-icons": "^1.3.0", + "@radix-ui/react-label": "^2.1.0", + "@radix-ui/react-progress": "^1.1.0", + "@radix-ui/react-scroll-area": "^1.1.0", + "@radix-ui/react-select": "^2.1.1", + "@radix-ui/react-separator": "^1.1.0", + "@radix-ui/react-slot": "^1.1.0", + "@radix-ui/react-tooltip": "^1.1.2", + "@remix-run/react": "^2.10.3", + "@remix-run/serve": "^2.10.3", + "@supabase/supabase-js": "^2.44.4", + "@tailwindcss/typography": "^0.5.13", + "@types/bull": "^4.10.0", + "bcryptjs": "^2.4.3", + "bull": "^4.15.1", + "class-variance-authority": "^0.7.0", + "clsx": "^2.1.1", + "html-react-parser": "^5.1.10", + "htmlparser2": "^9.1.0", + "isbot": "^5.1.13", + "isomorphic-dompurify": "^2.13.0", + "lucide-react": "^0.408.0", + "next-themes": "^0.3.0", + "react": "^18.3.1", + "react-dom": "^18.3.1", + "react-icons": "^5.2.1", + "remix-auth": "^3.7.0", + "remix-auth-form": "^1.5.0", + "remix-auth-google": "^2.0.0", + "remix-typedjson": "^0.4.1", + "tailwind-merge": "^2.4.0", + "tailwindcss-animate": "^1.0.7" + }, + "devDependencies": { + "@biomejs/biome": "^1.8.3", + "@remix-run/dev": "^2.10.3", + "@remix-run/testing": "^2.10.3", + "@testing-library/jest-dom": "^6.4.6", + "@testing-library/user-event": "^14.5.2", + "@types/bcryptjs": "^2.4.6", + "@types/react": "^18.3.3", + "@types/react-dom": "^18.3.0", + "autoprefixer": "^10.4.19", + "postcss": "^8.4.39", + "prisma": "^5.17.0", + "tailwindcss": "^3.4.6", + "tsx": "^4.16.2", + "typescript": "^5.5.3", + "vite": "^5.3.4", + "vite-tsconfig-paths": "^4.3.2", + "vitest": "^2.0.3", + "wrangler": "3.65.0" + }, + "engines": { + "node": ">=20.0.0" + } } diff --git a/web/public/_routes.json b/web/public/_routes.json index b042b3e..2307c58 100644 --- a/web/public/_routes.json +++ b/web/public/_routes.json @@ -1,5 +1,5 @@ { - "version": 1, - "include": ["/*"], - "exclude": ["/favicon.ico", "/assets/*"] + "version": 1, + "include": ["/*"], + "exclude": ["/favicon.ico", "/assets/*"] } diff --git a/web/tsconfig.json b/web/tsconfig.json index 9d87dd3..6f3de27 100644 --- a/web/tsconfig.json +++ b/web/tsconfig.json @@ -1,32 +1,32 @@ { - "include": [ - "**/*.ts", - "**/*.tsx", - "**/.server/**/*.ts", - "**/.server/**/*.tsx", - "**/.client/**/*.ts", - "**/.client/**/*.tsx" - ], - "compilerOptions": { - "lib": ["DOM", "DOM.Iterable", "ES2022"], - "types": ["@remix-run/node", "vite/client"], - "isolatedModules": true, - "esModuleInterop": true, - "jsx": "react-jsx", - "module": "ESNext", - "moduleResolution": "Bundler", - "resolveJsonModule": true, - "target": "ES2022", - "strict": true, - "allowJs": true, - "skipLibCheck": true, - "forceConsistentCasingInFileNames": true, - "baseUrl": ".", - "paths": { - "~/*": ["./app/*"] - }, + "include": [ + "**/*.ts", + "**/*.tsx", + "**/.server/**/*.ts", + "**/.server/**/*.tsx", + "**/.client/**/*.ts", + "**/.client/**/*.tsx" + ], + "compilerOptions": { + "lib": ["DOM", "DOM.Iterable", "ES2022"], + "types": ["@remix-run/node", "vite/client"], + "isolatedModules": true, + "esModuleInterop": true, + "jsx": "react-jsx", + "module": "ESNext", + "moduleResolution": "Bundler", + "resolveJsonModule": true, + "target": "ES2022", + "strict": true, + "allowJs": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "baseUrl": ".", + "paths": { + "~/*": ["./app/*"] + }, - // Vite takes care of building everything, not tsc. - "noEmit": true - } + // Vite takes care of building everything, not tsc. + "noEmit": true + } } From 14634eb8b990079f3e3703cdac1263ee3ba855af Mon Sep 17 00:00:00 2001 From: tomolld Date: Thu, 18 Jul 2024 16:53:47 +0900 Subject: [PATCH 3/4] =?UTF-8?q?bull=E3=81=A7=E3=82=AD=E3=83=A5=E3=83=BC?= =?UTF-8?q?=E3=81=AB=E3=81=97=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- web/app/libs/userAITranslationInfo.tsx | 5 +- .../components/URLTranslationForm.tsx | 8 +- .../components/UserAITranslationStatus.tsx | 110 ++++++++++++------ web/app/routes/translate/constants.ts | 2 +- web/app/routes/translate/libs/translation.ts | 10 ++ .../routes/translate/libs/translationUtils.ts | 39 +++++-- .../libs/userTranslationqueueService.ts | 28 ++++- web/app/routes/translate/route.tsx | 5 - web/app/routes/translate/types.ts | 7 ++ web/app/utils/gemini.ts | 19 ++- 10 files changed, 164 insertions(+), 69 deletions(-) diff --git a/web/app/libs/userAITranslationInfo.tsx b/web/app/libs/userAITranslationInfo.tsx index 43fd1b6..eb4e6a8 100644 --- a/web/app/libs/userAITranslationInfo.tsx +++ b/web/app/libs/userAITranslationInfo.tsx @@ -14,7 +14,10 @@ export async function getOrCreateUserAITranslationInfo( targetLanguage, }, }, - update: {}, + update: { + aiTranslationStatus: "pending", + aiTranslationProgress: 0, + }, create: { userId, pageVersionId, diff --git a/web/app/routes/translate/components/URLTranslationForm.tsx b/web/app/routes/translate/components/URLTranslationForm.tsx index 291678e..631fa46 100644 --- a/web/app/routes/translate/components/URLTranslationForm.tsx +++ b/web/app/routes/translate/components/URLTranslationForm.tsx @@ -6,13 +6,7 @@ import { z } from "zod"; import { LoadingSpinner } from "~/components/LoadingSpinner"; import { Button } from "~/components/ui/button"; import { Input } from "~/components/ui/input"; - -const urlTranslationSchema = z.object({ - url: z - .string() - .min(1, { message: "URLを入力してください" }) - .url("有効なURLを入力してください"), -}); +import { urlTranslationSchema } from "../types"; export function URLTranslationForm() { const navigation = useNavigation(); diff --git a/web/app/routes/translate/components/UserAITranslationStatus.tsx b/web/app/routes/translate/components/UserAITranslationStatus.tsx index 1be42d4..c6f56f2 100644 --- a/web/app/routes/translate/components/UserAITranslationStatus.tsx +++ b/web/app/routes/translate/components/UserAITranslationStatus.tsx @@ -4,6 +4,14 @@ import { Card, CardContent, CardHeader, CardTitle } from "~/components/ui/card"; import { Progress } from "~/components/ui/progress"; import { ScrollArea } from "~/components/ui/scroll-area"; import type { UserAITranslationInfoItem } from "../types"; +import { Button } from "~/components/ui/button"; +import { Form } from "@remix-run/react"; +import { useNavigation } from "@remix-run/react"; +import { LoadingSpinner } from "~/components/LoadingSpinner"; +import { RotateCcw } from 'lucide-react'; +import { urlTranslationSchema } from "../types"; +import { useForm } from "@conform-to/react"; +import { getZodConstraint, parseWithZod } from "@conform-to/zod"; type UserAITranslationStatusProps = { userAITranslationInfo: UserAITranslationInfoItem[]; @@ -14,6 +22,17 @@ export function UserAITranslationStatus({ userAITranslationInfo = [], targetLanguage, }: UserAITranslationStatusProps) { + + const navigation = useNavigation(); + const [form, fields] = useForm({ + id: "url-re-translation-form", + constraint: getZodConstraint(urlTranslationSchema), + shouldValidate: "onBlur", + shouldRevalidate: "onInput", + onValidate({ formData }) { + return parseWithZod(formData, { schema: urlTranslationSchema }); + }, + }); if (!userAITranslationInfo || userAITranslationInfo.length === 0) { return ( @@ -36,46 +55,61 @@ export function UserAITranslationStatus({
{userAITranslationInfo.map((item) => { - const translationInfo = - item.pageVersion.pageVersionTranslationInfo?.[0]; + const translationInfo = item.pageVersion.pageVersionTranslationInfo?.[0]; return ( - - - - - {item.pageVersion.title} - {translationInfo?.translationTitle && ( - - {translationInfo.translationTitle} - - )} - - - -

- {item.pageVersion.page.url} -

- + + + {item.pageVersion.title} + {translationInfo?.translationTitle && ( + + {translationInfo.translationTitle} + + )} + + + +

+ {item.pageVersion.page.url} +

+ + {item.aiTranslationStatus} + + +

+ {new Date(item.lastTranslatedAt).toLocaleString()} +

+
+ - {item.aiTranslationStatus} - - -

- Last updated:{" "} - {new Date(item.lastTranslatedAt).toLocaleString()} -

- - - + View + +
+ + +
+
+ + ); })}
diff --git a/web/app/routes/translate/constants.ts b/web/app/routes/translate/constants.ts index eab1693..e0d7c5a 100644 --- a/web/app/routes/translate/constants.ts +++ b/web/app/routes/translate/constants.ts @@ -1,3 +1,3 @@ export const REDIS_URL = process.env.REDIS_URL || "redis://localhost:6379"; -export const MAX_CHUNK_SIZE = 30000; +export const MAX_CHUNK_SIZE = 20000; export const AI_MODEL = "gemini-1.5-pro-latest"; diff --git a/web/app/routes/translate/libs/translation.ts b/web/app/routes/translate/libs/translation.ts index 18a4916..32c90d3 100644 --- a/web/app/routes/translate/libs/translation.ts +++ b/web/app/routes/translate/libs/translation.ts @@ -60,10 +60,20 @@ export async function processTranslationJob( ) { const { pageId, pageVersionId, targetLanguage, title, numberedElements } = job.data; + await updateUserAITranslationInfo( + userId, + pageVersionId, + targetLanguage, + "in_progress", + 0, + ); try { const chunks = splitNumberedElements(numberedElements); const totalChunks = chunks.length; for (let i = 0; i < chunks.length; i++) { + console.log(`Processing chunk ${i + 1} of ${totalChunks}`); + console.log("Chunk content:", JSON.stringify(chunks[i], null, 2)); + await getOrCreateTranslations( geminiApiKey, chunks[i], diff --git a/web/app/routes/translate/libs/translationUtils.ts b/web/app/routes/translate/libs/translationUtils.ts index 4b0d636..f77d722 100644 --- a/web/app/routes/translate/libs/translationUtils.ts +++ b/web/app/routes/translate/libs/translationUtils.ts @@ -32,22 +32,35 @@ export function splitNumberedElements( return chunks; } -export function extractTranslations(jsonString: string): NumberedElement[] { +export function extractTranslations( + text: string, +): { number: number; text: string }[] { try { - const parsedData = JSON.parse(jsonString); - - if (Array.isArray(parsedData)) { - return parsedData.map((item) => ({ - number: Number(item.number), - text: String(item.text), - })); + // まず、文字列をJSONとしてパースしてみる + const parsed = JSON.parse(text); + if (Array.isArray(parsed)) { + return parsed; } - console.error("Parsed data is not an array"); - return []; } catch (error) { - console.error("Failed to parse JSON:", error); - return []; + console.warn("Failed to parse JSON, falling back to regex parsing", error); + } + + const translations: { number: number; text: string }[] = []; + const regex = + /{\s*"number"\s*:\s*(\d+)\s*,\s*"text"\s*:\s*"((?:\\.|[^"\\])*)"\s*}/g; + let match: RegExpExecArray | null; + + while (true) { + match = regex.exec(text); + if (match === null) break; + + translations.push({ + number: Number.parseInt(match[1], 10), + text: match[2], + }); } + + return translations; } export async function getOrCreateTranslations( @@ -123,6 +136,7 @@ async function translateUntranslatedElements( const source_text = untranslatedElements .map((el) => JSON.stringify(el)) .join("\n"); + console.log("untranslatedText", source_text); const translatedText = await getGeminiModelResponse( geminiApiKey, AI_MODEL, @@ -132,6 +146,7 @@ async function translateUntranslatedElements( ); const extractedTranslations = extractTranslations(translatedText); + console.log("extractedNowTranslations", extractedTranslations); await getOrCreatePageVersionTranslationInfo( pageVersionId, targetLanguage, diff --git a/web/app/routes/translate/libs/userTranslationqueueService.ts b/web/app/routes/translate/libs/userTranslationqueueService.ts index 7f50e83..b7934c1 100644 --- a/web/app/routes/translate/libs/userTranslationqueueService.ts +++ b/web/app/routes/translate/libs/userTranslationqueueService.ts @@ -3,7 +3,12 @@ import { REDIS_URL } from "../constants"; import { processTranslationJob } from "./translation"; const createUserTranslationQueue = (userId: number) => - new Queue(`translation-user-${userId}`, REDIS_URL); + new Queue(`translation-user-${userId}`, REDIS_URL, { + defaultJobOptions: { + removeOnComplete: true, + removeOnFail: true + }, + }); const userTranslationQueues: { [userId: number]: QueueType } = {}; @@ -12,9 +17,26 @@ export function setupUserQueue(userId: number, geminiApiKey: string) { return userTranslationQueues[userId]; } const userTranslationQueue = createUserTranslationQueue(userId); - userTranslationQueue.process(async (job) => { - await processTranslationJob(job, geminiApiKey, userId); + userTranslationQueue.process(async (job) => { + console.log(`Starting job ${job.id} for user ${userId}`); + try { + await processTranslationJob(job, geminiApiKey, userId); + console.log(`Job ${job.id} completed successfully for user ${userId}`); + } catch (error) { + console.error(`Error processing job ${job.id} for user ${userId}:`, error); + throw error; + } }); + + userTranslationQueue.on('completed', async (job) => { + const activeCount = await userTranslationQueue.getActiveCount(); + const waitingCount = await userTranslationQueue.getWaitingCount(); + + if (activeCount === 0 && waitingCount === 0) { + console.log(`All translation jobs for user ${userId} have been completed.`); + } + }); + userTranslationQueues[userId] = userTranslationQueue; return userTranslationQueue; } diff --git a/web/app/routes/translate/route.tsx b/web/app/routes/translate/route.tsx index e4c6881..a22dca6 100644 --- a/web/app/routes/translate/route.tsx +++ b/web/app/routes/translate/route.tsx @@ -98,7 +98,6 @@ export async function action({ request }: ActionFunctionArgs) { const { content, title } = extractArticle(html); const numberedContent = addNumbersToContent(content); const extractedNumberedElements = extractNumberedElements(numberedContent); - console.log("extractedNumberedElements", extractedNumberedElements); const session = await getSession(request.headers.get("Cookie")); const targetLanguage = session.get("targetLanguage") || "ja"; @@ -135,10 +134,6 @@ export default function TranslatePage() {
-
diff --git a/web/app/routes/translate/types.ts b/web/app/routes/translate/types.ts index 76225c7..c07fe9c 100644 --- a/web/app/routes/translate/types.ts +++ b/web/app/routes/translate/types.ts @@ -4,6 +4,13 @@ export const geminiApiKeySchema = z.object({ geminiApiKey: z.string().min(1, "API key is required"), }); +export const urlTranslationSchema = z.object({ + url: z + .string() + .min(1, { message: "URLを入力してください" }) + .url("有効なURLを入力してください"), +}); + export const PageVersionTranslationInfoSchema = z.object({ id: z.number(), pageVersionId: z.number(), diff --git a/web/app/utils/gemini.ts b/web/app/utils/gemini.ts index 7f0bc5f..4ee09ee 100644 --- a/web/app/utils/gemini.ts +++ b/web/app/utils/gemini.ts @@ -2,10 +2,10 @@ import { GoogleGenerativeAI, HarmBlockThreshold, HarmCategory, + FunctionDeclarationSchemaType, } from "@google/generative-ai"; import { generateSystemMessage } from "./generateGeminiMessage"; const MAX_RETRIES = 3; -const RETRY_DELAY = 5000; export async function getGeminiModelResponse( geminiApiKey: string, @@ -38,6 +38,21 @@ export async function getGeminiModelResponse( safetySettings: safetySetting, generationConfig: { responseMimeType: "application/json", + responseSchema: { + type: FunctionDeclarationSchemaType.ARRAY, + items: { + type: FunctionDeclarationSchemaType.OBJECT, + properties: { + number: { + type: FunctionDeclarationSchemaType.INTEGER, + }, + text: { + type: FunctionDeclarationSchemaType.STRING, + }, + }, + required: ["number", "text"], + }, + }, }, }); let lastError: Error | null = null; @@ -59,7 +74,7 @@ export async function getGeminiModelResponse( if (retryCount < MAX_RETRIES - 1) { const delay = 1000 * (retryCount + 1); - console.log(`Retrying in ${delay / 1000} seconds...`); + console.log(`Retrying in ${delay / 100} seconds...`); await new Promise((resolve) => setTimeout(resolve, delay)); } } From f7a8b25910e2239d938650da56b1e730192a86a1 Mon Sep 17 00:00:00 2001 From: tomolld Date: Thu, 18 Jul 2024 17:49:23 +0900 Subject: [PATCH 4/4] f --- web/app/utils/generateGeminiMessage.ts | 41 +++++++++++++++++++------- web/package.json | 2 +- 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/web/app/utils/generateGeminiMessage.ts b/web/app/utils/generateGeminiMessage.ts index c735790..3ceb099 100644 --- a/web/app/utils/generateGeminiMessage.ts +++ b/web/app/utils/generateGeminiMessage.ts @@ -17,31 +17,50 @@ export function generateSystemMessage( Important instructions: - Do not explain your process or self-reference. - - Present the translated result as a JavaScript array of objects. - - Each object should have 'number' and 'text' properties. - - Include the translated title as the first item with 'number: 0'. - - For the content, the 'number' should correspond to the original text's position. - - The 'text' should contain the translation. - - Always enclose the 'text' value in double quotes. + - Present the translated result as a JSON array conforming to the following schema: + + { + "type": "array", + "items": { + "type": "object", + "properties": { + "number": { + "type": "integer", + "minimum": 0 + }, + "text": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": ["number", "text"] + } + } + + - Include the translated title as the first item with "number": 0. + - For the content, the "number" should correspond to the original text's position. + - The "text" should be an array containing the translation. If a paragraph consists of multiple sentences, include each sentence as a separate string in the array. - Maintain the original array structure and order, with the title translation added as the first item. - - Output ONLY the translated array. No additional text or explanations. + - Output ONLY the translated JSON array. No additional text or explanations. - Input array: + Input text: ${source_text} Translate to ${target_language} and output in the following format: [ { "number": 0, - "text": "Translated title" + "text": ["Translated title"] }, { "number": 1, - "text": "Translated text for item 1" + "text": ["Translated text for item 1, sentence 1", "Translated text for item 1, sentence 2"] }, { "number": 2, - "text": "Translated text for item 2" + "text": ["Translated text for item 2"] }, ... ]`; diff --git a/web/package.json b/web/package.json index d18c31c..54e00db 100644 --- a/web/package.json +++ b/web/package.json @@ -35,7 +35,7 @@ "bull": "^4.15.1", "class-variance-authority": "^0.7.0", "clsx": "^2.1.1", - "html-react-parser": "^5.1.10", + "html-react-parser": "^5.1.11", "htmlparser2": "^9.1.0", "isbot": "^5.1.13", "isomorphic-dompurify": "^2.13.0",