pageversionとsourceTextsの関係を修正 pageversionが違ってもnumberと内容が同じなら同一と判定 ジョブ…

…のバージョンシステムの導入 (#143)

web/app/feature/translate/functions/mutations.server.ts

@@ -0,0 +1,46 @@
+import { createHash } from "node:crypto";
+import { prisma } from "~/utils/prisma";
+
+export async function getOrCreateSourceTextIdAndPageVersionSourceText(
+	text: string,
+	number: number,
+	pageVersionId: number,
+): Promise<number> {
+	const textHash = Buffer.from(
+		createHash("sha256").update(text).digest("hex"),
+		"hex",
+	);
+
+	return prisma.$transaction(async (tx) => {
+		const sourceText = await tx.sourceText.upsert({
+			where: {
+				textHash_number: {
+					textHash,
+					number,
+				},
+			},
+			update: {},
+			create: {
+				text,
+				number,
+				textHash,
+			},
+		});
+
+		await tx.pageVersionSourceText.upsert({
+			where: {
+				pageVersionId_sourceTextId: {
+					pageVersionId,
+					sourceTextId: sourceText.id,
+				},
+			},
+			update: {},
+			create: {
+				pageVersionId,
+				sourceTextId: sourceText.id,
+			},
+		});
+
+		return sourceText.id;
+	});
+}

web/app/feature/translate/libs/translationUtils.ts

@@ -1,9 +1,9 @@
 import { getOrCreateAIUser } from "~/libs/db/user.server";
 import { getOrCreatePageVersionTranslationInfo } from "../../../libs/pageVersionTranslationInfo";
-import { getOrCreateSourceTextId } from "../../../libs/sourceTextService";
 import { MAX_CHUNK_SIZE } from "../../../routes/translate/constants";
 import type { NumberedElement } from "../../../routes/translate/types";
 import { prisma } from "../../../utils/prisma";
+import { getOrCreateSourceTextIdAndPageVersionSourceText } from "../functions/mutations.server";
 import { getGeminiModelResponse } from "../utils/gemini";
 
 export function splitNumberedElements(
@@ -75,10 +75,9 @@ export async function getOrCreateTranslations(
 	const untranslatedElements: NumberedElement[] = [];
 	const sourceTextsId = await Promise.all(
 		elements.map((element) =>
-			getOrCreateSourceTextId(
+			getOrCreateSourceTextIdAndPageVersionSourceText(
 				element.text,
 				element.number,
-				pageId,
 				pageVersionId,
 			),
 		),
@@ -168,18 +167,17 @@ async function translateUntranslatedElements(
 				return;
 			}
 
-			const sourceTextId = await getOrCreateSourceTextId(
-				sourceText,
-				translation.number,
-				pageId,
-				pageVersionId,
-			);
+			const sourceTextId =
+				await getOrCreateSourceTextIdAndPageVersionSourceText(
+					sourceText,
+					translation.number,
+					pageVersionId,
+				);
 			await prisma.translateText.create({
 				data: {
 					targetLanguage,
 					text: translation.text,
 					sourceTextId,
-					pageId,
 					userId: systemUserId,
 				},
 			});

web/app/feature/translate/translate-user-queue.ts

@@ -9,8 +9,10 @@ type TranslateJobData = {
 	aiModel: string;
 };
 
+const QUEUE_VERSION = 1;
+
 export const getTranslateUserQueue = (userId: number) => {
-	return Queue<TranslateJobData>(`translation-user-${userId}`, {
+	return Queue<TranslateJobData>(`translation-user-${userId}`, QUEUE_VERSION, {
 		processor: async (job) => {
 			console.log(`Starting job ${job.id} for user ${userId}`);
 			try {

web/app/routes/reader.$/components/ContentWithTranslations.tsx

@@ -1,18 +1,18 @@
 import DOMPurify from "dompurify";
 import parse from "html-react-parser";
 import { memo, useMemo } from "react";
-import type { SourceTextInfoWithTranslations } from "../types";
+import type { SourceTextWithTranslations } from "../types";
 import { Translation } from "./Translation";
 
 interface ContentWithTranslationsProps {
 	content: string;
-	sourceTextInfoWithTranslations: SourceTextInfoWithTranslations[];
+	sourceTextWithTranslations: SourceTextWithTranslations[];
 	userId: number | null;
 }
 
 export const ContentWithTranslations = memo(function ContentWithTranslations({
 	content,
-	sourceTextInfoWithTranslations,
+	sourceTextWithTranslations,
 	userId,
 }: ContentWithTranslationsProps) {
 	const parsedContent = useMemo(() => {
@@ -23,10 +23,7 @@ export const ContentWithTranslations = memo(function ContentWithTranslations({
 		const sanitizedContent = DOMPurify.sanitize(content);
 		const doc = new DOMParser().parseFromString(sanitizedContent, "text/html");
 		const translationMap = new Map(
-			sourceTextInfoWithTranslations.map((info) => [
-				info.number.toString(),
-				info,
-			]),
+			sourceTextWithTranslations.map((info) => [info.number.toString(), info]),
 		);
 
 		for (const [number] of translationMap) {
@@ -60,7 +57,7 @@ export const ContentWithTranslations = memo(function ContentWithTranslations({
 				return domNode;
 			},
 		});
-	}, [content, sourceTextInfoWithTranslations, userId]);
+	}, [content, sourceTextWithTranslations, userId]);
 
 	if (typeof window === "undefined") {
 		return <div>Loading...</div>;

web/app/routes/reader.$/functions/mutations.server.ts

@@ -52,7 +52,6 @@ export async function handleAddTranslationAction(
 ) {
 	const sourceText = await prisma.sourceText.findUnique({
 		where: { id: sourceTextId },
-		include: { page: true },
 	});
 
 	if (sourceText) {
@@ -61,7 +60,6 @@ export async function handleAddTranslationAction(
 				targetLanguage,
 				text,
 				sourceTextId,
-				pageId: sourceText.page.id,
 				userId,
 			},
 		});

web/app/routes/reader.$/functions/queries.server.ts

@@ -1,8 +1,5 @@
 import { prisma } from "../../../utils/prisma";
-import type {
-	LatestPageVersionWithTranslations,
-	SourceTextInfoWithTranslations,
-} from "../types";
+import type { LatestPageVersionWithTranslations } from "../types";
 
 export async function fetchLatestPageVersionWithTranslations(
 	url: string,
@@ -17,29 +14,38 @@ export async function fetchLatestPageVersionWithTranslations(
 			url: true,
 			content: true,
 			license: true,
-			sourceTexts: {
+			pageVersionSourceTexts: {
 				select: {
-					id: true,
-					number: true,
-					translateTexts: {
-						where: { targetLanguage },
+					sourceText: {
 						select: {
 							id: true,
-							text: true,
-							point: true,
-							user: { select: { name: true } },
-							votes: {
-								where: userId ? { userId } : undefined,
+							number: true,
+							translateTexts: {
+								where: { targetLanguage },
 								select: {
 									id: true,
-									isUpvote: true,
-									updatedAt: true,
+									text: true,
+									point: true,
+									user: { select: { name: true } },
+									votes: {
+										where: userId ? { userId } : undefined,
+										select: {
+											id: true,
+											isUpvote: true,
+											updatedAt: true,
+										},
+										orderBy: { updatedAt: "desc" },
+										take: 1,
+									},
 								},
-								orderBy: { updatedAt: "desc" },
-								take: 1,
+								orderBy: [{ point: "desc" }, { createdAt: "desc" }],
 							},
 						},
-						orderBy: [{ point: "desc" }, { createdAt: "desc" }],
+					},
+				},
+				orderBy: {
+					sourceText: {
+						number: "asc",
 					},
 				},
 			},
@@ -48,25 +54,26 @@ export async function fetchLatestPageVersionWithTranslations(
 
 	if (!pageVersion) return null;
 
-	const sourceTextInfoWithTranslations: SourceTextInfoWithTranslations[] =
-		pageVersion.sourceTexts.map((sourceText) => ({
-			number: sourceText.number,
-			sourceTextId: sourceText.id,
-			translationsWithVotes: sourceText.translateTexts.map((translateText) => ({
-				id: translateText.id,
-				text: translateText.text,
-				point: translateText.point,
-				userName: translateText.user.name,
-				userVote: translateText.votes[0] || null,
-			})),
-		}));
-
 	return {
 		title: pageVersion.title,
 		url: pageVersion.url,
 		license: pageVersion.license,
 		content: pageVersion.content,
-		sourceTextInfoWithTranslations,
+		sourceTextWithTranslations: pageVersion.pageVersionSourceTexts.map(
+			({ sourceText }) => ({
+				number: sourceText.number,
+				sourceTextId: sourceText.id,
+				translationsWithVotes: sourceText.translateTexts.map(
+					(translateText) => ({
+						id: translateText.id,
+						text: translateText.text,
+						point: translateText.point,
+						userName: translateText.user.name,
+						userVote: translateText.votes[0] || null,
+					}),
+				),
+			}),
+		),
 		userId,
 	};
 }

web/app/routes/reader.$/route.tsx

@@ -167,9 +167,7 @@ export default function ReaderView() {
 					<hr />
 					<ContentWithTranslations
 						content={pageData.content}
-						sourceTextInfoWithTranslations={
-							pageData.sourceTextInfoWithTranslations
-						}
+						sourceTextWithTranslations={pageData.sourceTextWithTranslations}
 						userId={safeUser?.id ?? null}
 					/>
 					<p>license: {pageData.license}</p>

web/app/routes/reader.$/types.ts

@@ -10,7 +10,7 @@ export type TranslationWithVote = Pick<
 	userVote: UserVote | null;
 };
 
-export interface SourceTextInfoWithTranslations {
+export interface SourceTextWithTranslations {
 	number: number;
 	sourceTextId: number;
 	translationsWithVotes: TranslationWithVote[];
@@ -21,7 +21,7 @@ export interface LatestPageVersionWithTranslations {
 	url: string;
 	content: string;
 	license: string;
-	sourceTextInfoWithTranslations: SourceTextInfoWithTranslations[];
+	sourceTextWithTranslations: SourceTextWithTranslations[];
 	userId: number | null;
 }
 

web/app/routes/translate/constants.ts

@@ -1,3 +1,2 @@
 export const REDIS_URL = process.env.REDIS_URL || "redis://localhost:6379";
 export const MAX_CHUNK_SIZE = 20000;
-export const AI_MODEL = "gemini-1.5-flash";

web/app/utils/queue.server.ts

@@ -4,6 +4,7 @@ import { RedisConfig } from "./redis-config";
 type RegisteredQueue = {
 	queue: BullQueue;
 	worker: Worker;
+	version: number;
 };
 
 declare global {
@@ -17,27 +18,36 @@ const registeredQueues = global.__registeredQueues;
 
 export function Queue<Payload>(
 	name: string,
+	version: number,
 	handlers: {
 		processor: Processor<Payload>;
 		onComplete: (job: Job<Payload>, queue: BullQueue<Payload>) => void;
 	},
 ): BullQueue<Payload> {
-	if (registeredQueues[name]) {
+	if (registeredQueues[name] && registeredQueues[name].version === version) {
 		return registeredQueues[name].queue;
 	}
+
+	if (registeredQueues[name]) {
+		registeredQueues[name].worker.close();
+	}
+
 	const queue = new BullQueue<Payload>(name, { connection: RedisConfig });
 	const worker = new Worker<Payload>(name, handlers.processor, {
 		connection: RedisConfig,
 	});
 	worker.on("completed", (job) => handlers.onComplete(job, queue));
-	registeredQueues[name] = { queue, worker };
+
+	registeredQueues[name] = { queue, worker, version };
 	return queue;
 }
 
 export async function clearAllQueues() {
-	for (const [name, { queue }] of Object.entries(registeredQueues)) {
+	for (const [name, { queue, worker }] of Object.entries(registeredQueues)) {
 		await queue.obliterate({ force: true });
+		await worker.close();
 		console.log(`Cleared queue: ${name}`);
 	}
+	global.__registeredQueues = {};
 	console.log("All queues have been cleared.");
 }