feature: improvements to evaluations results table

Fixes queue ordering as evaluations were being run after all document
runts had completed which is not the best experience.

Also added interval fetching of the table data to simulate real time
experience.

Finally, added collapsable convos in the log messages section in order
to make that section more readable.

To further improve this we need:
1. Decouple expensive queries from this view so that we can query table
   logs more freely
2. Add an independent queue dedicated to running evaluations in order to increase
   parallelism between document and evaluation runs

apps/web/src/actions/evaluations/computeEvaluationResultsWithMetadata.ts

@@ -0,0 +1,40 @@
+'use server'
+
+import {
+  CommitsRepository,
+  EvaluationsRepository,
+} from '@latitude-data/core/repositories'
+import { computeEvaluationResultsWithMetadata } from '@latitude-data/core/services/evaluationResults/computeEvaluationResultsWithMetadata'
+import { z } from 'zod'
+
+import { withProject } from '../procedures'
+
+export const computeEvaluationResultsWithMetadataAction = withProject
+  .createServerAction()
+  .input(
+    z.object({
+      evaluationId: z.number(),
+      documentUuid: z.string(),
+      commitUuid: z.string(),
+    }),
+  )
+  .handler(async ({ input, ctx }) => {
+    const { documentUuid } = input
+    const { workspace, project } = ctx
+    const commitsScope = new CommitsRepository(workspace.id)
+    const evaluationScope = new EvaluationsRepository(workspace.id)
+    const evaluation = await evaluationScope
+      .find(input.evaluationId)
+      .then((r) => r.unwrap())
+    const commit = await commitsScope
+      .getCommitByUuid({ projectId: project.id, uuid: input.commitUuid })
+      .then((r) => r.unwrap())
+
+    return await computeEvaluationResultsWithMetadata({
+      workspaceId: ctx.workspace.id,
+      evaluation,
+      documentUuid,
+      draft: commit,
+      limit: 1000,
+    }).then((r) => r.unwrap())
+  })

apps/web/src/actions/evaluations/runBatch.ts

@@ -10,11 +10,11 @@ import { nanoid } from 'nanoid'
 import { z } from 'zod'
 import { createServerActionProcedure } from 'zsa'
 
-import { widthDocument } from '../procedures'
+import { withDocument } from '../procedures'
 
 const USER_DECIDED_TO_IGNORE_THIS_PARAMETER = -1
 
-const withDataset = createServerActionProcedure(widthDocument)
+const withDataset = createServerActionProcedure(withDocument)
   .input(z.object({ datasetId: z.number() }))
   .handler(async ({ input, ctx }) => {
     const datasetsRepo = new DatasetsRepository(ctx.workspace.id)

apps/web/src/actions/procedures/index.ts

@@ -54,7 +54,7 @@ export const withProject = createServerActionProcedure(authProcedure)
     return { ...ctx, project }
   })
 
-export const widthDocument = createServerActionProcedure(withProject)
+export const withDocument = createServerActionProcedure(withProject)
   .input(z.object({ commitUuid: z.string(), documentUuid: z.string() }))
   .handler(async ({ input, ctx }) => {
     const repo = new DocumentVersionsRepository(ctx.workspace.id)

apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/EvaluationResults/EvaluationResultInfo/Messages.tsx

@@ -25,6 +25,7 @@ export function EvaluationResultMessages({
   return (
     <div className='flex flex-col gap-4 py-6 w-full max-h-full overflow-y-auto'>
       <MessageList
+        collapsed
         messages={messages}
         messageLayout='vertical'
         separator

apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/EvaluationResults/index.tsx

@@ -1,26 +1,56 @@
 'use client'
 
-import { useState } from 'react'
+import { useEffect, useState } from 'react'
 
 import { EvaluationDto } from '@latitude-data/core/browser'
 import { EvaluationResultWithMetadata } from '@latitude-data/core/repositories'
-import { Text } from '@latitude-data/web-ui'
+import {
+  Text,
+  useCurrentCommit,
+  useCurrentDocument,
+  useCurrentProject,
+} from '@latitude-data/web-ui'
+import useEvaluationResultsWithMetadata from '$/stores/evaluationResultsWithMetadata'
 import { useProviderLog } from '$/stores/providerLogs'
 
 import { EvaluationResultInfo } from './EvaluationResultInfo'
 import { EvaluationResultsTable } from './EvaluationResultsTable'
 import { EvaluationStatusBanner } from './EvaluationStatusBanner'
 
+const FIVE_SECONDS = 5000
+
 export function EvaluationResults({
   evaluation,
-  evaluationResults,
+  evaluationResults: serverData,
 }: {
   evaluation: EvaluationDto
   evaluationResults: EvaluationResultWithMetadata[]
 }) {
   const [selectedResult, setSelectedResult] = useState<
     EvaluationResultWithMetadata | undefined
   >(undefined)
+  const document = useCurrentDocument()
+  const { project } = useCurrentProject()
+  const { commit } = useCurrentCommit()
+  const { data: evaluationResults, mutate } = useEvaluationResultsWithMetadata(
+    {
+      evaluationId: evaluation.id,
+      documentUuid: document.documentUuid,
+      commitUuid: commit.uuid,
+      projectId: project.id,
+    },
+    {
+      fallbackData: serverData,
+    },
+  )
+
+  useEffect(() => {
+    const interval = setInterval(() => {
+      mutate()
+    }, FIVE_SECONDS)
+
+    return () => clearInterval(interval)
+  }, [mutate])
 
   const { data: providerLog } = useProviderLog(selectedResult?.providerLogId)
   return (

apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/layout.tsx

@@ -40,6 +40,7 @@ export default async function ConnectedEvaluationLayout({
     evaluation,
     documentUuid: params.documentUuid,
     draft: commit,
+    limit: 1000,
   }).then((r) => r.unwrap())
 
   return (

apps/web/src/stores/evaluationResultsWithMetadata.ts

@@ -0,0 +1,51 @@
+import { useMemo } from 'react'
+
+import { EvaluationResultWithMetadata } from '@latitude-data/core/repositories'
+import { useToast } from '@latitude-data/web-ui'
+import { computeEvaluationResultsWithMetadataAction } from '$/actions/evaluations/computeEvaluationResultsWithMetadata'
+import useSWR, { SWRConfiguration } from 'swr'
+
+const EMPTY_ARRAY: [] = []
+export default function useEvaluationResultsWithMetadata(
+  {
+    evaluationId,
+    documentUuid,
+    commitUuid,
+    projectId,
+  }: {
+    evaluationId: number
+    documentUuid: string
+    commitUuid: string
+    projectId: number
+  },
+  opts: SWRConfiguration,
+) {
+  const { toast } = useToast()
+  const { data = EMPTY_ARRAY, ...rest } = useSWR<
+    EvaluationResultWithMetadata[]
+  >(
+    ['evaluationResults', evaluationId, documentUuid, commitUuid, projectId],
+    async () => {
+      const [data, error] = await computeEvaluationResultsWithMetadataAction({
+        evaluationId,
+        documentUuid,
+        commitUuid,
+        projectId,
+      })
+
+      if (error) {
+        toast({
+          title: 'Error fetching evaluations',
+          description: error.formErrors?.[0] || error.message,
+          variant: 'destructive',
+        })
+        throw error
+      }
+
+      return data
+    },
+    opts,
+  )
+
+  return useMemo(() => ({ data, ...rest }), [data, rest])
+}

packages/core/src/services/evaluationResults/computeEvaluationResultsWithMetadata.ts

@@ -15,18 +15,20 @@ export async function computeEvaluationResultsWithMetadata(
     evaluation,
     documentUuid,
     draft,
+    limit,
   }: {
     workspaceId: number
     evaluation: Evaluation
     documentUuid: string
     draft?: Commit
+    limit?: number
   },
   db = database,
 ): Promise<TypedResult<EvaluationResultWithMetadata[], Error>> {
   const { evaluationResultsScope, documentLogsScope, baseQuery } =
     createEvaluationResultQuery(workspaceId, db)
 
-  const result = await baseQuery
+  const query = baseQuery
     .where(
       and(
         eq(evaluationResultsScope.evaluationId, evaluation.id),
@@ -36,5 +38,7 @@ export async function computeEvaluationResultsWithMetadata(
     )
     .orderBy(desc(evaluationResultsScope.createdAt))
 
+  const result = await (limit ? query.limit(limit) : query)
+
   return Result.ok(result)
 }

packages/jobs/src/job-definitions/batchEvaluations/runDocumentJob.test.ts

@@ -97,13 +97,16 @@ describe('runDocumentJob', () => {
 
     expect(
       mocks.queues.defaultQueue.jobs.enqueueRunEvaluationJob,
-    ).toHaveBeenCalledWith({
-      workspaceId: workspace.id,
-      documentUuid: document.documentUuid,
-      documentLogUuid: 'log1',
-      evaluationId: evaluation.id,
-      batchId: 'batch1',
-    })
+    ).toHaveBeenCalledWith(
+      {
+        workspaceId: workspace.id,
+        documentUuid: document.documentUuid,
+        documentLogUuid: 'log1',
+        evaluationId: evaluation.id,
+        batchId: 'batch1',
+      },
+      { lifo: true },
+    )
 
     expect(ProgressTracker.prototype.incrementErrors).not.toHaveBeenCalled()
     expect(ProgressTracker.prototype.decrementTotal).not.toHaveBeenCalled()

packages/jobs/src/job-definitions/batchEvaluations/runDocumentJob.ts

@@ -61,13 +61,16 @@ export const runDocumentJob = async (job: Job<RunDocumentJobData>) => {
     const queues = await setupJobs()
 
     // Enqueue the evaluation job
-    await queues.defaultQueue.jobs.enqueueRunEvaluationJob({
-      workspaceId,
-      documentUuid: document.documentUuid,
-      documentLogUuid: result.documentLogUuid,
-      evaluationId,
-      batchId,
-    })
+    await queues.defaultQueue.jobs.enqueueRunEvaluationJob(
+      {
+        workspaceId,
+        documentUuid: document.documentUuid,
+        documentLogUuid: result.documentLogUuid,
+        evaluationId,
+        batchId,
+      },
+      { lifo: true },
+    )
   } catch (error) {
     if (env.NODE_ENV !== 'production') {
       console.error(error)

packages/web-ui/src/ds/molecules/Chat/MessageList/index.tsx

@@ -1,36 +1,77 @@
+'use client'
+
+import { Fragment, useState } from 'react'
+
 import { Message as ConversationMessage } from '@latitude-data/compiler'
-import { Fragment } from 'react/jsx-runtime'
 
+import { Button } from '../../../atoms'
 import { Message, MessageProps } from '../Message'
 
 export function MessageList({
   messages,
   variant,
   messageLayout,
   separator = false,
+  collapsed = false,
   size,
 }: {
   messages: ConversationMessage[]
   variant?: MessageProps['variant']
   messageLayout?: MessageProps['layout']
+  collapsed?: boolean
   size?: MessageProps['size']
   separator?: boolean
 }) {
-  return messages.map((message, index) => (
-    <Fragment key={index}>
-      {separator && index > 0 && (
-        <div
-          key={`${index}-separator`}
-          className='h-px min-h-px w-full bg-border'
-        />
+  const [isCollapsed, setIsCollapsed] = useState(
+    collapsed && messages.length > 1,
+  )
+
+  const visibleMessages = isCollapsed ? messages.slice(-1) : messages
+  const hiddenMessagesCount = messages.length - visibleMessages.length
+
+  return (
+    <div className='flex flex-col gap-4'>
+      {isCollapsed && messages.length > 1 && (
+        <div className='relative cursor-pointer h-24 overflow-hidden'>
+          <div className='opacity-50 pointer-events-none absolute top-0 left-0 right-0 scale-90 origin-top'>
+            <Message
+              role={messages[messages.length - 2]!.role}
+              content={messages[messages.length - 2]!.content}
+              variant={variant}
+              layout={messageLayout}
+              size={size}
+            />
+          </div>
+          <div className='absolute inset-0 bg-gradient-to-t from-white via-white to-transparent' />
+          <div className='absolute bottom-0 left-0 right-0 text-center p-2 text-sm text-gray-600'>
+            <Button variant='secondary' onClick={() => setIsCollapsed(false)}>
+              {hiddenMessagesCount} previous{' '}
+              {hiddenMessagesCount > 1 ? 'messages' : 'message'}
+            </Button>
+          </div>
+        </div>
+      )}
+      {visibleMessages.map((message, index) => (
+        <Fragment key={index}>
+          {separator && index > 0 && (
+            <div className='h-px min-h-px w-full bg-border' />
+          )}
+          <Message
+            role={message.role}
+            content={message.content}
+            variant={variant}
+            layout={messageLayout}
+            size={size}
+          />
+        </Fragment>
+      ))}
+      {!isCollapsed && messages.length > 1 && (
+        <div className='text-center cursor-pointer text-sm text-gray-600 hover:text-gray-800'>
+          <Button variant='secondary' onClick={() => setIsCollapsed(true)}>
+            Collapse conversation
+          </Button>
+        </div>
       )}
-      <Message
-        role={message.role}
-        content={message.content}
-        variant={variant}
-        layout={messageLayout}
-        size={size}
-      />
-    </Fragment>
-  ))
+    </div>
+  )
 }