From d6b712428e124e080f02226fa598838dcf160c0a Mon Sep 17 00:00:00 2001 From: Nick Larew Date: Fri, 9 Aug 2024 13:15:39 -0400 Subject: [PATCH 1/7] (EAI-383) Fetch & Refine Top 250 Jira Tickets --- package-lock.json | 16 +- .../mongodb-artifact-generator/package.json | 4 +- .../src/ArtifactGeneratorEnvVars.ts | 5 + .../src/chat/makeGeneratePrompts.ts | 104 ++++ .../src/chat/makeGenerateResponse.ts | 109 ++++ .../src/chat/makeSummarizer.test.ts | 89 ++++ .../src/chat/makeSummarizer.ts | 115 +++++ .../src/chat/utils.test.ts | 66 +++ .../src/chat/utils.ts | 119 +++++ .../commands/generateJiraPromptResponse.ts | 376 ++++++++++++++ .../examples/ADMIN-10208.json | 400 +++++++++++++++ .../examples/WORKPLACE-119.json | 470 ++++++++++++++++++ .../src/jiraPromptResponse/topIssues.json | 236 +++++++++ .../src/runId.test.ts | 65 +++ .../mongodb-artifact-generator/src/runId.ts | 14 + .../src/runlogger.ts | 3 +- .../mongodb-artifact-generator/tsconfig.json | 5 +- 17 files changed, 2191 insertions(+), 5 deletions(-) create mode 100644 packages/mongodb-artifact-generator/src/chat/makeGeneratePrompts.ts create mode 100644 packages/mongodb-artifact-generator/src/chat/makeGenerateResponse.ts create mode 100644 packages/mongodb-artifact-generator/src/chat/makeSummarizer.test.ts create mode 100644 packages/mongodb-artifact-generator/src/chat/makeSummarizer.ts create mode 100644 packages/mongodb-artifact-generator/src/chat/utils.test.ts create mode 100644 packages/mongodb-artifact-generator/src/chat/utils.ts create mode 100644 packages/mongodb-artifact-generator/src/commands/generateJiraPromptResponse.ts create mode 100644 packages/mongodb-artifact-generator/src/jiraPromptResponse/examples/ADMIN-10208.json create mode 100644 packages/mongodb-artifact-generator/src/jiraPromptResponse/examples/WORKPLACE-119.json create mode 100644 packages/mongodb-artifact-generator/src/jiraPromptResponse/topIssues.json create mode 100644 packages/mongodb-artifact-generator/src/runId.test.ts create mode 100644 packages/mongodb-artifact-generator/src/runId.ts diff --git a/package-lock.json b/package-lock.json index f6c17eba0..b1b8d0ca0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -41152,6 +41152,18 @@ "zod": "^3.23.3" } }, + "node_modules/zod-validation-error": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/zod-validation-error/-/zod-validation-error-3.3.1.tgz", + "integrity": "sha512-uFzCZz7FQis256dqw4AhPQgD6f3pzNca/Zh62RNELavlumQB3nDIUFbF5JQfFLcMbO1s02Q7Xg/gpcOBlEnYZA==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + }, + "peerDependencies": { + "zod": "^3.18.0" + } + }, "node_modules/zwitch": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/zwitch/-/zwitch-2.0.4.tgz", @@ -42325,7 +42337,9 @@ "papaparse": "^5.4.1", "yaml": "^2.3.1", "yargs": "^17", - "zod": "^3.22.4" + "zod": "^3.22.4", + "zod-to-json-schema": "^3.23.2", + "zod-validation-error": "^3.3.1" }, "bin": { "mongodb-ai": "build/main.js" diff --git a/packages/mongodb-artifact-generator/package.json b/packages/mongodb-artifact-generator/package.json index b40334b0a..3da8f02bc 100644 --- a/packages/mongodb-artifact-generator/package.json +++ b/packages/mongodb-artifact-generator/package.json @@ -61,6 +61,8 @@ "papaparse": "^5.4.1", "yaml": "^2.3.1", "yargs": "^17", - "zod": "^3.22.4" + "zod": "^3.22.4", + "zod-to-json-schema": "^3.23.2", + "zod-validation-error": "^3.3.1" } } diff --git a/packages/mongodb-artifact-generator/src/ArtifactGeneratorEnvVars.ts b/packages/mongodb-artifact-generator/src/ArtifactGeneratorEnvVars.ts index bc7f48a8b..37132d47d 100644 --- a/packages/mongodb-artifact-generator/src/ArtifactGeneratorEnvVars.ts +++ b/packages/mongodb-artifact-generator/src/ArtifactGeneratorEnvVars.ts @@ -1,5 +1,10 @@ import { CORE_ENV_VARS } from "mongodb-rag-core"; +export const GENERATOR_LLM_ENV_VARS = { + GENERATOR_LLM_MAX_CONCURRENCY: "", +}; + export const ArtifactGeneratorEnvVars = { ...CORE_ENV_VARS, + ...GENERATOR_LLM_ENV_VARS, }; diff --git a/packages/mongodb-artifact-generator/src/chat/makeGeneratePrompts.ts b/packages/mongodb-artifact-generator/src/chat/makeGeneratePrompts.ts new file mode 100644 index 000000000..b35f1b3ba --- /dev/null +++ b/packages/mongodb-artifact-generator/src/chat/makeGeneratePrompts.ts @@ -0,0 +1,104 @@ +import { + ChatRequestMessage, + FunctionDefinition, + OpenAIClient, +} from "mongodb-rag-core"; +import { FormattedJiraIssueWithSummary } from "../commands/generateJiraPromptResponse"; +import { RunLogger } from "../runlogger"; +import { + asJsonSchema, + formatFewShotExamples, + formatMessagesForArtifact, + PromptExamplePair, +} from "./utils"; +import { z } from "zod"; +import { stripIndents } from "common-tags"; + +export type GeneratedPrompts = z.infer; +export const GeneratedPrompts = z.object({ + prompts: z.array(z.string()), +}); + +const generatePromptsTool: FunctionDefinition = { + name: "generatePrompts", + description: + "A list of generated example prompts that would elicit a given response.", + parameters: asJsonSchema(GeneratedPrompts), +}; + +export type MakeGeneratePromptsArgs = { + openAi: { + client: OpenAIClient; + deployment: string; + }; + logger?: RunLogger; + directions?: string; + examples?: PromptExamplePair[]; +}; + +export type GeneratePromptsArgs = FormattedJiraIssueWithSummary; + +export function makeGeneratePrompts({ + openAi, + logger, + directions, + examples = [], +}: MakeGeneratePromptsArgs) { + return async function generatePrompts({ + issue, + summary, + }: GeneratePromptsArgs) { + const messages = [ + { + role: "system", + content: [ + `Your task is to convert a provided input into a prompt-response format. The format mimics a conversation where one participant sends a prompt and the other replies with a response.`, + directions ?? "", + ].join("\n"), + }, + ...formatFewShotExamples({ + examples, + functionName: generatePromptsTool.name, + responseSchema: GeneratedPrompts, + }), + { + role: "user", + content: JSON.stringify({ issue, summary }), + }, + ] satisfies ChatRequestMessage[]; + const result = await openAi.client.getChatCompletions( + openAi.deployment, + messages, + { + temperature: 0, + maxTokens: 1500, + functions: [generatePromptsTool], + functionCall: { + name: generatePromptsTool.name, + }, + } + ); + const response = result.choices[0].message; + if (response === undefined) { + throw new Error("No response from OpenAI"); + } + if (response.functionCall === undefined) { + throw new Error("No function call in response from OpenAI"); + } + const generatedPrompts = GeneratedPrompts.parse( + JSON.parse(response.functionCall.arguments) + ); + + logger?.appendArtifact( + `chatTemplates/generatePrompts-${Date.now()}.json`, + stripIndents` + ${formatMessagesForArtifact(messages).join("\n")} + + ${JSON.stringify(summary)} + + ` + ); + + return generatedPrompts; + }; +} diff --git a/packages/mongodb-artifact-generator/src/chat/makeGenerateResponse.ts b/packages/mongodb-artifact-generator/src/chat/makeGenerateResponse.ts new file mode 100644 index 000000000..eda9d2266 --- /dev/null +++ b/packages/mongodb-artifact-generator/src/chat/makeGenerateResponse.ts @@ -0,0 +1,109 @@ +import { + ChatRequestMessage, + FunctionDefinition, + OpenAIClient, +} from "mongodb-rag-core"; +import { FormattedJiraIssue } from "../commands/generateJiraPromptResponse"; +import { RunLogger } from "../runlogger"; +import { + asJsonSchema, + formatFewShotExamples, + formatMessagesForArtifact, + PromptExamplePair, +} from "./utils"; +import { z } from "zod"; +import { stripIndents } from "common-tags"; +import { Summary } from "./makeSummarizer"; + +export type GeneratedResponse = z.infer; +export const GeneratedResponse = z.object({ + response: z.string().describe("The generated response text."), +}); + +const generateResponseTool: FunctionDefinition = { + name: "generateResponse", + description: "A response generated based on a given context.", + parameters: asJsonSchema(GeneratedResponse), +}; + +export type MakeGenerateResponseArgs = { + openAi: { + client: OpenAIClient; + deployment: string; + }; + logger?: RunLogger; + directions?: string; + examples?: PromptExamplePair[]; +}; + +export type GenerateResponseArgs = { + issue: FormattedJiraIssue; + summary: Summary; + prompt: string; +}; + +export function makeGenerateResponse({ + openAi, + logger, + directions, + examples = [], +}: MakeGenerateResponseArgs) { + return async function generateResponse({ + issue, + summary, + prompt, + }: GenerateResponseArgs) { + const messages = [ + { + role: "system", + content: [ + `Your task is to generate a response to a provided input. The response should be relevant to the input and based only on the provided context.`, + directions ?? "", + ].join("\n"), + }, + ...formatFewShotExamples({ + examples, + functionName: generateResponseTool.name, + responseSchema: GeneratedResponse, + }), + { + role: "user", + content: JSON.stringify({ issue, summary, prompt }), + }, + ] satisfies ChatRequestMessage[]; + const result = await openAi.client.getChatCompletions( + openAi.deployment, + messages, + { + temperature: 0, + maxTokens: 1500, + functions: [generateResponseTool], + functionCall: { + name: generateResponseTool.name, + }, + } + ); + const response = result.choices[0].message; + if (response === undefined) { + throw new Error("No response from OpenAI"); + } + if (response.functionCall === undefined) { + throw new Error("No function call in response from OpenAI"); + } + const generatedResponse = GeneratedResponse.parse( + JSON.parse(response.functionCall.arguments) + ); + + logger?.appendArtifact( + `chatTemplates/generateResponse-${Date.now()}.json`, + stripIndents` + ${formatMessagesForArtifact(messages).join("\n")} + + ${JSON.stringify(summary)} + + ` + ); + + return generatedResponse; + }; +} diff --git a/packages/mongodb-artifact-generator/src/chat/makeSummarizer.test.ts b/packages/mongodb-artifact-generator/src/chat/makeSummarizer.test.ts new file mode 100644 index 000000000..20154f883 --- /dev/null +++ b/packages/mongodb-artifact-generator/src/chat/makeSummarizer.test.ts @@ -0,0 +1,89 @@ +import { + AzureKeyCredential, + CORE_ENV_VARS, + OpenAIClient, + assertEnvVars, +} from "mongodb-rag-core"; +import { Summary, makeSummarizer } from "./makeSummarizer"; +import { PromptExamplePair } from "./utils"; + +const { OPENAI_CHAT_COMPLETION_DEPLOYMENT, OPENAI_ENDPOINT, OPENAI_API_KEY } = + assertEnvVars(CORE_ENV_VARS); + +const openAiClient = new OpenAIClient( + OPENAI_ENDPOINT, + new AzureKeyCredential(OPENAI_API_KEY) +); + +const openAi = { + client: openAiClient, + deployment: OPENAI_CHAT_COMPLETION_DEPLOYMENT, +}; + +// TODO: Convert this into a .eval file +describe.skip("makeSummarizer", () => { + it("creates a summarizer for content", async () => { + const summarizePoem = makeSummarizer({ + openAi, + directions: "The provided content will be a poem.", + examples: [ + [ + "Two roads diverged in a yellow wood,\nAnd sorry I could not travel both\nAnd be one traveler, long I stood\nAnd looked down one as far as I could\nTo where it bent in the undergrowth;", + { + description: + "A traveler encounters a fork in a road in a forest and reflects on the decision of choosing a path.", + topics: ["travel", "decisions", "forests", "fork in the road"], + }, + ], + [ + "Hope is the thing with feathers\nThat perches in the soul,\nAnd sings the tune without the words,\nAnd never stops at all,", + { + description: + "Hope is described as a bird that lives in the soul, continuously singing a wordless tune.", + // topics: ["hope", "soul", "birds", "music"], + }, + ], + [ + "I wandered lonely as a cloud\nThat floats on high o'er vales and hills,\nWhen all at once I saw a crowd,\nA host, of golden daffodils;", + { + descriptionz: + "A person feels lonely but then finds joy upon seeing a field of daffodils.", + topics: ["loneliness", "flowers"], + }, + ], + ], + }); + + const testCases = [ + [ + "The moon has a face like the clock in the hall;\nShe shines on thieves on the garden wall,\nOn streets and fields and harbor quays,\nAnd birdies asleep in the forks of the trees.", + { + description: + "The moon is compared to a clock, casting its light over different scenes, from thieves to sleeping birds.", + topics: ["moon", "night", "nature", "light"], + }, + ], + [ + "The rose is a rose,\nAnd was always a rose.\nBut the theory now goes\nThat the apple’s a rose,\nAnd the pear is, and so’s\nThe plum, I suppose.", + { + description: + "The poem reflects on the constancy and transformation of things, suggesting all things can be seen as a rose.", + topics: ["roses", "nature", "change", "metaphor"], + }, + ], + [ + "Fog drifts in,\nSoftly, softly,\nBlanketing the world\nIn a whisper.", + { + description: + "A quiet fog moves in, gently covering the surroundings in silence.", + topics: ["fog", "silence", "nature", "stillness"], + }, + ], + ] satisfies PromptExamplePair[]; + + for (const [poem, expected] of testCases) { + const summary = await summarizePoem({ input: poem }); + // expect(summary).toEqual(expected); + } + }); +}); diff --git a/packages/mongodb-artifact-generator/src/chat/makeSummarizer.ts b/packages/mongodb-artifact-generator/src/chat/makeSummarizer.ts new file mode 100644 index 000000000..4fad4dfee --- /dev/null +++ b/packages/mongodb-artifact-generator/src/chat/makeSummarizer.ts @@ -0,0 +1,115 @@ +import "dotenv/config"; +import { + ChatRequestMessage, + FunctionDefinition, + OpenAIClient, +} from "@azure/openai"; +import { stripIndents } from "common-tags"; +import { z } from "zod"; +import { RunLogger } from "../runlogger"; +import { + asJsonSchema, + formatFewShotExamples, + formatMessagesForArtifact, + PromptExamplePair, +} from "./utils"; + +export type Summarizer = (args: { + input: string; +}) => Summary | Promise; + +export type Summary = z.infer; +export const Summary = z.object({ + topics: z + .array( + z + .string() + .describe( + "The name of a topic in the input. This could be a product name, a feature, or any other relevant noun mentioned in the input." + + "\nExamples:\n" + + ["MongoDB Atlas", "Atlas Search", "Architecture", "Atlas SQL"].join( + "\n- " + ) + ) + ) + .describe("A list of topics mentioned in the input."), + description: z + .string() + .describe("A summarized text description of the input."), +}); + +const summarizeTool: FunctionDefinition = { + name: "summarize", + description: "A structured summary of the provided input", + parameters: asJsonSchema(Summary), +}; + +export type MakeSummarizerArgs = { + openAi: { + client: OpenAIClient; + deployment: string; + }; + logger?: RunLogger; + directions?: string; + examples?: PromptExamplePair[]; +}; + +export function makeSummarizer({ + openAi, + logger, + directions = "", + examples = [], +}: MakeSummarizerArgs): Summarizer { + return async function summarize({ input }: { input: string }) { + const messages = [ + { + role: "system", + content: [ + `Your task is to summarize a provided input. This information will be used to drive a generative process, so precision and correctness are incredibly important.`, + directions ?? "", + ].join("\n"), + }, + ...formatFewShotExamples({ + examples, + functionName: summarizeTool.name, + responseSchema: Summary, + }), + { + role: "user", + content: input, + }, + ] satisfies ChatRequestMessage[]; + const result = await openAi.client.getChatCompletions( + openAi.deployment, + messages, + { + temperature: 0, + maxTokens: 1500, + functions: [summarizeTool], + functionCall: { + name: summarizeTool.name, + }, + } + ); + const response = result.choices[0].message; + if (response === undefined) { + throw new Error("No response from OpenAI"); + } + if (response.functionCall === undefined) { + throw new Error("No function call in response from OpenAI"); + } + const summary = Summary.parse(JSON.parse(response.functionCall.arguments)); + + logger?.appendArtifact( + `chatTemplates/summarizer-${Date.now()}.json`, + stripIndents` + ${formatMessagesForArtifact(messages).join("\n")} + + ${JSON.stringify(summary)} + + ` + ); + + return summary; + }; +} diff --git a/packages/mongodb-artifact-generator/src/chat/utils.test.ts b/packages/mongodb-artifact-generator/src/chat/utils.test.ts new file mode 100644 index 000000000..61118b9be --- /dev/null +++ b/packages/mongodb-artifact-generator/src/chat/utils.test.ts @@ -0,0 +1,66 @@ +import { formatFewShotExamples } from "./utils"; +import { z } from "zod"; + +describe("formatFewShotExamples", () => { + test("formats few shot examples into an LLM conversation", () => { + const result = formatFewShotExamples({ + examples: [ + ["input1", "output1"], + ["input2", "output2"], + ], + responseSchema: z.string(), + functionName: "test-few-shot-function", + }); + expect(result).toEqual([ + { role: "user", content: "input1" }, + { + role: "assistant", + content: null, + functionCall: { + name: "test-few-shot-function", + arguments: JSON.stringify("output1"), + }, + }, + { role: "user", content: "input2" }, + { + role: "assistant", + content: null, + functionCall: { + name: "test-few-shot-function", + arguments: JSON.stringify("output2"), + }, + }, + ]); + }); + + test("allows you to pass objects as output", () => { + const result = formatFewShotExamples({ + examples: [ + ["input1", { key: "value1" }], + ["input2", { key: "value2" }], + ], + responseSchema: z.object({ key: z.string() }), + functionName: "test-few-shot-function", + }); + expect(result).toEqual([ + { role: "user", content: "input1" }, + { + role: "assistant", + content: null, + functionCall: { + name: "test-few-shot-function", + arguments: JSON.stringify({ key: "value1" }), + }, + }, + { role: "user", content: "input2" }, + { + role: "assistant", + content: null, + functionCall: { + name: "test-few-shot-function", + arguments: JSON.stringify({ key: "value2" }), + }, + }, + ]); + }); +}); diff --git a/packages/mongodb-artifact-generator/src/chat/utils.ts b/packages/mongodb-artifact-generator/src/chat/utils.ts new file mode 100644 index 000000000..7da6ccbae --- /dev/null +++ b/packages/mongodb-artifact-generator/src/chat/utils.ts @@ -0,0 +1,119 @@ +import { promises as fs } from "fs"; +import { ChatRequestMessage } from "mongodb-rag-core"; +import { z, ZodTypeAny } from "zod"; +import zodToJsonSchema from "zod-to-json-schema"; +import { fromError } from "zod-validation-error"; + +export function formatMessagesForArtifact(messages: ChatRequestMessage[]) { + const tagsByRole = { + system: "SystemMessage", + user: "UserMessage", + assistant: "AssistantMessage", + }; + return messages + .filter((message) => message.role in tagsByRole) + .map((message) => { + const tag = tagsByRole[message.role as keyof typeof tagsByRole]; + const isFunctionCall = + message.role === "assistant" && message.functionCall; + const content = !isFunctionCall + ? message.content + : `Function Call:\n${JSON.stringify(message.functionCall)}`; + + return `<${tag}>\n${content}\n`; + }); +} + +export async function loadPromptExamplePairFromFile( + path: string +): Promise { + const fileDataRaw = await fs.readFile(path, "utf-8"); + const [input, output] = JSON.parse(fileDataRaw); + return PromptExamplePair.parse([JSON.stringify(input), output]); +} + +export function toBulletPoint(text: string) { + return `* ${text}`; +} + +export function asBulletPoints(...lines: string[]) { + return lines.map(toBulletPoint).join("\n"); +} + +export type PromptExamplePair = z.infer; +export const PromptExamplePair = z.tuple([z.string(), z.unknown()]); + +export function formatFewShotExamples(args: { + examples: PromptExamplePair[]; + responseSchema?: ZodTypeAny; + functionName: string; +}) { + return args.examples.flatMap(([input, output], exampleIndex) => { + try { + const parsedOutput = args.responseSchema + ? args.responseSchema.parse(output) + : output; + + return [ + { role: "user", content: input }, + { + role: "assistant", + content: null, + functionCall: { + name: args.functionName, + arguments: JSON.stringify(parsedOutput), + }, + }, + ]; + } catch (error) { + if (error instanceof z.ZodError) { + throw new Error( + [ + `message: Error parsing few shot example at position ${exampleIndex}`, + `error: ${fromError(error)}`, + "given: |", + ...input.split("\n").map((line) => `\t${line}`), + "expected: |", + ...JSON.stringify(output, null, 2) + .split("\n") + .map((line) => `\t${line}`), + ].join("\n") + ); + } + throw error; + } + }) satisfies ChatRequestMessage[]; +} + +export const UserMessageMongoDbGuardrailFunctionSchema = z.object({ + reasoning: z + .string() + .describe( + "Reason for whether to reject the user query. Be concise. Think step by step. " + ), + rejectMessage: z + .boolean() + .describe( + "Set to true if the user query should be rejected. Set to false if the user query should be accepted." + ), +}); + +export type AsJsonSchemaOptions = { + // examples?: PromptExamplePair[]; + zodToJsonSchema?: Parameters[1]; +}; + +export function asJsonSchema( + schema: ZodTypeAny, + options: AsJsonSchemaOptions = {} +) { + if (typeof options.zodToJsonSchema === "string") { + const name = options.zodToJsonSchema; + options.zodToJsonSchema = { name }; + } + const convertedJsonSchema = zodToJsonSchema(schema, { + $refStrategy: "none", + ...(options.zodToJsonSchema ?? {}), + }); + return convertedJsonSchema; +} diff --git a/packages/mongodb-artifact-generator/src/commands/generateJiraPromptResponse.ts b/packages/mongodb-artifact-generator/src/commands/generateJiraPromptResponse.ts new file mode 100644 index 000000000..1c0d5f909 --- /dev/null +++ b/packages/mongodb-artifact-generator/src/commands/generateJiraPromptResponse.ts @@ -0,0 +1,376 @@ +import { + createConfiguredAction, + withConfig, + withConfigOptions, +} from "../withConfig"; +import { createCommand } from "../createCommand"; +import { makeRunLogger, type RunLogger } from "../runlogger"; +import path from "path"; +import { PromisePool } from "@supercharge/promise-pool"; +import { makeSummarizer, Summary } from "../chat/makeSummarizer"; +import { assertEnvVars } from "mongodb-rag-core"; +import { asBulletPoints, loadPromptExamplePairFromFile } from "../chat/utils"; +import { createRunId } from "../runId"; +import { makeGeneratePrompts } from "../chat/makeGeneratePrompts"; +import { makeGenerateResponse } from "../chat/makeGenerateResponse"; +import { GENERATOR_LLM_ENV_VARS } from "../ArtifactGeneratorEnvVars"; +import { promises as fs } from "fs"; +import { z } from "zod"; + +const { GENERATOR_LLM_MAX_CONCURRENCY, OPENAI_CHAT_COMPLETION_DEPLOYMENT } = + assertEnvVars({ + ...GENERATOR_LLM_ENV_VARS, + OPENAI_CHAT_COMPLETION_DEPLOYMENT: "", + }); + +let logger: RunLogger; + +export interface JiraComment { + id: string; + body: string; + author: { + emailAddress: string; + displayName: string; + }; + created: string; + updated: string; +} + +export interface FormattedJiraIssue { + key: string; + projectName: string; + summary: string; + status: string; + created: string; + updated: string; + description: string; + comments: JiraComment[]; +} + +export type FormattedJiraIssueWithSummary = { + issue: FormattedJiraIssue; + summary: Summary; +}; + +type GenerateJiraPromptResponseCommandArgs = { + runId?: string; + llmMaxConcurrency: number; +}; + +export default createCommand({ + command: "generateJiraPromptResponse", + builder(args) { + return withConfigOptions(args) + .option("runId", { + type: "string", + demandOption: false, + description: + "A unique name for the run. This controls where outputs artifacts and logs are stored.", + }) + .option("llmMaxConcurrency", { + type: "number", + demandOption: false, + default: z.coerce + .number() + .default(10) + .parse(GENERATOR_LLM_MAX_CONCURRENCY), + description: + "The maximum number of concurrent requests to the LLM API. Defaults to 10. Can be specified in the config file as GENERATOR_LLM_MAX_CONCURRENCY.", + }); + }, + async handler(args) { + const runId = args.runId ?? createRunId(); + logger = makeRunLogger({ + topic: "GenerateJiraPromptResponse", + runId, + }); + logger.logInfo(`Run ID: ${runId}`); + logger.logInfo(`Running command with args: ${JSON.stringify(args)}`); + try { + const result = await withConfig(action, args); + logger.logInfo(`Success`); + return result; + } finally { + await logger.flushArtifacts(); + await logger.flushLogs(); + } + }, + describe: + "Generate prompt-response pairs that crytallize knowledge from raw jira issues.", +}); + +export const action = + createConfiguredAction( + async ({ jiraApi, openAiClient }, { llmMaxConcurrency }) => { + logger.logInfo(`Setting up...`); + if (!jiraApi) { + throw new Error( + "jiraApi is required. Make sure to define it in the config." + ); + } + if (!openAiClient) { + throw new Error( + "openAiClient is required. Make sure to define it in the config." + ); + } + + const topIssueKeys = z + .array(z.string()) + .parse( + JSON.parse( + await fs.readFile( + path.join(__dirname, "../jiraPromptResponse/topIssues.json"), + "utf-8" + ) + ) + ) + .slice(0, 6); // TODO: Remove slice before merge. This helps make the run faster for testing. + + // Get each issue using a promise pool + const jiraMaxConcurrency = 12; + const { results: jiraIssues, errors: getJiraIssuesErrors } = + await PromisePool.for(topIssueKeys) + .withConcurrency(jiraMaxConcurrency) + .process(async (issueKey) => { + return await jiraApi.getIssue(issueKey); + }); + for (const error of getJiraIssuesErrors) { + logger.logError(`Error fetching issue: ${error.item}`); + } + + logger.appendArtifact("jiraIssues.raw.json", JSON.stringify(jiraIssues)); + + interface RawJiraIssue { + key: string; + fields: { + project: { + name: string; + }; + summary: string; + status: { + name: string; + }; + created: string; + updated: string; + description: string; + comment: { + comments: JiraComment[]; + }; + }; + } + + const formattedJiraIssues = (jiraIssues as RawJiraIssue[]).map( + (issue) => { + return { + key: issue.key, + projectName: issue.fields.project.name, + summary: issue.fields.summary, + status: issue.fields.status.name, + created: issue.fields.created, + updated: issue.fields.updated, + description: issue.fields.description, + comments: issue.fields.comment.comments.map((comment) => { + return { + id: comment.id, + body: comment.body, + author: { + emailAddress: comment.author.emailAddress, + displayName: comment.author.displayName, + }, + created: comment.created, + updated: comment.updated, + }; + }), + } satisfies FormattedJiraIssue; + } + ); + + logger.appendArtifact( + "jiraIssues.formatted.json", + JSON.stringify(formattedJiraIssues) + ); + + // Summarize each issue using a promise pool + const summarizeJiraIssue = makeSummarizer({ + openAi: { + client: openAiClient, + deployment: OPENAI_CHAT_COMPLETION_DEPLOYMENT, + }, + logger, + directions: asBulletPoints( + "The task is to summarize the provided Jira issue.", + "If there is a notable bug or issue being described, include that in the summary.", + "If there are any notable comments, include those in the summary.", + "If there are any notable actions that need to be taken to solve, include specific details about those in the summary." + ), + examples: await Promise.all([ + loadPromptExamplePairFromFile( + path.join( + __dirname, + "../jiraPromptResponse/examples/WORKPLACE-119.json" + ) + ), + loadPromptExamplePairFromFile( + path.join( + __dirname, + "../jiraPromptResponse/examples/ADMIN-10208.json" + ) + ), + ]), + }); + + const summariesByIssueKey = new Map(); + const { errors: summarizeJiraIssueErrors } = await PromisePool.for( + jiraIssues + ) + .withConcurrency(llmMaxConcurrency) + .process(async (issue) => { + const summary = await summarizeJiraIssue({ + input: JSON.stringify(issue), + }); + console.log("summarized issue", issue.key, summary); + summariesByIssueKey.set(issue.key, summary); + }); + for (const error of summarizeJiraIssueErrors) { + logger.logError(`Error summarizing issue: ${error.item}`); + console.log("Error summarizing issue", error.raw); + } + + logger.appendArtifact( + "summaries.json", + JSON.stringify(Object.fromEntries(summariesByIssueKey)) + ); + + // Append summaries to formatted issues + const formattedIssuesWithSummaries = formattedJiraIssues.map((issue) => { + const summary = summariesByIssueKey.get(issue.key); + if (!summary) { + throw new Error(`No summary found for issue ${issue.key}`); + } + return { + issue, + summary, + }; + }) satisfies FormattedJiraIssueWithSummary[]; + logger.appendArtifact( + "jiraIssues.formattedWithSummaries.json", + JSON.stringify(formattedIssuesWithSummaries) + ); + + // // Generate a list of N questions/prompts for each issue + const promptsByIssueKey = new Map(); + const generatePrompts = makeGeneratePrompts({ + openAi: { + client: openAiClient, + deployment: OPENAI_CHAT_COMPLETION_DEPLOYMENT, + }, + logger, + directions: asBulletPoints( + "Assume the prompter is not familiar with or looking for the provided Jira issue specifically.", + "Assume the prompter is looking for a specific piece of information about the topic or bug discussed in the Jira issue.", + "The prompt should be a question that can be answered with the information in the Jira issue but not about the issue itself.", + "If the Jira issue is for a specific language, framework, platform, driver, etc., indicate that in the prompt.", + "Do not reference hypothetical or speculative information." + ), + }); + const { errors: generatePromptsErrors } = await PromisePool.for( + formattedIssuesWithSummaries + ) + .withConcurrency(llmMaxConcurrency) + .process(async ({ issue, summary }) => { + const { prompts } = await generatePrompts({ issue, summary }); + console.log("generated prompts for issue", issue.key, summary); + promptsByIssueKey.set(issue.key, prompts); + }); + for (const error of generatePromptsErrors) { + logger.logError(`Error generating prompts: ${error.item}`); + } + const generatedPrompts: [issueKey: string, prompt: string][] = [ + ...promptsByIssueKey.entries(), + ].flatMap(([issueKey, prompts]) => { + return prompts.map((prompt) => [issueKey, prompt] as [string, string]); + }); + logger.appendArtifact( + "generatedPrompts.json", + JSON.stringify(generatedPrompts) + ); + + // Have the LLM generate a response to each prompt with the formatted/summarized issue as context + const responsesByIssueKey = new Map< + string, + [prompt: string, response: string][] + >(); + + const generateResponse = makeGenerateResponse({ + openAi: { + client: openAiClient, + deployment: OPENAI_CHAT_COMPLETION_DEPLOYMENT, + }, + logger, + directions: asBulletPoints( + "The response should be as accurate as possible based on the information in the Jira issue.", + "You may use your knowledge of the topic to provide a more detailed response. However, avoid answering the main request with information that is not in the Jira issue.", + "Do not make strong claims about MongoDB's product plans or future releases.", + "Do not directly refer to the product backlog or project management tools.", + "Do not reference the names of specific people or teams. If you must refer to a person, use a generic term like 'MongoDB employee', 'community member', 'developer', or 'engineer'." + ), + }); + + const prompts = generatedPrompts.map(([issueKey, prompt]) => { + const summary = summariesByIssueKey.get(issueKey); + if (!summary) { + throw new Error(`No summary found for key ${issueKey}`); + } + const issue = formattedJiraIssues.find( + (issue) => issue.key === issueKey + ); + if (!issue) { + throw new Error(`No issue data found for key ${issueKey}`); + } + return { + summary, + issue, + prompt, + }; + }); + let numGeneratedResponses = 0; + const { errors: generateResponsesErrors } = await PromisePool.for(prompts) + .withConcurrency(llmMaxConcurrency) + .process(async ({ summary, issue, prompt }, i) => { + const { response } = await generateResponse({ + summary, + issue, + prompt, + }); + console.log( + `generated response ${++numGeneratedResponses}/${prompts.length}` + ); + if (!responsesByIssueKey.has(issue.key)) { + responsesByIssueKey.set(issue.key, []); + } + responsesByIssueKey.get(issue.key)?.push([prompt, response]); + }); + for (const error of generateResponsesErrors) { + const { issue } = error.item; + logger.logError( + `Error generating responses for ${issue.key}: ${JSON.stringify( + error + )}` + ); + } + const generatedResponses = [...responsesByIssueKey.entries()].flatMap( + ([issueKey, responses]) => { + return responses.map(([prompt, response]) => ({ + issueKey, + prompt, + response, + })); + } + ); + + logger.appendArtifact( + "generatedResponses.json", + JSON.stringify(generatedResponses) + ); + } + ); diff --git a/packages/mongodb-artifact-generator/src/jiraPromptResponse/examples/ADMIN-10208.json b/packages/mongodb-artifact-generator/src/jiraPromptResponse/examples/ADMIN-10208.json new file mode 100644 index 000000000..2c61cfb87 --- /dev/null +++ b/packages/mongodb-artifact-generator/src/jiraPromptResponse/examples/ADMIN-10208.json @@ -0,0 +1,400 @@ +[ + { + "expand": "renderedFields,names,schema,operations,editmeta,changelog,versionedRepresentations", + "id": "108475", + "self": "https://jira.mongodb.org/rest/agile/1.0/issue/108475", + "key": "ADMIN-10208", + "fields": { + "customfield_14150": { + "self": "https://jira.mongodb.org/rest/api/2/customFieldOption/19783", + "value": "New York City", + "id": "19783", + "disabled": false + }, + "fixVersions": [], + "resolution": { + "self": "https://jira.mongodb.org/rest/api/2/resolution/9", + "id": "9", + "description": "GreenHopper Managed Resolution. Grab-bag resolution for other things (e.g.: re-directing users to google groups)", + "name": "Done" + }, + "lastViewed": null, + "priority": { + "self": "https://jira.mongodb.org/rest/api/2/priority/4", + "iconUrl": "https://jira.mongodb.org/images/icons/priorities/minor.svg", + "name": "Minor - P4", + "id": "4" + }, + "labels": [], + "aggregatetimeoriginalestimate": null, + "timeestimate": null, + "versions": [], + "issuelinks": [], + "assignee": null, + "status": { + "self": "https://jira.mongodb.org/rest/api/2/status/6", + "description": "The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.", + "iconUrl": "https://jira.mongodb.org/images/icons/statuses/closed.png", + "name": "Closed", + "id": "6", + "statusCategory": { + "self": "https://jira.mongodb.org/rest/api/2/statuscategory/3", + "id": 3, + "key": "done", + "colorName": "success", + "name": "Done" + } + }, + "components": [ + { + "self": "https://jira.mongodb.org/rest/api/2/component/11842", + "id": "11842", + "name": "CLI" + } + ], + "customfield_10051": [ + "jimbob@gmail.com(jimbob)", + "somebody.person@10gen.com(somebody.person@10gen.com)", + "excellent.sme@mongodb.com(excellent.sme@mongodb.com)" + ], + "aggregatetimeestimate": null, + "creator": { + "self": "https://jira.mongodb.org/rest/api/2/user?username=jimbob%40gmail.com", + "name": "jimbob@gmail.com", + "key": "jimbob@gmail.com", + "emailAddress": "jimbob@gmail.com", + "avatarUrls": { + "48x48": "https://jira.mongodb.org/secure/useravatar?ownerId=jimbob%40gmail.com&avatarId=11105", + "24x24": "https://jira.mongodb.org/secure/useravatar?size=small&ownerId=jimbob%40gmail.com&avatarId=11105", + "16x16": "https://jira.mongodb.org/secure/useravatar?size=xsmall&ownerId=jimbob%40gmail.com&avatarId=11105", + "32x32": "https://jira.mongodb.org/secure/useravatar?size=medium&ownerId=jimbob%40gmail.com&avatarId=11105" + }, + "displayName": "jimbob", + "active": true, + "timeZone": "America/New_York" + }, + "subtasks": [], + "reporter": { + "self": "https://jira.mongodb.org/rest/api/2/user?username=jimbob%40gmail.com", + "name": "jimbob@gmail.com", + "key": "jimbob@gmail.com", + "emailAddress": "jimbob@gmail.com", + "avatarUrls": { + "48x48": "https://jira.mongodb.org/secure/useravatar?ownerId=jimbob%40gmail.com&avatarId=11105", + "24x24": "https://jira.mongodb.org/secure/useravatar?size=small&ownerId=jimbob%40gmail.com&avatarId=11105", + "16x16": "https://jira.mongodb.org/secure/useravatar?size=xsmall&ownerId=jimbob%40gmail.com&avatarId=11105", + "32x32": "https://jira.mongodb.org/secure/useravatar?size=medium&ownerId=jimbob%40gmail.com&avatarId=11105" + }, + "displayName": "jimbob", + "active": true, + "timeZone": "America/New_York" + }, + "customfield_15850": "{summaryBean=com.atlassian.jira.plugin.devstatus.rest.SummaryBean@27f7a1d5[summary={pullrequest=com.atlassian.jira.plugin.devstatus.rest.SummaryItemBean@5b14819e[byInstanceType={},overall=PullRequestOverallBean{stateCount=0, state='OPEN', details=PullRequestOverallDetails{openCount=0, mergedCount=0, declinedCount=0}}], build=com.atlassian.jira.plugin.devstatus.rest.SummaryItemBean@2513a91f[byInstanceType={},overall=com.atlassian.jira.plugin.devstatus.summary.beans.BuildOverallBean@5af4c452[failedBuildCount=0,successfulBuildCount=0,unknownBuildCount=0,count=0,lastUpdated=,lastUpdatedTimestamp=]], review=com.atlassian.jira.plugin.devstatus.rest.SummaryItemBean@3075b425[byInstanceType={},overall=com.atlassian.jira.plugin.devstatus.summary.beans.ReviewsOverallBean@dc905a0[dueDate=,overDue=false,state=,stateCount=0,count=0,lastUpdated=,lastUpdatedTimestamp=]], deployment-environment=com.atlassian.jira.plugin.devstatus.rest.SummaryItemBean@352a1e57[byInstanceType={},overall=com.atlassian.jira.plugin.devstatus.summary.beans.DeploymentOverallBean@49809af4[showProjects=false,successfulCount=0,topEnvironments=[],count=0,lastUpdated=,lastUpdatedTimestamp=]], repository=com.atlassian.jira.plugin.devstatus.rest.SummaryItemBean@54c9db7d[byInstanceType={},overall=com.atlassian.jira.plugin.devstatus.summary.beans.CommitOverallBean@11e82041[count=0,lastUpdated=,lastUpdatedTimestamp=]], branch=com.atlassian.jira.plugin.devstatus.rest.SummaryItemBean@74555c8d[byInstanceType={},overall=com.atlassian.jira.plugin.devstatus.summary.beans.BranchOverallBean@394f3fc9[count=0,lastUpdated=,lastUpdatedTimestamp=]]},configErrors=[],errors=[]], devSummaryJson={\"cachedValue\":{\"errors\":[],\"configErrors\":[],\"summary\":{\"pullrequest\":{\"overall\":{\"count\":0,\"lastUpdated\":null,\"stateCount\":0,\"state\":\"OPEN\",\"details\":{\"openCount\":0,\"mergedCount\":0,\"declinedCount\":0,\"total\":0},\"open\":true},\"byInstanceType\":{}},\"build\":{\"overall\":{\"count\":0,\"lastUpdated\":null,\"failedBuildCount\":0,\"successfulBuildCount\":0,\"unknownBuildCount\":0},\"byInstanceType\":{}},\"review\":{\"overall\":{\"count\":0,\"lastUpdated\":null,\"stateCount\":0,\"state\":null,\"dueDate\":null,\"overDue\":false,\"completed\":false},\"byInstanceType\":{}},\"deployment-environment\":{\"overall\":{\"count\":0,\"lastUpdated\":null,\"topEnvironments\":[],\"showProjects\":false,\"successfulCount\":0},\"byInstanceType\":{}},\"repository\":{\"overall\":{\"count\":0,\"lastUpdated\":null},\"byInstanceType\":{}},\"branch\":{\"overall\":{\"count\":0,\"lastUpdated\":null},\"byInstanceType\":{}}}},\"isStale\":false}}", + "aggregateprogress": { "progress": 0, "total": 0 }, + "progress": { "progress": 0, "total": 0 }, + "votes": { + "self": "https://jira.mongodb.org/rest/api/2/issue/ADMIN-10208/votes", + "votes": 10, + "hasVoted": false + }, + "worklog": { "startAt": 0, "maxResults": 20, "total": 0, "worklogs": [] }, + "archivedby": null, + "issuetype": { + "self": "https://jira.mongodb.org/rest/api/2/issuetype/4", + "id": "4", + "description": "An improvement or enhancement to an existing feature or task.", + "iconUrl": "https://jira.mongodb.org/secure/viewavatar?size=xsmall&avatarId=14710&avatarType=issuetype", + "name": "Improvement", + "subtask": false, + "avatarId": 14710 + }, + "timespent": null, + "project": { + "self": "https://jira.mongodb.org/rest/api/2/project/11281", + "id": "11281", + "key": "ADMIN", + "name": "Admin Server", + "projectTypeKey": "service_desk", + "avatarUrls": { + "48x48": "https://jira.mongodb.org/secure/projectavatar?pid=11281&avatarId=17807", + "24x24": "https://jira.mongodb.org/secure/projectavatar?size=small&pid=11281&avatarId=17807", + "16x16": "https://jira.mongodb.org/secure/projectavatar?size=xsmall&pid=11281&avatarId=17807", + "32x32": "https://jira.mongodb.org/secure/projectavatar?size=medium&pid=11281&avatarId=17807" + }, + "projectCategory": { + "self": "https://jira.mongodb.org/rest/api/2/projectCategory/12111", + "id": "12111", + "description": "", + "name": "FinOps" + } + }, + "resolutiondate": "2014-03-20T17:21:15.000+0000", + "workratio": -1, + "watches": { + "self": "https://jira.mongodb.org/rest/api/2/issue/ADMIN-10208/watchers", + "watchCount": 10, + "isWatching": false + }, + "created": "2018-01-30T22:22:56.000+0000", + "updated": "2024-06-01T02:23:47.000+0000", + "timeoriginalestimate": null, + "summary": "Server is returning error when creating a new user", + "description": "The server is returning an error when creating a new user. The error message is: \"Error: User already exists.\". This is a problem because we need to be able to create new users. I don't think this user actually exists!", + "timetracking": {}, + "attachment": [], + "flagged": false, + "customfield_11452": { + "id": "34", + "name": "Time to first response", + "_links": { + "self": "https://jira.mongodb.org/rest/servicedeskapi/request/108475/sla/34" + }, + "completedCycles": [] + }, + "customfield_11451": { + "id": "33", + "name": "Time to resolution", + "_links": { + "self": "https://jira.mongodb.org/rest/servicedeskapi/request/108475/sla/33" + }, + "completedCycles": [] + }, + "environment": "Men's room", + "duedate": null, + "comment": { + "comments": [ + { + "self": "https://jira.mongodb.org/rest/api/2/issue/108475/comment/491172", + "id": "491172", + "author": { + "self": "https://jira.mongodb.org/rest/api/2/user?username=somebody.person%4010gen.com", + "name": "somebody.person@10gen.com", + "key": "somebody.person@10gen.com", + "emailAddress": "somebody.person@10gen.com", + "avatarUrls": { + "48x48": "https://jira.mongodb.org/secure/useravatar?avatarId=10123", + "24x24": "https://jira.mongodb.org/secure/useravatar?size=small&avatarId=10123", + "16x16": "https://jira.mongodb.org/secure/useravatar?size=xsmall&avatarId=10123", + "32x32": "https://jira.mongodb.org/secure/useravatar?size=medium&avatarId=10123" + }, + "displayName": "somebody.person@10gen.com", + "active": false, + "timeZone": "UTC" + }, + "body": "This happens in the UI sometimes too", + "updateAuthor": { + "self": "https://jira.mongodb.org/rest/api/2/user?username=somebody.person%4010gen.com", + "name": "somebody.person@10gen.com", + "key": "somebody.person@10gen.com", + "emailAddress": "?", + "avatarUrls": { + "48x48": "https://jira.mongodb.org/secure/useravatar?avatarId=10123", + "24x24": "https://jira.mongodb.org/secure/useravatar?size=small&avatarId=10123", + "16x16": "https://jira.mongodb.org/secure/useravatar?size=xsmall&avatarId=10123", + "32x32": "https://jira.mongodb.org/secure/useravatar?size=medium&avatarId=10123" + }, + "displayName": "somebody.person@10gen.com", + "active": false, + "timeZone": "UTC" + }, + "created": "2014-01-30T22:24:55.000+0000", + "updated": "2014-01-30T22:24:55.000+0000" + }, + { + "self": "https://jira.mongodb.org/rest/api/2/issue/108475/comment/491173", + "id": "491173", + "author": { + "self": "https://jira.mongodb.org/rest/api/2/user?username=triager%40mongodb.com", + "name": "triager@mongodb.com", + "key": "triager@gmail.com", + "emailAddress": "jimbob@gmail.com", + "avatarUrls": { + "48x48": "https://jira.mongodb.org/secure/useravatar?ownerId=triager%40gmail.com&avatarId=11105", + "24x24": "https://jira.mongodb.org/secure/useravatar?size=small&ownerId=triager%40gmail.com&avatarId=11105", + "16x16": "https://jira.mongodb.org/secure/useravatar?size=xsmall&ownerId=triager%40gmail.com&avatarId=11105", + "32x32": "https://jira.mongodb.org/secure/useravatar?size=medium&ownerId=triager%40gmail.com&avatarId=11105" + }, + "displayName": "jimbob", + "active": true, + "timeZone": "America/New_York" + }, + "body": "[~somebody.person@10gen.com]: That's weird. I wonder if it's related to the issue we had with the server last week. Do you have a reproduction case? cc [~excellent.sme@mongodb.com] any ideas?", + "updateAuthor": { + "self": "https://jira.mongodb.org/rest/api/2/user?username=triager%40mongodb.com", + "name": "triager@mongodb.com", + "key": "triager@gmail.com", + "emailAddress": "triager@mongodb.com", + "avatarUrls": { + "48x48": "https://jira.mongodb.org/secure/useravatar?ownerId=triager%40gmail.com&avatarId=11105", + "24x24": "https://jira.mongodb.org/secure/useravatar?size=small&ownerId=triager%40gmail.com&avatarId=11105", + "16x16": "https://jira.mongodb.org/secure/useravatar?size=xsmall&ownerId=triager%40gmail.com&avatarId=11105", + "32x32": "https://jira.mongodb.org/secure/useravatar?size=medium&ownerId=triager%40gmail.com&avatarId=11105" + }, + "displayName": "Triager", + "active": true, + "timeZone": "America/New_York" + }, + "created": "2014-01-30T22:26:53.000+0000", + "updated": "2014-01-30T22:26:53.000+0000" + }, + { + "self": "https://jira.mongodb.org/rest/api/2/issue/108475/comment/491176", + "id": "491176", + "author": { + "self": "https://jira.mongodb.org/rest/api/2/user?username=somebody.person%40mongodb.com", + "name": "somebody.person@mongodb.com", + "key": "somebody.person", + "emailAddress": "somebody.person@mongodb.com", + "avatarUrls": { + "48x48": "https://jira.mongodb.org/secure/useravatar?ownerId=somebody.person&avatarId=11409", + "24x24": "https://jira.mongodb.org/secure/useravatar?size=small&ownerId=somebody.person&avatarId=11409", + "16x16": "https://jira.mongodb.org/secure/useravatar?size=xsmall&ownerId=somebody.person&avatarId=11409", + "32x32": "https://jira.mongodb.org/secure/useravatar?size=medium&ownerId=somebody.person&avatarId=11409" + }, + "displayName": "somebody.person", + "active": true, + "timeZone": "America/New_York" + }, + "body": "Maybe a caching thing? A repro case would help.", + "updateAuthor": { + "self": "https://jira.mongodb.org/rest/api/2/user?username=somebody.person%40mongodb.com", + "name": "somebody.person@mongodb.com", + "key": "somebody.person", + "emailAddress": "somebody.person@mongodb.com", + "avatarUrls": { + "48x48": "https://jira.mongodb.org/secure/useravatar?ownerId=somebody.person&avatarId=11409", + "24x24": "https://jira.mongodb.org/secure/useravatar?size=small&ownerId=somebody.person&avatarId=11409", + "16x16": "https://jira.mongodb.org/secure/useravatar?size=xsmall&ownerId=somebody.person&avatarId=11409", + "32x32": "https://jira.mongodb.org/secure/useravatar?size=medium&ownerId=somebody.person&avatarId=11409" + }, + "displayName": "somebody.person", + "active": true, + "timeZone": "America/New_York" + }, + "created": "2014-01-30T22:31:38.000+0000", + "updated": "2014-01-30T22:31:38.000+0000" + }, + { + "self": "https://jira.mongodb.org/rest/api/2/issue/108475/comment/518934", + "id": "518934", + "author": { + "self": "https://jira.mongodb.org/rest/api/2/user?username=excellent.sme%40mongodb.com", + "name": "excellent.sme@mongodb.com", + "key": "excellent.sme", + "emailAddress": "excellent.sme@mongodb.com", + "avatarUrls": { + "48x48": "https://jira.mongodb.org/secure/useravatar?ownerId=excellent.sme&avatarId=11409", + "24x24": "https://jira.mongodb.org/secure/useravatar?size=small&ownerId=excellent.sme&avatarId=11409", + "16x16": "https://jira.mongodb.org/secure/useravatar?size=xsmall&ownerId=excellent.sme&avatarId=11409", + "32x32": "https://jira.mongodb.org/secure/useravatar?size=medium&ownerId=excellent.sme&avatarId=11409" + }, + "displayName": "excellent.sme", + "active": true, + "timeZone": "America/New_York" + }, + "body": "This could definitely be a cache thing - I ran into this on my test rig just now. Try running admin local cache delete and see if that helps.", + "updateAuthor": { + "self": "https://jira.mongodb.org/rest/api/2/user?username=excellent.sme%40mongodb.com", + "name": "excellent.sme@mongodb.com", + "key": "excellent.sme", + "emailAddress": "excellent.sme@mongodb.com", + "avatarUrls": { + "48x48": "https://jira.mongodb.org/secure/useravatar?ownerId=excellent.sme&avatarId=11409", + "24x24": "https://jira.mongodb.org/secure/useravatar?size=small&ownerId=excellent.sme&avatarId=11409", + "16x16": "https://jira.mongodb.org/secure/useravatar?size=xsmall&ownerId=excellent.sme&avatarId=11409", + "32x32": "https://jira.mongodb.org/secure/useravatar?size=medium&ownerId=excellent.sme&avatarId=11409" + }, + "displayName": "excellent.sme", + "active": true, + "timeZone": "America/New_York" + }, + "created": "2014-03-19T00:43:05.000+0000", + "updated": "2014-03-19T00:43:05.000+0000" + }, + { + "self": "https://jira.mongodb.org/rest/api/2/issue/108475/comment/518985", + "id": "518985", + "author": { + "self": "https://jira.mongodb.org/rest/api/2/user?username=somebody.person%40mongodb.com", + "name": "somebody.person@mongodb.com", + "key": "somebody.person@gmail.com", + "emailAddress": "somebody.person@mongodb.com", + "avatarUrls": { + "48x48": "https://jira.mongodb.org/secure/useravatar?ownerId=somebody.person%40gmail.com&avatarId=11105", + "24x24": "https://jira.mongodb.org/secure/useravatar?size=small&ownerId=somebody.person%40gmail.com&avatarId=11105", + "16x16": "https://jira.mongodb.org/secure/useravatar?size=xsmall&ownerId=somebody.person%40gmail.com&avatarId=11105", + "32x32": "https://jira.mongodb.org/secure/useravatar?size=medium&ownerId=somebody.person%40gmail.com&avatarId=11105" + }, + "displayName": "somebody.person%40mongodb.com", + "active": true, + "timeZone": "America/New_York" + }, + "body": "Nice! Please let us know if that works [~somebody.person@40gmail.com]", + "updateAuthor": { + "self": "https://jira.mongodb.org/rest/api/2/user?username=somebody.person%40mongodb.com", + "name": "somebody.person@mongodb.com", + "key": "somebody.person@gmail.com", + "emailAddress": "somebody.person@mongodb.com", + "avatarUrls": { + "48x48": "https://jira.mongodb.org/secure/useravatar?ownerId=somebody.person%40gmail.com&avatarId=11105", + "24x24": "https://jira.mongodb.org/secure/useravatar?size=small&ownerId=somebody.person%40gmail.com&avatarId=11105", + "16x16": "https://jira.mongodb.org/secure/useravatar?size=xsmall&ownerId=somebody.person%40gmail.com&avatarId=11105", + "32x32": "https://jira.mongodb.org/secure/useravatar?size=medium&ownerId=somebody.person%40gmail.com&avatarId=11105" + }, + "displayName": "somebody.person%40mongodb.com", + "active": true, + "timeZone": "America/New_York" + }, + "created": "2014-03-19T04:54:25.000+0000", + "updated": "2014-03-19T04:54:25.000+0000" + }, + + { + "self": "https://jira.mongodb.org/rest/api/2/issue/108475/comment/518985", + "id": "518985", + "author": { + "self": "https://jira.mongodb.org/rest/api/2/user?username=jimbob%40gmail.com", + "name": "jimbob@gmail.com", + "key": "jimbob@gmail.com", + "emailAddress": "jimbob@gmail.com", + "avatarUrls": { + "48x48": "https://jira.mongodb.org/secure/useravatar?ownerId=jimbob%40gmail.com&avatarId=11105", + "24x24": "https://jira.mongodb.org/secure/useravatar?size=small&ownerId=jimbob%40gmail.com&avatarId=11105", + "16x16": "https://jira.mongodb.org/secure/useravatar?size=xsmall&ownerId=jimbob%40gmail.com&avatarId=11105", + "32x32": "https://jira.mongodb.org/secure/useravatar?size=medium&ownerId=jimbob%40gmail.com&avatarId=11105" + }, + "displayName": "jimbob", + "active": true, + "timeZone": "America/New_York" + }, + "body": "Yes that solved it thanks!", + "updateAuthor": { + "self": "https://jira.mongodb.org/rest/api/2/user?username=jimbob%40gmail.com", + "name": "jimbob@gmail.com", + "key": "jimbob@gmail.com", + "emailAddress": "jimbob@gmail.com", + "avatarUrls": { + "48x48": "https://jira.mongodb.org/secure/useravatar?ownerId=jimbob%40gmail.com&avatarId=11105", + "24x24": "https://jira.mongodb.org/secure/useravatar?size=small&ownerId=jimbob%40gmail.com&avatarId=11105", + "16x16": "https://jira.mongodb.org/secure/useravatar?size=xsmall&ownerId=jimbob%40gmail.com&avatarId=11105", + "32x32": "https://jira.mongodb.org/secure/useravatar?size=medium&ownerId=jimbob%40gmail.com&avatarId=11105" + }, + "displayName": "jimbob", + "active": true, + "timeZone": "America/New_York" + }, + "created": "2014-03-19T04:54:25.000+0000", + "updated": "2014-03-19T04:54:25.000+0000" + } + ], + "maxResults": 7, + "total": 7, + "startAt": 0 + } + } + }, + { + "topics": ["server error", "user creation", "cache issue", "Admin Server"], + "description": "The issue reports a server error when creating a new user, with the message 'Error: User already exists.' The reporter believes the user does not actually exist. Comments suggest it might be a caching issue. The solution, confirmed to work by the reporter, involves clearing the cache by running `admin local cache delete`." + } +] diff --git a/packages/mongodb-artifact-generator/src/jiraPromptResponse/examples/WORKPLACE-119.json b/packages/mongodb-artifact-generator/src/jiraPromptResponse/examples/WORKPLACE-119.json new file mode 100644 index 000000000..f1b09f029 --- /dev/null +++ b/packages/mongodb-artifact-generator/src/jiraPromptResponse/examples/WORKPLACE-119.json @@ -0,0 +1,470 @@ +[ + { + "expand": "renderedFields,names,schema,operations,editmeta,changelog,versionedRepresentations", + "id": "108475", + "self": "https://jira.mongodb.org/rest/agile/1.0/issue/108475", + "key": "WORKPLACE-119", + "fields": { + "customfield_14150": { + "self": "https://jira.mongodb.org/rest/api/2/customFieldOption/19783", + "value": "New York City", + "id": "19783", + "disabled": false + }, + "fixVersions": [], + "resolution": { + "self": "https://jira.mongodb.org/rest/api/2/resolution/9", + "id": "9", + "description": "GreenHopper Managed Resolution. Grab-bag resolution for other things (e.g.: re-directing users to google groups)", + "name": "Done" + }, + "lastViewed": null, + "priority": { + "self": "https://jira.mongodb.org/rest/api/2/priority/4", + "iconUrl": "https://jira.mongodb.org/images/icons/priorities/minor.svg", + "name": "Minor - P4", + "id": "4" + }, + "labels": [], + "aggregatetimeoriginalestimate": null, + "timeestimate": null, + "versions": [], + "issuelinks": [], + "assignee": null, + "status": { + "self": "https://jira.mongodb.org/rest/api/2/status/6", + "description": "The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.", + "iconUrl": "https://jira.mongodb.org/images/icons/statuses/closed.png", + "name": "Closed", + "id": "6", + "statusCategory": { + "self": "https://jira.mongodb.org/rest/api/2/statuscategory/3", + "id": 3, + "key": "done", + "colorName": "success", + "name": "Done" + } + }, + "components": [ + { + "self": "https://jira.mongodb.org/rest/api/2/component/11842", + "id": "11842", + "name": "New York Office" + } + ], + "customfield_10051": [ + "akshay@mongodb.com(akshay)", + "carol.huang@10gen.com(carol.huang@10gen.com)", + "isabelle.davis(isabelle.davis)", + "jmikola@mongodb.com(jmikola@gmail.com)", + "kym.ganade(kym@10gen.com)" + ], + "aggregatetimeestimate": null, + "creator": { + "self": "https://jira.mongodb.org/rest/api/2/user?username=jmikola%40mongodb.com", + "name": "jmikola@mongodb.com", + "key": "jmikola@gmail.com", + "emailAddress": "jmikola@mongodb.com", + "avatarUrls": { + "48x48": "https://jira.mongodb.org/secure/useravatar?ownerId=jmikola%40gmail.com&avatarId=11105", + "24x24": "https://jira.mongodb.org/secure/useravatar?size=small&ownerId=jmikola%40gmail.com&avatarId=11105", + "16x16": "https://jira.mongodb.org/secure/useravatar?size=xsmall&ownerId=jmikola%40gmail.com&avatarId=11105", + "32x32": "https://jira.mongodb.org/secure/useravatar?size=medium&ownerId=jmikola%40gmail.com&avatarId=11105" + }, + "displayName": "Jeremy Mikola", + "active": true, + "timeZone": "America/New_York" + }, + "subtasks": [], + "reporter": { + "self": "https://jira.mongodb.org/rest/api/2/user?username=jmikola%40mongodb.com", + "name": "jmikola@mongodb.com", + "key": "jmikola@gmail.com", + "emailAddress": "jmikola@mongodb.com", + "avatarUrls": { + "48x48": "https://jira.mongodb.org/secure/useravatar?ownerId=jmikola%40gmail.com&avatarId=11105", + "24x24": "https://jira.mongodb.org/secure/useravatar?size=small&ownerId=jmikola%40gmail.com&avatarId=11105", + "16x16": "https://jira.mongodb.org/secure/useravatar?size=xsmall&ownerId=jmikola%40gmail.com&avatarId=11105", + "32x32": "https://jira.mongodb.org/secure/useravatar?size=medium&ownerId=jmikola%40gmail.com&avatarId=11105" + }, + "displayName": "Jeremy Mikola", + "active": true, + "timeZone": "America/New_York" + }, + "customfield_15850": "{summaryBean=com.atlassian.jira.plugin.devstatus.rest.SummaryBean@27f7a1d5[summary={pullrequest=com.atlassian.jira.plugin.devstatus.rest.SummaryItemBean@5b14819e[byInstanceType={},overall=PullRequestOverallBean{stateCount=0, state='OPEN', details=PullRequestOverallDetails{openCount=0, mergedCount=0, declinedCount=0}}], build=com.atlassian.jira.plugin.devstatus.rest.SummaryItemBean@2513a91f[byInstanceType={},overall=com.atlassian.jira.plugin.devstatus.summary.beans.BuildOverallBean@5af4c452[failedBuildCount=0,successfulBuildCount=0,unknownBuildCount=0,count=0,lastUpdated=,lastUpdatedTimestamp=]], review=com.atlassian.jira.plugin.devstatus.rest.SummaryItemBean@3075b425[byInstanceType={},overall=com.atlassian.jira.plugin.devstatus.summary.beans.ReviewsOverallBean@dc905a0[dueDate=,overDue=false,state=,stateCount=0,count=0,lastUpdated=,lastUpdatedTimestamp=]], deployment-environment=com.atlassian.jira.plugin.devstatus.rest.SummaryItemBean@352a1e57[byInstanceType={},overall=com.atlassian.jira.plugin.devstatus.summary.beans.DeploymentOverallBean@49809af4[showProjects=false,successfulCount=0,topEnvironments=[],count=0,lastUpdated=,lastUpdatedTimestamp=]], repository=com.atlassian.jira.plugin.devstatus.rest.SummaryItemBean@54c9db7d[byInstanceType={},overall=com.atlassian.jira.plugin.devstatus.summary.beans.CommitOverallBean@11e82041[count=0,lastUpdated=,lastUpdatedTimestamp=]], branch=com.atlassian.jira.plugin.devstatus.rest.SummaryItemBean@74555c8d[byInstanceType={},overall=com.atlassian.jira.plugin.devstatus.summary.beans.BranchOverallBean@394f3fc9[count=0,lastUpdated=,lastUpdatedTimestamp=]]},configErrors=[],errors=[]], devSummaryJson={\"cachedValue\":{\"errors\":[],\"configErrors\":[],\"summary\":{\"pullrequest\":{\"overall\":{\"count\":0,\"lastUpdated\":null,\"stateCount\":0,\"state\":\"OPEN\",\"details\":{\"openCount\":0,\"mergedCount\":0,\"declinedCount\":0,\"total\":0},\"open\":true},\"byInstanceType\":{}},\"build\":{\"overall\":{\"count\":0,\"lastUpdated\":null,\"failedBuildCount\":0,\"successfulBuildCount\":0,\"unknownBuildCount\":0},\"byInstanceType\":{}},\"review\":{\"overall\":{\"count\":0,\"lastUpdated\":null,\"stateCount\":0,\"state\":null,\"dueDate\":null,\"overDue\":false,\"completed\":false},\"byInstanceType\":{}},\"deployment-environment\":{\"overall\":{\"count\":0,\"lastUpdated\":null,\"topEnvironments\":[],\"showProjects\":false,\"successfulCount\":0},\"byInstanceType\":{}},\"repository\":{\"overall\":{\"count\":0,\"lastUpdated\":null},\"byInstanceType\":{}},\"branch\":{\"overall\":{\"count\":0,\"lastUpdated\":null},\"byInstanceType\":{}}}},\"isStale\":false}}", + "aggregateprogress": { "progress": 0, "total": 0 }, + "progress": { "progress": 0, "total": 0 }, + "votes": { + "self": "https://jira.mongodb.org/rest/api/2/issue/WORKPLACE-119/votes", + "votes": 10, + "hasVoted": false + }, + "worklog": { "startAt": 0, "maxResults": 20, "total": 0, "worklogs": [] }, + "archivedby": null, + "issuetype": { + "self": "https://jira.mongodb.org/rest/api/2/issuetype/4", + "id": "4", + "description": "An improvement or enhancement to an existing feature or task.", + "iconUrl": "https://jira.mongodb.org/secure/viewavatar?size=xsmall&avatarId=14710&avatarType=issuetype", + "name": "Improvement", + "subtask": false, + "avatarId": 14710 + }, + "timespent": null, + "project": { + "self": "https://jira.mongodb.org/rest/api/2/project/11281", + "id": "11281", + "key": "WORKPLACE", + "name": "Workplace & Real Estate", + "projectTypeKey": "service_desk", + "avatarUrls": { + "48x48": "https://jira.mongodb.org/secure/projectavatar?pid=11281&avatarId=17807", + "24x24": "https://jira.mongodb.org/secure/projectavatar?size=small&pid=11281&avatarId=17807", + "16x16": "https://jira.mongodb.org/secure/projectavatar?size=xsmall&pid=11281&avatarId=17807", + "32x32": "https://jira.mongodb.org/secure/projectavatar?size=medium&pid=11281&avatarId=17807" + }, + "projectCategory": { + "self": "https://jira.mongodb.org/rest/api/2/projectCategory/12111", + "id": "12111", + "description": "", + "name": "FinOps" + } + }, + "resolutiondate": "2014-03-20T17:21:15.000+0000", + "workratio": -1, + "watches": { + "self": "https://jira.mongodb.org/rest/api/2/issue/WORKPLACE-119/watchers", + "watchCount": 10, + "isWatching": false + }, + "created": "2014-01-30T22:22:56.000+0000", + "updated": "2020-06-01T02:23:47.000+0000", + "timeoriginalestimate": null, + "description": "!stall-gap.jpg!\n\nThere is a noticeable gap between the stall panels and the tile wall in the men's room (as illustrated in the left-most portion of the diagram). In the past, some resourceful individuals have hung tissue paper to cover the gap. Unfortunately, that is not a sustainable solution and the janitors tend to remove it (as illustrated in the right-most portion of the diagram).\n\nA somewhat related issue is the stability of the stall door locks. Specifically on the pair of stalls nearest the exit. Oftentimes, when someone enters or leaves an adjacent stall, a door lock will come loose. If the stall panels end up being realigned, it might be reasonable to also re-calibrate the door locks so they are not prone to becoming unlocked position so easily. ([~akshay] may be able to attest to this issue). Do let me know if I should open a separate issue for that, or if we should make it a sub-task of this issue.\n\nThank you,", + "timetracking": {}, + "attachment": [ + { + "self": "https://jira.mongodb.org/rest/api/2/attachment/37000", + "id": "37000", + "filename": "stall-gap.jpg", + "author": { + "self": "https://jira.mongodb.org/rest/api/2/user?username=jmikola%40mongodb.com", + "name": "jmikola@mongodb.com", + "key": "jmikola@gmail.com", + "emailAddress": "jmikola@mongodb.com", + "avatarUrls": { + "48x48": "https://jira.mongodb.org/secure/useravatar?ownerId=jmikola%40gmail.com&avatarId=11105", + "24x24": "https://jira.mongodb.org/secure/useravatar?size=small&ownerId=jmikola%40gmail.com&avatarId=11105", + "16x16": "https://jira.mongodb.org/secure/useravatar?size=xsmall&ownerId=jmikola%40gmail.com&avatarId=11105", + "32x32": "https://jira.mongodb.org/secure/useravatar?size=medium&ownerId=jmikola%40gmail.com&avatarId=11105" + }, + "displayName": "Jeremy Mikola", + "active": true, + "timeZone": "America/New_York" + }, + "created": "2014-01-30T22:22:56.000+0000", + "size": 144169, + "mimeType": "image/jpeg", + "content": "https://jira.mongodb.org/secure/attachment/37000/stall-gap.jpg", + "thumbnail": "https://jira.mongodb.org/secure/thumbnail/37000/_thumb_37000.png" + } + ], + "flagged": false, + "summary": "Significant stall panel gap in the men's room", + "customfield_11452": { + "id": "34", + "name": "Time to first response", + "_links": { + "self": "https://jira.mongodb.org/rest/servicedeskapi/request/108475/sla/34" + }, + "completedCycles": [] + }, + "customfield_11451": { + "id": "33", + "name": "Time to resolution", + "_links": { + "self": "https://jira.mongodb.org/rest/servicedeskapi/request/108475/sla/33" + }, + "completedCycles": [] + }, + "environment": "Men's room", + "duedate": null, + "comment": { + "comments": [ + { + "self": "https://jira.mongodb.org/rest/api/2/issue/108475/comment/491172", + "id": "491172", + "author": { + "self": "https://jira.mongodb.org/rest/api/2/user?username=carol.huang%4010gen.com", + "name": "carol.huang@10gen.com", + "key": "carol.huang@10gen.com", + "emailAddress": "?", + "avatarUrls": { + "48x48": "https://jira.mongodb.org/secure/useravatar?avatarId=10123", + "24x24": "https://jira.mongodb.org/secure/useravatar?size=small&avatarId=10123", + "16x16": "https://jira.mongodb.org/secure/useravatar?size=xsmall&avatarId=10123", + "32x32": "https://jira.mongodb.org/secure/useravatar?size=medium&avatarId=10123" + }, + "displayName": "carol.huang@10gen.com", + "active": false, + "timeZone": "UTC" + }, + "body": "The gap is also an issue in the women's restroom, for the record. ", + "updateAuthor": { + "self": "https://jira.mongodb.org/rest/api/2/user?username=carol.huang%4010gen.com", + "name": "carol.huang@10gen.com", + "key": "carol.huang@10gen.com", + "emailAddress": "?", + "avatarUrls": { + "48x48": "https://jira.mongodb.org/secure/useravatar?avatarId=10123", + "24x24": "https://jira.mongodb.org/secure/useravatar?size=small&avatarId=10123", + "16x16": "https://jira.mongodb.org/secure/useravatar?size=xsmall&avatarId=10123", + "32x32": "https://jira.mongodb.org/secure/useravatar?size=medium&avatarId=10123" + }, + "displayName": "carol.huang@10gen.com", + "active": false, + "timeZone": "UTC" + }, + "created": "2014-01-30T22:24:55.000+0000", + "updated": "2014-01-30T22:24:55.000+0000" + }, + { + "self": "https://jira.mongodb.org/rest/api/2/issue/108475/comment/491173", + "id": "491173", + "author": { + "self": "https://jira.mongodb.org/rest/api/2/user?username=jmikola%40mongodb.com", + "name": "jmikola@mongodb.com", + "key": "jmikola@gmail.com", + "emailAddress": "jmikola@mongodb.com", + "avatarUrls": { + "48x48": "https://jira.mongodb.org/secure/useravatar?ownerId=jmikola%40gmail.com&avatarId=11105", + "24x24": "https://jira.mongodb.org/secure/useravatar?size=small&ownerId=jmikola%40gmail.com&avatarId=11105", + "16x16": "https://jira.mongodb.org/secure/useravatar?size=xsmall&ownerId=jmikola%40gmail.com&avatarId=11105", + "32x32": "https://jira.mongodb.org/secure/useravatar?size=medium&ownerId=jmikola%40gmail.com&avatarId=11105" + }, + "displayName": "Jeremy Mikola", + "active": true, + "timeZone": "America/New_York" + }, + "body": "[~carol.huang@10gen.com]: Thank you for coming forward. I've heard rumors but this is the first confirmation I've had.", + "updateAuthor": { + "self": "https://jira.mongodb.org/rest/api/2/user?username=jmikola%40mongodb.com", + "name": "jmikola@mongodb.com", + "key": "jmikola@gmail.com", + "emailAddress": "jmikola@mongodb.com", + "avatarUrls": { + "48x48": "https://jira.mongodb.org/secure/useravatar?ownerId=jmikola%40gmail.com&avatarId=11105", + "24x24": "https://jira.mongodb.org/secure/useravatar?size=small&ownerId=jmikola%40gmail.com&avatarId=11105", + "16x16": "https://jira.mongodb.org/secure/useravatar?size=xsmall&ownerId=jmikola%40gmail.com&avatarId=11105", + "32x32": "https://jira.mongodb.org/secure/useravatar?size=medium&ownerId=jmikola%40gmail.com&avatarId=11105" + }, + "displayName": "Jeremy Mikola", + "active": true, + "timeZone": "America/New_York" + }, + "created": "2014-01-30T22:26:53.000+0000", + "updated": "2014-01-30T22:26:53.000+0000" + }, + { + "self": "https://jira.mongodb.org/rest/api/2/issue/108475/comment/491176", + "id": "491176", + "author": { + "self": "https://jira.mongodb.org/rest/api/2/user?username=akshay%40mongodb.com", + "name": "akshay@mongodb.com", + "key": "akshay", + "emailAddress": "akshay@mongodb.com", + "avatarUrls": { + "48x48": "https://jira.mongodb.org/secure/useravatar?ownerId=akshay&avatarId=11409", + "24x24": "https://jira.mongodb.org/secure/useravatar?size=small&ownerId=akshay&avatarId=11409", + "16x16": "https://jira.mongodb.org/secure/useravatar?size=xsmall&ownerId=akshay&avatarId=11409", + "32x32": "https://jira.mongodb.org/secure/useravatar?size=medium&ownerId=akshay&avatarId=11409" + }, + "displayName": "Akshay Kumar", + "active": true, + "timeZone": "America/New_York" + }, + "body": "ProTip: Use the Westin across the street.", + "updateAuthor": { + "self": "https://jira.mongodb.org/rest/api/2/user?username=akshay%40mongodb.com", + "name": "akshay@mongodb.com", + "key": "akshay", + "emailAddress": "akshay@mongodb.com", + "avatarUrls": { + "48x48": "https://jira.mongodb.org/secure/useravatar?ownerId=akshay&avatarId=11409", + "24x24": "https://jira.mongodb.org/secure/useravatar?size=small&ownerId=akshay&avatarId=11409", + "16x16": "https://jira.mongodb.org/secure/useravatar?size=xsmall&ownerId=akshay&avatarId=11409", + "32x32": "https://jira.mongodb.org/secure/useravatar?size=medium&ownerId=akshay&avatarId=11409" + }, + "displayName": "Akshay Kumar", + "active": true, + "timeZone": "America/New_York" + }, + "created": "2014-01-30T22:31:38.000+0000", + "updated": "2014-01-30T22:31:38.000+0000" + }, + { + "self": "https://jira.mongodb.org/rest/api/2/issue/108475/comment/491185", + "id": "491185", + "author": { + "self": "https://jira.mongodb.org/rest/api/2/user?username=isabelle.davis", + "name": "isabelle.davis", + "key": "isabelle.davis", + "emailAddress": "isabelle@10gen.com", + "avatarUrls": { + "48x48": "https://jira.mongodb.org/secure/useravatar?ownerId=isabelle.davis&avatarId=12232", + "24x24": "https://jira.mongodb.org/secure/useravatar?size=small&ownerId=isabelle.davis&avatarId=12232", + "16x16": "https://jira.mongodb.org/secure/useravatar?size=xsmall&ownerId=isabelle.davis&avatarId=12232", + "32x32": "https://jira.mongodb.org/secure/useravatar?size=medium&ownerId=isabelle.davis&avatarId=12232" + }, + "displayName": "Isabelle Davis", + "active": true, + "timeZone": "America/New_York" + }, + "body": "The building is aware. ", + "updateAuthor": { + "self": "https://jira.mongodb.org/rest/api/2/user?username=isabelle.davis", + "name": "isabelle.davis", + "key": "isabelle.davis", + "emailAddress": "isabelle@10gen.com", + "avatarUrls": { + "48x48": "https://jira.mongodb.org/secure/useravatar?ownerId=isabelle.davis&avatarId=12232", + "24x24": "https://jira.mongodb.org/secure/useravatar?size=small&ownerId=isabelle.davis&avatarId=12232", + "16x16": "https://jira.mongodb.org/secure/useravatar?size=xsmall&ownerId=isabelle.davis&avatarId=12232", + "32x32": "https://jira.mongodb.org/secure/useravatar?size=medium&ownerId=isabelle.davis&avatarId=12232" + }, + "displayName": "Isabelle Davis", + "active": true, + "timeZone": "America/New_York" + }, + "created": "2014-01-30T22:40:53.000+0000", + "updated": "2014-01-30T22:40:53.000+0000" + }, + { + "self": "https://jira.mongodb.org/rest/api/2/issue/108475/comment/518934", + "id": "518934", + "author": { + "self": "https://jira.mongodb.org/rest/api/2/user?username=akshay%40mongodb.com", + "name": "akshay@mongodb.com", + "key": "akshay", + "emailAddress": "akshay@mongodb.com", + "avatarUrls": { + "48x48": "https://jira.mongodb.org/secure/useravatar?ownerId=akshay&avatarId=11409", + "24x24": "https://jira.mongodb.org/secure/useravatar?size=small&ownerId=akshay&avatarId=11409", + "16x16": "https://jira.mongodb.org/secure/useravatar?size=xsmall&ownerId=akshay&avatarId=11409", + "32x32": "https://jira.mongodb.org/secure/useravatar?size=medium&ownerId=akshay&avatarId=11409" + }, + "displayName": "Akshay Kumar", + "active": true, + "timeZone": "America/New_York" + }, + "body": "This looks like it has been resolved!", + "updateAuthor": { + "self": "https://jira.mongodb.org/rest/api/2/user?username=akshay%40mongodb.com", + "name": "akshay@mongodb.com", + "key": "akshay", + "emailAddress": "akshay@mongodb.com", + "avatarUrls": { + "48x48": "https://jira.mongodb.org/secure/useravatar?ownerId=akshay&avatarId=11409", + "24x24": "https://jira.mongodb.org/secure/useravatar?size=small&ownerId=akshay&avatarId=11409", + "16x16": "https://jira.mongodb.org/secure/useravatar?size=xsmall&ownerId=akshay&avatarId=11409", + "32x32": "https://jira.mongodb.org/secure/useravatar?size=medium&ownerId=akshay&avatarId=11409" + }, + "displayName": "Akshay Kumar", + "active": true, + "timeZone": "America/New_York" + }, + "created": "2014-03-19T00:43:05.000+0000", + "updated": "2014-03-19T00:43:05.000+0000" + }, + { + "self": "https://jira.mongodb.org/rest/api/2/issue/108475/comment/518985", + "id": "518985", + "author": { + "self": "https://jira.mongodb.org/rest/api/2/user?username=jmikola%40mongodb.com", + "name": "jmikola@mongodb.com", + "key": "jmikola@gmail.com", + "emailAddress": "jmikola@mongodb.com", + "avatarUrls": { + "48x48": "https://jira.mongodb.org/secure/useravatar?ownerId=jmikola%40gmail.com&avatarId=11105", + "24x24": "https://jira.mongodb.org/secure/useravatar?size=small&ownerId=jmikola%40gmail.com&avatarId=11105", + "16x16": "https://jira.mongodb.org/secure/useravatar?size=xsmall&ownerId=jmikola%40gmail.com&avatarId=11105", + "32x32": "https://jira.mongodb.org/secure/useravatar?size=medium&ownerId=jmikola%40gmail.com&avatarId=11105" + }, + "displayName": "Jeremy Mikola", + "active": true, + "timeZone": "America/New_York" + }, + "body": "Quite masterfully, I might add. Good work team!", + "updateAuthor": { + "self": "https://jira.mongodb.org/rest/api/2/user?username=jmikola%40mongodb.com", + "name": "jmikola@mongodb.com", + "key": "jmikola@gmail.com", + "emailAddress": "jmikola@mongodb.com", + "avatarUrls": { + "48x48": "https://jira.mongodb.org/secure/useravatar?ownerId=jmikola%40gmail.com&avatarId=11105", + "24x24": "https://jira.mongodb.org/secure/useravatar?size=small&ownerId=jmikola%40gmail.com&avatarId=11105", + "16x16": "https://jira.mongodb.org/secure/useravatar?size=xsmall&ownerId=jmikola%40gmail.com&avatarId=11105", + "32x32": "https://jira.mongodb.org/secure/useravatar?size=medium&ownerId=jmikola%40gmail.com&avatarId=11105" + }, + "displayName": "Jeremy Mikola", + "active": true, + "timeZone": "America/New_York" + }, + "created": "2014-03-19T04:54:25.000+0000", + "updated": "2014-03-19T04:54:25.000+0000" + }, + { + "self": "https://jira.mongodb.org/rest/api/2/issue/108475/comment/520128", + "id": "520128", + "author": { + "self": "https://jira.mongodb.org/rest/api/2/user?username=kym.ganade", + "name": "kym.ganade", + "key": "kym@10gen.com", + "emailAddress": "kym@10gen.com", + "avatarUrls": { + "48x48": "https://jira.mongodb.org/secure/useravatar?avatarId=10122", + "24x24": "https://jira.mongodb.org/secure/useravatar?size=small&avatarId=10122", + "16x16": "https://jira.mongodb.org/secure/useravatar?size=xsmall&avatarId=10122", + "32x32": "https://jira.mongodb.org/secure/useravatar?size=medium&avatarId=10122" + }, + "displayName": "Kym Ganade [X]", + "active": true, + "timeZone": "America/Havana" + }, + "body": "Panels Have been added", + "updateAuthor": { + "self": "https://jira.mongodb.org/rest/api/2/user?username=kym.ganade", + "name": "kym.ganade", + "key": "kym@10gen.com", + "emailAddress": "kym@10gen.com", + "avatarUrls": { + "48x48": "https://jira.mongodb.org/secure/useravatar?avatarId=10122", + "24x24": "https://jira.mongodb.org/secure/useravatar?size=small&avatarId=10122", + "16x16": "https://jira.mongodb.org/secure/useravatar?size=xsmall&avatarId=10122", + "32x32": "https://jira.mongodb.org/secure/useravatar?size=medium&avatarId=10122" + }, + "displayName": "Kym Ganade [X]", + "active": true, + "timeZone": "America/Havana" + }, + "created": "2014-03-20T17:21:15.000+0000", + "updated": "2014-03-20T17:21:15.000+0000" + } + ], + "maxResults": 7, + "total": 7, + "startAt": 0 + } + } + }, + { + "topics": [ + "stall panel gap", + "men's restroom", + "women's restroom", + "New York office", + "stall door locks" + ], + "description": "This was filed to alert the workplace team of a significant gap between the stall panels and the tile wall in the men's room at the New York office. The issue also mentions the instability of the stall door locks, particularly those nearest the exit. A commenter confirmed that the gap is also present in the women's restroom. The issue was resolved by adding panels." + } +] diff --git a/packages/mongodb-artifact-generator/src/jiraPromptResponse/topIssues.json b/packages/mongodb-artifact-generator/src/jiraPromptResponse/topIssues.json new file mode 100644 index 000000000..14ffc64d1 --- /dev/null +++ b/packages/mongodb-artifact-generator/src/jiraPromptResponse/topIssues.json @@ -0,0 +1,236 @@ +[ + "PYTHON-3257", + "JAVA-4154", + "GODRIVER-1763", + "CSHARP-3653", + "SERVER-40578", + "SERVER-72774", + "SERVER-46977", + "JAVA-2407", + "SPARK-142", + "GODRIVER-2613", + "COMPASS-4657", + "CSHARP-3095", + "SERVER-46951", + "CSHARP-4058", + "MONGOID-4067", + "GODRIVER-897", + "SERVER-63143", + "CSHARP-1165", + "SERVER-32790", + "CSHARP-1895", + "NODE-5711", + "COMPASS-7283", + "KAFKA-395", + "SPARK-250", + "SERVER-19638", + "RUST-940", + "DOCS-15498", + "NODE-2313", + "SERVER-62699", + "COMPASS-4884", + "TOOLS-2151", + "SERVER-59766", + "SERVER-18335", + "JAVA-2609", + "SERVER-37846", + "SERVER-52537", + "MONGOID-210", + "SERVER-31459", + "JAVA-3452", + "PYTHON-1434", + "JAVA-2569", + "SERVER-32827", + "GODRIVER-1194", + "SERVER-74739", + "csharp-1309", + "CSHARP-4566", + "TOOLS-1059", + "SERVER-53337", + "server-6992", + "WT-3276", + "WT-10563", + "SERVER-35370", + "CSHARP-1592", + "SERVER-34879", + "JAVA-3668", + "SERVER-78369", + "WT-7126", + "SERVER-38549", + "TOOLS-1012", + "JAVA-466", + "NODE-3631", + "NODE-4209", + "CSHARP-1483", + "NODE-3523", + "JAVA-4044", + "SERVER-37554", + "DOCS-7669", + "COMPASS-4547", + "SERVER-59482", + "WT-11173", + "CSHARP-3578", + "SERVER-63417", + "TOOLS-896", + "CSHARP-3282", + "SERVER-47660", + "SERVER-49317", + "SERVER-37814", + "GODRIVER-1763", + "SERVER-46929", + "TOOLS-162", + "SERVER-51343", + "SERVER-46871", + "MOTOR-457", + "SERVER-64005", + "WT-9379", + "SERVER-41216", + "CSHARP-673", + "SERVER-48161", + "SERVER-54571", + "CSHARP-1726", + "SERVER-42366", + "SERVER-57782", + "CSHARP-863", + "PHPC-1180", + "NODE-5662", + "SERVER-31340", + "SERVER-58195", + "CDRIVER-1072", + "DOCS-5270", + "SERVER-31478", + "PYTHON-1880", + "SERVER-55459", + "RUBY-2167", + "SERVER-34604", + "SERVER-25883", + "SERVER-10777", + "RUBY-1917", + "SERVER-50067", + "SERVER-47553", + "GODRIVER-879", + "RUBY-2615", + "MONGOID-5627", + "SERVER-61680", + "NODE-3780", + "SERVER-4683", + "TOOLS-435", + "MONGOSH-1031", + "COMPASS-7575", + "SERVER-31602", + "CSHARP-3270", + "NODE-1649", + "SERVER-67014", + "DOCS-13727", + "SERVER-22056", + "SERVER-53177", + "JAVA-4076", + "CDRIVER-2045", + "MONGOID-518", + "SERVER-20243", + "SERVER-48398", + "SERVER-17357", + "SPARK-352", + "MONGOID-1023", + "COMPASS-2389", + "SERVER-31598", + "SERVER-25760", + "CSHARP-994", + "MONGOSH-818", + "SERVER-6798", + "SERVER-69199", + "SERVER-41791", + "COMPASS-4837", + "MONGOID-247", + "WT-3186", + "TOOLS-2352", + "WT-8003", + "NODE-1051", + "CSHARP-4474", + "SERVER-59926", + "DOCS-12798", + "SERVER-39562", + "SERVER-57086", + "TOOLS-2603", + "SERVER-19470", + "TOOLS-1549", + "SERVER-1603", + "SERVER-8985", + "DOCS-14793", + "DOCS-13654", + "MONGOSH-587", + "SERVER-5044", + "RUST-1779", + "CSHARP-1734", + "SERVER-41449", + "WT-7691", + "SERVER-58935", + "SERVER-9209", + "SERVER-38600", + "SERVER-13520", + "GODRIVER-1182", + "NODE-3654", + "NODE-5305", + "SERVER-45647", + "SERVER-31931", + "SERVER-64822", + "SERVER-10860", + "DOCS-12941", + "GODRIVER-3062", + "COMPASS-2437", + "SPARK-197", + "SERVER-27369", + "SERVER-34633", + "SERVER-18869", + "SERVER-38367", + "NODE-1917", + "MONGOID-3525", + "SERVER-27281", + "KAFKA-272", + "CDRIVER-4112", + "DOCS-8907", + "PHPC-682", + "SERVER-33122", + "MONGOSH-296", + "RUST-392", + "SERVER-30518", + "RUST-300", + "JAVA-1086", + "SERVER-59339", + "SERVER-59608", + "DOCS-6979", + "CSHARP-2455", + "NODE-962", + "SERVER-13025", + "WT-9805", + "SERVER-86419", + "CSHARP-1554", + "SERVER-62277", + "SERVER-58264", + "JAVA-3023", + "SPARK-60", + "WT-7475", + "SERVER-19445", + "SERVER-37319", + "SERVER-68365", + "MONGOID-4976", + "PYTHON-1167", + "SERVER-6720", + "SERVER-59178", + "SERVER-16975", + "SERVER-49429", + "SERVER-62759", + "MONGOID-3439", + "CSHARP-1984", + "JAVA-4471", + "SERVER-45906", + "TOOLS-2208", + "SERVER-64293", + "SERVER-27832", + "SERVER-69000", + "SERVER-48934", + "DOCS-11156", + "KAFKA-348", + "DOCS-10947", + "CSHARP-3903" +] diff --git a/packages/mongodb-artifact-generator/src/runId.test.ts b/packages/mongodb-artifact-generator/src/runId.test.ts new file mode 100644 index 000000000..28d18eddc --- /dev/null +++ b/packages/mongodb-artifact-generator/src/runId.test.ts @@ -0,0 +1,65 @@ +import { createRunId } from "./runId"; +import { promises as fs, writeFileSync } from "fs"; + +function mockStaticDate(input: ConstructorParameters[0]) { + const date = new Date(input); + jest.spyOn(global, "Date").mockImplementation(() => date); +} + +describe("createRunId", () => { + afterEach(() => { + jest.restoreAllMocks(); + }); + it("creates a runId based on the current date", () => { + mockStaticDate("2024-08-21T20:20:01.188Z"); + expect(new Date().getTime()).toBe(1724271601188); + expect(createRunId()).toBe("20240821-2020-4asie"); + }); + + it("does nothing to ensure that namespace is a valid filename", () => { + mockStaticDate("2024-08-21T20:20:01.188Z"); + const ns = `--_._((my-nam/es \\ pace-`; + const runId = createRunId(ns); + expect(runId).toBe("--_._((my-nam/es \\ pace--20240821-2020-4asie"); + expect(() => + writeFileSync( + `./${runId}.test.json`, + "this should error because of an invalid file name" + ) + ).toThrow(`ENOENT: no such file or directory, open './${runId}.test.json'`); + }); + + it("allows you to prepend a namespace prefix", () => { + mockStaticDate("2024-08-21T20:20:01.188Z"); + const ns = "my-namespace"; + const runIdWithoutNs = createRunId(); + const runIdWithNs = createRunId(ns); + expect(runIdWithoutNs).toBe("20240821-2020-4asie"); + expect(runIdWithNs).toBe("my-namespace-20240821-2020-4asie"); + }); + + it("creates runIds that sort by the time they were created", () => { + mockStaticDate("2024-08-21T20:20:01.188Z"); + const runId1 = createRunId(); + jest.restoreAllMocks(); + mockStaticDate("2024-08-21T20:20:17.223Z"); + const runId2 = createRunId(); + jest.restoreAllMocks(); + mockStaticDate("2024-08-21T20:21:01.011Z"); + const runId3 = createRunId(); + jest.restoreAllMocks(); + mockStaticDate("2024-09-01T01:00:00.003Z"); + const runId4 = createRunId(); + + expect(runId1).toBe("20240821-2020-4asie"); + expect(runId2).toBe("20240821-2020-4asur"); + expect(runId3).toBe("20240821-2021-4atsk"); + expect(runId4).toBe("20240901-0100-iv734"); + expect(runId1 < runId2).toBe(true); + expect(runId2 < runId3).toBe(true); + expect(runId3 < runId4).toBe(true); + expect(runId1.localeCompare(runId2)).toBe(-1); + expect(runId2.localeCompare(runId3)).toBe(-1); + expect(runId3.localeCompare(runId4)).toBe(-1); + }); +}); diff --git a/packages/mongodb-artifact-generator/src/runId.ts b/packages/mongodb-artifact-generator/src/runId.ts new file mode 100644 index 000000000..57be28fd0 --- /dev/null +++ b/packages/mongodb-artifact-generator/src/runId.ts @@ -0,0 +1,14 @@ +export function createRunId(namespace?: string) { + const now = new Date(); + const [date, time] = now.toISOString().split("T"); + // Format a string for the current day in YYYYMMDD format + const datestamp = date.replace(/-/g, ""); + // Format a string for the current time in 24H HHMM format + const minutestamp = time.replace(/:/g, "").slice(0, 4); + const millistamp = now + .getTime() + .toString(36) + .slice(2, 2 + 5); + const timestamp = `${datestamp}-${minutestamp}-${millistamp}`; + return namespace ? `${namespace}-${timestamp}` : timestamp; +} diff --git a/packages/mongodb-artifact-generator/src/runlogger.ts b/packages/mongodb-artifact-generator/src/runlogger.ts index 2965317ad..b26b1554c 100644 --- a/packages/mongodb-artifact-generator/src/runlogger.ts +++ b/packages/mongodb-artifact-generator/src/runlogger.ts @@ -2,6 +2,7 @@ import { promises as fs } from "fs"; import path from "path"; import { z } from "zod"; import { ObjectId } from "mongodb-rag-core"; +import { createRunId } from "./runId"; export type Artifact = z.infer; const ArtifactSchema = z @@ -63,7 +64,7 @@ export class RunLogger { topic: string; constructor(args: RunLoggerArgs) { - this.#runId = args.runId ?? new ObjectId().toHexString(); + this.#runId = args.runId ?? createRunId(); this.topic = args.topic; } diff --git a/packages/mongodb-artifact-generator/tsconfig.json b/packages/mongodb-artifact-generator/tsconfig.json index 58c159dd3..a8e7e3be8 100644 --- a/packages/mongodb-artifact-generator/tsconfig.json +++ b/packages/mongodb-artifact-generator/tsconfig.json @@ -3,7 +3,8 @@ "compilerOptions": { "lib": ["ES2022"], "target": "ES2022", - "outDir": "./build" + "outDir": "./build", + "resolveJsonModule": true, }, - "include": ["./src/**/*.ts"] + "include": ["./src/**/*.ts", "./src/**/*.json"], } From baf6ce7548aa3401a1e3cd3411ba76f67ac7fa2f Mon Sep 17 00:00:00 2001 From: Nick Larew Date: Wed, 16 Oct 2024 19:57:24 -0500 Subject: [PATCH 2/7] better issue input --- .../commands/generateJiraPromptResponse.ts | 81 ++++++++++++++----- 1 file changed, 59 insertions(+), 22 deletions(-) diff --git a/packages/mongodb-artifact-generator/src/commands/generateJiraPromptResponse.ts b/packages/mongodb-artifact-generator/src/commands/generateJiraPromptResponse.ts index 1c0d5f909..48f25fc30 100644 --- a/packages/mongodb-artifact-generator/src/commands/generateJiraPromptResponse.ts +++ b/packages/mongodb-artifact-generator/src/commands/generateJiraPromptResponse.ts @@ -55,8 +55,14 @@ export type FormattedJiraIssueWithSummary = { type GenerateJiraPromptResponseCommandArgs = { runId?: string; llmMaxConcurrency: number; + maxInputLength?: number; + issuesFilePath?: string; + issue?: string | string[]; }; +const issueKeysSchema = z.array(z.string()); +type IssueKeys = z.infer; + export default createCommand({ command: "generateJiraPromptResponse", builder(args) { @@ -76,6 +82,23 @@ export default createCommand({ .parse(GENERATOR_LLM_MAX_CONCURRENCY), description: "The maximum number of concurrent requests to the LLM API. Defaults to 10. Can be specified in the config file as GENERATOR_LLM_MAX_CONCURRENCY.", + }) + .option("maxInputLength", { + type: "number", + demandOption: false, + description: + "The maximum number of issues to process in this run. Any additional issues are skipped.", + }) + .option("issuesFilePath", { + type: "string", + demandOption: false, + description: + "Path to a JSON file containing an array of issue key strings to process.", + }) + .option("issue", { + type: "string", + demandOption: false, + description: "A single issue key to process.", }); }, async handler(args) { @@ -101,7 +124,10 @@ export default createCommand({ export const action = createConfiguredAction( - async ({ jiraApi, openAiClient }, { llmMaxConcurrency }) => { + async ( + { jiraApi, openAiClient }, + { llmMaxConcurrency, maxInputLength, issuesFilePath, issue } + ) => { logger.logInfo(`Setting up...`); if (!jiraApi) { throw new Error( @@ -114,30 +140,41 @@ export const action = ); } - const topIssueKeys = z - .array(z.string()) - .parse( - JSON.parse( - await fs.readFile( - path.join(__dirname, "../jiraPromptResponse/topIssues.json"), - "utf-8" - ) + // Determine which Jira issues to process + const issueKeys: string[] = []; + if (issue) { + issueKeys.push( + ...issueKeysSchema.parse(Array.isArray(issue) ? issue : [issue]) + ); + } + if (issuesFilePath) { + issueKeys.push( + ...issueKeysSchema.parse( + JSON.parse(await fs.readFile(issuesFilePath, "utf-8")) ) - ) - .slice(0, 6); // TODO: Remove slice before merge. This helps make the run faster for testing. - - // Get each issue using a promise pool - const jiraMaxConcurrency = 12; - const { results: jiraIssues, errors: getJiraIssuesErrors } = - await PromisePool.for(topIssueKeys) - .withConcurrency(jiraMaxConcurrency) - .process(async (issueKey) => { - return await jiraApi.getIssue(issueKey); - }); - for (const error of getJiraIssuesErrors) { - logger.logError(`Error fetching issue: ${error.item}`); + ); + } + if (issueKeys.length === 0) { + throw new Error("No issues provided."); } + // Fetch Jira issues + const jiraMaxConcurrency = 12; + const { results: jiraIssues } = await PromisePool.for( + issueKeys.slice(0, maxInputLength) + ) + .withConcurrency(jiraMaxConcurrency) + .handleError((error, issueKey) => { + const parsedErrorMessage = JSON.parse(error.message); + const logErrorMessage = + parsedErrorMessage?.errorMessages?.[0] ?? "Something went wrong."; + logger.logError( + `Error fetching issue: ${issueKey} - ${logErrorMessage}` + ); + }) + .process(async (issueKey) => { + return await jiraApi.getIssue(issueKey); + }); logger.appendArtifact("jiraIssues.raw.json", JSON.stringify(jiraIssues)); interface RawJiraIssue { From 39dc5d1e912c225a9c33c7dbc564b8e0745c6ba5 Mon Sep 17 00:00:00 2001 From: Nick Larew Date: Thu, 17 Oct 2024 12:28:33 -0500 Subject: [PATCH 3/7] add env var --- .drone.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.drone.yml b/.drone.yml index 0bb3742a3..eefb90ae9 100644 --- a/.drone.yml +++ b/.drone.yml @@ -31,6 +31,7 @@ steps: OPENAI_CHAT_COMPLETION_MODEL_VERSION: 2023-06-01-preview OPENAI_PREPROCESSOR_CHAT_COMPLETION_DEPLOYMENT: gpt-4o-mini OPENAI_API_VERSION: "2024-06-01" + GENERATOR_LLM_MAX_CONCURRENCY: 8 MONGODB_CONNECTION_URI: from_secret: mongodb_connection_uri OPENAI_ENDPOINT: From 112cd92bd0710a6fb5e97440d257bf59ccbca14c Mon Sep 17 00:00:00 2001 From: Nick Larew Date: Fri, 18 Oct 2024 16:27:40 -0500 Subject: [PATCH 4/7] Ensure unique keys, more config/args, less env --- .drone.yml | 1 - .../src/ArtifactGeneratorEnvVars.ts | 5 - .../mongodb-artifact-generator/src/Config.ts | 10 + .../src/chat/makeGeneratePrompts.ts | 2 +- .../src/chat/utils.ts | 13 -- .../commands/generateJiraPromptResponse.ts | 207 ++++++++++-------- 6 files changed, 130 insertions(+), 108 deletions(-) diff --git a/.drone.yml b/.drone.yml index eefb90ae9..0bb3742a3 100644 --- a/.drone.yml +++ b/.drone.yml @@ -31,7 +31,6 @@ steps: OPENAI_CHAT_COMPLETION_MODEL_VERSION: 2023-06-01-preview OPENAI_PREPROCESSOR_CHAT_COMPLETION_DEPLOYMENT: gpt-4o-mini OPENAI_API_VERSION: "2024-06-01" - GENERATOR_LLM_MAX_CONCURRENCY: 8 MONGODB_CONNECTION_URI: from_secret: mongodb_connection_uri OPENAI_ENDPOINT: diff --git a/packages/mongodb-artifact-generator/src/ArtifactGeneratorEnvVars.ts b/packages/mongodb-artifact-generator/src/ArtifactGeneratorEnvVars.ts index 37132d47d..bc7f48a8b 100644 --- a/packages/mongodb-artifact-generator/src/ArtifactGeneratorEnvVars.ts +++ b/packages/mongodb-artifact-generator/src/ArtifactGeneratorEnvVars.ts @@ -1,10 +1,5 @@ import { CORE_ENV_VARS } from "mongodb-rag-core"; -export const GENERATOR_LLM_ENV_VARS = { - GENERATOR_LLM_MAX_CONCURRENCY: "", -}; - export const ArtifactGeneratorEnvVars = { ...CORE_ENV_VARS, - ...GENERATOR_LLM_ENV_VARS, }; diff --git a/packages/mongodb-artifact-generator/src/Config.ts b/packages/mongodb-artifact-generator/src/Config.ts index 9104deb1f..e5268e23b 100644 --- a/packages/mongodb-artifact-generator/src/Config.ts +++ b/packages/mongodb-artifact-generator/src/Config.ts @@ -43,10 +43,20 @@ export type Config = { */ jiraApi?: Constructor; + /** + The maximum number of concurrent requests to make to the Jira API. + */ + jiraApiMaxConcurrency?: number; + /** The GitHub API client. */ githubApi?: Constructor; + + /** + The maximum number of concurrent requests to make to an LLM generator. + */ + llmMaxConcurrency?: number; }; export type Constructor = (() => T) | (() => Promise); diff --git a/packages/mongodb-artifact-generator/src/chat/makeGeneratePrompts.ts b/packages/mongodb-artifact-generator/src/chat/makeGeneratePrompts.ts index b35f1b3ba..f456a5541 100644 --- a/packages/mongodb-artifact-generator/src/chat/makeGeneratePrompts.ts +++ b/packages/mongodb-artifact-generator/src/chat/makeGeneratePrompts.ts @@ -16,7 +16,7 @@ import { stripIndents } from "common-tags"; export type GeneratedPrompts = z.infer; export const GeneratedPrompts = z.object({ - prompts: z.array(z.string()), + prompts: z.array(z.string()).min(1).max(4), }); const generatePromptsTool: FunctionDefinition = { diff --git a/packages/mongodb-artifact-generator/src/chat/utils.ts b/packages/mongodb-artifact-generator/src/chat/utils.ts index 7da6ccbae..62f44db6a 100644 --- a/packages/mongodb-artifact-generator/src/chat/utils.ts +++ b/packages/mongodb-artifact-generator/src/chat/utils.ts @@ -85,19 +85,6 @@ export function formatFewShotExamples(args: { }) satisfies ChatRequestMessage[]; } -export const UserMessageMongoDbGuardrailFunctionSchema = z.object({ - reasoning: z - .string() - .describe( - "Reason for whether to reject the user query. Be concise. Think step by step. " - ), - rejectMessage: z - .boolean() - .describe( - "Set to true if the user query should be rejected. Set to false if the user query should be accepted." - ), -}); - export type AsJsonSchemaOptions = { // examples?: PromptExamplePair[]; zodToJsonSchema?: Parameters[1]; diff --git a/packages/mongodb-artifact-generator/src/commands/generateJiraPromptResponse.ts b/packages/mongodb-artifact-generator/src/commands/generateJiraPromptResponse.ts index 48f25fc30..061e12c1f 100644 --- a/packages/mongodb-artifact-generator/src/commands/generateJiraPromptResponse.ts +++ b/packages/mongodb-artifact-generator/src/commands/generateJiraPromptResponse.ts @@ -13,56 +13,82 @@ import { asBulletPoints, loadPromptExamplePairFromFile } from "../chat/utils"; import { createRunId } from "../runId"; import { makeGeneratePrompts } from "../chat/makeGeneratePrompts"; import { makeGenerateResponse } from "../chat/makeGenerateResponse"; -import { GENERATOR_LLM_ENV_VARS } from "../ArtifactGeneratorEnvVars"; import { promises as fs } from "fs"; import { z } from "zod"; -const { GENERATOR_LLM_MAX_CONCURRENCY, OPENAI_CHAT_COMPLETION_DEPLOYMENT } = - assertEnvVars({ - ...GENERATOR_LLM_ENV_VARS, - OPENAI_CHAT_COMPLETION_DEPLOYMENT: "", - }); +const DEFAULT_JIRA_API_MAX_CONCURRENCY = 12; + +const DEFAULT_LLM_MAX_CONCURRENCY = 8; + +const { OPENAI_CHAT_COMPLETION_DEPLOYMENT } = assertEnvVars({ + OPENAI_CHAT_COMPLETION_DEPLOYMENT: "", +}); let logger: RunLogger; -export interface JiraComment { - id: string; - body: string; - author: { - emailAddress: string; - displayName: string; - }; - created: string; - updated: string; -} +export const JiraCommentSchema = z.object({ + id: z.string(), + body: z.string(), + author: z.object({ + emailAddress: z.string(), + displayName: z.string(), + }), + created: z.string(), + updated: z.string(), +}); +export type JiraComment = z.infer; -export interface FormattedJiraIssue { - key: string; - projectName: string; - summary: string; - status: string; - created: string; - updated: string; - description: string; - comments: JiraComment[]; -} +const RawJiraIssueSchema = z.object({ + key: z.string(), + fields: z.object({ + project: z.object({ + name: z.string(), + }), + summary: z.string(), + status: z.object({ + name: z.string(), + }), + created: z.string(), + updated: z.string(), + description: z.string(), + comment: z.object({ + comments: z.array(JiraCommentSchema), + }), + }), +}); -export type FormattedJiraIssueWithSummary = { - issue: FormattedJiraIssue; - summary: Summary; -}; +export type RawJiraIssue = z.infer; + +export const FormattedJiraIssueSchema = z.object({ + key: z.string(), + projectName: z.string(), + summary: z.string(), + status: z.string(), + created: z.string(), + updated: z.string(), + description: z.string(), + comments: z.array(JiraCommentSchema), +}); +export type FormattedJiraIssue = z.infer; + +export const FormattedJiraIssueWithSummarySchema = z.object({ + issue: FormattedJiraIssueSchema, + summary: Summary, +}); + +export type FormattedJiraIssueWithSummary = z.infer< + typeof FormattedJiraIssueWithSummarySchema +>; type GenerateJiraPromptResponseCommandArgs = { runId?: string; - llmMaxConcurrency: number; + jiraApiMaxConcurrency?: number; + llmMaxConcurrency?: number; maxInputLength?: number; issuesFilePath?: string; issue?: string | string[]; }; -const issueKeysSchema = z.array(z.string()); -type IssueKeys = z.infer; - export default createCommand({ command: "generateJiraPromptResponse", builder(args) { @@ -73,15 +99,17 @@ export default createCommand({ description: "A unique name for the run. This controls where outputs artifacts and logs are stored.", }) + .option("jiraApiMaxConcurrency", { + type: "number", + demandOption: false, + description: + "The maximum number of concurrent requests to the Jira API. Can be specified in the config file as `jiraApiMaxConcurrency`.", + }) .option("llmMaxConcurrency", { type: "number", demandOption: false, - default: z.coerce - .number() - .default(10) - .parse(GENERATOR_LLM_MAX_CONCURRENCY), description: - "The maximum number of concurrent requests to the LLM API. Defaults to 10. Can be specified in the config file as GENERATOR_LLM_MAX_CONCURRENCY.", + "The maximum number of concurrent requests to the LLM API. Can be specified in the config file as `llmMaxConcurrency`.", }) .option("maxInputLength", { type: "number", @@ -124,11 +152,10 @@ export default createCommand({ export const action = createConfiguredAction( - async ( - { jiraApi, openAiClient }, - { llmMaxConcurrency, maxInputLength, issuesFilePath, issue } - ) => { + async (config, args) => { logger.logInfo(`Setting up...`); + + const { jiraApi, openAiClient } = config; if (!jiraApi) { throw new Error( "jiraApi is required. Make sure to define it in the config." @@ -140,30 +167,46 @@ export const action = ); } + const jiraApiMaxConcurrency = + args.jiraApiMaxConcurrency ?? + config.jiraApiMaxConcurrency ?? + DEFAULT_JIRA_API_MAX_CONCURRENCY; + + const llmMaxConcurrency = + args.jiraApiMaxConcurrency ?? + config.jiraApiMaxConcurrency ?? + DEFAULT_LLM_MAX_CONCURRENCY; + // Determine which Jira issues to process - const issueKeys: string[] = []; - if (issue) { - issueKeys.push( - ...issueKeysSchema.parse(Array.isArray(issue) ? issue : [issue]) + const inputIssueKeys = new Set(); + const addInputIssueKeys = (issueKeys: unknown) => { + z.array(z.string()) + .parse(issueKeys) + .forEach((issueKey) => { + inputIssueKeys.add(issueKey); + }); + }; + + if (args.issue) { + addInputIssueKeys( + Array.isArray(args.issue) ? args.issue : [args.issue] ); } - if (issuesFilePath) { - issueKeys.push( - ...issueKeysSchema.parse( - JSON.parse(await fs.readFile(issuesFilePath, "utf-8")) - ) + if (args.issuesFilePath) { + addInputIssueKeys( + JSON.parse(await fs.readFile(args.issuesFilePath, "utf-8")) ); } - if (issueKeys.length === 0) { + if (inputIssueKeys.size === 0) { throw new Error("No issues provided."); } + const issueKeys = [...inputIssueKeys]; // Fetch Jira issues - const jiraMaxConcurrency = 12; const { results: jiraIssues } = await PromisePool.for( - issueKeys.slice(0, maxInputLength) + issueKeys.slice(0, args.maxInputLength) ) - .withConcurrency(jiraMaxConcurrency) + .withConcurrency(jiraApiMaxConcurrency) .handleError((error, issueKey) => { const parsedErrorMessage = JSON.parse(error.message); const logErrorMessage = @@ -177,28 +220,11 @@ export const action = }); logger.appendArtifact("jiraIssues.raw.json", JSON.stringify(jiraIssues)); - interface RawJiraIssue { - key: string; - fields: { - project: { - name: string; - }; - summary: string; - status: { - name: string; - }; - created: string; - updated: string; - description: string; - comment: { - comments: JiraComment[]; - }; - }; - } - - const formattedJiraIssues = (jiraIssues as RawJiraIssue[]).map( - (issue) => { - return { + const formattedJiraIssues = z + .array(RawJiraIssueSchema) + .parse(jiraIssues) + .map((issue) => { + return FormattedJiraIssueSchema.parse({ key: issue.key, projectName: issue.fields.project.name, summary: issue.fields.summary, @@ -218,9 +244,8 @@ export const action = updated: comment.updated, }; }), - } satisfies FormattedJiraIssue; - } - ); + }); + }); logger.appendArtifact( "jiraIssues.formatted.json", @@ -310,18 +335,24 @@ export const action = "Do not reference hypothetical or speculative information." ), }); - const { errors: generatePromptsErrors } = await PromisePool.for( - formattedIssuesWithSummaries - ) + await PromisePool.for(formattedIssuesWithSummaries) .withConcurrency(llmMaxConcurrency) + .handleError((error, { issue }) => { + logger.logError( + `Error generating prompts for ${issue.key}: ${JSON.stringify( + error + )}` + ); + }) .process(async ({ issue, summary }) => { const { prompts } = await generatePrompts({ issue, summary }); - console.log("generated prompts for issue", issue.key, summary); + console.log( + `generated ${prompts.length} prompts for issue`, + issue.key, + prompts + ); promptsByIssueKey.set(issue.key, prompts); }); - for (const error of generatePromptsErrors) { - logger.logError(`Error generating prompts: ${error.item}`); - } const generatedPrompts: [issueKey: string, prompt: string][] = [ ...promptsByIssueKey.entries(), ].flatMap(([issueKey, prompts]) => { From e3dc590646832747aaa6f7ab75f65556e9970995 Mon Sep 17 00:00:00 2001 From: Nick Larew Date: Fri, 18 Oct 2024 16:40:39 -0500 Subject: [PATCH 5/7] Add back some runtime logging --- .../commands/generateJiraPromptResponse.ts | 53 +++++++++++-------- 1 file changed, 32 insertions(+), 21 deletions(-) diff --git a/packages/mongodb-artifact-generator/src/commands/generateJiraPromptResponse.ts b/packages/mongodb-artifact-generator/src/commands/generateJiraPromptResponse.ts index 061e12c1f..373d0a7e1 100644 --- a/packages/mongodb-artifact-generator/src/commands/generateJiraPromptResponse.ts +++ b/packages/mongodb-artifact-generator/src/commands/generateJiraPromptResponse.ts @@ -203,6 +203,7 @@ export const action = const issueKeys = [...inputIssueKeys]; // Fetch Jira issues + console.log("Fetching Jira issues..."); const { results: jiraIssues } = await PromisePool.for( issueKeys.slice(0, args.maxInputLength) ) @@ -220,6 +221,8 @@ export const action = }); logger.appendArtifact("jiraIssues.raw.json", JSON.stringify(jiraIssues)); + // Format Jira issues + console.log("Formatting Jira issues..."); const formattedJiraIssues = z .array(RawJiraIssueSchema) .parse(jiraIssues) @@ -281,22 +284,22 @@ export const action = ]), }); + console.log(`Summarizing ${jiraIssues.length} Jira issues...`); const summariesByIssueKey = new Map(); - const { errors: summarizeJiraIssueErrors } = await PromisePool.for( - jiraIssues - ) + await PromisePool.for(jiraIssues) .withConcurrency(llmMaxConcurrency) + .handleError((error, issue) => { + logger.logError( + `Error summarizing issue ${issue.key}: ${JSON.stringify(error)}` + ); + }) .process(async (issue) => { const summary = await summarizeJiraIssue({ input: JSON.stringify(issue), }); - console.log("summarized issue", issue.key, summary); + logger.logInfo(`summarized issue ${issue.key}: ${summary}`); summariesByIssueKey.set(issue.key, summary); }); - for (const error of summarizeJiraIssueErrors) { - logger.logError(`Error summarizing issue: ${error.item}`); - console.log("Error summarizing issue", error.raw); - } logger.appendArtifact( "summaries.json", @@ -335,6 +338,7 @@ export const action = "Do not reference hypothetical or speculative information." ), }); + console.log("Generating prompts..."); await PromisePool.for(formattedIssuesWithSummaries) .withConcurrency(llmMaxConcurrency) .handleError((error, { issue }) => { @@ -347,9 +351,10 @@ export const action = .process(async ({ issue, summary }) => { const { prompts } = await generatePrompts({ issue, summary }); console.log( - `generated ${prompts.length} prompts for issue`, - issue.key, - prompts + ` generated ${prompts.length} prompts for issue ${issue.key}` + ); + logger.logInfo( + `generated ${prompts.length} prompts for issue ${issue.key}` ); promptsByIssueKey.set(issue.key, prompts); }); @@ -401,9 +406,18 @@ export const action = prompt, }; }); + + console.log("Generating responses..."); let numGeneratedResponses = 0; - const { errors: generateResponsesErrors } = await PromisePool.for(prompts) + await PromisePool.for(prompts) .withConcurrency(llmMaxConcurrency) + .handleError((error, { issue }) => { + logger.logError( + `Error generating response for ${issue.key}: ${JSON.stringify( + error + )}` + ); + }) .process(async ({ summary, issue, prompt }, i) => { const { response } = await generateResponse({ summary, @@ -411,21 +425,18 @@ export const action = prompt, }); console.log( - `generated response ${++numGeneratedResponses}/${prompts.length}` + ` generated response ${++numGeneratedResponses}/${ + prompts.length + } for issue ${issue.key}` + ); + logger.logInfo( + `generated response ${numGeneratedResponses}/${prompts.length} for issue ${issue.key}` ); if (!responsesByIssueKey.has(issue.key)) { responsesByIssueKey.set(issue.key, []); } responsesByIssueKey.get(issue.key)?.push([prompt, response]); }); - for (const error of generateResponsesErrors) { - const { issue } = error.item; - logger.logError( - `Error generating responses for ${issue.key}: ${JSON.stringify( - error - )}` - ); - } const generatedResponses = [...responsesByIssueKey.entries()].flatMap( ([issueKey, responses]) => { return responses.map(([prompt, response]) => ({ From e74acc4f1366448825049e60d9869291694e8c54 Mon Sep 17 00:00:00 2001 From: Nick Larew Date: Tue, 29 Oct 2024 11:57:39 -0500 Subject: [PATCH 6/7] Add defaults in jsdoc Co-authored-by: Ben Perlmutter <90647379+mongodben@users.noreply.github.com> --- packages/mongodb-artifact-generator/src/Config.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/mongodb-artifact-generator/src/Config.ts b/packages/mongodb-artifact-generator/src/Config.ts index e5268e23b..669f7c58b 100644 --- a/packages/mongodb-artifact-generator/src/Config.ts +++ b/packages/mongodb-artifact-generator/src/Config.ts @@ -45,6 +45,7 @@ export type Config = { /** The maximum number of concurrent requests to make to the Jira API. + @default 12 */ jiraApiMaxConcurrency?: number; @@ -55,6 +56,7 @@ export type Config = { /** The maximum number of concurrent requests to make to an LLM generator. + @default 8 */ llmMaxConcurrency?: number; }; From d59dda4330614c5611156869e266b8ecc819cec1 Mon Sep 17 00:00:00 2001 From: Nick Larew Date: Wed, 30 Oct 2024 00:24:09 -0500 Subject: [PATCH 7/7] Eval WIP --- package-lock.json | 424 +++++++++++++++++- .../mongodb-artifact-generator/package.json | 1 + .../src/chat/makeSummarizer.eval.ts | 123 +++++ 3 files changed, 547 insertions(+), 1 deletion(-) create mode 100644 packages/mongodb-artifact-generator/src/chat/makeSummarizer.eval.ts diff --git a/package-lock.json b/package-lock.json index b1b8d0ca0..d394ab332 100644 --- a/package-lock.json +++ b/package-lock.json @@ -37380,7 +37380,6 @@ "version": "1.6.6", "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.6.tgz", "integrity": "sha512-h+z7HKHYXj6wJU+AnS/+IH8Uh9fdcX1Lrhg1/VMdf9PwoBQXFcXiAdsy2tSK0P6gKwJLXp02r90ahUCqHk9rrw==", - "dev": true, "license": "MIT", "engines": { "node": ">=8.0.0" @@ -42329,6 +42328,7 @@ "@octokit/rest": "^20.1.0", "@supercharge/promise-pool": "^3.2.0", "@types/jira-client": "^7.1.9", + "braintrust": "^0.0.167", "common-tags": "^1.8.2", "dotenv": "^16.4.5", "front-matter": "^4.0.2", @@ -42371,6 +42371,195 @@ "npm": ">=8" } }, + "packages/mongodb-artifact-generator/node_modules/@ai-sdk/provider-utils": { + "version": "1.0.22", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-1.0.22.tgz", + "integrity": "sha512-YHK2rpj++wnLVc9vPGzGFP3Pjeld2MwhKinetA0zKXOoHAT/Jit5O8kZsxcSlJPu9wvcGT1UGZEjZrtO7PfFOQ==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "0.0.26", + "eventsource-parser": "^1.1.2", + "nanoid": "^3.3.7", + "secure-json-parse": "^2.7.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.0.0" + }, + "peerDependenciesMeta": { + "zod": { + "optional": true + } + } + }, + "packages/mongodb-artifact-generator/node_modules/@ai-sdk/provider-utils/node_modules/@ai-sdk/provider": { + "version": "0.0.26", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-0.0.26.tgz", + "integrity": "sha512-dQkfBDs2lTYpKM8389oopPdQgIU007GQyCbuPPrV+K6MtSII3HBfE0stUIMXUb44L+LK1t6GXPP7wjSzjO6uKg==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "packages/mongodb-artifact-generator/node_modules/@ai-sdk/react": { + "version": "0.0.68", + "resolved": "https://registry.npmjs.org/@ai-sdk/react/-/react-0.0.68.tgz", + "integrity": "sha512-dD7cm2UsPWkuWg+qKRXjF+sNLVcUzWUnV25FxvEliJP7I2ajOpq8c+/xyGlm+YodyvAB0fX+oSODOeIWi7lCKg==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider-utils": "1.0.22", + "@ai-sdk/ui-utils": "0.0.49", + "swr": "^2.2.5" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "react": "^18 || ^19 || ^19.0.0-rc", + "zod": "^3.0.0" + }, + "peerDependenciesMeta": { + "react": { + "optional": true + }, + "zod": { + "optional": true + } + } + }, + "packages/mongodb-artifact-generator/node_modules/@ai-sdk/solid": { + "version": "0.0.53", + "resolved": "https://registry.npmjs.org/@ai-sdk/solid/-/solid-0.0.53.tgz", + "integrity": "sha512-0yXkwTE75QKdmz40CBtAFy3sQdUnn/TNMTkTE2xfqC9YN7Ixql472TtC+3h6s4dPjRJm5bNnGJAWHwjT2PBmTw==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider-utils": "1.0.22", + "@ai-sdk/ui-utils": "0.0.49" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "solid-js": "^1.7.7" + }, + "peerDependenciesMeta": { + "solid-js": { + "optional": true + } + } + }, + "packages/mongodb-artifact-generator/node_modules/@ai-sdk/svelte": { + "version": "0.0.56", + "resolved": "https://registry.npmjs.org/@ai-sdk/svelte/-/svelte-0.0.56.tgz", + "integrity": "sha512-EmBHGxVkmC6Ugc2O3tH6+F0udYKUhdlqokKAdO3zZihpNCj4qC5msyzqbhRqX0415tD1eJib5SX2Sva47CHmLA==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider-utils": "1.0.22", + "@ai-sdk/ui-utils": "0.0.49", + "sswr": "^2.1.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "svelte": "^3.0.0 || ^4.0.0 || ^5.0.0" + }, + "peerDependenciesMeta": { + "svelte": { + "optional": true + } + } + }, + "packages/mongodb-artifact-generator/node_modules/@ai-sdk/ui-utils": { + "version": "0.0.49", + "resolved": "https://registry.npmjs.org/@ai-sdk/ui-utils/-/ui-utils-0.0.49.tgz", + "integrity": "sha512-urg0KYrfJmfEBSva9d132YRxAVmdU12ISGVlOV7yJkL86NPaU15qcRRWpOJqmMl4SJYkyZGyL1Rw9/GtLVurKw==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "0.0.26", + "@ai-sdk/provider-utils": "1.0.22", + "json-schema": "^0.4.0", + "secure-json-parse": "^2.7.0", + "zod-to-json-schema": "^3.23.3" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.0.0" + }, + "peerDependenciesMeta": { + "zod": { + "optional": true + } + } + }, + "packages/mongodb-artifact-generator/node_modules/@ai-sdk/ui-utils/node_modules/@ai-sdk/provider": { + "version": "0.0.26", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-0.0.26.tgz", + "integrity": "sha512-dQkfBDs2lTYpKM8389oopPdQgIU007GQyCbuPPrV+K6MtSII3HBfE0stUIMXUb44L+LK1t6GXPP7wjSzjO6uKg==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "packages/mongodb-artifact-generator/node_modules/@ai-sdk/vue": { + "version": "0.0.58", + "resolved": "https://registry.npmjs.org/@ai-sdk/vue/-/vue-0.0.58.tgz", + "integrity": "sha512-8cuIekJV+jYz68Z+EDp8Df1WNiBEO1NOUGNCy+5gqIi+j382YjuhZfzC78zbzg0PndfF5JzcXhWPqmcc0loUQA==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider-utils": "1.0.22", + "@ai-sdk/ui-utils": "0.0.49", + "swrv": "^1.0.4" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "vue": "^3.3.4" + }, + "peerDependenciesMeta": { + "vue": { + "optional": true + } + } + }, + "packages/mongodb-artifact-generator/node_modules/@braintrust/core": { + "version": "0.0.63", + "resolved": "https://registry.npmjs.org/@braintrust/core/-/core-0.0.63.tgz", + "integrity": "sha512-0b9hfso5Fy/ijta7lam41GTR1YYoQ5c+WZmRFnfHjw3FILbWVRZ7JxE+eo15X416T+UwtfhUaDu0k13JPe7RRA==", + "license": "MIT", + "dependencies": { + "@asteasolutions/zod-to-openapi": "^6.3.1", + "uuid": "^9.0.1", + "zod": "^3.22.4" + } + }, + "packages/mongodb-artifact-generator/node_modules/@esbuild/darwin-arm64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.18.20.tgz", + "integrity": "sha512-bxRHW5kHU38zS2lPTPOyuyTm+S+eobPUnTNkdJEfAddYgEcll4xkT8DB9d2008DtTbl7uJag2HuE5NZAZgnNEA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, "packages/mongodb-artifact-generator/node_modules/@octokit/auth-token": { "version": "4.0.0", "license": "MIT", @@ -42529,6 +42718,116 @@ "dev": true, "license": "MIT" }, + "packages/mongodb-artifact-generator/node_modules/ai": { + "version": "3.4.27", + "resolved": "https://registry.npmjs.org/ai/-/ai-3.4.27.tgz", + "integrity": "sha512-xM5EuCSnqnZ2+3VeIkghj5Tgu+SttOkUHsCBhZrspt7EqI1+kesEeLxJMk/9vLpb9jakF4Fd0+P7wirUPRWDRg==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "0.0.26", + "@ai-sdk/provider-utils": "1.0.22", + "@ai-sdk/react": "0.0.68", + "@ai-sdk/solid": "0.0.53", + "@ai-sdk/svelte": "0.0.56", + "@ai-sdk/ui-utils": "0.0.49", + "@ai-sdk/vue": "0.0.58", + "@opentelemetry/api": "1.9.0", + "eventsource-parser": "1.1.2", + "json-schema": "^0.4.0", + "jsondiffpatch": "0.6.0", + "secure-json-parse": "^2.7.0", + "zod-to-json-schema": "^3.23.3" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "openai": "^4.42.0", + "react": "^18 || ^19 || ^19.0.0-rc", + "sswr": "^2.1.0", + "svelte": "^3.0.0 || ^4.0.0 || ^5.0.0", + "zod": "^3.0.0" + }, + "peerDependenciesMeta": { + "openai": { + "optional": true + }, + "react": { + "optional": true + }, + "sswr": { + "optional": true + }, + "svelte": { + "optional": true + }, + "zod": { + "optional": true + } + } + }, + "packages/mongodb-artifact-generator/node_modules/ai/node_modules/@ai-sdk/provider": { + "version": "0.0.26", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-0.0.26.tgz", + "integrity": "sha512-dQkfBDs2lTYpKM8389oopPdQgIU007GQyCbuPPrV+K6MtSII3HBfE0stUIMXUb44L+LK1t6GXPP7wjSzjO6uKg==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "packages/mongodb-artifact-generator/node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "license": "Python-2.0" + }, + "packages/mongodb-artifact-generator/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "packages/mongodb-artifact-generator/node_modules/braintrust": { + "version": "0.0.167", + "resolved": "https://registry.npmjs.org/braintrust/-/braintrust-0.0.167.tgz", + "integrity": "sha512-1MWD4va6QIxdwDDHRPXeCaIBYQjwMwl/914PHMuwWJNovRbrUOmnZ9pggvYw8hHUg++y1SBSmdrVtLZn1ZB55g==", + "license": "MIT", + "dependencies": { + "@ai-sdk/provider": "^0.0.11", + "@braintrust/core": "0.0.63", + "@next/env": "^14.2.3", + "@vercel/functions": "^1.0.2", + "ai": "^3.2.16", + "argparse": "^2.0.1", + "chalk": "^4.1.2", + "cli-progress": "^3.12.0", + "dotenv": "^16.4.5", + "esbuild": "^0.18.20", + "eventsource-parser": "^1.1.2", + "graceful-fs": "^4.2.11", + "minimatch": "^9.0.3", + "mustache": "^4.2.0", + "pluralize": "^8.0.0", + "simple-git": "^3.21.0", + "slugify": "^1.6.6", + "source-map": "^0.7.4", + "uuid": "^9.0.1", + "zod": "^3.22.4", + "zod-to-json-schema": "^3.22.5" + }, + "bin": { + "braintrust": "dist/cli.js" + }, + "peerDependencies": { + "zod": "^3.0.0" + } + }, "packages/mongodb-artifact-generator/node_modules/cliui": { "version": "8.0.1", "license": "ISC", @@ -42545,6 +42844,120 @@ "version": "8.0.0", "license": "MIT" }, + "packages/mongodb-artifact-generator/node_modules/esbuild": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.18.20.tgz", + "integrity": "sha512-ceqxoedUrcayh7Y7ZX6NdbbDzGROiyVBgC4PriJThBKSVPWnnFHZAkfI1lJT8QFkOwH4qOS2SJkS4wvpGl8BpA==", + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=12" + }, + "optionalDependencies": { + "@esbuild/android-arm": "0.18.20", + "@esbuild/android-arm64": "0.18.20", + "@esbuild/android-x64": "0.18.20", + "@esbuild/darwin-arm64": "0.18.20", + "@esbuild/darwin-x64": "0.18.20", + "@esbuild/freebsd-arm64": "0.18.20", + "@esbuild/freebsd-x64": "0.18.20", + "@esbuild/linux-arm": "0.18.20", + "@esbuild/linux-arm64": "0.18.20", + "@esbuild/linux-ia32": "0.18.20", + "@esbuild/linux-loong64": "0.18.20", + "@esbuild/linux-mips64el": "0.18.20", + "@esbuild/linux-ppc64": "0.18.20", + "@esbuild/linux-riscv64": "0.18.20", + "@esbuild/linux-s390x": "0.18.20", + "@esbuild/linux-x64": "0.18.20", + "@esbuild/netbsd-x64": "0.18.20", + "@esbuild/openbsd-x64": "0.18.20", + "@esbuild/sunos-x64": "0.18.20", + "@esbuild/win32-arm64": "0.18.20", + "@esbuild/win32-ia32": "0.18.20", + "@esbuild/win32-x64": "0.18.20" + } + }, + "packages/mongodb-artifact-generator/node_modules/form-data-encoder": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz", + "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==", + "license": "MIT", + "optional": true, + "peer": true + }, + "packages/mongodb-artifact-generator/node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "license": "ISC" + }, + "packages/mongodb-artifact-generator/node_modules/minimatch": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", + "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "packages/mongodb-artifact-generator/node_modules/openai": { + "version": "4.68.4", + "resolved": "https://registry.npmjs.org/openai/-/openai-4.68.4.tgz", + "integrity": "sha512-LRinV8iU9VQplkr25oZlyrsYGPGasIwYN8KFMAAFTHHLHjHhejtJ5BALuLFrkGzY4wfbKhOhuT+7lcHZ+F3iEA==", + "license": "Apache-2.0", + "optional": true, + "peer": true, + "dependencies": { + "@types/node": "^18.11.18", + "@types/node-fetch": "^2.6.4", + "abort-controller": "^3.0.0", + "agentkeepalive": "^4.2.1", + "form-data-encoder": "1.7.2", + "formdata-node": "^4.3.2", + "node-fetch": "^2.6.7" + }, + "bin": { + "openai": "bin/cli" + }, + "peerDependencies": { + "zod": "^3.23.8" + }, + "peerDependenciesMeta": { + "zod": { + "optional": true + } + } + }, + "packages/mongodb-artifact-generator/node_modules/openai/node_modules/@types/node": { + "version": "18.19.61", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.61.tgz", + "integrity": "sha512-z8fH66NcVkDzBItOao+Nyh0fiy7CYdxIyxnNCcZ60aY0I+EA/y4TSi/S/W9i8DIQvwVo7a0pgzAxmDeNnqrpkw==", + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "undici-types": "~5.26.4" + } + }, + "packages/mongodb-artifact-generator/node_modules/source-map": { + "version": "0.7.4", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.4.tgz", + "integrity": "sha512-l3BikUxvPOcn5E74dZiq5BGsTb5yEwhaTSzccU6t4sDOH8NWJCstKO5QT2CvtFoK6F0saL7p9xHAqHOlCPJygA==", + "license": "BSD-3-Clause", + "engines": { + "node": ">= 8" + } + }, "packages/mongodb-artifact-generator/node_modules/string-width": { "version": "4.2.3", "license": "MIT", @@ -42602,6 +43015,15 @@ "node": ">=12" } }, + "packages/mongodb-artifact-generator/node_modules/zod-to-json-schema": { + "version": "3.23.5", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.23.5.tgz", + "integrity": "sha512-5wlSS0bXfF/BrL4jPAbz9da5hDlDptdEppYfe+x4eIJ7jioqKG9uUxOwPzqof09u/XeVdrgFu29lZi+8XNDJtA==", + "license": "ISC", + "peerDependencies": { + "zod": "^3.23.3" + } + }, "packages/mongodb-atlas": { "version": "1.0.0", "license": "Apache-2.0", diff --git a/packages/mongodb-artifact-generator/package.json b/packages/mongodb-artifact-generator/package.json index 3da8f02bc..3e480807c 100644 --- a/packages/mongodb-artifact-generator/package.json +++ b/packages/mongodb-artifact-generator/package.json @@ -53,6 +53,7 @@ "@octokit/rest": "^20.1.0", "@supercharge/promise-pool": "^3.2.0", "@types/jira-client": "^7.1.9", + "braintrust": "^0.0.167", "common-tags": "^1.8.2", "dotenv": "^16.4.5", "front-matter": "^4.0.2", diff --git a/packages/mongodb-artifact-generator/src/chat/makeSummarizer.eval.ts b/packages/mongodb-artifact-generator/src/chat/makeSummarizer.eval.ts new file mode 100644 index 000000000..7b1fe077d --- /dev/null +++ b/packages/mongodb-artifact-generator/src/chat/makeSummarizer.eval.ts @@ -0,0 +1,123 @@ +import { Eval, EvalCase, traced, EvalScorer } from "braintrust"; +import { PromptExamplePair } from "./utils"; +import { makeSummarizer } from "./makeSummarizer"; +import { + assertEnvVars, + AzureKeyCredential, + OpenAIClient, +} from "mongodb-rag-core"; + +interface SummarizerEvalCaseInput { + text: string; +} +type SummarizerEvalCase = EvalCase< + SummarizerEvalCaseInput, + unknown, + unknown +> & { + name: string; +}; + +const poems = [ + [ + "The moon has a face like the clock in the hall;\nShe shines on thieves on the garden wall,\nOn streets and fields and harbor quays,\nAnd birdies asleep in the forks of the trees.", + { + description: + "The moon is compared to a clock, casting its light over different scenes, from thieves to sleeping birds.", + topics: ["moon", "night", "nature", "light"], + }, + ], + [ + "The rose is a rose,\nAnd was always a rose.\nBut the theory now goes\nThat the apple’s a rose,\nAnd the pear is, and so’s\nThe plum, I suppose.", + { + description: + "The poem reflects on the constancy and transformation of things, suggesting all things can be seen as a rose.", + topics: ["roses", "nature", "change", "metaphor"], + }, + ], + [ + "Fog drifts in,\nSoftly, softly,\nBlanketing the world\nIn a whisper.", + { + description: + "A quiet fog moves in, gently covering the surroundings in silence.", + topics: ["fog", "silence", "nature", "stillness"], + }, + ], +] satisfies PromptExamplePair[]; + +const { OPENAI_ENDPOINT, OPENAI_API_KEY, OPENAI_CHAT_COMPLETION_DEPLOYMENT } = + assertEnvVars({ + OPENAI_ENDPOINT: "", + OPENAI_API_KEY: "", + OPENAI_CHAT_COMPLETION_DEPLOYMENT: "", + }); + +const summarizePoem = makeSummarizer({ + openAi: { + client: new OpenAIClient( + OPENAI_ENDPOINT, + new AzureKeyCredential(OPENAI_API_KEY) + ), + deployment: OPENAI_CHAT_COMPLETION_DEPLOYMENT, + }, + directions: "The provided content will be a poem.", + examples: [ + [ + "Two roads diverged in a yellow wood,\nAnd sorry I could not travel both\nAnd be one traveler, long I stood\nAnd looked down one as far as I could\nTo where it bent in the undergrowth;", + { + description: + "A traveler encounters a fork in a road in a forest and reflects on the decision of choosing a path.", + topics: ["travel", "decisions", "forests", "fork in the road"], + }, + ], + [ + "Hope is the thing with feathers\nThat perches in the soul,\nAnd sings the tune without the words,\nAnd never stops at all,", + { + description: + "Hope is described as a bird that lives in the soul, continuously singing a wordless tune.", + topics: ["hope", "soul", "birds", "music"], + }, + ], + [ + "I wandered lonely as a cloud\nThat floats on high o'er vales and hills,\nWhen all at once I saw a crowd,\nA host, of golden daffodils;", + { + descriptionz: + "A person feels lonely but then finds joy upon seeing a field of daffodils.", + topics: ["loneliness", "flowers"], + }, + ], + ], +}); + +Eval("mongodb-artifact-generator/chat/summarizer", { + experimentName: "summarizer-poetry", + metadata: { + description: + "Evaluates how well the MongoDB Artifact Generator summarizer works on poetry", + }, + maxConcurrency: 2, + data: async () => { + return poems.map(([text, expected]) => { + return { + name: `summarizes poem: ${expected.description}`, + input: { text }, + expected, + metadata: null, + } satisfies SummarizerEvalCase; + }); + }, + task: async (input) => { + return traced( + async () => { + const summary = await summarizePoem({ input: input.text }); + return summary; + }, + { + name: "summarizePoem", + } + ); + }, + scores: [ + // TODO: Add a scorer + ], +});