(EAI-383) Fetch & Refine Top Jira Tickets

.drone.yml

@@ -31,6 +31,7 @@ steps:
       OPENAI_CHAT_COMPLETION_MODEL_VERSION: 2023-06-01-preview
       OPENAI_PREPROCESSOR_CHAT_COMPLETION_DEPLOYMENT: gpt-4o-mini
       OPENAI_API_VERSION: "2024-06-01"
+      GENERATOR_LLM_MAX_CONCURRENCY: 8
       MONGODB_CONNECTION_URI:
         from_secret: mongodb_connection_uri
       OPENAI_ENDPOINT:

packages/mongodb-artifact-generator/package.json

@@ -61,6 +61,8 @@
     "papaparse": "^5.4.1",
     "yaml": "^2.3.1",
     "yargs": "^17",
-    "zod": "^3.22.4"
+    "zod": "^3.22.4",
+    "zod-to-json-schema": "^3.23.2",
+    "zod-validation-error": "^3.3.1"
   }
 }

packages/mongodb-artifact-generator/src/ArtifactGeneratorEnvVars.ts

@@ -1,5 +1,10 @@
 import { CORE_ENV_VARS } from "mongodb-rag-core";
 
+export const GENERATOR_LLM_ENV_VARS = {
+  GENERATOR_LLM_MAX_CONCURRENCY: "",
+};
+
 export const ArtifactGeneratorEnvVars = {
   ...CORE_ENV_VARS,
+  ...GENERATOR_LLM_ENV_VARS,
 };

packages/mongodb-artifact-generator/src/chat/makeGeneratePrompts.ts

@@ -0,0 +1,104 @@
+import {
+  ChatRequestMessage,
+  FunctionDefinition,
+  OpenAIClient,
+} from "mongodb-rag-core";
+import { FormattedJiraIssueWithSummary } from "../commands/generateJiraPromptResponse";
+import { RunLogger } from "../runlogger";
+import {
+  asJsonSchema,
+  formatFewShotExamples,
+  formatMessagesForArtifact,
+  PromptExamplePair,
+} from "./utils";
+import { z } from "zod";
+import { stripIndents } from "common-tags";
+
+export type GeneratedPrompts = z.infer<typeof GeneratedPrompts>;
+export const GeneratedPrompts = z.object({
+  prompts: z.array(z.string()),
+});
+
+const generatePromptsTool: FunctionDefinition = {
+  name: "generatePrompts",
+  description:
+    "A list of generated example prompts that would elicit a given response.",
+  parameters: asJsonSchema(GeneratedPrompts),
+};
+
+export type MakeGeneratePromptsArgs = {
+  openAi: {
+    client: OpenAIClient;
+    deployment: string;
+  };
+  logger?: RunLogger;
+  directions?: string;
+  examples?: PromptExamplePair[];
+};
+
+export type GeneratePromptsArgs = FormattedJiraIssueWithSummary;
+
+export function makeGeneratePrompts({
+  openAi,
+  logger,
+  directions,
+  examples = [],
+}: MakeGeneratePromptsArgs) {
+  return async function generatePrompts({
+    issue,
+    summary,
+  }: GeneratePromptsArgs) {
+    const messages = [
+      {
+        role: "system",
+        content: [
+          `Your task is to convert a provided input into a prompt-response format. The format mimics a conversation where one participant sends a prompt and the other replies with a response.`,
+          directions ?? "",
+        ].join("\n"),
+      },
+      ...formatFewShotExamples({
+        examples,
+        functionName: generatePromptsTool.name,
+        responseSchema: GeneratedPrompts,
+      }),
+      {
+        role: "user",
+        content: JSON.stringify({ issue, summary }),
+      },
+    ] satisfies ChatRequestMessage[];
+    const result = await openAi.client.getChatCompletions(
+      openAi.deployment,
+      messages,
+      {
+        temperature: 0,
+        maxTokens: 1500,
+        functions: [generatePromptsTool],
+        functionCall: {
+          name: generatePromptsTool.name,
+        },
+      }
+    );
+    const response = result.choices[0].message;
+    if (response === undefined) {
+      throw new Error("No response from OpenAI");
+    }
+    if (response.functionCall === undefined) {
+      throw new Error("No function call in response from OpenAI");
+    }
+    const generatedPrompts = GeneratedPrompts.parse(
+      JSON.parse(response.functionCall.arguments)
+    );
+
+    logger?.appendArtifact(
+      `chatTemplates/generatePrompts-${Date.now()}.json`,
+      stripIndents`
+        ${formatMessagesForArtifact(messages).join("\n")}
+        <Summary>
+          ${JSON.stringify(summary)}
+        </Summary>
+      `
+    );
+
+    return generatedPrompts;
+  };
+}

packages/mongodb-artifact-generator/src/chat/makeGenerateResponse.ts

@@ -0,0 +1,109 @@
+import {
+  ChatRequestMessage,
+  FunctionDefinition,
+  OpenAIClient,
+} from "mongodb-rag-core";
+import { FormattedJiraIssue } from "../commands/generateJiraPromptResponse";
+import { RunLogger } from "../runlogger";
+import {
+  asJsonSchema,
+  formatFewShotExamples,
+  formatMessagesForArtifact,
+  PromptExamplePair,
+} from "./utils";
+import { z } from "zod";
+import { stripIndents } from "common-tags";
+import { Summary } from "./makeSummarizer";
+
+export type GeneratedResponse = z.infer<typeof GeneratedResponse>;
+export const GeneratedResponse = z.object({
+  response: z.string().describe("The generated response text."),
+});
+
+const generateResponseTool: FunctionDefinition = {
+  name: "generateResponse",
+  description: "A response generated based on a given context.",
+  parameters: asJsonSchema(GeneratedResponse),
+};
+
+export type MakeGenerateResponseArgs = {
+  openAi: {
+    client: OpenAIClient;
+    deployment: string;
+  };
+  logger?: RunLogger;
+  directions?: string;
+  examples?: PromptExamplePair[];
+};
+
+export type GenerateResponseArgs = {
+  issue: FormattedJiraIssue;
+  summary: Summary;
+  prompt: string;
+};
+
+export function makeGenerateResponse({
+  openAi,
+  logger,
+  directions,
+  examples = [],
+}: MakeGenerateResponseArgs) {
+  return async function generateResponse({
+    issue,
+    summary,
+    prompt,
+  }: GenerateResponseArgs) {
+    const messages = [
+      {
+        role: "system",
+        content: [
+          `Your task is to generate a response to a provided input. The response should be relevant to the input and based only on the provided context.`,
+          directions ?? "",
+        ].join("\n"),
+      },
+      ...formatFewShotExamples({
+        examples,
+        functionName: generateResponseTool.name,
+        responseSchema: GeneratedResponse,
+      }),
+      {
+        role: "user",
+        content: JSON.stringify({ issue, summary, prompt }),
+      },
+    ] satisfies ChatRequestMessage[];
+    const result = await openAi.client.getChatCompletions(
+      openAi.deployment,
+      messages,
+      {
+        temperature: 0,
+        maxTokens: 1500,
+        functions: [generateResponseTool],
+        functionCall: {
+          name: generateResponseTool.name,
+        },
+      }
+    );
+    const response = result.choices[0].message;
+    if (response === undefined) {
+      throw new Error("No response from OpenAI");
+    }
+    if (response.functionCall === undefined) {
+      throw new Error("No function call in response from OpenAI");
+    }
+    const generatedResponse = GeneratedResponse.parse(
+      JSON.parse(response.functionCall.arguments)
+    );
+
+    logger?.appendArtifact(
+      `chatTemplates/generateResponse-${Date.now()}.json`,
+      stripIndents`
+        ${formatMessagesForArtifact(messages).join("\n")}
+        <Summary>
+          ${JSON.stringify(summary)}
+        </Summary>
+      `
+    );
+
+    return generatedResponse;
+  };
+}

packages/mongodb-artifact-generator/src/chat/makeSummarizer.test.ts

@@ -0,0 +1,89 @@
+import {
+  AzureKeyCredential,
+  CORE_ENV_VARS,
+  OpenAIClient,
+  assertEnvVars,
+} from "mongodb-rag-core";
+import { Summary, makeSummarizer } from "./makeSummarizer";
+import { PromptExamplePair } from "./utils";
+
+const { OPENAI_CHAT_COMPLETION_DEPLOYMENT, OPENAI_ENDPOINT, OPENAI_API_KEY } =
+  assertEnvVars(CORE_ENV_VARS);
+
+const openAiClient = new OpenAIClient(
+  OPENAI_ENDPOINT,
+  new AzureKeyCredential(OPENAI_API_KEY)
+);
+
+const openAi = {
+  client: openAiClient,
+  deployment: OPENAI_CHAT_COMPLETION_DEPLOYMENT,
+};
+
+// TODO: Convert this into a .eval file
+describe.skip("makeSummarizer", () => {
+  it("creates a summarizer for content", async () => {
+    const summarizePoem = makeSummarizer({
+      openAi,
+      directions: "The provided content will be a poem.",
+      examples: [
+        [
+          "Two roads diverged in a yellow wood,\nAnd sorry I could not travel both\nAnd be one traveler, long I stood\nAnd looked down one as far as I could\nTo where it bent in the undergrowth;",
+          {
+            description:
+              "A traveler encounters a fork in a road in a forest and reflects on the decision of choosing a path.",
+            topics: ["travel", "decisions", "forests", "fork in the road"],
+          },
+        ],
+        [
+          "Hope is the thing with feathers\nThat perches in the soul,\nAnd sings the tune without the words,\nAnd never stops at all,",
+          {
+            description:
+              "Hope is described as a bird that lives in the soul, continuously singing a wordless tune.",
+            // topics: ["hope", "soul", "birds", "music"],
+          },
+        ],
+        [
+          "I wandered lonely as a cloud\nThat floats on high o'er vales and hills,\nWhen all at once I saw a crowd,\nA host, of golden daffodils;",
+          {
+            descriptionz:
+              "A person feels lonely but then finds joy upon seeing a field of daffodils.",
+            topics: ["loneliness", "flowers"],
+          },
+        ],
+      ],
+    });
+
+    const testCases = [
+      [
+        "The moon has a face like the clock in the hall;\nShe shines on thieves on the garden wall,\nOn streets and fields and harbor quays,\nAnd birdies asleep in the forks of the trees.",
+        {
+          description:
+            "The moon is compared to a clock, casting its light over different scenes, from thieves to sleeping birds.",
+          topics: ["moon", "night", "nature", "light"],
+        },
+      ],
+      [
+        "The rose is a rose,\nAnd was always a rose.\nBut the theory now goes\nThat the apple’s a rose,\nAnd the pear is, and so’s\nThe plum, I suppose.",
+        {
+          description:
+            "The poem reflects on the constancy and transformation of things, suggesting all things can be seen as a rose.",
+          topics: ["roses", "nature", "change", "metaphor"],
+        },
+      ],
+      [
+        "Fog drifts in,\nSoftly, softly,\nBlanketing the world\nIn a whisper.",
+        {
+          description:
+            "A quiet fog moves in, gently covering the surroundings in silence.",
+          topics: ["fog", "silence", "nature", "stillness"],
+        },
+      ],
+    ] satisfies PromptExamplePair[];
+
+    for (const [poem, expected] of testCases) {
+      const summary = await summarizePoem({ input: poem });
+      // expect(summary).toEqual(expected);
+    }
+  });
+});