BugFixes: unquoted strings, openai-generic add api_key for bearer aut…

…h, support escape characters in quoted strings (#965) * Fix trailing comments on unquoted strings ("fooo //" is now correctly interpreted as "foo" + "// comment ..." * openai-generic should accept the api_key value for bearer auth (if not set, no header is provided) * Escape \" when inside of a quoted string. So "foo\"" should now parse.
BoundaryML · Sep 19, 2024 · 847f3a9 · 847f3a9
1 parent 919c79f
commit 847f3a9
Show file tree

Hide file tree

Showing 9 changed files with 81 additions and 34 deletions.
diff --git a/.github/workflows/primary.yml b/.github/workflows/primary.yml
@@ -10,6 +10,8 @@ on:
       - ".github/workflows/primary.yml"
     branches:
       - canary
+  merge_group:
+    types: [checks_requested]
   workflow_dispatch: {}
 
 concurrency:

diff --git a/docs/docs/snippets/clients/providers/openai-generic.mdx b/docs/docs/snippets/clients/providers/openai-generic.mdx
@@ -36,6 +36,13 @@ client<llm> MyClient {
   **Default: `system`**
 </ParamField>
 
+<ParamField path="api_key" type="string" default="<none>">
+  Will be used to build the `Authorization` header, like so: `Authorization: Bearer $api_key`
+  If `api_key` is not set, or is set to an empty string, the `Authorization` header will not be sent.
+
+  **Default: `<none>`**
+</ParamField>
+
 <ParamField path="headers" type="object">
   Additional headers to send with the request.
 

diff --git a/engine/baml-lib/baml/tests/validation_files/strings/quoted_strings.baml b/engine/baml-lib/baml/tests/validation_files/strings/quoted_strings.baml
@@ -0,0 +1,6 @@
+client<llm> Hello {
+  provider baml-openai-chat // Trailing chat
+  options {
+    thing "word with \"quotes\""
+  }
+}
diff --git a/engine/baml-lib/schema-ast/src/parser/datamodel.pest b/engine/baml-lib/schema-ast/src/parser/datamodel.pest
@@ -105,11 +105,11 @@ numeric_literal = @{ ("-")? ~ ASCII_DIGIT+ ~ ("." ~ ASCII_DIGIT+)? }
 // ######################################
 // String literals. These behave specially in BAML.
 // ######################################
-banned_chars            = @{ "#" | "@" | "{" | "(" | "[" | "<" | "}" | ")" | "]" | ">" | "," | "'" | NEWLINE }
+banned_chars            = @{ "#" | "@" | "{" | "(" | "[" | "<" | "}" | ")" | "]" | ">" | "," | "'" | " //" | NEWLINE }
 banned_start_chars      =  { WHITESPACE | banned_chars }
 banned_end_chars        =  { WHITESPACE | banned_chars }
-unquoted_string_literal =  { (!banned_start_chars ~ ANY) ~ (!banned_chars ~ ANY)* ~ (!banned_end_chars ~ ANY)* }
-quoted_string_content   = @{ (!"\"" ~ !NEWLINE ~ ANY)* }
+unquoted_string_literal =  { (!banned_start_chars ~ ANY) ~ (!banned_chars ~ !"\"" ~ ANY)* ~ (!banned_end_chars ~ ANY)* }
+quoted_string_content   = @{ (("\\\"" | !("\"" | NEWLINE) ~ ANY))* }
 quoted_string_literal   = ${ "\"" ~ quoted_string_content ~ "\"" }
 
 // TODO: Support comments in raw string literals

diff --git a/engine/baml-runtime/src/internal/llm_client/primitive/openai/properties/generic.rs b/engine/baml-runtime/src/internal/llm_client/primitive/openai/properties/generic.rs
@@ -47,10 +47,15 @@ pub fn resolve_properties(
         None => Default::default(),
     };
 
+    let api_key = properties
+        .remove("api_key")
+        .and_then(|v| v.as_str().map(|s| s.to_string()))
+        .filter(|s| !s.is_empty());
+
     Ok(PostRequestProperties {
         default_role,
         base_url,
-        api_key: None,
+        api_key,
         headers,
         properties,
         proxy_url: ctx

diff --git a/typescript/vscode-ext/packages/language-server/src/server.ts b/typescript/vscode-ext/packages/language-server/src/server.ts
@@ -94,7 +94,11 @@ export function startServer(options?: LSOptions): void {
           const warnings = params.errors.reduce((acc, [, diagnostics]) => {
             return acc + diagnostics.filter((d) => d.severity === 2).length
           }, 0)
-          connection.sendRequest('runtime_diagnostics', { errors, warnings })
+          try {
+            connection.sendRequest('runtime_diagnostics', { errors, warnings })
+          } catch (e) {
+            console.error('> Error sending runtime_diagnostics', e)
+          }
           break
         case 'error':
         case 'warn':

diff --git a/typescript/vscode-ext/packages/syntaxes/baml.tmLanguage.json b/typescript/vscode-ext/packages/syntaxes/baml.tmLanguage.json
@@ -145,10 +145,7 @@
           "beginCaptures": {
             "1": { "name": "storage.type.baml" }
           },
-          "patterns": [
-            { "include": "#type_definition" },
-            { "include": "#type_definition" }
-          ],
+          "patterns": [{ "include": "#type_definition" }, { "include": "#type_definition" }],
           "end": ">"
         },
         {
@@ -216,7 +213,7 @@
                   "patterns": [
                     {
                       "match": "\\w+",
-                      "name": "entity.name.other.client" 
+                      "name": "entity.name.other.client"
                     },
                     { "include": "#string_literal" }
                   ]
@@ -270,7 +267,7 @@
               "patterns": [
                 {
                   "match": "\\w+",
-                  "name": "entity.name.other.client" 
+                  "name": "entity.name.other.client"
                 },
                 { "include": "#string_literal" }
               ]
@@ -497,7 +494,6 @@
       "beginCaptures": {
         "1": { "name": "variable.other.readwrite" },
         "2": { "name": "string.quoted.block.baml.startquote" }
-
       },
       "end": "((\"){1,3}#)",
       "endCaptures": {
@@ -541,11 +537,25 @@
       }
     },
     "key_string_pair": {
-      "match": "(\"\\w+\"|\\b\\w+\\b)\\s+(\"[^\"]*\")",
-      "captures": {
+      "begin": "(\"\\w+\"|\\b\\w+\\b)\\s+(\")",
+      "beginCaptures": {
         "1": { "name": "variable.other.readwrite" },
         "2": { "name": "string.quoted.double" }
-      }
+      },
+      "end": "\"",
+      "endCaptures": {
+        "0": { "name": "string.quoted.double" }
+      },
+      "patterns": [
+        {
+          "name": "constant.character.escape",
+          "match": "\\\\."
+        },
+        {
+          "name": "string.quoted.double",
+          "match": "[^\"\\\\]+"
+        }
+      ]
     },
     "key_custom_string_pair": {
       "match": "(\"\\w+\"|\\b\\w+\\b)\\s+((?!null)[^\\s\\[\\{]+)",
@@ -584,13 +594,20 @@
       "contentName": "variable.other.readwrite.array",
       "patterns": [
         { "include": "#key_array_pair" },
-        { "include": "#string_quoted" },
+        { "include": "#string_quoted2" },
         { "include": "#constant_numeric" }
       ]
     },
-    "string_quoted": {
-      "match": "\"[^\"]*\"",
-      "name": "string.quoted.double"
+    "string_quoted2": {
+      "name": "string.quoted.double",
+      "begin": "\"",
+      "end": "\"",
+      "patterns": [
+        {
+          "name": "constant.character.escape",
+          "match": "\\\\."
+        }
+      ]
     },
     "string_unquoted": {
       "match": "\\b\\w+\\b",

diff --git a/typescript/vscode-ext/packages/vscode/src/panels/WebPanelView.ts b/typescript/vscode-ext/packages/vscode/src/panels/WebPanelView.ts
@@ -46,7 +46,7 @@ export class WebPanelView {
     panel: WebviewPanel,
     extensionUri: Uri,
     portLoader: () => number,
-    private reporter: TelemetryReporter,
+    private reporter?: TelemetryReporter,
   ) {
     this._panel = panel
     this._port = portLoader
@@ -105,7 +105,7 @@ export class WebPanelView {
 
   public postMessage<T>(command: string, content: T) {
     this._panel.webview.postMessage({ command: command, content })
-    this.reporter.sendTelemetryEvent({
+    this.reporter?.sendTelemetryEvent({
       event: `baml.webview.${command}`,
       properties: {},
     })
@@ -258,7 +258,7 @@ export class WebPanelView {
             }
             case 'telemetry': {
               const { action, data } = message.meta
-              this.reporter.sendTelemetryEvent({
+              this.reporter?.sendTelemetryEvent({
                 event: `baml.webview.${action}`,
                 properties: data,
               })

diff --git a/typescript/vscode-ext/packages/vscode/src/plugins/language-server/index.ts b/typescript/vscode-ext/packages/vscode/src/plugins/language-server/index.ts
@@ -53,7 +53,10 @@ export const requestDiagnostics = async () => {
 
 export const requestBamlCLIVersion = async () => {
   try {
-    const version = await client.sendRequest('bamlCliVersion')
+    const version = await client?.sendRequest('bamlCliVersion')
+    if (!version) {
+      return
+    }
     console.log('Got BAML CLI version', version)
     bamlConfig.cliVersion = version as string
   } catch (e) {
@@ -172,7 +175,7 @@ const activateClient = (
             },
             async (progress, token) => {
               let customCancellationToken: vscode.CancellationTokenSource | null = null
-              return new Promise(async (resolve) => {
+              const rest = new Promise<null>((resolve) => {
                 customCancellationToken = new vscode.CancellationTokenSource()
 
                 customCancellationToken.token.onCancellationRequested(() => {
@@ -187,14 +190,17 @@ const activateClient = (
                 const totalMs = message.durationMs || 1500 // Total duration in milliseconds (2 seconds)
                 const updateCount = 50 // Number of updates
                 const intervalMs = totalMs / updateCount // Interval between updates
-
-                for (let i = 0; i < updateCount; i++) {
-                  const prog = ((i + 1) / updateCount) * 100
-                  progress.report({ increment: prog, message: message.message })
-                  await sleep(intervalMs)
-                }
-                resolve(null)
+                ;(async () => {
+                  for (let i = 0; i < updateCount; i++) {
+                    const prog = ((i + 1) / updateCount) * 100
+                    progress.report({ increment: prog, message: message.message })
+                    await sleep(intervalMs)
+                  }
+                  resolve(null)
+                })()
               })
+
+              return rest
             },
           )
           break
@@ -245,9 +251,9 @@ const activateClient = (
     // And check again once every hour
     intervalTimers.push(
       setInterval(
-        async () => {
+        () => {
           console.log(`checking for updates ${new Date().toString()}`)
-          await checkForUpdates({ showIfNoUpdates: false })
+          checkForUpdates({ showIfNoUpdates: false })
         },
         60 * 60 * 1000 /* 1h in milliseconds: min/hr * secs/min * ms/sec */,
       ),
@@ -333,7 +339,7 @@ const plugin: BamlVSCodePlugin = {
       }),
 
       commands.registerCommand('baml.checkForUpdates', async () => {
-        checkForUpdates({ showIfNoUpdates: true }).catch((e) => {
+        await checkForUpdates({ showIfNoUpdates: true }).catch((e) => {
           console.error('Failed to check for updates', e)
         })
       }),