Add Azure OpenAI mixin

azure-openai-mixin/README.md

@@ -0,0 +1,166 @@
+# Hello world observ lib
+
+This lib can be used as a starter template of modular observ lib.
+
+
+## Import
+
+```sh
+jb init
+jb install https://github.com/grafana/jsonnet-libs/helloworld-observ-lib
+```
+
+## Modular observability lib format
+
+```jsonnet
+
+{
+  config: {
+    //common options
+  },
+
+  grafana: {
+
+   // grafana templated variables to reuse across mutltiple dashboards
+   variables: {
+      datasources: {
+        prometheus: {},
+        loki: {},
+      },
+      multiInstance: [],
+      singleInstace: [],
+  	  queriesSelector: "",
+   },
+
+   // grafana targets (queries) to attach to panels
+   targets: {
+    target1: <target1>,
+    target3: <target2>,
+    ...
+    targetN: <targetN>,
+   },
+
+   // grafana panels
+   panels: {
+    panel1: <panel1>,
+    panel2: <panel2>,
+    ...
+    panelN: <panelN>,
+   },
+
+   // grafana dashboards
+   dashboards: {
+    dashboard1: <dashboard1>,
+    dashboard2: <dashboard2>,
+    ...
+    dashboardN: <dashboardN>,
+   },
+
+   // grafana annotations
+   annotations: {
+    annotation1: <annotation1>,
+    ...
+   },
+
+   // grafana dashboard links
+   links: {
+      //common dashobard links
+   },
+ },
+
+ prometheus: {
+  alerts: {},
+  rules: {},
+ },
+}
+
+```
+
+## Pros of using modular observabilty format
+
+- Uses (grafonnet)[https://monitoring.mixins.dev]
+- Highly customizable and flexible:
+
+Any object like `panel`, `target` (query) can be easily referenced by key and then overriden before output of the lib is provided by using jsonnet (patching)[https://tanka.dev/tutorial/environments#patching] technique:
+
+```jsonnet
+local helloworldlib = import './main.libsonnet';
+
+local helloworld =
+  helloworldlib.new(
+    filteringSelector='job="integrations/helloworld"',
+    uid='myhelloworld',
+    groupLabels=['environment', 'cluster'],
+    instanceLabels=['host'],
+  )
+  + 
+  {
+    grafana+: {
+      panels+: {
+        panel1+: 
+          g.panel.timeSeries.withDescription("My new description for panel1")
+      }
+    }
+  };
+```
+
+- Due to high decomposition level, not only dashboards but single panels can be imported ('cherry-picked) from the library to be used in other dashboards
+- Format introduces mandatory arguments that each library should have: `filteringSelector`, `instanceLabels`, `groupLabels`, `uid`. Proper use of those parameters ensures library can be used to instantiate multiple copies of the observability package in the same enviroment without `ids` conflicts or timeSeries overlapping.
+
+## Examples
+
+### Monitoring-Mixin example
+
+You can use lib to fill in [monitoring-mixin](https://monitoring.mixins.dev/) structure:
+
+```jsonnet
+// mixin.libsonnet file
+
+local helloworldlib = import './main.libsonnet';
+
+local helloworld =
+  helloworldlib.new(
+    filteringSelector='job="integrations/helloworld"',
+    uid='myhelloworld',
+    groupLabels=['environment', 'cluster'],
+    instanceLabels=['host'],
+  );
+
+// populate monitoring-mixin:
+{
+  grafanaDashboards+:: helloworld.grafana.dashboards,
+  prometheusAlerts+:: helloworld.prometheus.alerts,
+  prometheusRules+:: helloworld.prometheus.recordingRules,
+}
+```
+
+### Logs collection
+
+Grafana Loki is used to populate logs dashboard and also for quering annotations.
+
+To opt-out, you can set `enableLokiLogs: false` in config:
+
+```
+local helloworldlib = import './main.libsonnet';
+
+local helloworld =
+  helloworldlib.new(
+    filteringSelector='job="integrations/helloworld"',
+    uid='myhelloworld',
+    groupLabels=['environment', 'cluster'],
+    instanceLabels=['host'],
+  )
+  + helloworldlib.withConfigMixin(
+    {
+      // disable loki logs
+      enableLokiLogs: false,
+    }
+  );
+
+// populate monitoring-mixin:
+{
+  grafanaDashboards+:: helloworld.grafana.dashboards,
+  prometheusAlerts+:: helloworld.prometheus.alerts,
+  prometheusRules+:: helloworld.prometheus.recordingRules,
+}
+```

azure-openai-mixin/alerts.libsonnet

@@ -0,0 +1,44 @@
+{
+  new(this): {
+
+    groups: [
+      {
+        name: 'alerts-' + this.config.uid,
+        rules: [
+          {
+            alert: 'HighLatency',
+            expr: |||
+              100 - (avg (rate(azure_microsoft_cognitiveservices_accounts_latency{%(filteringSelector)s}[5m])) * 100) > %(alertsThresholdLatency)s
+            ||| % this.config,
+            'for': '5m',
+            labels: {
+              severity: 'warning',
+            },
+            annotations: {
+              summary: 'High latency for Azure OpenAI calls.',
+              description: |||
+                Latency for Azure OpenAI calls on {{ $labels.resourceName }} is greater than %(alertsThresholdLatency)s%%. The currect value is {{ $value | printf "%%.2f" }}.
+              ||| % this.config,
+            },
+          },
+          {
+            alert: 'HighErrorRate',
+            expr: |||
+              100 - (avg (rate(azure_microsoft_cognitiveservices_accounts_successrate_average_percent{%(filteringSelector)s}[5m])) * 100) > %(alertsThresholdErrorRate)s
+            ||| % this.config,
+            'for': '5m',
+            labels: {
+              severity: 'warning',
+            },
+            annotations: {
+              summary: 'High error for Azure OpenAI calls.',
+              description: |||
+                Error rate for Azure OpenAI calls on {{ $labels.resourceName }} is greater than %(alertsThresholdErrorRate)s%%. The currect value is {{ $value | printf "%%.2f" }}.
+              ||| % this.config,
+            },
+          },
+        ],
+      },
+    ],
+  },
+}

azure-openai-mixin/annotations.libsonnet

@@ -0,0 +1,11 @@
+local commonlib = import 'common-lib/common/main.libsonnet';
+
+{
+  new(this):
+    local grafana = this.grafana;
+    local instanceLabels = this.config.instanceLabels;
+    local groupLabels = this.config.groupLabels;
+    {
+
+    },
+}

azure-openai-mixin/dashboards.libsonnet

@@ -0,0 +1,57 @@
+local g = import './g.libsonnet';
+local logslib = import 'github.com/grafana/jsonnet-libs/logs-lib/logs/main.libsonnet';
+{
+  local root = self,
+  new(this):
+    local prefix = this.config.dashboardNamePrefix;
+    local links = this.grafana.links;
+    local tags = this.config.dashboardTags;
+    local uid = g.util.string.slugify(this.config.uid);
+    local vars = this.grafana.variables;
+    local annotations = this.grafana.annotations;
+    local refresh = this.config.dashboardRefresh;
+    local period = this.config.dashboardPeriod;
+    local timezone = this.config.dashboardTimezone;
+    local panels = this.grafana.panels;
+    local stat = g.panel.stat;
+    {
+      'azure-openai-overview.json':
+        g.dashboard.new(prefix + 'Overview')
+        + g.dashboard.withPanels(
+          g.util.grid.wrapPanels(
+            [
+              g.panel.row.new('Overview'),
+              panels.totalCalls,
+              panels.successCalls,
+              panels.successRate,
+              panels.totalErrors,
+              panels.errorsRate,
+              g.panel.row.new('Tokens overview'),
+              panels.generatedTokens,
+              panels.tokenTransactions,
+              panels.processedPromptTokens,
+              panels.processedInferenceTokens,
+              g.panel.row.new('Troubleshooting'),
+              panels.rateLimitedCalls,
+              panels.blockedCalls,
+              panels.clientErrors,
+              panels.fineTunedTrainingHours,
+              panels.dataIn,
+              panels.dataOut,
+            ], 4, 3
+          )
+        )
+        + root.applyCommon(vars.singleInstance, uid + '-overview', tags, links { backToOverview+:: {} }, annotations, timezone, refresh, period),
+    },
+  //Apply common options(uids, tags, annotations etc..) to all dashboards above
+  applyCommon(vars, uid, tags, links, annotations, timezone, refresh, period):
+    g.dashboard.withTags(tags)
+    + g.dashboard.withUid(uid)
+    + g.dashboard.withLinks(std.objectValues(links))
+    + g.dashboard.withTimezone(timezone)
+    + g.dashboard.withRefresh(refresh)
+    + g.dashboard.time.withFrom(period)
+    + g.dashboard.withVariables(vars)
+    + g.dashboard.withAnnotations(std.objectValues(annotations)),
+
+}

azure-openai-mixin/g.libsonnet

@@ -0,0 +1 @@
+import 'github.com/grafana/grafonnet/gen/grafonnet-v10.0.0/main.libsonnet'

azure-openai-mixin/jsonnetfile.json

@@ -0,0 +1,33 @@
+{
+  "version": 1,
+  "dependencies": [
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/grafana/jsonnet-libs.git",
+          "subdir": "common-lib"
+        }
+      },
+      "version": "master"
+    },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/grafana/grafonnet.git",
+          "subdir": "gen/grafonnet-v10.0.0"
+        }
+      },
+      "version": "main"
+    },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/grafana/jsonnet-libs.git",
+          "subdir": "logs-lib"
+        }
+      },
+      "version": "master"
+    }
+  ],
+  "legacyImports": true
+}

azure-openai-mixin/main.libsonnet

@@ -0,0 +1,61 @@
+local alerts = import './alerts.libsonnet';
+local annotations = import './annotations.libsonnet';
+local dashboards = import './dashboards.libsonnet';
+local datasources = import './datasources.libsonnet';
+local g = import './g.libsonnet';
+local panels = import './panels.libsonnet';
+local targets = import './targets.libsonnet';
+local variables = import './variables.libsonnet';
+
+{
+  withConfigMixin(config): {
+    config+: config,
+  },
+  new(
+    filteringSelector,
+    groupLabels=['job'],
+    instanceLabels=['instance'],
+    dashboardNamePrefix='',
+    dashboardTags=[uid],
+    uid,
+  ): {
+
+    local this = self,
+    config: {
+      groupLabels: groupLabels,
+      instanceLabels: instanceLabels,
+      filteringSelector: filteringSelector,
+      dashboardTags: dashboardTags,
+      uid: uid,
+      dashboardNamePrefix: dashboardNamePrefix,
+
+      // additional params can be added if needed
+      criticalEvents: '90',
+      alertsThresholdLatency: '10',
+      alertsThresholdErrorRate: '5',
+      dashboardPeriod: 'now-1h',
+      dashboardTimezone: 'default',
+      dashboardRefresh: '1m',
+    },
+
+    grafana: {
+      variables: variables.new(this, varMetric='azure_microsoft_cognitiveservices_accounts_totalcalls_total_count'),
+      targets: targets.new(this),
+      annotations: annotations.new(this),
+      // common dashboards links here
+      links: {
+      },
+
+      panels: panels.new(this),
+      dashboards: dashboards.new(this),
+
+    },
+
+    prometheus: {
+      alerts: alerts.new(this),
+      recordingRules: {},
+    },
+
+  },
+
+}