install and doc changes needed to introduce builtin metrics (#750)

* install and doc changes needed to introduce builtin metrics Signed-off-by: Srinivasan Parthasarathy <[email protected]> * adding metrics task description Signed-off-by: Srinivasan Parthasarathy <[email protected]>
iter8-tools · Jun 8, 2021 · 0377ecd · 0377ecd
1 parent 69ee426
commit 0377ecd
Show file tree

Hide file tree

Showing 23 changed files with 353 additions and 140 deletions.
diff --git a/install/builtin-metrics/kustomization.yaml b/install/builtin-metrics/kustomization.yaml
@@ -0,0 +1,7 @@
+namespace: iter8-system
+
+commonLabels:
+  creator: iter8
+
+resources:
+- metrics.yaml
diff --git a/install/builtin-metrics/metrics.yaml b/install/builtin-metrics/metrics.yaml
@@ -0,0 +1,107 @@
+apiVersion: iter8.tools/v2alpha2
+kind: Metric
+metadata:
+  name: request-count
+spec:
+  description: Number of requests (Iter8 builtin metric)
+  type: Counter
+  provider: iter8
+  jqExpression: "dummy"
+  urlTemplate: https://iter8.tools
+---
+apiVersion: iter8.tools/v2alpha2
+kind: Metric
+metadata:
+  name: error-count
+spec:
+  description: Number of responses with HTTP status code 4xx or 5xx (Iter8 builtin metric)
+  type: Counter
+  provider: iter8
+  jqExpression: "dummy"
+  urlTemplate: https://iter8.tools
+---
+apiVersion: iter8.tools/v2alpha2
+kind: Metric
+metadata:
+  name: error-rate
+spec:
+  description: Fraction of responses with HTTP status code 4xx or 5xx (Iter8 builtin metric)
+  type: Gauge
+  sampleSize: iter8-system/request-count
+  provider: iter8
+  jqExpression: "dummy"
+  urlTemplate: https://iter8.tools
+---
+apiVersion: iter8.tools/v2alpha2
+kind: Metric
+metadata:
+  name: mean-latency
+spec:
+  description: Mean latency (Iter8 builtin metric)
+  type: Gauge
+  sampleSize: iter8-system/request-count
+  provider: iter8
+  jqExpression: "dummy"
+  urlTemplate: https://iter8.tools
+---
+apiVersion: iter8.tools/v2alpha2
+kind: Metric
+metadata:
+  name: latency-50th-percentile
+spec:
+  description: 50th percentile (median) latency (Iter8 builtin metric)
+  type: Gauge
+  sampleSize: iter8-system/request-count
+  provider: iter8
+  jqExpression: "dummy"
+  urlTemplate: https://iter8.tools
+---
+apiVersion: iter8.tools/v2alpha2
+kind: Metric
+metadata:
+  name: latency-75th-percentile
+spec:
+  description: 75th percentile latency (Iter8 builtin metric)
+  type: Gauge
+  sampleSize: iter8-system/request-count
+  provider: iter8
+  jqExpression: "dummy"
+  urlTemplate: https://iter8.tools
+---
+apiVersion: iter8.tools/v2alpha2
+kind: Metric
+metadata:
+  name: latency-90th-percentile
+spec:
+  description: 90th percentile latency (Iter8 builtin metric)
+  type: Gauge
+  sampleSize: iter8-system/request-count
+  provider: iter8
+  jqExpression: "dummy"
+  urlTemplate: https://iter8.tools
+---
+apiVersion: iter8.tools/v2alpha2
+kind: Metric
+metadata:
+  name: latency-95th-percentile
+spec:
+  description: 95th percentile latency (Iter8 builtin metric)
+  type: Gauge
+  sampleSize: iter8-system/request-count
+  provider: iter8
+  jqExpression: "dummy"
+  urlTemplate: https://iter8.tools
+---
+apiVersion: iter8.tools/v2alpha2
+kind: Metric
+metadata:
+  name: latency-99th-percentile
+spec:
+  description: 99th percentile latency (Iter8 builtin metric)
+  type: Gauge
+  sampleSize: iter8-system/request-count
+  provider: iter8
+  jqExpression: "dummy"
+  urlTemplate: https://iter8.tools
+---
+
diff --git a/install/core/iter8-analytics/kustomization.yaml b/install/core/iter8-analytics/kustomization.yaml
@@ -5,4 +5,4 @@ resources:
 images:
 - name: iter8/iter8-analytics
   # to change iter8-analytics version, edit the next line
-  newTag: 2.1.1
+  newTag: 2.1.2
diff --git a/install/core/iter8-controller/kustomization.yaml b/install/core/iter8-controller/kustomization.yaml
@@ -1,12 +1,12 @@
 resources:
 - etc3.yaml
 # to change Iter8 CRDs version, edit the next line
-- github.com/iter8-tools/etc3/config/crd/?ref=v0.1.21
+- github.com/iter8-tools/etc3/config/crd/?ref=v0.1.25
 
 images:
 - name: iter8/etc3:latest
   # to change etc3 version, edit the next line
-  newTag: 0.1.24
+  newTag: 0.1.25
 
 # even if you do not change the CRD, 
 # keep the two version numbers in sync to keep them the same

diff --git a/install/core/iter8-handler/handler.yaml b/install/core/iter8-handler/handler.yaml
@@ -9,7 +9,7 @@ spec:
       containers:
       - name: iter8-handler
         # to change handler version, edit the line below
-        image: iter8/handler:0.1.15
+        image: iter8/handler:0.1.17
         command: ["handler"]
         args: ["run", "-a", "$(ACTION)"]
         env:
@@ -19,6 +19,8 @@ spec:
           value: EXPERIMENT_NAMESPACE_VALUE
         - name: EXPERIMENT_NAME
           value: EXPERIMENT_NAME_VALUE
+        - name: LOG_LEVEL
+          value: trace
       restartPolicy: Never
   backoffLimit: 1
   activeDeadlineSeconds: 300
diff --git a/install/core/iter8-handler/kustomization.yaml b/install/core/iter8-handler/kustomization.yaml
@@ -2,7 +2,7 @@ configMapGenerator:
 - name: handlers
   files:
   - handler.yaml
-generatorOptions:
-  disableNameSuffixHash: true 
+# generatorOptions:
+#   disableNameSuffixHash: true 
 
 # if you want to change handler version, do so by editing handler.yaml
diff --git a/mkdocs/docs/images/conformance-iter8-knative-process.png b/mkdocs/docs/images/conformance-iter8-knative-process.png
diff --git a/mkdocs/docs/images/src/conformance-iter8-knative-process.pptx b/mkdocs/docs/images/src/conformance-iter8-knative-process.pptx
diff --git a/mkdocs/docs/metrics/builtin.md b/mkdocs/docs/metrics/builtin.md
@@ -0,0 +1,31 @@
+---
+template: main.html
+---
+
+# Builtin Metrics
+
+!!! tip "Builtin latency/error metrics"    
+    Iter8 ships with a set of nine builtin metrics that measure your app/ML model's performance in terms of latency and errors. You can collect and use these metrics in experiments without the need to configure any external databases. 
+
+    This feature enables you to get started with Iter8 experiments, especially, SLO validation experiments, quickly.
+
+## List of builtin metrics
+The following are the set of builtin Iter8 metrics.
+
+| Namespace | Name         | Type | Description |
+| ----- | ------------ | ----------- | -------- |
+| iter8-system | request-count | Counter | Number of requests |
+| iter8-system | error-count | Gauge | Number of responses with HTTP status code 4xx or 5xx |
+| iter8-system | error-rate | Gauge | Fraction of responses with HTTP status code 4xx or 5xx |
+| iter8-system | mean-latency | Gauge | Mean response latency |
+| iter8-system | latency-50th-percentile | Gauge | 50th percentile (median) response latency |
+| iter8-system | latency-75th-percentile | Gauge | 75th percentile response latency |
+| iter8-system | latency-90th-percentile | Gauge | 90th percentile response latency |
+| iter8-system | latency-95th-percentile | Gauge | 95th percentile response latency |
+| iter8-system | latency-99th-percentile | Gauge | 99th percentile response latency |
+
+## Collecting builtin metrics
+Use the [`metrics/collect` task](../../reference/tasks/metrics/) in an experiment to collect builtin metrics for your app/ML model versions.
+
+## Example
+For an example of an experiment that uses builtin metrics, look inside the Knative experiment in [this tutorial](../../tutorials/testing-patterns/conformance/#7-launch-experiment).
diff --git a/...cs/docs/metrics/defining-iter8-metrics.md → mkdocs/docs/metrics/custom.md b/...cs/docs/metrics/defining-iter8-metrics.md → mkdocs/docs/metrics/custom.md
@@ -2,9 +2,9 @@
 template: main.html
 ---
 
-# Defining Iter8 Metrics
+# Defining Custom Metrics
 
-This document describes how you can create Iter8 metrics and (optionally) supply authentication information that may be required by the metrics provider.
+Custom Iter8 metrics enable you to use data from any database for evaluating app/ML model versions within Iter8 experiments. This document describes how you can define custom Iter8 metrics and (optionally) supply authentication information that may be required by the metrics provider.
 
 Metric providers differ in the following aspects.
 
@@ -16,7 +16,7 @@ Metric providers differ in the following aspects.
 
 The examples in this document focus on Prometheus, NewRelic, Sysdig, and Elastic. However, the principles illustrated here will enable you to use metrics from any provider in experiments.
 
-## Defining metrics
+## Metrics with/without auth
 
 > **Note:** Metrics are defined by you, the **Iter8 end-user**.
 

diff --git a/mkdocs/docs/metrics/using-metrics.md b/mkdocs/docs/metrics/using-metrics.md
@@ -4,55 +4,42 @@ template: main.html
 
 # Using Metrics in Experiments
 
-!!! tip "Iter8 metrics API"    
-    Iter8 defines a new Kubernetes resource called Metric that makes it easy to use metrics in experiments from RESTful metric providers like Prometheus, New Relic, Sysdig and Elastic.
+!!! tip "Iter8 metric resources"    
+    Iter8 defines a custom Kubernetes resource (CRD) called **Metric** that makes it easy to define and use metrics in experiments. 
+
+    Iter8 installation includes a set of pre-defined [builtin metrics](../builtin) that pertain to app/ML model latency/errors. You can also [define custom metrics](../custom) that enable you to utilize data from Prometheus, New Relic, Sysdig, Elastic or any other database of your choice.
 
-    List metrics available in your cluster using the `kubectl get metrics.iter8.tools` command. Use metrics in experiments by referencing them in experiment criteria.
-
-## Listing metrics
-Iter8 metrics are Kubernetes resources which means you can list them using `kubectl get`.
+## List metrics
+Find the set Iter8 metrics available in your cluster using `kubectl get`.
 
 ``` shell
 kubectl get metrics.iter8.tools --all-namespaces
 ```
 
 ```shell
-NAMESPACE       NAME                           TYPE      DESCRIPTION
-iter8-knative   95th-percentile-tail-latency   Gauge     95th percentile tail latency
-iter8-knative   error-count                    Counter   Number of error responses
-iter8-knative   error-rate                     Gauge     Fraction of requests with error responses
-iter8-knative   mean-latency                   Gauge     Mean latency
-iter8-knative   request-count                  Counter   Number of requests
+NAMESPACE         NAME                      TYPE      DESCRIPTION
+iter8-kfserving   user-engagement           Gauge     Average duration of a session
+iter8-system      error-count               Counter   Number of responses with HTTP status code 4xx or 5xx (Iter8 builtin metric)
+iter8-system      error-rate                Gauge     Fraction of responses with HTTP status code 4xx or 5xx (Iter8 builtin metric)
+iter8-system      latency-50th-percentile   Gauge     50th percentile (median) latency (Iter8 builtin metric)
+iter8-system      latency-75th-percentile   Gauge     75th percentile latency (Iter8 builtin metric)
+iter8-system      latency-90th-percentile   Gauge     90th percentile latency (Iter8 builtin metric)
+iter8-system      latency-95th-percentile   Gauge     95th percentile latency (Iter8 builtin metric)
+iter8-system      latency-99th-percentile   Gauge     99th percentile latency (Iter8 builtin metric)
+iter8-system      mean-latency              Gauge     Mean latency (Iter8 builtin metric)
+iter8-system      request-count             Counter   Number of requests (Iter8 builtin metric)
 ```
 
-## Referencing metrics
+## Referencing metrics within experiments
 
-Use metrics in experiments by referencing them in criteria section. Reference metrics using the `namespace/name` or `name` [format](../../../reference/apispec/#criteria).
+Use metrics in experiments by referencing them in the criteria section of the experiment manifest. Reference metrics using the `namespace/name` or `name` [format](../../../reference/apispec/#criteria).
 
 ??? example "Sample experiment illustrating the use of metrics"
     ```yaml
-    apiVersion: iter8.tools/v2alpha2
     kind: Experiment
-    metadata:
-      name: quickstart-exp
+    ... 
     spec:
-      # target identifies the knative service under experimentation using its fully qualified name
-      target: default/sample-app
-      strategy:
-        # this experiment will perform a canary test
-        testingPattern: Canary
-        deploymentPattern: Progressive
-        actions:
-          start: # run the following sequence of tasks at the start of the experiment
-          - task: knative/init-experiment
-          finish: # run the following sequence of tasks at the end of the experiment
-          - task: common/exec # promote the winning version
-            with:
-              cmd: kubectl
-              args: 
-              - "apply"
-              - "-f"
-              - "https://raw.githubusercontent.com/iter8-tools/iter8/master/samples/knative/quickstart/{{ .promote }}.yaml"
+      ...
       criteria:
         requestCount: iter8-knative/request-count
         # mean latency of version should be under 50 milliseconds
@@ -65,27 +52,7 @@ Use metrics in experiments by referencing them in criteria section. Reference me
           upperLimit: 100
         - metric: iter8-knative/error-rate
           upperLimit: "0.01"
-      duration:
-        intervalSeconds: 10
-        iterationsPerLoop: 10
-      versionInfo:
-        # information about app versions used in this experiment
-        baseline:
-          name: current
-          variables:
-          - name: revision
-            value: sample-app-v1 
-          - name: promote
-            value: baseline
-        candidates:
-        - name: candidate
-          variables:
-          - name: revision
-            value: sample-app-v2
-          - name: promote
-            value: candidate 
     ```
 
 ## Observing metric values
-
-During an experiment, Iter8 reports the metric values observed for each version. Use `iter8ctl` to observe these metric values in realtime. See [here](../../../getting-started/quick-start/with-knative/#8-observe-experiment) for an example.
+During an experiment, Iter8 reports the metric values observed for each version. Use `iter8ctl` to observe these metric values in realtime. See [here](../../../getting-started/quick-start/#a-observe-metrics) for an example.
diff --git a/mkdocs/docs/reference/experiment.md b/mkdocs/docs/reference/experiment.md
@@ -109,7 +109,7 @@ Standard Kubernetes [meta.v1/ObjectMeta](https://kubernetes.io/docs/reference/ge
 | ----- | ---- | ----------- | -------- |
 | testingPattern | string | Determines the logic used to evaluate the app versions and determine the winner of the experiment. Iter8 supports two testing patterns, namely, `Canary` and `Conformance`. | Yes |
 | deploymentPattern | string | Determines if and how traffic is shifted during an experiment. This field is relevant only for experiments using the `Canary` testing pattern. Iter8 supports two deployment patterns, namely, `Progressive` and `FixedSplit`. | No |
-| actions | map[string][][TaskSpec](#taskspec) | An action is a sequence of tasks that can be executed by Iter8. `spec.strategy.actions` can be used to specify start and finish actions that will be run at the start and end of an experiment respectively. | No |
+| actions | map[ActionType][][TaskSpec](#taskspec) | An action is a sequence of tasks that can be executed by Iter8. `ActionType` is a string enum with three valid values: `start`, `loop`, and `finish`. The start action, if specified, is executed at the start of the experiment. The loop action, if specified, is executed during every loop of the experiment, after all the iterations within the loop have completed. The finish action, if specified, is executed at the end of the experiment after all the loops have completed. The `actions` field is used to specify all three types of actions. | No |
 
 ### TaskSpec
 
@@ -155,8 +155,11 @@ Standard Kubernetes [meta.v1/ObjectMeta](https://kubernetes.io/docs/reference/ge
 
 | Field name | Field type | Description | Required |
 | ----- | ---- | ----------- | -------- |
+| maxLoops | int32 | Maximum number of loops in the experiment. In case of a failure, the experiment may be terminated earlier. Default value = 1. | No |
+| iterationsPerLoop | int32 | Number of iterations *per experiment loop*. In case of a failure, the experiment may be terminated earlier. Default value = 15. | No |
 | intervalSeconds | int32 | Duration of a single iteration of the experiment in seconds. Default value = 20 seconds. | No |
-| maxIterations | int32 | Maximum number of iterations in the experiment. In case of failure, the experiment may be terminated earlier. Default value = 15. | No |
+
+> *Note*: Suppose an experiment has `maxLoops = x`, `iterationsPerLoop = y`, and `intervalSeconds = z`. Assuming the experiment does not terminate early due to failures, it would take a minimum of `x*y*z` seconds to complete. The actual duration may be more due to additional time incurred in [acquiring the target](#spec), and executing the `start`, `loop` and `finish` [actions](#strategy).
 
 ### VersionInfo
 
@@ -206,11 +209,22 @@ Standard Kubernetes [meta.v1/ObjectMeta](https://kubernetes.io/docs/reference/ge
 
 | Field name | Field type         | Description | Required |
 | ----- | ------------ | ----------- | -------- |
+| aggregatedBuiltinHists | [AggregatedBuiltinHists](#aggregatedbuiltinhists) | This field is used to store intermediate results from the [`metrics/collect` task](../tasks/metrics/#metrics-tasks) that enables [builtin metrics](../../metrics/builtin/). Reserved for Iter8 internal use. | No |
 | aggregatedMetrics | [AggregatedMetricsAnalysis](#aggregatedmetricsanalysis) | Most recently observed metric values for all metrics referenced in the experiment criteria. | No |
 | winnerAssessment | [WinnerAssessmentAnalysis](#winnerassessmentanalysis) | Information about the `winner` of the experiment. | No |
 | versionAssessments | [VersionAssessmentAnalysis](#versionassessmentanalysis) | For each version, a summary analysis identifying whether or not the version is satisfying the experiment criteria. | No |
 | weights | [WeightsAnalysis](#weightanalysis) | Recommended weight distribution to be applied before the next iteration of the experiment. | No |
 
+### AggregatedBuiltinHists
+
+| Field name | Field type         | Description | Required |
+| ----- | ------------ | ----------- | -------- |
+| provenance | string | Source of the data. Currently, Iter8 builtin metrics collect task is the only valid value for this field. Reserved for Iter8 internal use. | Yes |
+| timestamp | [metav1.Time](https://pkg.go.dev/k8s.io/[email protected]/pkg/apis/meta/v1#Time) | The time when this field was last updated. Reserved for Iter8 internal use. | Yes |
+| message | string | Human readable message. Reserved for Iter8 internal use. | No |
+| data | [apiextensionsv1.JSON](https://pkg.go.dev/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1#JSON) | Aggregated histogram data for storing intermediate results for builtin metics collection. Reserved for Iter8 internal use. | No |
+
+
 ### VersionAssessmentAnalysis
 
 | Field name | Field type         | Description | Required |

diff --git a/mkdocs/docs/reference/metrics.md b/mkdocs/docs/reference/metrics.md
@@ -39,7 +39,7 @@ Standard Kubernetes [meta.v1/ObjectMeta](https://kubernetes.io/docs/reference/ge
 | ----- | ------------ | ----------- | -------- |
 | description | string | Human readable description. This field is meant for informational purposes. | No |
 | units | string | Units of measurement. This field is meant for informational purposes. | No |
-| provider | string | Type of the metrics provider. This field is meant for informational purposes. | No |
+| provider | string | Type of the metrics provider (example, `prometheus`, `newrelic`, `sysdig`, `elastic`, ...). The keyword `iter8` is reserved for Iter8 builtin metrics. | No |
 | params | [][NamedValue](../experiment/#namedvalue) | List of name/value pairs corresponding to the name and value of the HTTP query parameters used by Iter8 when querying the metrics provider. Each name represents a parameter name; the corresponding value is a string template with placeholders; the placeholders will be dynamically substituted by Iter8 with values at query time. | No |
 | body | string | String used to construct the JSON body of the HTTP request. Body may be templated, in which Iter8 will attempt to substitute placeholders in the template at query time using version information. | No |
 | type | string | Metric type. Valid values are `Counter` and `Gauge`. Default value = `Gauge`. A `Counter` metric is one whose value never decreases over time. A `Gauge` metric is one whose value may increase or decrease over time. | No |

diff --git a/mkdocs/docs/reference/tasks.md b/mkdocs/docs/reference/tasks.md
@@ -6,7 +6,12 @@ template: main.html
 
 Tasks are an extension mechanism for enhancing the behavior of Iter8 experiments and can be specified within the [spec.strategy.actions](../experiment/#strategy) field of the experiment.
 
-Available tasks implemented are:
+Tasks are grouped into libraries. The following task libraries are available.
 
-- `common` [tasks](common/#common-tasks)
-- `notification` [tasks](notification/#notification-tasks)
+- `common` [library](common/#common-tasks)
+    * Task for executing a shell command.
+- `metrics` [library](metrics/#metrics-tasks)
+    * Task for collecting builtin metrics.
+    * The above task can also be used to generate requests for app/ML model versions without collecting builtin metrics.
+- `notification` [library](notification/#notification-tasks)
+    * Task for sending a slack notification.