Compare various KPIs to the threshold values and fail if any is excee…

…ded (#1271) (#1276) Signed-off-by: Pavel Macík <[email protected]> Signed-off-by: Pavel Macík <[email protected]> Signed-off-by: Pavel Macík <[email protected]> Co-authored-by: Pavel Macík <[email protected]>
redhat-developer · Oct 19, 2022 · 186b0ef · 186b0ef
1 parent 7ae0d7d
commit 186b0ef
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 2 deletions.
diff --git a/make/common.mk b/make/common.mk
@@ -144,7 +144,7 @@ endef
 
 YQ = $(shell pwd)/bin/yq
 yq:
-	$(call go-install-tool,$(YQ),github.com/mikefarah/yq/v4@v4.9.8)
+	$(call go-install-tool,$(YQ),github.com/mikefarah/yq/v4@v4.26.1)
 
 KUBECTL_SLICE = $(shell pwd)/bin/kubectl-slice
 kubectl-slice:

diff --git a/make/performance.mk b/make/performance.mk
@@ -7,6 +7,11 @@ OPENSHIFT_API ?=
 OPENSHIFT_USERNAME ?=
 OPENSHIFT_PASSWORD ?=
 
+TEST_PERFORMANCE_AVG_MEMORY ?= 150 # value in MiB
+TEST_PERFORMANCE_MAX_MEMORY ?= 200 # valie in MiB
+TEST_PERFORMANCE_AVG_CPU ?= 20 # value in millicores of vCPU
+TEST_PERFORMANCE_MAX_CPU ?= 100 # value in millicores of vCPU
+
 .PHONY: test-performance-setup
 ## Setup OpenShift cluster for performance test
 test-performance-setup:
@@ -29,4 +34,18 @@ test-performance-collect-kpi:
 test-performance-artifacts:
 	$(Q)echo "Gathering performance test artifacts"
 	$(Q)mkdir -p $(TEST_PERFORMANCE_ARTIFACTS) \
-		&& cp -rvf $(TEST_PERFORMANCE_OUTPUT_DIR) $(TEST_PERFORMANCE_ARTIFACTS)/
+		&& cp -rvf $(TEST_PERFORMANCE_OUTPUT_DIR) $(TEST_PERFORMANCE_ARTIFACTS)/
+
+.PHONY: test-performance-thresholds
+# Compare various KPIs to the threshold values and fail if any threshold is exceeded
+test-performance-thresholds: yq
+	@echo Evaluating KPI:
+	@$(YQ) eval '.kpi[] | select(.name == "usage")' $(TEST_PERFORMANCE_OUTPUT_DIR)/results/kpi.yaml
+	@echo "Checking if average value of memory "$$($(YQ) eval '(.kpi[] | select(.name == "usage").metrics.[] | select(.name == "Memory_MiB").average)' $(TEST_PERFORMANCE_OUTPUT_DIR)/results/kpi.yaml)" < $(TEST_PERFORMANCE_AVG_MEMORY) MiB"
+	@[ $$($(YQ) eval '(.kpi[] | select(.name == "usage").metrics.[] | select(.name == "Memory_MiB").average) < $(TEST_PERFORMANCE_AVG_MEMORY)' $(TEST_PERFORMANCE_OUTPUT_DIR)/results/kpi.yaml) == "true" ]
+	@echo "Checking if maximal value of memory "$$($(YQ) eval '(.kpi[] | select(.name == "usage").metrics.[] | select(.name == "Memory_MiB").maximum)' $(TEST_PERFORMANCE_OUTPUT_DIR)/results/kpi.yaml)" < $(TEST_PERFORMANCE_MAX_MEMORY) MiB"
+	@[ $$($(YQ) eval '(.kpi[] | select(.name == "usage").metrics.[] | select(.name == "Memory_MiB").maximum) < $(TEST_PERFORMANCE_MAX_MEMORY)' $(TEST_PERFORMANCE_OUTPUT_DIR)/results/kpi.yaml) == "true" ]
+	@echo "Checking if average value of CPU "$$($(YQ) eval '(.kpi[] | select(.name == "usage").metrics.[] | select(.name == "CPU_millicores").average)' $(TEST_PERFORMANCE_OUTPUT_DIR)/results/kpi.yaml)" < $(TEST_PERFORMANCE_AVG_CPU) milicores of vCPU"
+	@[ $$($(YQ) eval '(.kpi[] | select(.name == "usage").metrics.[] | select(.name == "CPU_millicores").average) < $(TEST_PERFORMANCE_AVG_CPU)' $(TEST_PERFORMANCE_OUTPUT_DIR)/results/kpi.yaml) == "true" ]
+	@echo "Checking if maximal value of CPU "$$($(YQ) eval '(.kpi[] | select(.name == "usage").metrics.[] | select(.name == "CPU_millicores").maximum)' $(TEST_PERFORMANCE_OUTPUT_DIR)/results/kpi.yaml)" < $(TEST_PERFORMANCE_MAX_CPU) milicores of vCPU"
+	@[ $$($(YQ) eval '(.kpi[] | select(.name == "usage").metrics.[] | select(.name == "CPU_millicores").maximum) < $(TEST_PERFORMANCE_MAX_CPU)' $(TEST_PERFORMANCE_OUTPUT_DIR)/results/kpi.yaml) == "true" ]