test: Yk/benchmarking scripts

test/README.md

@@ -0,0 +1,64 @@
+# HOW TO USE IT
+
+This document describe how to use the benchmarking scripts. 
+
+
+#### Folder tree
+
+<pre>
+.  
+├── bench    
+|  └── 100_pods  
+|        └── redis  
+|            ├── python_scripts  
+|            |   ├── cleaner.py  
+|            |   ├── merge_jsons.py  
+|            |   └── wjson.py  
+|            ├── stats  
+|            |   └── test_env.json  
+|            └── test_scripts  
+|                ├── bench_1k.sh  
+|                ├── bench_5k.sh  
+|                └── test.sh  
+├── htcmock  
+|   └── 100_pods  
+|        └── redis  
+|            ├── python_scripts  
+|            |   ├── cleaner.py  
+|            |   ├── merge_jsons.py  
+|            |   └── wjson.py  
+|            ├── stats  
+|            |   └── test_env.json  
+|            └── test_scripts  
+|                ├── htcmock_5k.sh  
+|                └── test.sh  
+├── README.md  
+
+</pre>
+
+## Bench and Htcmock
+
+### Run the tests
+
+* bench_1k.sh & bench_5k.sh : scripts for each test where we set the parameters of the test to run.
+* test.sh : script to run all the tests cases and  store the results in Json files.
+
+### Clean the output
+
+* cleaner.py : will clean the json output files.  
+Parameters : path to the json files we want to clean.  
+
+
+* merge_jsons.py : merges the cleaned results files with the parameters json file of the tested version of ArmoniK and est_env.json file (third party components of ArmoniK).
+* prerequisites: we have to install jsonmerge (pip install jsonmerge)
+
+### Analyse the results 
+
+wjson.py : will read the clean stats files so we can manipulate the data.
+
+* Parameters : List of clean json files
+
+### How to use it :
+* Run the script test_scripts/test.sh to run the tests, store the outputs, clean them and merge them with the environment and infrastructure description files.
+
+

test/bench/100_pods/redis/python_scripts/cleaner.py

@@ -0,0 +1,38 @@
+#! /usr/bin/env python3
+
+import json
+import re
+
+
+def clean_file(file):
+    lines = []
+    keep_stats = []
+    keep_tput = []
+    keep_options = []
+
+    # read the output file with no needed data
+    with open(file, 'r') as f:
+        # keep only stats lines in the json file
+        lines = f.read().split("\n")
+        for line in lines:
+            if not line: continue
+            # print(line)
+            jline = json.loads(line)
+            if "stats" in jline:
+                keep_stats.append(jline["stats"])
+                keep_options.append(jline["benchOptions"])
+            if "sessionThroughput" in jline:
+                keep_tput.append(jline["sessionThroughput"])
+    dic_json = [{"Test": "bench"} | stats | options | {"throughput": tput, "nb_pods": 100} for stats, tput, options in
+                zip(keep_stats, keep_tput, keep_options)]
+
+    # write a clean json file with needed data
+    with open(file, 'w') as f:
+        json.dump(dic_json, f)
+
+
+# clean the stats file
+file = "../stats/1k.json"
+clean_file(file)
+file = "../stats/5k.json"
+clean_file(file)

test/bench/100_pods/redis/python_scripts/merge_jsons.py

@@ -0,0 +1,19 @@
+#! /usr/bin/env python3
+
+import json
+import re
+from jsonmerge import merge
+
+result = []
+with open("../../../../../versions.tfvars.json", "r") as versions:
+    result.append(json.load(versions))
+with open("../stats/test_env.json", "r") as test_env:
+    result.append(json.load(test_env))
+with open("../stats/1k.json", "r") as stats:
+    result.append(json.load(stats))
+with open("../stats/5k.json", "r") as stats:
+    result2 = json.load(stats)
+# print(merged)
+with open("results.json", "w") as r:
+    dict_json = [merge(result[0], result[1]), result[2], result2]
+    json.dump(dict_json, r)

test/bench/100_pods/redis/python_scripts/wjson.py

@@ -0,0 +1,81 @@
+#! /usr/bin/env python3
+
+import json
+from pytimeparse import parse
+import matplotlib.pyplot as plt
+import numpy as np
+
+
+# Class test_case
+# An object store a lists of the stats of all the runs of the test case
+# So we could calculate the mean and the median of each stat
+class TestCase:
+
+    def __init__(self, file):
+        self.nbtasks = []
+        self.time = []
+        self.exec_time = []
+        self.sub_time = []
+        self.retrv_time = []
+        self.throughput = []
+        self.d_parallel = []
+        self.file = file
+        with open(file) as my_bench_file:
+            data = my_bench_file.read()
+        # case = TestCase()
+        runs = json.loads(data)
+
+        for run in runs:
+            if (run["nb_pods"] == 100):
+                self.nbtasks.append(run["TotalTasks"])
+                self.time.append(float(parse(run["ElapsedTime"])))
+                self.exec_time.append(float(parse(run["TasksExecutionTime"])))
+                self.sub_time.append(float(parse(run["SubmissionTime"])))
+                self.retrv_time.append(float(parse(run["ResultRetrievingTime"])))
+                self.throughput.append(float(run["throughput"]))
+                self.d_parallel.append(run["DegreeOfParallelism"])
+
+
+if __name__ == "__main__":
+
+    files = ['../stats/1k.json', '../stats/5k.json']
+    JsonFiles = [x for x in files if x.endswith(".json")]
+
+    # dictionary to store the stats of each test case
+    cases = {
+    }
+    cases["1k"] = TestCase(JsonFiles[0])
+    cases["5k"] = TestCase(JsonFiles[1])
+
+    # Dictionary to store the mean of each test case
+    mean = {
+        "time": {},
+        "exec_time": {},
+        "sub_time": {},
+        "retrv_time": {},
+        "throughput": {}
+    }
+
+    # calculte the mean of each test case
+    for file in files:
+        filename = file.split(".")[0]
+        mean["time"][filename] = np.mean(cases[filename].time)
+        mean["exec_time"][filename] = np.mean(cases[filename].exec_time)
+        mean["sub_time"][filename] = np.mean(cases[filename].sub_time)
+        mean["retrv_time"][filename] = np.mean(cases[filename].retrv_time)
+        mean["throughput"][filename] = np.mean(cases[filename].throughput)
+
+    # print the stats
+    for file in files:
+        filename = file.split(".")[0]
+        print('Degree of parallelism of retrieving time is : ' + str(cases[filename].d_parallel[0]))
+        print('mean total time for treatement of ' + filename + ' tasks on 100 pods is : ' + str(
+            mean["time"][filename]) + ' s')
+        print('mean time of the execution of ' + filename + ' tasks on 100 pods is : ' + str(
+            mean["exec_time"][filename]) + ' s')
+        print('mean time of the submission of ' + filename + ' tasks on 100 pods is : ' + str(
+            mean["sub_time"][filename]) + ' s')
+        print('mean time of the retrieving of ' + filename + ' tasks on 100 pods is : ' + str(
+            mean["retrv_time"][filename]) + ' s')
+        print('mean throughput for ' + filename + ' tasks on 100 pods is : ' + str(
+            mean["throughput"][filename]) + " tasks/s \n")

test/bench/100_pods/redis/stats/test_env.json

@@ -0,0 +1,16 @@
+{
+    "test_environnement" : {
+        "instance_type" : "c24.xlarge",
+        "Processor" : "Intel Xeon de 2e (Cascade Lake)",
+        "CPU_frequency" : "3.6 GHz - 3.9 GHz",
+        "nb_cpus" : "96",
+        "RAM" : "192",
+        "Network_bandwidth" : "25",
+        "EBS_bandwidth" : "19000",
+        "Kubernetes" : "AWS EKS 1.25",
+        "Object_object_type" : "AWS S3",
+        "Storage_queue_storage" : "Amazon MQ, broker : ActiveMQ 5.16.4",
+        "Storage_table_type" : "MongoDB 6.0.1",
+        "OS" : "Linux"
+    }
+}

test/bench/100_pods/redis/test_scripts/bench_1k.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+export CONTROL_PLANE_URL=$(cat ../../../../../infrastructure/quick-deploy/aws/armonik/generated/armonik-output.json | jq -r '.armonik.control_plane_url')
+
+#run  1000 tasks on 100 pods
+docker run --rm \
+	-e GrpcClient__Endpoint="${CONTROL_PLANE_URL}" \
+	-e BenchOptions__nTasks=1000 \
+	-e BenchOptions__TaskDurationMs=1 \
+	-e BenchOptions__PayloadSize=1 \
+	-e BenchOptions__ResultSize=1 \
+	-e BenchOptions__Partition=bench \
+    	-e BenchOptions__ShowEvents=false \
+	-e BenchOptions__BatchSize=50 \
+	-e BenchOptions__MaxRetries=5 \
+	-e BenchOptions__DegreeOfParallelism=5 \
+	dockerhubaneo/armonik_core_bench_test_client:$(cat ../../../../../versions.tfvars.json | jq -r '.armonik_versions.core')

test/bench/100_pods/redis/test_scripts/bench_5k.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+export CONTROL_PLANE_URL=$(cat ../../../../../infrastructure/quick-deploy/aws/armonik/generated/armonik-output.json | jq -r '.armonik.control_plane_url')
+
+#run  5000 tasks on 100 pods
+docker run --rm \
+	-e GrpcClient__Endpoint="${CONTROL_PLANE_URL}" \
+	-e BenchOptions__nTasks=5000 \
+	-e BenchOptions__TaskDurationMs=1 \
+	-e BenchOptions__PayloadSize=1 \
+	-e BenchOptions__ResultSize=1 \
+	-e BenchOptions__Partition=bench \
+    	-e BenchOptions__ShowEvents=false \
+	-e BenchOptions__BatchSize=50 \
+	-e BenchOptions__MaxRetries=5 \
+	-e BenchOptions__DegreeOfParallelism=5 \
+	dockerhubaneo/armonik_core_bench_test_client:$(cat ../../../../../versions.tfvars.json | jq -r '.armonik_versions.core')