Merge pull request #354 from SUNET/bugfix.compose_v2_compatible

Bugfix.compose v2 compatible

.github/workflows/run-unit-tests.yml

@@ -48,6 +48,7 @@ jobs:
           echo "PYTEST_POSTGRES_EXTERNAL=1" >> $GITHUB_ENV
           echo "PYTEST_REDIS_EXTERNAL=1" >> $GITHUB_ENV
           echo "JWT_AUTH_TOKEN=eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzI1NiJ9.eyJpYXQiOjE1NzEwNTk2MTgsIm5iZiI6MTU3MTA1OTYxOCwianRpIjoiNTQ2MDk2YTUtZTNmOS00NzFlLWE2NTctZWFlYTZkNzA4NmVhIiwic3ViIjoiYWRtaW4iLCJmcmVzaCI6ZmFsc2UsInR5cGUiOiJhY2Nlc3MifQ.Sfffg9oZg_Kmoq7Oe8IoTcbuagpP6nuUXOQzqJpgDfqDq_GM_4zGzt7XxByD4G0q8g4gZGHQnV14TpDer2hJXw" >> $GITHUB_ENV
+          echo "JWT_SECRET_KEY=unittestsharedsecret" >> $GITHUB_ENV
 
 #      - name: Create code coverage dir
 #        run: mkdir -p docker/coverage/
@@ -89,7 +90,7 @@ jobs:
         run: docker ps -a
 
       - name: Check API running version
-        run: curl -ks --connect-timeout 5 --max-time 10 --retry 5 --retry-delay 0 --retry-max-time 30 https://localhost/api/v1.0/system/version
+        run: curl -ks --connect-timeout 5 --max-time 10 --retry 10 --retry-delay 5 --retry-max-time 120 https://localhost/api/v1.0/system/version
 
       - name: Fix permissions for code coverage
         run: docker-compose -f docker/docker-compose_test.yaml exec -u root -T cnaas_api chown -R www-data:www-data /opt/cnaas/venv/cnaas-nms/src/

docker/api/config/uwsgi.ini

@@ -2,7 +2,7 @@
 uid=www-data
 gid=www-data
 chdir = /opt/cnaas/venv/cnaas-nms/src/
-plugins = gevent
+plugins = gevent3
 callable = cnaas_app
 module = cnaas_nms.run
 # separate single process for websocket

docker/docker-compose_test.yaml

@@ -33,6 +33,7 @@ services:
       - PYTEST_SETTINGS_CLONED
       - PYTEST_TEMPLATES_CLONED
       - JWT_AUTH_TOKEN
+      - JWT_SECRET_KEY
       - PERMISSIONS_DISABLED=True
     depends_on:
       - "cnaas_postgres"

src/cnaas_nms/run.py

@@ -29,13 +29,17 @@ def is_coverage_enabled():
     cov = coverage.coverage(data_file=".coverage-{}".format(os.getpid()), concurrency="gevent")
     cov.start()
 
+    def save_coverage_signal(signum, frame):
+        cov.stop()
+        cov.save()
+
     def save_coverage():
         cov.stop()
         cov.save()
 
     atexit.register(save_coverage)
-    gevent_signal.signal(signal.SIGTERM, save_coverage)
-    gevent_signal.signal(signal.SIGINT, save_coverage)
+    gevent_signal.signal(signal.SIGTERM, save_coverage_signal)
+    gevent_signal.signal(signal.SIGINT, save_coverage_signal)
 
 
 def get_app():

src/cnaas_nms/scheduler_mule.py

@@ -30,9 +30,13 @@ def save_coverage():
         cov.stop()
         cov.save()
 
+    def save_coverage_signal():
+        cov.stop()
+        cov.save()
+
     atexit.register(save_coverage)
-    signal.signal(signal.SIGTERM, save_coverage)
-    signal.signal(signal.SIGINT, save_coverage)
+    signal.signal(signal.SIGTERM, save_coverage_signal)
+    signal.signal(signal.SIGINT, save_coverage_signal)
 
 
 def pre_schedule_checks(scheduler, kwargs):

test/integrationtests.py

@@ -26,7 +26,7 @@ class GetTests(unittest.TestCase):
     def setUpClass(cls):
         for i in range(100):
             try:
-                r = requests.get(f"{URL}/api/v1.0/devices", headers=AUTH_HEADER, verify=TLS_VERIFY)
+                r = requests.get(f"{URL}/api/v1.0/system/version", headers=AUTH_HEADER, verify=TLS_VERIFY)
             except Exception as e:
                 print("Exception {}, retrying in 1 second".format(str(e)))
                 time.sleep(1)
@@ -99,9 +99,15 @@ def test_02_ztp(self):
         print("Discovered hostname, id: {}, {}".format(hostname, device_id))
         self.assertTrue(hostname, "No device in state discovered found for ZTP")
         data = {"hostname": "eosaccess", "device_type": "ACCESS"}
-        r = requests.post(
-            f"{URL}/api/v1.0/device_initcheck/{device_id}", headers=AUTH_HEADER, json=data, verify=TLS_VERIFY
-        )
+        initcheck_attempts = 3
+        for _ in range(initcheck_attempts):
+            time.sleep(5)
+            r = requests.post(
+                f"{URL}/api/v1.0/device_initcheck/{device_id}", headers=AUTH_HEADER, json=data, verify=TLS_VERIFY
+            )
+            if r.status_code == 200:
+                break
+
         self.assertEqual(r.status_code, 200, "Failed device_initcheck, http status")
         self.assertEqual(r.json()["status"], "success", "Failed device_initcheck, returned unsuccessful")
         #       this check fails when running integrationtests with one dist because of faked neighbor:

test/integrationtests.sh

@@ -19,10 +19,25 @@ export PYTEST_POSTGRES_EXTERNAL=1
 export PYTEST_REDIS_EXTERNAL=1
 export PYTEST_SETTINGS_CLONED=1
 export PYTEST_TEMPLATES_CLONED=1
+export OIDC_ENABLED=0
+export PERMISSIONS_DISABLED=1
 export JWT_AUTH_TOKEN="eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzI1NiJ9.eyJpYXQiOjE1NzEwNTk2MTgsIm5iZiI6MTU3MTA1OTYxOCwianRpIjoiNTQ2MDk2YTUtZTNmOS00NzFlLWE2NTctZWFlYTZkNzA4NmVhIiwic3ViIjoiYWRtaW4iLCJmcmVzaCI6ZmFsc2UsInR5cGUiOiJhY2Nlc3MifQ.Sfffg9oZg_Kmoq7Oe8IoTcbuagpP6nuUXOQzqJpgDfqDq_GM_4zGzt7XxByD4G0q8g4gZGHQnV14TpDer2hJXw"
 export JWT_SECRET_KEY="integrationtestkey"
 
-docker-compose down
+# select docker compose v 1 or 2
+set +e
+docker compose > /dev/null 2>&1
+if [ $? -eq 1 ]
+then
+	echo "detected docker-compose v1"
+	COMPOSE_COMMAND="docker-compose"
+else
+	echo "detected docker compose v2"
+	COMPOSE_COMMAND="docker compose"
+	COMPOSE_COMPATIBILITY=1
+fi
+
+$COMPOSE_COMMAND down -t 3
 
 if docker volume ls | egrep -q "cnaas-postgres-data$"
 then
@@ -59,20 +74,21 @@ docker volume create cnaas-cacert
 
 set -e
 
-docker-compose up -d
+$COMPOSE_COMMAND up -d
 
 docker cp ./jwt-cert/public.pem docker_cnaas_api_1:/opt/cnaas/jwtcert/public.pem
-docker-compose exec -u root -T cnaas_api /bin/chown -R www-data:www-data /opt/cnaas/jwtcert/
-docker-compose exec -u root -T cnaas_api /opt/cnaas/createca.sh
+$COMPOSE_COMMAND exec -u root -T cnaas_api /bin/chown -R www-data:www-data /opt/cnaas/jwtcert/
+$COMPOSE_COMMAND exec -u root -T cnaas_api /opt/cnaas/createca.sh
+
+sleep 5
 
-curl --connect-timeout 2 --max-time 2 --retry 10 --retry-delay 0 --retry-max-time 60 \
-    -ks "https://localhost/api/v1.0/system/version"
+curl --connect-timeout 2 --max-time 2 --retry 10 --retry-delay 0 --retry-max-time 60 -ks "https://localhost/api/v1.0/system/version"
 
 # optional copy and restart
-docker-compose exec -u root -T cnaas_api /bin/mv /opt/cnaas/venv/cnaas-nms/src/ /opt/cnaas/venv/cnaas-nms/src-bundled
+$COMPOSE_COMMAND exec -u root -T cnaas_api /bin/mv /opt/cnaas/venv/cnaas-nms/src/ /opt/cnaas/venv/cnaas-nms/src-bundled
 docker cp ../src docker_cnaas_api_1:/opt/cnaas/venv/cnaas-nms/src
-docker-compose exec -u root -T cnaas_api /bin/chown -R www-data:www-data /opt/cnaas/venv/cnaas-nms/src/
-docker-compose exec -u root -T cnaas_api /usr/bin/killall uwsgi
+$COMPOSE_COMMAND exec -u root -T cnaas_api /bin/chown -R www-data:www-data /opt/cnaas/venv/cnaas-nms/src/
+$COMPOSE_COMMAND exec -u root -T cnaas_api /usr/bin/killall uwsgi
 #
 
 if [ ! -z "$PRE_TEST_SCRIPT" ]
@@ -111,25 +127,25 @@ echo "Gathering coverage reports from integration tests:"
 MULE_PID="`docker logs docker_cnaas_api_1 | awk '/spawned uWSGI mule/{print $6}' | egrep -o "[0-9]+" | tail -n1`"
 echo "Found mule at pid $MULE_PID"
 # Allow for code coverage files to be saved
-docker-compose exec -u root -T cnaas_api chown -R www-data:www-data /opt/cnaas/venv/cnaas-nms/src/
+$COMPOSE_COMMAND exec -u root -T cnaas_api chown -R www-data:www-data /opt/cnaas/venv/cnaas-nms/src/
 curl -ks -H "Authorization: Bearer $JWT_AUTH_TOKEN" "https://localhost/api/v1.0/system/shutdown" -d "{}" -X POST -H "Content-Type: application/json"
 sleep 3
 
 echo "Starting unit tests..."
-docker-compose exec -u www-data -T cnaas_api /opt/cnaas/pytest.sh
+$COMPOSE_COMMAND exec -u www-data -T cnaas_api /opt/cnaas/pytest.sh
 echo "Try to generate coverage report:"
 if [ -z "$AUTOTEST" ]
 then
 	read -p "Do you want to upload coverage report to codecov.io? [y/N]" ans
 	case $ans in
-		[Yy]* ) docker-compose exec -u www-data -T cnaas_api /opt/cnaas/coverage.sh;;
+		[Yy]* ) $COMPOSE_COMMAND exec -u www-data -T cnaas_api /opt/cnaas/coverage.sh;;
 		* ) echo "Not uploading coverage report";;
 	esac
 else
-	docker-compose exec -u www-data -T cnaas_api /opt/cnaas/coverage.sh
+	$COMPOSE_COMMAND exec -u www-data -T cnaas_api /opt/cnaas/coverage.sh
 fi
 
 docker logs docker_cnaas_dhcpd_1
 docker logs docker_cnaas_api_1
 
-docker-compose down
+$COMPOSE_COMMAND down