Move Complement test matrix jobs definition to match Sytest and Trial.

.ci/scripts/calculate_jobs.py

@@ -126,3 +126,46 @@
 
 test_matrix = json.dumps(sytest_tests)
 print(f"::set-output name=sytest_test_matrix::{test_matrix}")
+
+
+# First calculate the workers used during complement jobs
+#
+# Will only be used on tests with SYNAPSE_COMPLEMENT_USE_WORKERS enabled
+# Github sees a none existent key and evaluates it to an empty string, according to
+# https://docs.github.com/en/actions/learn-github-actions/contexts
+# See the last sentence before "Determining when to use contexts"
+
+complement_test_jobs = [
+    {"arrangement": "monolith", "database": "SQLite"},
+    {"arrangement": "monolith", "database": "Postgres"},
+    {
+        "arrangement": "workers",
+        "database": "Postgres",
+        "worker_types": ", ".join(
+            worker
+            for worker in [
+                "event_persister",
+                "event_persister",
+                "background_worker",
+                "frontend_proxy",
+                "event_creator",
+                "user_dir",
+                "media_repository",
+                "federation_inbound",
+                "federation_reader",
+                "federation_sender",
+                "synchrotron",
+                "client_reader",
+                "appservice",
+                "pusher",
+            ]
+        ),
+    },
+]
+
+print("::group::Calculated complement job with workers")
+print(json.dumps(complement_test_jobs, indent=4))
+print("::endgroup::")
+
+test_matrix = json.dumps(complement_test_jobs)
+print(f"::set-output name=complement_test_matrix::{test_matrix}")

.github/workflows/tests.yml

@@ -148,6 +148,7 @@ jobs:
     outputs:
       trial_test_matrix: ${{ steps.get-matrix.outputs.trial_test_matrix }}
       sytest_test_matrix: ${{ steps.get-matrix.outputs.sytest_test_matrix }}
+      complement_test_matrix: ${{ steps.get-matrix.outputs.complement_test_matrix }}
 
   trial:
     if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
@@ -428,21 +429,13 @@ jobs:
 
   complement:
     if: "${{ !failure() && !cancelled() }}"
-    needs: linting-done
+    needs: calculate-test-jobs
     runs-on: ubuntu-latest
 
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - arrangement: monolith
-            database: SQLite
-
-          - arrangement: monolith
-            database: Postgres
-
-          - arrangement: workers
-            database: Postgres
+        job: ${{ fromJson(needs.calculate-test-jobs.outputs.complement_test_matrix) }}
 
     steps:
       - name: Run actions/checkout@v3 for synapse
@@ -460,11 +453,15 @@ jobs:
       - name: Prepare Complement's Prerequisites
         run: synapse/.ci/scripts/setup_complement_prerequisites.sh
 
-      - run: |
-          set -o pipefail
-          POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | synapse/.ci/scripts/gotestfmt
+      - name: Run Complement Tests
         shell: bash
-        name: Run Complement Tests
+        env:
+          POSTGRES: ${{ (matrix.job.database == 'Postgres') && 1 || '' }}
+          WORKERS: ${{ (matrix.job.arrangement == 'workers') && 1 || '' }}
+          WORKER_TYPES: ${{ matrix.worker_types }}
+        run: |
+          set -o pipefail
+          COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | synapse/.ci/scripts/gotestfmt
 
   cargo-test:
     if: ${{ needs.changes.outputs.rust == 'true' }}

changelog.d/14153.misc

@@ -0,0 +1 @@
+Move Complement CI job matrix definition to central location to match Sytest and Trial. Contributed by Jason Little.

docker/complement/conf/start_for_complement.sh

@@ -6,7 +6,9 @@ set -e
 
 echo "Complement Synapse launcher"
 echo "  Args: $@"
-echo "  Env: SYNAPSE_COMPLEMENT_DATABASE=$SYNAPSE_COMPLEMENT_DATABASE SYNAPSE_COMPLEMENT_USE_WORKERS=$SYNAPSE_COMPLEMENT_USE_WORKERS"
+echo "  Env: SYNAPSE_COMPLEMENT_DATABASE=$SYNAPSE_COMPLEMENT_DATABASE"
+echo "  Env: SYNAPSE_COMPLEMENT_USE_WORKERS=$SYNAPSE_COMPLEMENT_USE_WORKERS"
+echo "  Env: SYNAPSE_WORKER_TYPES=$SYNAPSE_WORKER_TYPES"
 
 function log {
     d=$(date +"%Y-%m-%d %H:%M:%S,%3N")
@@ -44,28 +46,8 @@ esac
 
 
 if [[ -n "$SYNAPSE_COMPLEMENT_USE_WORKERS" ]]; then
-  # Specify the workers to test with
-  export SYNAPSE_WORKER_TYPES="\
-      event_persister, \
-      event_persister, \
-      background_worker, \
-      frontend_proxy, \
-      event_creator, \
-      user_dir, \
-      media_repository, \
-      federation_inbound, \
-      federation_reader, \
-      federation_sender, \
-      synchrotron, \
-      client_reader, \
-      appservice, \
-      pusher"
-
   # Improve startup times by using a launcher based on fork()
   export SYNAPSE_USE_EXPERIMENTAL_FORKING_LAUNCHER=1
-else
-  # Empty string here means 'main process only'
-  export SYNAPSE_WORKER_TYPES=""
 fi
 
 

scripts-dev/complement.sh

@@ -139,6 +139,9 @@ if [[ -n "$WORKERS" ]]; then
   # Use workers.
   export PASS_SYNAPSE_COMPLEMENT_USE_WORKERS=true
 
+  # Pass through the workers defined. If none, it will be an empty string
+  export PASS_SYNAPSE_WORKER_TYPES="$WORKER_TYPES"
+
   # Workers can only use Postgres as a database.
   export PASS_SYNAPSE_COMPLEMENT_DATABASE=postgres