Separate sensor model behavior tests from integration tests

.github/workflows/build.yml

@@ -10,13 +10,13 @@ jobs:
 
     steps:
     - name: Checkout Model
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         submodules: true
 
     - name: Cache Protobuf
       id: cache-protobuf
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: protobuf-21.12
         key: ${{ runner.os }}-protobuf
@@ -55,7 +55,7 @@ jobs:
 
     - name: Cache Model FMU
       id: cache-model-fmu
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: /tmp/model_fmu
         key: ${{ runner.os }}-model-fmu-${{ github.sha }}

.github/workflows/build_esmini.yml

@@ -11,7 +11,7 @@ jobs:
     steps:
       - name: Cache esmini FMU
         id: cache-esmini-fmu
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: /tmp/esmini_fmu
           key: ${{ runner.os }}-esmini-fmu

.github/workflows/build_openmcx.yml

@@ -11,7 +11,7 @@ jobs:
     steps:
       - name: Cache OpenMCx
         id: cache-openmcx
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: openmcx
           key: ${{ runner.os }}-openmcx

.github/workflows/build_osi_field_checker.yml

@@ -11,7 +11,7 @@ jobs:
     steps:
       - name: Cache Protobuf
         id: cache-protobuf
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: protobuf-21.12
           key: ${{ runner.os }}-protobuf
@@ -31,7 +31,7 @@ jobs:
 
       - name: Cache OSI Field Checker FMU
         id: cache-osi-field-checker
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: /tmp/osi-field-checker
           key: ${{ runner.os }}-osi-field-checker

.github/workflows/build_tracefile_player.yml

@@ -11,7 +11,7 @@ jobs:
     steps:
       - name: Cache Protobuf
         id: cache-protobuf
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: protobuf-21.12
           key: ${{ runner.os }}-protobuf
@@ -31,7 +31,7 @@ jobs:
 
       - name: Cache Tracefile Player FMU
         id: cache-tracefile-player-fmu
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: /tmp/tracefile_player_fmu
           key: ${{ runner.os }}-tracefile-player-fmu

.github/workflows/build_tracefile_writer.yml

@@ -11,7 +11,7 @@ jobs:
     steps:
       - name: Cache Protobuf
         id: cache-protobuf
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: protobuf-21.12
           key: ${{ runner.os }}-protobuf
@@ -31,7 +31,7 @@ jobs:
 
       - name: Cache Tracefile Writer FMU
         id: cache-tracefile-writer-fmu
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: /tmp/tracefile_writer_fmu
           key: ${{ runner.os }}-tracefile-writer-fmu

.github/workflows/cl0.yml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
     - uses: enarx/spdx@master
       with:
         licenses: Apache-2.0 BSD-3-Clause BSD-2-Clause BSL-1.0 MIT MPL-2.0

.github/workflows/cl2.yml

@@ -29,7 +29,7 @@ jobs:
     name: Generate Integration Test Paths
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Generate matrix with all integration tests
         id: set-matrix
         working-directory: test/integration
@@ -52,55 +52,55 @@ jobs:
         run: echo ${{ github.workspace }}/${{ matrix.path }}
 
       - name: Checkout Model
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           submodules: true
 
       - name: Cache Protobuf
         id: cache-protobuf
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: protobuf-21.12
           key: ${{ runner.os }}-protobuf
 
       - name: Cache Model FMU
         id: cache-model-fmu
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: /tmp/model_fmu
           key: ${{ runner.os }}-model-fmu-${{ github.sha }}
 
       - name: Cache Tracefile Player FMU
         id: cache-tracefile-player-fmu
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: /tmp/tracefile_player_fmu
           key: ${{ runner.os }}-tracefile-player-fmu
 
       - name: Cache Tracefile Writer FMU
         id: cache-tracefile-writer-fmu
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: /tmp/tracefile_writer_fmu
           key: ${{ runner.os }}-tracefile-writer-fmu
 
       - name: Cache esmini FMU
         id: cache-esmini-fmu
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: /tmp/esmini_fmu
           key: ${{ runner.os }}-esmini-fmu
 
       - name: Cache OSI Field Checker
         id: cache-osi-field-checker
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: /tmp/osi-field-checker
           key: ${{ runner.os }}-osi-field-checker
 
       - name: Cache OpenMCx
         id: cache-openmcx
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: openmcx
           key: ${{ runner.os }}-openmcx
@@ -136,14 +136,13 @@ jobs:
 
       - name: Archive Trace File
         if: steps.check-for-trace-files.outputs.found_trace == 1
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ steps.check-for-trace-files.outputs.trace_file_name }}
           path: ./test/integration/${{ matrix.path }}/output/${{ steps.check-for-trace-files.outputs.trace_file_name }}
 
       - name: Check For Python Scripts
         id: check-for-python-scripts
-        if: steps.check-for-trace-files.outputs.found_trace == 1
         working-directory: ./test/integration/${{ matrix.path }}
         run: |
           ( count=`ls -1 *.py 2>/dev/null | wc -l`
@@ -165,14 +164,14 @@ jobs:
         run: sudo make install && sudo ldconfig
 
       - name: Install Analysis Dependencies
-        if: steps.check-for-python-scripts.outputs.found_script == 1 && steps.check-for-trace-files.outputs.found_trace == 1
+        if: steps.check-for-python-scripts.outputs.found_script == 1
         working-directory: ./lib/open-simulation-interface
         run: |
           python3 -m pip install protobuf==4.21.12
           python3 -m pip install .
 
       - name: Run Trace File Analysis
-        if: steps.check-for-python-scripts.outputs.found_script == 1 && steps.check-for-trace-files.outputs.found_trace == 1
+        if: steps.check-for-python-scripts.outputs.found_script == 1
         working-directory: ./test/integration/${{ matrix.path }}
         run: python3 ${{ steps.check-for-python-scripts.outputs.python_file_name }} output/${{ steps.check-for-trace-files.outputs.trace_file_name }}
 

.github/workflows/cl3.yml

@@ -0,0 +1,159 @@
+name: Credibility Assessment Level 3
+
+on:
+  workflow_call:
+
+jobs:
+  generate_behavior_test_paths:
+    runs-on: ubuntu-latest
+    name: Generate Behavior Test Paths
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Generate matrix with all behavior tests
+        id: set-matrix
+        working-directory: test/behavior
+        run: |
+          echo "matrix=$(ls -l | grep '^d' | awk -F ' ' '{print $9}' | grep -Po '\d{3}.*' | jq -R -s -c 'split("\n") | map(select(length > 0))')" >> $GITHUB_OUTPUT
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+
+  run_behavior_test:
+    needs: [generate_behavior_test_paths]
+    name: Behavior Test
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        path: ${{ fromJson(needs.generate_behavior_test_paths.outputs.matrix) }}
+
+    steps:
+      - name: Print matrix path
+        run: echo ${{ github.workspace }}/${{ matrix.path }}
+
+      - name: Checkout Model
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+
+      - name: Cache Protobuf
+        id: cache-protobuf
+        uses: actions/cache@v4
+        with:
+          path: protobuf-21.12
+          key: ${{ runner.os }}-protobuf
+
+      - name: Cache Model FMU
+        id: cache-model-fmu
+        uses: actions/cache@v4
+        with:
+          path: /tmp/model_fmu
+          key: ${{ runner.os }}-model-fmu-${{ github.sha }}
+
+      - name: Cache Tracefile Player FMU
+        id: cache-tracefile-player-fmu
+        uses: actions/cache@v4
+        with:
+          path: /tmp/tracefile_player_fmu
+          key: ${{ runner.os }}-tracefile-player-fmu
+
+      - name: Cache Tracefile Writer FMU
+        id: cache-tracefile-writer-fmu
+        uses: actions/cache@v4
+        with:
+          path: /tmp/tracefile_writer_fmu
+          key: ${{ runner.os }}-tracefile-writer-fmu
+
+      - name: Cache esmini FMU
+        id: cache-esmini-fmu
+        uses: actions/cache@v4
+        with:
+          path: /tmp/esmini_fmu
+          key: ${{ runner.os }}-esmini-fmu
+
+      - name: Cache OSI Field Checker
+        id: cache-osi-field-checker
+        uses: actions/cache@v4
+        with:
+          path: /tmp/osi-field-checker
+          key: ${{ runner.os }}-osi-field-checker
+
+      - name: Cache OpenMCx
+        id: cache-openmcx
+        uses: actions/cache@v4
+        with:
+          path: openmcx
+          key: ${{ runner.os }}-openmcx
+
+      - name: Install dependencies
+        run: sudo apt install -y libxml2-dev zlib1g-dev libzip-dev
+
+      - name: Install ProtoBuf
+        working-directory: protobuf-21.12
+        run: sudo make install && sudo ldconfig
+
+      - name: Check if analysis script exists
+        id: check_analysis_script
+        uses: andstor/file-existence-action@v3
+        with:
+          files: test/behavior/${{ matrix.path }}/analyze.py
+
+      - name: Check if trace file generator exists
+        id: check_trace_generator
+        uses: andstor/file-existence-action@v3
+        with:
+          files: test/behavior/${{ matrix.path }}/generate_test_trace.py
+
+      - name: Install OSI for python
+        if: steps.check_analysis_script.outputs.files_exists == 'true' || steps.check_trace_generator.outputs.files_exists == 'true'
+        working-directory: ./lib/open-simulation-interface
+        run: |
+          python3 -m pip install protobuf==4.21.12
+          python3 -m pip install .
+
+      - name: Run test trace generation
+        if: steps.check_trace_generator.outputs.files_exists == 'true'
+        working-directory: ./test/behavior/${{ matrix.path }}
+        run: python3 generate_test_trace.py
+
+      - name: Create Output Folder
+        working-directory: ./test/behavior/${{ matrix.path }}
+        run: mkdir output
+
+      - name: Run OpenMCx
+        id: cosimulation
+        run: ./openmcx/install/openmcx ./test/behavior/${{ matrix.path }}/SystemStructure.ssd
+
+      - name: Check For Trace Files
+        id: check-for-trace-files
+        working-directory: ./test/behavior/${{ matrix.path }}/output
+        run: |
+          ( count=`ls -1 *.osi 2>/dev/null | wc -l`
+          if [ $count == 0 ]; then
+          echo No trace file found.
+          echo "found_trace=0" >> $GITHUB_OUTPUT
+          elif [ $count == 1 ]; then
+          echo "found_trace=1" >> $GITHUB_OUTPUT
+          filename=$(ls -1 *.osi)
+          echo "trace_file_name=${filename}" >> $GITHUB_OUTPUT
+          else
+          echo More than one trace file.
+          echo "found_trace=0" >> $GITHUB_OUTPUT
+          fi
+          )
+
+      - name: Archive Trace File
+        if: steps.check-for-trace-files.outputs.found_trace == 1
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ steps.check-for-trace-files.outputs.trace_file_name }}
+          path: ./test/behavior/${{ matrix.path }}/output/${{ steps.check-for-trace-files.outputs.trace_file_name }}
+
+      - name: Run Trace File Analysis
+        if: steps.check_analysis_script.outputs.files_exists == 'true'
+        working-directory: ./test/behavior/${{ matrix.path }}
+        run: python3 analyze.py output/${{ steps.check-for-trace-files.outputs.trace_file_name }}
+
+      - name: Failed?
+        if: steps.cosimulation.outputs.failed == 1
+        run: exit 1

.github/workflows/cpp-linter.yml

@@ -9,13 +9,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout Model
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         submodules: true
 
     - name: Cache Protobuf
       id: cache-protobuf
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: protobuf-21.12
         key: ${{ runner.os }}-protobuf

.github/workflows/credibility_assessment.yml

@@ -19,3 +19,8 @@ jobs:
     name: CL 2
     needs: cl1
     uses: ./.github/workflows/cl2.yml
+
+  cl3:
+    name: CL 3
+    needs: cl2
+    uses: ./.github/workflows/cl3.yml