Merge pull request #435 from PelionIoT/dev

Merge dev to master

.github/workflows/align-pe-utils.yml

@@ -0,0 +1,46 @@
+name: align-pe-utils-versions
+on: 
+    push:
+        paths:
+            - 'snap/snapcraft.yaml'
+
+# This allows a subsequently queued workflow run to interrupt previous runs
+concurrency:
+  group: align-pe-utils-${{ github.event.pull_request.head.label || github.head_ref || github.ref }} 
+  cancel-in-progress: true
+
+jobs:
+    check-pe-utils-aligned:
+        runs-on: ubuntu-22.04
+        steps:
+        - name: Check out code
+          uses: actions/checkout@v4
+        - name: Install yq
+          run: sudo snap install yq
+        - name: Check tag/hash alignment
+          run: |
+            pe_source=$(yq ".parts.pe-utils.source-commit" < snap/snapcraft.yaml)
+            info_source=$(yq ".parts.edge-info.source-commit" < snap/snapcraft.yaml)
+            testnet_source=$(yq ".parts.edge-testnet.source-commit" < snap/snapcraft.yaml)
+            # We might use tags or source commit (during dev)
+            if [[ "$pe_source" != "null" ]] && [[ "$info_source" != "null" ]] && [[ "$testnet_source" != "null" ]] ; then
+              if [[ "$pe_source" != "$info_source" ]] || [[ "$pe_source" != "$testnet_source" ]]; then
+                echo "pe-utils, edge-info and/or edge-testnet not using same commit hash!"
+                echo "pe-utils    : $pe_source"
+                echo "edge-info   : $info_source"
+                echo "edge-testnet: $testnet_source"
+                exit 1
+              fi
+            else
+              pe_tag=$(yq ".parts.pe-utils.source-tag" < snap/snapcraft.yaml)
+              info_tag=$(yq ".parts.edge-info.source-tag" < snap/snapcraft.yaml)
+              testnet_tag=$(yq ".parts.edge-testnet.source-tag" < snap/snapcraft.yaml)
+              # shellcheck disable=SC2252
+              if [[ "$pe_tag" != "$info_tag" ]] || [[ "$pe_tag" != "$testnet_tag" ]]; then
+                echo "pe-utils, edge-info and/or edge-testnet not using same tag!"
+                echo "pe-utils    : $pe_tag"
+                echo "edge-info   : $info_tag"
+                echo "edge-testnet: $testnet_tag"
+                exit 1
+              fi
+            fi

.github/workflows/build.yml

@@ -1,25 +1,204 @@
 name: Build
-on: push
+on: 
+  push:
+  workflow_dispatch:
+    inputs:
+        branch:
+          description: 'Branch to run the workflow on'
+          required: true
+          default: 'master'
+
+# This allows a subsequently queued workflow run to interrupt previous runs
+concurrency:
+  group: snap-pelion-edge-${{ github.event.pull_request.head.label || github.head_ref || github.ref }} 
+  cancel-in-progress: true
 
 jobs:
-  build:
-    runs-on: ubuntu-latest
+  build-snap:
+    runs-on: [ "self-hosted", "snap" ]
     steps:
+      - name: Enable write on all files
+        run:  chmod a+w -R .
       - name: Check out code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
+        with:
+          # Use the branch specified by the workflow_dispatch input or the pull_request event
+          ref: ${{ github.event.inputs.branch || github.event.pull_request.head.ref }}
       - name: Copy mbed_cloud_dev_credentials.c
         env:
-          MBED_CLOUD_DEV_CREDENTIALS: ${{ secrets.MBED_CLOUD_DEV_CREDENTIALS_C_RYAN }}
+          MBED_CLOUD_DEV_CREDENTIALS: ${{ secrets.MBED_CLOUD_DEV_CREDENTIALS_SNAP_CI }}
         run: |
           echo "$MBED_CLOUD_DEV_CREDENTIALS" > mbed_cloud_dev_credentials.c
       - name: Copy update_default_resources.c
         env:
-          UPDATE_DEFAULT_RESOURCES: ${{ secrets.UPDATE_DEFAULT_RESOURCES_C_RYAN }}
+          UPDATE_DEFAULT_RESOURCES: ${{ secrets.UPDATE_DEFAULT_RESOURCES_SNAP_CI }}
         run: |
           echo "$UPDATE_DEFAULT_RESOURCES" > update_default_resources.c
       - name: Build Docker image
         run: |
           docker build --no-cache -f Dockerfile --label snapcore/snapcraft --tag ${USER}/snapcraft:latest .
       - name: Build snap
         run: |
+          # If previous run failed, set everything to write mode.
+          chmod a+w -R .
           docker run --rm -v "$PWD":/build -w /build ${USER}/snapcraft:latest bash -c "sudo apt-get update && snapcraft --debug"
+          # The build left files with no write rights, which ruins next test run
+          chmod a+w -R .
+      - name: Archive the snap-binary
+        uses: actions/upload-artifact@v4
+        with:
+          name: pelion-edge-amd64.snap
+          path: ./pelion-edge*.snap
+          if-no-files-found: error
+      - name: Archive connect.sh
+        uses: actions/upload-artifact@v4
+        with:
+          name: connect.sh
+          path: ./connect.sh
+          if-no-files-found: error
+
+  test:
+    runs-on: ubuntu-22.04
+    needs: build-snap
+    timeout-minutes: 40
+    env:
+      ACCESS_KEY: ${{ secrets.IZUMA_ACCESS_KEY }}
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+        with:
+          # Use the branch specified by the workflow_dispatch input or the pull_request event
+          ref: ${{ github.event.inputs.branch || github.event.pull_request.head.ref }}
+      - name: Check out scripts-internal & e2e tests repos
+        run: | 
+          git config --global url."https://${{ secrets.ACCESS_TOKEN }}@github.com/".insteadOf "[email protected]:"
+          git config --global url."https://${{ secrets.ACCESS_TOKEN }}@github".insteadOf "https://github"
+          git clone [email protected]:PelionIoT/e2e-edge-test-suite
+          git clone [email protected]:PelionIoT/scripts-internal.git
+          ls -al
+      - name: Set up Python v3.8
+        uses: actions/setup-python@v5
+        with:
+          python-version: v3.8
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r e2e-edge-test-suite/requirements.txt
+          sudo apt install virtualenv
+          curl -LO https://dl.k8s.io/release/v1.25.0/bin/linux/amd64/kubectl
+          sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
+          kubectl version --client
+      - name: Get pelion-edge.snap from storage
+        uses: actions/download-artifact@v4
+        with:
+          name: pelion-edge-amd64.snap
+          path: e2e-edge-test-suite
+      - name: Get connect.sh from storage
+        uses: actions/download-artifact@v4
+        with:
+          name: connect.sh
+          path: e2e-edge-test-suite
+      - name: Install snap as sudo and test
+        run: |
+          snapfile=$(ls e2e-edge-test-suite/*.snap)
+          sudo scripts-internal/snap/snap-install.sh -s $snapfile
+          sleep 5
+          if sudo snap restart pelion-edge; then
+            echo "sudo snap restart succeeded."
+          else
+            echo "sudo snap restart pelion-edge failed. Test results might be invalid."
+          fi
+          sleep 5
+          snap services
+          snap list
+          pelion-edge.help
+          pelion-edge.edge-info
+          pelion-edge.version
+          pelion-edge.edge-testnet
+          pelion-edge.curl --version
+      - name: Check snap-pelion-edge versions
+        run: |
+          sudo snap install yq
+          snapcraftyamlver=$(yq -r ".version" <snap/snapcraft.yaml |sed -r "s/\x1B\[([0-9]{1,3}(;[0-9]{1,2};?)?)?[mGK]//g")
+          version=$(pelion-edge.version |grep snap-pelion-edge | awk -F' ' '{ print $NF }')
+          if [[ "$snapcraftyamlver" != "$version" ]]; then
+              echo "FAIL - Snapcraft.yaml and pelion-edge.version versions differ: $snapcraftyamlver vs. $version"
+              exit 1
+          fi
+          snapver=$(snap list |grep "pelion-edge" | awk -F' ' '{ print $2 }')
+          if [[ "$snapcraftyamlver" != "$snapver" ]]; then
+              echo "FAIL - Snapcraft.yaml and snap list versions differ: '$snapcraftyamlver' vs. '$snapver'"
+              exit 1
+          fi
+          infover=$(pelion-edge.edge-info |grep "Pelion Edge Version:" | awk -F' ' '{ print $NF }' |sed -r "s/\x1B\[([0-9]{1,3}(;[0-9]{1,2};?)?)?[mGK]//g" )
+          if [[ "$snapcraftyamlver" != "$infover" ]]; then
+              echo "FAIL - Snapcraft.yaml and pelion-edge.edge-info versions differ: '$snapcraftyamlver' vs. '$infover'"
+              exit 1
+          fi
+
+# Disabled for now - reboot is required to get installation working correctly,
+# we cannot however reboot a GitHub runner mid-job.
+#      - name: Run snap-test
+#        if: ${{ failure() }} || ${{ success() }}
+#        run: |
+#          cd e2e-edge-test-suite
+#          ../scripts-internal/snap/snap-test.sh -a ${{ secrets.IZUMA_ACCESS_KEY }}
+      - name: Remove snap
+        if: always()
+        run: |
+          sudo scripts-internal/snap/snap-remove.sh -a ${{ secrets.IZUMA_ACCESS_KEY }} -s pelion-edge
+#      - name: Archive the pytest.log
+#        uses: actions/upload-artifact@v4
+#        with:
+#          name: pytest.log
+#          path: e2e-edge-test-suite/pytest.log
+#          if-no-files-found: error
+#      - name: Publish Test Results
+#        uses: EnricoMi/publish-unit-test-result-action@v2
+#        if: always()
+#        with:
+#          files: |
+#           results.xml
+      - name: Clean up .gitconfig
+        if: always()
+        run: rm -f ~/.gitconfig
+
+  pysh-check:
+    runs-on: client
+    steps:
+      - uses: actions/checkout@v4
+      - run: sudo apt-get install -y black pycodestyle pydocstyle shellcheck python3
+      - name: Check out scripts-internal
+        uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.ACCESS_TOKEN }}
+          repository: PelionIoT/scripts-internal
+          path: scripts-internal
+      - run: |
+          echo . >scripts-internal/.nopyshcheck
+          scripts-internal/ci/more-lines-checker.sh dev ${{ github.ref_name }} "scripts-internal/pysh-check/pysh-check.sh --workdir . pysh" > pysh-check.log
+          cat pysh-check.log
+      - name: Archive the logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: Shellcheck logs
+          path: "./*.log"
+
+  yamllint-snapcraft-yaml:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+      - run: sudo apt-get install yamllint
+      - run: yamllint -c lint-config.yaml snap/snapcraft.yaml
+      - name: Check yq compliancy
+        run: |
+          sudo snap install yq
+          yq -r "." <snap/snapcraft.yaml >yqout.yaml
+          if diff -u snap/snapcraft.yaml yqout.yaml
+          then
+            echo "yq compliancy check passed"
+          else
+            echo "yq compliancy check failed"
+            exit 1
+          fi