[FL-2649] Drop Docker in CI/CD (#1412)

* enable sparseCheckout, moving github actions from docker to raw shell * fix missing known_hosts while setting ssh priv key * fix build.yml * add ssh key to upload just in time * fixing rsync syntax * fix build.yml * try to fix build.yml again * testing rsync * test rsync again * add linters * add Black Python linter to submodules * add Black submodule * add working python linter target, dirty file list * up toolchain to version 4 * up toolchain to ver 5 * up toolchain version to 6 * fbt: using black 22.6.0 * remove Black submodule, up toolchain to ver 7 * fbt: added lint_py, format_py targets * add pvs_studio workflow * fix pvs_studio segfault * fix pvs_studio command * fix pvs_studio command 2 * show env before run pvs_studio * try to debug pvs_studio * try to strace pvs_studio.. * Add FBT_TOOLCHAIN_PATH, MacOS Rosseta check, and ignore non-x86_64 linux architectures * prevent redownloading toolchain on github-runners * fix toolchain download exitcode * add strace to debug pvs_studio segfault * disable strace to catch full code dump * Add './fbt cli' target to access Flipper CLI via PySerial * remove pvs_studio from this PR * removing clang-format from toolchain due errors * make source easy, and fix some mistakes found by @hedger * Add check_submodules workflow, some fixes * fixing mistakes Co-authored-by: hedger <[email protected]> Co-authored-by: hedger <[email protected]>
0xchocolate · Aug 2, 2022 · 93a4b9c · 93a4b9c
1 parent a1637e9
commit 93a4b9c
Show file tree

Hide file tree

Showing 17 changed files with 357 additions and 340 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -15,11 +15,8 @@ env:
 
 jobs:
   main:
-    runs-on: [self-hosted,FlipperZero]
+    runs-on: [self-hosted,FlipperZeroShell]
     steps:
-      - name: 'Cleanup workspace'
-        uses: AutoModality/action-clean@v1
-
       - name: 'Decontaminate previous build leftovers'
         run: |
           if [ -d .git ]
@@ -32,12 +29,8 @@ jobs:
         uses: actions/checkout@v2
         with:
           fetch-depth: 0
-          submodules: true
           ref: ${{ github.event.pull_request.head.sha }}
 
-      - name: 'Build docker image'
-        uses: ./.github/actions/docker
-
       - name: 'Make artifacts directory'
         run: |
           test -d artifacts && rm -rf artifacts || true
@@ -71,40 +64,34 @@ jobs:
         run: |
           tar czpf artifacts/flipper-z-any-scripts-${{steps.names.outputs.suffix}}.tgz scripts
 
-      - name: 'Build the firmware in docker'
-        uses: ./.github/actions/docker
-        with:
-          run: |
-            set -e
-            for TARGET in ${TARGETS}
-            do
-              ./fbt TARGET_HW=`echo ${TARGET} | sed 's/f//'` updater_package ${{ startsWith(github.ref, 'refs/tags') && 'DEBUG=0 COMPACT=1' || '' }}
-            done
+      - name: 'Build the firmware'
+        run: |
+          set -e
+          for TARGET in ${TARGETS}
+          do
+            FBT_TOOLCHAIN_PATH=/opt ./fbt TARGET_HW=`echo ${TARGET} | sed 's/f//'` updater_package ${{ startsWith(github.ref, 'refs/tags') && 'DEBUG=0 COMPACT=1' || '' }}
+          done
 
       - name: 'Move upload files'
         if: ${{ !github.event.pull_request.head.repo.fork }}
-        uses: ./.github/actions/docker
-        with:
-          run: |
-            set -e
-            for TARGET in ${TARGETS}
-            do
-              mv dist/${TARGET}-*/* artifacts/
-            done
+        run: |
+          set -e
+          for TARGET in ${TARGETS}
+          do
+            mv dist/${TARGET}-*/* artifacts/
+          done
 
       - name: 'Bundle self-update package'
         if: ${{ !github.event.pull_request.head.repo.fork }}
-        uses: ./.github/actions/docker
-        with:
-          run: |
-            set -e
-            for UPDATEBUNDLE in artifacts/*/
-            do
-              BUNDLE_NAME=`echo $UPDATEBUNDLE | cut -d'/' -f2`
-              echo Packaging ${BUNDLE_NAME}
-              tar czpf artifacts/flipper-z-${BUNDLE_NAME}.tgz -C artifacts ${BUNDLE_NAME}
-              rm -rf artifacts/${BUNDLE_NAME}
-            done
+        run: |
+          set -e
+          for UPDATEBUNDLE in artifacts/*/
+          do
+            BUNDLE_NAME=`echo $UPDATEBUNDLE | cut -d'/' -f2`
+            echo Packaging ${BUNDLE_NAME}
+            tar czpf artifacts/flipper-z-${BUNDLE_NAME}.tgz -C artifacts ${BUNDLE_NAME}
+            rm -rf artifacts/${BUNDLE_NAME}
+          done
 
       - name: "Check for uncommited changes"
         run: |
@@ -117,29 +104,23 @@ jobs:
 
       - name: 'Bundle core2 firmware'
         if: ${{ !github.event.pull_request.head.repo.fork }}
-        uses: ./.github/actions/docker
-        with:
-          run: |
-            ./fbt copro_dist
-            tar czpf artifacts/flipper-z-any-core2_firmware-${{steps.names.outputs.suffix}}.tgz -C assets core2_firmware
+        run: |
+          FBT_TOOLCHAIN_PATH=/opt ./fbt copro_dist
+          tar czpf artifacts/flipper-z-any-core2_firmware-${{steps.names.outputs.suffix}}.tgz -C assets core2_firmware
 
       - name: 'Upload artifacts to update server'
         if: ${{ !github.event.pull_request.head.repo.fork }}
-        uses: burnett01/[email protected]
-        with:
-          switches: -avzP --delete --mkpath
-          path: artifacts/
-          remote_path: "${{ secrets.RSYNC_DEPLOY_BASE_PATH }}${{steps.names.outputs.artifacts-path}}/"
-          remote_host: ${{ secrets.RSYNC_DEPLOY_HOST }}
-          remote_port: ${{ secrets.RSYNC_DEPLOY_PORT }}
-          remote_user: ${{ secrets.RSYNC_DEPLOY_USER }}
-          remote_key: ${{ secrets.RSYNC_DEPLOY_KEY }}
+        run: |
+          echo "${{ secrets.RSYNC_DEPLOY_KEY }}" > deploy_key;
+          chmod 600 ./deploy_key;
+          rsync -avzP --mkpath \
+              -e 'ssh -p ${{ secrets.RSYNC_DEPLOY_PORT }} -i ./deploy_key' \
+              artifacts/ ${{ secrets.RSYNC_DEPLOY_USER }}@${{ secrets.RSYNC_DEPLOY_HOST }}:"${{ secrets.RSYNC_DEPLOY_BASE_PATH }}${{steps.names.outputs.artifacts-path}}/";
+          rm ./deploy_key;
 
       - name: 'Trigger update server reindex'
         if: ${{ !github.event.pull_request.head.repo.fork }}
-        uses: wei/curl@master
-        with:
-          args: -X POST -F 'key=${{ secrets.REINDEX_KEY }}' ${{ secrets.REINDEX_URL }}
+        run: curl -X POST -F 'key=${{ secrets.REINDEX_KEY }}' ${{ secrets.REINDEX_URL }}
 
       - name: 'Find Previous Comment'
         if: ${{ !github.event.pull_request.head.repo.fork && github.event.pull_request }}
@@ -165,11 +146,8 @@ jobs:
 
   compact:
     if: ${{ !startsWith(github.ref, 'refs/tags') }}
-    runs-on: [self-hosted,FlipperZero]
+    runs-on: [self-hosted,FlipperZeroShell]
     steps:
-      - name: 'Cleanup workspace'
-        uses: AutoModality/action-clean@v1
-
       - name: 'Decontaminate previous build leftovers'
         run: |
           if [ -d .git ]
@@ -185,9 +163,6 @@ jobs:
           submodules: true
           ref: ${{ github.event.pull_request.head.sha }}
 
-      - name: 'Build docker image'
-        uses: ./.github/actions/docker
-
       - name: 'Generate suffix and folder name'
         id: names
         run: |
@@ -207,12 +182,10 @@ jobs:
           echo "WORKFLOW_BRANCH_OR_TAG=${BRANCH_OR_TAG}" >> $GITHUB_ENV
           echo "DIST_SUFFIX=${SUFFIX}" >> $GITHUB_ENV
 
-      - name: 'Build the firmware in docker'
-        uses: ./.github/actions/docker
-        with:
-          run: |
-            set -e
-            for TARGET in ${TARGETS}
-            do
-              ./fbt TARGET_HW=`echo ${TARGET} | sed 's/f//'` updater_package DEBUG=0 COMPACT=1
-            done
+      - name: 'Build the firmware'
+        run: |
+          set -e
+          for TARGET in ${TARGETS}
+          do
+            FBT_TOOLCHAIN_PATH=/opt ./fbt TARGET_HW=`echo ${TARGET} | sed 's/f//'` updater_package DEBUG=0 COMPACT=1
+          done
diff --git a/.github/workflows/build_toolchain.yml b/.github/workflows/build_toolchain.yml
diff --git a/.github/workflows/check_submodules.yml b/.github/workflows/check_submodules.yml
@@ -1,17 +1,47 @@
-name: 'Check submodules'
+name: 'Check submodules branch'
 
 on:
+  push:
+    branches:
+      - dev
+      - "release*"
+    tags:
+      - '*'
   pull_request:
 
 jobs:
-  protobuf:
-    runs-on: ubuntu-latest
+  check_protobuf:
+    runs-on: [self-hosted, FlipperZeroShell]
     steps:
-    - name: 'Checkout code'
-      uses: actions/checkout@v2
-    - name: 'Check submodule commit branch'
-      uses: jtmullen/submodule-branch-check-action@v1
-      with:
-        path: assets/protobuf
-        branch: dev
-        fetch_depth: 50
+      - name: 'Decontaminate previous build leftovers'
+        run: |
+          if [ -d .git ]
+          then
+            git submodule status \
+              || git checkout `git rev-list --max-parents=0 HEAD | tail -n 1`
+          fi
+
+      - name: 'Checkout code'
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: 'Check protobuf branch'
+        run: |
+          SUB_PATH="assets/protobuf";
+          SUB_BRANCH="dev";
+          SUB_COMMITS_MIN=40;
+          cd "$SUB_PATH";
+          SUBMODULE_HASH="$(git rev-parse HEAD)";
+          BRANCHES=$(git branch -r --contains "$SUBMODULE_HASH");
+          COMMITS_IN_BRANCH="$(git rev-list --count dev)";
+          if [ $COMMITS_IN_BRANCH -lt $SUB_COMMITS_MIN ]; then
+            echo "::set-output name=fails::error";
+            echo "::error::Error: Too low commits in $SUB_BRANCH of submodule $SUB_PATH: $COMMITS_IN_BRANCH(expected $SUB_COMMITS_MIN+)";
+            exit 1;
+          fi
+          if ! grep -q "/$SUB_BRANCH" <<< "$BRANCHES"; then
+            echo "::set-output name=fails::error";
+            echo "::error::Error: Submodule $SUB_PATH is not on branch $SUB_BRANCH";
+            exit 1;
+          fi
diff --git a/.github/workflows/lint_c.yml b/.github/workflows/lint_c.yml
@@ -1,6 +1,6 @@
 name: 'Lint C/C++ with clang-format'
 
-on: 
+on:
   push:
     branches:
       - dev
@@ -14,11 +14,8 @@ env:
 
 jobs:
   lint_c_cpp:
-    runs-on: [self-hosted,FlipperZero]
+    runs-on: [self-hosted,FlipperZeroShell]
     steps:
-      - name: 'Cleanup workspace'
-        uses: AutoModality/action-clean@v1
-
       - name: 'Decontaminate previous build leftovers'
         run: |
           if [ -d .git ]
@@ -31,23 +28,10 @@ jobs:
         uses: actions/checkout@v2
         with:
           fetch-depth: 0
-          submodules: true
-
-      - name: 'Docker cache'
-        uses: satackey/[email protected]
-        continue-on-error: true
-        with:
-          key: docker-cache-${{ hashFiles('docker/**') }}-{hash}
-          restore-keys: docker-cache-${{ hashFiles('docker/**') }}-
-
-      - name: 'Build docker image'
-        uses: ./.github/actions/docker
 
       - name: 'Check code formatting'
         id: syntax_check
-        uses: ./.github/actions/docker
-        with:
-          run: SET_GH_OUTPUT=1 ./fbt lint
+        run: SET_GH_OUTPUT=1 FBT_TOOLCHAIN_PATH=/opt ./fbt lint
 
       - name: Report code formatting errors
         if: failure() && steps.syntax_check.outputs.errors && github.event.pull_request
@@ -59,4 +43,4 @@ jobs:
             ```
             ${{ steps.syntax_check.outputs.errors }}
             ```
-            You might want to run `docker compose exec dev make format` for an auto-fix.
+            You might want to run `./fbt format` for an auto-fix.
diff --git a/.github/workflows/lint_python.yml b/.github/workflows/lint_python.yml
@@ -1,6 +1,6 @@
 name: 'Python Lint'
 
-on: 
+on:
   push:
     branches:
       - dev
@@ -11,11 +11,8 @@ on:
 
 jobs:
   lint_python:
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted,FlipperZeroShell]
     steps:
-      - name: 'Cleanup workspace'
-        uses: AutoModality/action-clean@v1
-
       - name: 'Decontaminate previous build leftovers'
         run: |
           if [ -d .git ]
@@ -29,8 +26,5 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: 'Setup python'
-        uses: actions/setup-python@v2
-
-      - name: 'Check python code with black'
-        uses: psf/[email protected]
+      - name: 'Check code formatting'
+        run: SET_GH_OUTPUT=1 FBT_TOOLCHAIN_PATH=/opt ./fbt lint_py
diff --git a/.github/workflows/reindex.yml b/.github/workflows/reindex.yml
@@ -7,9 +7,8 @@ on:
 jobs:
   reindex:
     name: 'Reindex updates'
-    runs-on: [self-hosted,FlipperZero]
+    runs-on: [self-hosted,FlipperZeroShell]
     steps:
       - name: Trigger reindex
-        uses: wei/curl@master
-        with:
-          args: -X POST -F 'key=${{ secrets.REINDEX_KEY }}' ${{ secrets.REINDEX_URL }}
+        run: |
+          curl -X POST -F 'key=${{ secrets.REINDEX_KEY }}' ${{ secrets.REINDEX_URL }}