Move deploy from TravisCI to CircleCI (#3075)

Our travisci keeps breaking for different reasons so we decided to move our deploy to circleci.
algorand · Oct 14, 2021 · 1e87c71 · 1e87c71
1 parent 83740b4
commit 1e87c71
Show file tree

Hide file tree

Showing 2 changed files with 101 additions and 58 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -5,22 +5,33 @@ orbs:
   go: circleci/[email protected]
   slack: circleci/[email protected]
 
+parameters:
+  ubuntu_image:
+    type: string
+    default: "ubuntu-2004:202104-01"
+  build_dir:
+    type: string
+    default: "/opt/cibuild"
+  result_path:
+    type: string
+    default: "/tmp/build_test_results"
+
 executors:
   amd64_medium:
     machine:
-      image: ubuntu-2004:202104-01
+      image: << pipeline.parameters.ubuntu_image >>
     resource_class: medium
   amd64_large:
     machine:
-      image: ubuntu-2004:202104-01
+      image: << pipeline.parameters.ubuntu_image >>
     resource_class: large
   arm64_medium:
     machine:
-      image: ubuntu-2004:202101-01
+      image: << pipeline.parameters.ubuntu_image >>
     resource_class: arm.medium
   arm64_large:
     machine:
-      image: ubuntu-2004:202101-01
+      image: << pipeline.parameters.ubuntu_image >>
     resource_class: arm.large
   mac_amd64_medium:
     macos:
@@ -138,6 +149,22 @@ workflows:
           requires:
             - << matrix.platform >>_<< matrix.job_type >>
 
+      - upload_binaries:
+          name: << matrix.platform >>_upload_binaries
+          matrix:
+            <<: *matrix-default
+          requires:
+            - << matrix.platform >>_test_nightly_verification
+            - << matrix.platform >>_integration_nightly_verification
+            - << matrix.platform >>_e2e_expect_nightly_verification
+          filters:
+            branches:
+              only:
+                - /rel\/.*/
+          context:
+            - slack-secrets
+            - aws-secrets
+
       #- windows_x64_build
 
 commands:
@@ -146,18 +173,20 @@ commands:
     steps:
       - run: |
           sudo rm -rf ${HOME}/.go_workspace /usr/local/go
+
   prepare_build_dir:
     description: Set up build directory
     parameters:
       build_dir:
         type: string
-        default: "/opt/cibuild"
+        default: << pipeline.parameters.build_dir >>
     steps:
       - run:
           working_directory: /tmp
           command: |
             sudo mkdir -p << parameters.build_dir >>
             sudo chown -R $USER:$GROUP << parameters.build_dir >>
+
   prepare_windows:
     description: Prepare windows image
     steps:
@@ -180,7 +209,7 @@ commands:
     parameters:
       build_dir:
         type: string
-        default: "/opt/cibuild"
+        default: << pipeline.parameters.build_dir >>
     steps:
       - restore_libsodium
       - restore_cache:
@@ -246,7 +275,7 @@ commands:
     parameters:
       build_dir:
         type: string
-        default: "/opt/cibuild"
+        default: << pipeline.parameters.build_dir >>
       result_subdir:
         type: string
       no_output_timeout:
@@ -257,11 +286,14 @@ commands:
         default: ""
       result_path:
         type: string
-        default: "/tmp/results"
+        default: << pipeline.parameters.result_path >>
     steps:
       - attach_workspace:
           at: << parameters.build_dir >>
-      - run: mkdir -p << parameters.result_path >>/<< parameters.result_subdir >>/$CIRCLE_NODE_INDEX
+      - run: |
+          mkdir -p << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX}
+          touch << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX}/results.xml
+          touch << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX}/testresults.json
       - restore_cache:
           keys:
             - 'go-cache-v2-{{ .Environment.CIRCLE_STAGE }}-'
@@ -284,8 +316,8 @@ commands:
             scripts/buildtools/install_buildtools.sh -o "gotest.tools/gotestsum"
             PACKAGES="$(go list ./... | grep -v /go-algorand/test/)"
             export PACKAGE_NAMES=$(echo $PACKAGES | tr -d '\n')
-            export PARTITION_TOTAL=$CIRCLE_NODE_TOTAL
-            export PARTITION_ID=$CIRCLE_NODE_INDEX
+            export PARTITION_TOTAL=${CIRCLE_NODE_TOTAL}
+            export PARTITION_ID=${CIRCLE_NODE_INDEX}
             export PARALLEL_FLAG="-p 1"
             gotestsum --format testname --junitfile << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX}/results.xml --jsonfile << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX}/testresults.json -- --tags "sqlite_unlock_notify sqlite_omit_load_extension" << parameters.short_test_flag >> -race -timeout 1h -coverprofile=coverage.txt -covermode=atomic -p 1 $PACKAGE_NAMES
       - store_artifacts:
@@ -316,7 +348,7 @@ commands:
     parameters:
       build_dir:
         type: string
-        default: "/opt/cibuild"
+        default: << pipeline.parameters.build_dir >>
       result_subdir:
         type: string
       no_output_timeout:
@@ -327,11 +359,14 @@ commands:
         default: ""
       result_path:
         type: string
-        default: "/tmp/results"
+        default: << pipeline.parameters.result_path >>
     steps:
       - attach_workspace:
           at: << parameters.build_dir >>
-      - run: mkdir -p << parameters.result_path >>/<< parameters.result_subdir >>/$CIRCLE_NODE_INDEX
+      - run: |
+          mkdir -p << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX}
+          touch << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX}/results.xml
+          touch << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX}/testresults.json
       - run:
           name: Run integration tests
           no_output_timeout: << parameters.no_output_timeout >>
@@ -351,9 +386,9 @@ commands:
             scripts/buildtools/install_buildtools.sh -o "gotest.tools/gotestsum"
             export ALGOTEST=1
             export SHORTTEST=<< parameters.short_test_flag >>
-            export TEST_RESULTS=<< parameters.result_path >>/<< parameters.result_subdir >>/$CIRCLE_NODE_INDEX
-            export PARTITION_TOTAL=$CIRCLE_NODE_TOTAL
-            export PARTITION_ID=$CIRCLE_NODE_INDEX
+            export TEST_RESULTS=<< parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX}
+            export PARTITION_TOTAL=${CIRCLE_NODE_TOTAL}
+            export PARTITION_ID=${CIRCLE_NODE_INDEX}
             test/scripts/run_integration_tests.sh
       - store_artifacts:
           path: << parameters.result_path >>
@@ -370,7 +405,7 @@ commands:
     parameters:
       result_path:
         type: string
-        default: "/tmp/results"
+        default: << pipeline.parameters.result_path >>
       result_subdir:
         type: string
     steps:
@@ -385,6 +420,27 @@ commands:
           path: << parameters.result_path >>/<< parameters.result_subdir >>
           destination: << parameters.result_subdir >>/combined-test-results
 
+  upload_binaries_command:
+    description: save build artifacts for potential deployments
+    parameters:
+      platform:
+        type: string
+      build_dir:
+        type: string
+        default: << pipeline.parameters.build_dir >>
+    steps:
+        - run:
+            name: Upload binaries << parameters.platform >>
+            environment:
+              TRAVIS_BRANCH: ${CIRCLE_BRANCH}
+            command: |
+              scripts/travis/deploy_packages.sh
+        - when:
+            condition:
+              equal: [ "amd64", << parameters.platform >> ]
+            steps:
+              - run: scripts/travis/test_release.sh
+
 jobs:
   codegen_verification:
     executor: amd64_medium
@@ -401,7 +457,7 @@ jobs:
       platform:
         type: string
     executor: << parameters.platform >>_medium
-    working_directory: /opt/cibuild/project
+    working_directory: << pipeline.parameters.build_dir >>/project
     steps:
       - prepare_build_dir
       - checkout
@@ -413,7 +469,7 @@ jobs:
       platform:
         type: string
     executor: << parameters.platform >>_large
-    working_directory: /opt/cibuild/project
+    working_directory: << pipeline.parameters.build_dir >>/project
     parallelism: 4
     steps:
       - prepare_build_dir
@@ -428,7 +484,7 @@ jobs:
       platform:
         type: string
     executor: << parameters.platform >>_large
-    working_directory: /opt/cibuild/project
+    working_directory: << pipeline.parameters.build_dir >>/project
     parallelism: 4
     steps:
       - prepare_build_dir
@@ -446,7 +502,7 @@ jobs:
       platform:
         type: string
     executor: << parameters.platform >>_large
-    working_directory: /opt/cibuild/project
+    working_directory: << pipeline.parameters.build_dir >>/project
     parallelism: 4
     environment:
       E2E_TEST_FILTER: "GO"
@@ -462,7 +518,7 @@ jobs:
       platform:
         type: string
     executor: << parameters.platform >>_large
-    working_directory: /opt/cibuild/project
+    working_directory: << pipeline.parameters.build_dir >>/project
     parallelism: 4
     environment:
       E2E_TEST_FILTER: "GO"
@@ -480,7 +536,7 @@ jobs:
       platform:
         type: string
     executor: << parameters.platform >>_large
-    working_directory: /opt/cibuild/project
+    working_directory: << pipeline.parameters.build_dir >>/project
     parallelism: 4
     environment:
       E2E_TEST_FILTER: "EXPECT"
@@ -496,7 +552,7 @@ jobs:
       platform:
         type: string
     executor: << parameters.platform >>_large
-    working_directory: /opt/cibuild/project
+    working_directory: << pipeline.parameters.build_dir >>/project
     parallelism: 4
     environment:
       E2E_TEST_FILTER: "EXPECT"
@@ -514,7 +570,7 @@ jobs:
       platform:
         type: string
     executor: << parameters.platform >>_large
-    working_directory: /opt/cibuild/project
+    working_directory: << pipeline.parameters.build_dir >>/project
     parallelism: 4
     environment:
       E2E_TEST_FILTER: "SCRIPTS"
@@ -530,7 +586,7 @@ jobs:
       platform:
         type: string
     executor: << parameters.platform >>_large
-    working_directory: /opt/cibuild/project
+    working_directory: << pipeline.parameters.build_dir >>/project
     parallelism: 4
     environment:
       E2E_TEST_FILTER: "SCRIPTS"
@@ -566,6 +622,7 @@ jobs:
     docker:
       - image: python:3.9.6-alpine
     resource_class: small
+    working_directory: << pipeline.parameters.build_dir >>/project
     parameters:
       platform: # platform: ["amd64", "arm64", "mac_amd64"]
         type: string
@@ -575,3 +632,19 @@ jobs:
       - checkout
       - tests_verification_command:
           result_subdir: << parameters.platform >>_<< parameters.job_type >>
+
+  upload_binaries:
+    working_directory: << pipeline.parameters.build_dir >>/project
+    parameters:
+      platform:
+        type: string
+    executor: << parameters.platform >>_medium
+    steps:
+      - prepare_build_dir
+      - checkout
+      - prepare_go
+      - upload_binaries_command:
+          platform: << parameters.platform >>
+      - slack/notify: &slack-fail-event
+          event: fail
+          template: basic_fail_1
diff --git a/.travis.yml b/.travis.yml
@@ -12,17 +12,11 @@ stages:
     if: type = pull_request
   - name: build_release
     if: (branch =~ /^hotfix\// OR branch =~ /^rel\//) AND type != pull_request
-  - name: deploy
-    if: branch =~ /^rel\// AND type != pull_request
-  - name: post_deploy
-    if: branch =~ /^rel\// AND type != pull_request
 
 jobs:
   allow_failures:
-    - name: External ARM64 Deploy
     - name: External ARM64 Integration Test
     - name: External ARM Build
-    - name: External ARM Deploy
     - name: Test Release Builds
   include:
     - stage: build_commit
@@ -142,30 +136,6 @@ jobs:
        script:
          - $mingw64 scripts/travis/build_test.sh
 
-    - stage: deploy
-      name: Ubuntu Deploy
-      os: linux
-      script:
-        - scripts/travis/deploy_packages.sh
-        - scripts/travis/test_release.sh
-    - # same stage, parallel job
-      name: MacOS Deploy
-      os: osx
-      osx_image: xcode11
-      script: scripts/travis/deploy_packages.sh
-    - # same stage, parallel job
-      name: External ARM64 Deploy
-      os: linux
-      env:
-        - BUILD_TYPE: "external_build"
-        - TARGET_PLATFORM: "linux-arm64"
-      addons:
-        apt:
-          packages:
-            - awscli
-      script:
-        - scripts/travis/external_build.sh ./scripts/travis/deploy_packages.sh
-
 # Don't rebuild libsodium every time
 cache:
   directories:
@@ -225,4 +195,4 @@ notifications:
   slack:
     if: branch = "rel/nightly"
     rooms:
-      - secure: FvM+GHkfVQyJqounsRO8kcQT2fYljdBp/fODIlgNE1pcd08WKqHa14cpjQN2T/zC7/eeJAwDuk4ADUqgLdb85vSd6KgcYwBaPMLj8b1La/JY17ULpVuD1uke3uDb+OhC084Uo7cYUAcmpPOtX7ujRVXbeRFoj5R6xqyCYEBGeRNaQIJh36Jd1jkKjQGZgcQIMy0JZXTnVwxP1o3ed6mYtVMt0qlBCzHKXTr7CUTFp+TwFoZ99jxvjNEA6cpn1UE82uq08CiQ4ANY51SCMqq6L9y76SRREznjtR3CGGhZYlmewVnlsco6IB0Xe5k4j8GlIlZfV+bJiBaE4GqNKOpt0DcB/hxZtDbe7LcSfdtwU/NG8L1rw4Ktj+ALcEJDuprRhYaafEaL6iOVsOy6YwW2Tbj6xcJzeorTs/p0URkSJ/M1Us7zdOTzLkVeUKAcHIIl4SClHq3svzSwoSyjlwuq18Wp2VWNfQFY3qFd7eZo/meM8UuhRrTai18eX5ZJ9leAdwvwfma1uxS2ZlG8aDupx2Ww19AOZZ8txQtDTn+rzP8U1UnlaOMJStOmdEKfExGy348f1yQcJtOv4armZuEuC6eoIkeznNS8wO/ZkRSWuxDcc5Q1DsQADXfGSMEL36Aa6aDU5Hy7els00qYDImPVL+7rd2pAfSUzFCfA+7klefg=
+      - secure: FvM+GHkfVQyJqounsRO8kcQT2fYljdBp/fODIlgNE1pcd08WKqHa14cpjQN2T/zC7/eeJAwDuk4ADUqgLdb85vSd6KgcYwBaPMLj8b1La/JY17ULpVuD1uke3uDb+OhC084Uo7cYUAcmpPOtX7ujRVXbeRFoj5R6xqyCYEBGeRNaQIJh36Jd1jkKjQGZgcQIMy0JZXTnVwxP1o3ed6mYtVMt0qlBCzHKXTr7CUTFp+TwFoZ99jxvjNEA6cpn1UE82uq08CiQ4ANY51SCMqq6L9y76SRREznjtR3CGGhZYlmewVnlsco6IB0Xe5k4j8GlIlZfV+bJiBaE4GqNKOpt0DcB/hxZtDbe7LcSfdtwU/NG8L1rw4Ktj+ALcEJDuprRhYaafEaL6iOVsOy6YwW2Tbj6xcJzeorTs/p0URkSJ/M1Us7zdOTzLkVeUKAcHIIl4SClHq3svzSwoSyjlwuq18Wp2VWNfQFY3qFd7eZo/meM8UuhRrTai18eX5ZJ9leAdwvwfma1uxS2ZlG8aDupx2Ww19AOZZ8txQtDTn+rzP8U1UnlaOMJStOmdEKfExGy348f1yQcJtOv4armZuEuC6eoIkeznNS8wO/ZkRSWuxDcc5Q1DsQADXfGSMEL36Aa6aDU5Hy7els00qYDImPVL+7rd2pAfSUzFCfA+7klefg=