From b0aea0ce890c302065ae0ee326d27235cc3a5a49 Mon Sep 17 00:00:00 2001 From: Shahzad Lone Date: Wed, 3 Aug 2022 01:49:04 -0400 Subject: [PATCH] tools: Migrate from CircleCi to GitHub Actions (#679) - Resolves #570 - Description: * Moves all our CircleCi workflows to GitHub Actions (Change Detection, Test Runs). * Change detection will only run in [PRs] or [pushes to tags, master, or develop]. * Change detection for GitHub Action Workflow needed explicit private and public ssh keys (generated and added the corresponding keys to secret section, and deploy key sections), there is an authentication step that takes care of the SSH authentication for that workflow. * Added a new Workflow to build defradb binary and call `defradb start` on it to ensure the start up command doesn't crash. * Removed CircleCi config file validation tool. --- .circleci/config.yml | 100 ---------------------- .github/workflows/code-test-coverage.yml | 4 +- .github/workflows/detect-change.yml | 48 +++++++++++ .github/workflows/lint-then-benchmark.yml | 8 +- .github/workflows/run-tests.yml | 27 ++++++ .github/workflows/start-binary.yml | 50 +++++++++++ Makefile | 30 ++++--- 7 files changed, 150 insertions(+), 117 deletions(-) delete mode 100644 .circleci/config.yml create mode 100644 .github/workflows/detect-change.yml create mode 100644 .github/workflows/run-tests.yml create mode 100644 .github/workflows/start-binary.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 86bb5559e8..0000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,100 +0,0 @@ -# Use the latest 2.1 version of CircleCI pipeline process engine. -# See: https://circleci.com/docs/2.0/configuration-reference -version: 2.1 - -#========================================================================================================[ COMMANDS ] -commands: - - # A reusable command with a boolean parameter to specify if we want to run change detection or not. - build_and_test: - - parameters: - no_change_detection: - description: "If true then just builds and runs tests, otherwise detects for database changes." - type: boolean - default: false - - steps: - - checkout - - - restore_cache: - keys: - - go-mod-v5-cimg-go-1.17-{{ checksum "go.sum" }} - - - run: - name: Install go modules into the module cache. - command: make deps:modules - - - save_cache: - key: go-mod-v5-cimg-go-1.17-{{ checksum "go.sum" }} - paths: - - "/go/pkg/mod" - - - when: - condition: << parameters.no_change_detection >> - steps: - - run: - name: Run tests - command: | - make test:verbose - environment: - DEFRA_BADGER_MEMORY: true - DEFRA_BADGER_FILE: true - DEFRA_MAP: true - - - when: - condition: - not: << parameters.no_change_detection >> - steps: - - run: - name: Run tests with change detection - command: | - make test:changes - environment: - DEFRA_DETECT_DATABASE_CHANGES: true - - - store_test_results: - path: "/tmp/defradb-dev" - - -#============================================================================================================[ JOBS ] -# Define jobs to be invoked later in a workflow. -# See: https://circleci.com/docs/2.0/configuration-reference/#jobs -jobs: - - # Job builds and runs all the test. - build: - - working_directory: ~/repo - - docker: - # https://circleci.com/developer/images/image/cimg/go - - image: cimg/go:1.17 - - steps: - - build_and_test: - no_change_detection: true - - # Job detects if we have a database change. - change_detection: - - working_directory: ~/repo - - docker: - # https://circleci.com/developer/images/image/cimg/go - - image: cimg/go:1.17 - - steps: - - build_and_test: - no_change_detection: false - - -#=======================================================================================================[ WORKFLOWS ] -# Invoke jobs via workflows -workflows: - - build-test: - - jobs: - - build - - change_detection diff --git a/.github/workflows/code-test-coverage.yml b/.github/workflows/code-test-coverage.yml index 84bf07d216..b5dbf78880 100644 --- a/.github/workflows/code-test-coverage.yml +++ b/.github/workflows/code-test-coverage.yml @@ -1,9 +1,11 @@ -name: Code Test Coverage +name: Code Test Coverage Workflow on: [push] jobs: code-test-coverage: + name: Code test coverage job + runs-on: ubuntu-latest steps: diff --git a/.github/workflows/detect-change.yml b/.github/workflows/detect-change.yml new file mode 100644 index 0000000000..bdc224cfe9 --- /dev/null +++ b/.github/workflows/detect-change.yml @@ -0,0 +1,48 @@ +name: Detect Change Workflow + +on: + pull_request: + + push: + tags: + - v* + branches: + - master + - develop + +jobs: + detect-change: + name: Detect change job + + runs-on: ubuntu-latest + + steps: + - name: Checkout code into the directory + uses: actions/checkout@v3 + + - name: Setup Go environment explicitly + uses: actions/setup-go@v3 + with: + go-version: "1.17" + check-latest: true + + - name: Build dependencies + run: | + make deps:modules + make deps:test + + # We need SSH Authorization because change detection needs use `ls-remote` like: + # git ls-remote git@github.com:sourcenetwork/defradb.git refs/heads/develop + - uses: webfactory/ssh-agent@v0.5.4 + with: + ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} + + - name: Run detection for changes + run: make test:changes + + ## Uncomment to enable ability to SSH into the runner. + #- name: Setup upterm ssh session for debugging + # uses: lhotari/action-upterm@v1 + # with: + # limit-access-to-actor: true + # limit-access-to-users: shahzadlone diff --git a/.github/workflows/lint-then-benchmark.yml b/.github/workflows/lint-then-benchmark.yml index 1ae596cd95..94d86f9d2e 100644 --- a/.github/workflows/lint-then-benchmark.yml +++ b/.github/workflows/lint-then-benchmark.yml @@ -1,4 +1,4 @@ -name: Lint and then Benchmark +name: Lint Then Benchmark Workflow on: pull_request: @@ -23,7 +23,8 @@ env: jobs: # ========================================================= Step-1: Run the lint check. golangci: - name: Lint Check + name: Lint check job + strategy: matrix: os: [ubuntu-latest] @@ -75,6 +76,7 @@ jobs: # This job acts like a switch to simplify our ci control flow later on. decide-benchmark-type: name: Decide which benchmarks to run based on flags + strategy: matrix: os: [ubuntu-latest] @@ -112,7 +114,7 @@ jobs: # ================== Step-3: Start the runner and get it registered as a github runner. start-runner: - name: Start self-hosted EC2 runner + name: Start self-hosted EC2 runner job needs: - golangci # only run if the linter check passed. diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml new file mode 100644 index 0000000000..26f51838e7 --- /dev/null +++ b/.github/workflows/run-tests.yml @@ -0,0 +1,27 @@ +name: Run Tests Workflow + +on: [push] + +jobs: + run-tests: + name: Run tests job + + runs-on: ubuntu-latest + + steps: + - name: Checkout code into the directory + uses: actions/checkout@v3 + + - name: Setup Go environment explicitly + uses: actions/setup-go@v3 + with: + go-version: "1.17" + check-latest: true + + - name: Build dependencies + run: | + make deps:modules + make deps:test + + - name: Run the tests, showing name of each test + run: make test:names diff --git a/.github/workflows/start-binary.yml b/.github/workflows/start-binary.yml new file mode 100644 index 0000000000..d0f18668c5 --- /dev/null +++ b/.github/workflows/start-binary.yml @@ -0,0 +1,50 @@ +name: Start Binary Workflow + +on: + pull_request: + + push: + tags: + - v* + branches: + - master + - develop + +jobs: + start-binary: + name: Start binary job + + runs-on: ubuntu-latest + + steps: + + - name: Checkout code into the directory + uses: actions/checkout@v3 + + - name: Setup Go environment explicitly + uses: actions/setup-go@v3 + with: + go-version: "1.17" + check-latest: true + + - name: Build modules + run: make deps:modules + + - name: Build binary + run: make build + + - name: Attempt to start binary + run: | + ./build/defradb start & + sleep 5 + + - name: Check if binary is still running + run: | + FOUND=$(pgrep -c "defradb"); + echo "Process(es) we found = [${FOUND}]"; + if [[ ${FOUND} == 0 ]]; then + echo "DefraDB start command failed."; + exit 1; + else + echo "DefraDB running."; + fi diff --git a/Makefile b/Makefile index 11adf08358..ac27756f9c 100644 --- a/Makefile +++ b/Makefile @@ -47,11 +47,6 @@ start: @$(MAKE) build ./build/defradb start -.PHONY: dev\:start -dev\:start: - @$(MAKE) build - DEFRA_ENV=dev ./build/defradb start - .PHONY: client\:dump client\:dump: ./build/defradb client dump @@ -84,14 +79,19 @@ deps\:chglog: deps\:modules: go mod download -.PHONY: deps\:ci -deps\:ci: - curl -fLSs https://raw.githubusercontent.com/CircleCI-Public/circleci-cli/master/install.sh | DESTDIR=${HOME}/bin bash - .PHONY: deps deps: - @$(MAKE) deps:lint && $(MAKE) deps:coverage && $(MAKE) deps:bench && $(MAKE) deps:chglog && \ - $(MAKE) deps:modules && $(MAKE) deps:ci && $(MAKE) deps:test + @$(MAKE) deps:modules && \ + $(MAKE) deps:bench && \ + $(MAKE) deps:chglog && \ + $(MAKE) deps:coverage && \ + $(MAKE) deps:lint && \ + $(MAKE) deps:test + +.PHONY: dev\:start +dev\:start: + @$(MAKE) build + DEFRA_ENV=dev ./build/defradb start .PHONY: tidy tidy: @@ -114,9 +114,13 @@ test: test\:go: go test ./... -race -shuffle=on +.PHONY: test\:names +test\:names: + gotestsum --format testname -- ./... -race -shuffle=on + .PHONY: test\:verbose test\:verbose: - gotestsum --format testname --junitfile /tmp/defradb-dev/test.xml -- ./... -race -shuffle=on + gotestsum --format standard-verbose -- ./... -race -shuffle=on .PHONY: test\:watch test\:watch: @@ -160,7 +164,7 @@ test\:coverage-html: .PHONY: test\:changes test\:changes: - env DEFRA_DETECT_DATABASE_CHANGES=true gotestsum --junitfile /tmp/defradb-dev/changes.xml -- ./... -shuffle=on -p 1 + env DEFRA_DETECT_DATABASE_CHANGES=true gotestsum -- ./... -shuffle=on -p 1 .PHONY: validate\:codecov validate\:codecov: