feat(CI): add venv caching (#6843)

Co-authored-by: Harshal Sheth <[email protected]>
datahub-project · Dec 28, 2022 · 9c794af · 9c794af
1 parent 988b3d4
commit 9c794af
Show file tree

Hide file tree

Showing 2 changed files with 33 additions and 0 deletions.
diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml
@@ -466,6 +466,7 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: "3.7"
+          cache: 'pip'
       - name: Pull images in background
         run: |
           docker pull acryldata/datahub-actions:head &
@@ -514,6 +515,20 @@ jobs:
         if: ${{ needs.setup.outputs.publish != 'true' }}
         with:
           image: ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
+      - name: Calculate pip install plan
+        id: "packages_checksum"
+        run: |
+          pip install pip -U # only 22.2 and above contains the --dry-run flag
+
+          # only the last line of the output is the packages that will be installed
+          pip install --dry-run -r ./smoke/requirements.txt  | tail -n 1 > /tmp/would_be_installed.txt
+          CHECKSUM=$(shasum /tmp/would_be_installed.txt | awk '{print $1}')
+          echo "packages_checksum=$CHECKSUM" >> $GITHUB_OUTPUT
+      - uses: actions/cache@v3
+        id: cache-venv
+        with:
+          path: ./smoke-test/venv/
+          key: smoke-venv-${{ runner.os }}-${{ steps.packages_checksum.outputs.packages_checksum }}
       - name: check background image download status
         run: |
           jobs

diff --git a/.github/workflows/metadata-ingestion.yml b/.github/workflows/metadata-ingestion.yml
@@ -4,12 +4,14 @@ on:
     branches:
       - master
     paths:
+      - ".github/**"
       - "metadata-ingestion/**"
       - "metadata-models/**"
   pull_request:
     branches:
       - master
     paths:
+      - ".github/**"
       - "metadata-ingestion/**"
       - "metadata-models/**"
   release:
@@ -49,12 +51,28 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
+          cache: 'pip'
       - uses: vemonet/setup-spark@v1 # spark is required for pyspark+pydeequ data lake profiling
         with:
           spark-version: "3.0.3"
           hadoop-version: "3.2"
       - name: Install dependencies
         run: ./metadata-ingestion/scripts/install_deps.sh
+      - name: Calculate pip install plan
+        id: "packages_checksum"
+        run: |
+          cd metadata-ingestion
+          pip install pip -U # only 22.2 and above contains the --dry-run flag
+
+          # only the last line of the output is the packages that will be installed
+          pip install --dry-run -e .[dev]  ${{ matrix.extraPythonRequirement }} | tail -n 1 > /tmp/would_be_installed.txt
+          CHECKSUM=$(shasum /tmp/would_be_installed.txt | awk '{print $1}')
+          echo "packages_checksum=$CHECKSUM" >> $GITHUB_OUTPUT
+      - uses: actions/cache@v3
+        id: cache-venv
+        with:
+          path: ./metadata-ingestion/venv/
+          key: ${{ runner.os }}-venv-${{ steps.packages_checksum.outputs.packages_checksum }}-${{ matrix.python-version }}-${{ matrix.command }}
       - name: Run metadata-ingestion tests (extras ${{ matrix.extraPythonRequirement }})
         run: ./gradlew -Pextra_pip_requirements='${{ matrix.extraPythonRequirement }}' :metadata-ingestion:${{ matrix.command }}
       - name: pip freeze show list installed