From f033b951d0b41d9b60bf7dd0d7ca3c38dcc84745 Mon Sep 17 00:00:00 2001 From: Sajid Alam <90610031+SajidAlamQB@users.noreply.github.com> Date: Thu, 18 May 2023 14:18:00 +0100 Subject: [PATCH] ci: Migrate the release workflow from CircleCI to GitHub Actions (#203) * Create check-release.yml * change from test pypi to pypi * split into jobs and move version logic into script * update github actions output * lint * changes based on review * changes based on review * fix script to not append continuously * change pypi api token logic Signed-off-by: Tom Kurian --- .github/workflows/check-release.yml | 93 +++++++++++++++++++ .../github_actions/github_actions_release.py | 54 +++++++++++ 2 files changed, 147 insertions(+) create mode 100644 .github/workflows/check-release.yml create mode 100755 tools/github_actions/github_actions_release.py diff --git a/.github/workflows/check-release.yml b/.github/workflows/check-release.yml new file mode 100644 index 000000000..386810bbd --- /dev/null +++ b/.github/workflows/check-release.yml @@ -0,0 +1,93 @@ +name: Check versions and build-publish + +on: + push: + branches: + - main + +jobs: + check-version: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: '3.10' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install requests + - name: Check version + run: python tools/github_actions/github_actions_release.py + - name: Set outputs + id: version_check + run: | + echo "new_release=${{ env.NEW_RELEASE }}" >> $GITHUB_OUTPUT + echo "package_name=${{ env.PACKAGE_NAME }}" >> $GITHUB_OUTPUT + echo "package_version=${{ env.PACKAGE_VERSION }}" >> $GITHUB_OUTPUT + outputs: + new_release: ${{ steps.version_check.outputs.new_release }} + package_name: ${{ steps.version_check.outputs.package_name }} + package_version: ${{ steps.version_check.outputs.package_version }} + + test: + needs: check-version + if: ${{ needs.check-version.outputs.new_release == 'true' }} + uses: ./.github/workflows/check-plugin.yml + with: + plugin: ${{ needs.check-version.outputs.package_name }} + + build-publish: + needs: [check-version, test] + if: ${{ needs.check-version.outputs.new_release == 'true' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: '3.10' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build + - name: Build package + run: | + export plugin=${{ needs.check-version.outputs.package_name }} + make package + - name: Create GitHub Release + uses: actions/github-script@v6 + with: + github-token: ${{ secrets.GH_TAGGING_TOKEN }} + script: | + const package_name = "${{ needs.check-version.outputs.package_name }}" + const package_version = "${{ needs.check-version.outputs.package_version }}" + const response = await github.rest.repos.createRelease({ + owner: context.repo.owner, + repo: context.repo.repo, + tag_name: `${package_name}-${package_version}`, + target_commitish: 'main', + name: `${package_name}-${package_version}`, + body: `Release ${package_version}`, + draft: false, + prerelease: false, + }); + return response.data; + - name: Set PyPI token + run: | + if [ "${{ needs.check-version.outputs.PACKAGE_NAME }}" == "kedro-airflow" ]; then + echo 'PYPI_TOKEN=${{ secrets.AIRFLOW_PYPI_TOKEN }}' >> $GITHUB_ENV + elif [ "${{ needs.check-version.outputs.PACKAGE_NAME }}" == "kedro-datasets" ]; then + echo 'PYPI_TOKEN=${{ secrets.DATASETS_PYPI_TOKEN }}' >> $GITHUB_ENV + elif [ "${{ needs.check-version.outputs.PACKAGE_NAME }}" == "kedro-docker" ]; then + echo 'PYPI_TOKEN=${{ secrets.DOCKER_PYPI_TOKEN }}' >> $GITHUB_ENV + elif [ "${{ needs.check-version.outputs.PACKAGE_NAME }}" == "kedro-telemetry" ]; then + echo 'PYPI_TOKEN=${{ secrets.TELEMETRY_PYPI_TOKEN }}' >> $GITHUB_ENV + fi + - name: Publish distribution 📦 to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: ${{ needs.check-version.outputs.package_name }}/dist + password: ${{ env.PYPI_TOKEN }} + diff --git a/tools/github_actions/github_actions_release.py b/tools/github_actions/github_actions_release.py new file mode 100755 index 000000000..cec1a8b97 --- /dev/null +++ b/tools/github_actions/github_actions_release.py @@ -0,0 +1,54 @@ +import os +import sys +import re +import requests +from pathlib import Path + +VERSION_MATCHSTR = r'\s*__version__\s*=\s*"(\d+\.\d+\.\d+)"' +PACKAGE_PATHS = ( + "kedro-datasets/kedro_datasets", + "kedro-telemetry/kedro_telemetry", + "kedro-airflow/kedro_airflow", + "kedro-docker/kedro_docker", +) + + +def get_package_version(base_path, package_path): + init_file_path = Path(base_path) / package_path / "__init__.py" + match_obj = re.search(VERSION_MATCHSTR, Path(init_file_path).read_text()) + return match_obj.group(1) + + +def check_no_version_pypi(pypi_endpoint, package_name, package_version): + print(f"Check if {package_name} {package_version} is on pypi") + response = requests.get(pypi_endpoint, timeout=10) + if response.status_code == 404: + # Version doesn't exist on Pypi - do release + print(f"Starting the release of {package_name} {package_version}") + return True + else: + print(f"Skipped: {package_name} {package_version} already exists on PyPI") + return False + + +if __name__ == "__main__": + """Check if a package needs to be released""" + base_path = Path() + new_release = "false" + package_name = None + package_version = None + + for package_path in PACKAGE_PATHS: + package_name, _ = package_path.split("/") + package_version = get_package_version(base_path, package_path) + pypi_endpoint = f"https://pypi.org/pypi/{package_name}/{package_version}/json/" + + if check_no_version_pypi(pypi_endpoint, package_name, package_version): + new_release = "true" + break + + env_file = os.getenv('GITHUB_ENV') + with open(env_file, "a") as env_file: + env_file.write(f"NEW_RELEASE={new_release}\n") + if new_release == "true": + env_file.write(f"PACKAGE_NAME={package_name}\nPACKAGE_VERSION={package_version}\n")