Setup the new auto-release CircleCI Release (#43)

* put back the circle ci config Signed-off-by: Nok Chan <[email protected]> * initial setup for datasets cI Signed-off-by: Nok Chan <[email protected]> * Add window config Signed-off-by: Nok Chan <[email protected]> * dummy changes to trigger CI Signed-off-by: Nok Chan <[email protected]> * Bug fix the path-filtering, wrong repository Signed-off-by: Nok Chan <[email protected]> * More attempts to fix CI - wish I can test locally Signed-off-by: Nok Chan <[email protected]> * Need to cd to the plugins directory before install Signed-off-by: Nok Chan <[email protected]> * Add window setup for tensorflow test Signed-off-by: Nok Chan <[email protected]> * fixing Spark related test Signed-off-by: Nok Chan <[email protected]> * Fix config Signed-off-by: Nok Chan <[email protected]> * Trigger CI Signed-off-by: Nok Chan <[email protected]> * reverse the boolean condition... Signed-off-by: Nok Chan <[email protected]> * fix conda env Signed-off-by: Nok Chan <[email protected]> * Fix Window Test and some linting Signed-off-by: Nok Chan <[email protected]> * fix coverage issue Signed-off-by: Nok Chan <[email protected]> * add snippets to check env variable Signed-off-by: Nok Chan <[email protected]> * Comment out invalid release config for testing Signed-off-by: Nok Chan <[email protected]> * testing Signed-off-by: Nok Chan <[email protected]> * use older python syntax Signed-off-by: Nok Chan <[email protected]> * remove auto-release change & split PR Signed-off-by: Nok Chan <[email protected]> * remove empty lines Signed-off-by: Nok Chan <[email protected]> * add CI scripts Signed-off-by: Nok Chan <[email protected]> * Revert "add CI scripts" This reverts commit eb6ade0. * add test to cover loading csv in remote storage Signed-off-by: Nok Chan <[email protected]> * Revert "add test to cover loading csv in remote storage" This reverts commit 547b938. * add build scripts Signed-off-by: Nok Chan <[email protected]> * update file for permission issue Signed-off-by: Nok Chan <[email protected]> * setup env Signed-off-by: Nok Chan <[email protected]> * update config Signed-off-by: Nok Chan <[email protected]> * Fix broken path Signed-off-by: Nok Chan <[email protected]> * update requirements Signed-off-by: Nok Chan <[email protected]> * bug fix Signed-off-by: Nok Chan <[email protected]> * Make the logging niceer Signed-off-by: Nok Chan <[email protected]> * update parameter in circleci config Signed-off-by: Nok Chan <[email protected]> * add condition to trigger main_updated so it won't be triggered in a dead loop Signed-off-by: Nok Chan <[email protected]> * add more logs Signed-off-by: Nok Chan <[email protected]> * update condition Signed-off-by: Nok Chan <[email protected]> * Try trigger release Signed-off-by: Nok Chan <[email protected]> * temporarily turn off the condition since unittest are not passing Signed-off-by: Nok Chan <[email protected]> * fix config Signed-off-by: Nok Chan <[email protected]> * fix plugin release Signed-off-by: Nok Chan <[email protected]> * Fix the argument for tag name Signed-off-by: Nok Chan <[email protected]> * tidy out the config a little bit Signed-off-by: Nok Chan <[email protected]> * more bug fix Signed-off-by: Nok Chan <[email protected]> * fix pypi folder Signed-off-by: Nok Chan <[email protected]> * remove the TWINE_REPOSITORY_URL argument Signed-off-by: Nok Chan <[email protected]> * update the CI config Signed-off-by: Nok Chan <[email protected]> * trigger a proper release Signed-off-by: Nok Chan <[email protected]> * fix config Signed-off-by: Nok Chan <[email protected]> * add dummy e2etest Signed-off-by: Nok Chan <[email protected]> * add comments Signed-off-by: Nok Chan <[email protected]> * fix lint Signed-off-by: Nok Chan <[email protected]> * format nicer Signed-off-by: Nok Chan <[email protected]> * Clean up for review Signed-off-by: Nok Chan <[email protected]> * apply review comments
kedro-org · Jul 12, 2022 · 5813362 · 5813362
1 parent 190b990
commit 5813362
Show file tree

Hide file tree

Showing 10 changed files with 317 additions and 9 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,5 +1,13 @@
 version: 2.1
 
+parameters:
+  release_package:
+    type: string
+    default: ""
+  release_version:
+    type: string
+    default: ""
+
 setup: true
 
 # the path-filtering orb is required to continue a pipeline based on

diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml
@@ -18,6 +18,12 @@ parameters:
   run-build-kedro-datasets:
     type: boolean
     default: false
+  release_package:
+    type: string
+    default: ""
+  release_version:
+    type: string
+    default: ""
 
 commands:
   setup_conda:
@@ -297,12 +303,101 @@ jobs:
                 name: Run unit tests without spark sequentially
                 command: conda activate kedro_plugins; make test-no-spark-sequential
 
+  sync:
+    parameters:
+      python_version:
+        type: string
+    docker:
+      # https://circleci.com/docs/2.0/circleci-images/#circleci-base-image
+      - image: cimg/base:2020.01
+    steps:
+      - checkout
+      - add_ssh_keys
+      - run:
+          name: Set git email and name
+          command: |
+            git config --global user.email "[email protected]"
+            git config --global user.name "Kedro"
+      # - run:
+      #     name: Trigger Read The Docs build
+      #     command: ./tools/circleci/rtd-build.sh ${RTD_TOKEN} latest
+      - setup_conda:
+          python_version: <<parameters.python_version>>
+      - run:
+          name: Maybe trigger the release workflow
+          command: |
+            conda activate kedro_plugins;
+            pip install requests
+            ./tools/circleci/circleci_release.py
+
+
+  # This is effectively just a combination of the lint, unit_tests and e2e_tests jobs.
+  # It's used to check that the nightly docker image is working ok and before publishing a release.
+  build_package:
+    parameters:
+      python_version:
+        type: string
+    machine:
+      image: ubuntu-2004:202201-02
+      docker_layer_caching: true
+    steps:
+      - setup:
+          python_version: <<parameters.python_version>>  # Just need one Python version here
+          plugin: <<pipeline.parameters.release_package>>
+      - run:
+          name: Run linters
+          command: export plugin=<<pipeline.parameters.release_package>>; make lint
+      - unless:
+          condition:
+            equal: ["3.10", <<parameters.python_version>>]
+          steps:
+            - run:
+                name: Run unit tests in parallel
+                command: export plugin=<<pipeline.parameters.release_package>>; make test
+      - when:
+          condition:
+            equal: [ "3.10", <<parameters.python_version>> ]
+          steps:
+            - run:
+                name: Run unit tests sequentially
+                command: export plugin=<<pipeline.parameters.release_package>>; make test-sequential
+      - run:
+          name: Run e2e tests
+          command: export plugin=<<pipeline.parameters.release_package>>; make e2e-tests
+
+  publish_package:
+    machine:
+      image: ubuntu-2004:202201-02
+      docker_layer_caching: true
+    steps:
+      - run:
+          name: Print the release package and version
+          command: |
+              echo "Release package: <<pipeline.parameters.release_package>> <<pipeline.parameters.release_version>>"
+      - setup:
+          python_version: "3.8"  # Just need one Python version here
+          plugin: <<pipeline.parameters.release_package>>  # From circle_release.py
+      - add_ssh_keys
+      - run:
+          name: Tag and publish release on Github
+          command: ./tools/circleci/github_release.py <<pipeline.parameters.release_package>> <<pipeline.parameters.release_version>>
+      - run:
+          name: Publish to PyPI
+          command: |
+            export plugin=<<pipeline.parameters.release_package>>
+            make package
+            make pypi
+
 
 workflows:
   # when pipeline parameter, run-build-kedro-telemetry is true, the
   # kedro-telemetry job is triggered.
   kedro-telemetry:
-    when: <<pipeline.parameters.run-build-kedro-telemetry>>
+    when:
+      and:
+        - <<pipeline.parameters.run-build-kedro-telemetry>>
+        - not: <<pipeline.parameters.release_package>>
+        - not: <<pipeline.parameters.release_version>>
     jobs:
       - unit_tests:
           plugin: "kedro-telemetry"
@@ -319,7 +414,11 @@ workflows:
   # when pipeline parameter, run-build-kedro-docker is true, the
   # kedro-docker job is triggered.
   kedro-docker:
-    when: <<pipeline.parameters.run-build-kedro-docker>>
+    when:
+      and:
+        -  <<pipeline.parameters.run-build-kedro-docker>>
+        - not: <<pipeline.parameters.release_package>>
+        - not: <<pipeline.parameters.release_version>>
     jobs:
       - unit_tests:
           plugin: "kedro-docker"
@@ -341,7 +440,11 @@ workflows:
   # when pipeline parameter, run-build-kedro-airflow is true, the
   # kedro-airflow job is triggered.
   kedro-airflow:
-    when: <<pipeline.parameters.run-build-kedro-airflow>>
+    when:
+      and:
+        - <<pipeline.parameters.run-build-kedro-airflow>>
+        - not: <<pipeline.parameters.release_package>>
+        - not: <<pipeline.parameters.release_version>>
     jobs:
       - unit_tests:
           plugin: "kedro-airflow"
@@ -363,7 +466,11 @@ workflows:
   # when pipeline parameter, run-build-kedro-datasets is true, the
   # kedro-datasets job is triggered.
   kedro-datasets:
-    when: <<pipeline.parameters.run-build-kedro-datasets>>
+    when:
+      and:
+        - <<pipeline.parameters.run-build-kedro-datasets>>
+        - not: <<pipeline.parameters.release_package>>
+        - not: <<pipeline.parameters.release_version>>
     jobs:
       - unit_tests:
           plugin: "kedro-datasets"
@@ -377,3 +484,33 @@ workflows:
               python_version: ["3.7", "3.8", "3.9", "3.10"]
       - lint:
           plugin: "kedro-datasets"
+
+  # For release
+  main_updated:
+    when:
+      and:
+        - not: <<pipeline.parameters.release_package>>
+        - not: <<pipeline.parameters.release_version>>
+    jobs:
+      - sync:
+          filters:
+            branches:
+              only: main
+          matrix:
+            # We just need one Python enviornment to trigger the job
+            parameters:
+              python_version: ["3.8"]
+
+  package_release:
+    when:
+      and:
+        -  <<pipeline.parameters.release_package>>
+        -  <<pipeline.parameters.release_version>>
+    jobs:
+      - build_package:
+          matrix:
+            parameters:
+              python_version: ["3.7", "3.8", "3.9", "3.10"]
+      - publish_package:
+          requires:
+            - build_package
diff --git a/Makefile b/Makefile
@@ -4,6 +4,7 @@ package:
 	python setup.py sdist bdist_wheel
 
 pypi:
+	python -m pip install twine -U
 	python -m twine upload $(plugin)/dist/*
 
 install: package
@@ -18,6 +19,9 @@ lint:
 test:
 	cd $(plugin) && pytest tests --cov-config pyproject.toml --numprocesses 4 --dist loadfile
 
+test-sequential:
+	cd $(plugin) && pytest tests --cov-config pyproject.toml
+
 e2e-tests:
 	cd $(plugin) && behave
 

diff --git a/kedro-datasets/features/steps/dummy.feature b/kedro-datasets/features/steps/dummy.feature
@@ -0,0 +1 @@
+# A dummy file to keep CI behave correctly
diff --git a/kedro-datasets/kedro_datasets/__init__.py b/kedro-datasets/kedro_datasets/__init__.py
@@ -1,6 +1,3 @@
 """``kedro_datasets`` is where you can find all of Kedro's data connectors."""
 
-__version__ = "0.0.2"
-
-
-# todo: remove this before merge Trigger CI with file changes
+__version__ = "0.0.4"
diff --git a/kedro-datasets/pyproject.toml b/kedro-datasets/pyproject.toml
@@ -33,7 +33,7 @@ min-public-methods = 1
 [tool.coverage.report]
 fail_under = 100
 show_missing = true
-omit = ["tests/*"]
+omit = ["tests/*", "kedro_datasets/datasets/holoviews/*"]
 exclude_lines = ["pragma: no cover", "raise NotImplementedError"]
 
 [tool.pytest.ini_options]

diff --git a/tools/circleci/circleci_release.py b/tools/circleci/circleci_release.py
@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+"""
+CircleCI pipeline to check if it needs to trigger a release
+"""
+
+import os
+
+import requests
+from requests.structures import CaseInsensitiveDict
+
+from utils.check_no_version_pypi import check_no_version_pypi
+from utils.package_version import get_package_version
+
+PACKAGE_PATHS = (
+    "kedro-datasets/kedro_datasets",
+    "kedro-telemetry/kedro_telemetry",
+    "kedro-airflow/kedro_airflow",
+    "kedro-docker/kedro_docker",
+)
+PROJECT_SLUG = "github/kedro-org/kedro-plugins"
+# CIRCLE_BRANCH = "feat/cicd-auto-release"
+CIRCLE_BRANCH = os.environ.get("CIRCLE_BRANCH")
+
+
+def circleci_release(project_slug, payload, circle_endpoint, circle_release_token):
+    """Trigging the CircleCI Release Pipeline"""
+    # See https://circleci.com/docs/2.0/api-developers-guide
+    print("Starting the CircleCI Release Pipeline")
+    CIRCLE_ENDPOINT = f"https://circleci.com/api/v2/project/{project_slug}/pipeline"
+
+    headers = CaseInsensitiveDict()
+    headers["Content-Type"] = "application/json"
+    headers["Circle-Token"] = circle_release_token
+
+    resp = requests.post(circle_endpoint, headers=headers, json=payload)
+    print(f"Status Code: {resp.status_code}")
+    if resp.status_code == 201:
+        print("Creating CircleCI Pipeline successfully")
+        print(resp.content)
+    else:
+        print("Failed to create CircleCI Pipeline")
+    return resp
+
+
+if __name__ == "__main__":
+    """Trigger the CircleCI Release Process"""
+    from pathlib import Path
+
+    # Personal API Tokens - https://circleci.com/docs/managing-api-tokens
+    CIRCLE_RELEASE_TOKEN = os.environ.get("CIRCLE_RELEASE_TOKEN")
+    if not CIRCLE_RELEASE_TOKEN:
+        raise ValueError("CIRCLE_RELEASE_TOKEN is not defined as envionrmnet variable.")
+
+    base_path = Path()
+    # Loop for all 4 repositories
+    for package_path in PACKAGE_PATHS:
+        package_name, _ = package_path.split("/")
+        package_version = get_package_version(base_path, package_path)
+        pypi_endpoint = f"https://pypi.org/pypi/{package_name}/{package_version}/json/"
+        circleci_endpoint = (
+            f"https://circleci.com/api/v2/project/{PROJECT_SLUG}/pipeline"
+        )
+        payload = {
+            "branch": CIRCLE_BRANCH,
+            "parameters": {
+                "release_package": package_name,
+                "release_version": package_version,
+            },
+        }
+
+        print(package_name, package_version)
+        if check_no_version_pypi(pypi_endpoint, package_name, package_version):
+            circleci_release(
+                PROJECT_SLUG, payload, circleci_endpoint, CIRCLE_RELEASE_TOKEN
+            )
diff --git a/tools/circleci/github_release.py b/tools/circleci/github_release.py
@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+import os
+import sys
+
+import requests
+from requests.structures import CaseInsensitiveDict
+
+GITHUB_USER = "kedro-org"
+GITHUB_REPO = "kedro-plugins"
+# On GitHub select "Settings" > "Developer Setting" -> "Personal access Token""
+GITHUB_TAGGING_TOKEN = os.environ.get("GITHUB_TAGGING_TOKEN")
+
+
+def github_release(
+    package_name,
+    version,
+    github_user=GITHUB_USER,
+    github_repo=GITHUB_REPO,
+    github_tagging_token=GITHUB_TAGGING_TOKEN,
+):
+    """Trigger the GitHub Release to create artifacts and tags"""
+    print("Starting GitHub Release")
+
+    github_endpoint = (
+        f"https://api.github.com/repos/{github_user}/{github_repo}/releases"
+    )
+    payload = {
+        "tag_name": f"{package_name}-{version}",  # kedro-datasets 0.0.1
+        "target_commitish": "main",
+        "name": f"{version}",
+        "body": f"Release {version}",
+        "draft": False,
+        "prerelease": False,
+    }
+
+    headers = CaseInsensitiveDict()
+    headers["Content-Type"] = "application/json"
+    headers["Authorization"] = f"token {github_tagging_token}"
+    resp = requests.post(github_endpoint, headers=headers, json=payload)
+    if resp.status_code == 200:
+        print("Create GitHub release successfully")
+        print(resp.content)
+    else:
+        print("Failed to create Github release")
+        print(resp.content)
+    return resp
+
+
+if __name__ == "__main__":
+    package_name = sys.argv[1]
+    package_version = sys.argv[2]
+    res = github_release(package_name, package_version)
diff --git a/tools/circleci/utils/check_no_version_pypi.py b/tools/circleci/utils/check_no_version_pypi.py
@@ -0,0 +1,13 @@
+import requests
+
+
+def check_no_version_pypi(pypi_endpoint, package_name, package_version):
+    print("Check if {package_name} {package_version} is on pypi")
+    response = requests.get(pypi_endpoint)
+    if response.status_code == 404:
+        # Not exist on Pypi - do release
+        print(f"Starting the release of {package_name} {package_version}")
+        return True
+    else:
+        print(f"Skipped: {package_name} {package_version} already exists on PyPI")
+        return False