-
Notifications
You must be signed in to change notification settings - Fork 84
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Setup the new auto-release CircleCI Release #43
Changes from all commits
ecffa72
79df35b
009c3d6
6284409
a55ff3d
5e11bd6
11a6a25
3616dda
6c79e13
77f4566
63f0002
d3fa5b5
fffdec9
935af8c
ee6581f
e4a0e54
b857f20
8afecfb
9c20426
42258b8
bc3c510
e53ab6f
eb6ade0
315d236
547b938
4ee29fe
2a0ee5b
7ac9451
cea2141
8914067
23b74b8
1ff22b5
15a875d
52c69d7
55d75f8
aeb5249
9e9fa4b
a555187
efedb0e
afa701a
cf2f5a1
54a4f05
eff737f
711a7e2
6df8cc8
6795635
2aaa207
0eb5611
a0b5efe
fa9395e
ebbe3ac
55a57e6
40ea125
b802213
022bbe1
d036af2
ce5cbf2
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -18,6 +18,12 @@ parameters: | |
run-build-kedro-datasets: | ||
type: boolean | ||
default: false | ||
release_package: | ||
type: string | ||
default: "" | ||
release_version: | ||
type: string | ||
default: "" | ||
|
||
commands: | ||
setup_conda: | ||
|
@@ -297,12 +303,101 @@ jobs: | |
name: Run unit tests without spark sequentially | ||
command: conda activate kedro_plugins; make test-no-spark-sequential | ||
|
||
sync: | ||
parameters: | ||
python_version: | ||
type: string | ||
docker: | ||
# https://circleci.com/docs/2.0/circleci-images/#circleci-base-image | ||
- image: cimg/base:2020.01 | ||
steps: | ||
- checkout | ||
- add_ssh_keys | ||
- run: | ||
name: Set git email and name | ||
command: | | ||
git config --global user.email "[email protected]" | ||
git config --global user.name "Kedro" | ||
# - run: | ||
# name: Trigger Read The Docs build | ||
# command: ./tools/circleci/rtd-build.sh ${RTD_TOKEN} latest | ||
- setup_conda: | ||
python_version: <<parameters.python_version>> | ||
- run: | ||
name: Maybe trigger the release workflow | ||
command: | | ||
conda activate kedro_plugins; | ||
pip install requests | ||
./tools/circleci/circleci_release.py | ||
|
||
|
||
# This is effectively just a combination of the lint, unit_tests and e2e_tests jobs. | ||
# It's used to check that the nightly docker image is working ok and before publishing a release. | ||
build_package: | ||
parameters: | ||
python_version: | ||
type: string | ||
machine: | ||
image: ubuntu-2004:202201-02 | ||
docker_layer_caching: true | ||
steps: | ||
- setup: | ||
python_version: <<parameters.python_version>> # Just need one Python version here | ||
plugin: <<pipeline.parameters.release_package>> | ||
- run: | ||
name: Run linters | ||
command: export plugin=<<pipeline.parameters.release_package>>; make lint | ||
- unless: | ||
condition: | ||
equal: ["3.10", <<parameters.python_version>>] | ||
steps: | ||
- run: | ||
name: Run unit tests in parallel | ||
command: export plugin=<<pipeline.parameters.release_package>>; make test | ||
- when: | ||
condition: | ||
equal: [ "3.10", <<parameters.python_version>> ] | ||
steps: | ||
- run: | ||
name: Run unit tests sequentially | ||
command: export plugin=<<pipeline.parameters.release_package>>; make test-sequential | ||
- run: | ||
name: Run e2e tests | ||
command: export plugin=<<pipeline.parameters.release_package>>; make e2e-tests | ||
|
||
publish_package: | ||
machine: | ||
image: ubuntu-2004:202201-02 | ||
docker_layer_caching: true | ||
steps: | ||
- run: | ||
name: Print the release package and version | ||
command: | | ||
echo "Release package: <<pipeline.parameters.release_package>> <<pipeline.parameters.release_version>>" | ||
- setup: | ||
python_version: "3.8" # Just need one Python version here | ||
plugin: <<pipeline.parameters.release_package>> # From circle_release.py | ||
- add_ssh_keys | ||
- run: | ||
name: Tag and publish release on Github | ||
command: ./tools/circleci/github_release.py <<pipeline.parameters.release_package>> <<pipeline.parameters.release_version>> | ||
- run: | ||
name: Publish to PyPI | ||
command: | | ||
export plugin=<<pipeline.parameters.release_package>> | ||
make package | ||
make pypi | ||
|
||
|
||
workflows: | ||
# when pipeline parameter, run-build-kedro-telemetry is true, the | ||
# kedro-telemetry job is triggered. | ||
kedro-telemetry: | ||
when: <<pipeline.parameters.run-build-kedro-telemetry>> | ||
when: | ||
and: | ||
- <<pipeline.parameters.run-build-kedro-telemetry>> | ||
- not: <<pipeline.parameters.release_package>> | ||
- not: <<pipeline.parameters.release_version>> | ||
jobs: | ||
- unit_tests: | ||
plugin: "kedro-telemetry" | ||
|
@@ -319,7 +414,11 @@ workflows: | |
# when pipeline parameter, run-build-kedro-docker is true, the | ||
# kedro-docker job is triggered. | ||
kedro-docker: | ||
when: <<pipeline.parameters.run-build-kedro-docker>> | ||
when: | ||
and: | ||
- <<pipeline.parameters.run-build-kedro-docker>> | ||
- not: <<pipeline.parameters.release_package>> | ||
- not: <<pipeline.parameters.release_version>> | ||
jobs: | ||
- unit_tests: | ||
plugin: "kedro-docker" | ||
|
@@ -341,7 +440,11 @@ workflows: | |
# when pipeline parameter, run-build-kedro-airflow is true, the | ||
# kedro-airflow job is triggered. | ||
kedro-airflow: | ||
when: <<pipeline.parameters.run-build-kedro-airflow>> | ||
when: | ||
and: | ||
- <<pipeline.parameters.run-build-kedro-airflow>> | ||
- not: <<pipeline.parameters.release_package>> | ||
- not: <<pipeline.parameters.release_version>> | ||
jobs: | ||
- unit_tests: | ||
plugin: "kedro-airflow" | ||
|
@@ -363,7 +466,11 @@ workflows: | |
# when pipeline parameter, run-build-kedro-datasets is true, the | ||
# kedro-datasets job is triggered. | ||
kedro-datasets: | ||
when: <<pipeline.parameters.run-build-kedro-datasets>> | ||
when: | ||
and: | ||
- <<pipeline.parameters.run-build-kedro-datasets>> | ||
- not: <<pipeline.parameters.release_package>> | ||
- not: <<pipeline.parameters.release_version>> | ||
jobs: | ||
- unit_tests: | ||
plugin: "kedro-datasets" | ||
|
@@ -377,3 +484,33 @@ workflows: | |
python_version: ["3.7", "3.8", "3.9", "3.10"] | ||
- lint: | ||
plugin: "kedro-datasets" | ||
|
||
# For release | ||
main_updated: | ||
when: | ||
and: | ||
- not: <<pipeline.parameters.release_package>> | ||
- not: <<pipeline.parameters.release_version>> | ||
jobs: | ||
- sync: | ||
filters: | ||
branches: | ||
only: main | ||
matrix: | ||
# We just need one Python enviornment to trigger the job | ||
parameters: | ||
python_version: ["3.8"] | ||
|
||
package_release: | ||
when: | ||
and: | ||
- <<pipeline.parameters.release_package>> | ||
- <<pipeline.parameters.release_version>> | ||
jobs: | ||
- build_package: | ||
matrix: | ||
parameters: | ||
python_version: ["3.7", "3.8", "3.9", "3.10"] | ||
- publish_package: | ||
requires: | ||
- build_package |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
# A dummy file to keep CI behave correctly |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,6 +1,3 @@ | ||
"""``kedro_datasets`` is where you can find all of Kedro's data connectors.""" | ||
|
||
__version__ = "0.0.2" | ||
|
||
|
||
# todo: remove this before merge Trigger CI with file changes | ||
__version__ = "0.0.4" |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -33,7 +33,7 @@ min-public-methods = 1 | |
[tool.coverage.report] | ||
fail_under = 100 | ||
show_missing = true | ||
omit = ["tests/*"] | ||
omit = ["tests/*", "kedro_datasets/datasets/holoviews/*"] | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Carries from |
||
exclude_lines = ["pragma: no cover", "raise NotImplementedError"] | ||
|
||
[tool.pytest.ini_options] | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,75 @@ | ||
#!/usr/bin/env python3 | ||
""" | ||
CircleCI pipeline to check if it needs to trigger a release | ||
""" | ||
|
||
import os | ||
|
||
import requests | ||
from requests.structures import CaseInsensitiveDict | ||
|
||
from utils.check_no_version_pypi import check_no_version_pypi | ||
from utils.package_version import get_package_version | ||
|
||
PACKAGE_PATHS = ( | ||
"kedro-datasets/kedro_datasets", | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. In theory, we can just use |
||
"kedro-telemetry/kedro_telemetry", | ||
"kedro-airflow/kedro_airflow", | ||
"kedro-docker/kedro_docker", | ||
) | ||
PROJECT_SLUG = "github/kedro-org/kedro-plugins" | ||
# CIRCLE_BRANCH = "feat/cicd-auto-release" | ||
CIRCLE_BRANCH = os.environ.get("CIRCLE_BRANCH") | ||
|
||
|
||
def circleci_release(project_slug, payload, circle_endpoint, circle_release_token): | ||
"""Trigging the CircleCI Release Pipeline""" | ||
# See https://circleci.com/docs/2.0/api-developers-guide | ||
print("Starting the CircleCI Release Pipeline") | ||
CIRCLE_ENDPOINT = f"https://circleci.com/api/v2/project/{project_slug}/pipeline" | ||
|
||
headers = CaseInsensitiveDict() | ||
headers["Content-Type"] = "application/json" | ||
headers["Circle-Token"] = circle_release_token | ||
|
||
resp = requests.post(circle_endpoint, headers=headers, json=payload) | ||
print(f"Status Code: {resp.status_code}") | ||
if resp.status_code == 201: | ||
print("Creating CircleCI Pipeline successfully") | ||
print(resp.content) | ||
else: | ||
print("Failed to create CircleCI Pipeline") | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Do we need any more info here? Error code or something like that to help us figure out what went wrong? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I agree with Merel, maybe put the response code in the print statement as well. |
||
return resp | ||
|
||
|
||
if __name__ == "__main__": | ||
"""Trigger the CircleCI Release Process""" | ||
from pathlib import Path | ||
|
||
# Personal API Tokens - https://circleci.com/docs/managing-api-tokens | ||
CIRCLE_RELEASE_TOKEN = os.environ.get("CIRCLE_RELEASE_TOKEN") | ||
if not CIRCLE_RELEASE_TOKEN: | ||
raise ValueError("CIRCLE_RELEASE_TOKEN is not defined as envionrmnet variable.") | ||
|
||
base_path = Path() | ||
# Loop for all 4 repositories | ||
for package_path in PACKAGE_PATHS: | ||
package_name, _ = package_path.split("/") | ||
package_version = get_package_version(base_path, package_path) | ||
pypi_endpoint = f"https://pypi.org/pypi/{package_name}/{package_version}/json/" | ||
circleci_endpoint = ( | ||
f"https://circleci.com/api/v2/project/{PROJECT_SLUG}/pipeline" | ||
) | ||
payload = { | ||
"branch": CIRCLE_BRANCH, | ||
"parameters": { | ||
"release_package": package_name, | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We used to check the version twice in two different jobs, I think it's better to keep a single source of truth and just pass this information in the requests. |
||
"release_version": package_version, | ||
}, | ||
} | ||
|
||
print(package_name, package_version) | ||
if check_no_version_pypi(pypi_endpoint, package_name, package_version): | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Everytime |
||
circleci_release( | ||
PROJECT_SLUG, payload, circleci_endpoint, CIRCLE_RELEASE_TOKEN | ||
) |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,52 @@ | ||
#!/usr/bin/env python3 | ||
import os | ||
import sys | ||
|
||
import requests | ||
from requests.structures import CaseInsensitiveDict | ||
|
||
GITHUB_USER = "kedro-org" | ||
GITHUB_REPO = "kedro-plugins" | ||
# On GitHub select "Settings" > "Developer Setting" -> "Personal access Token"" | ||
GITHUB_TAGGING_TOKEN = os.environ.get("GITHUB_TAGGING_TOKEN") | ||
|
||
|
||
def github_release( | ||
package_name, | ||
version, | ||
github_user=GITHUB_USER, | ||
github_repo=GITHUB_REPO, | ||
github_tagging_token=GITHUB_TAGGING_TOKEN, | ||
): | ||
"""Trigger the GitHub Release to create artifacts and tags""" | ||
print("Starting GitHub Release") | ||
|
||
github_endpoint = ( | ||
f"https://api.github.com/repos/{github_user}/{github_repo}/releases" | ||
) | ||
payload = { | ||
"tag_name": f"{package_name}-{version}", # kedro-datasets 0.0.1 | ||
"target_commitish": "main", | ||
"name": f"{version}", | ||
"body": f"Release {version}", | ||
"draft": False, | ||
"prerelease": False, | ||
} | ||
|
||
headers = CaseInsensitiveDict() | ||
headers["Content-Type"] = "application/json" | ||
headers["Authorization"] = f"token {github_tagging_token}" | ||
resp = requests.post(github_endpoint, headers=headers, json=payload) | ||
if resp.status_code == 200: | ||
print("Create GitHub release successfully") | ||
print(resp.content) | ||
else: | ||
print("Failed to create Github release") | ||
print(resp.content) | ||
return resp | ||
|
||
|
||
if __name__ == "__main__": | ||
package_name = sys.argv[1] | ||
package_version = sys.argv[2] | ||
res = github_release(package_name, package_version) |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,13 @@ | ||
import requests | ||
|
||
|
||
def check_no_version_pypi(pypi_endpoint, package_name, package_version): | ||
print("Check if {package_name} {package_version} is on pypi") | ||
response = requests.get(pypi_endpoint) | ||
if response.status_code == 404: | ||
# Not exist on Pypi - do release | ||
print(f"Starting the release of {package_name} {package_version}") | ||
return True | ||
else: | ||
print(f"Skipped: {package_name} {package_version} already exists on PyPI") | ||
return False |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Why do you need these parameters here?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
It's a CircleCI thing, otherwise, it will just throw an error. We called these 2 parameters in the POST request.