From f51747b136429001b3ad83c25d6a7f53639eb097 Mon Sep 17 00:00:00 2001 From: Ben Leggett Date: Wed, 7 Sep 2022 18:35:08 -0400 Subject: [PATCH 1/9] Add support for Helm OCI registries --- chartpress.py | 115 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 106 insertions(+), 9 deletions(-) diff --git a/chartpress.py b/chartpress.py index a516dcb..fb36053 100644 --- a/chartpress.py +++ b/chartpress.py @@ -843,6 +843,94 @@ def build_chart( # return version return version +def publish_chart_oci( + chart_name, + chart_base, + chart_version, + chart_oci_repo, + chart_oci_prefix, + force=False, +): + """ + Update a Helm chart stored in an OCI registry (e.g. ghcr.io). + + The strategy adopted to do this is: + + 1. Clone the Helm chart registry as found in the gh-pages branch of a git + reposistory. + 2. If --force-publish-chart isn't specified, then verify that we won't + overwrite an existing chart version. + 3. Create a temporary directory and `helm package` the chart into a file + within this temporary directory now only containing the chart .tar file. + 4. Generate a index.yaml with `helm repo index` based on charts found in the + temporary directory folder (a single one), and then merge in the bigger + and existing index.yaml from the cloned Helm chart registry using the + --merge flag. + 5. Copy the new index.yaml and packaged Helm chart .tar into the gh-pages + branch, commit it, and push it back to the origin remote. + + Note that if we would add the new chart .tar file next to the other .tar + files and use `helm repo index` we would recreate `index.yaml` and update + all the timestamps etc. which is something we don't want. Using `helm repo + index` on a directory with only the new chart .tar file allows us to avoid + this issue. + + Also note that the --merge flag will not override existing entries to the + fresh index.yaml file with the index.yaml from the --merge flag. Due to + this, it is as we would have a --force-publish-chart by default. + """ + + # clone/fetch the Helm chart repo and checkout its gh-pages branch, note the + # use of cwd (current working directory) + + chart_dir = f"{chart_base}/{chart_name}" + _check_call(["git", "fetch"], cwd=chart_dir, echo=True) + + # # check if a chart with the same name and version has already been published. If + # # there is, the behaviour depends on `--force-publish-chart` + # # and chart_version and make a decision based on the --force-publish-chart + # # flag if that is the case, but always log what's done + # if os.path.isfile(os.path.join(chart_dir, "index.yaml")): + # with open(os.path.join(checkout_dir, "index.yaml")) as f: + # chart_repo_index = yaml.load(f) + # published_charts = chart_repo_index["entries"].get(chart_name, []) + + # if published_charts and any( + # c["version"] == chart_version for c in published_charts + # ): + # if force: + # _log( + # f"Chart of version {chart_version} already exists, overwriting it." + # ) + # else: + # _log( + # f"Skipping chart publishing of version {chart_version}, it is already published" + # ) + # return + + # package the latest version into a temporary directory + # and run helm repo index with --merge to update index.yaml + # without refreshing all of the timestamps + with TemporaryDirectory() as td: + _check_call( + [ + "helm", + "package", + chart_name, + "--dependency-update", + "--destination", + td + "/", + ] + ) + + _check_call( + [ + "helm", + "push", + chart_name + ".tgz", + "oci://" + chart_oci_repo + "/" + chart_oci_prefix, + ] + ) def publish_pages( chart_name, @@ -1240,15 +1328,24 @@ def main(argv=None): # publish chart if args.publish_chart: - publish_pages( - chart_name=chart["name"], - chart_version=chart_version, - chart_repo_github_path=chart["repo"]["git"], - chart_repo_url=chart["repo"]["published"], - extra_message=args.extra_message, - force=args.force_publish_chart, - ) - + if "oci" in chart["repo"]: + publish_chart_oci( + chart_name=chart["name"], + chart_base=chart["basepath"], + chart_version=chart_version, + chart_oci_repo=chart["repo"]["oci"], + chart_oci_prefix=chart["repo"]["prefix"], + force=args.force_publish_chart, + ) + if "git" in chart["repo"]: + publish_pages( + chart_name=chart["name"], + chart_version=chart_version, + chart_repo_github_path=chart["repo"]["git"], + chart_repo_url=chart["repo"]["published"], + extra_message=args.extra_message, + force=args.force_publish_chart, + ) if __name__ == "__main__": main() From 3248a2e6f9c97cdf296aeb15c1a3b58934d8cedc Mon Sep 17 00:00:00 2001 From: Ben Leggett Date: Thu, 8 Sep 2022 13:13:14 -0400 Subject: [PATCH 2/9] Fix this --- README.md | 13 ++++++++++ chartpress.py | 67 ++++++++++++++++++++++++++++++--------------------- 2 files changed, 52 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 5388b1e..9273a16 100644 --- a/README.md +++ b/README.md @@ -172,6 +172,9 @@ charts: # --reset flag. It defaults to "0.0.1-set.by.chartpress". This is a valid # SemVer 2 version, which is required for a helm lint command to succeed. resetVersion: 1.2.3-dev + # Base path, relative to `chartpress.yaml`, where named charts are kept. + # Default is same directory as `chartpress.yaml` + basePath: ./helmcharts # baseVersion sets the base version for development tags, # instead of using the latest tag from `git describe`. @@ -189,6 +192,16 @@ charts: repo: git: jupyterhub/helm-chart published: https://jupyterhub.github.io/helm-chart + + # Publishing Helm charts to OCI registries (with a custom path prefix) is also supported, + # via defining an `oci` key under `repo`. + # For example, the following will push a chart named `binderhub` to Github's OCI registry under the path + # `ghcr.io/jupyterhub/helm-charts/binderhub` + # repo: + # oci: ghcr.io/jupyterhub + # prefix: helm-charts + + # Additional paths that when modified should lead to an updated Chart.yaml # version, other than the chart directory in or any path that # influence the images of the chart. These paths should be set relative to diff --git a/chartpress.py b/chartpress.py index fb36053..8d974b4 100644 --- a/chartpress.py +++ b/chartpress.py @@ -697,7 +697,7 @@ def build_images( return values_file_modifications -def _update_values_file_with_modifications(name, modifications): +def _update_values_file_with_modifications(name, base_path, modifications): """ Update /values.yaml file with a dictionary of modifications with its root level keys representing a path within the values.yaml file. @@ -715,7 +715,7 @@ def _update_values_file_with_modifications(name, modifications): } } """ - values_file = os.path.join(name, "values.yaml") + values_file = os.path.join(base_path, name, "values.yaml") with open(values_file) as f: values = yaml.load(f) @@ -799,6 +799,7 @@ def build_chart( long=False, strict_version=False, base_version=None, + base_path=None, ): """ Update Chart.yaml's version, using specified version or by constructing one. @@ -818,7 +819,7 @@ def build_chart( - 0.9.0 """ # read Chart.yaml - chart_file = os.path.join(name, "Chart.yaml") + chart_file = os.path.join(base_path, name, "Chart.yaml") with open(chart_file) as f: chart = yaml.load(f) @@ -886,27 +887,36 @@ def publish_chart_oci( chart_dir = f"{chart_base}/{chart_name}" _check_call(["git", "fetch"], cwd=chart_dir, echo=True) - # # check if a chart with the same name and version has already been published. If - # # there is, the behaviour depends on `--force-publish-chart` - # # and chart_version and make a decision based on the --force-publish-chart - # # flag if that is the case, but always log what's done - # if os.path.isfile(os.path.join(chart_dir, "index.yaml")): - # with open(os.path.join(checkout_dir, "index.yaml")) as f: - # chart_repo_index = yaml.load(f) - # published_charts = chart_repo_index["entries"].get(chart_name, []) - - # if published_charts and any( - # c["version"] == chart_version for c in published_charts - # ): - # if force: - # _log( - # f"Chart of version {chart_version} already exists, overwriting it." - # ) - # else: - # _log( - # f"Skipping chart publishing of version {chart_version}, it is already published" - # ) - # return + # check if a chart with the same name and version has already been published. If + # there is, the behaviour depends on `--force-publish-chart` + # and chart_version and make a decision based on the --force-publish-chart + # flag if that is the case, but always log what's done + + try: + _check_call( + [ + "helm", + "show", + "chart", + "oci://" + chart_oci_repo + "/" + chart_oci_prefix + "/" + chart_name, + "--version", + chart_version, + ] + ) + except subprocess.CalledProcessError: + _log( + f"Chart of version {chart_version} not already published, continuing." + ) + else: + if force: + _log( + f"Chart of version {chart_version} already exists, overwriting it." + ) + else: + _log( + f"Skipping chart publishing of version {chart_version}, it is already published" + ) + return # package the latest version into a temporary directory # and run helm repo index with --merge to update index.yaml @@ -916,7 +926,7 @@ def publish_chart_oci( [ "helm", "package", - chart_name, + chart_dir, "--dependency-update", "--destination", td + "/", @@ -927,7 +937,7 @@ def publish_chart_oci( [ "helm", "push", - chart_name + ".tgz", + os.path.join(td, chart_name + "-" + chart_version + ".tgz"), "oci://" + chart_oci_repo + "/" + chart_oci_prefix, ] ) @@ -1285,6 +1295,7 @@ def main(argv=None): base_version=base_version, long=args.long, strict_version=args.publish_chart, + base_path=chart["basePath"], ) if "images" in chart: @@ -1323,7 +1334,7 @@ def main(argv=None): # update values.yaml _update_values_file_with_modifications( - chart["name"], values_file_modifications + chart["name"], chart["basePath"], values_file_modifications ) # publish chart @@ -1331,7 +1342,7 @@ def main(argv=None): if "oci" in chart["repo"]: publish_chart_oci( chart_name=chart["name"], - chart_base=chart["basepath"], + chart_base=chart["basePath"], chart_version=chart_version, chart_oci_repo=chart["repo"]["oci"], chart_oci_prefix=chart["repo"]["prefix"], From 18e0f5ba689d047af4be07eaf182b27a1829b806 Mon Sep 17 00:00:00 2001 From: Ben Leggett Date: Thu, 8 Sep 2022 13:15:45 -0400 Subject: [PATCH 3/9] Fix default --- chartpress.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/chartpress.py b/chartpress.py index 8d974b4..be715a7 100644 --- a/chartpress.py +++ b/chartpress.py @@ -697,7 +697,7 @@ def build_images( return values_file_modifications -def _update_values_file_with_modifications(name, base_path, modifications): +def _update_values_file_with_modifications(name, modifications, base_path="./"): """ Update /values.yaml file with a dictionary of modifications with its root level keys representing a path within the values.yaml file. @@ -799,7 +799,7 @@ def build_chart( long=False, strict_version=False, base_version=None, - base_path=None, + base_path="./", ): """ Update Chart.yaml's version, using specified version or by constructing one. @@ -846,11 +846,11 @@ def build_chart( def publish_chart_oci( chart_name, - chart_base, chart_version, chart_oci_repo, chart_oci_prefix, force=False, + chart_base="./", ): """ Update a Helm chart stored in an OCI registry (e.g. ghcr.io). @@ -1334,7 +1334,7 @@ def main(argv=None): # update values.yaml _update_values_file_with_modifications( - chart["name"], chart["basePath"], values_file_modifications + chart["name"], values_file_modifications, chart["basePath"] ) # publish chart @@ -1342,11 +1342,11 @@ def main(argv=None): if "oci" in chart["repo"]: publish_chart_oci( chart_name=chart["name"], - chart_base=chart["basePath"], chart_version=chart_version, chart_oci_repo=chart["repo"]["oci"], chart_oci_prefix=chart["repo"]["prefix"], force=args.force_publish_chart, + chart_base=chart["basePath"], ) if "git" in chart["repo"]: publish_pages( From 6cc113b6a4779c911c5d30bcd1d50cf8f552cd97 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 8 Sep 2022 17:18:43 +0000 Subject: [PATCH 4/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- README.md | 7 +++---- chartpress.py | 11 +++++------ 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 9273a16..cdfcfbd 100644 --- a/README.md +++ b/README.md @@ -192,16 +192,15 @@ charts: repo: git: jupyterhub/helm-chart published: https://jupyterhub.github.io/helm-chart - + # Publishing Helm charts to OCI registries (with a custom path prefix) is also supported, # via defining an `oci` key under `repo`. - # For example, the following will push a chart named `binderhub` to Github's OCI registry under the path + # For example, the following will push a chart named `binderhub` to Github's OCI registry under the path # `ghcr.io/jupyterhub/helm-charts/binderhub` # repo: # oci: ghcr.io/jupyterhub # prefix: helm-charts - - + # Additional paths that when modified should lead to an updated Chart.yaml # version, other than the chart directory in or any path that # influence the images of the chart. These paths should be set relative to diff --git a/chartpress.py b/chartpress.py index be715a7..ca6679c 100644 --- a/chartpress.py +++ b/chartpress.py @@ -844,6 +844,7 @@ def build_chart( # return version return version + def publish_chart_oci( chart_name, chart_version, @@ -904,14 +905,10 @@ def publish_chart_oci( ] ) except subprocess.CalledProcessError: - _log( - f"Chart of version {chart_version} not already published, continuing." - ) + _log(f"Chart of version {chart_version} not already published, continuing.") else: if force: - _log( - f"Chart of version {chart_version} already exists, overwriting it." - ) + _log(f"Chart of version {chart_version} already exists, overwriting it.") else: _log( f"Skipping chart publishing of version {chart_version}, it is already published" @@ -942,6 +939,7 @@ def publish_chart_oci( ] ) + def publish_pages( chart_name, chart_version, @@ -1358,5 +1356,6 @@ def main(argv=None): force=args.force_publish_chart, ) + if __name__ == "__main__": main() From 57259980f8a8665a2f341b79738f59371a0f514f Mon Sep 17 00:00:00 2001 From: Ben Leggett Date: Thu, 8 Sep 2022 13:32:54 -0400 Subject: [PATCH 5/9] Set defaults along the lines of other values --- chartpress.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/chartpress.py b/chartpress.py index ca6679c..cb1bdd7 100644 --- a/chartpress.py +++ b/chartpress.py @@ -331,6 +331,15 @@ def _get_all_image_paths(name, options): paths.extend(options.get("paths", [])) return list(set(paths)) +def _get_chart_base_path(options): + """ + Return the image's contextPath configuration value, or a default value based + on the image name. + """ + if options.get("basePath"): + return options["basePath"] + else: + return "./" def _get_all_chart_paths(options): """ @@ -697,7 +706,7 @@ def build_images( return values_file_modifications -def _update_values_file_with_modifications(name, modifications, base_path="./"): +def _update_values_file_with_modifications(name, modifications, base_path): """ Update /values.yaml file with a dictionary of modifications with its root level keys representing a path within the values.yaml file. @@ -799,7 +808,7 @@ def build_chart( long=False, strict_version=False, base_version=None, - base_path="./", + base_path, ): """ Update Chart.yaml's version, using specified version or by constructing one. @@ -1284,6 +1293,8 @@ def main(argv=None): if base_version: base_version = _check_base_version(base_version) + + chart_base_path = _get_chart_base_path(chart) if not args.list_images: # update Chart.yaml with a version chart_version = build_chart( @@ -1293,7 +1304,7 @@ def main(argv=None): base_version=base_version, long=args.long, strict_version=args.publish_chart, - base_path=chart["basePath"], + base_path=chart_base_path, ) if "images" in chart: @@ -1332,7 +1343,7 @@ def main(argv=None): # update values.yaml _update_values_file_with_modifications( - chart["name"], values_file_modifications, chart["basePath"] + chart["name"], values_file_modifications, chart_base_path ) # publish chart @@ -1344,7 +1355,7 @@ def main(argv=None): chart_oci_repo=chart["repo"]["oci"], chart_oci_prefix=chart["repo"]["prefix"], force=args.force_publish_chart, - chart_base=chart["basePath"], + chart_base=chart_base_path, ) if "git" in chart["repo"]: publish_pages( From 786c6acb1538289f5558765b917012276f009e12 Mon Sep 17 00:00:00 2001 From: Ben Leggett Date: Thu, 8 Sep 2022 13:36:18 -0400 Subject: [PATCH 6/9] Fix this --- chartpress.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/chartpress.py b/chartpress.py index cb1bdd7..fc722fa 100644 --- a/chartpress.py +++ b/chartpress.py @@ -808,7 +808,7 @@ def build_chart( long=False, strict_version=False, base_version=None, - base_path, + base_path=None, ): """ Update Chart.yaml's version, using specified version or by constructing one. @@ -857,10 +857,10 @@ def build_chart( def publish_chart_oci( chart_name, chart_version, + chart_base, chart_oci_repo, chart_oci_prefix, force=False, - chart_base="./", ): """ Update a Helm chart stored in an OCI registry (e.g. ghcr.io). @@ -1352,10 +1352,10 @@ def main(argv=None): publish_chart_oci( chart_name=chart["name"], chart_version=chart_version, + chart_base=chart_base_path, chart_oci_repo=chart["repo"]["oci"], chart_oci_prefix=chart["repo"]["prefix"], force=args.force_publish_chart, - chart_base=chart_base_path, ) if "git" in chart["repo"]: publish_pages( From 93e2e79e757769603ab0ff795d06b36806b74618 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 8 Sep 2022 17:36:35 +0000 Subject: [PATCH 7/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- chartpress.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/chartpress.py b/chartpress.py index fc722fa..f0eba56 100644 --- a/chartpress.py +++ b/chartpress.py @@ -331,6 +331,7 @@ def _get_all_image_paths(name, options): paths.extend(options.get("paths", [])) return list(set(paths)) + def _get_chart_base_path(options): """ Return the image's contextPath configuration value, or a default value based @@ -341,6 +342,7 @@ def _get_chart_base_path(options): else: return "./" + def _get_all_chart_paths(options): """ Returns the unique paths that when changed should trigger a version update @@ -1293,7 +1295,6 @@ def main(argv=None): if base_version: base_version = _check_base_version(base_version) - chart_base_path = _get_chart_base_path(chart) if not args.list_images: # update Chart.yaml with a version From 1ad666095d76cc283d48ece3e9ef75118b5b4d34 Mon Sep 17 00:00:00 2001 From: Ben Leggett Date: Thu, 8 Sep 2022 13:40:02 -0400 Subject: [PATCH 8/9] Fix this --- chartpress.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/chartpress.py b/chartpress.py index f0eba56..a0cb60a 100644 --- a/chartpress.py +++ b/chartpress.py @@ -334,13 +334,13 @@ def _get_all_image_paths(name, options): def _get_chart_base_path(options): """ - Return the image's contextPath configuration value, or a default value based - on the image name. + Return the basePath which will be prepended to the chart name when loading the chart directory, + or an empty value, meaning the chart directory is assumed to be in the same root as `chartpress.yaml`. """ if options.get("basePath"): return options["basePath"] else: - return "./" + return "" def _get_all_chart_paths(options): From 4030d6119c233a5bcf377557e87f21a721938746 Mon Sep 17 00:00:00 2001 From: Ben Leggett Date: Thu, 8 Sep 2022 13:45:44 -0400 Subject: [PATCH 9/9] Fix this --- chartpress.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chartpress.py b/chartpress.py index a0cb60a..a355272 100644 --- a/chartpress.py +++ b/chartpress.py @@ -810,7 +810,7 @@ def build_chart( long=False, strict_version=False, base_version=None, - base_path=None, + base_path="", ): """ Update Chart.yaml's version, using specified version or by constructing one.