From e2580a2e8f549c1c43bbd5904bdd3bfc151f2e02 Mon Sep 17 00:00:00 2001 From: jgraham Date: Mon, 1 Jun 2020 15:31:45 +0100 Subject: [PATCH] Add a wpt docker-push command (#23742) Co-authored-by: Robert Ma --- tools/docker/README.md | 18 +++---- tools/docker/commands.json | 28 +++++++++-- tools/docker/frontend.py | 99 ++++++++++++++++++++++++++++++++++++-- 3 files changed, 128 insertions(+), 17 deletions(-) diff --git a/tools/docker/README.md b/tools/docker/README.md index 9342aded298bdd..2203727ef96f31 100644 --- a/tools/docker/README.md +++ b/tools/docker/README.md @@ -4,15 +4,13 @@ images must be updated as well. Doing this requires you be part of the 'webplatformtests' organization on Docker Hub; ping @Hexcles or @stephenmcgruer if you are not a member. -In this directory, run the following, where `` is of the form -`webplatformtests/wpt:{current-version + 0.01}`: +The tag for a new docker image is of the form +`webplatformtests/wpt:{current-version + 0.01}` -```sh -# --pull forces Docker to get the newest base image. -docker build --pull -t . -docker push -``` +To update the docker image: -Then update the following Taskcluster configurations to use the new image: -* `.taskcluster.yml` (the decision task) -* `tools/ci/tc/tasks/test.yml` (all the other tasks) +* Update the following Taskcluster configurations to use the new image: + - `.taskcluster.yml` (the decision task) + - `tools/ci/tc/tasks/test.yml` (all the other tasks) + +* Run `wpt docker-push` diff --git a/tools/docker/commands.json b/tools/docker/commands.json index 15182cc9e6531d..421b0a636c2593 100644 --- a/tools/docker/commands.json +++ b/tools/docker/commands.json @@ -1,6 +1,26 @@ { - "docker-run": {"path": "frontend.py", "script": "run", "parser": "parser_run", "help": "Run wpt docker image", - "virtualenv": false}, - "docker-build": {"path": "frontend.py", "script": "build", "help": "Build wpt docker image", - "virtualenv": false} + "docker-run": { + "path": "frontend.py", + "script": "run", + "parser": "parser_run", + "help": "Run wpt docker image", + "virtualenv": false + }, + "docker-build": { + "path": "frontend.py", + "script": "build", + "help": "Build wpt docker image", + "virtualenv": false + }, + "docker-push": { + "path": "frontend.py", + "script": "push", + "parser": "parser_push", + "help": "Build and push wpt docker image", + "virtualenv": true, + "install": [ + "requests", + "pyyaml" + ] + } } diff --git a/tools/docker/frontend.py b/tools/docker/frontend.py index 59a1cff2ebd0ef..6d35d4c3826311 100644 --- a/tools/docker/frontend.py +++ b/tools/docker/frontend.py @@ -1,18 +1,111 @@ import argparse -import subprocess +import logging import os +import re +import subprocess +import sys + +from six import iteritems here = os.path.abspath(os.path.dirname(__file__)) wpt_root = os.path.abspath(os.path.join(here, os.pardir, os.pardir)) -def build(*args, **kwargs): +logger = logging.getLogger() + + +def build(tag="wpt:local", *args, **kwargs): subprocess.check_call(["docker", "build", "--pull", - "--tag", "wpt:local", + "--tag", tag, here]) +def parser_push(): + parser = argparse.ArgumentParser() + parser.add_argument("--tag", action="store", + help="Tag to use (default is taken from .taskcluster.yml)") + parser.add_argument("--force", action="store_true", + help="Ignore warnings and push anyway") + return parser + + +def walk_yaml(root, target): + rv = [] + if isinstance(root, list): + for value in root: + if isinstance(value, (dict, list)): + rv.extend(walk_yaml(value, target)) + elif isinstance(root, dict): + for key, value in iteritems(root): + if isinstance(value, (dict, list)): + rv.extend(walk_yaml(value, target)) + elif key == target: + rv.append(value) + return rv + + +def read_image_name(): + import yaml + with open(os.path.join(wpt_root, ".taskcluster.yml")) as f: + taskcluster_data = yaml.safe_load(f) + taskcluster_values = set(walk_yaml(taskcluster_data, "image")) + with open(os.path.join(wpt_root, "tools", "ci", "tc", "tasks", "test.yml")) as f: + test_data = yaml.safe_load(f) + tests_value = test_data["components"]["wpt-base"]["image"] + return taskcluster_values, tests_value + + +def lookup_tag(tag): + import requests + org, repo_version = tag.split("/", 1) + repo, version = repo_version.rsplit(":", 1) + resp = requests.get("https://hub.docker.com/v2/repositories/%s/%s/tags/%s" % + (org, repo, version)) + if resp.status_code == 200: + return True + if resp.status_code == 404: + return False + resp.raise_for_status() + + +def push(venv, tag=None, force=False, *args, **kwargs): + taskcluster_tags, tests_tag = read_image_name() + + taskcluster_tag = taskcluster_tags.pop() + + error_log = logger.warning if force else logger.error + if len(taskcluster_tags) != 0 or tests_tag != taskcluster_tag: + error_log("Image names in .taskcluster.yml and tools/ci/tc/tasks/test.yml " + "don't match.") + if not force: + sys.exit(1) + if tag is not None and tag != taskcluster_tag: + error_log("Supplied tag doesn't match .taskcluster.yml or " + "tools/ci/tc/tasks/test.yml; remember to update before pushing") + if not force: + sys.exit(1) + if tag is None: + logger.info("Using tag %s from .taskcluster.yml" % taskcluster_tag) + tag = taskcluster_tag + + tag_re = re.compile(r"webplatformtests/wpt:\d\.\d+") + if not tag_re.match(tag): + error_log("Tag doesn't match expected format webplatformtests/wpt:0.x") + if not force: + sys.exit(1) + + if lookup_tag(tag): + # No override for this case + logger.critical("Tag %s already exists" % tag) + sys.exit(1) + + build(tag) + subprocess.check_call(["docker", + "push", + tag]) + + def parser_run(): parser = argparse.ArgumentParser() parser.add_argument("--rebuild", action="store_true", help="Force rebuild of image")