github actions for mbed-os-env docker management

.github/workflows/ci_scripts/ghcr_utils.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python
+
+"""
+Copyright (c) 2017-2021 ARM Limited. All rights reserved.
+SPDX-License-Identifier: Apache-2.0
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations 
+"""
+import click
+import requests
+import logging
+import sys
+import time
+import json
+import subprocess
+
+"""
+This file contains ghcr utlity wrapper used for:
+- retrieving digest of docker image
+- deleting images in ghcr
+"""
+
+
+@click.command()
+@click.pass_context
+@click.option("-r", "--repository", required=True)
+@click.option("-t", "--tag", required=True)
+@click.option("-p", "--platform", required=False)
+def get_digest(ctx, repository, tag, platform=None):
+    """
+    Prints docker digest of specific platform of multi architecture image
+    :param ctx: click context
+    :param repository: docker repository
+    :param tag: docker tag
+    :param platform: platform for e.g, linux/arm64
+    """
+    command = f"docker run quay.io/skopeo/stable --creds={ctx.obj['username']}:{ctx.obj['passwd']} inspect docker://ghcr.io/{ctx.obj['username']}/{repository}:{tag} --raw"
+    output = subprocess.run(
+        command.split(), stdout=subprocess.PIPE, check=True
+    ).stdout.decode("utf-8")
+    output = json.loads(output)
+
+    images = output["manifests"]
+    digest = ""
+    if len(images) and platform is None:
+        logging.error(
+            "This tag has more than one platform associated to it, please input a platform"
+        )
+        sys.exit(1)
+
+    for image in images:
+        if platform != None:
+            if (platform.split("/")[0] == image["platform"]["os"]) and (
+                platform.split("/")[1] == image["platform"]["architecture"]
+            ):
+                digest = image["digest"]
+        else:
+            digest = image["digest"]
+
+    if digest == "":
+        logging.error("Digest not found. image not in repo for the given platform")
+        sys.exit(1)
+
+    print(digest)
+
+
+@click.command()
+@click.pass_context
+@click.option("-r", "--repository", required=True)
+@click.option(
+    "-n",
+    "--number_of_days",
+    default="10",
+    help="number of days since image was created",
+    required=False,
+)
+def delete_old_images(ctx, repository, number_of_days):
+    """
+    delete old images from docker repository
+
+    :param ctx: click context
+    :param repository: docker repository
+    :param number_of_days: delete older than these number of days
+    """
+    with requests.Session() as s:
+        github_api_accept = "application/vnd.github.v3+json"
+        s.headers.update(
+            {"Authorization": f'token {ctx.obj["passwd"]}', "Accept": github_api_accept}
+        )
+        r = s.get(
+            f"https://api.github.com/user/packages/container/{repository}/versions"
+        )
+        versions = r.json()
+        version_id = None
+        pattern = "%d.%m.%Y %H:%M:%S"
+        pattern = "%Y-%m-%dT%H:%M:%SZ"
+        current_time = time.time()
+        for version in versions:
+            logging.info(version)
+            epoch = int(time.mktime(time.strptime(version["updated_at"], pattern)))
+
+            if (current_time - epoch) / (24 * 60 * 60) > int(number_of_days):
+                version_id = version["id"]
+                logging.debug(f"deleteing image with version id {version_id}")
+
+                url = f"https://api.github.com/user/packages/container/{repository}/versions/{version_id}"
+                resp = s.delete(url)
+                resp.raise_for_status()
+
+
+@click.group()
+@click.pass_context
+@click.option("-u", "--username", required=False)
+@click.option("-p", "--passwd", required=False)
+@click.option("-v", "--verbose", is_flag=True, default=False)
+def main(ctx, username, passwd, verbose):
+    ctx.obj = {"username": username, "passwd": passwd}
+
+    if verbose:
+        logging.basicConfig(
+            stream=sys.stdout,
+            format="%(levelname)s %(asctime)s %(message)s",
+            datefmt="%m/%d/%Y %I:%M:%S %p",
+        )
+        logging.getLogger().setLevel(logging.DEBUG)
+    else:
+        logging.basicConfig(
+            format="%(levelname)s %(asctime)s %(message)s",
+            datefmt="%m/%d/%Y %I:%M:%S %p",
+        )
+        logging.getLogger().setLevel(logging.INFO)
+
+
+if __name__ == "__main__":
+    main.add_command(get_digest)
+    main.add_command(delete_old_images)
+    main()

.github/workflows/docker_management.branch.yml

@@ -0,0 +1,237 @@
+name: Publish or Update docker image for head of branch
+# Design details in https://github.com/ARMmbed/mbed-os/blob/master/docs/design-documents/docker_management
+
+on:
+
+  # passive update once a week
+  schedule:
+    - cron:  '15 4 * * 6'
+
+  # build on master branch when there is changes for active update
+  push:
+    branches: 
+      - master
+
+    paths:
+      - requirements.txt
+      - docker_images/mbed-os-env/**
+      - .github/workflows/docker_management.branch.yml
+
+
+  # manual trigger when needed 
+  workflow_dispatch: 
+
+
+jobs:
+  prepare-tags:
+    runs-on: ubuntu-latest
+
+    steps:
+      - 
+        name: Extract branch name
+        shell: bash
+        run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+        id: extract_branch
+
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - 
+        name: Set UUID
+        id: generate-uuid
+        uses: filipstefansson/uuid-action@v1
+
+# set docker tags we are building, and intending to publish
+# dev-tag is temporary for testing purpose. This should be considered as unstable.
+# dated-tag is created for versioning purpose
+# prod-tag-latest could be used by customers, CI etc for keeping up to date
+      - 
+        name: Get build information
+        shell: bash
+        run: |
+          mkdir -p build_info
+          date=$(date +"%Y.%m.%dT%H.%M.%S")
+          echo dev-${{ steps.extract_branch.outputs.branch }}-${date}-${{ steps.generate-uuid.outputs.uuid }} > build_info/dev_tag
+          echo ${{ steps.extract_branch.outputs.branch }}-${date} > build_info/prod_tag_dated
+          echo ${{ steps.extract_branch.outputs.branch }}-latest > build_info/prod_tag_latest
+          echo ${{ steps.extract_branch.outputs.branch }} > build_info/mbed_os_version
+          
+      - 
+        name: Archive information 
+        uses: actions/upload-artifact@v2
+        with:
+          name: build-info
+          path: build_info
+
+
+  build-container:
+    runs-on: ubuntu-latest
+    needs: prepare-tags
+    outputs:
+      DEV_DIGEST: ${{ steps.docker_info_dev.outputs.DIGEST }}
+      PROD_DIGEST: ${{ steps.docker_info_prod.outputs.DIGEST }}
+
+    steps:
+      - 
+        name: unarchive artefacts
+        uses: actions/download-artifact@v2
+        with:
+          name: build-info
+
+      - 
+        name: Get build info from archive
+        shell: bash
+        id: build_info
+        run: |
+          value=`cat dev_tag`
+          echo "DEV TAG is $value"
+          echo "::set-output name=DOCKER_DEV_TAG::$value"
+          value=`cat prod_tag_dated`
+          echo "PROD TAG DATED is $value"
+          echo "::set-output name=DOCKER_PROD_TAG_DATED::$value"
+          value=`cat prod_tag_latest`
+          echo "::set-output name=DOCKER_PROD_TAG_LATEST::$value"
+          echo "PROD TAG is $value"
+
+      - 
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - 
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - 
+        name: Login to DockerHub
+        uses: docker/login-action@v1 
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+
+      -
+        name: Build docker containers
+        uses: docker/build-push-action@v2
+        id: docker_build_dev
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: true
+          file: ./docker_images/mbed-os-env/Dockerfile
+          tags: ghcr.io/${{ github.actor }}/mbed-os-env-tmp:${{ steps.build_info.outputs.DOCKER_DEV_TAG }}
+
+  test-container:
+    runs-on: ubuntu-latest
+    needs: build-container
+    strategy:
+      matrix:
+        platform: [linux/amd64, linux/arm64]
+
+    steps:
+      - 
+        name: unarchive artefacts
+        uses: actions/download-artifact@v2
+        with:
+          name: build-info
+
+      - 
+        name: Get build info from archive
+        shell: bash
+        id: build_info
+        run: |
+          value=`cat dev_tag`
+          echo "TAG is $value"
+          echo "::set-output name=DOCKER_DEV_TAG::$value"
+          value=`cat prod_tag_dated`
+          echo "TAG is $value"
+          echo "::set-output name=DOCKER_PROD_TAG_DATED::$value"
+          value=`cat prod_tag_latest`
+          echo "::set-output name=DOCKER_PROD_TAG_LATEST::$value"
+          value=`cat mbed_os_version`
+          echo "::set-output name=MBED_OS_VERSION::$value"
+
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+
+      - 
+        name: Find DEV DOCKER DIGEST
+        id: docker_info_dev
+        run: |
+          DIGEST=$(python ./.github/workflows/ci_scripts/ghcr_utils.py -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} get-digest -r mbed-os-env-tmp -t ${{ steps.build_info.outputs.DOCKER_DEV_TAG }} -p ${{ matrix.platform }} )
+          echo "::set-output name=DIGEST::$DIGEST"
+          echo "Docker DIGEST: $DIGEST"
+
+      # as the dev images are created only for master branch, run test against
+      # development branch of blinky
+      - 
+        name: Checkout
+        uses: actions/checkout@v2
+        with:
+          repository: ARMmbed/mbed-os-example-blinky
+          path: mbed-os-example-blinky
+          ref: development
+      - 
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - 
+        name: test the container
+        id: test
+        uses: addnab/docker-run-action@v3
+        with:
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+          registry: ghcr.io
+          options: -v ${{ github.workspace }}:/work -w=/work
+          image: ghcr.io/${{ github.actor }}/mbed-os-env-tmp@${{ steps.docker_info_dev.outputs.DIGEST }}
+          shell: bash
+
+          run: |
+            uname -m
+            cd mbed-os-example-blinky
+            mbed deploy
+            # build using CLI1
+            mbed compile -m K64F -t GCC_ARM
+
+            # build using CLI2
+            mbed-tools compile -m K64F -t GCC_ARM         
+            
+      
+  deploy-container:
+    runs-on: ubuntu-latest
+    needs: test-container
+
+    steps:
+      - 
+        name: unarchive artefacts
+        uses: actions/download-artifact@v2
+        with:
+          name: build-info
+
+      - 
+        name: Get build info from archive
+        shell: bash
+        id: build_info
+        run: |
+          value=`cat dev_tag`
+          echo "TAG is $value"
+          echo "::set-output name=DOCKER_DEV_TAG::$value"
+          value=`cat prod_tag_dated`
+          echo "TAG is $value"
+          echo "::set-output name=DOCKER_PROD_TAG_DATED::$value"
+          value=`cat prod_tag_latest`
+          echo "::set-output name=DOCKER_PROD_TAG_LATEST::$value"
+          
+      - 
+        name: copy dev tag to prod
+        run: |
+          docker run quay.io/skopeo/stable --src-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} --dest-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} copy --all docker://ghcr.io/${{ github.actor }}/mbed-os-env-tmp:${{ steps.build_info.outputs.DOCKER_DEV_TAG }}  docker://ghcr.io/${{ github.actor }}/mbed-os-env:${{ steps.build_info.outputs.DOCKER_PROD_TAG_LATEST }}
+          docker run quay.io/skopeo/stable --src-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} --dest-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} copy --all docker://ghcr.io/${{ github.actor }}/mbed-os-env-tmp:${{ steps.build_info.outputs.DOCKER_DEV_TAG }}  docker://ghcr.io/${{ github.actor }}/mbed-os-env:${{ steps.build_info.outputs.DOCKER_PROD_TAG_DATED }}