From 83925ebe6ecccb0b1b35c1a472063d7d29c6f774 Mon Sep 17 00:00:00 2001 From: Konstantin Skaburskas Date: Fri, 18 Feb 2022 15:33:49 +0100 Subject: [PATCH 1/8] fix issue --- src/notify-email.py | 26 ++++++++++++++++++-------- src/notify-slack.py | 8 ++++++-- tests/notify_deps.py | 1 + tests/test_notify_email.py | 14 ++++++++++++++ 4 files changed, 39 insertions(+), 10 deletions(-) create mode 120000 tests/notify_deps.py create mode 100644 tests/test_notify_email.py diff --git a/src/notify-email.py b/src/notify-email.py index c6c8ba4..ef4ed0f 100755 --- a/src/notify-email.py +++ b/src/notify-email.py @@ -1,14 +1,18 @@ #!/usr/bin/env python3 -from notify_deps import * - -import smtplib +import multiprocessing +import os import requests +import smtplib +import time from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from jinja2 import Template from datetime import datetime +from notify_deps import get_logger, timestamp_convert, main +from notify_deps import NUVLA_ENDPOINT + log_local = get_logger('email') @@ -33,7 +37,7 @@ def get_nuvla_config(): headers = {'nuvla-authn-info': nuvla_api_authn_header} resp = requests.get(config_url, headers=headers) if resp.status_code != 200: - raise Exception(f'Failed to get response from server: status {resp.status_code}') + raise EnvironmentError(f'Failed to get response from server: status {resp.status_code}') return resp.json() @@ -47,7 +51,7 @@ def set_smpt_params(): try: SMTP_PORT = int(os.environ['SMTP_PORT']) except ValueError: - raise Exception(f"Incorrect value for SMTP_PORT number: {os.environ['SMTP_PORT']}") + raise ValueError(f"Incorrect value for SMTP_PORT number: {os.environ['SMTP_PORT']}") SMTP_SSL = os.environ['SMTP_SSL'].lower() in ['true', 'True'] else: nuvla_config = get_nuvla_config() @@ -59,7 +63,7 @@ def set_smpt_params(): except Exception as ex: msg = f'Provide full SMTP config either via env vars or in configuration/nuvla: {ex}' log_local.error(msg) - raise Exception(msg) + raise ValueError(msg) KAFKA_TOPIC = os.environ.get('KAFKA_TOPIC') or 'NOTIFICATIONS_EMAIL_S' @@ -81,7 +85,6 @@ def get_smtp_server(debug_level=0) -> smtplib.SMTP: def html_content(values: dict): - # subs_config_id = values.get('SUBS_ID') subs_config_link = f'Notification configuration' r_uri = values.get('RESOURCE_URI') @@ -135,12 +138,19 @@ def send(server: smtplib.SMTP, recipients, subject, html, attempts=SEND_EMAIL_AT raise SendFailedMaxAttempts(f'Failed sending email after {attempts} attempts.') +def get_recipients(v: dict): + return list(filter(lambda x: x != '', v.get('DESTINATION', '').split(' '))) + + def worker(workq: multiprocessing.Queue): smtp_server = get_smtp_server() while True: msg = workq.get() if msg: - recipients = msg.value['DESTINATION'].split(',') + recipients = get_recipients(msg.value) + if len(recipients) == 0: + log_local.warning(f'No recipients provided in: {msg.value}') + continue r_id = msg.value.get('RESOURCE_ID') r_name = msg.value.get('NAME') subject = msg.value.get('SUBS_NAME') or f'{r_name or r_id} alert' diff --git a/src/notify-slack.py b/src/notify-slack.py index e24183a..7b1c820 100755 --- a/src/notify-slack.py +++ b/src/notify-slack.py @@ -1,9 +1,14 @@ #!/usr/bin/env python3 +import json +import datetime +import multiprocessing import requests +import os import re -from notify_deps import * +from notify_deps import get_logger, timestamp_convert, main +from notify_deps import NUVLA_ENDPOINT KAFKA_TOPIC = os.environ.get('KAFKA_TOPIC') or 'NOTIFICATIONS_SLACK_S' KAFKA_GROUP_ID = 'nuvla-notification-slack' @@ -18,7 +23,6 @@ def message_content(values: dict): - # subs_config_id = values.get('SUBS_ID') subs_name = lt.sub('<', gt.sub('>', values.get('SUBS_NAME', ''))) subs_config_txt = f'<{NUVLA_ENDPOINT}/ui/notifications|{subs_name}>' diff --git a/tests/notify_deps.py b/tests/notify_deps.py new file mode 120000 index 0000000..b26ed92 --- /dev/null +++ b/tests/notify_deps.py @@ -0,0 +1 @@ +../src/notify_deps.py \ No newline at end of file diff --git a/tests/test_notify_email.py b/tests/test_notify_email.py new file mode 100644 index 0000000..fcd4b26 --- /dev/null +++ b/tests/test_notify_email.py @@ -0,0 +1,14 @@ +import unittest + +from notify_email import get_recipients + + +class NotifyEmail(unittest.TestCase): + + def test_get_recipients(self): + assert 0 == len(get_recipients({})) + e1 = 'a@b.c' + e2 = 'a@b.c' + assert [e1] == get_recipients({'DESTINATION': e1}) + assert [e1, e2] == get_recipients({'DESTINATION': f'{e1} {e2}'}) + assert [e1, e2] == get_recipients({'DESTINATION': f' {e1} {e2} '}) From 84b898ef0cf4801d3037fc073396921c093cf029 Mon Sep 17 00:00:00 2001 From: Konstantin Skaburskas Date: Fri, 18 Feb 2022 15:35:44 +0100 Subject: [PATCH 2/8] test and build in CI --- .dockerignore | 9 ++++ .github/workflows/main.yml | 29 +++++++++---- container-release.sh | 88 ++++++++++++++++++++++++++++++++++++++ requirements.tests.txt | 2 + 4 files changed, 119 insertions(+), 9 deletions(-) create mode 100644 .dockerignore create mode 100644 container-release.sh create mode 100644 requirements.tests.txt diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..4c82899 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +target +.travis? +pom.xml +*.md +.gitignore +.idea +code/tests +code/requirements.tests.txt +__pycache__ diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 21a96d3..ac4c4cc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,6 +1,4 @@ -# This is a basic workflow to help you get started with Actions - -name: CI +name: "Test and build notification scripts." on: push: @@ -13,18 +11,31 @@ on: jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - - name: Run a one-line script - run: echo Hello, world! + - uses: actions/setup-python@v2 + with: + python-version: '3.8' - - name: Run a multi-line script + - name: Run unit tests run: | - echo Add other actions to build, - echo test, and deploy your project. + pip install -r requirements.test.txt + pytest tests/ --junitxml=test-report.xml -v + + - name: Build and deploy on architecture + env: + DOCKER_USERNAME: ${{ secrets.SIXSQ_DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.SIXSQ_DOCKER_PASSWORD }} + run: ./container-release.sh + + - name: Publish Unit Test Results + uses: EnricoMi/publish-unit-test-result-action@v1 + if: always() + with: + files: test-report.xml notify: if: always() diff --git a/container-release.sh b/container-release.sh new file mode 100644 index 0000000..586f422 --- /dev/null +++ b/container-release.sh @@ -0,0 +1,88 @@ +#!/bin/bash -xe + +############################### +# CHANGE THIS ON EVERY REPO # +DOCKER_IMAGE=$(basename `git rev-parse --show-toplevel`) +############################### + +# default env vars in GH actions +GIT_BRANCH=$(echo ${GITHUB_REF} | awk -F'/' '{print $(NF)}' | sed -e 's/[^a-z0-9\._-]/-/g') + +# non-tagged builds are not releases, so they always go on nuvladev +DOCKER_ORG=${DOCKER_ORG:-nuvladev} + +MANIFEST=${DOCKER_ORG}/${DOCKER_IMAGE}:${GIT_BRANCH} + +platforms=(amd64 arm64) + + +# +# remove any previous builds +# + +rm -Rf target/*.tar +mkdir -p target + +# +# generate image for each platform +# + +for platform in "${platforms[@]}"; do + GIT_BUILD_TIME=$(date --utc +%FT%T.%3NZ) + docker run --rm --privileged -v ${PWD}:/tmp/work --entrypoint buildctl-daemonless.sh moby/buildkit:master \ + build \ + --frontend dockerfile.v0 \ + --opt platform=linux/${platform} \ + --opt filename=./Dockerfile \ + --opt build-arg:GIT_BRANCH=${GIT_BRANCH} \ + --opt build-arg:GIT_BUILD_TIME=${GIT_BUILD_TIME} \ + --opt build-arg:GIT_COMMIT_ID=${GITHUB_SHA} \ + --opt build-arg:GITHUB_RUN_NUMBER=${GITHUB_RUN_NUMBER} \ + --opt build-arg:GITHUB_RUN_ID=${GITHUB_RUN_ID} \ + --opt build-arg:PROJECT_URL=${GIHUB_SERVER_URL}/${GITHUB_REPOSITORY} \ + --output type=docker,name=${MANIFEST}-${platform},dest=/tmp/work/target/${DOCKER_IMAGE}-${platform}.docker.tar \ + --local context=/tmp/work \ + --local dockerfile=/tmp/work \ + --progress plain + +done + +# +# load all generated images +# + +for platform in "${platforms[@]}"; do + docker load --input ./target/${DOCKER_IMAGE}-${platform}.docker.tar +done + + +manifest_args=(${MANIFEST}) + +# +# login to docker hub +# + +unset HISTFILE +echo ${DOCKER_PASSWORD} | docker login -u ${DOCKER_USERNAME} --password-stdin + +# +# push all generated images +# + +for platform in "${platforms[@]}"; do + docker push ${MANIFEST}-${platform} + manifest_args+=("${MANIFEST}-${platform}") +done + +# +# create manifest, update, and push +# + +export DOCKER_CLI_EXPERIMENTAL=enabled +docker manifest create "${manifest_args[@]}" + +for platform in "${platforms[@]}"; do + docker manifest annotate ${MANIFEST} ${MANIFEST}-${platform} --arch ${platform} +done + +docker manifest push --purge ${MANIFEST} \ No newline at end of file diff --git a/requirements.tests.txt b/requirements.tests.txt new file mode 100644 index 0000000..5154d6f --- /dev/null +++ b/requirements.tests.txt @@ -0,0 +1,2 @@ +-r requerements.txt +pytest==7.0.1 From 2f7ddac6123ba98378a7c4433de8ce61f9badbb1 Mon Sep 17 00:00:00 2001 From: Konstantin Skaburskas Date: Fri, 18 Feb 2022 15:37:27 +0100 Subject: [PATCH 3/8] minor --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ac4c4cc..32e816e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -22,7 +22,7 @@ jobs: - name: Run unit tests run: | - pip install -r requirements.test.txt + pip install -r requirements.tests.txt pytest tests/ --junitxml=test-report.xml -v - name: Build and deploy on architecture From fa032d7d6c86dbddaa301b62f5e5d64e32a639c1 Mon Sep 17 00:00:00 2001 From: Konstantin Skaburskas Date: Fri, 18 Feb 2022 15:38:57 +0100 Subject: [PATCH 4/8] typo --- requirements.tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.tests.txt b/requirements.tests.txt index 5154d6f..25b96d4 100644 --- a/requirements.tests.txt +++ b/requirements.tests.txt @@ -1,2 +1,2 @@ --r requerements.txt +-r requirements.txt pytest==7.0.1 From fdf1da713ecc843170bfb5622db78a1b8b4ea350 Mon Sep 17 00:00:00 2001 From: Konstantin Skaburskas Date: Fri, 18 Feb 2022 15:59:23 +0100 Subject: [PATCH 5/8] fix for https://github.com/aws/aws-sam-cli/issues/3661 --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index 1dd4ad1..00c8c38 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,5 @@ kafka-python==2.0.2 requests==2.25.0 urllib3==1.26.2 Jinja2==2.11.2 +# To fix https://github.com/aws/aws-sam-cli/issues/3661 +markupsafe==2.0.1 From 0a977b6f6d85651a04bc1285ed7c9c0a2f23ed02 Mon Sep 17 00:00:00 2001 From: Konstantin Skaburskas Date: Fri, 18 Feb 2022 16:01:59 +0100 Subject: [PATCH 6/8] exec permission for container-release.sh --- container-release.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 container-release.sh diff --git a/container-release.sh b/container-release.sh old mode 100644 new mode 100755 From 9acac4dfdd8938214a8c3ffcd7afaec877723af5 Mon Sep 17 00:00:00 2001 From: Konstantin Skaburskas Date: Fri, 18 Feb 2022 16:06:50 +0100 Subject: [PATCH 7/8] multi-arch docker image build required --- .github/workflows/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 32e816e..69aea17 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -25,6 +25,9 @@ jobs: pip install -r requirements.tests.txt pytest tests/ --junitxml=test-report.xml -v + - name: Multi-arch docker image build prerequired + run: sudo docker run --privileged linuxkit/binfmt:v0.7 + - name: Build and deploy on architecture env: DOCKER_USERNAME: ${{ secrets.SIXSQ_DOCKER_USERNAME }} From 1155d8ceded5d087ad07a419deade92a4fbb419d Mon Sep 17 00:00:00 2001 From: Konstantin Skaburskas Date: Fri, 18 Feb 2022 16:14:33 +0100 Subject: [PATCH 8/8] add release ci job --- .github/workflows/release.yml | 41 +++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..b790f23 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,41 @@ +# Automatic release for every new tag +name: "Release" + +on: + push: + tags: + - "*.*.*" + +jobs: + build: + runs-on: ubuntu-20.04 + + steps: + - uses: actions/checkout@v2 + + - name: Multi-arch docker image build prerequired + run: sudo docker run --privileged linuxkit/binfmt:v0.7 + + - name: Build and deploy on architecture + env: + DOCKER_USERNAME: ${{ secrets.SIXSQ_DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.SIXSQ_DOCKER_PASSWORD }} + DOCKER_ORG: nuvla + run: ./container-release.sh + + + notify: + if: always() + name: Post Workflow Status To Slack + needs: + - build + runs-on: ubuntu-latest + steps: + - name: Slack Workflow Notification + uses: Gamesight/slack-workflow-status@master + with: + # Required Input + repo_token: ${{secrets.GITHUB_TOKEN}} + slack_webhook_url: ${{secrets.SLACK_WEBHOOK_URL}} + # Optional Input + icon_emoji: ':rocket:'