From aac3acfc0b2df6d3fd1a30e3f991b00f7e085c02 Mon Sep 17 00:00:00 2001 From: Amit Bhanja Date: Wed, 7 Feb 2024 11:16:51 +0100 Subject: [PATCH 01/20] Added prometheus client -- Defined metrics --- src/metrics.py | 6 ++++++ src/notify-email.py | 6 ++++++ src/notify-slack.py | 2 ++ 3 files changed, 14 insertions(+) create mode 100644 src/metrics.py diff --git a/src/metrics.py b/src/metrics.py new file mode 100644 index 0000000..a942b01 --- /dev/null +++ b/src/metrics.py @@ -0,0 +1,6 @@ +from prometheus_client import Counter + +NOTIFICATIONS_SENT = Counter('notifications_sent', 'Number of notifications sent', + ['type', 'subscription_name', 'endpoint']) +NOTIFICATIONS_ERROR = Counter('notifications_error', 'Number of notifications that could not be sent due to error', + ['type', 'subscription_id', 'endpoint', 'exception']) diff --git a/src/notify-email.py b/src/notify-email.py index c1e2076..046e83f 100755 --- a/src/notify-email.py +++ b/src/notify-email.py @@ -12,6 +12,8 @@ from notify_deps import get_logger, timestamp_convert, main from notify_deps import NUVLA_ENDPOINT +from metrics import NOTIFICATIONS_SENT, NOTIFICATIONS_ERROR +from prometheus_client import start_http_server log_local = get_logger('email') @@ -185,13 +187,16 @@ def worker(workq: multiprocessing.Queue): html = html_content(msg.value) send(smtp_server, recipients, subject, html) log_local.info(f'sent: {msg} to {recipients}') + NOTIFICATIONS_SENT.labels('email', r_name, ','.join(recipients)).inc() except smtplib.SMTPException as ex: log_local.error(f'Failed sending email due to SMTP error: {ex}') log_local.warning('Reconnecting to SMTP server...') smtp_server = get_smtp_server() log_local.warning('Reconnecting to SMTP server... done.') + NOTIFICATIONS_ERROR.labels('email', r_name, ','.join(recipients), type(ex)).inc() except Exception as ex: log_local.error(f'Failed sending email: {ex}') + NOTIFICATIONS_ERROR.labels('email', r_name, ','.join(recipients), type(ex)).inc() def email_template(template_file=EMAIL_TEMPLATE_DEFAULT_FILE): @@ -207,4 +212,5 @@ def init_email_templates(default=EMAIL_TEMPLATE_DEFAULT_FILE, if __name__ == "__main__": init_email_templates() set_smtp_params() + start_http_server(9128) main(worker, KAFKA_TOPIC, KAFKA_GROUP_ID) diff --git a/src/notify-slack.py b/src/notify-slack.py index 0591585..37a91e8 100755 --- a/src/notify-slack.py +++ b/src/notify-slack.py @@ -9,6 +9,7 @@ from notify_deps import get_logger, timestamp_convert, main from notify_deps import NUVLA_ENDPOINT +from prometheus_client import start_http_server KAFKA_TOPIC = os.environ.get('KAFKA_TOPIC') or 'NOTIFICATIONS_SLACK_S' KAFKA_GROUP_ID = 'nuvla-notification-slack' @@ -132,4 +133,5 @@ def worker(workq: multiprocessing.Queue): if __name__ == "__main__": + start_http_server(9129) main(worker, KAFKA_TOPIC, KAFKA_GROUP_ID) From 814e876d10f49c6a4f286c84f8ec330cfff59938 Mon Sep 17 00:00:00 2001 From: Amit Bhanja Date: Wed, 7 Feb 2024 11:16:51 +0100 Subject: [PATCH 02/20] Added prometheus client -- Defined metrics --- src/notify-email.py | 2 +- src/notify-slack.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/notify-email.py b/src/notify-email.py index 046e83f..80eabf5 100755 --- a/src/notify-email.py +++ b/src/notify-email.py @@ -12,7 +12,7 @@ from notify_deps import get_logger, timestamp_convert, main from notify_deps import NUVLA_ENDPOINT -from metrics import NOTIFICATIONS_SENT, NOTIFICATIONS_ERROR +from src.metrics import NOTIFICATIONS_SENT, NOTIFICATIONS_ERROR from prometheus_client import start_http_server diff --git a/src/notify-slack.py b/src/notify-slack.py index 37a91e8..b6eacff 100755 --- a/src/notify-slack.py +++ b/src/notify-slack.py @@ -9,6 +9,7 @@ from notify_deps import get_logger, timestamp_convert, main from notify_deps import NUVLA_ENDPOINT +from src.metrics import NOTIFICATIONS_SENT, NOTIFICATIONS_ERROR from prometheus_client import start_http_server KAFKA_TOPIC = os.environ.get('KAFKA_TOPIC') or 'NOTIFICATIONS_SLACK_S' @@ -128,7 +129,9 @@ def worker(workq: multiprocessing.Queue): resp = send_message(dest, message_content(msg.value)) if not resp.ok: log_local.error(f'Failed sending {msg} to {dest}: {resp.text}') + NOTIFICATIONS_ERROR.labels('slack', msg.value['SUBS_NAME'], dest, resp.text).inc() else: + NOTIFICATIONS_SENT.labels('slack', msg.value['SUBS_NAME'], dest).inc() log_local.info(f'sent: {msg} to {dest}') From d8fa2f6a97cd995645e749684d64eaec5a46e839 Mon Sep 17 00:00:00 2001 From: Amit Bhanja Date: Wed, 7 Feb 2024 11:16:51 +0100 Subject: [PATCH 03/20] Added prometheus client -- Defined metrics --- src/metrics.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/metrics.py b/src/metrics.py index a942b01..2acc8fa 100644 --- a/src/metrics.py +++ b/src/metrics.py @@ -1,6 +1,8 @@ from prometheus_client import Counter -NOTIFICATIONS_SENT = Counter('notifications_sent', 'Number of notifications sent', +NAMESPACE = 'kafka_notify' + +NOTIFICATIONS_SENT = Counter(f'{NAMESPACE}_notifications_sent', 'Number of notifications sent', ['type', 'subscription_name', 'endpoint']) -NOTIFICATIONS_ERROR = Counter('notifications_error', 'Number of notifications that could not be sent due to error', +NOTIFICATIONS_ERROR = Counter(f'{NAMESPACE}_notifications_error', 'Number of notifications that could not be sent due to error', ['type', 'subscription_id', 'endpoint', 'exception']) From 5acc50f7b56e8c0337aa8178d86a300ee7ed845b Mon Sep 17 00:00:00 2001 From: Amit Bhanja Date: Wed, 7 Feb 2024 11:16:51 +0100 Subject: [PATCH 04/20] Added prometheus client -- Defined metrics --- src/metrics.py | 9 +++++++-- src/notify-email.py | 6 +++++- src/notify-slack.py | 5 ++++- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/metrics.py b/src/metrics.py index 2acc8fa..7eaf945 100644 --- a/src/metrics.py +++ b/src/metrics.py @@ -1,8 +1,13 @@ -from prometheus_client import Counter +from prometheus_client import Counter, Enum NAMESPACE = 'kafka_notify' +PROCESS_STATES = Enum(f'{NAMESPACE}_process_states', 'State of the process', states=['idle', 'processing', + 'error - recoverable', 'error - ' + 'need restart']) + NOTIFICATIONS_SENT = Counter(f'{NAMESPACE}_notifications_sent', 'Number of notifications sent', ['type', 'subscription_name', 'endpoint']) -NOTIFICATIONS_ERROR = Counter(f'{NAMESPACE}_notifications_error', 'Number of notifications that could not be sent due to error', +NOTIFICATIONS_ERROR = Counter(f'{NAMESPACE}_notifications_error', + 'Number of notifications that could not be sent due to error', ['type', 'subscription_id', 'endpoint', 'exception']) diff --git a/src/notify-email.py b/src/notify-email.py index 80eabf5..57df861 100755 --- a/src/notify-email.py +++ b/src/notify-email.py @@ -12,7 +12,7 @@ from notify_deps import get_logger, timestamp_convert, main from notify_deps import NUVLA_ENDPOINT -from src.metrics import NOTIFICATIONS_SENT, NOTIFICATIONS_ERROR +from src.metrics import NOTIFICATIONS_SENT, NOTIFICATIONS_ERROR, PROCESS_STATES from prometheus_client import start_http_server @@ -174,7 +174,9 @@ def get_recipients(v: dict): def worker(workq: multiprocessing.Queue): smtp_server = get_smtp_server() while True: + PROCESS_STATES.state('idle') msg = workq.get() + PROCESS_STATES.state('processing') if msg: recipients = get_recipients(msg.value) if len(recipients) == 0: @@ -194,9 +196,11 @@ def worker(workq: multiprocessing.Queue): smtp_server = get_smtp_server() log_local.warning('Reconnecting to SMTP server... done.') NOTIFICATIONS_ERROR.labels('email', r_name, ','.join(recipients), type(ex)).inc() + PROCESS_STATES.state('error - recoverable') except Exception as ex: log_local.error(f'Failed sending email: {ex}') NOTIFICATIONS_ERROR.labels('email', r_name, ','.join(recipients), type(ex)).inc() + PROCESS_STATES.state('error - recoverable') def email_template(template_file=EMAIL_TEMPLATE_DEFAULT_FILE): diff --git a/src/notify-slack.py b/src/notify-slack.py index b6eacff..eb173f1 100755 --- a/src/notify-slack.py +++ b/src/notify-slack.py @@ -9,7 +9,7 @@ from notify_deps import get_logger, timestamp_convert, main from notify_deps import NUVLA_ENDPOINT -from src.metrics import NOTIFICATIONS_SENT, NOTIFICATIONS_ERROR +from src.metrics import NOTIFICATIONS_SENT, NOTIFICATIONS_ERROR, PROCESS_STATES from prometheus_client import start_http_server KAFKA_TOPIC = os.environ.get('KAFKA_TOPIC') or 'NOTIFICATIONS_SLACK_S' @@ -123,12 +123,15 @@ def send_message(dest, message): def worker(workq: multiprocessing.Queue): while True: + PROCESS_STATES.state('idle') msg = workq.get() + PROCESS_STATES.state('processing') if msg: dest = msg.value['DESTINATION'] resp = send_message(dest, message_content(msg.value)) if not resp.ok: log_local.error(f'Failed sending {msg} to {dest}: {resp.text}') + PROCESS_STATES.state('error - recoverable') NOTIFICATIONS_ERROR.labels('slack', msg.value['SUBS_NAME'], dest, resp.text).inc() else: NOTIFICATIONS_SENT.labels('slack', msg.value['SUBS_NAME'], dest).inc() From c05bb7994a03a2fb0437228d9d8dfffbc74cb1f2 Mon Sep 17 00:00:00 2001 From: Amit Bhanja Date: Wed, 7 Feb 2024 11:16:51 +0100 Subject: [PATCH 05/20] Added prometheus client -- Defined metrics --- src/notify-email.py | 2 +- src/notify-slack.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/notify-email.py b/src/notify-email.py index 57df861..ce8ad35 100755 --- a/src/notify-email.py +++ b/src/notify-email.py @@ -12,7 +12,7 @@ from notify_deps import get_logger, timestamp_convert, main from notify_deps import NUVLA_ENDPOINT -from src.metrics import NOTIFICATIONS_SENT, NOTIFICATIONS_ERROR, PROCESS_STATES +from metrics import NOTIFICATIONS_SENT, NOTIFICATIONS_ERROR, PROCESS_STATES from prometheus_client import start_http_server diff --git a/src/notify-slack.py b/src/notify-slack.py index eb173f1..7e1b6ac 100755 --- a/src/notify-slack.py +++ b/src/notify-slack.py @@ -9,7 +9,7 @@ from notify_deps import get_logger, timestamp_convert, main from notify_deps import NUVLA_ENDPOINT -from src.metrics import NOTIFICATIONS_SENT, NOTIFICATIONS_ERROR, PROCESS_STATES +from metrics import NOTIFICATIONS_SENT, NOTIFICATIONS_ERROR, PROCESS_STATES from prometheus_client import start_http_server KAFKA_TOPIC = os.environ.get('KAFKA_TOPIC') or 'NOTIFICATIONS_SLACK_S' From 7ae6f1fa37c74f32b9b07174912e39d96e6328c3 Mon Sep 17 00:00:00 2001 From: Amit Bhanja Date: Wed, 7 Feb 2024 11:16:51 +0100 Subject: [PATCH 06/20] Added prometheus client -- Defined metrics --- tests/metrics.py | 1 + 1 file changed, 1 insertion(+) create mode 120000 tests/metrics.py diff --git a/tests/metrics.py b/tests/metrics.py new file mode 120000 index 0000000..bcc1ec8 --- /dev/null +++ b/tests/metrics.py @@ -0,0 +1 @@ +../src/metrics.py \ No newline at end of file From 7ce930774c296c4dfefa09c40633c76ac6910841 Mon Sep 17 00:00:00 2001 From: Amit Bhanja Date: Wed, 7 Feb 2024 11:16:51 +0100 Subject: [PATCH 07/20] Added prometheus client -- Defined metrics --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 00c8c38..f6f1d6f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ urllib3==1.26.2 Jinja2==2.11.2 # To fix https://github.com/aws/aws-sam-cli/issues/3661 markupsafe==2.0.1 +prometheus_client==0.19.0 \ No newline at end of file From 3e522fba07cf4630804c68a69dc29aacf1641997 Mon Sep 17 00:00:00 2001 From: Amit Bhanja Date: Wed, 7 Feb 2024 11:16:51 +0100 Subject: [PATCH 08/20] Added prometheus client -- Defined metrics --- src/notify-email.py | 4 ++-- src/notify-slack.py | 4 ++-- src/notify_deps.py | 8 ++++++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/notify-email.py b/src/notify-email.py index ce8ad35..6af199a 100755 --- a/src/notify-email.py +++ b/src/notify-email.py @@ -11,7 +11,7 @@ from datetime import datetime from notify_deps import get_logger, timestamp_convert, main -from notify_deps import NUVLA_ENDPOINT +from notify_deps import NUVLA_ENDPOINT, prometheus_exporter_port from metrics import NOTIFICATIONS_SENT, NOTIFICATIONS_ERROR, PROCESS_STATES from prometheus_client import start_http_server @@ -216,5 +216,5 @@ def init_email_templates(default=EMAIL_TEMPLATE_DEFAULT_FILE, if __name__ == "__main__": init_email_templates() set_smtp_params() - start_http_server(9128) + start_http_server(prometheus_exporter_port()) main(worker, KAFKA_TOPIC, KAFKA_GROUP_ID) diff --git a/src/notify-slack.py b/src/notify-slack.py index 7e1b6ac..c27ad77 100755 --- a/src/notify-slack.py +++ b/src/notify-slack.py @@ -8,7 +8,7 @@ import re from notify_deps import get_logger, timestamp_convert, main -from notify_deps import NUVLA_ENDPOINT +from notify_deps import NUVLA_ENDPOINT, prometheus_exporter_port from metrics import NOTIFICATIONS_SENT, NOTIFICATIONS_ERROR, PROCESS_STATES from prometheus_client import start_http_server @@ -139,5 +139,5 @@ def worker(workq: multiprocessing.Queue): if __name__ == "__main__": - start_http_server(9129) + start_http_server(prometheus_exporter_port()) main(worker, KAFKA_TOPIC, KAFKA_GROUP_ID) diff --git a/src/notify_deps.py b/src/notify_deps.py index e6d9802..3340419 100644 --- a/src/notify_deps.py +++ b/src/notify_deps.py @@ -8,7 +8,6 @@ from datetime import datetime from kafka import KafkaConsumer - log_formatter = logging.Formatter( '%(asctime)s - %(name)s - %(process)d - %(levelname)s - %(message)s') stdout_handler = logging.StreamHandler(sys.stdout) @@ -37,6 +36,8 @@ def get_logger(who): work_queue = multiprocessing.Queue() +DEFAULT_PROMETHEUS_EXPORTER_PORT = 9140 + def kafka_consumer(topic, bootstrap_servers, group_id, auto_offset_reset='latest'): consumer = KafkaConsumer(topic, @@ -54,6 +55,10 @@ def timestamp_convert(ts): strftime('%Y-%m-%d %H:%M:%S UTC') +def prometheus_exporter_port(): + return int(os.environ.get('PROMETHEUS_EXPORTER_PORT', DEFAULT_PROMETHEUS_EXPORTER_PORT)) + + def main(worker, kafka_topic, group_id): pool = multiprocessing.Pool(5, worker, (work_queue,)) for msg in kafka_consumer(kafka_topic, KAFKA_BOOTSTRAP_SERVERS, group_id=group_id): @@ -68,4 +73,3 @@ def main(worker, kafka_topic, group_id): work_queue.join_thread() pool.close() pool.join() - From 5cd8a9b1080b935a65fd3d5597d30c85a3350e5c Mon Sep 17 00:00:00 2001 From: Amit Bhanja Date: Wed, 7 Feb 2024 11:16:51 +0100 Subject: [PATCH 09/20] Added prometheus client -- Defined metrics --- Dockerfile | 2 ++ src/metrics.py | 13 +++++++------ src/notify-email.py | 9 +++++++-- src/notify-slack.py | 6 ++++-- 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index 99b6010..83a3402 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,8 +9,10 @@ RUN pip3 install --prefix=/install -r /requirements.txt FROM base COPY --from=builder /install /usr/local COPY src/notify* /app/ +COPY src/metrics.py /app/ COPY src/run.sh /app/ COPY src/templates /app/templates WORKDIR /app +EXPOSE 9140 ENTRYPOINT [ "./run.sh" ] diff --git a/src/metrics.py b/src/metrics.py index 7eaf945..1cacbfb 100644 --- a/src/metrics.py +++ b/src/metrics.py @@ -2,12 +2,13 @@ NAMESPACE = 'kafka_notify' -PROCESS_STATES = Enum(f'{NAMESPACE}_process_states', 'State of the process', states=['idle', 'processing', +PROCESS_STATES = Enum('process_states', 'State of the process', states=['idle', 'processing', 'error - recoverable', 'error - ' - 'need restart']) + 'need restart'] + , namespace=NAMESPACE) -NOTIFICATIONS_SENT = Counter(f'{NAMESPACE}_notifications_sent', 'Number of notifications sent', - ['type', 'subscription_name', 'endpoint']) -NOTIFICATIONS_ERROR = Counter(f'{NAMESPACE}_notifications_error', +NOTIFICATIONS_SENT = Counter('notifications_sent', 'Number of notifications sent', + ['type', 'name', 'endpoint'], namespace=NAMESPACE) +NOTIFICATIONS_ERROR = Counter('notifications_error', 'Number of notifications that could not be sent due to error', - ['type', 'subscription_id', 'endpoint', 'exception']) + ['type', 'name', 'endpoint', 'exception'], namespace=NAMESPACE) diff --git a/src/notify-email.py b/src/notify-email.py index 6af199a..a55b2ef 100755 --- a/src/notify-email.py +++ b/src/notify-email.py @@ -12,8 +12,8 @@ from notify_deps import get_logger, timestamp_convert, main from notify_deps import NUVLA_ENDPOINT, prometheus_exporter_port -from metrics import NOTIFICATIONS_SENT, NOTIFICATIONS_ERROR, PROCESS_STATES -from prometheus_client import start_http_server +from metrics import NOTIFICATIONS_SENT, NOTIFICATIONS_ERROR, PROCESS_STATES, NAMESPACE +from prometheus_client import start_http_server, PROCESS_COLLECTOR, REGISTRY, ProcessCollector log_local = get_logger('email') @@ -216,5 +216,10 @@ def init_email_templates(default=EMAIL_TEMPLATE_DEFAULT_FILE, if __name__ == "__main__": init_email_templates() set_smtp_params() +<<<<<<< HEAD +======= + REGISTRY.unregister(PROCESS_COLLECTOR) + ProcessCollector(namespace=NAMESPACE) +>>>>>>> 2164fe2 (Added prometheus client) start_http_server(prometheus_exporter_port()) main(worker, KAFKA_TOPIC, KAFKA_GROUP_ID) diff --git a/src/notify-slack.py b/src/notify-slack.py index c27ad77..5149d1d 100755 --- a/src/notify-slack.py +++ b/src/notify-slack.py @@ -9,8 +9,8 @@ from notify_deps import get_logger, timestamp_convert, main from notify_deps import NUVLA_ENDPOINT, prometheus_exporter_port -from metrics import NOTIFICATIONS_SENT, NOTIFICATIONS_ERROR, PROCESS_STATES -from prometheus_client import start_http_server +from metrics import NOTIFICATIONS_SENT, NOTIFICATIONS_ERROR, PROCESS_STATES, NAMESPACE +from prometheus_client import start_http_server, PROCESS_COLLECTOR, REGISTRY, ProcessCollector KAFKA_TOPIC = os.environ.get('KAFKA_TOPIC') or 'NOTIFICATIONS_SLACK_S' KAFKA_GROUP_ID = 'nuvla-notification-slack' @@ -139,5 +139,7 @@ def worker(workq: multiprocessing.Queue): if __name__ == "__main__": + REGISTRY.unregister(PROCESS_COLLECTOR) + ProcessCollector(namespace=NAMESPACE) start_http_server(prometheus_exporter_port()) main(worker, KAFKA_TOPIC, KAFKA_GROUP_ID) From fe42b644c7e0bd03643cd39bed3cba81fc640738 Mon Sep 17 00:00:00 2001 From: Amit Bhanja Date: Wed, 7 Feb 2024 11:16:51 +0100 Subject: [PATCH 10/20] Added prometheus client -- Defined metrics --- src/notify-email.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/notify-email.py b/src/notify-email.py index a55b2ef..199946f 100755 --- a/src/notify-email.py +++ b/src/notify-email.py @@ -216,10 +216,7 @@ def init_email_templates(default=EMAIL_TEMPLATE_DEFAULT_FILE, if __name__ == "__main__": init_email_templates() set_smtp_params() -<<<<<<< HEAD -======= REGISTRY.unregister(PROCESS_COLLECTOR) ProcessCollector(namespace=NAMESPACE) ->>>>>>> 2164fe2 (Added prometheus client) start_http_server(prometheus_exporter_port()) main(worker, KAFKA_TOPIC, KAFKA_GROUP_ID) From 162bc1cab5f9e09535447a3ba1844a09810ebe0e Mon Sep 17 00:00:00 2001 From: Amit Bhanja Date: Wed, 7 Feb 2024 11:16:51 +0100 Subject: [PATCH 11/20] Added prometheus client -- Defined metrics --- Dockerfile | 2 ++ src/notify-email.py | 29 ++++++++++++++++++++++++----- src/notify-slack.py | 27 +++++++++++++++++++++++---- 3 files changed, 49 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 83a3402..58734c5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,6 +12,8 @@ COPY src/notify* /app/ COPY src/metrics.py /app/ COPY src/run.sh /app/ COPY src/templates /app/templates +RUN mkdir /app/prom_data +ENV PROMETHEUS_MULTIPROC_DIR /app/prom_data WORKDIR /app EXPOSE 9140 diff --git a/src/notify-email.py b/src/notify-email.py index 199946f..9fce4e8 100755 --- a/src/notify-email.py +++ b/src/notify-email.py @@ -9,12 +9,12 @@ from email.mime.text import MIMEText from jinja2 import Template from datetime import datetime +import shutil from notify_deps import get_logger, timestamp_convert, main from notify_deps import NUVLA_ENDPOINT, prometheus_exporter_port -from metrics import NOTIFICATIONS_SENT, NOTIFICATIONS_ERROR, PROCESS_STATES, NAMESPACE -from prometheus_client import start_http_server, PROCESS_COLLECTOR, REGISTRY, ProcessCollector - +from prometheus_client import start_http_server, multiprocess, CollectorRegistry +from prometheus_client import Counter, Enum log_local = get_logger('email') @@ -33,6 +33,27 @@ IMG_ALERT_OK = 'ui/images/nuvla-alert-ok.png' IMG_ALERT_NOK = 'ui/images/nuvla-alert-nok.png' +path = os.environ.get('PROMETHEUS_MULTIPROC_DIR') +if os.path.exists(path): + shutil.rmtree(path) + os.mkdir(path) + +registry = CollectorRegistry() +multiprocess.MultiProcessCollector(registry) + +PROCESS_STATES = Enum('process_states', 'State of the process', states=['idle', 'processing', + 'error - recoverable', 'error - ' + 'need restart'] + , namespace='kafka_notify', registry=registry) + +NOTIFICATIONS_SENT = Counter('notifications_sent', 'Number of notifications sent', + ['type', 'name', 'endpoint'], namespace='kafka_notify', + registry=registry) +NOTIFICATIONS_ERROR = Counter('notifications_error', + 'Number of notifications that could not be sent due to error', + ['type', 'name', 'endpoint', 'exception'], namespace='kafka_notify', + registry=registry) + class SendFailedMaxAttempts(Exception): pass @@ -216,7 +237,5 @@ def init_email_templates(default=EMAIL_TEMPLATE_DEFAULT_FILE, if __name__ == "__main__": init_email_templates() set_smtp_params() - REGISTRY.unregister(PROCESS_COLLECTOR) - ProcessCollector(namespace=NAMESPACE) start_http_server(prometheus_exporter_port()) main(worker, KAFKA_TOPIC, KAFKA_GROUP_ID) diff --git a/src/notify-slack.py b/src/notify-slack.py index 5149d1d..9e593b3 100755 --- a/src/notify-slack.py +++ b/src/notify-slack.py @@ -6,11 +6,12 @@ import requests import os import re +import shutil from notify_deps import get_logger, timestamp_convert, main from notify_deps import NUVLA_ENDPOINT, prometheus_exporter_port -from metrics import NOTIFICATIONS_SENT, NOTIFICATIONS_ERROR, PROCESS_STATES, NAMESPACE -from prometheus_client import start_http_server, PROCESS_COLLECTOR, REGISTRY, ProcessCollector +from prometheus_client import start_http_server, multiprocess, CollectorRegistry +from prometheus_client import Counter, Enum KAFKA_TOPIC = os.environ.get('KAFKA_TOPIC') or 'NOTIFICATIONS_SLACK_S' KAFKA_GROUP_ID = 'nuvla-notification-slack' @@ -23,6 +24,26 @@ COLOR_OK = "#2C9442" COLOR_NOK = "#B70B0B" +path = os.environ.get('PROMETHEUS_MULTIPROC_DIR') +if os.path.exists(path): + shutil.rmtree(path) + os.mkdir(path) + +registry = CollectorRegistry() +multiprocess.MultiProcessCollector(registry) + +PROCESS_STATES = Enum('process_states', 'State of the process', states=['idle', 'processing', + 'error - recoverable', 'error - ' + 'need restart'] + , namespace='kafka_notify', registry=registry) + +NOTIFICATIONS_SENT = Counter('notifications_sent', 'Number of notifications sent', + ['type', 'name', 'endpoint'], namespace='kafka_notify', + registry=registry) +NOTIFICATIONS_ERROR = Counter('notifications_error', + 'Number of notifications that could not be sent due to error', + ['type', 'name', 'endpoint', 'exception'], namespace='kafka_notify', + registry=registry) def now_timestamp(): return datetime.now().timestamp() @@ -139,7 +160,5 @@ def worker(workq: multiprocessing.Queue): if __name__ == "__main__": - REGISTRY.unregister(PROCESS_COLLECTOR) - ProcessCollector(namespace=NAMESPACE) start_http_server(prometheus_exporter_port()) main(worker, KAFKA_TOPIC, KAFKA_GROUP_ID) From ae4b745f833361b5983eac7be02e7bbdd6f20057 Mon Sep 17 00:00:00 2001 From: Amit Bhanja Date: Wed, 7 Feb 2024 11:16:51 +0100 Subject: [PATCH 12/20] Added prometheus client -- Defined metrics --- src/notify-slack.py | 19 ++++++++++--------- tests/test_notify_email.py | 9 +++++++++ tests/test_notify_slack.py | 11 +++++++++++ 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/src/notify-slack.py b/src/notify-slack.py index 9e593b3..a6edcc5 100755 --- a/src/notify-slack.py +++ b/src/notify-slack.py @@ -45,6 +45,7 @@ ['type', 'name', 'endpoint', 'exception'], namespace='kafka_notify', registry=registry) + def now_timestamp(): return datetime.now().timestamp() @@ -124,15 +125,15 @@ def message_content(msg_params: dict): ) attachments = [{ - 'color': color, - 'author_name': 'Nuvla.io', - 'author_link': 'https://nuvla.io', - 'author_icon': 'https://sixsq.com/assets/img/logo-sixsq.svg', - 'fields': fields, - 'footer': 'https://sixsq.com', - 'footer_icon': 'https://sixsq.com/assets/img/logo-sixsq.svg', - 'ts': now_timestamp() - } + 'color': color, + 'author_name': 'Nuvla.io', + 'author_link': 'https://nuvla.io', + 'author_icon': 'https://sixsq.com/assets/img/logo-sixsq.svg', + 'fields': fields, + 'footer': 'https://sixsq.com', + 'footer_icon': 'https://sixsq.com/assets/img/logo-sixsq.svg', + 'ts': now_timestamp() + } ] return {'attachments': attachments} diff --git a/tests/test_notify_email.py b/tests/test_notify_email.py index 027e568..c88cd4f 100644 --- a/tests/test_notify_email.py +++ b/tests/test_notify_email.py @@ -1,6 +1,15 @@ import os import unittest +from unittest.mock import Mock +import shutil +from prometheus_client import multiprocess +os.environ['PROMETHEUS_MULTIPROC_DIR'] = '/tmp/test' +os.path.exists = Mock(return_value=True) +os.mkdir = Mock() +shutil.rmtree = Mock() + +multiprocess.MultiProcessCollector = Mock() import notify_email from notify_email import get_recipients, html_content, email_template diff --git a/tests/test_notify_slack.py b/tests/test_notify_slack.py index a9ee83b..52803de 100644 --- a/tests/test_notify_slack.py +++ b/tests/test_notify_slack.py @@ -1,4 +1,15 @@ import unittest +import os +from unittest.mock import Mock +import shutil +from prometheus_client import multiprocess + +os.environ['PROMETHEUS_MULTIPROC_DIR'] = '/tmp/test' +os.path.exists = Mock(return_value=True) +os.mkdir = Mock() +shutil.rmtree = Mock() + +multiprocess.MultiProcessCollector = Mock() from notify_slack import now_timestamp, message_content From 7c4eeec956199cc50844509aac8047883b76574b Mon Sep 17 00:00:00 2001 From: Amit Bhanja Date: Wed, 7 Feb 2024 11:16:51 +0100 Subject: [PATCH 13/20] Added prometheus client -- Defined metrics --- src/metrics.py | 23 +++++++++++++++++------ src/notify-email.py | 27 +++------------------------ src/notify-slack.py | 26 +++----------------------- tests/test_notify_email.py | 2 +- tests/test_notify_slack.py | 2 +- 5 files changed, 25 insertions(+), 55 deletions(-) diff --git a/src/metrics.py b/src/metrics.py index 1cacbfb..aa4c5f0 100644 --- a/src/metrics.py +++ b/src/metrics.py @@ -1,14 +1,25 @@ +import os +import shutil +from prometheus_client import multiprocess, CollectorRegistry from prometheus_client import Counter, Enum -NAMESPACE = 'kafka_notify' +path = os.environ.get('PROMETHEUS_MULTIPROC_DIR') +if os.path.exists(path): + shutil.rmtree(path) + os.mkdir(path) + +registry = CollectorRegistry() +multiprocess.MultiProcessCollector(registry) PROCESS_STATES = Enum('process_states', 'State of the process', states=['idle', 'processing', - 'error - recoverable', 'error - ' - 'need restart'] - , namespace=NAMESPACE) + 'error - recoverable', 'error - ' + 'need restart'] + , namespace='kafka_notify', registry=registry) NOTIFICATIONS_SENT = Counter('notifications_sent', 'Number of notifications sent', - ['type', 'name', 'endpoint'], namespace=NAMESPACE) + ['type', 'name', 'endpoint'], namespace='kafka_notify', + registry=registry) NOTIFICATIONS_ERROR = Counter('notifications_error', 'Number of notifications that could not be sent due to error', - ['type', 'name', 'endpoint', 'exception'], namespace=NAMESPACE) + ['type', 'name', 'endpoint', 'exception'], namespace='kafka_notify', + registry=registry) diff --git a/src/notify-email.py b/src/notify-email.py index 9fce4e8..5483f09 100755 --- a/src/notify-email.py +++ b/src/notify-email.py @@ -13,8 +13,8 @@ from notify_deps import get_logger, timestamp_convert, main from notify_deps import NUVLA_ENDPOINT, prometheus_exporter_port -from prometheus_client import start_http_server, multiprocess, CollectorRegistry -from prometheus_client import Counter, Enum +from prometheus_client import start_http_server +from metrics import PROCESS_STATES, NOTIFICATIONS_SENT, NOTIFICATIONS_ERROR, registry log_local = get_logger('email') @@ -33,27 +33,6 @@ IMG_ALERT_OK = 'ui/images/nuvla-alert-ok.png' IMG_ALERT_NOK = 'ui/images/nuvla-alert-nok.png' -path = os.environ.get('PROMETHEUS_MULTIPROC_DIR') -if os.path.exists(path): - shutil.rmtree(path) - os.mkdir(path) - -registry = CollectorRegistry() -multiprocess.MultiProcessCollector(registry) - -PROCESS_STATES = Enum('process_states', 'State of the process', states=['idle', 'processing', - 'error - recoverable', 'error - ' - 'need restart'] - , namespace='kafka_notify', registry=registry) - -NOTIFICATIONS_SENT = Counter('notifications_sent', 'Number of notifications sent', - ['type', 'name', 'endpoint'], namespace='kafka_notify', - registry=registry) -NOTIFICATIONS_ERROR = Counter('notifications_error', - 'Number of notifications that could not be sent due to error', - ['type', 'name', 'endpoint', 'exception'], namespace='kafka_notify', - registry=registry) - class SendFailedMaxAttempts(Exception): pass @@ -237,5 +216,5 @@ def init_email_templates(default=EMAIL_TEMPLATE_DEFAULT_FILE, if __name__ == "__main__": init_email_templates() set_smtp_params() - start_http_server(prometheus_exporter_port()) + start_http_server(prometheus_exporter_port(), registry=registry) main(worker, KAFKA_TOPIC, KAFKA_GROUP_ID) diff --git a/src/notify-slack.py b/src/notify-slack.py index a6edcc5..2e3d454 100755 --- a/src/notify-slack.py +++ b/src/notify-slack.py @@ -10,8 +10,8 @@ from notify_deps import get_logger, timestamp_convert, main from notify_deps import NUVLA_ENDPOINT, prometheus_exporter_port -from prometheus_client import start_http_server, multiprocess, CollectorRegistry -from prometheus_client import Counter, Enum +from prometheus_client import start_http_server +from metrics import PROCESS_STATES, NOTIFICATIONS_SENT, NOTIFICATIONS_ERROR, registry KAFKA_TOPIC = os.environ.get('KAFKA_TOPIC') or 'NOTIFICATIONS_SLACK_S' KAFKA_GROUP_ID = 'nuvla-notification-slack' @@ -24,26 +24,6 @@ COLOR_OK = "#2C9442" COLOR_NOK = "#B70B0B" -path = os.environ.get('PROMETHEUS_MULTIPROC_DIR') -if os.path.exists(path): - shutil.rmtree(path) - os.mkdir(path) - -registry = CollectorRegistry() -multiprocess.MultiProcessCollector(registry) - -PROCESS_STATES = Enum('process_states', 'State of the process', states=['idle', 'processing', - 'error - recoverable', 'error - ' - 'need restart'] - , namespace='kafka_notify', registry=registry) - -NOTIFICATIONS_SENT = Counter('notifications_sent', 'Number of notifications sent', - ['type', 'name', 'endpoint'], namespace='kafka_notify', - registry=registry) -NOTIFICATIONS_ERROR = Counter('notifications_error', - 'Number of notifications that could not be sent due to error', - ['type', 'name', 'endpoint', 'exception'], namespace='kafka_notify', - registry=registry) def now_timestamp(): @@ -161,5 +141,5 @@ def worker(workq: multiprocessing.Queue): if __name__ == "__main__": - start_http_server(prometheus_exporter_port()) + start_http_server(prometheus_exporter_port(), registry=registry) main(worker, KAFKA_TOPIC, KAFKA_GROUP_ID) diff --git a/tests/test_notify_email.py b/tests/test_notify_email.py index c88cd4f..7dba962 100644 --- a/tests/test_notify_email.py +++ b/tests/test_notify_email.py @@ -4,7 +4,7 @@ import shutil from prometheus_client import multiprocess -os.environ['PROMETHEUS_MULTIPROC_DIR'] = '/tmp/test' +os.environ['PROMETHEUS_MULTIPROC_DIR'] = '' os.path.exists = Mock(return_value=True) os.mkdir = Mock() shutil.rmtree = Mock() diff --git a/tests/test_notify_slack.py b/tests/test_notify_slack.py index 52803de..c37cb01 100644 --- a/tests/test_notify_slack.py +++ b/tests/test_notify_slack.py @@ -4,7 +4,7 @@ import shutil from prometheus_client import multiprocess -os.environ['PROMETHEUS_MULTIPROC_DIR'] = '/tmp/test' +os.environ['PROMETHEUS_MULTIPROC_DIR'] = '' os.path.exists = Mock(return_value=True) os.mkdir = Mock() shutil.rmtree = Mock() From 57111a95aaaf177ef75e3c0bf04c16c2c78855ab Mon Sep 17 00:00:00 2001 From: Amit Bhanja Date: Wed, 7 Feb 2024 11:16:51 +0100 Subject: [PATCH 14/20] Added prometheus client -- Defined metrics --- src/notify-email.py | 2 +- src/notify-slack.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/notify-email.py b/src/notify-email.py index 5483f09..16fab3d 100755 --- a/src/notify-email.py +++ b/src/notify-email.py @@ -189,7 +189,7 @@ def worker(workq: multiprocessing.Queue): html = html_content(msg.value) send(smtp_server, recipients, subject, html) log_local.info(f'sent: {msg} to {recipients}') - NOTIFICATIONS_SENT.labels('email', r_name, ','.join(recipients)).inc() + NOTIFICATIONS_SENT.labels('email', f'{r_name or r_id}', ','.join(recipients)).inc() except smtplib.SMTPException as ex: log_local.error(f'Failed sending email due to SMTP error: {ex}') log_local.warning('Reconnecting to SMTP server...') diff --git a/src/notify-slack.py b/src/notify-slack.py index 2e3d454..392cae2 100755 --- a/src/notify-slack.py +++ b/src/notify-slack.py @@ -134,9 +134,12 @@ def worker(workq: multiprocessing.Queue): if not resp.ok: log_local.error(f'Failed sending {msg} to {dest}: {resp.text}') PROCESS_STATES.state('error - recoverable') - NOTIFICATIONS_ERROR.labels('slack', msg.value['SUBS_NAME'], dest, resp.text).inc() + NOTIFICATIONS_ERROR.labels('slack', + f'{msg.value.get("NAME") or msg.value["RESOURCE_ID"] or msg.value["SUBS_NAME"]}' + , dest, resp.text).inc() else: - NOTIFICATIONS_SENT.labels('slack', msg.value['SUBS_NAME'], dest).inc() + NOTIFICATIONS_SENT.labels('slack', f'{msg.value.get("NAME") or msg.value["RESOURCE_ID"] or msg.value["SUBS_NAME"]}' + , dest).inc() log_local.info(f'sent: {msg} to {dest}') From 2b76a9befc066ebdf6630c799de468ecb51b9d53 Mon Sep 17 00:00:00 2001 From: Amit Bhanja Date: Wed, 7 Feb 2024 11:16:51 +0100 Subject: [PATCH 15/20] Added prometheus client -- Defined metrics --- src/notify-slack.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/notify-slack.py b/src/notify-slack.py index 392cae2..41cc9e6 100755 --- a/src/notify-slack.py +++ b/src/notify-slack.py @@ -135,10 +135,10 @@ def worker(workq: multiprocessing.Queue): log_local.error(f'Failed sending {msg} to {dest}: {resp.text}') PROCESS_STATES.state('error - recoverable') NOTIFICATIONS_ERROR.labels('slack', - f'{msg.value.get("NAME") or msg.value["RESOURCE_ID"] or msg.value["SUBS_NAME"]}' + f'{msg.value.get("NAME") or msg.value["SUBS_NAME"]}' , dest, resp.text).inc() else: - NOTIFICATIONS_SENT.labels('slack', f'{msg.value.get("NAME") or msg.value["RESOURCE_ID"] or msg.value["SUBS_NAME"]}' + NOTIFICATIONS_SENT.labels('slack', f'{msg.value.get("NAME") or msg.value["SUBS_NAME"]}' , dest).inc() log_local.info(f'sent: {msg} to {dest}') From 731e4c5c0fd93046af478e06423c9b2c52ed6aa8 Mon Sep 17 00:00:00 2001 From: Konstantin Skaburskas Date: Wed, 14 Feb 2024 15:50:20 +0100 Subject: [PATCH 16/20] catch requests errors, log and produce error metrics --- src/notify-slack.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/notify-slack.py b/src/notify-slack.py index 41cc9e6..c32b678 100755 --- a/src/notify-slack.py +++ b/src/notify-slack.py @@ -130,16 +130,22 @@ def worker(workq: multiprocessing.Queue): PROCESS_STATES.state('processing') if msg: dest = msg.value['DESTINATION'] - resp = send_message(dest, message_content(msg.value)) + try: + resp = send_message(dest, message_content(msg.value)) + except requests.exceptions.RequestException as ex: + log_local.error(f'Failed sending {msg} to {dest}: {ex}') + PROCESS_STATES.state('error - recoverable') + NOTIFICATIONS_ERROR.labels('slack', f'{msg.value.get("NAME") or msg.value["SUBS_NAME"]}', + dest, type(ex)).inc() + continue if not resp.ok: log_local.error(f'Failed sending {msg} to {dest}: {resp.text}') PROCESS_STATES.state('error - recoverable') - NOTIFICATIONS_ERROR.labels('slack', - f'{msg.value.get("NAME") or msg.value["SUBS_NAME"]}' - , dest, resp.text).inc() + NOTIFICATIONS_ERROR.labels('slack', f'{msg.value.get("NAME") or msg.value["SUBS_NAME"]}', + dest, resp.text).inc() else: - NOTIFICATIONS_SENT.labels('slack', f'{msg.value.get("NAME") or msg.value["SUBS_NAME"]}' - , dest).inc() + NOTIFICATIONS_SENT.labels('slack', f'{msg.value.get("NAME") or msg.value["SUBS_NAME"]}', + dest).inc() log_local.info(f'sent: {msg} to {dest}') From b3d395e46f8452dbae658f8dad6b7010b0a8480a Mon Sep 17 00:00:00 2001 From: Konstantin Skaburskas Date: Mon, 19 Feb 2024 15:05:50 +0100 Subject: [PATCH 17/20] add changelog entry --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 58295bf..dda8f5b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +### Add + +- Expose internal business metrics for Prometheus scraping. + ## [0.7.1] - 2023-12-14 ### Changed From fcbc8f4315576183fd75dd84a4118c78ed87ebd6 Mon Sep 17 00:00:00 2001 From: Konstantin Skaburskas Date: Mon, 19 Feb 2024 15:15:54 +0100 Subject: [PATCH 18/20] only define multiprocessor collector when path is defined --- src/metrics.py | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/src/metrics.py b/src/metrics.py index aa4c5f0..d63b29b 100644 --- a/src/metrics.py +++ b/src/metrics.py @@ -3,23 +3,25 @@ from prometheus_client import multiprocess, CollectorRegistry from prometheus_client import Counter, Enum -path = os.environ.get('PROMETHEUS_MULTIPROC_DIR') -if os.path.exists(path): - shutil.rmtree(path) - os.mkdir(path) - registry = CollectorRegistry() -multiprocess.MultiProcessCollector(registry) -PROCESS_STATES = Enum('process_states', 'State of the process', states=['idle', 'processing', - 'error - recoverable', 'error - ' - 'need restart'] - , namespace='kafka_notify', registry=registry) +path = os.environ.get('PROMETHEUS_MULTIPROC_DIR') +if path: + multiprocess.MultiProcessCollector(registry, path=path) + if os.path.exists(path): + shutil.rmtree(path) + os.mkdir(path) + +PROCESS_STATES = Enum('process_states', + 'State of the process', + states=['idle', 'processing', 'error - recoverable', 'error - need restart'], + namespace='kafka_notify', registry=registry) -NOTIFICATIONS_SENT = Counter('notifications_sent', 'Number of notifications sent', - ['type', 'name', 'endpoint'], namespace='kafka_notify', - registry=registry) +NOTIFICATIONS_SENT = Counter('notifications_sent', + 'Number of notifications sent', + ['type', 'name', 'endpoint'], + namespace='kafka_notify', registry=registry) NOTIFICATIONS_ERROR = Counter('notifications_error', 'Number of notifications that could not be sent due to error', - ['type', 'name', 'endpoint', 'exception'], namespace='kafka_notify', - registry=registry) + ['type', 'name', 'endpoint', 'exception'], + namespace='kafka_notify', registry=registry) From d067ed959b688f8853266e23662fc043c24e7871 Mon Sep 17 00:00:00 2001 From: Konstantin Skaburskas Date: Mon, 19 Feb 2024 15:16:48 +0100 Subject: [PATCH 19/20] refactor sending of email loop; add SMTPServerDisconnected exception handling --- src/notify-email.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/notify-email.py b/src/notify-email.py index 16fab3d..0672ff7 100755 --- a/src/notify-email.py +++ b/src/notify-email.py @@ -9,7 +9,6 @@ from email.mime.text import MIMEText from jinja2 import Template from datetime import datetime -import shutil from notify_deps import get_logger, timestamp_convert, main from notify_deps import NUVLA_ENDPOINT, prometheus_exporter_port @@ -144,26 +143,29 @@ def html_content(msg_params: dict): return get_email_template(msg_params).render(**params) -def send(server: smtplib.SMTP, recipients, subject, html, attempts=SEND_EMAIL_ATTEMPTS): +def send(server: smtplib.SMTP, recipients, subject, html, attempts=SEND_EMAIL_ATTEMPTS, + sleep_interval=0.5): msg = MIMEMultipart('alternative') msg['Subject'] = subject msg['From'] = f'Nuvla <{server.user}>' msg['To'] = ', '.join(recipients) msg.attach(MIMEText(html, 'html', 'utf-8')) for i in range(attempts): - if i > 0: - log_local.warning(f'Failed sending email: retry {i}') - time.sleep(.5) - log_local.warning('Reconnecting to SMTP server...') - server = get_smtp_server() - log_local.warning('Reconnecting to SMTP server... done.') try: resp = server.sendmail(server.user, recipients, msg.as_string()) if resp: log_local.error(f'SMTP failed to deliver email to: {resp}') return + except smtplib.SMTPServerDisconnected: + if i < attempts - 1: # no need to sleep on the last iteration + time.sleep(sleep_interval) + log_local.warning('Reconnecting to SMTP server...') + server = get_smtp_server() + log_local.warning('Reconnecting to SMTP server... done.') except smtplib.SMTPException as ex: log_local.error(f'Failed sending email due to SMTP error: {ex}') + NOTIFICATIONS_ERROR.labels('email', subject, ','.join(recipients), type(ex)).inc() + PROCESS_STATES.state('error - recoverable') raise SendFailedMaxAttempts(f'Failed sending email after {attempts} attempts.') From ab67f0fb0a21a3c028822bfd9efaa4db5c3af00d Mon Sep 17 00:00:00 2001 From: Konstantin Skaburskas Date: Tue, 20 Feb 2024 08:45:46 +0100 Subject: [PATCH 20/20] remove unused import --- src/notify-slack.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/notify-slack.py b/src/notify-slack.py index c32b678..fdf6ea8 100755 --- a/src/notify-slack.py +++ b/src/notify-slack.py @@ -6,7 +6,6 @@ import requests import os import re -import shutil from notify_deps import get_logger, timestamp_convert, main from notify_deps import NUVLA_ENDPOINT, prometheus_exporter_port @@ -25,7 +24,6 @@ COLOR_NOK = "#B70B0B" - def now_timestamp(): return datetime.now().timestamp()