From aacdae5f20bf36745c0c8b3208f347b41f3eb7c4 Mon Sep 17 00:00:00 2001 From: Tommy Li Date: Mon, 7 Dec 2020 10:08:48 -0800 Subject: [PATCH 1/3] add email notification example --- samples/exit-handler-email/README.md | 45 ++++++++++++++ samples/exit-handler-email/component.yaml | 58 ++++++++++++++++++ samples/exit-handler-email/secret.yaml | 10 ++++ samples/exit-handler-email/send-email.py | 73 +++++++++++++++++++++++ 4 files changed, 186 insertions(+) create mode 100644 samples/exit-handler-email/README.md create mode 100644 samples/exit-handler-email/component.yaml create mode 100644 samples/exit-handler-email/secret.yaml create mode 100644 samples/exit-handler-email/send-email.py diff --git a/samples/exit-handler-email/README.md b/samples/exit-handler-email/README.md new file mode 100644 index 0000000000..b54935c9fd --- /dev/null +++ b/samples/exit-handler-email/README.md @@ -0,0 +1,45 @@ +# Email notification via SMTP server + +This pipeline demonstrates how to use the exit handler in the Kubeflow pipeline to send email notifications via the SMTP server. The exit component is based on the [send-email](https://github.com/tektoncd/catalog/tree/master/task/sendmail/0.1) Tekton catalog task. + +## prerequisites +- Install [KFP Tekton prerequisites](/samples/README.md) +- Host or subscribe to a [SMTP server](https://en.wikipedia.org/wiki/Simple_Mail_Transfer_Protocol) + +## Instructions +1. Modify [secret.yaml](secret.yaml) with the following information. Then create the secret under the Kubeflow namespace (for single user) or User namespace (for multi-user). + +* **url**: The IP address of the SMTP server + +* **port**: The port number of the SMTP server + +* **user**: User name for the SMTP server + +* **password**: Password for the SMTP server + +* **tls**: The tls enabled or not ("True" or "False") + + ```shell + NAMESPACE=kubeflow + kubectl apply -f secret.yaml -n ${NAMESPACE} + ``` + +2. Compile the send-email pipeline using the compiler inside the python code. The kfp-tekton SDK will produce a Tekton pipeline yaml definition in the same directory called `email_pipeline.yaml`. + ```shell + # Compile the python code + python send-email.py + ``` + +Then, upload the `email_pipeline.yaml` file to the Kubeflow pipeline dashboard with Tekton Backend to run this pipeline. + +### Pipeline parameters + +* **server**: The name of the secret that has the SMTP server information + +* **subject**: Email subject (plain text) + +* **body**: Email body (plain text) + +* **sender**: Email sender email address + +* **recipients**: Email recipients email addresses (space delimited) diff --git a/samples/exit-handler-email/component.yaml b/samples/exit-handler-email/component.yaml new file mode 100644 index 0000000000..957f454128 --- /dev/null +++ b/samples/exit-handler-email/component.yaml @@ -0,0 +1,58 @@ +name: sendmail +description: | + This task sends a simple email to receivers via SMTP server +inputs: + - {name: server, description: 'secret name for SMTP server information (url, port, password)'} + - {name: subject, description: 'plain text email subject'} + - {name: body, description: 'plain text email body'} + - {name: sender, description: 'sender email address'} + - {name: recipients, description: 'recipient email addresses (space delimited list)'} +implementation: + container: + image: docker.io/library/python:3.8-alpine@sha256:c31682a549a3cc0a02f694a29aed07fd252ad05935a8560237aed99b8e87bf77 #tag: 3.8-alpine + command: + - python3 + - -u + - -c + - | + #!/usr/bin/env python3 + import smtplib, ssl, os + port = os.getenv('PORT') + smtp_server = os.getenv('SERVER') + sender_email = "$(params.sender)" + receiver_emails = "$(params.recipients)" + user = os.getenv('USER') + password = os.getenv('PASSWORD') + tls = os.getenv('TLS') + message = f"""\ + Subject: $(params.subject) + To: {receiver_emails} + From: {sender_email} + $(params.body)""" + print(message) + if tls == 'True': + context = ssl.create_default_context() + server = smtplib.SMTP_SSL(smtp_server, port, context=context) + else: + server = smtplib.SMTP(smtp_server, port) + if password != '': + server.login(user, password) + for receiver in receiver_emails.split(' '): + server.sendmail(sender_email, receiver, message) + server.quit() + + import argparse + _parser = argparse.ArgumentParser('sendmail inputs') + _parser.add_argument("--server", type=str, required=True) + _parser.add_argument("--subject", type=str, required=True) + _parser.add_argument("--body", type=str, required=True) + _parser.add_argument("--sender", type=str, required=True) + _parser.add_argument("--recipients", type=str, required=True) + _parsed_args = vars(_parser.parse_args()) + args: [ + --server, {inputValue: server}, + --subject, {inputValue: subject}, + --body, {inputValue: body}, + --sender, {inputValue: sender}, + --recipients, {inputValue: recipients}, + ] diff --git a/samples/exit-handler-email/secret.yaml b/samples/exit-handler-email/secret.yaml new file mode 100644 index 0000000000..6f94d652c3 --- /dev/null +++ b/samples/exit-handler-email/secret.yaml @@ -0,0 +1,10 @@ +kind: Secret +apiVersion: v1 +metadata: + name: server-secret +stringData: + url: "smtp.server.com" + port: "25" + user: "userid" + password: "password" + tls: "False" diff --git a/samples/exit-handler-email/send-email.py b/samples/exit-handler-email/send-email.py new file mode 100644 index 0000000000..fc659f50d0 --- /dev/null +++ b/samples/exit-handler-email/send-email.py @@ -0,0 +1,73 @@ +# Copyright 2020 kubeflow.org +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from kfp import dsl +import kfp +from kubernetes import client as k8s_client + + +def echo_op(): + return dsl.ContainerOp( + name='echo', + image='busybox', + command=['sh', '-c'], + arguments=['echo "Got scheduled"'] + ) + + +def env_from_secret(env_name, secret_name, secret_key): + return k8s_client.V1EnvVar( + name=env_name, + value_from=k8s_client.V1EnvVarSource( + secret_key_ref=k8s_client.V1SecretKeySelector( + name=secret_name, + key=secret_key + ) + ) + ) + + +email_op = kfp.components.load_component_from_file('component.yaml') + + +@dsl.pipeline( + name='email_pipeline', + description='email pipeline' +) +def email_pipeline( + server="server-secret", + subject="Hi, again!", + body="Tekton email", + sender="", + recipients=" " +): + email = email_op(server=server, + subject=subject, + body=body, + sender=sender, + recipients=recipients) + email.add_env_variable(env_from_secret('USER', '$(params.server)', 'user')) + email.add_env_variable(env_from_secret('PASSWORD', '$(params.server)', 'password')) + email.add_env_variable(env_from_secret('TLS', '$(params.server)', 'tls')) + email.add_env_variable(env_from_secret('SERVER', '$(params.server)', 'url')) + email.add_env_variable(env_from_secret('PORT', '$(params.server)', 'port')) + + with dsl.ExitHandler(email): + echo = echo_op() + + + +if __name__ == '__main__': + from kfp_tekton.compiler import TektonCompiler + TektonCompiler().compile(email_pipeline, 'email_pipeline.yaml') From fe9680865f908b463c321198f4287c5ba3b188d9 Mon Sep 17 00:00:00 2001 From: Tommy Li Date: Fri, 11 Dec 2020 17:50:24 -0800 Subject: [PATCH 2/3] add workaround for passing possible files --- samples/exit-handler-email/README.md | 12 +++++- samples/exit-handler-email/component.yaml | 49 +++++++++++++++-------- samples/exit-handler-email/pvc.yaml | 10 +++++ samples/exit-handler-email/send-email.py | 27 ++++++++----- 4 files changed, 69 insertions(+), 29 deletions(-) create mode 100644 samples/exit-handler-email/pvc.yaml diff --git a/samples/exit-handler-email/README.md b/samples/exit-handler-email/README.md index b54935c9fd..a7b86482b0 100644 --- a/samples/exit-handler-email/README.md +++ b/samples/exit-handler-email/README.md @@ -24,7 +24,13 @@ This pipeline demonstrates how to use the exit handler in the Kubeflow pipeline kubectl apply -f secret.yaml -n ${NAMESPACE} ``` -2. Compile the send-email pipeline using the compiler inside the python code. The kfp-tekton SDK will produce a Tekton pipeline yaml definition in the same directory called `email_pipeline.yaml`. +2. Create a shared persistent volume claim for passing optional attachment. + + ```shell + kubectl apply -f pvc.yaml -n ${NAMESPACE} + ``` + +3. Compile the send-email pipeline using the compiler inside the python code. The kfp-tekton SDK will produce a Tekton pipeline yaml definition in the same directory called `email_pipeline.yaml`. ```shell # Compile the python code python send-email.py @@ -42,4 +48,6 @@ Then, upload the `email_pipeline.yaml` file to the Kubeflow pipeline dashboard w * **sender**: Email sender email address -* **recipients**: Email recipients email addresses (space delimited) +* **recipients**: Email recipients email addresses (comma space delimited) + +* **attachment_path**: Optional attachment path from the previous path diff --git a/samples/exit-handler-email/component.yaml b/samples/exit-handler-email/component.yaml index 957f454128..e6e2c8dbc1 100644 --- a/samples/exit-handler-email/component.yaml +++ b/samples/exit-handler-email/component.yaml @@ -7,6 +7,7 @@ inputs: - {name: body, description: 'plain text email body'} - {name: sender, description: 'sender email address'} - {name: recipients, description: 'recipient email addresses (space delimited list)'} + - {name: attachment_path, description: 'email attachment file path'} implementation: container: image: docker.io/library/python:3.8-alpine@sha256:c31682a549a3cc0a02f694a29aed07fd252ad05935a8560237aed99b8e87bf77 #tag: 3.8-alpine @@ -16,7 +17,23 @@ implementation: - -c - | #!/usr/bin/env python3 + import argparse + _parser = argparse.ArgumentParser('sendmail inputs') + _parser.add_argument("--server", type=str, required=True) + _parser.add_argument("--subject", type=str, required=True) + _parser.add_argument("--body", type=str, required=True) + _parser.add_argument("--sender", type=str, required=True) + _parser.add_argument("--recipients", type=str, required=True) + _parser.add_argument("--attachment_path", type=str, default='') + _parsed_args = _parser.parse_args() + import smtplib, ssl, os + from pathlib import Path + from email.mime.multipart import MIMEMultipart + from email.mime.base import MIMEBase + from email.mime.text import MIMEText + from email.utils import COMMASPACE, formatdate + from email import encoders port = os.getenv('PORT') smtp_server = os.getenv('SERVER') sender_email = "$(params.sender)" @@ -24,12 +41,13 @@ implementation: user = os.getenv('USER') password = os.getenv('PASSWORD') tls = os.getenv('TLS') - message = f"""\ - Subject: $(params.subject) - To: {receiver_emails} - From: {sender_email} - $(params.body)""" - print(message) + path = _parsed_args.attachment_path + + msg = MIMEMultipart() + msg['From'] = sender_email + msg['To'] = receiver_emails + msg['Subject'] = "$(params.subject)" + msg.attach(MIMEText("$(params.body)")) if tls == 'True': context = ssl.create_default_context() server = smtplib.SMTP_SSL(smtp_server, port, context=context) @@ -37,22 +55,21 @@ implementation: server = smtplib.SMTP(smtp_server, port) if password != '': server.login(user, password) + part = MIMEBase('application', "octet-stream") + with open(path, 'rb') as file: + part.set_payload(file.read()) + encoders.encode_base64(part) + part.add_header('Content-Disposition', + 'attachment; filename="{}"'.format(Path(path).name)) + msg.attach(part) for receiver in receiver_emails.split(' '): - server.sendmail(sender_email, receiver, message) + server.sendmail(sender_email, receiver, msg.as_string()) server.quit() - - import argparse - _parser = argparse.ArgumentParser('sendmail inputs') - _parser.add_argument("--server", type=str, required=True) - _parser.add_argument("--subject", type=str, required=True) - _parser.add_argument("--body", type=str, required=True) - _parser.add_argument("--sender", type=str, required=True) - _parser.add_argument("--recipients", type=str, required=True) - _parsed_args = vars(_parser.parse_args()) args: [ --server, {inputValue: server}, --subject, {inputValue: subject}, --body, {inputValue: body}, --sender, {inputValue: sender}, --recipients, {inputValue: recipients}, + --attachment_path, {inputValue: attachment_path}, ] diff --git a/samples/exit-handler-email/pvc.yaml b/samples/exit-handler-email/pvc.yaml new file mode 100644 index 0000000000..0571aa7d57 --- /dev/null +++ b/samples/exit-handler-email/pvc.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: shared-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi diff --git a/samples/exit-handler-email/send-email.py b/samples/exit-handler-email/send-email.py index fc659f50d0..40d29d921f 100644 --- a/samples/exit-handler-email/send-email.py +++ b/samples/exit-handler-email/send-email.py @@ -15,15 +15,15 @@ from kfp import dsl import kfp from kubernetes import client as k8s_client +from kfp.components import func_to_container_op +from kfp import onprem +import os -def echo_op(): - return dsl.ContainerOp( - name='echo', - image='busybox', - command=['sh', '-c'], - arguments=['echo "Got scheduled"'] - ) +@func_to_container_op +def write_file(output_text_path: str): + with open(output_text_path, 'w') as writer: + writer.write('hello world') def env_from_secret(env_name, secret_name, secret_key): @@ -39,6 +39,8 @@ def env_from_secret(env_name, secret_name, secret_key): email_op = kfp.components.load_component_from_file('component.yaml') +# pvc mount point has to be string, not pipeline param. +attachment_path = "/tmp/data" @dsl.pipeline( @@ -49,22 +51,25 @@ def email_pipeline( server="server-secret", subject="Hi, again!", body="Tekton email", - sender="", - recipients=" " + sender="me@myserver.com", + recipients="him@hisserver.com, her@herserver.com", + attachment_filepath="/tmp/data/output.txt" ): email = email_op(server=server, subject=subject, body=body, sender=sender, - recipients=recipients) + recipients=recipients, + attachment_path=attachment_filepath) email.add_env_variable(env_from_secret('USER', '$(params.server)', 'user')) email.add_env_variable(env_from_secret('PASSWORD', '$(params.server)', 'password')) email.add_env_variable(env_from_secret('TLS', '$(params.server)', 'tls')) email.add_env_variable(env_from_secret('SERVER', '$(params.server)', 'url')) email.add_env_variable(env_from_secret('PORT', '$(params.server)', 'port')) + email.apply(onprem.mount_pvc('shared-pvc', 'shared-pvc', attachment_path)) with dsl.ExitHandler(email): - echo = echo_op() + write_file_task = write_file(attachment_filepath).apply(onprem.mount_pvc('shared-pvc', 'shared-pvc', attachment_path)) From bbdee307a34a1f2ac72f66a444dc90a82f32c66b Mon Sep 17 00:00:00 2001 From: Tommy Li Date: Mon, 4 Jan 2021 13:52:28 -0800 Subject: [PATCH 3/3] Update send-email.py --- samples/exit-handler-email/send-email.py | 1 - 1 file changed, 1 deletion(-) diff --git a/samples/exit-handler-email/send-email.py b/samples/exit-handler-email/send-email.py index 40d29d921f..dd3c4f2ed8 100644 --- a/samples/exit-handler-email/send-email.py +++ b/samples/exit-handler-email/send-email.py @@ -72,7 +72,6 @@ def email_pipeline( write_file_task = write_file(attachment_filepath).apply(onprem.mount_pvc('shared-pvc', 'shared-pvc', attachment_path)) - if __name__ == '__main__': from kfp_tekton.compiler import TektonCompiler TektonCompiler().compile(email_pipeline, 'email_pipeline.yaml')