wml-pipeline.yaml

# Copyright 2021 The MLX Contributors
#
# SPDX-License-Identifier: Apache-2.0
apiVersion: tekton.dev/v1beta1
kind: PipelineRun
metadata:
  name: kfp-on-wml-training
  annotations:
    tekton.dev/output_artifacts: '{"create-secret-kubernetes-cluster": [{"key": "artifacts/$PIPELINERUN/create-secret-kubernetes-cluster/secret_name.tgz",
      "name": "create-secret-kubernetes-cluster-secret_name", "path": "/tmp/outputs/secret_name/data"}],
      "deploy-model-watson-machine-learning": [{"key": "artifacts/$PIPELINERUN/deploy-model-watson-machine-learning/model_uid.tgz",
      "name": "deploy-model-watson-machine-learning-model_uid", "path": "/tmp/outputs/model_uid/data"},
      {"key": "artifacts/$PIPELINERUN/deploy-model-watson-machine-learning/scoring_endpoint.tgz",
      "name": "deploy-model-watson-machine-learning-scoring_endpoint", "path": "/tmp/outputs/scoring_endpoint/data"}],
      "store-model-watson-machine-learning": [{"key": "artifacts/$PIPELINERUN/store-model-watson-machine-learning/model_uid.tgz",
      "name": "store-model-watson-machine-learning-model_uid", "path": "/tmp/outputs/model_uid/data"}],
      "train-model-watson-machine-learning": [{"key": "artifacts/$PIPELINERUN/train-model-watson-machine-learning/run_uid.tgz",
      "name": "train-model-watson-machine-learning-run_uid", "path": "/tmp/outputs/run_uid/data"},
      {"key": "artifacts/$PIPELINERUN/train-model-watson-machine-learning/training_uid.tgz",
      "name": "train-model-watson-machine-learning-training_uid", "path": "/tmp/outputs/training_uid/data"}]}'
    tekton.dev/input_artifacts: '{"deploy-model-watson-machine-learning": [{"name":
      "store-model-watson-machine-learning-model_uid", "parent_task": "store-model-watson-machine-learning"}],
      "store-model-watson-machine-learning": [{"name": "train-model-watson-machine-learning-run_uid",
      "parent_task": "train-model-watson-machine-learning"}], "train-model-watson-machine-learning":
      [{"name": "create-secret-kubernetes-cluster-secret_name", "parent_task": "create-secret-kubernetes-cluster"}]}'
    tekton.dev/artifact_bucket: mlpipeline
    tekton.dev/artifact_endpoint: minio-service.kubeflow:9000
    tekton.dev/artifact_endpoint_scheme: http://
    tekton.dev/artifact_items: '{"create-secret-kubernetes-cluster": [["secret_name",
      "$(results.secret-name.path)"]], "deploy-model-watson-machine-learning": [["model_uid",
      "$(results.model-uid.path)"], ["scoring_endpoint", "$(results.scoring-endpoint.path)"]],
      "store-model-watson-machine-learning": [["model_uid", "$(results.model-uid.path)"]],
      "train-model-watson-machine-learning": [["run_uid", "$(results.run-uid.path)"],
      ["training_uid", "$(results.training-uid.path)"]]}'
    sidecar.istio.io/inject: "false"
    pipelines.kubeflow.org/big_data_passing_format: $(workspaces.$TASK_NAME.path)/artifacts/$ORIG_PR_NAME/$TASKRUN_NAME/$TASK_PARAM_NAME
    pipelines.kubeflow.org/pipeline_spec: '{"description": "Kubeflow pipelines running
      on WML performing tensorflow image recognition.", "inputs": [{"default": "",
      "name": "GITHUB_TOKEN", "optional": true}, {"default": "https://raw.githubusercontent.com/user/repository/branch/creds.ini",
      "name": "CONFIG_FILE_URL", "optional": true}, {"default": "tf-model.zip", "name":
      "train_code", "optional": true}, {"default": "''python3 convolutional_network.py
      --trainImagesFile ${DATA_DIR}/train-images-idx3-ubyte.gz --trainLabelsFile ${DATA_DIR}/train-labels-idx1-ubyte.gz
      --testImagesFile ${DATA_DIR}/t10k-images-idx3-ubyte.gz --testLabelsFile ${DATA_DIR}/t10k-labels-idx1-ubyte.gz
      --learningRate 0.001 --trainingIters 20000''", "name": "execution_command",
      "optional": true}, {"default": "tensorflow", "name": "framework", "optional":
      true}, {"default": "1.15", "name": "framework_version", "optional": true}, {"default":
      "python", "name": "runtime", "optional": true}, {"default": "3.6", "name": "runtime_version",
      "optional": true}, {"default": "wml-tensorflow-definition", "name": "run_definition",
      "optional": true}, {"default": "wml-tensorflow-run", "name": "run_name", "optional":
      true}, {"default": "wml-tensorflow-mnist", "name": "model_name", "optional":
      true}, {"default": "tf-mnist-test-payload.json", "name": "scoring_payload",
      "optional": true}, {"default": "k80", "name": "compute_name", "optional": true},
      {"default": "1", "name": "compute_nodes", "optional": true}], "name": "Watson
      Machine Learning"}'
spec:
  params:
  - name: CONFIG_FILE_URL
    value: https://raw.githubusercontent.com/user/repository/branch/creds.ini
  - name: GITHUB_TOKEN
    value: ''
  - name: compute_name
    value: k80
  - name: compute_nodes
    value: '1'
  - name: execution_command
    value: '''python3 convolutional_network.py --trainImagesFile ${DATA_DIR}/train-images-idx3-ubyte.gz
      --trainLabelsFile ${DATA_DIR}/train-labels-idx1-ubyte.gz --testImagesFile ${DATA_DIR}/t10k-images-idx3-ubyte.gz
      --testLabelsFile ${DATA_DIR}/t10k-labels-idx1-ubyte.gz --learningRate 0.001
      --trainingIters 20000'''
  - name: framework
    value: tensorflow
  - name: framework_version
    value: '1.15'
  - name: model_name
    value: wml-tensorflow-mnist
  - name: run_definition
    value: wml-tensorflow-definition
  - name: run_name
    value: wml-tensorflow-run
  - name: runtime
    value: python
  - name: runtime_version
    value: '3.6'
  - name: scoring_payload
    value: tf-mnist-test-payload.json
  - name: train_code
    value: tf-model.zip
  pipelineSpec:
    params:
    - name: CONFIG_FILE_URL
      default: https://raw.githubusercontent.com/user/repository/branch/creds.ini
    - name: GITHUB_TOKEN
      default: ''
    - name: compute_name
      default: k80
    - name: compute_nodes
      default: '1'
    - name: execution_command
      default: '''python3 convolutional_network.py --trainImagesFile ${DATA_DIR}/train-images-idx3-ubyte.gz
        --trainLabelsFile ${DATA_DIR}/train-labels-idx1-ubyte.gz --testImagesFile
        ${DATA_DIR}/t10k-images-idx3-ubyte.gz --testLabelsFile ${DATA_DIR}/t10k-labels-idx1-ubyte.gz
        --learningRate 0.001 --trainingIters 20000'''
    - name: framework
      default: tensorflow
    - name: framework_version
      default: '1.15'
    - name: model_name
      default: wml-tensorflow-mnist
    - name: run_definition
      default: wml-tensorflow-definition
    - name: run_name
      default: wml-tensorflow-run
    - name: runtime
      default: python
    - name: runtime_version
      default: '3.6'
    - name: scoring_payload
      default: tf-mnist-test-payload.json
    - name: train_code
      default: tf-model.zip
    tasks:
    - name: create-secret-kubernetes-cluster
      params:
      - name: CONFIG_FILE_URL
        value: $(params.CONFIG_FILE_URL)
      - name: GITHUB_TOKEN
        value: $(params.GITHUB_TOKEN)
      taskSpec:
        steps:
        - name: main
          args:
          - /app/config.py
          - --token
          - $(inputs.params.GITHUB_TOKEN)
          - --url
          - $(inputs.params.CONFIG_FILE_URL)
          - --name
          - kfp-creds
          - --output-secret-name-file
          - $(results.secret-name.path)
          command:
          - python3
          image: docker.io/aipipeline/wml-config:latest
        params:
        - name: CONFIG_FILE_URL
        - name: GITHUB_TOKEN
        results:
        - name: secret-name
          description: /tmp/outputs/secret_name/data
        metadata:
          labels:
            pipelines.kubeflow.org/pipelinename: ''
            pipelines.kubeflow.org/generation: ''
            pipelines.kubeflow.org/cache_enabled: "true"
          annotations:
            pipelines.kubeflow.org/component_spec_digest: '{"name": "Create Secret
              - Kubernetes Cluster", "outputs": [{"description": "Kubernetes secret
              name", "name": "secret_name"}], "version": "Create Secret - Kubernetes
              Cluster@sha256=902baeb855ce424cc0efea20e61f1b51acfad6cdcaef65ca9f3e9e503a5bbae6"}'
            tekton.dev/template: ''
      timeout: 525600m
    - name: train-model-watson-machine-learning
      params:
      - name: compute_name
        value: $(params.compute_name)
      - name: compute_nodes
        value: $(params.compute_nodes)
      - name: create-secret-kubernetes-cluster-secret_name
        value: $(tasks.create-secret-kubernetes-cluster.results.secret-name)
      - name: execution_command
        value: $(params.execution_command)
      - name: framework
        value: $(params.framework)
      - name: framework_version
        value: $(params.framework_version)
      - name: run_definition
        value: $(params.run_definition)
      - name: run_name
        value: $(params.run_name)
      - name: runtime
        value: $(params.runtime)
      - name: runtime_version
        value: $(params.runtime_version)
      - name: train_code
        value: $(params.train_code)
      taskSpec:
        steps:
        - name: main
          args:
          - -u
          - /app/wml-train.py
          - --config
          - $(inputs.params.create-secret-kubernetes-cluster-secret_name)
          - --train-code
          - $(inputs.params.train_code)
          - --execution-command
          - $(inputs.params.execution_command)
          - --framework
          - $(inputs.params.framework)
          - --framework-version
          - $(inputs.params.framework_version)
          - --runtime
          - $(inputs.params.runtime)
          - --runtime-version
          - $(inputs.params.runtime_version)
          - --run-definition
          - $(inputs.params.run_definition)
          - --run-name
          - $(inputs.params.run_name)
          - --author-name
          - default-author
          - --compute-name
          - $(inputs.params.compute_name)
          - --compute-nodes
          - $(inputs.params.compute_nodes)
          - --output-run-uid-path
          - $(results.run-uid.path)
          - --output-training-uid-path
          - $(results.training-uid.path)
          command:
          - python3
          image: docker.io/aipipeline/wml-train:latest
          imagePullPolicy: Always
          volumeMounts:
          - mountPath: /app/secrets
            name: kfp-creds
        params:
        - name: compute_name
        - name: compute_nodes
        - name: create-secret-kubernetes-cluster-secret_name
        - name: execution_command
        - name: framework
        - name: framework_version
        - name: run_definition
        - name: run_name
        - name: runtime
        - name: runtime_version
        - name: train_code
        results:
        - name: run-uid
          description: /tmp/outputs/run_uid/data
        - name: training-uid
          description: /tmp/outputs/training_uid/data
        volumes:
        - name: kfp-creds
          secret:
            secretName: kfp-creds
        metadata:
          labels:
            pipelines.kubeflow.org/pipelinename: ''
            pipelines.kubeflow.org/generation: ''
            pipelines.kubeflow.org/cache_enabled: "true"
          annotations:
            platform: IBM Watson Machine Learning
            pipelines.kubeflow.org/component_spec_digest: '{"name": "Train Model -
              Watson Machine Learning", "outputs": [{"description": "UID for the Watson
              Machine Learning training-runs", "name": "run_uid"}, {"description":
              "Training Location UID for the Watson Machine Learning training-runs",
              "name": "training_uid"}], "version": "Train Model - Watson Machine Learning@sha256=b4c691df9f8a58439e3b232f23da34c30695dd93d8f3d79954142f185d377c7a"}'
            tekton.dev/template: ''
      timeout: 525600m
    - name: store-model-watson-machine-learning
      params:
      - name: framework
        value: $(params.framework)
      - name: framework_version
        value: $(params.framework_version)
      - name: model_name
        value: $(params.model_name)
      - name: runtime_version
        value: $(params.runtime_version)
      - name: train-model-watson-machine-learning-run_uid
        value: $(tasks.train-model-watson-machine-learning.results.run-uid)
      taskSpec:
        steps:
        - name: main
          args:
          - -u
          - /app/wml-store.py
          - --run-uid
          - $(inputs.params.train-model-watson-machine-learning-run_uid)
          - --model-name
          - $(inputs.params.model_name)
          - --framework
          - $(inputs.params.framework)
          - --framework-version
          - $(inputs.params.framework_version)
          - --runtime-version
          - $(inputs.params.runtime_version)
          - --output-model-uid-path
          - $(results.model-uid.path)
          command:
          - python3
          image: docker.io/aipipeline/wml-store:latest
          imagePullPolicy: Always
          volumeMounts:
          - mountPath: /app/secrets
            name: kfp-creds
        params:
        - name: framework
        - name: framework_version
        - name: model_name
        - name: runtime_version
        - name: train-model-watson-machine-learning-run_uid
        results:
        - name: model-uid
          description: /tmp/outputs/model_uid/data
        volumes:
        - name: kfp-creds
          secret:
            secretName: kfp-creds
        metadata:
          labels:
            pipelines.kubeflow.org/pipelinename: ''
            pipelines.kubeflow.org/generation: ''
            pipelines.kubeflow.org/cache_enabled: "true"
          annotations:
            platform: IBM Watson Machine Learning
            pipelines.kubeflow.org/component_spec_digest: '{"name": "Store model -
              Watson Machine Learning", "outputs": [{"description": "UID for the stored
              model on Watson Machine Learning", "name": "model_uid"}], "version":
              "Store model - Watson Machine Learning@sha256=feee7b0338b0e788c49b6b5f2b9c10f1f129ce1483aa99d349297236d830353c"}'
            tekton.dev/template: ''
      timeout: 525600m
    - name: deploy-model-watson-machine-learning
      params:
      - name: model_name
        value: $(params.model_name)
      - name: scoring_payload
        value: $(params.scoring_payload)
      - name: store-model-watson-machine-learning-model_uid
        value: $(tasks.store-model-watson-machine-learning.results.model-uid)
      taskSpec:
        steps:
        - name: main
          args:
          - -u
          - /app/wml-deploy.py
          - --model-uid
          - $(inputs.params.store-model-watson-machine-learning-model_uid)
          - --model-name
          - $(inputs.params.model_name)
          - --scoring-payload
          - $(inputs.params.scoring_payload)
          - --deployment-name
          - ''
          - --output-scoring-endpoint-path
          - $(results.scoring-endpoint.path)
          - --output-model-uid-path
          - $(results.model-uid.path)
          command:
          - python
          image: docker.io/aipipeline/wml-deploy:latest
          imagePullPolicy: Always
          volumeMounts:
          - mountPath: /app/secrets
            name: kfp-creds
        params:
        - name: model_name
        - name: scoring_payload
        - name: store-model-watson-machine-learning-model_uid
        results:
        - name: scoring-endpoint
          description: /tmp/outputs/scoring_endpoint/data
        - name: model-uid
          description: /tmp/outputs/model_uid/data
        volumes:
        - name: kfp-creds
          secret:
            secretName: kfp-creds
        metadata:
          labels:
            pipelines.kubeflow.org/pipelinename: ''
            pipelines.kubeflow.org/generation: ''
            pipelines.kubeflow.org/cache_enabled: "true"
          annotations:
            platform: IBM Watson Machine Learning
            pipelines.kubeflow.org/component_spec_digest: '{"name": "Deploy Model
              - Watson Machine Learning", "outputs": [{"description": "Link to the
              deployed model web service", "name": "scoring_endpoint"}, {"description":
              "UID for the stored model on Watson Machine Learning", "name": "model_uid"}],
              "version": "Deploy Model - Watson Machine Learning@sha256=11bfa9e2fe70d7acd77dfe31505d1aa230570f326c6b55ef7fa691216b8f0bcd"}'
            tekton.dev/template: ''
      timeout: 525600m
  timeout: 525600m