parallel_join.yaml

# Copyright 2021-2023 kubeflow.org
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

apiVersion: tekton.dev/v1
kind: PipelineRun
metadata:
  name: parallel-pipeline
  annotations:
    tekton.dev/output_artifacts: '{"gcs-download": [{"key": "artifacts/$PIPELINERUN/gcs-download/data.tgz",
      "name": "gcs-download-data", "path": "/tmp/outputs/data/data"}], "gcs-download-2":
      [{"key": "artifacts/$PIPELINERUN/gcs-download-2/data.tgz", "name": "gcs-download-2-data",
      "path": "/tmp/outputs/data/data"}]}'
    tekton.dev/input_artifacts: '{"echo": [{"name": "gcs-download-2-data", "parent_task":
      "gcs-download-2"}, {"name": "gcs-download-data", "parent_task": "gcs-download"}]}'
    tekton.dev/artifact_bucket: mlpipeline
    tekton.dev/artifact_endpoint: minio-service.kubeflow:9000
    tekton.dev/artifact_endpoint_scheme: http://
    tekton.dev/artifact_items: '{"echo": [], "gcs-download": [["data", "$(results.data.path)"]],
      "gcs-download-2": [["data", "$(results.data.path)"]]}'
    sidecar.istio.io/inject: "false"
    tekton.dev/template: ''
    pipelines.kubeflow.org/big_data_passing_format: $(workspaces.$TASK_NAME.path)/artifacts/$ORIG_PR_NAME/$TASKRUN_NAME/$TASK_PARAM_NAME
    pipelines.kubeflow.org/pipeline_spec: '{"description": "Download two messages
      in parallel and prints the concatenated result.", "inputs": [{"default": "gs://ml-pipeline-playground/shakespeare1.txt",
      "name": "url1", "optional": true, "type": "String"}, {"default": "gs://ml-pipeline-playground/shakespeare2.txt",
      "name": "url2", "optional": true, "type": "String"}], "name": "parallel-pipeline"}'
  labels:
    pipelines.kubeflow.org/pipelinename: ''
    pipelines.kubeflow.org/generation: ''
spec:
  params:
  - name: url1
    value: gs://ml-pipeline-playground/shakespeare1.txt
  - name: url2
    value: gs://ml-pipeline-playground/shakespeare2.txt
  pipelineSpec:
    params:
    - name: url1
      default: gs://ml-pipeline-playground/shakespeare1.txt
    - name: url2
      default: gs://ml-pipeline-playground/shakespeare2.txt
    tasks:
    - name: gcs-download
      params:
      - name: url1
        value: $(params.url1)
      taskSpec:
        steps:
        - name: main
          args:
          - |
            gsutil cat $0 | tee $1
          - $(inputs.params.url1)
          - $(results.data.path)
          command:
          - sh
          - -c
          image: google/cloud-sdk:279.0.0
        params:
        - name: url1
        results:
        - name: data
          type: string
          description: /tmp/outputs/data/data
        metadata:
          labels:
            pipelines.kubeflow.org/cache_enabled: "true"
          annotations:
            pipelines.kubeflow.org/component_spec_digest: '{"name": "gcs-download",
              "outputs": [{"name": "data", "type": "String"}], "version": "gcs-download@sha256=511cfab40b31f87d8013514f1aee58997608048664cd54fa424e5167bcfcf97c"}'
    - name: gcs-download-2
      params:
      - name: url2
        value: $(params.url2)
      taskSpec:
        steps:
        - name: main
          args:
          - |
            gsutil cat $0 | tee $1
          - $(inputs.params.url2)
          - $(results.data.path)
          command:
          - sh
          - -c
          image: google/cloud-sdk:279.0.0
        params:
        - name: url2
        results:
        - name: data
          type: string
          description: /tmp/outputs/data/data
        metadata:
          labels:
            pipelines.kubeflow.org/cache_enabled: "true"
          annotations:
            pipelines.kubeflow.org/component_spec_digest: '{"name": "gcs-download",
              "outputs": [{"name": "data", "type": "String"}], "version": "gcs-download@sha256=511cfab40b31f87d8013514f1aee58997608048664cd54fa424e5167bcfcf97c"}'
    - name: echo
      params:
      - name: gcs-download-2-data
        value: $(tasks.gcs-download-2.results.data)
      - name: gcs-download-data
        value: $(tasks.gcs-download.results.data)
      taskSpec:
        steps:
        - name: main
          args:
          - |
            echo "Text 1: $0"; echo "Text 2: $1"
          - $(inputs.params.gcs-download-data)
          - $(inputs.params.gcs-download-2-data)
          command:
          - sh
          - -c
          image: library/bash:4.4.23
        params:
        - name: gcs-download-2-data
        - name: gcs-download-data
        metadata:
          labels:
            pipelines.kubeflow.org/cache_enabled: "true"
          annotations:
            pipelines.kubeflow.org/component_spec_digest: '{"name": "echo", "outputs":
              [], "version": "echo@sha256=6ba9f3d0658a72a763f9ed77d34a378b19b686b649512adee9831ed4ec7ddcd4"}'