Deploy model from MR in GitLab (#279)

* Deploy model from MR in GitLab * new default gitlab workflow
iterative · Dec 18, 2023 · 21712e9 · 21712e9
1 parent 4cd2eaa
commit 21712e9
Show file tree

Hide file tree

Showing 2 changed files with 90 additions and 51 deletions.
diff --git a/example-get-started-experiments/code/.gitlab-ci.yml b/example-get-started-experiments/code/.gitlab-ci.yml
@@ -1,52 +1,39 @@
-variables:
-  EXP_RUN_ARGS: ""
-deploy-runner:
-  image: iterativeai/cml:0-dvc2-base1
-  script:
-  - pip install awscli
-  - >
-    CREDENTIALS=($(aws sts assume-role-with-web-identity
-    --region=us-east-1
-    --role-arn=arn:aws:iam::342840881361:role/SandboxUser
-    --role-session-name=GitLab
-    --duration-seconds=3600
-    --web-identity-token="$CI_JOB_JWT_V2"
-    --query="Credentials.[AccessKeyId,SecretAccessKey,SessionToken]"
-    --output=text))
-  - export AWS_ACCESS_KEY_ID="${CREDENTIALS[0]}"
-  - export AWS_SECRET_ACCESS_KEY="${CREDENTIALS[1]}"
-  - export AWS_SESSION_TOKEN="${CREDENTIALS[2]}"
-  - aws sts get-caller-identity
-  - >
-    cml runner launch --single \
-      --labels=cml \
-      --cloud=aws \
-      --cloud-region=us-east \
-      --cloud-hdd-size=40 \
-      --cloud-type=g5.2xlarge
-runner-job:
+# Deploy Model (Template)
+
+workflow:
+  rules:
+    # Run the pipeline whenever a tag is pushed to the repository
+    - if: $CI_COMMIT_TAG
+
+parse:
+  # This job parses the model tag to identify model registry actions
+  image: python:3.11-slim
+  script: 
+  # Install GTO to parse model tags
+  - pip install gto
+  # This job parses the model tags to identify model registry actions
+  - echo "CI_COMMIT_TAG - ${CI_COMMIT_TAG}" 
+  - echo MODEL_NAME="$(gto check-ref ${CI_COMMIT_TAG} --name)" >> parse.env
+  - echo MODEL_VERSION="$(gto check-ref ${CI_COMMIT_TAG} --version)" >> parse.env
+  - echo MODEL_EVENT="$(gto check-ref ${CI_COMMIT_TAG} --event)" >> parse.env
+  - echo MODEL_STAGE="$(gto check-ref ${CI_COMMIT_TAG} --stage)" >> parse.env
+  # Print variables saved to parse.env
+  - cat parse.env
+  artifacts:
+    reports:
+      dotenv: parse.env
+
+deploy-model:
   needs:
-    - deploy-runner
-  tags:
-    - cml
-  image: iterativeai/cml:0-dvc2-base1
-  script:
-  - pip install awscli
-  - >
-    CREDENTIALS=($(aws sts assume-role-with-web-identity
-    --region=us-east-1
-    --role-arn=arn:aws:iam::342840881361:role/SandboxUser
-    --role-session-name=GitLab
-    --duration-seconds=3600
-    --web-identity-token="$CI_JOB_JWT_V2"
-    --query="Credentials.[AccessKeyId,SecretAccessKey,SessionToken]"
-    --output=text))
-  - export AWS_ACCESS_KEY_ID="${CREDENTIALS[0]}"
-  - export AWS_SECRET_ACCESS_KEY="${CREDENTIALS[1]}"
-  - export AWS_SESSION_TOKEN="${CREDENTIALS[2]}"
-  - aws sts get-caller-identity
-  - pip install -r requirements.txt
-  - cml ci
-  - dvc exp run --pull --allow-missing $EXP_RUN_ARGS
-  - dvc remote add --local push_remote s3://dvc-public/remote/get-started-pools 
-  - dvc exp push origin -r push_remote
+  - job: parse
+    artifacts: true
+  image: python:3.11-slim
+  script: 
+  # Check if the model is assigned to prod (variables from parse.env are only available in the 'script' section)
+  - if [[ $MODEL_EVENT == 'assignment' && $MODEL_STAGE == 'prod' ]]; then echo "Deploy model"; else exit 1; fi
+  # Install DVC
+  - pip install dvc
+  # Build commands to download and deploy the model
+  - dvc config --global studio.token ${DVC_STUDIO_TOKEN}
+  - dvc artifacts get  ${CI_REPOSITORY_URL} ${MODEL_NAME} --rev ${MODEL_VERSION}
+  - echo "The right model is available and you can use the rest of this command to deploy it. Good job!"
diff --git a/example-get-started-experiments/code/gitlab-workflows/cloud-experiment.gitlab-ci.yml b/example-get-started-experiments/code/gitlab-workflows/cloud-experiment.gitlab-ci.yml
@@ -0,0 +1,52 @@
+variables:
+  EXP_RUN_ARGS: ""
+deploy-runner:
+  image: iterativeai/cml:0-dvc2-base1
+  script:
+  - pip install awscli
+  - >
+    CREDENTIALS=($(aws sts assume-role-with-web-identity
+    --region=us-east-1
+    --role-arn=arn:aws:iam::342840881361:role/SandboxUser
+    --role-session-name=GitLab
+    --duration-seconds=3600
+    --web-identity-token="$CI_JOB_JWT_V2"
+    --query="Credentials.[AccessKeyId,SecretAccessKey,SessionToken]"
+    --output=text))
+  - export AWS_ACCESS_KEY_ID="${CREDENTIALS[0]}"
+  - export AWS_SECRET_ACCESS_KEY="${CREDENTIALS[1]}"
+  - export AWS_SESSION_TOKEN="${CREDENTIALS[2]}"
+  - aws sts get-caller-identity
+  - >
+    cml runner launch --single \
+      --labels=cml \
+      --cloud=aws \
+      --cloud-region=us-east \
+      --cloud-hdd-size=40 \
+      --cloud-type=g5.2xlarge
+runner-job:
+  needs:
+    - deploy-runner
+  tags:
+    - cml
+  image: iterativeai/cml:0-dvc2-base1
+  script:
+  - pip install awscli
+  - >
+    CREDENTIALS=($(aws sts assume-role-with-web-identity
+    --region=us-east-1
+    --role-arn=arn:aws:iam::342840881361:role/SandboxUser
+    --role-session-name=GitLab
+    --duration-seconds=3600
+    --web-identity-token="$CI_JOB_JWT_V2"
+    --query="Credentials.[AccessKeyId,SecretAccessKey,SessionToken]"
+    --output=text))
+  - export AWS_ACCESS_KEY_ID="${CREDENTIALS[0]}"
+  - export AWS_SECRET_ACCESS_KEY="${CREDENTIALS[1]}"
+  - export AWS_SESSION_TOKEN="${CREDENTIALS[2]}"
+  - aws sts get-caller-identity
+  - pip install -r requirements.txt
+  - cml ci
+  - dvc exp run --pull --allow-missing $EXP_RUN_ARGS
+  - dvc remote add --local push_remote s3://dvc-public/remote/get-started-pools 
+  - dvc exp push origin -r push_remote