Merge branch 'origin/master'

.github/workflows/build.yaml

@@ -0,0 +1,27 @@
+name: build
+on: [push, pull_request]
+
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python3 -m venv venv
+          . venv/bin/activate
+          pip install setuptools wheel twine
+          pip install -r requirements.txt
+      - name: run tests
+        env: 
+          RETRY_ATTEMPTS: 1
+        run: |
+          . venv/bin/activate
+          pip install nose2
+          nose2

.github/workflows/package.yaml

@@ -0,0 +1,65 @@
+name: package
+on:
+  release:
+    types: [created]
+
+jobs:
+
+  version:
+    runs-on: ubuntu-latest
+    outputs:
+      app-version: ${{ steps.set-version.outputs.version }}
+    steps:
+    - uses: actions/checkout@v4
+    - run: echo "APP_VERSION=$(python setup.py --version)" >> $GITHUB_ENV
+    - run: echo app version is $APP_VERSION
+    - id: set-version
+      run: echo "::set-output name=version::$APP_VERSION"
+
+  deploy:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.10"
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python3 -m venv venv
+        . venv/bin/activate
+        pip install setuptools wheel twine
+        pip install -r requirements.txt
+    - name: verify git tag vs. version
+      run: |
+        python3 -m venv venv
+        . venv/bin/activate
+        python setup.py verify        
+    - name: Build
+      run: |
+        . venv/bin/activate
+        python setup.py bdist_wheel --universal
+    - name: Publish
+      env:
+        TWINE_USERNAME: __token__
+        TWINE_PASSWORD: ${{ secrets.PYPI_APIKEY }}
+      run: |
+        . venv/bin/activate
+        twine upload dist/*
+
+  container-build:
+    needs: version
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4   
+    - run: echo version ${{needs.version.outputs.app-version}}
+    - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
+    - name: build & push image
+      run: |
+        IMAGE_ID=ghcr.io/duke-gcb/calrissian/calrissian
+        docker build . --file Dockerfile --tag calrissian
+        docker tag calrissian $IMAGE_ID:${{needs.version.outputs.app-version}}
+        docker push $IMAGE_ID:${{needs.version.outputs.app-version}}
+

.gitignore

@@ -10,3 +10,4 @@ build
 *.egg
 *.whl
 *.zip
+env-calrissian*

CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [v0.15.0] - 2023-06-19
+
+### Fixes
+
+- #101 and 158 (PR #159)
+
 ## [v0.14.0] - 2023-06-19
 
 ### Added

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.7
+FROM python:3.10.0-slim-buster
 LABEL maintainer="[email protected]"
 
 # cwltool requires nodejs

calrissian/job.py

@@ -25,6 +25,7 @@ def _cuda_check(cuda_req, requestCount):
 from cwltool.utils import visit_class, ensure_writable
 
 log = logging.getLogger("calrissian.job")
+log_main = logging.getLogger("calrissian.main")
 
 
 K8S_UNSAFE_REGEX = re.compile('[^-a-z0-9]')
@@ -66,8 +67,8 @@ def read_yaml(filename):
 
 
 def quoted_arg_list(arg_list):
-        shouldquote = needs_shell_quoting_re.search
-        return [shellescape.quote(arg) if shouldquote(arg) else arg for arg in arg_list]
+    shouldquote = needs_shell_quoting_re.search
+    return [shellescape.quote(arg) if shouldquote(arg) else arg for arg in arg_list]
 
 
 def total_size(outputs):
@@ -428,6 +429,10 @@ def dump_tool_logs(self, name, completion_result: CompletionResult, runtime_cont
         """
         Dumps the tool logs
         """
+        if not os.path.exists(runtime_context.tool_logs_basepath):
+            log.debug(f'os.makedirs({runtime_context.tool_logs_basepath})')
+            os.makedirs(runtime_context.tool_logs_basepath)
+
         log_filename = os.path.join(runtime_context.tool_logs_basepath, f"{name}.log")
 
         log.info(f"Writing pod {name} logs to {log_filename}")
@@ -687,6 +692,12 @@ def _required_env(self) -> Dict[str, str]:
 
     def run(self, runtimeContext, tmpdir_lock=None):
 
+        def get_pod_command(pod):
+            return pod['spec']['containers'][0]['args']
+
+        def get_pod_name(pod):
+            return pod['spec']['containers'][0]['name']
+
         self.check_requirements(runtimeContext)
 
         if tmpdir_lock:
@@ -702,6 +713,9 @@ def run(self, runtimeContext, tmpdir_lock=None):
         pod = self.create_kubernetes_runtime(runtimeContext) # analogous to create_runtime()
         self.execute_kubernetes_pod(pod) # analogous to _execute()
         completion_result = self.wait_for_kubernetes_pod()
+        if completion_result.exit_code != 0:
+            log_main.error(f"ERROR the command below failed in pod {get_pod_name(pod)}:")
+            log_main.error("\t" + " ".join(get_pod_command(pod)))
         self.finish(completion_result, runtimeContext)
 
     def setup_kubernetes(self, runtime_context):

calrissian/k8s.py

@@ -1,7 +1,7 @@
 from typing import List, Union
 from kubernetes import client, config, watch
 from kubernetes.client.models import V1ContainerState, V1Container, V1ContainerStatus
-from kubernetes.client.api_client import ApiException
+from kubernetes.client.rest import ApiException
 from kubernetes.config.config_exception import ConfigException
 from calrissian.executor import IncompleteStatusException
 from calrissian.retry import retry_exponential_if_exception_type

conformance/README.md

@@ -135,7 +135,7 @@ kubectl --namespace="$NAMESPACE_NAME" create -f StageConformanceTestsData.yaml
 Build a container that installs `calrissian` and `cwltest` with: 
 
 ```
-minikube image build .. -t ghcr.io/calrissian/conformance:latest -f conformance/Dockerfile.conformance 
+minikube image build .. -t conformance:latest -f conformance/Dockerfile.conformance 
 ```
 
 This will build `calrissian:conformance` from the current source tree and this image is available in the minikube cluster node.

requirements.in

@@ -0,0 +1,8 @@
+urllib3
+kubernetes
+cwltool
+tenacity
+importlib-metadata
+msgpack
+typing-extensions
+freezegun