fixed batch jobs on Creo #34

2022-03-24 11:07:28,481:P62:INFO:openeogeotrellis.deploy.batch_job:batch_job.py main: fail 2022-03-24 11:07:28.480997, elapsed 0:00:00.000687
Traceback (most recent call last):
 File "/opt/openeo/lib64/python3.8/site-packages/openeogeotrellis/deploy/batch_job.py", line 478, in <module>
   main(sys.argv)
 File "/opt/openeo/lib64/python3.8/site-packages/openeogeotrellis/deploy/batch_job.py", line 229, in main
   raise Exception(
Exception: usage: /opt/openeo/lib64/python3.8/site-packages/openeogeotrellis/deploy/batch_job.py <job specification input file> <job directory> <results output file name> <user log file name> <metadata file name> <api version> <dependencies> <user id> <soft errors>

openeogeotrellis/backend.py

@@ -944,7 +944,7 @@ def _start_job(self, job_id: str, user_id: str, batch_process_dependencies: Unio
             max_executors = extra_options.get("max-executors", "200")
             queue = extra_options.get("queue", "default")
             profile = extra_options.get("profile", "false")
-            soft_errors = extra_options.get("soft-errors", "false")
+            soft_errors = extra_options.get("soft-errors", "false")  # TODO: accept real booleans
 
             def serialize_dependencies() -> str:
                 dependencies = batch_process_dependencies or job_info.get('dependencies') or []
@@ -1021,6 +1021,7 @@ def as_arg_element(dependency: dict) -> dict:
                     api_version=api_version,
                     dependencies="[]",  # TODO: use `serialize_dependencies()` here instead? It's probably messy to get that JSON string correctly encoded in the rendered YAML.
                     user_id=user_id,
+                    soft_errors=soft_errors,
                     current_time=int(time.time()),
                     aws_access_key_id=os.environ.get("AWS_ACCESS_KEY_ID"),
                     aws_secret_access_key=os.environ.get("AWS_SECRET_ACCESS_KEY"),

openeogeotrellis/deploy/batch_job.py

@@ -240,7 +240,7 @@ def main(argv: List[str]) -> None:
     api_version = argv[6]
     dependencies = _deserialize_dependencies(argv[7])
     user_id = argv[8]
-    soft_errors = argv[9].upper() == "TRUE"
+    soft_errors = argv[9].lower() == "true"
 
     _create_job_dir(job_dir)
 

openeogeotrellis/deploy/sparkapplication.yaml.j2

@@ -21,6 +21,7 @@ spec:
     - "{{ api_version }}"
     - "{{ dependencies }}"
     - "{{ user_id }}"
+    - "{{ soft_errors }}"
   dynamicAllocation:
     enabled: true
     initialExecutors: 2