Add support for http/https inputs (#103)

miniwdl-plugins/s3upload/miniwdl_s3upload.py

@@ -22,6 +22,7 @@
 """
 
 import os
+import re
 import subprocess
 import threading
 import json
@@ -119,7 +120,7 @@ def remove_temporary_flag(s3uri, retry=0):
 
 
 def inode(link: str):
-    if link.startswith("s3://"):
+    if re.match(r'^\w+://', link):
         return link
     st = os.stat(os.path.realpath(link))
     return (st.st_dev, st.st_ino)

test/test_wdl.py

@@ -387,6 +387,21 @@ def test_simple_sfn_wdl_workflow(self):
             json.loads(messages[0]["Body"])["detail"]["lastCompletedStage"], "run"
         )
 
+    def test_https_inputs(self):
+        output_prefix = "out-https-1"
+        sfn_input: Dict[str, Any] = {
+            "RUN_WDL_URI": f"s3://{self.wdl_obj.bucket_name}/{self.wdl_obj.key}",
+            "OutputPrefix": f"s3://{self.input_obj.bucket_name}/{output_prefix}",
+            "Input": {
+                "Run": {
+                    "hello": "https://raw.githubusercontent.com/chanzuckerberg/czid-workflows/main/README.md",
+                    "docker_image_id": "ubuntu",
+                }
+            },
+        }
+
+        self._wait_sfn(sfn_input, self.single_sfn_arn)
+
     def test_failing_wdl_workflow(self):
         output_prefix = "out-fail-1"
         sfn_input: Dict[str, Any] = {

version

@@ -1 +1 @@
-v1.3.2
+v1.4.0