116 reduce debug logging (#117)

Can also help us identify weird package push "errors"

CHANGELOG.md

@@ -1,5 +1,14 @@
 # Changelog
 
+## [0.4.5] 2023-08-23
+
+- fix metadata in README.md
+- write out nf-quilt/*.json files
+- add summarize wildcards (md, html) for QuiltProduct
+- redo normalizedPaths
+- reduce logging (especially stack traces and filesystem calls)
+- convert non-fatal errors to warnings
+
 ## [0.4.4] 2023-08-22
 
 - JDK 11 compatibility (i.e., remove stripIndent)

Makefile

@@ -25,7 +25,7 @@ check-env:
 
 clean:
 	./gradlew clean
-	rm -rf work null results
+	rm -rf null results
 	rm -rf */*/build plugins/nf-quilt/bin
 	rm -f .nextflow.log* .launch*classpath
 

README.md

@@ -92,9 +92,9 @@ If a plugin is not yet available in the NextFlow plugin registry, you can use a
 From the command-line, do, e.g.:
 
 ```bash
-# export NXF_VER=23.04.0
-export NXF_PLUGINS_TEST_REPOSITORY=https://github.com/quiltdata/nf-quilt/releases/download/0.4.4/nf-quilt-0.4.4-meta.json
-nextflow run main.nf -plugins [email protected].4
+# export NXF_VER=23.04.3
+export NXF_PLUGINS_TEST_REPOSITORY=https://github.com/quiltdata/nf-quilt/releases/download/0.4.5/nf-quilt-0.4.5-meta.json
+nextflow run main.nf -plugins [email protected].5
 ```
 
 For Tower, you can use the "Pre-run script" to set the environment variables.

VERSION

@@ -1 +1 @@
-23.04.0
+23.04.3

example.env

@@ -1,3 +1,3 @@
-export NXF_VER=23.04.0
+export NXF_VER=23.04.3
 export TOWER_ACCESS_TOKEN=eyxxxxxxxxxxxxxxxQ1ZTE=
 export TOWER_WORKSPACE_ID=000000000000000

plugins/nf-quilt/build.gradle

@@ -55,7 +55,7 @@ sourceSets {
 }
 
 ext{
-    nextflowVersion = '23.04.1'
+    nextflowVersion = '23.04.3'
 }
 
 dependencies {

plugins/nf-quilt/src/main/nextflow/quilt/QuiltObserver.groovy

@@ -35,8 +35,9 @@ import nextflow.trace.TraceObserver
 @CompileStatic
 class QuiltObserver implements TraceObserver {
 
-    private final Set<QuiltPath> paths = [] as Set
     private Session session
+    final private Map<String,String> uniqueURIs = [:]
+    final private Map<String,String> publishedURIs = [:]
 
     static QuiltPath asQuiltPath(Path path) {
         if (path in QuiltPath) {
@@ -50,51 +51,67 @@ class QuiltObserver implements TraceObserver {
         return null
     }
 
-    static QuiltPath normalizePath(QuiltPath path, Map params) {
-        String bkt = path.getBucket()
-        String pname = path.getPackageName()
-        QuiltPath result = path
-        params.each { key, value ->
+    static String pkgKey(QuiltPath path) {
+        return "${path.getBucket()}/${path.getPackageName()}"
+    }
+
+    static String pathless(String uri) {
+        return uri.replaceFirst(/&path=[^&]+/, '')
+    }
+
+    String checkPath(QuiltPath path, boolean published = false) {
+        log.debug("checkPath[$path] published[$published]")
+        String key = pkgKey(path)
+        String uri = pathless(path.toUriString())
+        // only keep the longest pathless URI for each key
+        if (uniqueURIs[key]?.length() < uri.length()) {
+            uniqueURIs[key] = uri
+        }
+        if (published) {
+            publishedURIs[key] = uniqueURIs[key]
+        }
+        return uniqueURIs[key]
+    }
+
+    void checkParams(Map params) {
+        log.debug("checkParams[$params]")
+        params.each { k, value ->
             String uri = "$value"
-            if (uri.contains(bkt) && uri.contains(pname)) {
-                if (uri.endsWith(pname)) {
-                    return QuiltPathFactory.parse(uri)
-                }
-                String pathless = uri.replaceFirst(/&path=[^&]+/, '')
-                log.debug("`normalizePath.pathless` $uri\n -> $pathless")
-                QuiltPath npath = QuiltPathFactory.parse(pathless)
-                result = npath
+            if (uri.startsWith(QuiltParser.SCHEME)) {
+                log.debug("checkParams.uri[$k]: $uri")
+                QuiltPath path = QuiltPathFactory.parse(uri)
+                checkPath(path)
             }
         }
-        return result
     }
 
     @Override
     void onFlowCreate(Session session) {
         log.debug("`onFlowCreate` $this")
         this.session = session
-        this.paths
+        checkParams(session.getParams())
     }
 
+    // NOTE: TraceFileObserver calls onFilePublish _before_ onFlowCreate
     @Override
     void onFilePublish(Path path) { //, Path source
         log.debug("onFilePublish.Path[$path]") //.Source[$source]
         QuiltPath qPath = asQuiltPath(path)
-
         if (qPath) {
-            QuiltPath npath = normalizePath(qPath, session.getParams())
-            this.paths.add(npath)
-            log.debug("onFilePublish.QuiltPath[$qPath]: paths=${paths}")
+            checkPath(qPath, true)
         } else {
-            log.warn("onFilePublish.QuiltPath missing: $path")
+            log.warn("onFilePublish.not.QuiltPath: $path")
         }
     }
 
     @Override
     void onFlowComplete() {
-        log.debug("`onFlowComplete` ${paths}")
-        // publish pkgs to repository
-        this.paths.each { path -> new QuiltProduct(path, session) }
+        log.debug("`onFlowComplete` ${publishedURIs}")
+        // create QuiltProduct for each unique package URI
+        publishedURIs.each { k, uri ->
+            QuiltPath path = QuiltPathFactory.parse(uri)
+            new QuiltProduct(path, session)
+        }
     }
 
 }

plugins/nf-quilt/src/main/nextflow/quilt/QuiltObserverFactory.groovy

@@ -32,7 +32,7 @@ class QuiltObserverFactory implements TraceObserverFactory {
 
     @Override
     Collection<TraceObserver> create(Session session) {
-        log.debug("`create` ${this}")
+        //log.debug("`create` ${this}")
         return (Collection<TraceObserver>) [new QuiltObserver()]
     }