From 27d2f76c9fc36a9b65bc6593d71a254675580a32 Mon Sep 17 00:00:00 2001 From: "Dr. Ernie Prabhakar" Date: Wed, 11 Sep 2024 21:00:12 -0700 Subject: [PATCH] feat(235) explicit output uri (#237) #235 and friends --------- Co-authored-by: Dr. Ernie Prabhakar <19791+drernie@users.noreply.github.com> --- .github/workflows/mega-linter.yml | 2 +- .github/workflows/test.yml | 6 +- CHANGELOG.md | 31 ++- README.md | 21 +- plugins/nf-quilt/build.gradle | 2 +- .../main/nextflow/quilt/QuiltObserver.groovy | 191 +++++++++++------ .../main/nextflow/quilt/QuiltProduct.groovy | 24 ++- .../nextflow/quilt/jep/QuiltPackage.groovy | 21 +- .../quilt/nio/QuiltFileSystemProvider.groovy | 25 +-- .../main/nextflow/quilt/nio/QuiltPath.groovy | 10 +- .../src/resources/META-INF/MANIFEST.MF | 2 +- .../nextflow/quilt/QuiltObserverTest.groovy | 199 +++++++++++------- .../test/nextflow/quilt/QuiltPkgTest.groovy | 4 +- .../nextflow/quilt/QuiltProductTest.groovy | 5 +- .../nextflow/quilt/QuiltSpecification.groovy | 8 +- .../quilt/jep/QuiltPackageTest.groovy | 15 +- .../nextflow/quilt/jep/QuiltParserTest.groovy | 6 +- .../nio/QuiltFileSystemProviderTest.groovy | 149 ++++++++++++- .../nextflow/quilt/nio/QuiltNioTest.groovy | 24 ++- .../nextflow/quilt/nio/QuiltPathTest.groovy | 142 +++++++++++-- 20 files changed, 647 insertions(+), 240 deletions(-) diff --git a/.github/workflows/mega-linter.yml b/.github/workflows/mega-linter.yml index 3d7c2630..dfb37e32 100644 --- a/.github/workflows/mega-linter.yml +++ b/.github/workflows/mega-linter.yml @@ -29,7 +29,7 @@ jobs: - name: Checkout Code uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 with: - token: ${{ secrets.PAT || secrets.GITHUB_TOKEN }} + token: ${{ secrets.GITHUB_TOKEN }} # secrets.PAT || fetch-depth: 0 # If you use VALIDATE_ALL_CODEBASE = true, you can remove this line to improve performances # MegaLinter diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3786751b..d9614b46 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,7 +20,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest, windows-latest] + os: [windows-latest, ubuntu-latest, macos-latest] java_version: [11, 17, 19] runs-on: ${{ matrix.os }} @@ -60,7 +60,7 @@ jobs: with: name: nf-quilt-test-reports-${{ matrix.os }}-${{ matrix.java_version }} path: | - D:\a\nf-quilt\nf-quilt\plugins\nf-quilt\build\reports\tests\test\ + D:\a\nf-quilt\nf-quilt\plugins\nf-quilt\build\reports\ overwrite: true - name: Archive production artifacts (Linux and MacOS) uses: actions/upload-artifact@v4 @@ -68,6 +68,6 @@ jobs: with: name: nf-quilt-test-reports-${{ matrix.os }}-${{ matrix.java_version }} path: | - ${{ github.workspace }}/plugins/nf-quilt/build/reports/tests/test/ + ${{ github.workspace }}/plugins/nf-quilt/build/reports/ overwrite: true diff --git a/CHANGELOG.md b/CHANGELOG.md index 13028869..51450b2d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,35 @@ # Changelog -## [0.8.1] UNRELEASED +## [0.8.6] 2024-09-11 -- Fix bug in path extraction from S3 URIs +- Fix addOverlay bug on subfolders +- Fix Windows tests +- Improve test coverage + +## [0.8.5] 2024-09-10a + +- Error with packaging subfolders on S3 overlay +- Improved overlay debugging + +## [0.8.4] 2024-09-10 + +- Fix bug with unrecognized output URIs + +## [0.8.3] 2024-09-08 + +- Fix Windows bug with overlay files + +## [0.8.2] 2024-09-07 + +- Use copyFile rather than writeString for overlay files [requires NextFlow 23 or later] +- Restore README and quilt_summarize to output + +## [0.8.1] 2024-09-05 + +- Get output URI directly from params +- Add `dest` parameter to Quilt URIs inferred from S3 URIs +- Specify `outputPrefixes` using `quilt` section of `nextflow.config` +- Stop proactively installing packages ## [0.8.0] 2024-08-31 diff --git a/README.md b/README.md index 0077f759..ff322e63 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,22 @@ Nextflow plugin for reading and writing Quilt packages as a FileSystem developed by [Quilt Data](https://quiltdata.com/) that enables you read and write directly to Quilt packages using `quilt+s3` URIs wherever your Nextflow pipeline currently use `s3` URIs. -In v0.8+, the plugin can even be used with "native" URIs, and it will automatically register a Quilt package at the root of the bucket. +## NEW: Use nf-quilt plugin with existing S3 URIs -Inspired by the original [`nf-quilt`](https://github.com/nextflow-io/nf-quilt) plugin (v0.2.0) developed by Seqera labs. +In v0.8+, the plugin can even be used with "native" S3 URIs. You can continue using your exising S3 URIs,and Nextflow will write the data out as usual. However, simply by adding the `nf-quilt` plugin, you can also "overlay" that data with a Quilt package containing all the metadata from that run. + +For example: + +```shell +nextflow run nf-core/rnaseq -plugins nf-quilt --outdir "s3://quilt-example-bucket/test/nf_quilt_rnaseq" +# other parameters omitted for brevity +``` + +will automatically create the package: + +```url +quilt+s3://quilt-example-bucket#package=test/nf_quilt_rnaseq +``` ## I. Using the nf-quilt plugin in Production @@ -80,8 +93,8 @@ From the command-line, do, e.g.: ```bash # export NXF_VER=23.04.3 export LOG4J_DEBUG=true # for verbose logging -export NXF_PLUGINS_TEST_REPOSITORY=https://github.com/quiltdata/nf-quilt/releases/download/0.8.0/nf-quilt-0.8.0-meta.json -nextflow run main.nf -plugins nf-quilt@0.8.0 +export NXF_PLUGINS_TEST_REPOSITORY=https://github.com/quiltdata/nf-quilt/releases/download/0.8.6/nf-quilt-0.8.6-meta.json +nextflow run main.nf -plugins nf-quilt@0.8.6 ``` For Tower, you can use the "Pre-run script" to set the environment variables. diff --git a/plugins/nf-quilt/build.gradle b/plugins/nf-quilt/build.gradle index 8b77bf35..767f18f2 100644 --- a/plugins/nf-quilt/build.gradle +++ b/plugins/nf-quilt/build.gradle @@ -125,7 +125,7 @@ jacocoTestCoverageVerification { violationRules { rule { limit { - minimum = 0.65 + minimum = 0.7 } } diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltObserver.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltObserver.groovy index 874c0934..5381fbd9 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltObserver.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltObserver.groovy @@ -17,6 +17,7 @@ package nextflow.quilt import nextflow.Session import nextflow.quilt.jep.QuiltParser +import nextflow.quilt.jep.QuiltPackage import nextflow.quilt.nio.QuiltPath import nextflow.quilt.nio.QuiltPathFactory import nextflow.trace.TraceObserver @@ -38,10 +39,11 @@ import groovy.util.logging.Slf4j class QuiltObserver implements TraceObserver { private Session session - final private Map uniqueURIs = [:] - final private Map publishedURIs = [:] + private String workDir - // Is this overkill? Do we only ever have one output package per run? + // Is this overkill? Do we only ever have one output URI per run? + private String[] outputPrefixes = ['pub', 'out'] + final private Map outputURIs = [:] final private Map> packageOverlays = [:] final private Lock lock = new ReentrantLock() // Need this because of threads @@ -57,107 +59,162 @@ class QuiltObserver implements TraceObserver { return null } - static String pkgKey(QuiltPath path) { - return "${path.getBucket()}/${path.getPackageName()}" + static String quiltURIfromS3(String s3uri) { + log.debug("quiltURIfromS3: $s3uri") + String[] partsArray = s3uri.split('/') + List parts = new ArrayList(partsArray.toList()) + // parts.eachWithIndex { p, i -> println("quiltURIfromS3.parts[$i]: $p") } + + if (parts.size() < 2) { + throw new IllegalArgumentException("Invalid s3uri[${parts.size()}]: $parts") + } + parts = parts.drop(2) + if (parts[0].endsWith(':')) { + parts = parts.drop(1) + } + String bucket = parts.remove(0) + String dest = parts.join('%2f') + String suffix = parts.size() > 1 ? parts.removeLast() : 'default_suffix' + String prefix = parts.size() > 0 ? parts.removeLast() : 'default_prefix' + String base = "quilt+s3://${bucket}#package=${prefix}%2f${suffix}" + String uri = base + '&dest=' + ((dest) ?: '/') + return uri } - static String pathless(String uri) { - return uri.replaceFirst(/&path=[^&]+/, '') + static String pkgKey(QuiltPath path) { + return QuiltPackage.osConvert("${path.getBucket()}/${path.getPackageName()}") } - String checkPath(QuiltPath path, boolean published = false) { - log.debug("checkPath[$path] published[$published]") - String key = pkgKey(path) - String uri = pathless(path.toUriString()) - // only keep the longest pathless URI for each key - if (uniqueURIs[key]?.length() < uri.length()) { - uniqueURIs[key] = uri - } - if (published) { - publishedURIs[key] = uniqueURIs[key] + void findOutputParams(Map params) { + log.debug("findOutputParams[$params]") + params.each { key, value -> + String uri = "$value" + if (outputPrefixes.any { key.startsWith(it) && !key.contains('-') }) { + String[] splits = uri.split(':') + if (splits.size() < 2) { + log.debug("Unrecognized URI[$uri] for key[$key] matching $outputPrefixes") + return + } + String scheme = splits[0] + if (scheme == 's3') { + uri = quiltURIfromS3(uri) + } else if (scheme != 'quilt+s3') { + log.warn("Unrecognized scheme:$scheme for output URI[$key]: $uri") + return + } + QuiltPath path = QuiltPathFactory.parse(uri) + String pkgKey = pkgKey(path) + outputURIs[pkgKey] = uri + } } - return uniqueURIs[key] } - String extractPackageURI(Path nonQuiltPath) { - String pathString = nonQuiltPath.toUri() - // println("extractPackageURI.pathString[${nonQuiltPath}] -> $pathString") - String[] partsArray = pathString.split('/') - List parts = new ArrayList(partsArray.toList()) - // parts.eachWithIndex { p, i -> println("extractPackageURI.parts[$i]: $p") } - - if (parts.size() < 3) { - throw new IllegalArgumentException("Invalid pathString: $pathString ($nonQuiltPath)") + void checkConfig(Map> config) { + Object prefixes = config.get('quilt')?.get('outputPrefixes') + if (prefixes) { + outputPrefixes = prefixes as String[] } - parts = parts.drop(3) - if (parts[0].endsWith(':')) { - parts = parts.drop(1) + } + + String workRelative(Path src) { + Path source = src.toAbsolutePath().normalize() + Path workDir = session.workDir.toAbsolutePath().normalize() + try { + Path subPath = workDir.relativize(source) + // drop first two components, which are the workDir + Path relPath = subPath.subpath(2, subPath.getNameCount()) + return relPath.toString() + } catch (IllegalArgumentException e) { + log.error("workRelative.fallback: $e") + log.warn("Cannot relativize source:${source.getClass()} to workDir:${workDir.getClass()}") + return source.toString() } - String bucket = parts.remove(0) - String file_path = parts.remove(parts.size() - 1) - String prefix = parts.size() > 0 ? parts.remove(0) : 'default_prefix' - String suffix = parts.size() > 0 ? parts.remove(0) : 'default_suffix' - if (parts.size() > 0) { - String folder_path = parts.join('/') - file_path = folder_path + '/' + file_path + } + + String pkgRelative(String key, Path dest) { + String destString = QuiltPackage.osConvert(dest.toAbsolutePath().normalize().toString()) + String pkgKey = QuiltPackage.osConvert(key) + // find pkgKey in destination.toString() + int index = destString.indexOf(pkgKey) + println("pkgRelative[$index]: $pkgKey in $destString") + // return the portion after the end of pkgKey + int len = index + pkgKey.length() + 1 + if (index >= 0 && len < destString.length()) { + return destString.substring(len) } + return null + } - // TODO: should overlay packages always force to new versions? - String base = "quilt+s3://${bucket}#package=${prefix}%2f${suffix}" - String uri = "${base}&path=${file_path}" - - String key = pkgKey(QuiltPathFactory.parse(uri)) - Map current = packageOverlays.get(key, [:]) as Map - current[file_path] = nonQuiltPath - lock.withLock { - uniqueURIs[key] = base - publishedURIs[key] = base - packageOverlays[key] = current + String addOverlay(String pkgKey, Path dest, Path source) { + lock.lock() + try { + Map overlays = packageOverlays.get(pkgKey, [:]) as Map + String relPath = pkgRelative(pkgKey, dest) + println("addOverlay.relPath: $relPath") + log.debug("addOverlay[$relPath] = dest:$dest <= source:$source") + overlays[relPath] = source + packageOverlays[pkgKey] = overlays + return relPath + } finally { + lock.unlock() } - return uri + return null } - void checkParams(Map params) { - log.debug("checkParams[$params]") - params.each { k, value -> - String uri = "$value" - if (uri.startsWith(QuiltParser.SCHEME)) { - log.debug("checkParams.uri[$k]: $uri") - QuiltPath path = QuiltPathFactory.parse(uri) - checkPath(path) + boolean confirmQuiltPath(QuiltPath qPath) { + log.debug("confirmQuiltPath[$qPath]") + String key = pkgKey(qPath) + log.debug("confirmQuiltPath: key[$key] in outputURIs[${outputURIs.size()}]: $outputURIs") + return outputURIs.containsKey(key) ? true : false + } + + boolean canOverlayPath(Path dest, Path source) { + log.debug("canOverlayPath[$dest] <- $source") + Set keys = outputURIs.keySet() + for (String key : keys) { + if (dest.toString().contains(key)) { + log.debug("canOverlayPath: matched key[$key] to $dest") + addOverlay(key, dest, source) + return true } } + log.error("canOverlayPath: no key found for $dest in $keys") + return false } @Override void onFlowCreate(Session session) { log.debug("`onFlowCreate` $this") this.session = session - checkParams(session.getParams()) + this.workDir = session.config.workDir + findOutputParams(session.getParams()) + checkConfig(session.config) } - // NOTE: TraceFileObserver calls onFilePublish _before_ onFlowCreate @Override void onFilePublish(Path destination, Path source) { // Path source may be null, won't work with older versions of Nextflow log.debug("onFilePublish.Path[$destination] <- $source") + if (!outputURIs) { + // NOTE: TraceFileObserver calls onFilePublish _before_ onFlowCreate + log.debug('onFilePublish: no outputURIs yet') + return + } QuiltPath qPath = asQuiltPath(destination) - if (qPath) { - checkPath(qPath, true) - } else { - String uri = extractPackageURI(destination) - log.debug("onFilePublish.NonQuiltPath[$destination]: $uri") + boolean ok = (qPath != null) ? confirmQuiltPath(qPath) : canOverlayPath(destination, source) + if (!ok) { + log.error("onFilePublish: no match for $destination") } } @Override void onFlowComplete() { - log.debug("onFlowComplete.publishedURIs[${publishedURIs.size()}]: $publishedURIs") + log.debug("onFlowComplete.outputURIs[${outputURIs.size()}]: $outputURIs") // create QuiltProduct for each unique package URI - publishedURIs.each { key, uri -> + outputURIs.each { key, uri -> QuiltPath path = QuiltPathFactory.parse(uri) Map overlays = packageOverlays.get(key, [:]) as Map - // log.debug("onFlowComplete.pkg: $path overlays[${overlays?.size()}]: $overlays") + log.debug("onFlowComplete.pkg: $path overlays[${overlays?.size()}]: $overlays") new QuiltProduct(path, session, overlays) } } diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy index 2c4b3143..a82f7559 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy @@ -106,6 +106,17 @@ ${nextflow} } } + static void copyFile(Path source, String destRoot, String relpath) { + Path dest = Paths.get(destRoot, relpath.split('/') as String[]) + try { + dest.getParent().toFile().mkdirs() // ensure directories exist first + Files.copy(source, dest) + } + catch (Exception e) { + log.error("writeString: cannot write `$source` to `$dest` in `${destRoot}`") + } + } + static String now() { LocalDateTime time = LocalDateTime.now() return time.toString() @@ -127,7 +138,7 @@ ${nextflow} if (session.isSuccess() || pkg.is_force()) { if (overlays) { - log.info("publishing overlays: ${overlays.size()}") + log.debug("publishing overlays: ${overlays.size()}") publishOverlays(overlays) } else { log.info('No overlays to publish.') @@ -139,15 +150,20 @@ ${nextflow} } void publishOverlays(Map overlays) { - overlays.each { key, overlay -> - log.info("publishing overlay[$key]: ${overlay}") - writeString(overlay.text, pkg, key) + /// Copying published files to inside package directory + /// for (re)upload to the package + /// FIXME: Replace this with in-place packaging + overlays.each { relpath, source -> + log.info("publishing overlay[$relpath]: ${source}") + copyFile(source, pkg.packageDest().toString(), relpath) } } void publish() { log.debug("publish($msg)") meta = setupMeta() + setupReadme() + setupSummarize() try { log.info("publish.pushing: ${pkg}") def m = pkg.push(msg, meta) diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy index be5e19cb..893dff9a 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy @@ -20,6 +20,7 @@ package nextflow.quilt.jep import groovy.json.JsonOutput import groovy.transform.CompileStatic import groovy.util.logging.Slf4j +import java.nio.file.FileSystems import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths @@ -55,6 +56,18 @@ class QuiltPackage { private final Map meta private boolean installed + static String osSep() { + return FileSystems.getDefault().getSeparator() + } + + static String osJoin(String... parts) { + return parts.join(osSep()) + } + + static String osConvert(String path) { + return path.replace('/', FileSystems.getDefault().getSeparator()) + } + static String today() { LocalDate date = LocalDate.now() return date.toString() @@ -153,7 +166,7 @@ class QuiltPackage { */ List relativeChildren(String subpath) { Path subfolder = folder.resolve(subpath) - String base = subfolder.toString() + '/' + String base = subfolder.toString() + osSep() List result = [] final String[] children = subfolder.list().sort() //log.debug("relativeChildren[${base}] $children") @@ -172,10 +185,6 @@ class QuiltPackage { void setup() { Files.createDirectories(this.folder) this.installed = false - if (!this.is_force()) { - log.debug("QuiltPackage.setup.install.options: $parsed.options") - install(true) // FIXME: only needed for nextflow < 23.12? - } } boolean is_force() { @@ -215,7 +224,7 @@ class QuiltPackage { manifest.install(dest) log.info("install: ${implicitStr}installed into $dest)") - println("QuiltPackage.install.Children: ${relativeChildren('')}") + log.debug("QuiltPackage.install.Children: ${relativeChildren('')}") } catch (IOException e) { if (!implicit) { log.error("failed to install $packageName", e) diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy index 47fa2a05..8165affc 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy @@ -33,6 +33,7 @@ import java.nio.file.LinkOption import java.nio.file.NoSuchFileException import java.nio.file.OpenOption import java.nio.file.Path +import java.nio.file.Paths import java.nio.file.StandardOpenOption import java.nio.file.attribute.BasicFileAttributeView import java.nio.file.attribute.BasicFileAttributes @@ -71,11 +72,6 @@ class QuiltFileSystemProvider extends FileSystemProvider implements FileSystemTr if (path in QuiltPath) { return (QuiltPath)path } - String pathString = path?.toString() ?: '-' - if (pathString.startsWith(QuiltParser.SCHEME + ':/')) { - QuiltPath qPath = QuiltPathFactory.parse(pathString) - return qPath - } String pathClassName = path?.class?.name ?: '-' throw new IllegalArgumentException( "Not a valid Quilt blob storage path object: `${path}` [${pathClassName}]" @@ -85,10 +81,6 @@ class QuiltFileSystemProvider extends FileSystemProvider implements FileSystemTr static QuiltFileSystem getQuiltFilesystem(Path path) { final qPath = asQuiltPath(path) final fs = qPath.getFileSystem() - if (fs !in QuiltFileSystem) { - String pathClassName = path?.class?.name ?: '-' - throw new IllegalArgumentException("Not a valid Quilt file system: `$fs` [${pathClassName}]") - } return (QuiltFileSystem)fs } @@ -114,12 +106,9 @@ class QuiltFileSystemProvider extends FileSystemProvider implements FileSystemTr log.debug "QuiltFileSystemProvider.download: ${remoteFile} -> ${localDestination}" QuiltPath qPath = asQuiltPath(remoteFile) Path cachedFile = qPath.localPath() - /* - * UNUSED: QuiltPackage is always installed - * QuiltPackage pkg = qPath.pkg() + log.info "download Quilt package: ${pkg} installed? ${pkg.installed}" if (!pkg.installed) { - log.info "download.install Quilt package: ${pkg}" Path dest = pkg.install() if (!dest) { log.error "download.install failed: ${pkg}" @@ -127,12 +116,6 @@ class QuiltFileSystemProvider extends FileSystemProvider implements FileSystemTr } log.info "download.installed Quilt package to: $dest" } - */ - - if (!Files.exists(cachedFile)) { - log.error "download: File ${cachedFile} not found" - throw new NoSuchFileException(remoteFile.toString()) - } final CopyOptions opts = CopyOptions.parse(options) // delete target if it exists and REPLACE_EXISTING is specified @@ -323,7 +306,7 @@ class QuiltFileSystemProvider extends FileSystemProvider implements FileSystemTr } void checkRoot(Path path) { - if (path.toString() == '/') { + if (path == Paths.get('/')) { throw new UnsupportedOperationException("Operation 'checkRoot' not supported on root path") } } @@ -479,7 +462,7 @@ class QuiltFileSystemProvider extends FileSystemProvider implements FileSystemTr QuiltFileSystem fs = qPath.filesystem return (V)fs.getFileAttributeView(qPath) } - throw new UnsupportedOperationException("Operation 'getFileAttributeView' is not supported by QuiltFileSystem") + throw new UnsupportedOperationException("Operation 'getFileAttributeView' is not supported for type $type") } @Override diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltPath.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltPath.groovy index e8e3135a..423831f6 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltPath.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltPath.groovy @@ -214,15 +214,17 @@ final class QuiltPath implements Path, Comparable { Path relativize(Path other) { if (this == other) { return null } String file = (other in QuiltPath) ? ((QuiltPath)other).localPath() : other.toString() - String base = [pkg().toString(), parsed.getPath()].join(QuiltParser.SEP) - //log.debug("relativize[$base] in [$file]") + String base = QuiltPackage.osConvert("${pkg()}/${parsed.getPath()}") + log.debug("relativize[$base] in [$file]") int i = file.indexOf(base) if (i < 1) { - throw new UnsupportedOperationException("other[$file] does not contain package[$base]") + throw new IllegalArgumentException("other[$file] does not contain package[$base]") } String tail = file.substring(i + base.size()) - if (tail.size() > 0 && tail[0] == '/') { tail = tail.substring(1) } // drop leading "/" + if (tail.size() > 0 && (tail[0] == '/' || tail[0] == '\\')) { + tail = tail.substring(1) + } // drop leading separator //log.debug("tail[$i] -> $tail") return Paths.get(tail) } diff --git a/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF b/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF index 8c805a79..dbeabc13 100644 --- a/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF +++ b/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF @@ -1,7 +1,7 @@ Manifest-Version: 1.0 Plugin-Class: nextflow.quilt.QuiltPlugin Plugin-Id: nf-quilt -Plugin-Version: 0.8.0 +Plugin-Version: 0.8.6 Plugin-Provider: Quilt Data Plugin-Requires: >=22.10.6 diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltObserverTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltObserverTest.groovy index 439c2ceb..f6154f6a 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltObserverTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltObserverTest.groovy @@ -18,12 +18,13 @@ package nextflow.quilt.nio import nextflow.quilt.QuiltSpecification import nextflow.quilt.QuiltObserver +import nextflow.quilt.jep.QuiltPackage import nextflow.Session +//import spock.lang.Ignore import java.nio.file.Path import java.nio.file.Paths import groovy.transform.CompileDynamic -import spock.lang.Ignore /** * @@ -32,115 +33,153 @@ import spock.lang.Ignore @CompileDynamic class QuiltObserverTest extends QuiltSpecification { - void 'should extract Quilt path from appropriate UNIX Path'() { - given: - Path pkg = Paths.get('/var/tmp/output/quilt-example#package=examples%2fhurdat') - Path unpkg = Paths.get('/var/tmp/output/quilt-example/examples/hurdat') + private static final String SPEC_KEY = 'udp-spec/nf-quilt/source' + private static final String TEST_KEY = 'bkt/pre/suf' + + Session mockSession(boolean success = false) { + String quilt_uri = 'quilt+s3://bucket#package=prefix%2fsuffix' + return GroovyMock(Session) { + getParams() >> [pubNot: 'foo', pubBad: 'foo:bar', outdir: SpecURI(), pubDir: testURI, inDir: quilt_uri] + isSuccess() >> success + config >> [quilt: [outputPrefixes: ['pub']]] + workDir >> Paths.get('./work') + } + } + QuiltObserver makeObserver(boolean success = false) { + QuiltObserver observer = new QuiltObserver() + observer.onFlowCreate(mockSession(success)) + return observer + } + + void 'should extract appropriate UNIX Path asQuiltPath'() { expect: - QuiltObserver.asQuiltPath(unpkg) == null - QuiltObserver.asQuiltPath(pkg).toString() == 'quilt-example#package=examples%2fhurdat' + String unixFolder = "/var/tmp/output/${pkgString}" + Path unixPath = Paths.get(unixFolder) + QuiltObserver.asQuiltPath(unixPath).toString() == pkgString + where: + pkgString << ['quilt-example#package=examples%2fhurdat', 'udp-spec#package=nf-quilt%2fsource'] } - void 'normalized Paths from params, and match'() { + void 'should form pkgKey from QuiltPath'() { given: - String subURL = fullURL.replace('?key=val&key2=val2', '') - String noURL = fullURL.replace('bkt', 'bucket') - String newURL = fullURL.replace('bkt', 'new-bucket') - Session session = Stub(Session) - session.getParams() >> [subdir: subURL, outdir: fullURL, nodir: noURL] - QuiltObserver observer = new QuiltObserver() + Path testPath = QuiltPathFactory.parse(testURI) + Path specPath = QuiltPathFactory.parse(SpecURI()) + expect: + QuiltObserver.pkgKey(testPath) == QuiltPackage.osConvert(TEST_KEY) + QuiltObserver.pkgKey(specPath) == QuiltPackage.osConvert(SPEC_KEY) + } - when: - observer.onFlowCreate(session) - String n_bkt = observer.uniqueURIs['bkt/pre/suf'] - String n_bucket = observer.uniqueURIs['bucket/pre/suf'] - String n_new = QuiltObserver.pathless(newURL).replace('pre/suf', 'pre%2fsuf') + void 'should extract quiltURIfromS3'() { + expect: + QuiltObserver.quiltURIfromS3(s3_uri) == quilt_uri + where: + s3_uri | quilt_uri + 's3://bucket/prefix/suffix' | 'quilt+s3://bucket#package=prefix%2fsuffix&dest=prefix%2fsuffix' + 's3://bucket/prefix' | 'quilt+s3://bucket#package=prefix%2fdefault_suffix&dest=prefix' + 's3://bucket' | 'quilt+s3://bucket#package=default_prefix%2fdefault_suffix&dest=/' + 's3://bucket/folder/prefix/suffix' | 'quilt+s3://bucket#package=prefix%2fsuffix&dest=folder%2fprefix%2fsuffix' + } - then: - observer - n_bkt != null - n_bucket != null - fullURL.contains('&path=') - !n_bkt.contains('&path=') - !n_bucket.contains('&path=') - n_bkt.split('#')[0] == 'quilt+s3://bkt?key=val&key2=val2' - n_bkt.contains('quilt+s3://bkt?key=val&key2=val2') - n_bucket.contains('quilt+s3://bucket?key=val&key2=val2') + void 'should return workRelative path for source'() { + given: + QuiltObserver observer = makeObserver() + Path workDir = observer.session.workDir + String subPath = 'output/file.txt' + String workPath = "job/hash/${subPath}" + Path source = Paths.get(workDir.toString(), workPath) + expect: + String relPath = observer.workRelative(source) + relPath == QuiltPackage.osConvert(subPath) + } - when: - Path fullPath = QuiltPathFactory.parse(fullURL) - Path subPath = QuiltPathFactory.parse(subURL) - Path noPath = QuiltPathFactory.parse(noURL) - Path newPath = QuiltPathFactory.parse(newURL) + void 'should return pkgRelative path for dest'() { + given: + QuiltObserver observer = makeObserver() + expect: + Path dest = Paths.get(TEST_KEY, folderPath) + String relPath = observer.pkgRelative(offset, dest) + rc == (relPath == QuiltPackage.osConvert(folderPath)) + where: + rc | offset | folderPath + true | TEST_KEY | 'output/file.txt' + false | SPEC_KEY | 'output/file.txt' + } - then: - observer.checkPath(fullPath) == n_bkt - observer.checkPath(subPath) == n_bkt - observer.checkPath(noPath) == n_bucket - observer.checkPath(newPath) == n_new + void 'should findOutputParams'() { + given: + QuiltObserver observer = makeObserver() + String targetKey = QuiltPackage.osConvert('bucket/prefix/suffix') + expect: + String key = QuiltPackage.osConvert(unixKey) + observer.outputURIs + !observer.outputURIs.containsKey(targetKey) + observer.outputURIs.size() == 2 + + observer.outputURIs.containsKey(key) + observer.outputURIs[key] == uri + observer.confirmQuiltPath(QuiltPathFactory.parse(uri)) + where: + unixKey | uri + SPEC_KEY | SpecURI() + TEST_KEY | testURI } - // FIXME: Should infer package name from pubdir/outdir parameters, not just each file's path - void 'should extract package URI from output path'() { + void 'should set outputPrefixes from config'() { given: QuiltObserver observer = new QuiltObserver() - Path path = Paths.get(filepath) - String uri = observer.extractPackageURI(path) + Map> config = ['quilt': ['outputPrefixes': ['bucket', 'file']]] + observer.checkConfig(config) expect: - uri == quilt_uri - where: - filepath | quilt_uri - '/bucket/prefix/suffix/folder/file.ext' | 'quilt+s3://bucket#package=prefix%2fsuffix&path=folder/file.ext' - '/bucket/prefix/suffix/file.ext' | 'quilt+s3://bucket#package=prefix%2fsuffix&path=file.ext' - '/bucket/prefix/file.ext' | 'quilt+s3://bucket#package=prefix%2fdefault_suffix&path=file.ext' - '/bucket/file.ext' | 'quilt+s3://bucket#package=default_prefix%2fdefault_suffix&path=file.ext' + observer.outputPrefixes.size() == 2 + observer.outputPrefixes.contains('bucket') + observer.outputPrefixes.contains('file') } - void 'should recover URI from onFilePublish QuiltPath'() { + void 'should not confirmQuiltPath for non-output URIs'() { given: QuiltObserver observer = new QuiltObserver() - Path path = Paths.get(key) - Path quiltPath = QuiltPathFactory.parse(quilt_uri) - observer.onFilePublish(quiltPath, path) - String pkgKey = observer.pkgKey(quiltPath) + QuiltPath specPath = QuiltPathFactory.parse(SpecURI()) + QuiltPath testPath = QuiltPathFactory.parse(testURI) expect: - pkgKey == key - observer.uniqueURIs[key] == quilt_uri - observer.publishedURIs[key] == quilt_uri + !observer.confirmQuiltPath(specPath) + !observer.confirmQuiltPath(testPath) + } + + void 'should return: #rc if canOverlayPath with: #path in: #root'() { + given: + QuiltObserver observer = makeObserver() + expect: + rc == observer.canOverlayPath(Paths.get(root, path), Paths.get(path)) where: - key | quilt_uri - 'bucket/prefix/suffix' | 'quilt+s3://bucket#package=prefix%2fsuffix' + rc | root | path + true | SPEC_KEY | 'output/file.txt' + true | TEST_KEY | 'output/file.txt' + false | '/root' | 'output/file.txt' } - @Ignore('FIXME: handle onFilePublish with local Path') - void 'should extract URI from onFilePublish local Path'() { + /// source usually lacks subfolder paths + void 'should addOverlay logical path with subfolders'() { given: - QuiltObserver observer = new QuiltObserver() - Path quiltPath = QuiltPathFactory.parse(quilt_uri) - Path path = Paths.get('/'+key) - observer.onFilePublish(path, quiltPath) - String pkgKey = observer.pkgKey(quiltPath) + QuiltObserver observer = makeObserver() + String file_path = 'source' + Path source = Paths.get(root, file_path) + Path dest = Paths.get(root, path) expect: - pkgKey == key - observer.uniqueURIs[key] == quilt_uri - observer.publishedURIs[key] == quilt_uri + String relPath = observer.addOverlay(TEST_KEY, dest, source) + relPath == result ?: QuiltPackage.osConvert(result) where: - key | quilt_uri - 'bucket/prefix/suffix' | 'quilt+s3://bucket#package=prefix%2fsuffix' + root | path | result + TEST_KEY | SPEC_KEY | SPEC_KEY + SPEC_KEY | TEST_KEY | null } - void 'should not error on onFlowComplete'() { + void 'should not error on onFlowComplete success'() { given: String quilt_uri = 'quilt+s3://bucket#package=prefix%2fsuffix' QuiltObserver observer = new QuiltObserver() QuiltPath qPath = QuiltPathFactory.parse(quilt_uri) - Session session = GroovyMock(Session) { - // getWorkflowMetadata() >> metadata - getParams() >> [outdir: quilt_uri] - isSuccess() >> false - } - observer.onFlowCreate(session) - observer.onFilePublish(qPath, qPath) + observer.onFlowCreate(mockSession(false)) + observer.onFilePublish(qPath.localPath()) when: observer.onFlowComplete() then: diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltPkgTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltPkgTest.groovy index 2f44656a..668df423 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltPkgTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltPkgTest.groovy @@ -40,9 +40,9 @@ class QuiltPkgTest extends QuiltSpecification { ] private static QuiltPackage GetPackage(String suffix) { - String baseURI = 'quilt+s3://udp-spec#package=nf-quilt/' + String baseURI = SpecURI().replace('source', suffix) QuiltPathFactory factory = new QuiltPathFactory() - QuiltPath qpath = factory.parseUri(baseURI + suffix) + QuiltPath qpath = factory.parseUri(baseURI) println("Parsed: $qpath") QuiltPackage pkg = qpath.pkg() println("Package: $pkg") diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltProductTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltProductTest.groovy index a1840fcd..727c0674 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltProductTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltProductTest.groovy @@ -40,7 +40,7 @@ import spock.lang.Unroll class QuiltProductTest extends QuiltSpecification { QuiltProduct makeProduct(String query=null, boolean success = false) { - String subURL = query ? fullURL.replace('key=val&key2=val2', query) : fullURL + String subURL = query ? testURI.replace('key=val&key2=val2', query) : testURI WorkflowMetadata metadata = GroovyMock(WorkflowMetadata) { toMap() >> [start:'2022-01-01', complete:'2022-01-02'] } @@ -218,7 +218,8 @@ class QuiltProductTest extends QuiltSpecification { } @Unroll - @IgnoreIf({ env.WRITE_BUCKET == 'quilt-example' || env.WRITE_BUCKET == null }) + @IgnoreIf({ env.WRITE_BUCKET == null }) + @Ignore('Invalid test: top-level summarize') void 'should summarize top-level readable files + multiqc '() { given: String sumURL = writeableURL('summarized') diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltSpecification.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltSpecification.groovy index db46b46f..b38ab544 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltSpecification.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltSpecification.groovy @@ -47,7 +47,11 @@ import spock.lang.Specification @CompileDynamic class QuiltSpecification extends Specification { - @Shared String fullURL + static String SpecURI() { + return 'quilt+s3://udp-spec#package=nf-quilt/source' + } + + @Shared String testURI @Shared String pluginsMode @@ -59,7 +63,7 @@ class QuiltSpecification extends Specification { // reset previous instances PluginExtensionProvider.reset() // this need to be set *before* the plugin manager class is created - fullURL = 'quilt+s3://bkt?key=val&key2=val2' + + testURI = 'quilt+s3://bkt?key=val&key2=val2' + '#package=pre/suf@abcdef314159265'+ '&path=p/t&property=prop&workflow=wf&catalog=quiltdata.com' pluginsMode = System.getProperty('pf4j.mode') diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy index c33fc9c1..9f8e93d4 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy @@ -196,7 +196,8 @@ class QuiltPackageTest extends QuiltSpecification { println("read-only-bucket:TEST_URL: ${READONLY_URL}") def qout = factory.parseUri(READONLY_URL) def opkg = qout.pkg() - println("opkg: ${opkg}") + opkg.install() + println("opkg: ${opkg} installed: ${opkg.isInstalled()}") def outPath = Paths.get(opkg.packageDest().toString(), 'foo/bar.txt') println("outPath: ${outPath}") Files.writeString(outPath, "Time: ${timestamp}") @@ -236,7 +237,7 @@ class QuiltPackageTest extends QuiltSpecification { } // TODO: ensure metadata is correctly inserted into package - @IgnoreIf({ env.WRITE_BUCKET == 'quilt-example' || env.WRITE_BUCKET == null }) + @IgnoreIf({ env.WRITE_BUCKET == null }) void 'should not fail pushing invalid metadata '() { given: QuiltPackage opkg = writeablePackage('observer') @@ -245,7 +246,7 @@ class QuiltPackageTest extends QuiltSpecification { opkg.push('msg', meta) } - @IgnoreIf({ env.WRITE_BUCKET == 'quilt-example' || env.WRITE_BUCKET == null }) + @IgnoreIf({ env.WRITE_BUCKET == null }) void 'should fail if invalid workflow'() { given: String pkgName = "workflow-bad-${timestamp}" @@ -253,10 +254,10 @@ class QuiltPackageTest extends QuiltSpecification { when: bad_wf.push('missing-workflow first time', [:]) then: - thrown(com.quiltdata.quiltcore.workflows.WorkflowException) + thrown(RuntimeException) } - @IgnoreIf({ env.WRITE_BUCKET == 'quilt-example' || env.WRITE_BUCKET == null }) + @IgnoreIf({ env.WRITE_BUCKET == null }) void 'should fail push if unsatisfied workflow'() { given: Map meta = [ @@ -271,12 +272,12 @@ class QuiltPackageTest extends QuiltSpecification { when: good_wf.push('empty meta', [:]) then: - thrown(com.quiltdata.quiltcore.workflows.WorkflowException) + thrown(RuntimeException) when: good_wf.push('bad_meta', bad_meta) then: - thrown(com.quiltdata.quiltcore.workflows.WorkflowException) + thrown(RuntimeException) expect: good_wf.push('my-workflow', meta) diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltParserTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltParserTest.groovy index 0d570e14..dece83a1 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltParserTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltParserTest.groovy @@ -99,7 +99,7 @@ class QuiltParserTest extends QuiltSpecification { void 'should extract other parameters from URI'() { when: - QuiltParser parser = QuiltParser.forUriString(fullURL) + QuiltParser parser = QuiltParser.forUriString(testURI) Map meta = parser.getMetadata() then: @@ -116,11 +116,11 @@ class QuiltParserTest extends QuiltSpecification { void 'should unparse other parameters back to URI'() { when: - QuiltParser parser = QuiltParser.forUriString(fullURL) + QuiltParser parser = QuiltParser.forUriString(testURI) String unparsed = parser.toUriString() then: - unparsed.replace('%2f', '/') == fullURL + unparsed.replace('%2f', '/') == testURI } void 'should collect array parameters from query string'() { diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltFileSystemProviderTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltFileSystemProviderTest.groovy index 562a9465..a4a71f6c 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltFileSystemProviderTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltFileSystemProviderTest.groovy @@ -8,7 +8,9 @@ import java.nio.file.Paths import java.nio.file.Files import java.nio.file.CopyOption import java.nio.file.StandardCopyOption +import java.nio.file.DirectoryStream import groovy.util.logging.Slf4j +import spock.lang.IgnoreIf /** * @@ -19,7 +21,7 @@ import groovy.util.logging.Slf4j class QuiltFileSystemProviderTest extends QuiltSpecification { static Path parsedURIWithPath(boolean withPath = false) { - String packageURI = 'quilt+s3://udp-spec#package=nf-quilt/source' + String packageURI = SpecURI() if (withPath) { packageURI += '&path=COPY_THIS.md' } @@ -68,6 +70,9 @@ class QuiltFileSystemProviderTest extends QuiltSpecification { Path tempFolder = Files.createTempDirectory('quilt') Path tempFile = tempFolder.resolve(filename) + expect: + !Files.exists(tempFile) + when: provider.download(remoteFile, tempFile) @@ -90,6 +95,42 @@ class QuiltFileSystemProviderTest extends QuiltSpecification { Files.list(tempFolder).count() > 0 } + void 'should fail to download a file if already exists'() { + given: + QuiltFileSystemProvider provider = new QuiltFileSystemProvider() + Path remoteFolder = parsedURIWithPath(false) + Path tempFolder = Files.createTempDirectory('quilt') + when: + provider.download(remoteFolder, tempFolder, null) + + then: + thrown java.nio.file.FileAlreadyExistsException + } + + @IgnoreIf({ env.WRITE_BUCKET == null }) + void 'should upload file to test bucket'() { + given: + QuiltFileSystemProvider provider = new QuiltFileSystemProvider() + String url = writeableURL('upload') + String filename = 'UPLOAD_THIS.md' + QuiltPath remotePath = QuiltPathFactory.parse(url) + QuiltPath remoteFile = remotePath.resolveSibling(filename) + Path tempFolder = Files.createTempDirectory('quilt') + Path tempFile = tempFolder.resolve(filename) + // write test file + Files.writeString(tempFile, 'This is a test file') + + expect: + !Files.exists(remoteFile.localPath()) + + when: + provider.upload(tempFile, remoteFile) + + then: + Files.exists(remoteFile) + Files.size(remoteFile) > 0 + } + void 'should fail to upload a file to itself'() { given: QuiltFileSystemProvider provider = new QuiltFileSystemProvider() @@ -102,6 +143,28 @@ class QuiltFileSystemProviderTest extends QuiltSpecification { thrown java.nio.file.FileAlreadyExistsException } + void 'should throw error when checkRoot is root'() { + given: + QuiltFileSystemProvider provider = new QuiltFileSystemProvider() + Path remoteFile = Paths.get('/') + + when: + provider.checkRoot(remoteFile) + + then: + thrown UnsupportedOperationException + } + + void 'should return DirectoryStream for emptyStream'() { + given: + QuiltFileSystemProvider provider = new QuiltFileSystemProvider() + DirectoryStream stream = provider.emptyStream() + + expect: + stream != null + stream.iterator().hasNext() == false + } + void 'should error when copying from remote to local path'() { given: QuiltFileSystemProvider provider = new QuiltFileSystemProvider() @@ -129,4 +192,88 @@ class QuiltFileSystemProviderTest extends QuiltSpecification { Files.exists(remoteFile.localPath()) } + void 'should error when moving from remote to local path'() { + given: + QuiltFileSystemProvider provider = new QuiltFileSystemProvider() + Path remoteFile = parsedURIWithPath(true) + String filename = remoteFile.getFileName() + Path tempFolder = Files.createTempDirectory('quilt') + Path tempFile = tempFolder.resolve(filename) + + when: + provider.move(remoteFile, tempFile) + + then: + thrown org.codehaus.groovy.runtime.powerassert.PowerAssertionError + } + + void 'should recognize when path isHidden'() { + given: + QuiltFileSystemProvider provider = new QuiltFileSystemProvider() + + expect: + provider.isHidden(Paths.get(remoteFile)) == isHidden + + where: + remoteFile | isHidden + 'foo' | false + '.foo' | true + } + + void 'should throw error on getFileStore'() { + given: + QuiltFileSystemProvider provider = new QuiltFileSystemProvider() + + when: + provider.getFileStore(Paths.get('foo')) + + then: + thrown UnsupportedOperationException + } + + void 'should throw error on getFileAttributeView with unknown type'() { + given: + QuiltFileSystemProvider provider = new QuiltFileSystemProvider() + + when: + provider.getFileAttributeView(Paths.get('foo'), DirectoryStream) + + then: + thrown UnsupportedOperationException + } + + void 'should throw error on readAttributes with unknown type'() { + given: + QuiltFileSystemProvider provider = new QuiltFileSystemProvider() + QuiltPath qPath = parsedURIWithPath(true) + + when: + provider.readAttributes(qPath, DirectoryStream) + + then: + thrown UnsupportedOperationException + } + + void 'should throw error on readAttributes with String'() { + given: + QuiltFileSystemProvider provider = new QuiltFileSystemProvider() + + when: + provider.readAttributes(Paths.get('foo'), 'basic:isDirectory') + + then: + thrown UnsupportedOperationException + } + + void 'should throw error on setAttributes'() { + given: + QuiltFileSystemProvider provider = new QuiltFileSystemProvider() + + when: + provider.setAttribute(Paths.get('foo'), 'basic:isDirectory', provider) + + then: + thrown UnsupportedOperationException + } + } diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltNioTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltNioTest.groovy index 79a04465..59ff61d3 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltNioTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltNioTest.groovy @@ -79,6 +79,17 @@ class QuiltNioTest extends QuiltSpecification { text.startsWith('id') } + void 'should pretend to read file attributes'() { + given: + Path path = Paths.get(new URI(WRITE_URL)) + makeObject(path, TEXT) + + when: + BasicFileAttributes attrs = Files.readAttributes(path, BasicFileAttributes) + then: + attrs != null + } + @IgnoreIf({ System.getProperty('os.name').toLowerCase().contains('windows') }) @IgnoreIf({ System.getProperty('os.name').toLowerCase().contains('linux') }) void 'should read file attributes'() { @@ -213,6 +224,7 @@ class QuiltNioTest extends QuiltSpecification { } @IgnoreIf({ env.WRITE_BUCKET == null }) + @Ignore('Invalid test: top-level summarize') void 'move a remote file to a bucket'() { given: Path path = Paths.get(new URI(WRITE_URL)) @@ -512,9 +524,7 @@ class QuiltNioTest extends QuiltSpecification { list == [ 'file4.txt' ] } - // \QuiltPackage.quilt_dev_null_test_null\foo - @IgnoreIf({ System.getProperty('os.name').toLowerCase().contains('windows') }) - void 'should check walkTree'() { + void 'should check walkTree 1'() { given: makeObject(null_path('foo/file1.txt'), 'A') makeObject(null_path('foo/file2.txt'), 'BB') @@ -554,8 +564,8 @@ class QuiltNioTest extends QuiltSpecification { dirs.size() == 4 dirs.contains('null') dirs.contains('foo') - dirs.contains('foo/bar') - dirs.contains('foo/bar/baz') + dirs.contains(QuiltPackage.osConvert('foo/bar')) + dirs.contains(QuiltPackage.osConvert('foo/bar/baz')) when: dirs = [] @@ -646,8 +656,8 @@ class QuiltNioTest extends QuiltSpecification { dirs.contains('foo') files.size() == 3 files.containsKey('foo') - files.containsKey('foo/bar') - files.containsKey('foo/baz') + files.containsKey(QuiltPackage.osConvert('foo/bar')) + files.containsKey(QuiltPackage.osConvert('foo/baz')) } void 'should handle file names with same prefix'() { diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltPathTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltPathTest.groovy index 65bb9050..4f24cb44 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltPathTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltPathTest.groovy @@ -13,7 +13,6 @@ import java.nio.file.ProviderMismatchException import spock.lang.Unroll import spock.lang.Ignore -import spock.lang.IgnoreIf /** * @@ -192,10 +191,10 @@ class QuiltPathTest extends QuiltSpecification { thrown ProviderMismatchException } - @Ignore('FIXME: subpath not yet implemented') + @Ignore('FIXME: test subpath in QuiltParser first') void 'should validate subpath: #expected'() { expect: - pathify(path).subpath(from, to) == pathify(expected) + pathify(path).subpath(from, to).getPath() == expected where: path | from | to | expected 'bucket#package=some%2fbig%2fdata%2ffile.txt' | 0 | 1 | 'data' @@ -210,13 +209,13 @@ class QuiltPathTest extends QuiltSpecification { pathify(path).startsWith(pathify(prefix)) == expected where: - path | prefix | expected + path | prefix | expected 'bucket#package=s/d/file.txt' | 'bucket#package=s%2fd' | true 'bucket#package=s/d/file.txt' | 'bucket#package=s' | true 'bucket#package=s/d/file.txt' | 'bucket' | true 'bucket#package=s/d/file.txt' | 'file.txt' | false - 'data%2ffile.txt' | 'data' | true - 'data%2ffile.txt' | 'file.txt' | false + 'data%2ffile.txt' | 'data' | true + 'data%2ffile.txt' | 'file.txt' | false } @Unroll @@ -226,13 +225,13 @@ class QuiltPathTest extends QuiltSpecification { //pathify(path).endsWith(pathify(suffix)) == expected where: - path | suffix | expected - SUB_PATH | 'file.txt' | true - SUB_PATH | 'f%2ffile.txt' | true - SUB_PATH | '/f%2ffile.txt' | false - SUB_PATH | 'bucket' | false - 'data%2ffile.txt' | 'data' | false - 'data%2ffile.txt' | 'file.txt' | true + path | suffix | expected + SUB_PATH | 'file.txt' | true + SUB_PATH | 'f%2ffile.txt' | true + SUB_PATH | '/f%2ffile.txt' | false + SUB_PATH | 'bucket' | false + 'data%2ffile.txt' | 'data' | false + 'data%2ffile.txt' | 'file.txt' | true } @Unroll @@ -240,10 +239,10 @@ class QuiltPathTest extends QuiltSpecification { expect: pathify(path).normalize() == pathify(expected) where: - path | expected - 'bucket#path=s/d/file.txt' | 'bucket#path=s/d/file.txt' + path | expected + 'bucket#path=s/d/file.txt' | 'bucket#path=s/d/file.txt' 'bucket#path=some%2f..%2ffile.txt' | 'bucket#path=file.txt' - 'file.txt' | 'file.txt' + 'file.txt' | 'file.txt' } @Unroll @@ -258,22 +257,38 @@ class QuiltPathTest extends QuiltSpecification { 'bucket' | 'some%2ffile-name.txt' | 'bucket#path=some%2ffile-name.txt' } + String based(String fragment = '') { + return "bucket#package=so%2fme${fragment}" + } + + String sepJoin(String... parts) { + if (QuiltPackage.osSep() == '/') { + return parts.join('%2f') + } + return parts.join('%5c') + } @Unroll - @IgnoreIf({ System.getProperty('os.name').toLowerCase().contains('windows') }) void 'should validate relativize'() { expect: pathify(path).relativize(pathify(other)).toString() == pathify(expected).toString() where: - path | other | expected - 'bucket#package=so%2fme' | 'bucket#package=so%2fme%2fdata%2ffile.txt' | 'data%2ffile.txt' - 'bucket#package=so%2fme%2fdata' | 'bucket#package=so%2fme%2fdata%2ffile.txt' | 'file.txt' - 'bucket#package=so%2fme&path=foo' | 'bucket#package=so%2fme&path=foo%2fbar' | 'bar' + path | other | expected + based() | based('%2fdata%2ffile.txt') | sepJoin('data', 'file.txt') + based('%2fdata') | based('%2fdata%2ffile.txt') | 'file.txt' + based('&path=foo') | based('&path=foo%2fbar') | 'bar' + } + + void 'should error on relativize if no common path'() { + when: + pathify('bucket#package=so%2fme&path=bar').relativize(pathify('bucket#package=so%2fme&path=foo')) + then: + thrown IllegalArgumentException } void 'should reconstruct full URLs'() { given: QuiltPath pkgPath = QuiltPathFactory.parse(PKG_URL) - QuiltPath fullPath = QuiltPathFactory.parse(fullURL) + QuiltPath fullPath = QuiltPathFactory.parse(testURI) expect: !pkgPath.toUriString().contains('?') fullPath.toUriString().contains('?') @@ -299,4 +314,87 @@ class QuiltPathTest extends QuiltSpecification { path.pkg() == prior } + void 'should getNameCount'() { + expect: + pathify(path).getNameCount() == expected + where: + path | expected + 'bucket#package=so%2fme' | 0 + 'bucket#package=so%2fme&path=file-name.txt' | 1 + 'bucket#package=so%2fme&path=folder/name.txt' | 2 + 'bucket#package=so%2fme&path=folder%2fname.txt' | 2 + } + + void 'should error on getName'() { + when: + pathify('bucket#package=so%2fme').getName(-1) + then: + thrown UnsupportedOperationException + } + + void 'should return subPaths'() { + given: + Path path = pathify('bucket#package=so%2fme&path=folder/name.txt') + Path subPath = path.subpath(1, 2) + expect: + subPath + subPath.toString() == 'bucket#package=so%2fme&path=name.txt' + } + + void 'should match endsWith'() { + given: + QuiltPath path = pathify('bucket#package=so%2fme&path=folder/name.txt') + expect: + path.endsWith('name.txt') + !path.endsWith('folder') + } + + void 'should error on resolveSibling'() { + when: + Path path = pathify('bucket#package=so%2fme&path=folder/name.txt') + pathify('bucket#package=so%2fme').resolveSibling(path) + then: + thrown UnsupportedOperationException + } + + void 'should error on toAbsolutePath if not absolute'() { + when: + pathify('bucket').toAbsolutePath() + then: + thrown UnsupportedOperationException + } + + void 'should error on toRealPath'() { + when: + pathify('bucket#package=so%2fme').toRealPath() + then: + thrown UnsupportedOperationException + } + + void 'should error on toFile'() { + when: + pathify('bucket#package=so%2fme').toFile() + then: + thrown UnsupportedOperationException + } + + void 'should error on iterator'() { + when: + pathify('bucket#package=so%2fme').iterator() + then: + thrown UnsupportedOperationException + } + + void 'should error on register'() { + when: + pathify('bucket#package=so%2fme').register(null) + then: + thrown UnsupportedOperationException + + when: + pathify('bucket#package=so%2fme').register(null, null) + then: + thrown UnsupportedOperationException + } + }