Merge pull request #15 from seqeralabs/release_fixes

Move nf-test files and container definitions

.github/workflows/ci.yml

@@ -1,12 +1,15 @@
 name: nf-aggregate CI
 # This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors
 on:
-  push:
-    branches:
-      - dev
   pull_request:
   release:
-    types: [published]
+    types:
+      - published
+  merge_group:
+    types:
+      - checks_requested
+    branches:
+      - master
 
 env:
   NXF_ANSI_LOG: false
@@ -16,26 +19,26 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  test:
   changes:
-    name: Check for changes
+    name: Find tags to test
     runs-on: ubuntu-latest
     outputs:
-      # Expose matched filters as job 'tags' output variable
-      tags: ${{ steps.filter.outputs.changes }}
+      tags: ${{ steps.get_tags.outputs.result }}
     steps:
       - uses: actions/checkout@v3
       - name: Combine all tags.yml files
         id: get_username
         run: find . -name "tags.yml" -not -path "./.github/*" -exec cat {} + > .github/tags.yml
       - name: debug
         run: cat .github/tags.yml
-      - uses: dorny/paths-filter@v2
-        id: filter
+      - name: Get tags using yq
+        id: get_tags
+        uses: mikefarah/yq@master
         with:
-          filters: ".github/tags.yml"
+          cmd: yq -e '. | keys | tojson' .github/tags.yml
+
   nf-test:
-    name: nf-test ${{ matrix.tags }} ${{ matrix.profile }}
+    name: nf-test ${{ matrix.tags }} ${{ matrix.profile }} nf-${{ matrix.NXF_VER }}
     needs: [changes]
     if: needs.changes.outputs.tags != '[]'
     runs-on: ubuntu-latest
@@ -47,10 +50,9 @@ jobs:
         NXF_VER:
           - "23.10.0"
           - "latest-everything"
-        tags: ["${{ fromJson(needs.changes.outputs.tags) }}"]
+        tags: ${{ fromJson(needs.changes.outputs.tags) }}
         profile:
           - "docker"
-          - "conda"
 
     steps:
       - name: Check out pipeline code
@@ -61,22 +63,6 @@ jobs:
         with:
           version: "${{ matrix.NXF_VER }}"
 
-      - name: Set up miniconda
-        uses: conda-incubator/setup-miniconda@v2
-        with:
-          miniconda-version: "latest"
-          auto-update-conda: true
-          channels: conda-forge,bioconda,defaults
-          python-version: ${{ matrix.python-version }}
-
-      - name: Conda setup
-        run: |
-          conda clean -a
-          conda install -n base conda-libmamba-solver
-          conda config --set solver libmamba
-          echo $(realpath $CONDA)/condabin >> $GITHUB_PATH
-          echo $(realpath python) >> $GITHUB_PATH
-
       - name: Cache nf-test installation
         id: cache-software
         uses: actions/cache@v3
@@ -94,7 +80,7 @@ jobs:
 
       - name: Run nf-test
         run: |
-          nf-test test --tag ${{ matrix.tags }} --profile "test,${{ matrix.profile }}" --junitxml=test.xml
+          nf-test test --tag ${{ matrix.tags }} --profile "test,${{ matrix.profile }}" --junitxml=test.xml --verbose
 
       - name: Output log on failure
         if: failure()
@@ -107,3 +93,21 @@ jobs:
         if: always() # always run even if the previous step fails
         with:
           report_paths: test.xml
+
+  confirm-pass:
+    runs-on: ubuntu-latest
+    needs: [changes, nf-test]
+    if: always()
+    steps:
+      - name: All tests ok
+        if: ${{ success() || !contains(needs.*.result, 'failure') }}
+        run: exit 0
+      - name: One or more tests failed
+        if: ${{ contains(needs.*.result, 'failure') }}
+        run: exit 1
+
+      - name: debug-print
+        if: always()
+        run: |
+          echo "toJSON(needs) = ${{ toJSON(needs) }}"
+          echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}"

.gitignore

@@ -6,4 +6,4 @@ results/
 testing/
 testing*
 *.pyc
-.nf-test
+.nf-test*

main.nf

@@ -43,7 +43,10 @@ workflow {
     NF_AGGREGATE (
         PIPELINE_INITIALISATION.out.ids,
         ch_multiqc_custom_config,
-        ch_multiqc_logo
+        ch_multiqc_logo,
+        params.seqera_api_endpoint,
+        params.skip_run_gantt,
+        params.skip_multiqc
     )
 }
 

modules.json

@@ -7,7 +7,7 @@
                 "nf-core": {
                     "multiqc": {
                         "branch": "master",
-                        "git_sha": "1537442a7be4a78efa3d1ff700a923c627bbda5d",
+                        "git_sha": "4ab13872435962dadc239979554d13709e20bf29",
                         "installed_by": ["modules"]
                     }
                 }
@@ -21,7 +21,7 @@
                     },
                     "utils_nfvalidation_plugin": {
                         "branch": "master",
-                        "git_sha": "c81784b47d44464b35e04220e00051bb5be27524",
+                        "git_sha": "f0f8f4dc889970251eb251b85610f7ec2eafded9",
                         "installed_by": ["subworkflows"]
                     }
                 }

modules/local/plot_run_gantt/main.nf

@@ -1,6 +1,7 @@
 process PLOT_RUN_GANTT {
     tag "$meta.id"
     conda 'click=8.0.1 pandas=1.1.5 plotly_express=0.4.1 typing=3.10.0.0'
+    container 'seqeralabs/nf-aggregate:click-8.0.1_pandas-1.1.5_plotly_express-0.4.1_typing-3.10.0.0--342dabfe6548a051'
 
     input:
     tuple val(meta), path(run_dump)

modules/local/plot_run_gantt/tests/main.nf.test

@@ -3,7 +3,6 @@ nextflow_process {
     name "Test Process PLOT_RUN_GANTT"
     script "../main.nf"
     process "PLOT_RUN_GANTT"
-    config "./nextflow.config"
 
     tag "modules"
     tag "modules_local"
@@ -25,7 +24,6 @@ nextflow_process {
 
         when {
             process {
-
                 """
                 input[0] = SEQERA_RUNS_DUMP.out.run_dump
                 """

modules/local/seqera_runs_dump/main.nf

@@ -3,6 +3,7 @@ include { getRunMetadata } from './functions'
 process SEQERA_RUNS_DUMP {
     tag "$meta.id"
     conda 'tower-cli=0.9.0'
+    container 'seqeralabs/nf-aggregate:tower-cli-0.9.0--2cb0f2e9d85d026b'
 
     input:
     val meta

modules/local/seqera_runs_dump/tests/main.nf.test

@@ -3,7 +3,6 @@ nextflow_process {
     name "Test Process SEQERA_RUNS_DUMP"
     script "../main.nf"
     process "SEQERA_RUNS_DUMP"
-    config "./nextflow.config"
 
     tag "modules"
     tag "modules_local"

nextflow.config

@@ -51,23 +51,27 @@ process {
     maxErrors     = '-1'
 }
 
+// Ignore process selector warnings by default
+nextflow.enable.configProcessNamesValidation = false
+
 profiles {
     test { 
         params.input = "${projectDir}/workflows/nf_aggregate/assets/test_run_ids.csv"
     }
-    wave {
-        wave.enabled = true
-        wave.strategy = ['conda']
-        wave.build.repository = 'public.ecr.aws/seqera-labs/wave/containers'
-        wave.freeze = true   
-    }
     test_full {
         params.input = "${projectDir}/workflows/nf_aggregate/assets/test_run_ids.csv"
     }
     debug {
         dumpHashes             = true
         process.beforeScript   = 'echo $HOSTNAME'
         cleanup                = false
+        nextflow.enable.configProcessNamesValidation = true
+    }
+    wave {
+        wave.enabled = true
+        wave.strategy = ['conda', 'container', 'dockerfile', 'spack']
+        wave.build.repository = 'quay.io/seqeralabs/nf-aggregate'
+        wave.freeze = true   
     }
     conda {
         conda.enabled          = true
@@ -157,10 +161,10 @@ profiles {
 // Set default registry for Apptainer, Docker, Podman and Singularity independent of -profile
 // Will not be used unless Apptainer / Docker / Podman / Singularity are enabled
 // Set to your registry if you have a mirror of containers
-apptainer.registry   = 'public.ecr.aws/seqera-labs/wave/containers'
-docker.registry      = 'public.ecr.aws/seqera-labs/wave/containers'
-podman.registry      = 'public.ecr.aws/seqera-labs/wave/containers'
-singularity.registry = 'public.ecr.aws/seqera-labs/wave/containers'
+apptainer.registry   = 'quay.io'
+docker.registry      = 'quay.io'
+podman.registry      = 'quay.io'
+singularity.registry = 'quay.io'
 
 // Nextflow plugins
 plugins {
@@ -182,9 +186,6 @@ env {
 // Load Nextflow config for nf_aggregate workflow
 includeConfig 'workflows/nf_aggregate/nextflow.config'
 
-// Load container config for nf_aggregate workflow
-includeConfig 'conf/container.config'
-
 // Capture exit codes from upstream processes when piping
 process.shell = ['/bin/bash', '-euo', 'pipefail']
 

nf-test.config

@@ -3,13 +3,10 @@ config {
     testsDir "."
 
     // nf-test directory including temporary files for each test
-    workDir "/tmp"
-
-    // location of library folder that is added automatically to the classpath
-    libDir "lib/"
+    workDir System.getenv("TMPDIR") ?: '.nf-test'
 
     // location of an optional nextflow.config file specific for executing tests
-    configFile "nextflow.config"
+    configFile "tests/nextflow.config"
 
     // run all test with the defined docker profile from the main nextflow.config
     profile ""

subworkflows/local/utils_nf_aggregate/tests/main.nf.test

@@ -0,0 +1,37 @@
+nextflow_workflow {
+
+    name "Test Workflow PIPELINE_INITIALISATION"
+    script "../main.nf"
+    workflow "PIPELINE_INITIALISATION"
+
+    tag "subworkflows"
+    tag "subworkflows_local_utils_nf_aggregate"
+
+    stage {
+        symlink "nextflow_schema.json"
+        symlink "workflows/nf_aggregate/assets/test_run_ids.csv"
+    }
+
+    test("Should run without failures") {
+
+        when {
+            workflow {
+                """
+                // define inputs of the workflow here. Example:
+                // input[0] = file("test-file.txt")
+                """
+            }
+        }
+
+        then {
+            assertAll(
+                { assert workflow.success },
+                { assert snapshot(
+                    workflow.out.ids
+                 ).match() }
+            )
+        }
+
+    }
+
+}