From 8acb9c4856cddac922536682a2873946ef8aa56b Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 8 Aug 2024 13:32:08 +0200 Subject: [PATCH 01/11] handle template features with a yaml file --- .../create-test-lint-wf-template.yml | 78 +++---- .prettierignore | 1 + nf_core/pipelines/create/create.py | 200 ++++------------ nf_core/pipelines/create/templatefeatures.yml | 213 ++++++++++++++++++ tests/data/pipeline_create_template_skip.yml | 13 +- tests/test_create.py | 18 +- 6 files changed, 297 insertions(+), 226 deletions(-) create mode 100644 nf_core/pipelines/create/templatefeatures.yml diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index dbe143fda6..d4ff3fc557 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -31,31 +31,41 @@ env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} jobs: + install-yq: + name: Install yq + runs-on: ubuntu-latest + steps: + - name: πŸ— Set up yq + uses: frenck/action-setup-yq@v1 + + prepare-matrix: + name: Retrieve all template features + runs-on: ubuntu-latest + outputs: + all_features: ${{ steps.create_matrix.outputs.matrix }} + steps: + - name: checkout + uses: actions/checkout@v3 + - name: Create Matrix + id: create_matrix + run: echo "matrix=$(yq 'keys' ${{ github.workspace }}/nf_core/pipelines/create/templatefeatures.yml)" >> $GITHUB_OUTPUT + RunTestWorkflow: runs-on: ${{ matrix.runner }} env: NXF_ANSI_LOG: false strategy: matrix: - TEMPLATE: - - "template_skip_github_badges.yml" - - "template_skip_igenomes.yml" - - "template_skip_ci.yml" - - "template_skip_code_linters.yml" - - "template_skip_citations.yml" - - "template_skip_gitpod.yml" - - "template_skip_codespaces.yml" - - "template_skip_multiqc.yml" - - "template_skip_changelog.yml" + TEMPLATE: ${{ fromJSON(needs.prepare-matrix.outputs.all_features) }} runner: # use the runner given by the input if it is dispatched manually, run on github if it is a rerun or on self-hosted by default - ${{ github.event.inputs.runners || github.run_number > 1 && 'ubuntu-latest' || 'self-hosted' }} profile: ["self_hosted_runner"] include: - - TEMPLATE: "template_skip_all.yml" + - TEMPLATE: all runner: ubuntu-latest profile: "docker" - - TEMPLATE: "template_skip_nf_core_configs.yml" + - TEMPLATE: nf_core_configs runner: ubuntu-latest profile: "docker" fail-fast: false @@ -90,53 +100,17 @@ jobs: run: | mkdir create-test-lint-wf export NXF_WORK=$(pwd) - printf "org: my-prefix\nskip: ['ci', 'github_badges', 'igenomes', 'nf_core_configs']" > create-test-lint-wf/template_skip_all.yml - - - name: Create template skip github_badges - run: | - printf "org: my-prefix\nskip: github_badges" > create-test-lint-wf/template_skip_github_badges.yml - - - name: Create template skip igenomes - run: | - printf "org: my-prefix\nskip: igenomes" > create-test-lint-wf/template_skip_igenomes.yml - - - name: Create template skip ci - run: | - printf "org: my-prefix\nskip: ci" > create-test-lint-wf/template_skip_ci.yml - - - name: Create template skip nf_core_configs - run: | - printf "org: my-prefix\nskip: nf_core_configs" > create-test-lint-wf/template_skip_nf_core_configs.yml - - - name: Create template skip code_linters - run: | - printf "org: my-prefix\nskip: code_linters" > create-test-lint-wf/template_skip_code_linters.yml - - - name: Create template skip citations - run: | - printf "org: my-prefix\nskip: citations" > create-test-lint-wf/template_skip_citations.yml - - - name: Create template skip gitpod - run: | - printf "org: my-prefix\nskip: gitpod" > create-test-lint-wf/template_skip_gitpod.yml - - - name: Create template skip multiqc - run: | - printf "org: my-prefix\nskip: multiqc" > create-test-lint-wf/template_skip_multiqc.yml - - - name: Create template skip changelog - run: | - printf "org: my-prefix\nskip: changelog" > create-test-lint-wf/template_skip_changelog.yml + printf "org: my-prefix\nskip: ${{ needs.prepare-matrix.outputs.all_features }}" > create-test-lint-wf/template_skip_all.yml - - name: Create template skip codespaces + - name: Create template skip {{ matrix.TEMPLATE }} run: | - printf "org: my-prefix\nskip: codespaces" > create-test-lint-wf/template_skip_codespaces.yml + printf "org: my-prefix\nskip: {{ matrix.TEMPLATE }}" > create-test-lint-wf/template_skip_{{ matrix.TEMPLATE }}.yml # Create a pipeline from the template - name: create a pipeline from the template ${{ matrix.TEMPLATE }} run: | cd create-test-lint-wf - nf-core --log-file log.txt pipelines create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" --template-yaml ${{ matrix.TEMPLATE }} + nf-core --log-file log.txt pipelines create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" --template-yaml template_skip_${{ matrix.TEMPLATE }}.yml - name: run the pipeline run: | diff --git a/.prettierignore b/.prettierignore index 059007ab03..9387ee9509 100644 --- a/.prettierignore +++ b/.prettierignore @@ -7,6 +7,7 @@ nf_core/module-template/meta.yml nf_core/pipeline-template/nextflow_schema.json nf_core/pipeline-template/modules.json nf_core/pipeline-template/tower.yml +tests/data/pipeline_create_template_skip.yml # don't run on things handled by ruff *.py *.pyc diff --git a/nf_core/pipelines/create/create.py b/nf_core/pipelines/create/create.py index b04b28c473..db0833c68e 100644 --- a/nf_core/pipelines/create/create.py +++ b/nf_core/pipelines/create/create.py @@ -76,43 +76,26 @@ def __init__( else: raise UserWarning("The template configuration was not provided.") - self.jinja_params, skip_paths = self.obtain_jinja_params_dict( + # Read features yaml file + features_yml_path = Path(nf_core.__file__).parent / "pipelines" / "create" / "templatefeatures.yml" + with open(features_yml_path) as fh: + self.template_features_yml = yaml.safe_load(fh) + + self.jinja_params, self.skip_areas = self.obtain_jinja_params_dict( self.config.skip_features or [], self.config.outdir ) - skippable_paths = { - "github": [ - ".github/", - ".gitignore", - ], - "ci": [".github/workflows/"], - "igenomes": ["conf/igenomes.config"], - "is_nfcore": [ - ".github/ISSUE_TEMPLATE/config", - "CODE_OF_CONDUCT.md", - ".github/workflows/awsfulltest.yml", - ".github/workflows/awstest.yml", - ".github/workflows/release-announcements.yml", - ], - "code_linters": [ - ".editorconfig", - ".pre-commit-config.yaml", - ".prettierignore", - ".prettierrc.yml", - ".github/workflows/fix-linting.yml", - ], - "citations": ["assets/methods_description_template.yml"], - "gitpod": [".gitpod.yml"], - "codespaces": [".devcontainer/devcontainer.json"], - "multiqc": [ - "assets/multiqc_config.yml", - "assets/methods_description_template.yml", - "modules/nf-core/multiqc/", - ], - "changelog": ["CHANGELOG.md"], - } + # format strings in features yaml + short_name = self.jinja_params["short_name"] + env = jinja2.Environment(loader=jinja2.PackageLoader("nf_core", "pipelines"), keep_trailing_newline=True) + features_template = env.get_template( + str(features_yml_path.relative_to(Path(nf_core.__file__).parent / "pipelines")) + ) + rendered_features = features_template.render({"short_name": short_name}) + self.template_features_yml = yaml.safe_load(rendered_features) + # Get list of files we're skipping with the supplied skip keys - self.skip_paths = set(sp for k in skip_paths for sp in skippable_paths[k]) + self.skip_paths = set(sp for k in self.skip_areas for sp in self.template_features_yml[k]["skippable_paths"]) # Set convenience variables self.name = self.config.name @@ -205,43 +188,27 @@ def obtain_jinja_params_dict(self, features_to_skip, pipeline_dir): Returns: jinja_params (dict): Dictionary of template areas to skip with values true/false. - skip_paths (list): List of template areas which contain paths to skip. + skip_areas (list): List of template areas which contain paths to skip. """ # Try reading config file _, config_yml = nf_core.utils.load_tools_config(pipeline_dir) - # Define the different template areas, and what actions to take for each - # if they are skipped - template_areas = { - "github": {"file": True, "content": False}, - "ci": {"file": True, "content": False}, - "github_badges": {"file": False, "content": True}, - "igenomes": {"file": True, "content": True}, - "nf_core_configs": {"file": False, "content": True}, - "code_linters": {"file": True, "content": True}, - "citations": {"file": True, "content": True}, - "gitpod": {"file": True, "content": True}, - "codespaces": {"file": True, "content": True}, - "multiqc": {"file": True, "content": True}, - "changelog": {"file": True, "content": False}, - } - # Set the parameters for the jinja template jinja_params = self.config.model_dump() # Add template areas to jinja params and create list of areas with paths to skip - skip_paths = [] - for t_area in template_areas: + skip_areas = [] + for t_area in self.template_features_yml.keys(): if t_area in features_to_skip: - if template_areas[t_area]["file"]: - skip_paths.append(t_area) + if self.template_features_yml[t_area]["skippable_paths"]: + skip_areas.append(t_area) jinja_params[t_area] = False else: jinja_params[t_area] = True # Add is_nfcore as an area to skip for non-nf-core pipelines, to skip all nf-core files - if not jinja_params["is_nfcore"]: - skip_paths.append("is_nfcore") + if not self.config.is_nfcore: + skip_areas.append("is_nfcore") # Set the last parameters based on the ones provided jinja_params["short_name"] = ( @@ -259,7 +226,7 @@ def obtain_jinja_params_dict(self, features_to_skip, pipeline_dir): and "nextflow_config" in config_yml["lint"] and "manifest.name" in config_yml["lint"]["nextflow_config"] ): - return jinja_params, skip_paths + return jinja_params, skip_areas # Check that the pipeline name matches the requirements if not re.match(r"^[a-z]+$", jinja_params["short_name"]): @@ -270,7 +237,7 @@ def obtain_jinja_params_dict(self, features_to_skip, pipeline_dir): "Your workflow name is not lowercase without punctuation. This may cause Nextflow errors.\nConsider changing the name to avoid special characters." ) - return jinja_params, skip_paths + return jinja_params, skip_areas def init_pipeline(self): """Creates the nf-core pipeline.""" @@ -438,109 +405,20 @@ def fix_linting(self): for a customized pipeline. """ # Create a lint config - short_name = self.jinja_params["short_name"] - if not self.config.is_nfcore: - lint_config = { - "files_exist": [ - "CODE_OF_CONDUCT.md", - f"assets/nf-core-{short_name}_logo_light.png", - f"docs/images/nf-core-{short_name}_logo_light.png", - f"docs/images/nf-core-{short_name}_logo_dark.png", - ".github/ISSUE_TEMPLATE/config.yml", - ".github/workflows/awstest.yml", - ".github/workflows/awsfulltest.yml", - ], - "files_unchanged": [ - "CODE_OF_CONDUCT.md", - f"assets/nf-core-{short_name}_logo_light.png", - f"docs/images/nf-core-{short_name}_logo_light.png", - f"docs/images/nf-core-{short_name}_logo_dark.png", - ], - "nextflow_config": [ - "manifest.name", - "manifest.homePage", - ], - "multiqc_config": ["report_comment"], - } - else: - lint_config = {} - - # Add GitHub hosting specific configurations - if not self.jinja_params["github"]: - lint_config["files_exist"].extend( - [ - ".github/ISSUE_TEMPLATE/bug_report.yml", - ".github/ISSUE_TEMPLATE/feature_request.yml", - ".github/PULL_REQUEST_TEMPLATE.md", - ".github/CONTRIBUTING.md", - ".github/.dockstore.yml", - ".gitignore", - ] - ) - lint_config["files_unchanged"].extend( - [ - ".github/ISSUE_TEMPLATE/bug_report.yml", - ".github/ISSUE_TEMPLATE/config.yml", - ".github/ISSUE_TEMPLATE/feature_request.yml", - ".github/PULL_REQUEST_TEMPLATE.md", - ".github/workflows/branch.yml", - ".github/workflows/linting_comment.yml", - ".github/workflows/linting.yml", - ".github/CONTRIBUTING.md", - ".github/.dockstore.yml", - ] - ) - - # Add CI specific configurations - if not self.jinja_params["ci"]: - lint_config["files_exist"].extend( - [ - ".github/workflows/branch.yml", - ".github/workflows/ci.yml", - ".github/workflows/linting_comment.yml", - ".github/workflows/linting.yml", - ] - ) - - # Add custom config specific configurations - if not self.jinja_params["nf_core_configs"]: - lint_config["files_exist"].extend(["conf/igenomes.config"]) - lint_config["nextflow_config"].extend( - [ - "process.cpus", - "process.memory", - "process.time", - "custom_config", - ] - ) - - # Add igenomes specific configurations - if not self.jinja_params["igenomes"]: - lint_config["files_exist"].extend(["conf/igenomes.config"]) - - # Add github badges specific configurations - if not self.jinja_params["github_badges"] or not self.jinja_params["github"]: - lint_config["readme"] = ["nextflow_badge"] - - # Add codespaces specific configurations - if not self.jinja_params["codespaces"]: - lint_config["files_unchanged"].extend([".github/CONTRIBUTING.md"]) - - # Add multiqc specific configurations - if not self.jinja_params["multiqc"]: - lint_config.setdefault("files_unchanged", []).extend( - [".github/CONTRIBUTING.md", "assets/sendmail_template.txt"] - ) - lint_config.setdefault("files_exist", []).extend(["assets/multiqc_config.yml"]) - lint_config["multiqc_config"] = False - - # Add changelog specific configurations - if not self.jinja_params["changelog"]: - lint_config["files_exist"].extend(["CHANGELOG.md"]) - - # If the pipeline is not nf-core - if not self.config.is_nfcore: - lint_config["files_unchanged"].extend([".github/ISSUE_TEMPLATE/bug_report.yml"]) + lint_config = {} + for area in self.skip_areas: + try: + for lint_test in self.template_features_yml[area]["linting"]: + if not lint_config[lint_test]: + pass + if self.template_features_yml[area]["linting"][lint_test]: + lint_config.setdefault(lint_test, []).extend( + self.template_features_yml[area]["linting"][lint_test] + ) + else: + lint_config[lint_test] = False + except KeyError: + pass # Areas without linting # Add the lint content to the preexisting nf-core config config_fn, nf_core_yml = nf_core.utils.load_tools_config(self.outdir) diff --git a/nf_core/pipelines/create/templatefeatures.yml b/nf_core/pipelines/create/templatefeatures.yml new file mode 100644 index 0000000000..843203c7f8 --- /dev/null +++ b/nf_core/pipelines/create/templatefeatures.yml @@ -0,0 +1,213 @@ +github: + skippable_paths: + - ".github/" + - ".gitignore" + short_description: "" + description: "" + help: "" + linting: + files_exist: + - ".github/ISSUE_TEMPLATE/bug_report.yml" + - ".github/ISSUE_TEMPLATE/feature_request.yml" + - ".github/PULL_REQUEST_TEMPLATE.md" + - ".github/CONTRIBUTING.md" + - ".github/.dockstore.yml" + - ".gitignore" + files_unchanged: + - ".github/ISSUE_TEMPLATE/bug_report.yml" + - ".github/ISSUE_TEMPLATE/config.yml" + - ".github/ISSUE_TEMPLATE/feature_request.yml" + - ".github/PULL_REQUEST_TEMPLATE.md" + - ".github/workflows/branch.yml" + - ".github/workflows/linting_comment.yml" + - ".github/workflows/linting.yml" + - ".github/CONTRIBUTING.md" + - ".github/.dockstore.yml" + readme: + - "nextflow_badge" +ci: + skippable_paths: + - ".github/workflows/" + short_description: "Add Github CI tests" + description: "The pipeline will include several GitHub actions for Continuous Integration (CI) testing" + help: | + Nf-core provides a set of Continuous Integration (CI) tests for Github. + When you open a pull request (PR) on your pipeline repository, these tests will run automatically. + + There are different types of tests: + * Linting tests check that your code is formatted correctly and that it adheres to nf-core standards + For code linting they will use [prettier](https://prettier.io/). + * Pipeline tests run your pipeline on a small dataset to check that it works + These tests are run with a small test dataset on GitHub and a larger test dataset on AWS + * Marking old issues as stale + linting: + files_exist: + - ".github/workflows/branch.yml" + - ".github/workflows/ci.yml" + - ".github/workflows/linting_comment.yml" + - ".github/workflows/linting.yml" +igenomes: + skippable_paths: + - "conf/igenomes.config" + short_description: "Use reference genomes" + description: "The pipeline will be configured to use a copy of the most common reference genome files from iGenomes" + help: | + Nf-core pipelines are configured to use a copy of the most common reference genome files. + + By selecting this option, your pipeline will include a configuration file specifying the paths to these files. + + The required code to use these files will also be included in the template. + When the pipeline user provides an appropriate genome key, + the pipeline will automatically download the required reference files. + + For more information about reference genomes in nf-core pipelines, + see the [nf-core docs](https://nf-co.re/docs/usage/reference_genomes). + linting: + files_exist: + - "conf/igenomes.config" +github_badges: + skippable_paths: False + short_description: "Add Github badges" + description: "The README.md file of the pipeline will include GitHub badges" + help: | + The pipeline `README.md` will include badges for: + * AWS CI Tests + * Zenodo DOI + * Nextflow + * Conda + * Docker + * Singularity + * Launching on Nextflow Tower + linting: + readme: + - "nextflow_badge" +nf_core_configs: + skippable_paths: False + short_description: "Add configuration files" + description: "The pipeline will include configuration profiles containing custom parameters requried to run nf-core pipelines at different institutions" + help: | + Nf-core has a repository with a collection of configuration profiles. + + Those config files define a set of parameters which are specific to compute environments at different Institutions. + They can be used within all nf-core pipelines. + If you are likely to be running nf-core pipelines regularly it is a good idea to use or create a custom config file for your organisation. + + For more information about nf-core configuration profiles, see the [nf-core/configs repository](https://github.com/nf-core/configs) + linting: + files_exist: + - "conf/igenomes.config" + nextflow_config: + - "process.cpus" + - "process.memory" + - "process.time" + - "custom_config" +is_nfcore: + skippable_paths: + - ".github/ISSUE_TEMPLATE/config" + - "CODE_OF_CONDUCT.md" + - ".github/workflows/awsfulltest.yml" + - ".github/workflows/awstest.yml" + - ".github/workflows/release-announcements.yml" + short_description: "" + description: "" + help: "" + linting: + files_exist: + - "CODE_OF_CONDUCT.md" + - "assets/nf-core-{{short_name}}_logo_light.png" + - "docs/images/nf-core-{{short_name}}_logo_light.png" + - "docs/images/nf-core-{{short_name}}_logo_dark.png" + - ".github/ISSUE_TEMPLATE/config.yml" + - ".github/workflows/awstest.yml" + - ".github/workflows/awsfulltest.yml" + files_unchanged: + - "CODE_OF_CONDUCT.md" + - "assets/nf-core-{{short_name}}_logo_light.png" + - "docs/images/nf-core-{{short_name}}_logo_light.png" + - "docs/images/nf-core-{{short_name}}_logo_dark.png" + - ".github/ISSUE_TEMPLATE/bug_report.yml" + nextflow_config: + - "manifest.name" + - "manifest.homePage" + multiqc_config: + - "report_comment" +code_linters: + skippable_paths: + - ".editorconfig" + - ".pre-commit-config.yaml" + - ".prettierignore" + - ".prettierrc.yml" + - ".github/workflows/fix-linting.yml" + short_description: "Use code linters" + description: "The pipeline will include code linters and CI tests to lint your code: pre-commit, editor-config and prettier." + help: | + Pipelines include code linters to check the formatting of your code in order to harmonize code styles between developers. + Linters will check all non-ignored files, e.g., JSON, YAML, Nextlow or Python files in your repository. + The available code linters are: + + - pre-commit (https://pre-commit.com/): used to run all code-linters on every PR and on ever commit if you run `pre-commit install` to install it in your local repository. + - editor-config (https://github.com/editorconfig-checker/editorconfig-checker): checks rules such as indentation or trailing spaces. + - prettier (https://github.com/prettier/prettier): enforces a consistent style (indentation, quoting, line length, etc). +citations: + skippable_paths: + - "assets/methods_description_template.yml" + short_description: "Include citations" + description: "Include pipeline tools citations in CITATIONS.md and a method description in the MultiQC report (if enabled)." + help: | + If adding citations, the pipeline template will contain a `CITATIONS.md` file to add the citations of all tools used in the pipeline. + + Additionally, it will include a YAML file (`assets/methods_description_template.yml`) to add a Materials & Methods section describing the tools used in the pieline, + and the logics to add this section to the output MultiQC report (if the report is generated). +gitpod: + skippable_paths: + - ".gitpod.yml" + short_description: "Include a gitpod environment" + description: "Include the configuration required to use Gitpod." + help: | + Gitpod (https://www.gitpod.io/) provides standardized and automated development environments. + + Including this to your pipeline will provide an environment with the latest version of nf-core/tools installed and all its requirements. + This is useful to have all the tools ready for pipeline development. +codespaces: + skippable_paths: + - ".devcontainer/devcontainer.json" + short_description: "Include GitHub Codespaces" + description: "The pipeline will include a devcontainer configuration for GitHub Codespaces, providing a development environment with nf-core/tools and Nextflow installed." + help: | + The pipeline will include a devcontainer configuration. + The devcontainer will create a GitHub Codespaces for Nextflow development with nf-core/tools and Nextflow installed. + + Github Codespaces (https://github.com/features/codespaces) is an online developer environment that runs in your browser, complete with VSCode and a terminal. + linting: + files_unchanged: + - ".github/CONTRIBUTING.md" +multiqc: + skippable_paths: + - "assets/multiqc_config.yml" + - "assets/methods_description_template.yml" + - "modules/nf-core/multiqc/" + short_description: "Use multiqc" + description: "The pipeline will include the MultiQC module which generates an HTML report for quality control." + help: | + MultiQC is a visualization tool that generates a single HTML report summarising all samples in your project. Most of the pipeline quality control results can be visualised in the report and further statistics are available in the report data directory. + + The pipeline will include the MultiQC module and will have special steps which also allow the software versions to be reported in the MultiQC output for future traceability. For more information about how to use MultiQC reports, see http://multiqc.info. + linting: + files_unchanged: + - ".github/CONTRIBUTING.md" + - "assets/sendmail_template.txt" + files_exist: + - "assets/multiqc_config.yml" + multiqc_config: False +changelog: + skippable_paths: + - "CHANGELOG.md" + short_description: "Add a changelog" + description: "Add a CHANGELOG.md file." + help: | + Having a `CHANGELOG.md` file in the pipeline root directory is useful to track the changes added to each version. + + You can read more information on the recommended format here: https://keepachangelog.com/en/1.0.0/ + linting: + files_exist: + - "CHANGELOG.md" diff --git a/tests/data/pipeline_create_template_skip.yml b/tests/data/pipeline_create_template_skip.yml index 21cdc0c3d5..3ab8f69eca 100644 --- a/tests/data/pipeline_create_template_skip.yml +++ b/tests/data/pipeline_create_template_skip.yml @@ -5,15 +5,4 @@ version: 1.0.0 force: True org: testprefix is_nfcore: False -skip_features: - - github - - ci - - github_badges - - igenomes - - nf_core_configs - - code_linters - - citations - - gitpod - - codespaces - - multiqc - - changelog +skip_features: {{ all_features }} diff --git a/tests/test_create.py b/tests/test_create.py index 920a0f4750..92f7f2a9d9 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -5,6 +5,7 @@ from pathlib import Path import git +import jinja2 import yaml import nf_core.pipelines.create.create @@ -98,9 +99,24 @@ def test_pipeline_creation_initiation_customize_template(self, tmp_path): @with_temporary_folder def test_pipeline_creation_with_yml_skip(self, tmp_path): + # Update pipeline_create_template_skip.yml file + features_yml_path = Path(nf_core.__file__).parent / "pipelines" / "create" / "templatefeatures.yml" + with open(features_yml_path) as fh: + template_features_yml = yaml.safe_load(fh) + all_features = list(template_features_yml.keys()) + all_features.remove("is_nfcore") + env = jinja2.Environment(loader=jinja2.PackageLoader("tests", "data"), keep_trailing_newline=True) + skip_template = env.get_template( + str(PIPELINE_TEMPLATE_YML_SKIP.relative_to(Path(nf_core.__file__).parent.parent / "tests" / "data")) + ) + rendered_content = skip_template.render({"all_features": all_features}) + rendered_yaml = Path(tmp_path) / "pipeline_create_template_skip.yml" + with open(rendered_yaml, "w") as fh: + fh.write(rendered_content) + pipeline = nf_core.pipelines.create.create.PipelineCreate( outdir=tmp_path, - template_config=PIPELINE_TEMPLATE_YML_SKIP, + template_config=rendered_yaml, default_branch=self.default_branch, ) pipeline.init_pipeline() From 637317a3422f39d3f98fe2aafcc4efc6d0a88a80 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Thu, 8 Aug 2024 14:23:01 +0000 Subject: [PATCH 02/11] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 609c3aea06..c1e07f3bff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ - add option to exclude codespaces from pipeline template ([#3105](https://github.com/nf-core/tools/pull/3105)) - add option to exclude multiqc from pipeline template ([#3103](https://github.com/nf-core/tools/pull/3103)) - add option to exclude changelog from custom pipeline template ([#3104](https://github.com/nf-core/tools/pull/3104)) +- handle template features with a yaml file ([#3108](https://github.com/nf-core/tools/pull/3108)) ### Linting From efff9eada0641bc4ee42cf73f2c6943182f00886 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 9 Aug 2024 10:30:55 +0200 Subject: [PATCH 03/11] use template features yaml file to generate textual layout --- nf_core/pipelines/create/__init__.py | 8 + nf_core/pipelines/create/custompipeline.py | 142 ++---------------- nf_core/pipelines/create/nfcorepipeline.py | 25 ++- nf_core/pipelines/create/templatefeatures.yml | 24 +++ nf_core/pipelines/create/utils.py | 21 --- 5 files changed, 50 insertions(+), 170 deletions(-) diff --git a/nf_core/pipelines/create/__init__.py b/nf_core/pipelines/create/__init__.py index 56e25bf1d5..ae766b4e26 100644 --- a/nf_core/pipelines/create/__init__.py +++ b/nf_core/pipelines/create/__init__.py @@ -1,10 +1,13 @@ """A Textual app to create a pipeline.""" import logging +from pathlib import Path +import yaml from textual.app import App from textual.widgets import Button +import nf_core from nf_core.pipelines.create import utils from nf_core.pipelines.create.basicdetails import BasicDetails from nf_core.pipelines.create.custompipeline import CustomPipeline @@ -66,6 +69,11 @@ class PipelineCreateApp(App[utils.CreateConfig]): # Logging state LOGGING_STATE = None + # Template features + features_yml_path = Path(nf_core.__file__).parent / "pipelines" / "create" / "templatefeatures.yml" + with open(features_yml_path) as fh: + template_features_yml = yaml.safe_load(fh) + def on_mount(self) -> None: self.push_screen("welcome") diff --git a/nf_core/pipelines/create/custompipeline.py b/nf_core/pipelines/create/custompipeline.py index 96b6df5062..fdb7a46d28 100644 --- a/nf_core/pipelines/create/custompipeline.py +++ b/nf_core/pipelines/create/custompipeline.py @@ -6,76 +6,7 @@ from textual.screen import Screen from textual.widgets import Button, Footer, Header, Markdown, Switch -from nf_core.pipelines.create.utils import PipelineFeature, markdown_genomes, markdown_multiqc - -markdown_ci = """ -Nf-core provides a set of Continuous Integration (CI) tests for Github. -When you open a pull request (PR) on your pipeline repository, these tests will run automatically. - -There are different types of tests: -* Linting tests check that your code is formatted correctly and that it adheres to nf-core standards - For code linting they will use [prettier](https://prettier.io/). -* Pipeline tests run your pipeline on a small dataset to check that it works - These tests are run with a small test dataset on GitHub and a larger test dataset on AWS -* Marking old issues as stale -""" - -markdown_badges = """ -The pipeline `README.md` will include badges for: -* AWS CI Tests -* Zenodo DOI -* Nextflow -* Conda -* Docker -* Singularity -* Launching on Nextflow Tower -""" - -markdown_configuration = """ -Nf-core has a repository with a collection of configuration profiles. - -Those config files define a set of parameters which are specific to compute environments at different Institutions. -They can be used within all nf-core pipelines. -If you are likely to be running nf-core pipelines regularly it is a good idea to use or create a custom config file for your organisation. - -For more information about nf-core configuration profiles, see the [nf-core/configs repository](https://github.com/nf-core/configs) -""" - -markdown_code_linters = """ -Pipelines include code linters to check the formatting of your code in order to harmonize code styles between developers. -Linters will check all non-ignored files, e.g., JSON, YAML, Nextlow or Python files in your repository. -The available code linters are: - -- pre-commit (https://pre-commit.com/): used to run all code-linters on every PR and on ever commit if you run `pre-commit install` to install it in your local repository. -- editor-config (https://github.com/editorconfig-checker/editorconfig-checker): checks rules such as indentation or trailing spaces. -- prettier (https://github.com/prettier/prettier): enforces a consistent style (indentation, quoting, line length, etc). -""" - -markdown_citations = """ -If adding citations, the pipeline template will contain a `CITATIONS.md` file to add the citations of all tools used in the pipeline. - -Additionally, it will include a YAML file (`assets/methods_description_template.yml`) to add a Materials & Methods section describing the tools used in the pieline, -and the logics to add this section to the output MultiQC report (if the report is generated). -""" - -markdown_gitpod = """ -Gitpod (https://www.gitpod.io/) provides standardized and automated development environments. - -Including this to your pipeline will provide an environment with the latest version of nf-core/tools installed and all its requirements. -This is useful to have all the tools ready for pipeline development. -""" - -markdown_codespaces = """ -The pipeline will include a devcontainer configuration. -The devcontainer will create a GitHub Codespaces for Nextflow development with nf-core/tools and Nextflow installed. - -Github Codespaces (https://github.com/features/codespaces) is an online developer environment that runs in your browser, complete with VSCode and a terminal. -""" - -markdown_changelog = """ -Having a `CHANGELOG.md` file in the pipeline root directory is useful to track the changes added to each version. -You can read more information on the recommended format here: https://keepachangelog.com/en/1.0.0/ -""" +from nf_core.pipelines.create.utils import PipelineFeature class CustomPipeline(Screen): @@ -91,75 +22,20 @@ def compose(self) -> ComposeResult: """ ) ) - yield ScrollableContainer( - PipelineFeature( - markdown_genomes, - "Use reference genomes", - "The pipeline will be configured to use a copy of the most common reference genome files from iGenomes", - "igenomes", - ), - PipelineFeature( - markdown_ci, - "Add Github CI tests", - "The pipeline will include several GitHub actions for Continuous Integration (CI) testing", - "ci", - ), - PipelineFeature( - markdown_badges, - "Add Github badges", - "The README.md file of the pipeline will include GitHub badges", - "github_badges", - ), - PipelineFeature( - markdown_configuration, - "Add configuration files", - "The pipeline will include configuration profiles containing custom parameters requried to run nf-core pipelines at different institutions", - "nf_core_configs", - ), - PipelineFeature( - markdown_code_linters, - "Use code linters", - "The pipeline will include code linters and CI tests to lint your code: pre-commit, editor-config and prettier.", - "code_linters", - ), - PipelineFeature( - markdown_citations, - "Include citations", - "Include pipeline tools citations in CITATIONS.md and a method description in the MultiQC report (if enabled).", - "citations", - ), - PipelineFeature( - markdown_gitpod, - "Include a gitpod environment", - "Include the configuration required to use Gitpod.", - "gitpod", - ), - PipelineFeature( - markdown_codespaces, - "Include GitHub Codespaces", - "The pipeline will include a devcontainer configuration for GitHub Codespaces, providing a development environment with nf-core/tools and Nextflow installed.", - "codespaces", - ), - PipelineFeature( - markdown_multiqc, - "Use multiqc", - "The pipeline will include the MultiQC module which generates an HTML report for quality control.", - "multiqc", - ), - PipelineFeature( - markdown_changelog, - "Add a changelog", - "Add a CHANGELOG.md file.", - "changelog", - ), - classes="features-container", - ) + yield ScrollableContainer(id="features") yield Center( Button("Back", id="back", variant="default"), Button("Continue", id="continue", variant="success"), classes="cta", ) + def on_mount(self) -> None: + for name, feature in self.parent.template_features_yml.items(): + if feature["custom_pipelines"]: + self.query_one("#features").mount( + PipelineFeature(feature["help"], feature["short_description"], feature["description"], name) + ) + @on(Button.Pressed, "#continue") def on_button_pressed(self, event: Button.Pressed) -> None: """Save fields to the config.""" diff --git a/nf_core/pipelines/create/nfcorepipeline.py b/nf_core/pipelines/create/nfcorepipeline.py index 8319cb044c..b6a8231694 100644 --- a/nf_core/pipelines/create/nfcorepipeline.py +++ b/nf_core/pipelines/create/nfcorepipeline.py @@ -6,7 +6,7 @@ from textual.screen import Screen from textual.widgets import Button, Footer, Header, Markdown, Switch -from nf_core.pipelines.create.utils import PipelineFeature, markdown_genomes, markdown_multiqc +from nf_core.pipelines.create.utils import PipelineFeature class NfcorePipeline(Screen): @@ -22,27 +22,20 @@ def compose(self) -> ComposeResult: """ ) ) - yield ScrollableContainer( - PipelineFeature( - markdown_genomes, - "Use reference genomes", - "The pipeline will be configured to use a copy of the most common reference genome files from iGenomes", - "igenomes", - ), - PipelineFeature( - markdown_multiqc, - "Use multiqc", - "The pipeline will include the MultiQC module which generates an HTML report for quality control.", - "multiqc", - ), - classes="features-container", - ) + yield ScrollableContainer(id="features") yield Center( Button("Back", id="back", variant="default"), Button("Continue", id="continue", variant="success"), classes="cta", ) + def on_mount(self) -> None: + for name, feature in self.parent.template_features_yml.items(): + if feature["nfcore_pipelines"]: + self.query_one("#features").mount( + PipelineFeature(feature["help"], feature["short_description"], feature["description"], name) + ) + @on(Button.Pressed, "#continue") def on_button_pressed(self, event: Button.Pressed) -> None: """Save fields to the config.""" diff --git a/nf_core/pipelines/create/templatefeatures.yml b/nf_core/pipelines/create/templatefeatures.yml index 843203c7f8..a91887a891 100644 --- a/nf_core/pipelines/create/templatefeatures.yml +++ b/nf_core/pipelines/create/templatefeatures.yml @@ -25,6 +25,8 @@ github: - ".github/.dockstore.yml" readme: - "nextflow_badge" + nfcore_pipelines: False + custom_pipelines: False ci: skippable_paths: - ".github/workflows/" @@ -46,6 +48,8 @@ ci: - ".github/workflows/ci.yml" - ".github/workflows/linting_comment.yml" - ".github/workflows/linting.yml" + nfcore_pipelines: False + custom_pipelines: True igenomes: skippable_paths: - "conf/igenomes.config" @@ -65,6 +69,8 @@ igenomes: linting: files_exist: - "conf/igenomes.config" + nfcore_pipelines: True + custom_pipelines: True github_badges: skippable_paths: False short_description: "Add Github badges" @@ -81,6 +87,8 @@ github_badges: linting: readme: - "nextflow_badge" + nfcore_pipelines: False + custom_pipelines: True nf_core_configs: skippable_paths: False short_description: "Add configuration files" @@ -101,6 +109,8 @@ nf_core_configs: - "process.memory" - "process.time" - "custom_config" + nfcore_pipelines: False + custom_pipelines: True is_nfcore: skippable_paths: - ".github/ISSUE_TEMPLATE/config" @@ -131,6 +141,8 @@ is_nfcore: - "manifest.homePage" multiqc_config: - "report_comment" + nfcore_pipelines: False + custom_pipelines: False code_linters: skippable_paths: - ".editorconfig" @@ -148,6 +160,8 @@ code_linters: - pre-commit (https://pre-commit.com/): used to run all code-linters on every PR and on ever commit if you run `pre-commit install` to install it in your local repository. - editor-config (https://github.com/editorconfig-checker/editorconfig-checker): checks rules such as indentation or trailing spaces. - prettier (https://github.com/prettier/prettier): enforces a consistent style (indentation, quoting, line length, etc). + nfcore_pipelines: False + custom_pipelines: True citations: skippable_paths: - "assets/methods_description_template.yml" @@ -158,6 +172,8 @@ citations: Additionally, it will include a YAML file (`assets/methods_description_template.yml`) to add a Materials & Methods section describing the tools used in the pieline, and the logics to add this section to the output MultiQC report (if the report is generated). + nfcore_pipelines: False + custom_pipelines: True gitpod: skippable_paths: - ".gitpod.yml" @@ -168,6 +184,8 @@ gitpod: Including this to your pipeline will provide an environment with the latest version of nf-core/tools installed and all its requirements. This is useful to have all the tools ready for pipeline development. + nfcore_pipelines: False + custom_pipelines: True codespaces: skippable_paths: - ".devcontainer/devcontainer.json" @@ -181,6 +199,8 @@ codespaces: linting: files_unchanged: - ".github/CONTRIBUTING.md" + nfcore_pipelines: False + custom_pipelines: True multiqc: skippable_paths: - "assets/multiqc_config.yml" @@ -199,6 +219,8 @@ multiqc: files_exist: - "assets/multiqc_config.yml" multiqc_config: False + nfcore_pipelines: True + custom_pipelines: True changelog: skippable_paths: - "CHANGELOG.md" @@ -211,3 +233,5 @@ changelog: linting: files_exist: - "CHANGELOG.md" + nfcore_pipelines: False + custom_pipelines: True diff --git a/nf_core/pipelines/create/utils.py b/nf_core/pipelines/create/utils.py index 43094ab531..dba498d181 100644 --- a/nf_core/pipelines/create/utils.py +++ b/nf_core/pipelines/create/utils.py @@ -244,24 +244,3 @@ def add_hide_class(app, widget_id: str) -> None: def remove_hide_class(app, widget_id: str) -> None: """Remove class 'hide' to a widget. Display widget.""" app.get_widget_by_id(widget_id).remove_class("hide") - - -## Markdown text to reuse in different screens -markdown_genomes = """ -Nf-core pipelines are configured to use a copy of the most common reference genome files. - -By selecting this option, your pipeline will include a configuration file specifying the paths to these files. - -The required code to use these files will also be included in the template. -When the pipeline user provides an appropriate genome key, -the pipeline will automatically download the required reference files. - -For more information about reference genomes in nf-core pipelines, -see the [nf-core docs](https://nf-co.re/docs/usage/reference_genomes). -""" - -markdown_multiqc = """ -MultiQC is a visualization tool that generates a single HTML report summarising all samples in your project. Most of the pipeline quality control results can be visualised in the report and further statistics are available in the report data directory. - -The pipeline will include the MultiQC module and will have special steps which also allow the software versions to be reported in the MultiQC output for future traceability. For more information about how to use MultiQC reports, see http://multiqc.info. -""" From 2fa7a033ac1a4d99563e63cdaf76611c223b5da8 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 9 Aug 2024 11:26:29 +0200 Subject: [PATCH 04/11] update textual snapshots --- tests/__snapshots__/test_create_app.ambr | 764 +++++++++++------------ 1 file changed, 382 insertions(+), 382 deletions(-) diff --git a/tests/__snapshots__/test_create_app.ambr b/tests/__snapshots__/test_create_app.ambr index f2c1c45c45..f4eb255087 100644 --- a/tests/__snapshots__/test_create_app.ambr +++ b/tests/__snapshots__/test_create_app.ambr @@ -851,257 +851,257 @@ font-weight: 700; } - .terminal-463758155-matrix { + .terminal-3220763577-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-463758155-title { + .terminal-3220763577-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-463758155-r1 { fill: #c5c8c6 } - .terminal-463758155-r2 { fill: #e3e3e3 } - .terminal-463758155-r3 { fill: #989898 } - .terminal-463758155-r4 { fill: #e1e1e1 } - .terminal-463758155-r5 { fill: #4ebf71;font-weight: bold } - .terminal-463758155-r6 { fill: #1e1e1e } - .terminal-463758155-r7 { fill: #0178d4 } - .terminal-463758155-r8 { fill: #454a50 } - .terminal-463758155-r9 { fill: #e2e2e2 } - .terminal-463758155-r10 { fill: #808080 } - .terminal-463758155-r11 { fill: #e2e3e3;font-weight: bold } - .terminal-463758155-r12 { fill: #000000 } - .terminal-463758155-r13 { fill: #e4e4e4 } - .terminal-463758155-r14 { fill: #14191f } - .terminal-463758155-r15 { fill: #507bb3 } - .terminal-463758155-r16 { fill: #dde6ed;font-weight: bold } - .terminal-463758155-r17 { fill: #001541 } - .terminal-463758155-r18 { fill: #7ae998 } - .terminal-463758155-r19 { fill: #0a180e;font-weight: bold } - .terminal-463758155-r20 { fill: #008139 } - .terminal-463758155-r21 { fill: #fea62b;font-weight: bold } - .terminal-463758155-r22 { fill: #a7a9ab } - .terminal-463758155-r23 { fill: #e2e3e3 } + .terminal-3220763577-r1 { fill: #c5c8c6 } + .terminal-3220763577-r2 { fill: #e3e3e3 } + .terminal-3220763577-r3 { fill: #989898 } + .terminal-3220763577-r4 { fill: #e1e1e1 } + .terminal-3220763577-r5 { fill: #4ebf71;font-weight: bold } + .terminal-3220763577-r6 { fill: #1e1e1e } + .terminal-3220763577-r7 { fill: #507bb3 } + .terminal-3220763577-r8 { fill: #e2e2e2 } + .terminal-3220763577-r9 { fill: #808080 } + .terminal-3220763577-r10 { fill: #dde6ed;font-weight: bold } + .terminal-3220763577-r11 { fill: #001541 } + .terminal-3220763577-r12 { fill: #0178d4 } + .terminal-3220763577-r13 { fill: #454a50 } + .terminal-3220763577-r14 { fill: #e2e3e3;font-weight: bold } + .terminal-3220763577-r15 { fill: #000000 } + .terminal-3220763577-r16 { fill: #e4e4e4 } + .terminal-3220763577-r17 { fill: #14191f } + .terminal-3220763577-r18 { fill: #7ae998 } + .terminal-3220763577-r19 { fill: #0a180e;font-weight: bold } + .terminal-3220763577-r20 { fill: #008139 } + .terminal-3220763577-r21 { fill: #fea62b;font-weight: bold } + .terminal-3220763577-r22 { fill: #a7a9ab } + .terminal-3220763577-r23 { fill: #e2e3e3 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - nf-core create + nf-core create - - - - ⭘nf-core create β€” Create a new pipeline with the nf-core pipeline template - - - Template features - - - β–Šβ–”β–”β–”β–”β–”β–”β–”β–”β–Žβ–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–” - β–Šβ–Ž        Use reference The pipeline will  Hide help  - β–Šβ–β–β–β–β–β–β–β–β–Ž        genomesbe configured to β–β–β–β–β–β–β–β–β–β–β–β–β–β–β–β– - use a copy of the  - most common  - reference genome  - files from  - iGenomes - - - Nf-core pipelines are configured to use a copy of the most common  - reference genome files. - - By selecting this option, your pipeline will include a configuration - file specifying the paths to these files.β–‚β–‚ - - The required code to use these files will also be included in the  - template. When the pipeline user provides an appropriate genome key, - the pipeline will automatically download the required reference β–‚β–‚ - files. - - - - β–Šβ–”β–”β–”β–”β–”β–”β–”β–”β–Žβ–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–” - β–Šβ–Ž        Add Github CI The pipeline will  Show help  - β–Šβ–β–β–β–β–β–β–β–β–Ž        testsinclude several β–β–β–β–β–β–β–β–β–β–β–β–β–β–β–β– - GitHub actions for - Continuous  - Integration (CI)  - testing - - β–Šβ–”β–”β–”β–”β–”β–”β–”β–”β–Žβ–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–” - β–Šβ–Ž        Add Github badgesThe README.md file Show help  - β–Šβ–β–β–β–β–β–β–β–β–Žof the pipeline β–β–β–β–β–β–β–β–β–β–β–β–β–β–β–β– - will include  - GitHub badges - - β–Šβ–”β–”β–”β–”β–”β–”β–”β–”β–Žβ–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–” - - β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–” -  Back  Continue  - ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - -  d Toggle dark mode  q Quit  + + + + ⭘nf-core create β€” Create a new pipeline with the nf-core pipeline template + + + Template features + + + β–Šβ–”β–”β–”β–”β–”β–”β–”β–”β–Žβ–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–” + β–Šβ–Ž        Add Github CI testsThe pipeline will  Show help  + β–Šβ–β–β–β–β–β–β–β–β–Žinclude several GitHub▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + actions for Continuous + Integration (CI)  + testing + + β–Šβ–”β–”β–”β–”β–”β–”β–”β–”β–Žβ–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–” + β–Šβ–Ž        Use reference genomesThe pipeline will be  Hide help  + β–Šβ–β–β–β–β–β–β–β–β–Žconfigured to use a β–β–β–β–β–β–β–β–β–β–β–β–β–β–β–β– + copy of the most  + common reference  + genome files from  + iGenomes + + + Nf-core pipelines are configured to use a copy of the most common reference  + genome files.β–‚β–‚ + + By selecting this option, your pipeline will include a configuration file  + specifying the paths to these files. + + The required code to use these files will also be included in the template.  + When the pipeline user provides an appropriate genome key, the pipeline will + automatically download the required reference files. + β–…β–… + For more information about reference genomes in nf-core pipelines, see the  + + + β–Šβ–”β–”β–”β–”β–”β–”β–”β–”β–Žβ–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–” + β–Šβ–Ž        Add Github badgesThe README.md file of  Show help  + β–Šβ–β–β–β–β–β–β–β–β–Žthe pipeline will β–β–β–β–β–β–β–β–β–β–β–β–β–β–β–β– + include GitHub badges + + β–Šβ–”β–”β–”β–”β–”β–”β–”β–”β–Žβ–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–” + β–Šβ–Ž        Add configuration The pipeline will  Show help  + β–Šβ–β–β–β–β–β–β–β–β–Ž        filesinclude configuration β–β–β–β–β–β–β–β–β–β–β–β–β–β–β–β– + profiles containing  + custom parameters  + β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–” +  Back  Continue  + ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + +  d Toggle dark mode  q Quit  @@ -2233,255 +2233,255 @@ font-weight: 700; } - .terminal-2213530577-matrix { + .terminal-537214554-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2213530577-title { + .terminal-537214554-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2213530577-r1 { fill: #c5c8c6 } - .terminal-2213530577-r2 { fill: #e3e3e3 } - .terminal-2213530577-r3 { fill: #989898 } - .terminal-2213530577-r4 { fill: #e1e1e1 } - .terminal-2213530577-r5 { fill: #4ebf71;font-weight: bold } - .terminal-2213530577-r6 { fill: #1e1e1e } - .terminal-2213530577-r7 { fill: #507bb3 } - .terminal-2213530577-r8 { fill: #e2e2e2 } - .terminal-2213530577-r9 { fill: #808080 } - .terminal-2213530577-r10 { fill: #dde6ed;font-weight: bold } - .terminal-2213530577-r11 { fill: #001541 } - .terminal-2213530577-r12 { fill: #14191f } - .terminal-2213530577-r13 { fill: #454a50 } - .terminal-2213530577-r14 { fill: #7ae998 } - .terminal-2213530577-r15 { fill: #e2e3e3;font-weight: bold } - .terminal-2213530577-r16 { fill: #0a180e;font-weight: bold } - .terminal-2213530577-r17 { fill: #000000 } - .terminal-2213530577-r18 { fill: #008139 } - .terminal-2213530577-r19 { fill: #fea62b;font-weight: bold } - .terminal-2213530577-r20 { fill: #a7a9ab } - .terminal-2213530577-r21 { fill: #e2e3e3 } + .terminal-537214554-r1 { fill: #c5c8c6 } + .terminal-537214554-r2 { fill: #e3e3e3 } + .terminal-537214554-r3 { fill: #989898 } + .terminal-537214554-r4 { fill: #e1e1e1 } + .terminal-537214554-r5 { fill: #4ebf71;font-weight: bold } + .terminal-537214554-r6 { fill: #1e1e1e } + .terminal-537214554-r7 { fill: #507bb3 } + .terminal-537214554-r8 { fill: #e2e2e2 } + .terminal-537214554-r9 { fill: #808080 } + .terminal-537214554-r10 { fill: #dde6ed;font-weight: bold } + .terminal-537214554-r11 { fill: #001541 } + .terminal-537214554-r12 { fill: #14191f } + .terminal-537214554-r13 { fill: #454a50 } + .terminal-537214554-r14 { fill: #7ae998 } + .terminal-537214554-r15 { fill: #e2e3e3;font-weight: bold } + .terminal-537214554-r16 { fill: #0a180e;font-weight: bold } + .terminal-537214554-r17 { fill: #000000 } + .terminal-537214554-r18 { fill: #008139 } + .terminal-537214554-r19 { fill: #fea62b;font-weight: bold } + .terminal-537214554-r20 { fill: #a7a9ab } + .terminal-537214554-r21 { fill: #e2e3e3 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - nf-core create + nf-core create - - - - ⭘nf-core create β€” Create a new pipeline with the nf-core pipeline template - - - Template features - - - β–Šβ–”β–”β–”β–”β–”β–”β–”β–”β–Žβ–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–” - β–Šβ–Ž        Use reference The pipeline will  Show help  - β–Šβ–β–β–β–β–β–β–β–β–Ž        genomesbe configured to β–β–β–β–β–β–β–β–β–β–β–β–β–β–β–β– - use a copy of the  - most common  - reference genome  - files from  - iGenomes - - β–Šβ–”β–”β–”β–”β–”β–”β–”β–”β–Žβ–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–” - β–Šβ–Ž        Add Github CI The pipeline will  Show help  - β–Šβ–β–β–β–β–β–β–β–β–Ž        testsinclude several β–β–β–β–β–β–β–β–β–β–β–β–β–β–β–β– - GitHub actions for - Continuous  - Integration (CI)  - testing - - β–Šβ–”β–”β–”β–”β–”β–”β–”β–”β–Žβ–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–†β–† - β–Šβ–Ž        Add Github badgesThe README.md file Show help  - β–Šβ–β–β–β–β–β–β–β–β–Žof the pipeline β–β–β–β–β–β–β–β–β–β–β–β–β–β–β–β– - will include  - GitHub badges - - β–Šβ–”β–”β–”β–”β–”β–”β–”β–”β–Žβ–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–” - β–Šβ–Ž        Add configuration         The pipeline will  Show help  - β–Šβ–β–β–β–β–β–β–β–β–Ž        filesinclude β–β–β–β–β–β–β–β–β–β–β–β–β–β–β–β– - configuration  - profiles  - containing custom  - parameters  - requried to run  - nf-core pipelines  - at different  - institutions - - β–Šβ–”β–”β–”β–”β–”β–”β–”β–”β–Žβ–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–” - β–Šβ–Ž        Use code lintersThe pipeline will  Show help  - β–Šβ–β–β–β–β–β–β–β–β–Žinclude code β–β–β–β–β–β–β–β–β–β–β–β–β–β–β–β– - - β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–” -  Back  Continue  - ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - -  d Toggle dark mode  q Quit  + + + + ⭘nf-core create β€” Create a new pipeline with the nf-core pipeline template + + + Template features + + + β–Šβ–”β–”β–”β–”β–”β–”β–”β–”β–Žβ–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–” + β–Šβ–Ž        Add Github CI testsThe pipeline will  Show help  + β–Šβ–β–β–β–β–β–β–β–β–Žinclude several GitHub▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + actions for Continuous + Integration (CI)  + testing + + β–Šβ–”β–”β–”β–”β–”β–”β–”β–”β–Žβ–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–” + β–Šβ–Ž        Use reference genomesThe pipeline will be  Show help  + β–Šβ–β–β–β–β–β–β–β–β–Žconfigured to use a β–β–β–β–β–β–β–β–β–β–β–β–β–β–β–β– + copy of the most  + common reference  + genome files from  + iGenomes + + β–Šβ–”β–”β–”β–”β–”β–”β–”β–”β–Žβ–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–” + β–Šβ–Ž        Add Github badgesThe README.md file of  Show help  + β–Šβ–β–β–β–β–β–β–β–β–Žthe pipeline will β–β–β–β–β–β–β–β–β–β–β–β–β–β–β–β– + include GitHub badges + + β–Šβ–”β–”β–”β–”β–”β–”β–”β–”β–Žβ–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–” + β–Šβ–Ž        Add configuration The pipeline will  Show help β–‡β–‡ + β–Šβ–β–β–β–β–β–β–β–β–Ž        filesinclude configuration β–β–β–β–β–β–β–β–β–β–β–β–β–β–β–β– + profiles containing  + custom parameters  + requried to run  + nf-core pipelines at  + different institutions + + β–Šβ–”β–”β–”β–”β–”β–”β–”β–”β–Žβ–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–” + β–Šβ–Ž        Use code lintersThe pipeline will  Show help  + β–Šβ–β–β–β–β–β–β–β–β–Žinclude code linters β–β–β–β–β–β–β–β–β–β–β–β–β–β–β–β– + and CI tests to lint  + your code: pre-commit, + editor-config and  + prettier. + + β–Šβ–”β–”β–”β–”β–”β–”β–”β–”β–Žβ–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–” + β–Šβ–Ž        Include citationsInclude pipeline tools Show help  + β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–” +  Back  Continue  + ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + +  d Toggle dark mode  q Quit  @@ -2511,254 +2511,254 @@ font-weight: 700; } - .terminal-157696122-matrix { + .terminal-1633351929-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-157696122-title { + .terminal-1633351929-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-157696122-r1 { fill: #c5c8c6 } - .terminal-157696122-r2 { fill: #e3e3e3 } - .terminal-157696122-r3 { fill: #989898 } - .terminal-157696122-r4 { fill: #e1e1e1 } - .terminal-157696122-r5 { fill: #4ebf71;font-weight: bold } - .terminal-157696122-r6 { fill: #1e1e1e } - .terminal-157696122-r7 { fill: #507bb3 } - .terminal-157696122-r8 { fill: #e2e2e2 } - .terminal-157696122-r9 { fill: #808080 } - .terminal-157696122-r10 { fill: #dde6ed;font-weight: bold } - .terminal-157696122-r11 { fill: #001541 } - .terminal-157696122-r12 { fill: #454a50 } - .terminal-157696122-r13 { fill: #7ae998 } - .terminal-157696122-r14 { fill: #e2e3e3;font-weight: bold } - .terminal-157696122-r15 { fill: #0a180e;font-weight: bold } - .terminal-157696122-r16 { fill: #000000 } - .terminal-157696122-r17 { fill: #008139 } - .terminal-157696122-r18 { fill: #fea62b;font-weight: bold } - .terminal-157696122-r19 { fill: #a7a9ab } - .terminal-157696122-r20 { fill: #e2e3e3 } + .terminal-1633351929-r1 { fill: #c5c8c6 } + .terminal-1633351929-r2 { fill: #e3e3e3 } + .terminal-1633351929-r3 { fill: #989898 } + .terminal-1633351929-r4 { fill: #e1e1e1 } + .terminal-1633351929-r5 { fill: #4ebf71;font-weight: bold } + .terminal-1633351929-r6 { fill: #1e1e1e } + .terminal-1633351929-r7 { fill: #507bb3 } + .terminal-1633351929-r8 { fill: #e2e2e2 } + .terminal-1633351929-r9 { fill: #808080 } + .terminal-1633351929-r10 { fill: #dde6ed;font-weight: bold } + .terminal-1633351929-r11 { fill: #001541 } + .terminal-1633351929-r12 { fill: #454a50 } + .terminal-1633351929-r13 { fill: #7ae998 } + .terminal-1633351929-r14 { fill: #e2e3e3;font-weight: bold } + .terminal-1633351929-r15 { fill: #0a180e;font-weight: bold } + .terminal-1633351929-r16 { fill: #000000 } + .terminal-1633351929-r17 { fill: #008139 } + .terminal-1633351929-r18 { fill: #fea62b;font-weight: bold } + .terminal-1633351929-r19 { fill: #a7a9ab } + .terminal-1633351929-r20 { fill: #e2e3e3 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - nf-core create + nf-core create - - - - ⭘nf-core create β€” Create a new pipeline with the nf-core pipeline template - - - Template features - - - β–Šβ–”β–”β–”β–”β–”β–”β–”β–”β–Žβ–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–” - β–Šβ–Ž        Use reference The pipeline will  Show help  - β–Šβ–β–β–β–β–β–β–β–β–Ž        genomesbe configured to β–β–β–β–β–β–β–β–β–β–β–β–β–β–β–β– - use a copy of the  - most common  - reference genome  - files from iGenomes - - β–Šβ–”β–”β–”β–”β–”β–”β–”β–”β–Žβ–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–” - β–Šβ–Ž        Use multiqcThe pipeline will  Show help  - β–Šβ–β–β–β–β–β–β–β–β–Žinclude the MultiQC▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - module which  - generates an HTML  - report for quality  - control. - - - - - - - - - - - - - - - - - - - - - - - - - β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–” -  Back  Continue  - ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - -  d Toggle dark mode  q Quit  + + + + ⭘nf-core create β€” Create a new pipeline with the nf-core pipeline template + + + Template features + + + β–Šβ–”β–”β–”β–”β–”β–”β–”β–”β–Žβ–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–” + β–Šβ–Ž        Use reference genomesThe pipeline will be  Show help  + β–Šβ–β–β–β–β–β–β–β–β–Žconfigured to use a β–β–β–β–β–β–β–β–β–β–β–β–β–β–β–β– + copy of the most common + reference genome files  + from iGenomes + + β–Šβ–”β–”β–”β–”β–”β–”β–”β–”β–Žβ–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–” + β–Šβ–Ž        Use multiqcThe pipeline will  Show help  + β–Šβ–β–β–β–β–β–β–β–β–Žinclude the MultiQC β–β–β–β–β–β–β–β–β–β–β–β–β–β–β–β– + module which generates  + an HTML report for  + quality control. + + + + + + + + + + + + + + + + + + + + + + + + + + + β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–”β–” +  Back  Continue  + ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + +  d Toggle dark mode  q Quit  From a009e68f594767a90cccd0df0e4fbbbceaf4a3f2 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 9 Aug 2024 11:37:18 +0200 Subject: [PATCH 05/11] try reading yaml keys with a list format --- .github/workflows/create-test-lint-wf-template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index d4ff3fc557..15582951de 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -48,7 +48,7 @@ jobs: uses: actions/checkout@v3 - name: Create Matrix id: create_matrix - run: echo "matrix=$(yq 'keys' ${{ github.workspace }}/nf_core/pipelines/create/templatefeatures.yml)" >> $GITHUB_OUTPUT + run: echo "matrix=$(yq 'keys' ${{ github.workspace }}/nf_core/pipelines/create/templatefeatures.yml | awk '{print $2}' | paste -sd "," - | awk '{print "[" $0 "]"}')" >> $GITHUB_OUTPUT RunTestWorkflow: runs-on: ${{ matrix.runner }} From e36bd7fc6411b68c8b22b17aa783ae9680f4fb2c Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 12 Aug 2024 09:37:06 +0200 Subject: [PATCH 06/11] don't read from json on matrix creation --- .github/workflows/create-test-lint-wf-template.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index 15582951de..fd7a494d0d 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -56,7 +56,7 @@ jobs: NXF_ANSI_LOG: false strategy: matrix: - TEMPLATE: ${{ fromJSON(needs.prepare-matrix.outputs.all_features) }} + TEMPLATE: ${{ needs.prepare-matrix.outputs.all_features }} runner: # use the runner given by the input if it is dispatched manually, run on github if it is a rerun or on self-hosted by default - ${{ github.event.inputs.runners || github.run_number > 1 && 'ubuntu-latest' || 'self-hosted' }} @@ -68,6 +68,9 @@ jobs: - TEMPLATE: nf_core_configs runner: ubuntu-latest profile: "docker" + exclude: + - TEMPLATE: github + - TEMPLATE: is_nfcore fail-fast: false steps: From 78c64c719db446bf84f33137cbce80cd54ee37bc Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 13 Aug 2024 11:07:04 +0200 Subject: [PATCH 07/11] apply suggestions from code review by @mashehu --- .../create-test-lint-wf-template.yml | 9 ++---- nf_core/pipelines/create/create.py | 4 ++- nf_core/pipelines/create/custompipeline.py | 2 +- nf_core/pipelines/create/nfcorepipeline.py | 2 +- nf_core/pipelines/create/templatefeatures.yml | 28 +++++++++---------- 5 files changed, 21 insertions(+), 24 deletions(-) diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index fd7a494d0d..bd661ce07c 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -31,19 +31,14 @@ env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} jobs: - install-yq: - name: Install yq - runs-on: ubuntu-latest - steps: - - name: πŸ— Set up yq - uses: frenck/action-setup-yq@v1 - prepare-matrix: name: Retrieve all template features runs-on: ubuntu-latest outputs: all_features: ${{ steps.create_matrix.outputs.matrix }} steps: + - name: πŸ— Set up yq + uses: frenck/action-setup-yq@v1 - name: checkout uses: actions/checkout@v3 - name: Create Matrix diff --git a/nf_core/pipelines/create/create.py b/nf_core/pipelines/create/create.py index 584a4673da..1df6cc3f46 100644 --- a/nf_core/pipelines/create/create.py +++ b/nf_core/pipelines/create/create.py @@ -185,7 +185,9 @@ def update_config(self, organisation, version, force, outdir): if self.config.is_nfcore is None: self.config.is_nfcore = self.config.org == "nf-core" - def obtain_jinja_params_dict(self, features_to_skip: List[str], pipeline_dir: Union[str, Path]): + def obtain_jinja_params_dict( + self, features_to_skip: List[str], pipeline_dir: Union[str, Path] + ) -> tuple[dict, list[str]]: """Creates a dictionary of parameters for the new pipeline. Args: diff --git a/nf_core/pipelines/create/custompipeline.py b/nf_core/pipelines/create/custompipeline.py index fdb7a46d28..5debcfee7f 100644 --- a/nf_core/pipelines/create/custompipeline.py +++ b/nf_core/pipelines/create/custompipeline.py @@ -33,7 +33,7 @@ def on_mount(self) -> None: for name, feature in self.parent.template_features_yml.items(): if feature["custom_pipelines"]: self.query_one("#features").mount( - PipelineFeature(feature["help"], feature["short_description"], feature["description"], name) + PipelineFeature(feature["help_text"], feature["short_description"], feature["description"], name) ) @on(Button.Pressed, "#continue") diff --git a/nf_core/pipelines/create/nfcorepipeline.py b/nf_core/pipelines/create/nfcorepipeline.py index b6a8231694..ebb9866986 100644 --- a/nf_core/pipelines/create/nfcorepipeline.py +++ b/nf_core/pipelines/create/nfcorepipeline.py @@ -33,7 +33,7 @@ def on_mount(self) -> None: for name, feature in self.parent.template_features_yml.items(): if feature["nfcore_pipelines"]: self.query_one("#features").mount( - PipelineFeature(feature["help"], feature["short_description"], feature["description"], name) + PipelineFeature(feature["help_text"], feature["short_description"], feature["description"], name) ) @on(Button.Pressed, "#continue") diff --git a/nf_core/pipelines/create/templatefeatures.yml b/nf_core/pipelines/create/templatefeatures.yml index a91887a891..48ce200551 100644 --- a/nf_core/pipelines/create/templatefeatures.yml +++ b/nf_core/pipelines/create/templatefeatures.yml @@ -2,9 +2,9 @@ github: skippable_paths: - ".github/" - ".gitignore" - short_description: "" + short_description: "Skip the creation of a local Git repository." description: "" - help: "" + help_text: "" linting: files_exist: - ".github/ISSUE_TEMPLATE/bug_report.yml" @@ -32,7 +32,7 @@ ci: - ".github/workflows/" short_description: "Add Github CI tests" description: "The pipeline will include several GitHub actions for Continuous Integration (CI) testing" - help: | + help_text: | Nf-core provides a set of Continuous Integration (CI) tests for Github. When you open a pull request (PR) on your pipeline repository, these tests will run automatically. @@ -55,7 +55,7 @@ igenomes: - "conf/igenomes.config" short_description: "Use reference genomes" description: "The pipeline will be configured to use a copy of the most common reference genome files from iGenomes" - help: | + help_text: | Nf-core pipelines are configured to use a copy of the most common reference genome files. By selecting this option, your pipeline will include a configuration file specifying the paths to these files. @@ -75,7 +75,7 @@ github_badges: skippable_paths: False short_description: "Add Github badges" description: "The README.md file of the pipeline will include GitHub badges" - help: | + help_text: | The pipeline `README.md` will include badges for: * AWS CI Tests * Zenodo DOI @@ -93,7 +93,7 @@ nf_core_configs: skippable_paths: False short_description: "Add configuration files" description: "The pipeline will include configuration profiles containing custom parameters requried to run nf-core pipelines at different institutions" - help: | + help_text: | Nf-core has a repository with a collection of configuration profiles. Those config files define a set of parameters which are specific to compute environments at different Institutions. @@ -118,9 +118,9 @@ is_nfcore: - ".github/workflows/awsfulltest.yml" - ".github/workflows/awstest.yml" - ".github/workflows/release-announcements.yml" - short_description: "" + short_description: "A custom pipeline which won't be part of the nf-core organisation but be compatible with nf-core/tools." description: "" - help: "" + help_text: "" linting: files_exist: - "CODE_OF_CONDUCT.md" @@ -152,7 +152,7 @@ code_linters: - ".github/workflows/fix-linting.yml" short_description: "Use code linters" description: "The pipeline will include code linters and CI tests to lint your code: pre-commit, editor-config and prettier." - help: | + help_text: | Pipelines include code linters to check the formatting of your code in order to harmonize code styles between developers. Linters will check all non-ignored files, e.g., JSON, YAML, Nextlow or Python files in your repository. The available code linters are: @@ -167,7 +167,7 @@ citations: - "assets/methods_description_template.yml" short_description: "Include citations" description: "Include pipeline tools citations in CITATIONS.md and a method description in the MultiQC report (if enabled)." - help: | + help_text: | If adding citations, the pipeline template will contain a `CITATIONS.md` file to add the citations of all tools used in the pipeline. Additionally, it will include a YAML file (`assets/methods_description_template.yml`) to add a Materials & Methods section describing the tools used in the pieline, @@ -179,7 +179,7 @@ gitpod: - ".gitpod.yml" short_description: "Include a gitpod environment" description: "Include the configuration required to use Gitpod." - help: | + help_text: | Gitpod (https://www.gitpod.io/) provides standardized and automated development environments. Including this to your pipeline will provide an environment with the latest version of nf-core/tools installed and all its requirements. @@ -191,7 +191,7 @@ codespaces: - ".devcontainer/devcontainer.json" short_description: "Include GitHub Codespaces" description: "The pipeline will include a devcontainer configuration for GitHub Codespaces, providing a development environment with nf-core/tools and Nextflow installed." - help: | + help_text: | The pipeline will include a devcontainer configuration. The devcontainer will create a GitHub Codespaces for Nextflow development with nf-core/tools and Nextflow installed. @@ -208,7 +208,7 @@ multiqc: - "modules/nf-core/multiqc/" short_description: "Use multiqc" description: "The pipeline will include the MultiQC module which generates an HTML report for quality control." - help: | + help_text: | MultiQC is a visualization tool that generates a single HTML report summarising all samples in your project. Most of the pipeline quality control results can be visualised in the report and further statistics are available in the report data directory. The pipeline will include the MultiQC module and will have special steps which also allow the software versions to be reported in the MultiQC output for future traceability. For more information about how to use MultiQC reports, see http://multiqc.info. @@ -226,7 +226,7 @@ changelog: - "CHANGELOG.md" short_description: "Add a changelog" description: "Add a CHANGELOG.md file." - help: | + help_text: | Having a `CHANGELOG.md` file in the pipeline root directory is useful to track the changes added to each version. You can read more information on the recommended format here: https://keepachangelog.com/en/1.0.0/ From bf07bf24a31850250486000c708c7b6e31b60f01 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 13 Aug 2024 11:32:58 +0200 Subject: [PATCH 08/11] load feature yaml file with a utils function --- nf_core/pipelines/create/__init__.py | 4 +--- nf_core/pipelines/create/create.py | 6 ++---- nf_core/pipelines/create/utils.py | 11 +++++++++++ tests/pipelines/test_create.py | 5 ++--- 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/nf_core/pipelines/create/__init__.py b/nf_core/pipelines/create/__init__.py index 1774d0a010..8b0edf34cf 100644 --- a/nf_core/pipelines/create/__init__.py +++ b/nf_core/pipelines/create/__init__.py @@ -72,9 +72,7 @@ class PipelineCreateApp(App[utils.CreateConfig]): LOGGING_STATE = None # Template features - features_yml_path = Path(nf_core.__file__).parent / "pipelines" / "create" / "templatefeatures.yml" - with open(features_yml_path) as fh: - template_features_yml = yaml.safe_load(fh) + template_features_yml = utils.load_features_yaml() def on_mount(self) -> None: self.push_screen("welcome") diff --git a/nf_core/pipelines/create/create.py b/nf_core/pipelines/create/create.py index 1df6cc3f46..b3135435ee 100644 --- a/nf_core/pipelines/create/create.py +++ b/nf_core/pipelines/create/create.py @@ -18,7 +18,7 @@ import nf_core import nf_core.pipelines.schema import nf_core.utils -from nf_core.pipelines.create.utils import CreateConfig +from nf_core.pipelines.create.utils import CreateConfig, features_yml_path, load_features_yaml from nf_core.pipelines.create_logo import create_logo from nf_core.pipelines.lint_utils import run_prettier_on_file from nf_core.utils import LintConfigType, NFCoreTemplateConfig @@ -82,9 +82,7 @@ def __init__( raise UserWarning("The template configuration was not provided.") # Read features yaml file - features_yml_path = Path(nf_core.__file__).parent / "pipelines" / "create" / "templatefeatures.yml" - with open(features_yml_path) as fh: - self.template_features_yml = yaml.safe_load(fh) + self.template_features_yml = load_features_yaml() if self.config.outdir is None: self.config.outdir = str(Path.cwd()) diff --git a/nf_core/pipelines/create/utils.py b/nf_core/pipelines/create/utils.py index 89cf0ab65a..2e0015ed90 100644 --- a/nf_core/pipelines/create/utils.py +++ b/nf_core/pipelines/create/utils.py @@ -5,6 +5,7 @@ from pathlib import Path from typing import Any, Dict, Iterator, Union +import yaml from pydantic import ConfigDict, ValidationError, ValidationInfo, field_validator from rich.logging import RichHandler from textual import on @@ -16,6 +17,7 @@ from textual.widget import Widget from textual.widgets import Button, Input, Markdown, RichLog, Static, Switch +import nf_core from nf_core.utils import NFCoreTemplateConfig # Use ContextVar to define a context on the model initialization @@ -34,6 +36,9 @@ def init_context(value: Dict[str, Any]) -> Iterator[None]: # Define a global variable to store the pipeline type NFCORE_PIPELINE_GLOBAL: bool = True +# YAML file describing template features +features_yml_path = Path(nf_core.__file__).parent / "pipelines" / "create" / "templatefeatures.yml" + class CreateConfig(NFCoreTemplateConfig): """Pydantic model for the nf-core create config.""" @@ -242,3 +247,9 @@ def add_hide_class(app, widget_id: str) -> None: def remove_hide_class(app, widget_id: str) -> None: """Remove class 'hide' to a widget. Display widget.""" app.get_widget_by_id(widget_id).remove_class("hide") + + +def load_features_yaml() -> dict: + """Load the YAML file describing template features.""" + with open(features_yml_path) as fh: + return yaml.safe_load(fh) diff --git a/tests/pipelines/test_create.py b/tests/pipelines/test_create.py index b684a4669e..13fd3b24cd 100644 --- a/tests/pipelines/test_create.py +++ b/tests/pipelines/test_create.py @@ -9,6 +9,7 @@ import yaml import nf_core.pipelines.create.create +from nf_core.pipelines.create.utils import load_features_yaml from ..utils import TEST_DATA_DIR, with_temporary_folder @@ -101,9 +102,7 @@ def test_pipeline_creation_initiation_customize_template(self, tmp_path): @with_temporary_folder def test_pipeline_creation_with_yml_skip(self, tmp_path): # Update pipeline_create_template_skip.yml file - features_yml_path = Path(nf_core.__file__).parent / "pipelines" / "create" / "templatefeatures.yml" - with open(features_yml_path) as fh: - template_features_yml = yaml.safe_load(fh) + template_features_yml = load_features_yaml() all_features = list(template_features_yml.keys()) all_features.remove("is_nfcore") env = jinja2.Environment(loader=jinja2.PackageLoader("tests", "data"), keep_trailing_newline=True) From 2c2ec5db394128a947bc874461582e45711648d1 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 13 Aug 2024 11:51:26 +0200 Subject: [PATCH 09/11] set force to true when creating a pipeline from the app --- nf_core/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/utils.py b/nf_core/utils.py index d0546a5c5e..96c16130ca 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -1055,7 +1055,7 @@ class NFCoreTemplateConfig(BaseModel): description: Optional[str] = None author: Optional[str] = None version: Optional[str] = None - force: Optional[bool] = None + force: Optional[bool] = True outdir: Optional[Union[str, Path]] = None skip_features: Optional[list] = None is_nfcore: Optional[bool] = None From 2535b6fc136fc24bef77249f3fc755b344100b9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Tue, 13 Aug 2024 12:16:51 +0200 Subject: [PATCH 10/11] Update nf_core/pipelines/create/utils.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias HΓΆrtenhuber --- nf_core/pipelines/create/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipelines/create/utils.py b/nf_core/pipelines/create/utils.py index 2e0015ed90..0b72c2bcf5 100644 --- a/nf_core/pipelines/create/utils.py +++ b/nf_core/pipelines/create/utils.py @@ -249,7 +249,7 @@ def remove_hide_class(app, widget_id: str) -> None: app.get_widget_by_id(widget_id).remove_class("hide") -def load_features_yaml() -> dict: +def load_features_yaml() -> Dict: """Load the YAML file describing template features.""" with open(features_yml_path) as fh: return yaml.safe_load(fh) From 6af86cf39306c2bf3c9bd4d0f61057fdea24f590 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 13 Aug 2024 12:29:02 +0200 Subject: [PATCH 11/11] fix typing --- nf_core/pipelines/create/create.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipelines/create/create.py b/nf_core/pipelines/create/create.py index b3135435ee..200507d8a0 100644 --- a/nf_core/pipelines/create/create.py +++ b/nf_core/pipelines/create/create.py @@ -8,7 +8,7 @@ import re import shutil from pathlib import Path -from typing import Dict, List, Optional, Union, cast +from typing import Dict, List, Optional, Tuple, Union, cast import git import git.config @@ -185,7 +185,7 @@ def update_config(self, organisation, version, force, outdir): def obtain_jinja_params_dict( self, features_to_skip: List[str], pipeline_dir: Union[str, Path] - ) -> tuple[dict, list[str]]: + ) -> Tuple[Dict, List[str]]: """Creates a dictionary of parameters for the new pipeline. Args: