diff --git a/.bumpversion.cfg b/.bumpversion.cfg index c1b79aee74..1557280522 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 38.3.0 +current_version = 39.0.0 commit = True tag = True tag_name = v{new_version} diff --git a/cg/__init__.py b/cg/__init__.py index 9342351f4f..4dcfc131d8 100644 --- a/cg/__init__.py +++ b/cg/__init__.py @@ -1,4 +1,4 @@ import pkg_resources __title__ = "cg" -__version__ = "38.3.0" +__version__ = "39.0.0" diff --git a/cg/apps/cgstats/db/models/demux_sample.py b/cg/apps/cgstats/db/models/demux_sample.py index 747f07f179..bc71ff6e8d 100644 --- a/cg/apps/cgstats/db/models/demux_sample.py +++ b/cg/apps/cgstats/db/models/demux_sample.py @@ -1,5 +1,5 @@ import logging -from pydantic import BaseModel, validator +from pydantic.v1 import BaseModel, validator from cg.apps.cgstats.parsers.conversion_stats import SampleConversionResults from cg.apps.cgstats.parsers.demux_stats import SampleBarcodeStats diff --git a/cg/apps/cgstats/db/models/dragen_demux_sample.py b/cg/apps/cgstats/db/models/dragen_demux_sample.py index e210e4d410..c3eb3f7469 100644 --- a/cg/apps/cgstats/db/models/dragen_demux_sample.py +++ b/cg/apps/cgstats/db/models/dragen_demux_sample.py @@ -3,7 +3,7 @@ import logging -from pydantic import BaseModel +from pydantic.v1 import BaseModel LOG = logging.getLogger(__name__) diff --git a/cg/apps/cgstats/parsers/conversion_stats.py b/cg/apps/cgstats/parsers/conversion_stats.py index ee6d518d9a..848f84b461 100644 --- a/cg/apps/cgstats/parsers/conversion_stats.py +++ b/cg/apps/cgstats/parsers/conversion_stats.py @@ -27,7 +27,7 @@ from typing import Dict, List, Optional, Set from xml.etree.ElementTree import Element, iterparse -from pydantic import BaseModel +from pydantic.v1 import BaseModel LOG = logging.getLogger(__name__) diff --git a/cg/apps/cgstats/parsers/demux_stats.py b/cg/apps/cgstats/parsers/demux_stats.py index bfce80a055..97cd102fbe 100644 --- a/cg/apps/cgstats/parsers/demux_stats.py +++ b/cg/apps/cgstats/parsers/demux_stats.py @@ -4,7 +4,7 @@ from typing import Dict, Optional, Set from xml.etree.ElementTree import Element, iterparse -from pydantic import BaseModel +from pydantic.v1 import BaseModel LOG = logging.getLogger(__name__) diff --git a/cg/apps/cgstats/parsers/dragen_demultiplexing_stats.py b/cg/apps/cgstats/parsers/dragen_demultiplexing_stats.py index beb86c4baf..82ae983355 100644 --- a/cg/apps/cgstats/parsers/dragen_demultiplexing_stats.py +++ b/cg/apps/cgstats/parsers/dragen_demultiplexing_stats.py @@ -4,7 +4,7 @@ from pathlib import Path from typing import Dict -from pydantic import BaseModel +from pydantic.v1 import BaseModel LOG = logging.getLogger(__name__) diff --git a/cg/apps/crunchy/crunchy.py b/cg/apps/crunchy/crunchy.py index 675b1343b2..788e83811e 100644 --- a/cg/apps/crunchy/crunchy.py +++ b/cg/apps/crunchy/crunchy.py @@ -10,9 +10,8 @@ from pathlib import Path from typing import Dict, Optional -from cgmodels.crunchy.metadata import CrunchyFile, CrunchyMetadata - from cg.apps.crunchy import files +from cg.apps.crunchy.models import CrunchyFile, CrunchyMetadata from cg.apps.slurm.slurm_api import SlurmAPI from cg.constants import FASTQ_DELTA from cg.models import CompressionData diff --git a/cg/apps/crunchy/files.py b/cg/apps/crunchy/files.py index 672004b3fe..b4812954ae 100644 --- a/cg/apps/crunchy/files.py +++ b/cg/apps/crunchy/files.py @@ -5,10 +5,10 @@ from pathlib import Path from typing import Dict, List, Optional +from cg.apps.crunchy.models import CrunchyFile, CrunchyMetadata from cg.constants.constants import FileFormat from cg.io.controller import ReadFile, ReadStream, WriteFile from cg.utils.date import get_date -from cgmodels.crunchy.metadata import CrunchyFile, CrunchyMetadata LOG = logging.getLogger(__name__) diff --git a/cg/apps/crunchy/models.py b/cg/apps/crunchy/models.py new file mode 100644 index 0000000000..f3800177b6 --- /dev/null +++ b/cg/apps/crunchy/models.py @@ -0,0 +1,17 @@ +from datetime import date +from typing import Optional + +from pydantic.v1 import BaseModel, conlist +from typing_extensions import Literal + + +class CrunchyFile(BaseModel): + path: str + file: Literal["first_read", "second_read", "spring"] + checksum: Optional[str] + algorithm: Literal["sha1", "md5", "sha256"] = None + updated: date = None + + +class CrunchyMetadata(BaseModel): + files: conlist(CrunchyFile, max_items=3, min_items=3) diff --git a/cg/apps/demultiplex/demultiplex_api.py b/cg/apps/demultiplex/demultiplex_api.py index 72f937b787..b0fa5de9fb 100644 --- a/cg/apps/demultiplex/demultiplex_api.py +++ b/cg/apps/demultiplex/demultiplex_api.py @@ -139,7 +139,7 @@ def is_demultiplexing_completed(self, flow_cell: FlowCellDirectoryData) -> bool: LOG.info(f"Check if demultiplexing is ready for {flow_cell.path}") logfile: Path = self.get_stderr_logfile(flow_cell) if not logfile.exists(): - LOG.warning("Could not find logfile!") + LOG.warning(f"Could not find logfile: {logfile}!") return False return self.demultiplexing_completed_path(flow_cell).exists() diff --git a/cg/apps/demultiplex/demux_report.py b/cg/apps/demultiplex/demux_report.py index 3b2d626649..b021f9a827 100644 --- a/cg/apps/demultiplex/demux_report.py +++ b/cg/apps/demultiplex/demux_report.py @@ -6,7 +6,7 @@ SampleConversionResults, UnknownBarcode, ) -from pydantic import BaseModel +from pydantic.v1 import BaseModel from typing_extensions import Literal LOG = logging.getLogger(__name__) diff --git a/cg/apps/demultiplex/sample_sheet/index.py b/cg/apps/demultiplex/sample_sheet/index.py index 894f3d05ca..3f4d1cb21e 100644 --- a/cg/apps/demultiplex/sample_sheet/index.py +++ b/cg/apps/demultiplex/sample_sheet/index.py @@ -10,7 +10,7 @@ from cg.resources import VALID_INDEXES_PATH from cg.utils.utils import get_hamming_distance from packaging import version -from pydantic import BaseModel +from pydantic.v1 import BaseModel LOG = logging.getLogger(__name__) DNA_COMPLEMENTS: Dict[str, str] = {"A": "T", "C": "G", "G": "C", "T": "A"} diff --git a/cg/apps/demultiplex/sample_sheet/models.py b/cg/apps/demultiplex/sample_sheet/models.py index 5a182c2108..418de0f4ee 100644 --- a/cg/apps/demultiplex/sample_sheet/models.py +++ b/cg/apps/demultiplex/sample_sheet/models.py @@ -1,5 +1,5 @@ import logging -from pydantic import BaseModel, Extra, Field +from pydantic.v1 import BaseModel, Extra, Field from typing import List from cg.constants.constants import GenomeVersion diff --git a/cg/apps/demultiplex/sample_sheet/read_sample_sheet.py b/cg/apps/demultiplex/sample_sheet/read_sample_sheet.py index caed0e194c..32b87400f2 100644 --- a/cg/apps/demultiplex/sample_sheet/read_sample_sheet.py +++ b/cg/apps/demultiplex/sample_sheet/read_sample_sheet.py @@ -1,7 +1,7 @@ import logging from pathlib import Path from typing import Dict, List, Type -from pydantic import parse_obj_as +from pydantic.v1 import parse_obj_as from cg.apps.demultiplex.sample_sheet.models import FlowCellSample, SampleSheet from cg.constants.constants import FileFormat diff --git a/cg/apps/hermes/models.py b/cg/apps/hermes/models.py index 7f6da1ddf2..d59842bd65 100644 --- a/cg/apps/hermes/models.py +++ b/cg/apps/hermes/models.py @@ -3,7 +3,7 @@ from pathlib import Path from typing import List, Optional -from pydantic import BaseModel, validator +from pydantic.v1 import BaseModel, validator LOG = logging.getLogger(__name__) diff --git a/cg/apps/housekeeper/models.py b/cg/apps/housekeeper/models.py index f15225b9ab..f165806b52 100644 --- a/cg/apps/housekeeper/models.py +++ b/cg/apps/housekeeper/models.py @@ -2,7 +2,7 @@ from datetime import datetime from typing import List, Optional -from pydantic import BaseModel +from pydantic.v1 import BaseModel class InputFile(BaseModel): diff --git a/cg/apps/orderform/excel_orderform_parser.py b/cg/apps/orderform/excel_orderform_parser.py index 0a645e230f..2e347de914 100644 --- a/cg/apps/orderform/excel_orderform_parser.py +++ b/cg/apps/orderform/excel_orderform_parser.py @@ -6,7 +6,7 @@ from openpyxl.cell.cell import Cell from openpyxl.workbook import Workbook from openpyxl.worksheet.worksheet import Worksheet -from pydantic import parse_obj_as +from pydantic.v1 import parse_obj_as from cg.apps.orderform.orderform_parser import OrderformParser from cg.constants import DataDelivery diff --git a/cg/apps/orderform/orderform_parser.py b/cg/apps/orderform/orderform_parser.py index 01aeae32f1..30f5e529f3 100644 --- a/cg/apps/orderform/orderform_parser.py +++ b/cg/apps/orderform/orderform_parser.py @@ -2,7 +2,7 @@ from pathlib import Path from typing import Dict, List, Optional, Set, Hashable, Iterable -from pydantic import constr, BaseModel +from pydantic.v1 import constr, BaseModel from cg.constants import DataDelivery from cg.exc import OrderFormError diff --git a/cg/apps/scout/scout_export.py b/cg/apps/scout/scout_export.py index 6c3931396d..bff85b2e3b 100644 --- a/cg/apps/scout/scout_export.py +++ b/cg/apps/scout/scout_export.py @@ -3,7 +3,7 @@ from datetime import datetime from typing import List, Optional, Dict -from pydantic import BaseModel, Field, validator +from pydantic.v1 import BaseModel, Field, validator from typing_extensions import Literal from cg.constants.gene_panel import GENOME_BUILD_37 diff --git a/cg/apps/sequencing_metrics_parser/models/bcl2fastq_metrics.py b/cg/apps/sequencing_metrics_parser/models/bcl2fastq_metrics.py index 35be3722ca..7abe811b59 100644 --- a/cg/apps/sequencing_metrics_parser/models/bcl2fastq_metrics.py +++ b/cg/apps/sequencing_metrics_parser/models/bcl2fastq_metrics.py @@ -1,5 +1,5 @@ from typing import List, Dict -from pydantic import BaseModel, Field, validator +from pydantic.v1 import BaseModel, Field, validator class IndexMetric(BaseModel): diff --git a/cg/apps/sequencing_metrics_parser/models/bcl_convert.py b/cg/apps/sequencing_metrics_parser/models/bcl_convert.py index 7a9cbcc516..497cb30412 100644 --- a/cg/apps/sequencing_metrics_parser/models/bcl_convert.py +++ b/cg/apps/sequencing_metrics_parser/models/bcl_convert.py @@ -1,5 +1,4 @@ -from pydantic import BaseModel, BaseConfig, Field -import xml.etree.ElementTree as ET +from pydantic.v1 import BaseModel, BaseConfig, Field from cg.constants.bcl_convert_metrics import ( BclConvertQualityMetricsColumnNames, BclConvertDemuxMetricsColumnNames, diff --git a/cg/apps/sequencing_metrics_parser/parsers/bcl_convert_to_sequencing_statistics.py b/cg/apps/sequencing_metrics_parser/parsers/bcl_convert_to_sequencing_statistics.py index aef0938fe4..31ac1f21b7 100644 --- a/cg/apps/sequencing_metrics_parser/parsers/bcl_convert_to_sequencing_statistics.py +++ b/cg/apps/sequencing_metrics_parser/parsers/bcl_convert_to_sequencing_statistics.py @@ -25,7 +25,7 @@ def create_sample_lane_sequencing_metrics_from_bcl_convert_metrics_for_flow_cell sample_internal_id=sample_internal_id, flow_cell_name=FlowCellDirectoryData( flow_cell_path=flow_cell_dir, bcl_converter=BclConverter.BCLCONVERT - ).flow_cell_name, + ).id, flow_cell_lane_number=lane, sample_total_reads_in_lane=metrics_parser.calculate_total_reads_for_sample_in_lane( sample_internal_id=sample_internal_id, lane=lane diff --git a/cg/apps/tb/models.py b/cg/apps/tb/models.py index cf06f3236f..deea3a0a24 100644 --- a/cg/apps/tb/models.py +++ b/cg/apps/tb/models.py @@ -3,7 +3,7 @@ from typing import Optional from dateutil.parser import parse as parse_datestr -from pydantic import BaseModel, validator +from pydantic.v1 import BaseModel, validator class TrailblazerAnalysis(BaseModel): diff --git a/cg/cli/demultiplex/finish.py b/cg/cli/demultiplex/finish.py index 5837eb5c16..a3755e9aad 100644 --- a/cg/cli/demultiplex/finish.py +++ b/cg/cli/demultiplex/finish.py @@ -32,6 +32,11 @@ def finish_all_cmd(context: CGConfig, bcl_converter: str, dry_run: bool) -> None demux_post_processing_api.set_dry_run(dry_run=dry_run) demux_post_processing_api.finish_all_flow_cells(bcl_converter=bcl_converter) + # Temporary finish flow cell logic will replace logic above when validated + demux_post_processing_api_temp: DemuxPostProcessingAPI = DemuxPostProcessingAPI(config=context) + demux_post_processing_api_temp.set_dry_run(dry_run=dry_run) + demux_post_processing_api_temp.finish_all_flow_cells_temp() + @finish_group.command(name="flow-cell") @click.argument("flow-cell-name") @@ -54,6 +59,10 @@ def finish_flow_cell( demux_post_processing_api.finish_flow_cell( flow_cell_name=flow_cell_name, force=force, bcl_converter=bcl_converter ) + # Temporary finish flow cell logic will replace logic above when validated + demux_post_processing_api_temp: DemuxPostProcessingAPI = DemuxPostProcessingAPI(config=context) + demux_post_processing_api_temp.set_dry_run(dry_run) + demux_post_processing_api_temp.finish_flow_cell_temp(flow_cell_directory_name=flow_cell_name) @finish_group.command(name="temporary") @@ -66,6 +75,16 @@ def finish_flow_cell_temporary(context: CGConfig, flow_cell_directory_name: str) ) +@finish_group.command(name="temporary-all") +@click.pass_obj +@DRY_RUN +def finish_flow_cell_temporary_all(context: CGConfig, dry_run: bool): + # Temporary finish flow cell logic will replace logic above when validated + demux_post_processing_api_temp: DemuxPostProcessingAPI = DemuxPostProcessingAPI(config=context) + demux_post_processing_api_temp.set_dry_run(dry_run=dry_run) + demux_post_processing_api_temp.finish_all_flow_cells_temp() + + @finish_group.command(name="all-hiseq-x") @DRY_RUN @click.pass_obj diff --git a/cg/cli/demultiplex/sample_sheet.py b/cg/cli/demultiplex/sample_sheet.py index 29a64fdd10..0ccca3dd06 100644 --- a/cg/cli/demultiplex/sample_sheet.py +++ b/cg/cli/demultiplex/sample_sheet.py @@ -14,7 +14,7 @@ from cg.io.controller import WriteFile, WriteStream from cg.models.cg_config import CGConfig from cg.models.demultiplex.flow_cell import FlowCellDirectoryData -from pydantic import ValidationError +from pydantic.v1 import ValidationError LOG = logging.getLogger(__name__) diff --git a/cg/cli/generate/report/base.py b/cg/cli/generate/report/base.py index 798b87310f..c46be93acf 100644 --- a/cg/cli/generate/report/base.py +++ b/cg/cli/generate/report/base.py @@ -1,23 +1,13 @@ -"""Commands to generate delivery reports""" - +"""Commands to generate delivery reports.""" import logging import sys from datetime import datetime from pathlib import Path -from typing import TextIO, Optional +from typing import Optional import click +from housekeeper.store.models import Version -from cg.store.models import Family -from cgmodels.cg.constants import Pipeline -from housekeeper.store import models as hk_models - -from cg.cli.generate.report.utils import ( - get_report_case, - get_report_api, - get_report_analysis_started, - get_report_api_pipeline, -) from cg.cli.generate.report.options import ( ARGUMENT_CASE_ID, OPTION_FORCE_REPORT, @@ -25,9 +15,16 @@ OPTION_STARTED_AT, OPTION_PIPELINE, ) -from cg.constants import EXIT_SUCCESS, EXIT_FAIL +from cg.cli.generate.report.utils import ( + get_report_case, + get_report_api, + get_report_analysis_started, + get_report_api_pipeline, +) +from cg.constants import EXIT_SUCCESS, EXIT_FAIL, Pipeline from cg.exc import CgError from cg.meta.report.report_api import ReportAPI +from cg.store.models import Family LOG = logging.getLogger(__name__) @@ -44,11 +41,9 @@ def generate_delivery_report( force_report: bool, dry_run: bool, analysis_started_at: str = None, -): +) -> None: """Creates a delivery report for the provided case.""" - click.echo(click.style("--------------- DELIVERY REPORT ---------------")) - case: Family = get_report_case(context, case_id) report_api: ReportAPI = get_report_api(context, case) analysis_date: datetime = get_report_analysis_started(case, report_api, analysis_started_at) @@ -61,22 +56,32 @@ def generate_delivery_report( click.echo(delivery_report_html) return - delivery_report_file: TextIO = report_api.create_delivery_report_file( - case_id, - file_path=Path(report_api.analysis_api.root, case_id), + version: Version = report_api.housekeeper_api.version(bundle=case_id, date=analysis_date) + delivery_report: Optional[str] = report_api.get_delivery_report_from_hk( + case_id=case_id, version=version + ) + if delivery_report: + click.echo( + click.style(f"Delivery report already in housekeeper: {delivery_report}", fg="yellow") + ) + return + + created_delivery_report: Path = report_api.create_delivery_report_file( + case_id=case_id, + directory=Path(report_api.analysis_api.root, case_id), analysis_date=analysis_date, force_report=force_report, ) - - hk_report_file: Optional[hk_models.File] = report_api.add_delivery_report_to_hk( - delivery_report_file, case_id, analysis_date + report_api.add_delivery_report_to_hk( + case_id=case_id, delivery_report_file=created_delivery_report, version=version ) - - if hk_report_file: - click.echo(click.style("Uploaded delivery report to housekeeper", fg="green")) - report_api.update_delivery_report_date(case, analysis_date) - else: - click.echo(click.style("Delivery report already uploaded to housekeeper", fg="yellow")) + click.echo( + click.style( + f"Uploaded delivery report to housekeeper: {created_delivery_report.as_posix()}", + fg="green", + ) + ) + report_api.update_delivery_report_date(case=case, analysis_date=analysis_date) @click.command("available-delivery-reports") @@ -86,7 +91,7 @@ def generate_delivery_report( @click.pass_context def generate_available_delivery_reports( context: click.Context, pipeline: Pipeline, force_report: bool, dry_run: bool -): +) -> None: """Generates delivery reports for all cases that need one and stores them in housekeeper.""" click.echo(click.style("--------------- AVAILABLE DELIVERY REPORTS ---------------")) diff --git a/cg/cli/generate/report/utils.py b/cg/cli/generate/report/utils.py index a4f9194729..3233c8f5cf 100644 --- a/cg/cli/generate/report/utils.py +++ b/cg/cli/generate/report/utils.py @@ -4,14 +4,13 @@ from typing import Optional, List, Dict import click -from cg.meta.report.rnafusion import RnafusionReportAPI -from cgmodels.cg.constants import Pipeline -from cg.constants import REPORT_SUPPORTED_PIPELINES, REPORT_SUPPORTED_DATA_DELIVERY -from cg.meta.report.report_api import ReportAPI +from cg.constants import REPORT_SUPPORTED_PIPELINES, REPORT_SUPPORTED_DATA_DELIVERY, Pipeline from cg.meta.report.balsamic import BalsamicReportAPI from cg.meta.report.balsamic_umi import BalsamicUmiReportAPI from cg.meta.report.mip_dna import MipDNAReportAPI +from cg.meta.report.report_api import ReportAPI +from cg.meta.report.rnafusion import RnafusionReportAPI from cg.meta.workflow.balsamic import BalsamicAnalysisAPI from cg.meta.workflow.balsamic_umi import BalsamicUmiAnalysisAPI from cg.meta.workflow.mip_dna import MipDNAAnalysisAPI @@ -50,7 +49,7 @@ def get_report_case(context: click.Context, case_id: str) -> Family: else: LOG.error("Provide one of the following case IDs:") for case in cases_without_delivery_report: - click.echo(f"{case.internal_id} ({ case.data_analysis})") + click.echo(f"{case.internal_id} ({case.data_analysis})") raise click.Abort if case.data_analysis not in REPORT_SUPPORTED_PIPELINES: LOG.error( diff --git a/cg/cli/upload/delivery_report.py b/cg/cli/upload/delivery_report.py index 900de3e06b..ec1dc3a9e8 100644 --- a/cg/cli/upload/delivery_report.py +++ b/cg/cli/upload/delivery_report.py @@ -1,13 +1,14 @@ -"""Delivery report upload to scout commands""" - +"""Delivery report upload to scout commands.""" import logging +from typing import Optional import click -from cg.store.models import Family +from housekeeper.store.models import Version -from cg.cli.generate.report.utils import get_report_case, get_report_api from cg.cli.generate.report.options import ARGUMENT_CASE_ID +from cg.cli.generate.report.utils import get_report_case, get_report_api from cg.meta.report.report_api import ReportAPI +from cg.store.models import Family LOG = logging.getLogger(__name__) @@ -15,19 +16,26 @@ @click.command("delivery-report-to-scout") @ARGUMENT_CASE_ID @click.option( - "-d", "--dry-run", is_flag=True, default=False, help="Run command without uploading to scout" + "-r", "--re-upload", is_flag=True, default=False, help="Re-upload existing delivery report" +) +@click.option( + "-d", "--dry-run", is_flag=True, default=False, help="Run command without uploading to Scout" ) @click.pass_context -def upload_delivery_report_to_scout(context: click.Context, case_id: str, dry_run: bool): - """Fetches a delivery report from housekeeper and uploads it to scout.""" - +def upload_delivery_report_to_scout( + context: click.Context, case_id: str, re_upload: bool, dry_run: bool +) -> None: + """Fetches a delivery report from Housekeeper and uploads it to Scout.""" click.echo(click.style("--------------- DELIVERY REPORT UPLOAD ---------------")) - case: Family = get_report_case(context, case_id) report_api: ReportAPI = get_report_api(context, case) - report_path: str = report_api.get_delivery_report_from_hk(case_id) - - if not dry_run: + version: Version = report_api.housekeeper_api.last_version(case_id) + delivery_report: Optional[str] = report_api.get_delivery_report_from_hk( + case_id=case_id, version=version + ) + if delivery_report and not dry_run: report_api.scout_api.upload_delivery_report( - report_path=report_path, case_id=case.internal_id, update=True + report_path=delivery_report, case_id=case.internal_id, update=re_upload ) + return + LOG.error("Delivery report not uploaded to Scout") diff --git a/cg/cli/upload/observations/observations.py b/cg/cli/upload/observations/observations.py index 2a993d98b3..353dadc86f 100644 --- a/cg/cli/upload/observations/observations.py +++ b/cg/cli/upload/observations/observations.py @@ -8,7 +8,7 @@ import click from sqlalchemy.orm import Query from cgmodels.cg.constants import Pipeline -from pydantic import ValidationError +from pydantic.v1 import ValidationError from cg.cli.upload.observations.utils import get_observations_case_to_upload, get_observations_api from cg.exc import LoqusdbError, CaseNotFoundError diff --git a/cg/cli/workflow/balsamic/base.py b/cg/cli/workflow/balsamic/base.py index b70de386ee..bcea1e2985 100644 --- a/cg/cli/workflow/balsamic/base.py +++ b/cg/cli/workflow/balsamic/base.py @@ -22,7 +22,7 @@ from cg.meta.workflow.balsamic import BalsamicAnalysisAPI from cg.models.cg_config import CGConfig from cg.store import Store -from pydantic import ValidationError +from pydantic.v1 import ValidationError LOG = logging.getLogger(__name__) diff --git a/cg/cli/workflow/rnafusion/base.py b/cg/cli/workflow/rnafusion/base.py index cb7382b21b..af1c5a4891 100644 --- a/cg/cli/workflow/rnafusion/base.py +++ b/cg/cli/workflow/rnafusion/base.py @@ -5,7 +5,7 @@ from typing import Optional import click -from pydantic import ValidationError +from pydantic.v1 import ValidationError from cg.apps.housekeeper.hk import HousekeeperAPI from cg.cli.workflow.commands import ARGUMENT_CASE_ID, resolve_compression diff --git a/cg/constants/housekeeper_tags.py b/cg/constants/housekeeper_tags.py index 86b5005d0a..5d1457c5f4 100644 --- a/cg/constants/housekeeper_tags.py +++ b/cg/constants/housekeeper_tags.py @@ -40,6 +40,7 @@ class SequencingFileTag(StrEnum): SAMPLE_SHEET: str = "samplesheet" SPRING: str = "spring" SPRING_METADATA: str = "spring-metadata" + DEMUX_LOG: str = "log" HK_MULTIQC_HTML_TAG = ["multiqc-html"] diff --git a/cg/meta/archive/ddn_dataflow.py b/cg/meta/archive/ddn_dataflow.py index c2a9de772d..a0607267e0 100644 --- a/cg/meta/archive/ddn_dataflow.py +++ b/cg/meta/archive/ddn_dataflow.py @@ -4,7 +4,7 @@ from typing import Dict, List, Optional from urllib.parse import urljoin -from pydantic import BaseModel +from pydantic.v1 import BaseModel from requests.models import Response from datetime import datetime diff --git a/cg/meta/demultiplex/demux_post_processing.py b/cg/meta/demultiplex/demux_post_processing.py index 2437ed1546..c2f4314a29 100644 --- a/cg/meta/demultiplex/demux_post_processing.py +++ b/cg/meta/demultiplex/demux_post_processing.py @@ -17,9 +17,9 @@ create_sample_lane_sequencing_metrics_for_flow_cell, ) from cg.constants.cgstats import STATS_HEADER -from cg.constants.demultiplexing import DemultiplexingDirsAndFiles from cg.constants.housekeeper_tags import SequencingFileTag from cg.constants.sequencing import Sequencers +from cg.constants.demultiplexing import DemultiplexingDirsAndFiles from cg.exc import FlowCellError from cg.meta.demultiplex import files from cg.meta.demultiplex.utils import ( @@ -30,6 +30,7 @@ get_sample_fastqs_from_flow_cell, get_sample_sheet_path, parse_flow_cell_directory_data, + copy_sample_sheet, ) from cg.apps.demultiplex.sample_sheet.read_sample_sheet import ( get_sample_internal_ids_from_sample_sheet, @@ -42,8 +43,8 @@ from cg.store import Store from cg.store.models import Flowcell, SampleLaneSequencingMetrics from cg.utils import Process -from cg.utils.files import get_file_in_directory -from cg.constants.demultiplexing import DemultiplexingDirsAndFiles +from cg.utils.files import get_files_matching_pattern + LOG = logging.getLogger(__name__) @@ -68,22 +69,6 @@ def set_dry_run(self, dry_run: bool) -> None: if dry_run: self.demux_api.set_dry_run(dry_run=dry_run) - def copy_sample_sheet(self) -> None: - """Copy the sample sheet from run dir to demux dir""" - sample_sheet_path = get_file_in_directory( - self.demux_api.run_dir, DemultiplexingDirsAndFiles.SAMPLE_SHEET_FILE_NAME - ) - target_sample_sheet_path = Path( - self.demux_api.out_dir, DemultiplexingDirsAndFiles.SAMPLE_SHEET_FILE_NAME - ) - LOG.info( - f"Copy sample sheet {sample_sheet_path} from flow cell to demuxed result dir {target_sample_sheet_path}" - ) - shutil.copy( - sample_sheet_path.as_posix(), - target_sample_sheet_path.as_posix(), - ) - def transfer_flow_cell( self, flow_cell_dir: Path, flow_cell_id: str, store: bool = True ) -> None: @@ -103,6 +88,7 @@ def finish_flow_cell_temp(self, flow_cell_directory_name: str) -> None: """Store data for the demultiplexed flow cell and mark it as ready for delivery. This function: + - Copies the sample sheet to the demultiplexed flow cell directory - Parses and validates the flow cell directory data - Stores the flow cell in the status database - Stores sequencing metrics in the status database @@ -116,24 +102,43 @@ def finish_flow_cell_temp(self, flow_cell_directory_name: str) -> None: Raises: FlowCellError: If the flow cell directory or the data it contains is not valid. """ - + if self.dry_run: + LOG.info(f"Dry run will not finish flow cell {flow_cell_directory_name}") + return LOG.info(f"Finish flow cell {flow_cell_directory_name}") - self.copy_sample_sheet() - flow_cell_directory_path: Path = Path(self.demux_api.out_dir, flow_cell_directory_name) - bcl_converter: str = get_bcl_converter_name(flow_cell_directory_path) + + flow_cell_directory: Path = Path(self.demux_api.out_dir, flow_cell_directory_name) + flow_cell_run_directory: Path = Path(self.demux_api.run_dir, flow_cell_directory_name) + + bcl_converter: str = get_bcl_converter_name(flow_cell_directory) parsed_flow_cell: FlowCellDirectoryData = parse_flow_cell_directory_data( - flow_cell_directory=flow_cell_directory_path, + flow_cell_directory=flow_cell_directory, bcl_converter=bcl_converter, ) + copy_sample_sheet( + sample_sheet_source_directory=flow_cell_run_directory, + sample_sheet_destination_directory=flow_cell_directory, + ) + try: self.store_flow_cell_data(parsed_flow_cell) except Exception as e: LOG.error(f"Failed to store flow cell data: {str(e)}") raise - create_delivery_file_in_flow_cell_directory(flow_cell_directory_path) + create_delivery_file_in_flow_cell_directory(flow_cell_directory) + + def finish_all_flow_cells_temp(self) -> None: + """Finish all flow cells that need it.""" + flow_cell_dirs = self.demux_api.get_all_demultiplexed_flow_cell_dirs() + for flow_cell_dir in flow_cell_dirs: + try: + self.finish_flow_cell_temp(flow_cell_dir.name) + except FlowCellError as e: + LOG.error(f"Failed to finish flow cell {flow_cell_dir.name}: {str(e)}") + continue def store_flow_cell_data(self, parsed_flow_cell: FlowCellDirectoryData) -> None: """Store data from the flow cell directory in status db and housekeeper.""" @@ -223,6 +228,24 @@ def store_flow_cell_data_in_housekeeper(self, flow_cell: FlowCellDirectoryData) flow_cell_directory=flow_cell.path, flow_cell_name=flow_cell.id ) self.add_sample_fastq_files_to_housekeeper(flow_cell) + self.add_demux_logs_to_housekeeper(flow_cell) + + def add_demux_logs_to_housekeeper(self, flow_cell: FlowCellDirectoryData) -> None: + """Add demux logs to Housekeeper.""" + log_pattern: str = r"*_demultiplex.std*" + demux_log_file_paths: List[Path] = get_files_matching_pattern( + directory=Path(self.demux_api.run_dir, flow_cell.full_name), pattern=log_pattern + ) + + tag_names: List[str] = [SequencingFileTag.DEMUX_LOG, flow_cell.id] + for file_path in demux_log_file_paths: + try: + self.add_file_to_bundle_if_non_existent( + file_path=file_path, bundle_name=flow_cell.id, tag_names=tag_names + ) + LOG.info(f"Added demux log file {file_path} to Housekeeper.") + except FileNotFoundError as e: + LOG.error(f"Cannot find demux log file {file_path}. Error: {e}.") def add_sample_fastq_files_to_housekeeper(self, flow_cell: FlowCellDirectoryData) -> None: """Add sample fastq files from flow cell to Housekeeper.""" @@ -639,6 +662,7 @@ def finish_flow_cell( bcl_converter=bcl_converter, ) except FlowCellError: + LOG.warning(f"Could not find flow cell {flow_cell_name}") return if not self.demux_api.is_demultiplexing_completed(flow_cell=flow_cell): LOG.warning("Demultiplex is not ready for %s", flow_cell_name) @@ -657,6 +681,7 @@ def finish_flow_cell( LOG.warning("Flow cell is already finished!") if not force: return + LOG.info("Post processing flow cell anyway") self.post_process_flow_cell(demux_results=demux_results) diff --git a/cg/meta/demultiplex/utils.py b/cg/meta/demultiplex/utils.py index 40ac02207b..7c79dd66b0 100644 --- a/cg/meta/demultiplex/utils.py +++ b/cg/meta/demultiplex/utils.py @@ -1,10 +1,10 @@ import logging -import os import re +import shutil from pathlib import Path from typing import List, Optional -from cg.apps.demultiplex.sample_sheet.models import FlowCellSample + from cg.constants.constants import FileExtensions from cg.constants.demultiplexing import BclConverter, DemultiplexingDirsAndFiles from cg.constants.sequencing import FLOWCELL_Q30_THRESHOLD, Sequencers @@ -18,6 +18,7 @@ from cg.utils.files import get_file_in_directory, get_files_matching_pattern + LOG = logging.getLogger(__name__) @@ -89,11 +90,34 @@ def get_sample_sheet_path( def parse_flow_cell_directory_data( flow_cell_directory: Path, bcl_converter: str ) -> FlowCellDirectoryData: - """Parse flow cell data from the flow cell directory.""" + """Return flow cell data from the flow cell directory.""" + is_flow_cell_directory_valid(flow_cell_directory) + return FlowCellDirectoryData(flow_cell_path=flow_cell_directory, bcl_converter=bcl_converter) - try: - is_flow_cell_directory_valid(flow_cell_directory) - except FlowCellError as e: - raise FlowCellError(f"Flow cell directory was not valid: {flow_cell_directory}, {e}") - return FlowCellDirectoryData(flow_cell_path=flow_cell_directory, bcl_converter=bcl_converter) +def copy_sample_sheet( + sample_sheet_source_directory: Path, sample_sheet_destination_directory: Path +) -> None: + """Copy the sample sheet from the flow-cell-run dir to demultiplex-runs dir for a flow cell.""" + sample_sheet_source: Path = Path( + sample_sheet_source_directory, DemultiplexingDirsAndFiles.SAMPLE_SHEET_FILE_NAME + ) + sample_sheet_destination: Path = Path( + sample_sheet_destination_directory, DemultiplexingDirsAndFiles.SAMPLE_SHEET_FILE_NAME + ) + + if not sample_sheet_destination.exists(): + LOG.debug( + f"Copy sample sheet {sample_sheet_source_directory} from flow cell to demuxed result dir {sample_sheet_destination_directory}" + ) + try: + shutil.copy( + sample_sheet_source.as_posix(), + sample_sheet_destination.as_posix(), + ) + except FileNotFoundError as e: + raise FileNotFoundError( + f"Could not copy sample sheet from {sample_sheet_source_directory} to {sample_sheet_destination_directory}: {e}" + ) + return + LOG.warning(f"Sample sheet already exists: {sample_sheet_destination}, skipping copy.") diff --git a/cg/meta/invoice.py b/cg/meta/invoice.py index f7bcb3e4cc..fec202390f 100644 --- a/cg/meta/invoice.py +++ b/cg/meta/invoice.py @@ -11,7 +11,7 @@ CustomerNames, ) from cg.models.invoice.invoice import InvoiceContact, InvoiceApplication, InvoiceReport, InvoiceInfo -from pydantic import ValidationError +from pydantic.v1 import ValidationError class InvoiceAPI: diff --git a/cg/meta/report/balsamic.py b/cg/meta/report/balsamic.py index e7b8794a80..17d7342414 100644 --- a/cg/meta/report/balsamic.py +++ b/cg/meta/report/balsamic.py @@ -1,7 +1,6 @@ import logging from typing import List, Union, Optional, Dict -from cgmodels.cg.constants import Pipeline from housekeeper.store.models import Version, File from cg.constants import ( @@ -18,11 +17,12 @@ REQUIRED_SAMPLE_METADATA_BALSAMIC_TN_WGS_FIELDS, BALSAMIC_ANALYSIS_TYPE, REQUIRED_SAMPLE_METADATA_BALSAMIC_TO_WGS_FIELDS, + Pipeline, ) from cg.constants.scout_upload import BALSAMIC_CASE_TAGS from cg.meta.report.field_validators import get_million_read_pairs -from cg.meta.workflow.balsamic import BalsamicAnalysisAPI from cg.meta.report.report_api import ReportAPI +from cg.meta.workflow.balsamic import BalsamicAnalysisAPI from cg.models.balsamic.analysis import BalsamicAnalysis from cg.models.balsamic.config import BalsamicVarCaller from cg.models.balsamic.metrics import ( diff --git a/cg/meta/report/balsamic_umi.py b/cg/meta/report/balsamic_umi.py index 333551e4c0..13186c7509 100644 --- a/cg/meta/report/balsamic_umi.py +++ b/cg/meta/report/balsamic_umi.py @@ -2,7 +2,6 @@ from cg.constants.scout_upload import BALSAMIC_UMI_CASE_TAGS from cg.meta.report.balsamic import BalsamicReportAPI - from cg.meta.workflow.balsamic_umi import BalsamicUmiAnalysisAPI from cg.models.cg_config import CGConfig diff --git a/cg/meta/report/mip_dna.py b/cg/meta/report/mip_dna.py index 20fe8a800b..35e648f397 100644 --- a/cg/meta/report/mip_dna.py +++ b/cg/meta/report/mip_dna.py @@ -1,7 +1,6 @@ import logging from typing import List, Optional, Iterable -from cgmodels.cg.constants import Pipeline from housekeeper.store.models import Version, File from cg.constants import ( @@ -15,18 +14,19 @@ REQUIRED_SAMPLE_TIMESTAMP_FIELDS, REQUIRED_SAMPLE_METADATA_MIP_DNA_FIELDS, REQUIRED_SAMPLE_METADATA_MIP_DNA_WGS_FIELDS, + Pipeline, ) from cg.constants.scout_upload import MIP_CASE_TAGS from cg.meta.report.field_validators import get_million_read_pairs -from cg.models.cg_config import CGConfig from cg.meta.report.report_api import ReportAPI from cg.meta.workflow.mip_dna import MipDNAAnalysisAPI +from cg.models.cg_config import CGConfig from cg.models.mip.mip_analysis import MipAnalysis +from cg.models.mip.mip_metrics_deliverables import get_sample_id_metric from cg.models.report.metadata import MipDNASampleMetadataModel from cg.models.report.report import CaseModel from cg.models.report.sample import SampleModel -from cg.models.mip.mip_metrics_deliverables import get_sample_id_metric -from cg.store.models import Family, Sample, Application +from cg.store.models import Family, Sample LOG = logging.getLogger(__name__) diff --git a/cg/meta/report/report_api.py b/cg/meta/report/report_api.py index 17ceda41f8..add766de52 100644 --- a/cg/meta/report/report_api.py +++ b/cg/meta/report/report_api.py @@ -1,25 +1,24 @@ -"""Module to create delivery reports""" - -from datetime import datetime +"""Module to create delivery reports.""" import logging +from datetime import datetime from pathlib import Path -from typing import TextIO, Optional, List +from typing import Optional, List import requests -from sqlalchemy.orm import Query - -from cgmodels.cg.constants import Pipeline from housekeeper.store.models import File, Version +from jinja2 import Environment, PackageLoader, select_autoescape, Template +from sqlalchemy.orm import Query +from cg.constants import Pipeline from cg.constants.constants import FileFormat, MAX_ITEMS_TO_RETRIEVE +from cg.constants.housekeeper_tags import HK_DELIVERY_REPORT_TAG from cg.exc import DeliveryReportError from cg.io.controller import WriteStream +from cg.meta.meta import MetaAPI from cg.meta.report.field_validators import get_missing_report_data, get_empty_report_data from cg.meta.workflow.analysis import AnalysisAPI -from cg.constants.housekeeper_tags import HK_DELIVERY_REPORT_TAG from cg.models.analysis import AnalysisModel from cg.models.cg_config import CGConfig -from cg.meta.meta import MetaAPI from cg.models.report.metadata import SampleMetadataModel from cg.models.report.report import ( ReportModel, @@ -30,7 +29,6 @@ ) from cg.models.report.sample import SampleModel, ApplicationModel, TimestampModel, MethodsModel from cg.store.models import Analysis, Application, Family, Sample, FamilySample -from jinja2 import Environment, PackageLoader, select_autoescape, Template LOG = logging.getLogger(__name__) @@ -56,47 +54,38 @@ def create_delivery_report( return rendered_report def create_delivery_report_file( - self, case_id: str, file_path: Path, analysis_date: datetime, force_report: bool - ) -> TextIO: - """Generates a temporary file containing a delivery report.""" - file_path.mkdir(parents=True, exist_ok=True) + self, case_id: str, directory: Path, analysis_date: datetime, force_report: bool + ) -> Path: + """Generates a file containing the delivery report content.""" + directory.mkdir(parents=True, exist_ok=True) delivery_report: str = self.create_delivery_report( case_id=case_id, analysis_date=analysis_date, force_report=force_report ) - report_file_path: Path = Path(file_path / "delivery-report.html") - with open(report_file_path, "w") as delivery_report_file: - delivery_report_file.write(delivery_report) - return delivery_report_file + report_file_path: Path = Path(directory, "delivery-report.html") + with open(report_file_path, "w") as delivery_report_stream: + delivery_report_stream.write(delivery_report) + return report_file_path def add_delivery_report_to_hk( - self, delivery_report_file: Path, case_id: str, analysis_date: datetime + self, case_id: str, delivery_report_file: Path, version: Version ) -> Optional[File]: - """ - Adds a delivery report file, if it has not already been generated, to an analysis bundle for a specific case - in HK and returns a pointer to it. - """ - version = self.housekeeper_api.version(case_id, analysis_date) - try: - self.get_delivery_report_from_hk(case_id=case_id) - except FileNotFoundError: - LOG.info(f"Adding a new delivery report to housekeeper for {case_id}") - file: File = self.housekeeper_api.add_file( - delivery_report_file.name, version, [case_id, HK_DELIVERY_REPORT_TAG] - ) - self.housekeeper_api.include_file(file, version) - self.housekeeper_api.add_commit(file) - return file - return None - - def get_delivery_report_from_hk(self, case_id: str) -> str: - """Extracts the delivery reports of a specific case stored in HK.""" - version: Version = self.housekeeper_api.last_version(case_id) + """Add a delivery report file to a case bundle and return its file object.""" + LOG.info(f"Adding a new delivery report to housekeeper for {case_id}") + file: File = self.housekeeper_api.add_file( + path=delivery_report_file, version_obj=version, tags=[case_id, HK_DELIVERY_REPORT_TAG] + ) + self.housekeeper_api.include_file(file, version) + self.housekeeper_api.add_commit(file) + return file + + def get_delivery_report_from_hk(self, case_id: str, version: Version) -> Optional[str]: + """Return path of a delivery report stored in HK.""" delivery_report: File = self.housekeeper_api.get_latest_file( bundle=case_id, tags=[HK_DELIVERY_REPORT_TAG], version=version.id ) if not delivery_report: - LOG.warning(f"No existing delivery report found in housekeeper for {case_id}") - raise FileNotFoundError + LOG.warning(f"No delivery report found in housekeeper for {case_id}") + return None return delivery_report.full_path def get_scout_uploaded_file_from_hk(self, case_id: str, scout_tag: str) -> Optional[str]: diff --git a/cg/meta/report/rnafusion.py b/cg/meta/report/rnafusion.py index 837cf99cd6..d313ba4354 100644 --- a/cg/meta/report/rnafusion.py +++ b/cg/meta/report/rnafusion.py @@ -1,8 +1,6 @@ """RNAfusion delivery report API.""" from typing import List, Optional -from cgmodels.cg.constants import Pipeline - from cg.constants import ( REQUIRED_APPLICATION_FIELDS, REQUIRED_CASE_FIELDS, @@ -13,6 +11,7 @@ REQUIRED_SAMPLE_METHODS_FIELDS, REQUIRED_SAMPLE_RNAFUSION_FIELDS, REQUIRED_SAMPLE_TIMESTAMP_FIELDS, + Pipeline, ) from cg.constants.constants import GenomeVersion from cg.meta.report.field_validators import get_million_read_pairs diff --git a/cg/meta/upload/gisaid/models.py b/cg/meta/upload/gisaid/models.py index 0f7c7b0724..e226dbb91d 100644 --- a/cg/meta/upload/gisaid/models.py +++ b/cg/meta/upload/gisaid/models.py @@ -1,6 +1,6 @@ from pathlib import Path from typing import Optional -from pydantic import BaseModel, validator +from pydantic.v1 import BaseModel, validator from datetime import datetime from cg.meta.upload.gisaid.constants import AUTHORS diff --git a/cg/meta/upload/nipt/models.py b/cg/meta/upload/nipt/models.py index 9e89c64225..2d689b8909 100644 --- a/cg/meta/upload/nipt/models.py +++ b/cg/meta/upload/nipt/models.py @@ -1,5 +1,5 @@ from typing import Optional -from pydantic import BaseModel +from pydantic.v1 import BaseModel class StatinaUploadFiles(BaseModel): diff --git a/cg/meta/upload/scout/hk_tags.py b/cg/meta/upload/scout/hk_tags.py index 0c336294b2..4236b51e94 100644 --- a/cg/meta/upload/scout/hk_tags.py +++ b/cg/meta/upload/scout/hk_tags.py @@ -2,7 +2,7 @@ from typing import Optional, Set -from pydantic import BaseModel, Field +from pydantic.v1 import BaseModel, Field class CaseTags(BaseModel): diff --git a/cg/meta/upload/scout/uploadscoutapi.py b/cg/meta/upload/scout/uploadscoutapi.py index 656c7cc42f..858fe48aa3 100644 --- a/cg/meta/upload/scout/uploadscoutapi.py +++ b/cg/meta/upload/scout/uploadscoutapi.py @@ -23,7 +23,7 @@ from cg.store import Store from cg.store.models import Analysis, Customer, Family, Sample from housekeeper.store.models import File, Version -from pydantic.dataclasses import dataclass +from pydantic.v1.dataclasses import dataclass LOG = logging.getLogger(__name__) diff --git a/cg/meta/workflow/balsamic.py b/cg/meta/workflow/balsamic.py index 86fa06011e..9cfb2a9630 100644 --- a/cg/meta/workflow/balsamic.py +++ b/cg/meta/workflow/balsamic.py @@ -5,7 +5,7 @@ from typing import Dict, List, Optional, Union from housekeeper.store.models import Version, File -from pydantic import ValidationError +from pydantic.v1 import ValidationError from cg.constants import Pipeline from cg.constants.constants import FileFormat, SampleType, AnalysisType diff --git a/cg/meta/workflow/mip.py b/cg/meta/workflow/mip.py index c1287d8ae0..8354abb4c6 100644 --- a/cg/meta/workflow/mip.py +++ b/cg/meta/workflow/mip.py @@ -3,7 +3,7 @@ from pathlib import Path from typing import Any, List, Optional, Dict, Union -from pydantic import ValidationError +from pydantic.v1 import ValidationError from cg.apps.mip.confighandler import ConfigHandler from cg.constants import COLLABORATORS, COMBOS, GenePanelMasterList, Pipeline, FileExtensions diff --git a/cg/meta/workflow/rnafusion.py b/cg/meta/workflow/rnafusion.py index 93ba782d84..9cd25ead92 100644 --- a/cg/meta/workflow/rnafusion.py +++ b/cg/meta/workflow/rnafusion.py @@ -4,7 +4,7 @@ from pathlib import Path from typing import Dict, List, Optional -from pydantic import ValidationError +from pydantic.v1 import ValidationError from cg import resources from cg.constants import Pipeline diff --git a/cg/meta/workflow/taxprofiler.py b/cg/meta/workflow/taxprofiler.py index c053e767d1..d91297df30 100644 --- a/cg/meta/workflow/taxprofiler.py +++ b/cg/meta/workflow/taxprofiler.py @@ -1,10 +1,9 @@ """Module for Taxprofiler Analysis API.""" import logging -from typing import Dict, List, Optional, Tuple +from typing import Dict, List, Optional -from pydantic import ValidationError -from cg.constants import Pipeline +from pydantic.v1 import ValidationError from cg.meta.workflow.analysis import AnalysisAPI from cg.models.cg_config import CGConfig from cg.constants import Pipeline diff --git a/cg/models/analysis.py b/cg/models/analysis.py index 86dae49a3a..930373d418 100644 --- a/cg/models/analysis.py +++ b/cg/models/analysis.py @@ -1,4 +1,4 @@ -from pydantic import BaseModel +from pydantic.v1 import BaseModel class AnalysisModel(BaseModel): diff --git a/cg/models/balsamic/config.py b/cg/models/balsamic/config.py index a4d10d75ee..35940f7490 100644 --- a/cg/models/balsamic/config.py +++ b/cg/models/balsamic/config.py @@ -2,7 +2,7 @@ from pathlib import Path from typing import Dict, List, Union, Optional -from pydantic import BaseModel, validator +from pydantic.v1 import BaseModel, validator class BalsamicConfigAnalysis(BaseModel): diff --git a/cg/models/balsamic/metrics.py b/cg/models/balsamic/metrics.py index 10aff85df1..bad0b42821 100644 --- a/cg/models/balsamic/metrics.py +++ b/cg/models/balsamic/metrics.py @@ -1,6 +1,6 @@ from typing import Optional -from pydantic import BaseModel, validator +from pydantic.v1 import BaseModel, validator from cg.models.deliverables.metric_deliverables import MetricCondition, MetricsBase diff --git a/cg/models/cg_config.py b/cg/models/cg_config.py index 864d826ab8..6718192807 100644 --- a/cg/models/cg_config.py +++ b/cg/models/cg_config.py @@ -1,7 +1,7 @@ import logging from typing import Optional -from pydantic import BaseModel, EmailStr, Field +from pydantic.v1 import BaseModel, EmailStr, Field from typing_extensions import Literal from cg.apps.cgstats.stats import StatsAPI diff --git a/cg/models/cgstats/flowcell.py b/cg/models/cgstats/flowcell.py index e4a40f9d2f..05b1812f69 100644 --- a/cg/models/cgstats/flowcell.py +++ b/cg/models/cgstats/flowcell.py @@ -1,7 +1,7 @@ from datetime import datetime from typing import List, Optional -from pydantic import BaseModel +from pydantic.v1 import BaseModel class StatsSample(BaseModel): diff --git a/cg/models/cgstats/stats_sample.py b/cg/models/cgstats/stats_sample.py index 427015b777..da2e3326ea 100644 --- a/cg/models/cgstats/stats_sample.py +++ b/cg/models/cgstats/stats_sample.py @@ -1,6 +1,6 @@ from typing import List, Set -from pydantic import BaseModel, Field, validator +from pydantic.v1 import BaseModel, Field, validator class Unaligned(BaseModel): diff --git a/cg/models/deliverables/metric_deliverables.py b/cg/models/deliverables/metric_deliverables.py index 70aac9b160..2fb15b7503 100644 --- a/cg/models/deliverables/metric_deliverables.py +++ b/cg/models/deliverables/metric_deliverables.py @@ -1,7 +1,7 @@ import operator from typing import Any, Callable, Dict, List, Optional -from pydantic import BaseModel, Field, validator +from pydantic.v1 import BaseModel, Field, validator from cg.exc import CgError, MetricsQCError diff --git a/cg/models/demultiplex/demux_results.py b/cg/models/demultiplex/demux_results.py index c842a9b13f..5d9cbe320c 100644 --- a/cg/models/demultiplex/demux_results.py +++ b/cg/models/demultiplex/demux_results.py @@ -4,7 +4,7 @@ from pathlib import Path from typing import Iterable, Optional, Union -from pydantic import BaseModel +from pydantic.v1 import BaseModel from typing_extensions import Literal from cg.apps.cgstats.parsers.adapter_metrics import AdapterMetrics diff --git a/cg/models/demultiplex/flow_cell.py b/cg/models/demultiplex/flow_cell.py index 3a73bac0d8..d48a7c6b84 100644 --- a/cg/models/demultiplex/flow_cell.py +++ b/cg/models/demultiplex/flow_cell.py @@ -4,7 +4,7 @@ from pathlib import Path from typing import List, Optional, Type, Union -from pydantic import ValidationError +from pydantic.v1 import ValidationError from typing_extensions import Literal from cg.apps.demultiplex.sample_sheet.models import ( @@ -26,7 +26,6 @@ RunParametersNovaSeq6000, RunParametersNovaSeqX, ) -from cg.utils.files import get_file_in_directory LOG = logging.getLogger(__name__) @@ -45,7 +44,6 @@ def __init__(self, flow_cell_path: Path, bcl_converter: Optional[str] = "bcl2fas self.base_name: str = "" # Base name is flow cell-id + flow cell position self.id: str = "" self.position: Literal["A", "B"] = "A" - self.flow_cell_name: str = "" self.parse_flow_cell_dir_name() def parse_flow_cell_dir_name(self): @@ -65,7 +63,6 @@ def parse_flow_cell_dir_name(self): LOG.debug(f"Set flow cell id to {base_name}") self.id = base_name[1:] self.position = base_name[0] - self.flow_cell_name = base_name[1:] @property def split_flow_cell_name(self) -> List[str]: diff --git a/cg/models/demultiplex/sbatch.py b/cg/models/demultiplex/sbatch.py index 24eb464eae..ac05ce6905 100644 --- a/cg/models/demultiplex/sbatch.py +++ b/cg/models/demultiplex/sbatch.py @@ -1,4 +1,4 @@ -from pydantic import BaseModel +from pydantic.v1 import BaseModel from typing_extensions import Literal diff --git a/cg/models/invoice/invoice.py b/cg/models/invoice/invoice.py index 9edcb53158..3c27fb6504 100644 --- a/cg/models/invoice/invoice.py +++ b/cg/models/invoice/invoice.py @@ -1,5 +1,5 @@ """Module for defining invoice models.""" -from pydantic import BaseModel +from pydantic.v1 import BaseModel from typing import List, Optional, Any from cg.constants.priority import PriorityTerms from cg.constants.sequencing import RecordType diff --git a/cg/models/lims/sample.py b/cg/models/lims/sample.py index cef05e0d34..876fe2002b 100644 --- a/cg/models/lims/sample.py +++ b/cg/models/lims/sample.py @@ -1,6 +1,6 @@ from typing import Optional -from pydantic import BaseModel, validator +from pydantic.v1 import BaseModel, validator from typing_extensions import Literal from cg.constants import Priority diff --git a/cg/models/mip/mip_config.py b/cg/models/mip/mip_config.py index 790bc41c94..2ec48f826a 100644 --- a/cg/models/mip/mip_config.py +++ b/cg/models/mip/mip_config.py @@ -1,6 +1,6 @@ """Model MIP config""" -from pydantic import BaseModel, EmailStr, Field, validator +from pydantic.v1 import BaseModel, EmailStr, Field, validator from typing import List from cg.constants.priority import SlurmQos diff --git a/cg/models/mip/mip_metrics_deliverables.py b/cg/models/mip/mip_metrics_deliverables.py index 3f049c13ec..47011dd30b 100644 --- a/cg/models/mip/mip_metrics_deliverables.py +++ b/cg/models/mip/mip_metrics_deliverables.py @@ -1,6 +1,6 @@ from typing import Any, Dict, List, Optional -from pydantic import validator +from pydantic.v1 import validator from cg.constants.subject import Gender from cg.models.deliverables.metric_deliverables import ( diff --git a/cg/models/mip/mip_sample_info.py b/cg/models/mip/mip_sample_info.py index c555bcd843..be4992796f 100644 --- a/cg/models/mip/mip_sample_info.py +++ b/cg/models/mip/mip_sample_info.py @@ -3,7 +3,7 @@ import datetime -from pydantic import BaseModel, Field, validator +from pydantic.v1 import BaseModel, Field, validator class MipBaseSampleInfo(BaseModel): diff --git a/cg/models/nextflow/deliverables.py b/cg/models/nextflow/deliverables.py index cc00dbab01..18e000291a 100644 --- a/cg/models/nextflow/deliverables.py +++ b/cg/models/nextflow/deliverables.py @@ -1,6 +1,6 @@ import collections -from pydantic import BaseModel, validator +from pydantic.v1 import BaseModel, validator from cg.constants.nextflow import DELIVER_FILE_HEADERS diff --git a/cg/models/nextflow/sample.py b/cg/models/nextflow/sample.py index b47047ecc1..60925daf62 100644 --- a/cg/models/nextflow/sample.py +++ b/cg/models/nextflow/sample.py @@ -1,6 +1,6 @@ from typing import List -from pydantic import BaseModel, validator +from pydantic.v1 import BaseModel, validator class NextflowSample(BaseModel): diff --git a/cg/models/observations/input_files.py b/cg/models/observations/input_files.py index 6a11766b83..01ab69a140 100644 --- a/cg/models/observations/input_files.py +++ b/cg/models/observations/input_files.py @@ -4,7 +4,7 @@ from pathlib import Path from typing import Optional -from pydantic import BaseModel, validator +from pydantic.v1 import BaseModel, validator LOG = logging.getLogger(__name__) diff --git a/cg/models/orders/excel_sample.py b/cg/models/orders/excel_sample.py index 8561c60d24..40eee4a990 100644 --- a/cg/models/orders/excel_sample.py +++ b/cg/models/orders/excel_sample.py @@ -1,5 +1,5 @@ from typing import List, Optional -from pydantic import Field, validator +from pydantic.v1 import Field, validator from cg.constants.orderforms import REV_SEX_MAP, SOURCE_TYPES from cg.models.orders.sample_base import OrderSample diff --git a/cg/models/orders/json_sample.py b/cg/models/orders/json_sample.py index 88408c7902..8153793f4f 100644 --- a/cg/models/orders/json_sample.py +++ b/cg/models/orders/json_sample.py @@ -2,7 +2,7 @@ from cg.constants import DataDelivery, Pipeline from cg.models.orders.sample_base import OrderSample -from pydantic import constr, validator +from pydantic.v1 import constr, validator class JsonSample(OrderSample): diff --git a/cg/models/orders/order.py b/cg/models/orders/order.py index 84b94fc502..4354095fb4 100644 --- a/cg/models/orders/order.py +++ b/cg/models/orders/order.py @@ -1,6 +1,6 @@ from typing import Optional, Any -from pydantic import BaseModel, constr, conlist +from pydantic.v1 import BaseModel, constr, conlist from cg.models.orders.constants import OrderType from cg.models.orders.samples import ( diff --git a/cg/models/orders/orderform_schema.py b/cg/models/orders/orderform_schema.py index c2f425ef8c..850370ce6b 100644 --- a/cg/models/orders/orderform_schema.py +++ b/cg/models/orders/orderform_schema.py @@ -1,6 +1,6 @@ from typing import List, Optional -from pydantic import BaseModel +from pydantic.v1 import BaseModel from cg.models.orders.sample_base import OrderSample diff --git a/cg/models/orders/sample_base.py b/cg/models/orders/sample_base.py index 1e2e3d2ab9..62c47fd386 100644 --- a/cg/models/orders/sample_base.py +++ b/cg/models/orders/sample_base.py @@ -2,7 +2,7 @@ from typing import List, Optional from cg.constants import DataDelivery, Pipeline -from pydantic import BaseModel, constr, validator +from pydantic.v1 import BaseModel, constr, validator from cg.store.models import Application, Family, Customer, Pool, Sample diff --git a/cg/models/orders/samples.py b/cg/models/orders/samples.py index 04d2a0ac62..ed5b66ce37 100644 --- a/cg/models/orders/samples.py +++ b/cg/models/orders/samples.py @@ -13,7 +13,7 @@ ) from cg.store.models import Application, Family, Organism, Panel, Pool, Sample from cgmodels.cg.constants import Pipeline -from pydantic import BaseModel, constr, validator +from pydantic.v1 import BaseModel, constr, validator class OptionalIntValidator: diff --git a/cg/models/report/metadata.py b/cg/models/report/metadata.py index 52d168ae86..2bc688bbb2 100644 --- a/cg/models/report/metadata.py +++ b/cg/models/report/metadata.py @@ -1,6 +1,6 @@ from typing import Optional, Union -from pydantic import BaseModel, validator +from pydantic.v1 import BaseModel, validator from cg.models.report.validators import ( validate_empty_field, validate_float, diff --git a/cg/models/report/report.py b/cg/models/report/report.py index 4ad3c98f62..0e73f6cea4 100644 --- a/cg/models/report/report.py +++ b/cg/models/report/report.py @@ -1,7 +1,7 @@ from datetime import datetime from typing import List, Optional, Union -from pydantic import BaseModel, validator, root_validator +from pydantic.v1 import BaseModel, validator, root_validator from cg.constants import Pipeline, DataDelivery from cg.models.report.sample import SampleModel, ApplicationModel from cg.models.report.validators import ( diff --git a/cg/models/report/sample.py b/cg/models/report/sample.py index ca0ae93fae..6d417ae9d7 100644 --- a/cg/models/report/sample.py +++ b/cg/models/report/sample.py @@ -2,7 +2,7 @@ from typing import Optional, Union from cg.constants.subject import Gender -from pydantic import BaseModel, validator +from pydantic.v1 import BaseModel, validator from cg.models.report.metadata import SampleMetadataModel from cg.models.report.validators import ( validate_empty_field, diff --git a/cg/models/rnafusion/command_args.py b/cg/models/rnafusion/command_args.py index 7fc761e73e..6255cbd8d2 100644 --- a/cg/models/rnafusion/command_args.py +++ b/cg/models/rnafusion/command_args.py @@ -1,7 +1,7 @@ from pathlib import Path from typing import Optional, Union -from pydantic import BaseModel +from pydantic.v1 import BaseModel class CommandArgs(BaseModel): diff --git a/cg/models/rnafusion/metrics.py b/cg/models/rnafusion/metrics.py index 6c68202d59..e94139c670 100644 --- a/cg/models/rnafusion/metrics.py +++ b/cg/models/rnafusion/metrics.py @@ -1,7 +1,7 @@ """Rnafusion quality control metrics model.""" from typing import Optional -from pydantic import BaseModel +from pydantic.v1 import BaseModel class RnafusionQCMetrics(BaseModel): diff --git a/cg/models/scout/scout_load_config.py b/cg/models/scout/scout_load_config.py index fea2add067..9a0d0349ca 100644 --- a/cg/models/scout/scout_load_config.py +++ b/cg/models/scout/scout_load_config.py @@ -3,7 +3,7 @@ from datetime import datetime from typing import List, Optional -from pydantic import BaseModel, validator +from pydantic.v1 import BaseModel, validator from typing_extensions import Literal diff --git a/cg/models/slurm/sbatch.py b/cg/models/slurm/sbatch.py index 18c7673408..3a01f02dd7 100644 --- a/cg/models/slurm/sbatch.py +++ b/cg/models/slurm/sbatch.py @@ -1,6 +1,6 @@ from typing import Optional -from pydantic import BaseModel +from pydantic.v1 import BaseModel from cg.constants.constants import HastaSlurmPartitions from cg.constants.priority import SlurmQos diff --git a/cg/models/workflow/mutant.py b/cg/models/workflow/mutant.py index 279fc697a4..d3628d3901 100644 --- a/cg/models/workflow/mutant.py +++ b/cg/models/workflow/mutant.py @@ -1,4 +1,4 @@ -from pydantic import BaseModel, validator +from pydantic.v1 import BaseModel, validator class MutantSampleConfig(BaseModel): diff --git a/cg/server/api.py b/cg/server/api.py index 88cba5000a..eb0933c0db 100644 --- a/cg/server/api.py +++ b/cg/server/api.py @@ -22,7 +22,7 @@ from cg.store.models import Analysis, Application, Customer, Family, Flowcell, Pool, Sample, User from flask import Blueprint, abort, current_app, g, jsonify, make_response, request from google.auth import jwt -from pydantic import ValidationError +from pydantic.v1 import ValidationError from requests.exceptions import HTTPError from sqlalchemy.exc import IntegrityError from urllib3.exceptions import MaxRetryError, NewConnectionError diff --git a/requirements.txt b/requirements.txt index b365e3b0b3..706725b1a7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,7 +33,7 @@ packaging pandas paramiko petname -pydantic<2.0 +pydantic>2.0 python-dateutil pyyaml setuptools>=39.2.0 # due to WeasyPrint 45, tinycss2 1.0.1 and cairocffi file-.cairocffi-VERSION diff --git a/setup.py b/setup.py index 620f958830..caf04d7c92 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,7 @@ def parse_requirements(req_path="./requirements.txt"): setup( name=NAME, - version="38.3.0", + version="39.0.0", description=DESCRIPTION, long_description=LONG_DESCRIPTION, long_description_content_type="text/markdown", diff --git a/tests/apps/cgstats/conftest.py b/tests/apps/cgstats/conftest.py index 73568b75fb..b18c885938 100644 --- a/tests/apps/cgstats/conftest.py +++ b/tests/apps/cgstats/conftest.py @@ -3,7 +3,7 @@ from typing import Dict import pytest -from pydantic import BaseModel +from pydantic.v1 import BaseModel from cg.apps.cgstats.crud import create from cg.apps.cgstats.db.models import Supportparams, Sample, Project, Datasource, Demux diff --git a/tests/apps/crunchy/conftest.py b/tests/apps/crunchy/conftest.py index 044f360e12..e698ef17cb 100644 --- a/tests/apps/crunchy/conftest.py +++ b/tests/apps/crunchy/conftest.py @@ -5,10 +5,10 @@ import pytest +from cg.apps.crunchy.models import CrunchyMetadata from cg.constants.constants import FileFormat from cg.io.controller import WriteFile from cg.models import CompressionData -from cgmodels.crunchy.metadata import CrunchyMetadata LOG = logging.getLogger(__name__) diff --git a/tests/apps/crunchy/test_compress_fastq.py b/tests/apps/crunchy/test_compress_fastq.py index 0b70933949..d57fde44bd 100644 --- a/tests/apps/crunchy/test_compress_fastq.py +++ b/tests/apps/crunchy/test_compress_fastq.py @@ -4,6 +4,8 @@ from typing import Dict, List import pytest +from pydantic.v1 import ValidationError + from cg.apps.crunchy import CrunchyAPI from cg.apps.crunchy.files import ( get_crunchy_metadata, @@ -11,13 +13,12 @@ get_log_dir, get_spring_archive_files, ) +from cg.apps.crunchy.models import CrunchyFile, CrunchyMetadata from cg.apps.slurm.slurm_api import SlurmAPI from cg.constants.constants import FileFormat from cg.io.controller import WriteFile from cg.models import CompressionData from cg.utils import Process -from cgmodels.crunchy.metadata import CrunchyFile, CrunchyMetadata -from pydantic import ValidationError def test_get_spring_metadata(spring_metadata_file: Path): diff --git a/tests/apps/crunchy/test_config.py b/tests/apps/crunchy/test_config.py index cda2b83aa2..edc22af129 100644 --- a/tests/apps/crunchy/test_config.py +++ b/tests/apps/crunchy/test_config.py @@ -3,9 +3,9 @@ from typing import Dict, Any from cg.apps.crunchy.files import get_crunchy_metadata, update_metadata_date, update_metadata_paths +from cg.apps.crunchy.models import CrunchyMetadata from cg.constants.constants import FileFormat from cg.io.controller import ReadFile -from cgmodels.crunchy.metadata import CrunchyMetadata def test_get_spring_metadata_real_file( diff --git a/tests/apps/demultiplex/conftest.py b/tests/apps/demultiplex/conftest.py index dbff174d67..7025c7229c 100644 --- a/tests/apps/demultiplex/conftest.py +++ b/tests/apps/demultiplex/conftest.py @@ -36,12 +36,33 @@ def fixture_demux_run_dir_dragen(flow_cell_runs_dir: Path) -> Path: return Path(flow_cell_runs_dir, BclConverter.DRAGEN) +@pytest.fixture(name="index_obj") +def fixture_index_obj() -> Index: + return Index(name="C07 - UDI0051", sequence="AACAGGTT-ATACCAAG") + + @pytest.fixture(name="valid_index") def fixture_valid_index_() -> Index: """Return a valid index.""" return Index(name="C07 - UDI0051", sequence="AACAGGTT-ATACCAAG") +@pytest.fixture(name="lims_novaseq_bcl2fastq_samples") +def fixture_lims_novaseq_bcl2fastq_samples( + lims_novaseq_samples_raw: List[dict], +) -> List[FlowCellSampleNovaSeq6000Bcl2Fastq]: + """Return a list of parsed Bcl2fastq flow cell samples""" + return [FlowCellSampleNovaSeq6000Bcl2Fastq(**sample) for sample in lims_novaseq_samples_raw] + + +@pytest.fixture(name="lims_novaseq_dragen_samples") +def fixture_lims_novaseq_dragen_samples( + lims_novaseq_samples_raw: List[dict], +) -> List[FlowCellSampleNovaSeq6000Dragen]: + """Return a list of parsed Dragen flow cell samples""" + return [FlowCellSampleNovaSeq6000Dragen(**sample) for sample in lims_novaseq_samples_raw] + + @pytest.fixture(name="lims_novaseq_x_samples") def fixture_lims_novaseq_x_samples( lims_novaseq_samples_raw: List[dict], @@ -347,18 +368,3 @@ def fixture_novaseq6000_flow_cell_sample_before_adapt_indexes() -> ( SampleName="814206", Project="814206", ) - - -@pytest.fixture(name="novaseq6000_bcl_convert_sample_sheet_path") -def fixture_novaseq6000_sample_sheet_path() -> Path: - """Return the path to a NovaSeq 6000 BCL convert sample sheet.""" - return Path( - "tests", - "fixtures", - "apps", - "sequencing_metrics_parser", - "230622_A00621_0864_AHY7FFDRX2", - "Unaligned", - "Reports", - "SampleSheet.csv", - ) diff --git a/tests/apps/orderform/test_excel_sample_schema.py b/tests/apps/orderform/test_excel_sample_schema.py index 1734768c5c..50f17119a7 100644 --- a/tests/apps/orderform/test_excel_sample_schema.py +++ b/tests/apps/orderform/test_excel_sample_schema.py @@ -1,5 +1,5 @@ import pytest -from pydantic import ValidationError +from pydantic.v1 import ValidationError from cg.constants import DataDelivery from cg.models.orders.excel_sample import ExcelSample diff --git a/tests/apps/scout/test_scout_load_config.py b/tests/apps/scout/test_scout_load_config.py index b97a613233..ad1e09b513 100644 --- a/tests/apps/scout/test_scout_load_config.py +++ b/tests/apps/scout/test_scout_load_config.py @@ -2,7 +2,7 @@ from typing import Any import pytest -from pydantic import ValidationError +from pydantic.v1 import ValidationError from cg.models.scout import scout_load_config from cg.models.scout.scout_load_config import MipLoadConfig, ScoutMipIndividual diff --git a/tests/cli/demultiplex/test_finish_demux.py b/tests/cli/demultiplex/test_finish_demux.py index fad574f446..11b5520cf5 100644 --- a/tests/cli/demultiplex/test_finish_demux.py +++ b/tests/cli/demultiplex/test_finish_demux.py @@ -40,7 +40,7 @@ def test_finish_all_cmd( ): caplog.set_level(logging.INFO) - # GIVEN a demultiplex flow cell finished output directory that exist + # GIVEN a demultiplex flow cell finished output directory that does not exist # GIVEN a demultiplex context @@ -80,7 +80,7 @@ def test_finish_flow_cell_dry_run( assert result.exit_code == EXIT_SUCCESS -def test_finish_flow_cell( +def test_finish_flow_cell_fail( caplog, cli_runner: testing.CliRunner, demultiplex_context: CGConfig, @@ -89,7 +89,7 @@ def test_finish_flow_cell( ): caplog.set_level(logging.INFO) - # GIVEN a demultiplex flow cell finished output directory that exist + # GIVEN a demultiplex flow cell finished output directory that does not exist # GIVEN a demultiplex context @@ -103,7 +103,7 @@ def test_finish_flow_cell( ) # THEN assert the command exits successfully - assert result.exit_code == EXIT_SUCCESS + assert result.exit_code != EXIT_SUCCESS def test_finish_all_hiseq_x_dry_run( diff --git a/tests/cli/generate/report/conftest.py b/tests/cli/generate/report/conftest.py index ee375309bf..a881ecd6ad 100644 --- a/tests/cli/generate/report/conftest.py +++ b/tests/cli/generate/report/conftest.py @@ -2,9 +2,9 @@ import click import pytest -from cgmodels.cg.constants import Pipeline from cg.cli.generate.report.base import generate_delivery_report +from cg.constants import Pipeline from cg.models.cg_config import CGConfig from tests.mocks.report import MockMipDNAReportAPI, MockMipDNAAnalysisAPI diff --git a/tests/cli/generate/report/test_cli_delivery_report.py b/tests/cli/generate/report/test_cli_delivery_report.py index f7e0b07670..81918e8111 100644 --- a/tests/cli/generate/report/test_cli_delivery_report.py +++ b/tests/cli/generate/report/test_cli_delivery_report.py @@ -1,24 +1,26 @@ -"""Tests the cli for generating delivery reports""" - +"""Tests the cli for generating delivery reports.""" import logging from datetime import datetime +from _pytest.logging import LogCaptureFixture +from click.testing import Result, CliRunner + from cg.cli.generate.report.base import generate_delivery_report from cg.constants import EXIT_SUCCESS, EXIT_FAIL +from cg.models.cg_config import CGConfig -def test_delivery_report_invalid_case(mip_dna_context, cli_runner, caplog): - """Tests the delivery report command for an invalid case""" - +def test_delivery_report_invalid_case( + mip_dna_context: CGConfig, cli_runner: CliRunner, caplog: LogCaptureFixture +): + """Tests the delivery report command for an invalid case.""" caplog.set_level(logging.INFO) # GIVEN a MIP DNA context object # WHEN calling delivery_report with an invalid case name - result = cli_runner.invoke( - generate_delivery_report, - ["not a case"], - obj=mip_dna_context, + result: Result = cli_runner.invoke( + generate_delivery_report, ["not a case"], obj=mip_dna_context ) # THEN the command should fail due to an invalid case ID @@ -27,18 +29,17 @@ def test_delivery_report_invalid_case(mip_dna_context, cli_runner, caplog): assert result.exit_code == EXIT_FAIL -def test_delivery_report_dry_run(mip_dna_context, cli_runner, case_id, caplog): - """Tests the delivery report command with a dry run option""" - +def test_delivery_report_dry_run( + mip_dna_context: CGConfig, cli_runner: CliRunner, case_id: str, caplog: LogCaptureFixture +): + """Tests the delivery report command with a dry run option.""" caplog.set_level(logging.INFO) # GIVEN a MIP DNA context object # WHEN calling delivery_report with a dry option - result = cli_runner.invoke( - generate_delivery_report, - [case_id, "--dry-run"], - obj=mip_dna_context, + result: Result = cli_runner.invoke( + generate_delivery_report, [case_id, "--dry-run"], obj=mip_dna_context ) # THEN the command should be invoked successfully @@ -49,20 +50,18 @@ def test_delivery_report_dry_run(mip_dna_context, cli_runner, case_id, caplog): assert result.exit_code == EXIT_SUCCESS -def test_delivery_report(mip_dna_context, cli_runner, case_id, caplog): - """Tests the delivery report command expecting a rendered html file""" - +def test_delivery_report( + mip_dna_context: CGConfig, cli_runner: CliRunner, case_id: str, caplog: LogCaptureFixture +): + """Tests the delivery report command expecting a rendered html file.""" caplog.set_level(logging.INFO) # GIVEN a MIP DNA context object # WHEN calling delivery_report with a dry option - cli_runner.invoke( - generate_delivery_report, - [case_id], - obj=mip_dna_context, - ) + result: Result = cli_runner.invoke(generate_delivery_report, [case_id], obj=mip_dna_context) # THEN the report html should have been created and added to HK assert "create_delivery_report_file" in caplog.text assert "add_delivery_report_to_hk" in caplog.text + assert result.exit_code == EXIT_SUCCESS diff --git a/tests/cli/generate/report/test_utils.py b/tests/cli/generate/report/test_utils.py index 9018500459..9b52f406ee 100644 --- a/tests/cli/generate/report/test_utils.py +++ b/tests/cli/generate/report/test_utils.py @@ -3,7 +3,7 @@ import click.exceptions import pytest -from cgmodels.cg.constants import Pipeline +from cg.constants import Pipeline from cg.cli.generate.report.utils import ( get_report_case, diff --git a/tests/cli/upload/test_cli_upload_delivery_report.py b/tests/cli/upload/test_cli_upload_delivery_report.py index 357aeae0d6..64856ef005 100644 --- a/tests/cli/upload/test_cli_upload_delivery_report.py +++ b/tests/cli/upload/test_cli_upload_delivery_report.py @@ -1,38 +1,41 @@ -"""Tests the cli for uploading delivery reports""" +"""Tests the cli for uploading delivery reports.""" +from click.testing import CliRunner, Result +from cg.apps.housekeeper.hk import HousekeeperAPI from cg.cli.upload.delivery_report import upload_delivery_report_to_scout from cg.constants import EXIT_FAIL, EXIT_SUCCESS from cg.meta.report.mip_dna import MipDNAReportAPI +from cg.models.cg_config import CGConfig -def test_delivery_report_to_scout_no_params(upload_context, cli_runner): - """Tests the upload to Scout without specifying the case""" +def test_delivery_report_to_scout_no_params(upload_context: CGConfig, cli_runner: CliRunner): + """Tests the upload to Scout without specifying the case.""" # GIVEN a MIP-DNA report api assert isinstance(upload_context.meta_apis.get("report_api"), MipDNAReportAPI) # WHEN invoking the delivery report upload without parameters - result = cli_runner.invoke( - upload_delivery_report_to_scout, - obj=upload_context, - ) + result: Result = cli_runner.invoke(upload_delivery_report_to_scout, obj=upload_context) # THEN the command should fail due to a missing case ID assert "There are no valid cases to perform delivery report actions" in result.output assert result.exit_code == EXIT_FAIL -def test_delivery_report_to_scout(upload_context, cli_runner, upload_report_hk_api, case_id): - """Tests the upload to Scout of a MIP DNA delivery report""" +def test_delivery_report_to_scout( + upload_context: CGConfig, + cli_runner: CliRunner, + upload_report_hk_api: HousekeeperAPI, + case_id: str, +): + """Tests the upload to Scout of a MIP DNA delivery report.""" # GIVEN a Housekeeper context with a delivery report file that is ready for upload upload_context.housekeeper_api_ = upload_report_hk_api # WHEN uploading the delivery report - result = cli_runner.invoke( - upload_delivery_report_to_scout, - [case_id], - obj=upload_context, + result: Result = cli_runner.invoke( + upload_delivery_report_to_scout, [case_id], obj=upload_context ) # THEN check that the command exits with success and that the mock file has been uploaded to Scout diff --git a/tests/cli/workflow/balsamic/test_store_housekeeper.py b/tests/cli/workflow/balsamic/test_store_housekeeper.py index d5d76d1a5d..4bb60a25bc 100644 --- a/tests/cli/workflow/balsamic/test_store_housekeeper.py +++ b/tests/cli/workflow/balsamic/test_store_housekeeper.py @@ -13,7 +13,7 @@ from cg.models.cg_config import CGConfig from cg.utils import Process from click.testing import CliRunner -from pydantic import ValidationError +from pydantic.v1 import ValidationError def test_without_options(cli_runner: CliRunner, balsamic_context: CGConfig): diff --git a/tests/cli/workflow/rnafusion/test_cli_rnafusion_store_housekeeper.py b/tests/cli/workflow/rnafusion/test_cli_rnafusion_store_housekeeper.py index ad99fa2d24..09726a07ac 100644 --- a/tests/cli/workflow/rnafusion/test_cli_rnafusion_store_housekeeper.py +++ b/tests/cli/workflow/rnafusion/test_cli_rnafusion_store_housekeeper.py @@ -4,7 +4,7 @@ import pytest from _pytest.logging import LogCaptureFixture from click.testing import CliRunner -from pydantic import ValidationError +from pydantic.v1 import ValidationError from pytest_mock import MockFixture from cg.apps.hermes.hermes_api import HermesApi diff --git a/tests/conftest.py b/tests/conftest.py index 07b21f0b10..fb4cddc3a0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1087,6 +1087,21 @@ def fixture_populated_stats_api( return stats_api +@pytest.fixture(name="novaseq6000_bcl_convert_sample_sheet_path") +def fixture_novaseq6000_sample_sheet_path() -> Path: + """Return the path to a NovaSeq 6000 BCL convert sample sheet.""" + return Path( + "tests", + "fixtures", + "apps", + "sequencing_metrics_parser", + "230622_A00621_0864_AHY7FFDRX2", + "Unaligned", + "Reports", + "SampleSheet.csv", + ) + + @pytest.fixture(name="demultiplex_fixtures", scope="session") def fixture_demultiplex_fixtures(apps_dir: Path) -> Path: """Return the path to the demultiplex fixture directory.""" @@ -2688,8 +2703,8 @@ def fixture_store_with_sequencing_metrics( yield store -@pytest.fixture -def demultiplexed_flow_cells_directory(tmp_path) -> Path: +@pytest.fixture(name="demultiplexed_flow_cells_tmp_directory") +def fixture_demultiplexed_flow_cells_tmp_directory(tmp_path) -> Path: original_dir = Path( Path(__file__).parent, "fixtures", "apps", "demultiplexing", "demultiplexed-runs" ) diff --git a/tests/fixtures/apps/demultiplexing/flow-cell-runs/hiseq/170407_ST-E00198_0209_BHHKVCALXX/SampleSheet.csv b/tests/fixtures/apps/demultiplexing/flow-cell-runs/hiseq/170407_ST-E00198_0209_BHHKVCALXX/SampleSheet.csv new file mode 100644 index 0000000000..44f9ddf862 --- /dev/null +++ b/tests/fixtures/apps/demultiplexing/flow-cell-runs/hiseq/170407_ST-E00198_0209_BHHKVCALXX/SampleSheet.csv @@ -0,0 +1,3 @@ +[Data] +FCID,Lane,SampleID,SampleRef,index,SampleName,Control,Recipe,Operator,Project +HHKVCALXX,1,SVE2528A1_CTGAAGCT,hg19,CTGAAGCT,123456,N,R1,script,123456 diff --git a/tests/fixtures/apps/demultiplexing/flow-cell-runs/hiseq/180522_ST-E00198_0301_BHLCKNCCXY/copy_complete.txt b/tests/fixtures/apps/demultiplexing/flow-cell-runs/hiseq/180522_ST-E00198_0301_BHLCKNCCXY/copy_complete.txt deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/fixtures/apps/demultiplexing/flow-cell-runs/hiseq/180522_ST-E00198_0301_BHLCKNCCXY/runParameters.xml b/tests/fixtures/apps/demultiplexing/flow-cell-runs/hiseq/180522_ST-E00198_0301_BHLCKNCCXY/runParameters.xml deleted file mode 100644 index c2ecd8aa23..0000000000 --- a/tests/fixtures/apps/demultiplexing/flow-cell-runs/hiseq/180522_ST-E00198_0301_BHLCKNCCXY/runParameters.xml +++ /dev/null @@ -1,165 +0,0 @@ - - - - false - 2 - HLCKNCCXY - -999 - B - SINGLEINDEX - 151 - 8 - 0 - 151 - X:\Runs - Save All Thumbnails - HiSeq X - true - - HiSeq X SBS - HiSeq X PE - HiSeq X Sequencing Primer - None - false - HiSeq Control Software - HD 3.5.0.7 - 180522_ST-E00198_0301_BHLCKNCCXY - 180522 - BaseSpace - - clinical.genomics@scilifelab.se - 89045957 - 89045957 - O:\Illumina\HiSeqTemp - true - true - true - - ST-E00198 - 301 - HWI-E00198 - 10.37.13 - 3.0.0 - 2.7.7 - 2.9.2.15 - Illumina,Bruno Fluidics Controller,0,v2.0420 - 2.21-C00-R03 - 7.5.180.4395 - 2.12.0.0 - - HLCKNCCXY - HHHHHCCX2 - sbsuser - -
-
-
-
-
-
-
-
- - DynamicITF - BothLaneSurfaces - true - AutoSwath - true - true - true - true - false - - 3 - 7 - 0 - 250 - 250 - 0 - 200 - 21 - 1 - 100 - 50 - 20 - 65535 - 50 - 4 - - 3200 - 7241 - 3200 - 174420 - 60.175 - 24 - 2 - false - - - - - - - - - - false - 325 - false - false - true - - - - - - - - - - - - - CMR - - - - SRE - - - - EDP - - - - LFN - - - - ICB - - - - SB1 - - - - SB2 - - - - SB3 - - - - - - O:\Illumina\HiSeqTemp\180522_ST-E00198_0301_BHLCKNCCXY - HD 3.6.0.4 - false - false - - false - - 1 - \ No newline at end of file diff --git a/tests/fixtures/apps/demultiplexing/flow-cell-runs/nova_seq_6000/230504_A00689_0804_BHY7FFDRX2/SampleSheet.csv b/tests/fixtures/apps/demultiplexing/flow-cell-runs/nova_seq_6000/230504_A00689_0804_BHY7FFDRX2/SampleSheet.csv new file mode 100644 index 0000000000..cb2e59e745 --- /dev/null +++ b/tests/fixtures/apps/demultiplexing/flow-cell-runs/nova_seq_6000/230504_A00689_0804_BHY7FFDRX2/SampleSheet.csv @@ -0,0 +1,9 @@ +[Settings] +BarcodeMismatchesIndex1,0 +BarcodeMismatchesIndex2,0 +[Data] +FCID,Lane,Sample_ID,SampleRef,index,index2,SampleName,Control,Recipe,Operator,Sample_Project +HY7FFDRX2,1,ACC11927A2,hg19,GTCCTTCGGC,CTGTGCATGA,anonymous_1,N,R1,script,405887 +HY7FFDRX2,1,ACC11927A5,hg19,GTTTCACGAT,TTTGGCCGAA,anonymous_2,N,R1,script,405887 +HY7FFDRX2,2,ACC11927A2,hg19,GTCCTTCGGC,CTGTGCATGA,anonymous_1,N,R1,script,405887 +HY7FFDRX2,2,ACC11927A5,hg19,GTTTCACGAT,TTTGGCCGAA,anonymous_2,N,R1,script,405887 diff --git a/tests/meta/demultiplex/test_demux_post_processing.py b/tests/meta/demultiplex/test_demux_post_processing.py index 45e223306d..66965c14ef 100644 --- a/tests/meta/demultiplex/test_demux_post_processing.py +++ b/tests/meta/demultiplex/test_demux_post_processing.py @@ -785,22 +785,48 @@ def test_post_processing_of_flow_cell_demultiplexed_with_bclconvert( assert delivery_path.exists() -def test_copy_sample_sheet(demultiplex_context: CGConfig): +def test_add_demux_logs_to_housekeeper( + demultiplex_context: CGConfig, dragen_flow_cell: FlowCellDirectoryData +): # GIVEN a DemuxPostProcessing API demux_post_processing_api = DemuxPostProcessingAPI(demultiplex_context) - # GIVEN a sample sheet in the run directory - sample_sheet_path = Path( - demux_post_processing_api.demux_api.run_dir, - DemultiplexingDirsAndFiles.SAMPLE_SHEET_FILE_NAME, - ) - sample_sheet_path.touch() + # GIVEN a bundle and flow cell version exists in housekeeper + demux_post_processing_api.add_bundle_and_version_if_non_existent( + bundle_name=dragen_flow_cell.id + ) + + # GIVEN a demux log in the run directory + demux_log_file_paths: List[Path] = [ + Path( + demux_post_processing_api.demux_api.run_dir, + f"{dragen_flow_cell.full_name}", + f"{dragen_flow_cell.id}_demultiplex.stdout", + ), + Path( + demux_post_processing_api.demux_api.run_dir, + f"{dragen_flow_cell.full_name}", + f"{dragen_flow_cell.id}_demultiplex.stderr", + ), + ] + for file_path in demux_log_file_paths: + file_path.parent.mkdir(parents=True, exist_ok=True) + file_path.touch() + + # WHEN adding the demux logs to housekeeper + demux_post_processing_api.add_demux_logs_to_housekeeper(flow_cell=dragen_flow_cell) + + # THEN the demux log was added to housekeeper + files = demux_post_processing_api.hk_api.get_files( + tags=[SequencingFileTag.DEMUX_LOG], + bundle=dragen_flow_cell.id, + ).all() - # WHEN copying the sample sheet - demux_post_processing_api.copy_sample_sheet() + expected_file_names: List[str] = [] + for file_path in demux_log_file_paths: + expected_file_names.append(file_path.name.split("/")[-1]) - # THEN the sample sheet was copied to the out directory - assert Path( - demux_post_processing_api.demux_api.out_dir, - DemultiplexingDirsAndFiles.SAMPLE_SHEET_FILE_NAME, - ).exists() + # THEN the demux logs were added to housekeeper with the correct names + assert len(files) == 2 + for file in files: + assert file.path.split("/")[-1] in expected_file_names diff --git a/tests/meta/demultiplex/test_utils.py b/tests/meta/demultiplex/test_utils.py index 6e2e6fec48..8bc413d999 100644 --- a/tests/meta/demultiplex/test_utils.py +++ b/tests/meta/demultiplex/test_utils.py @@ -14,9 +14,14 @@ get_q30_threshold, get_sample_sheet_path, parse_flow_cell_directory_data, + copy_sample_sheet, ) from cg.meta.demultiplex.validation import is_bcl2fastq_demux_folder_structure from cg.models.demultiplex.flow_cell import FlowCellDirectoryData +from cg.meta.demultiplex.demux_post_processing import ( + DemuxPostProcessingAPI, +) +from cg.models.cg_config import CGConfig def test_get_lane_from_sample_fastq_file_path(): @@ -186,3 +191,30 @@ def test_parse_flow_cell_directory_data_valid(mocked_function): # THEN the flow cell path and bcl converter should be set assert result.path == Path(flow_cell_run_directory) assert result.bcl_converter == "dummy_bcl_converter" + + +def test_copy_sample_sheet(demultiplex_context: CGConfig): + # GIVEN a DemuxPostProcessing API + demux_post_processing_api = DemuxPostProcessingAPI(demultiplex_context) + + # GIVEN a sample sheet in the run directory + sample_sheet_path = Path( + demux_post_processing_api.demux_api.run_dir, + ) + sample_sheet_path.mkdir(parents=True, exist_ok=True) + Path(sample_sheet_path, DemultiplexingDirsAndFiles.SAMPLE_SHEET_FILE_NAME).touch() + + # GIVEN a sample sheet target path + target_sample_sheet_path = Path(demux_post_processing_api.demux_api.out_dir) + + # WHEN copying the sample sheet + copy_sample_sheet( + sample_sheet_source_directory=sample_sheet_path, + sample_sheet_destination_directory=target_sample_sheet_path, + ) + + # THEN the sample sheet was copied to the out directory + assert Path( + target_sample_sheet_path, + DemultiplexingDirsAndFiles.SAMPLE_SHEET_FILE_NAME, + ).exists() diff --git a/tests/fixtures/apps/demultiplexing/flow-cell-runs/hiseq/180522_ST-E00198_0301_BHLCKNCCXY/RTAComplete.txt b/tests/meta/demultiplex/tests/fixtures/apps/demultiplexing/flow-cell-runs/hiseq/SampleSheet.csv similarity index 100% rename from tests/fixtures/apps/demultiplexing/flow-cell-runs/hiseq/180522_ST-E00198_0301_BHLCKNCCXY/RTAComplete.txt rename to tests/meta/demultiplex/tests/fixtures/apps/demultiplexing/flow-cell-runs/hiseq/SampleSheet.csv diff --git a/tests/meta/report/conftest.py b/tests/meta/report/conftest.py index 57b85a27c3..b0cad831d4 100644 --- a/tests/meta/report/conftest.py +++ b/tests/meta/report/conftest.py @@ -3,8 +3,8 @@ from typing import Dict, List import pytest -from cgmodels.cg.constants import Pipeline +from cg.constants import Pipeline from cg.constants.constants import FileFormat from cg.io.controller import ReadFile from cg.meta.report.balsamic import BalsamicReportAPI diff --git a/tests/meta/report/test_report_api.py b/tests/meta/report/test_report_api.py index 3ec817ac9a..1f75b5e95d 100644 --- a/tests/meta/report/test_report_api.py +++ b/tests/meta/report/test_report_api.py @@ -1,22 +1,27 @@ +"""Test delivery report API methods.""" import logging -import os from datetime import datetime, timedelta -from typing import List, TextIO +from pathlib import Path +from typing import List -from cg.models.report.report import DataAnalysisModel, ReportModel, CustomerModel, CaseModel +from _pytest.logging import LogCaptureFixture +from cg.meta.workflow.mip_dna import MipDNAAnalysisAPI -from cg.models.report.sample import SampleModel, ApplicationModel, MethodsModel, TimestampModel +from cg.store import Store -from cg.models.mip.mip_analysis import MipAnalysis - -from cg.store.models import Analysis, FamilySample +from cg.meta.report.mip_dna import MipDNAReportAPI from cg.constants import REPORT_GENDER from cg.exc import DeliveryReportError +from cg.models.mip.mip_analysis import MipAnalysis +from cg.models.report.report import DataAnalysisModel, ReportModel, CustomerModel, CaseModel +from cg.models.report.sample import SampleModel, ApplicationModel, MethodsModel, TimestampModel +from cg.store.models import Analysis, FamilySample, Family from tests.meta.report.helper import recursive_assert +from tests.store_helpers import StoreHelpers -def test_create_delivery_report(report_api_mip_dna, case_mip_dna): +def test_create_delivery_report(report_api_mip_dna: MipDNAReportAPI, case_mip_dna: Family): """Tests the creation of the rendered delivery report.""" # GIVEN a pre-built case @@ -32,24 +37,27 @@ def test_create_delivery_report(report_api_mip_dna, case_mip_dna): assert len(delivery_report) > 0 -def test_create_delivery_report_file(report_api_mip_dna, case_mip_dna, tmp_path): +def test_create_delivery_report_file( + report_api_mip_dna: MipDNAReportAPI, case_mip_dna: Family, tmp_path: Path +): """Tests file generation containing the delivery report data.""" # GIVEN a pre-built case # WHEN creating the report file - created_report_file: TextIO = report_api_mip_dna.create_delivery_report_file( + created_report_file: Path = report_api_mip_dna.create_delivery_report_file( case_id=case_mip_dna.internal_id, - file_path=tmp_path, + directory=tmp_path, analysis_date=case_mip_dna.analyses[0].started_at, force_report=False, ) - # THEN check if an html report has been created and saved - assert os.path.isfile(created_report_file.name) + # THEN check if a html report has been created and saved + assert created_report_file.is_file() + assert created_report_file.exists() -def test_render_delivery_report(report_api_mip_dna, case_mip_dna): +def test_render_delivery_report(report_api_mip_dna: MipDNAReportAPI, case_mip_dna: Family): """Tests delivery report rendering.""" # GIVEN a generated report @@ -65,7 +73,7 @@ def test_render_delivery_report(report_api_mip_dna, case_mip_dna): assert "html" in rendered_report -def test_get_validated_report_data(report_api_mip_dna, case_mip_dna): +def test_get_validated_report_data(report_api_mip_dna: MipDNAReportAPI, case_mip_dna: Family): """Tests report data retrieval.""" # GIVEN a valid case @@ -85,9 +93,10 @@ def test_get_validated_report_data(report_api_mip_dna, case_mip_dna): recursive_assert(report_data.dict()) -def test_validate_report_empty_fields(report_api_mip_dna, case_mip_dna, caplog): +def test_validate_report_empty_fields( + report_api_mip_dna: MipDNAReportAPI, case_mip_dna: Family, caplog: LogCaptureFixture +): """Tests the validations of allowed empty report fields.""" - caplog.set_level(logging.INFO) # GIVEN a delivery report @@ -110,7 +119,9 @@ def test_validate_report_empty_fields(report_api_mip_dna, case_mip_dna, caplog): assert "library_prep" in caplog.text -def test_validate_report_missing_fields(report_api_mip_dna, case_mip_dna, caplog): +def test_validate_report_missing_fields( + report_api_mip_dna: MipDNAReportAPI, case_mip_dna: Family, caplog: LogCaptureFixture +): """Tests the validations of empty required report fields.""" # GIVEN a delivery report @@ -134,7 +145,9 @@ def test_validate_report_missing_fields(report_api_mip_dna, case_mip_dna, caplog assert "accredited" in caplog.text -def test_get_validated_report_data_external_sample(report_api_mip_dna, case_mip_dna): +def test_get_validated_report_data_external_sample( + report_api_mip_dna: MipDNAReportAPI, case_mip_dna: Family +): """Tests report data retrieval.""" # GIVEN a delivery report with external sample data @@ -153,13 +166,13 @@ def test_get_validated_report_data_external_sample(report_api_mip_dna, case_mip_ assert report_data -def test_get_customer_data(report_api_mip_dna, case_mip_dna): +def test_get_customer_data(report_api_mip_dna: MipDNAReportAPI, case_mip_dna: Family): """Checks that the retrieved customer data is the expected one.""" # GIVEN a pre-built case # GIVEN an expected output - expected_customer = { + expected_customer: dict = { "name": "Production", "id": "cust000", "invoice_address": "Test street", @@ -173,7 +186,12 @@ def test_get_customer_data(report_api_mip_dna, case_mip_dna): assert customer_data == expected_customer -def test_get_report_version_version(report_api_mip_dna, store, helpers, timestamp_yesterday): +def test_get_report_version_version( + report_api_mip_dna: MipDNAReportAPI, + store: Store, + helpers: StoreHelpers, + timestamp_yesterday: datetime, +): """Validates the extracted report versions of two analyses.""" # GIVEN a specific set of analyses @@ -191,7 +209,12 @@ def test_get_report_version_version(report_api_mip_dna, store, helpers, timestam assert first_analysis_version == 1 -def test_get_case_data(report_api_mip_dna, mip_analysis_api, case_mip_dna, family_name): +def test_get_case_data( + report_api_mip_dna: MipDNAReportAPI, + mip_analysis_api: MipDNAAnalysisAPI, + case_mip_dna: Family, + family_name: str, +): """Tests the extracted case data.""" # GIVEN a pre-built case @@ -212,7 +235,11 @@ def test_get_case_data(report_api_mip_dna, mip_analysis_api, case_mip_dna, famil def test_get_samples_data( - report_api_mip_dna, mip_analysis_api, case_mip_dna, case_samples_data, lims_samples + report_api_mip_dna: MipDNAReportAPI, + mip_analysis_api: MipDNAAnalysisAPI, + case_mip_dna: Family, + case_samples_data: List[FamilySample], + lims_samples: List[dict], ): """Validates the retrieved sample data.""" @@ -242,7 +269,11 @@ def test_get_samples_data( assert samples_data.timestamps -def test_get_lims_sample(report_api_mip_dna, case_samples_data, lims_samples): +def test_get_lims_sample( + report_api_mip_dna: MipDNAReportAPI, + case_samples_data: List[FamilySample], + lims_samples: List[dict], +): """Tests lims data extraction.""" # GIVEN a family samples instance @@ -257,7 +288,11 @@ def test_get_lims_sample(report_api_mip_dna, case_samples_data, lims_samples): assert lims_data == expected_lims_data -def test_get_sample_application_data(report_api_mip_dna, case_samples_data, lims_samples): +def test_get_sample_application_data( + report_api_mip_dna: MipDNAReportAPI, + case_samples_data: List[FamilySample], + lims_samples: List[dict], +): """Tests sample application data extraction.""" # GIVEN a lims sample instance @@ -279,7 +314,9 @@ def test_get_sample_application_data(report_api_mip_dna, case_samples_data, lims assert application_data.accredited == expected_application_data.get("is_accredited") -def test_get_unique_applications(report_api_mip_dna, mip_analysis_api, case_mip_dna): +def test_get_unique_applications( + report_api_mip_dna: MipDNAReportAPI, mip_analysis_api: MipDNAAnalysisAPI, case_mip_dna: Family +): """Tests unique applications filtering.""" # GIVEN a list of samples sharing the same application @@ -295,7 +332,9 @@ def test_get_unique_applications(report_api_mip_dna, mip_analysis_api, case_mip_ assert len(unique_applications) == 1 -def test_get_sample_methods_data(report_api_mip_dna, case_samples_data): +def test_get_sample_methods_data( + report_api_mip_dna: MipDNAReportAPI, case_samples_data: List[FamilySample] +): """Tests sample methods retrieval from lims.""" # GIVEN a sample ID @@ -314,7 +353,9 @@ def test_get_sample_methods_data(report_api_mip_dna, case_samples_data): assert sample_methods == expected_sample_methods -def test_get_case_analysis_data(report_api_mip_dna, mip_analysis_api, case_mip_dna): +def test_get_case_analysis_data( + report_api_mip_dna: MipDNAReportAPI, mip_analysis_api: MipDNAAnalysisAPI, case_mip_dna: Family +): """Tests data analysis parameters retrieval.""" # GIVEN a pre-built case @@ -333,7 +374,11 @@ def test_get_case_analysis_data(report_api_mip_dna, mip_analysis_api, case_mip_d assert case_analysis_data.scout_files -def test_get_sample_timestamp_data(report_api_mip_dna, case_samples_data, timestamp_yesterday): +def test_get_sample_timestamp_data( + report_api_mip_dna: MipDNAReportAPI, + case_samples_data: List[FamilySample], + timestamp_yesterday: datetime, +): """Checks that the sample timestamp information is correctly retrieved from StatusDB.""" # GIVEN a mock sample data diff --git a/tests/mocks/limsmock.py b/tests/mocks/limsmock.py index f72e1ae48e..47b3e0d56a 100644 --- a/tests/mocks/limsmock.py +++ b/tests/mocks/limsmock.py @@ -1,7 +1,7 @@ from typing import List, Optional from cg.apps.lims import LimsAPI -from pydantic import BaseModel +from pydantic.v1 import BaseModel from typing_extensions import Literal diff --git a/tests/mocks/report.py b/tests/mocks/report.py index 7fa4a6e3ef..4f0e831f53 100644 --- a/tests/mocks/report.py +++ b/tests/mocks/report.py @@ -1,111 +1,96 @@ +"""Delivery report mock classes and methods.""" import logging from datetime import datetime from pathlib import Path -from typing import Union, Optional +from typing import Union, Optional, Dict + +from housekeeper.store.models import Version from cg.apps.coverage import ChanjoAPI -from cg.meta.workflow.mip_dna import MipDNAAnalysisAPI from cg.meta.report.mip_dna import MipDNAReportAPI from cg.meta.workflow.analysis import AnalysisAPI +from cg.meta.workflow.mip_dna import MipDNAAnalysisAPI from cg.models.cg_config import CGConfig LOG = logging.getLogger(__name__) class MockMipDNAAnalysisAPI(MipDNAAnalysisAPI): - """Mock MipDNAAnalysisAPI for CLI tests""" + """Mock MipDNAAnalysisAPI for CLI tests.""" def __init__(self, config: CGConfig): super().__init__(config) @property def root(self): - """docstring for root""" - return "/root/path" class MockHousekeeperMipDNAReportAPI(MipDNAReportAPI): - """Mock ReportAPI for CLI tests overwriting HK methods""" + """Mock ReportAPI for CLI tests overwriting Housekeeper methods.""" def __init__(self, config: CGConfig, analysis_api: AnalysisAPI): super().__init__(config, analysis_api) def add_delivery_report_to_hk( - self, delivery_report_file: Path, case_id: str, analysis_date: datetime - ): - """docstring for add_delivery_report_to_hk""" - + self, case_id: str, delivery_report_file: Path, version: Version + ) -> None: + """Mocked add_delivery_report_to_hk method.""" LOG.info( - "add_delivery_report_to_hk called with the following args: delivery_report_file=%s, case=%s, " - "analysis_date=%s", - delivery_report_file, - case_id, - analysis_date, + f"add_delivery_report_to_hk called with the following args: case={case_id}, delivery_report_file=" + f"{delivery_report_file}, version={version}" ) - def get_delivery_report_from_hk(self, case_id: str) -> str: - """docstring for get_delivery_report_from_hk""" - - LOG.info(f"get_delivery_report_from_hk called with the following args: case={case_id}") - - return "/path/to/delivery_report.html" - - def get_scout_uploaded_file_from_hk(self, case_id: str, scout_tag: str) -> Optional[str]: - """docstring for get_scout_uploaded_file_from_hk""" - + def get_delivery_report_from_hk(self, case_id: str, version: Version) -> None: + """Return mocked delivery report path stored in Housekeeper.""" LOG.info( - "add_delivery_report_to_hk called with the following args: case=%s, scout_tag=%s", - case_id, - scout_tag, + f"get_delivery_report_from_hk called with the following args: case={case_id}, version={version}" ) + return None + def get_scout_uploaded_file_from_hk(self, case_id: str, scout_tag: str) -> str: + """Return mocked uploaded to Scout file.""" + LOG.info( + f"get_scout_uploaded_file_from_hk called with the following args: case={case_id}, scout_tag={scout_tag}" + ) return f"path/to/{scout_tag}" class MockMipDNAReportAPI(MockHousekeeperMipDNAReportAPI): - """Mock ReportAPI for CLI tests""" + """Mock ReportAPI for CLI tests.""" def __init__(self, config: CGConfig, analysis_api: AnalysisAPI): super().__init__(config, analysis_api) - def create_delivery_report(self, case_id: str, analysis_date: datetime, force_report: bool): - """docstring for create_delivery_report""" - + def create_delivery_report( + self, case_id: str, analysis_date: datetime, force_report: bool + ) -> None: + """Mocked create_delivery_report method.""" LOG.info( - "create_delivery_report called with the following args: case=%s, analysis_date=%s, force_report=%s", - case_id, - analysis_date, - force_report, + f"create_delivery_report called with the following args: case={case_id}, analysis_date={analysis_date}, " + f"force_report={force_report}", ) def create_delivery_report_file( - self, case_id: str, file_path: Path, analysis_date: datetime, force_report: bool - ): - """docstring for create_delivery_report_file""" - + self, case_id: str, directory: Path, analysis_date: datetime, force_report: bool + ) -> Path: + """Return mocked delivery report file path.""" LOG.info( - "create_delivery_report_file called with the following args: case=%s, file_path=%s, analysis_date=%s, " - "force_report=%s", - case_id, - file_path, - analysis_date, - force_report, + f"create_delivery_report_file called with the following args: case={case_id}, directory={directory}, " + f"analysis_date={analysis_date}, force_report={force_report}" ) - - return file_path + return directory class MockChanjo(ChanjoAPI): - """Chanjo mock class""" + """Chanjo mocked class.""" def __init__(self): mock_config = {"chanjo": {"config_path": "/mock/path", "binary_path": "/mock/path"}} super().__init__(mock_config) - def sample_coverage(self, sample_id: str, panel_genes: list) -> Union[None, dict]: - """Calculates sample coverage for a specific panel""" - + def sample_coverage(self, sample_id: str, panel_genes: list) -> Optional[Dict[str, float]]: + """Return mocked sample dictionary.""" sample_coverage = None if sample_id == "ADM1": sample_coverage = {"mean_coverage": 38.342, "mean_completeness": 99.1} @@ -113,5 +98,4 @@ def sample_coverage(self, sample_id: str, panel_genes: list) -> Union[None, dict sample_coverage = {"mean_coverage": 37.342, "mean_completeness": 97.1} elif sample_id == "ADM3": sample_coverage = {"mean_coverage": 39.342, "mean_completeness": 98.1} - return sample_coverage diff --git a/tests/mocks/scout.py b/tests/mocks/scout.py index 16eb368647..2d4cccc1f7 100644 --- a/tests/mocks/scout.py +++ b/tests/mocks/scout.py @@ -2,7 +2,7 @@ import logging from pathlib import Path -from pydantic import BaseModel, validator +from pydantic.v1 import BaseModel, validator from typing import List from typing_extensions import Literal diff --git a/tests/models/nextflow/test_nextflow_deliver.py b/tests/models/nextflow/test_nextflow_deliver.py index 398ce26602..61f0ead9da 100644 --- a/tests/models/nextflow/test_nextflow_deliver.py +++ b/tests/models/nextflow/test_nextflow_deliver.py @@ -1,6 +1,6 @@ from typing import Dict, Optional -import pydantic +from pydantic.v1 import ValidationError as PydanticValidationError import pytest from cg.models.nextflow.deliverables import NextflowDeliverables @@ -30,7 +30,7 @@ def test_instantiate_nextflow_deliverables_with_empty_entry( Tests nextflow delivery object with empty entry. """ # WHEN instantiating a deliverables object with an empty entry THEN assert that it was successfully created - with pytest.raises(pydantic.ValidationError): + with pytest.raises(PydanticValidationError): NextflowDeliverables(deliverables=nextflow_deliverables_with_empty_entry) @@ -41,5 +41,5 @@ def test_instantiate_nextflow_deliverables_with_faulty_entry( Tests nextflow delivery object with empty entry. """ # WHEN instantiating a deliverables object with an empty entry THEN assert that it was successfully created - with pytest.raises(pydantic.ValidationError): + with pytest.raises(PydanticValidationError): NextflowDeliverables(deliverables=nextflow_deliverables_with_faulty_entry) diff --git a/tests/models/report/test_validators.py b/tests/models/report/test_validators.py index fefa09b454..d218f11100 100644 --- a/tests/models/report/test_validators.py +++ b/tests/models/report/test_validators.py @@ -1,7 +1,5 @@ """Tests delivery report models validators.""" -from cgmodels.cg.constants import Pipeline - -from cg.constants import NA_FIELD, YES_FIELD, REPORT_GENDER +from cg.constants import NA_FIELD, YES_FIELD, REPORT_GENDER, Pipeline from cg.constants.subject import Gender from cg.models.orders.constants import OrderType from cg.models.report.validators import ( diff --git a/tests/models/rnafusion/test_rnafusion_sample.py b/tests/models/rnafusion/test_rnafusion_sample.py index 512e9b017d..cdb0a9879b 100644 --- a/tests/models/rnafusion/test_rnafusion_sample.py +++ b/tests/models/rnafusion/test_rnafusion_sample.py @@ -1,6 +1,6 @@ from typing import List -import pydantic +from pydantic.v1 import ValidationError as PydanticValidationError import pytest from cg.models.rnafusion.rnafusion_sample import RnafusionSample @@ -43,7 +43,7 @@ def test_instantiate_rnafusion_sample_fastq_r1_r2_different_length( # WHEN instantiating a sample object THEN throws a ValidationError - with pytest.raises(pydantic.ValidationError): + with pytest.raises(PydanticValidationError): RnafusionSample( sample=rnafusion_sample, fastq_r1=rnafusion_fastq_r1, @@ -87,7 +87,7 @@ def test_instantiate_rnafusion_strandedness_not_acceptable( """ # WHEN instantiating a sample object THEN throws a ValidationError - with pytest.raises(pydantic.ValidationError): + with pytest.raises(PydanticValidationError): RnafusionSample( sample=rnafusion_sample, fastq_r1=rnafusion_fastq_r1,