Merge master

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 38.3.0
+current_version = 39.0.0
 commit = True
 tag = True
 tag_name = v{new_version}

cg/__init__.py

@@ -1,4 +1,4 @@
 import pkg_resources
 
 __title__ = "cg"
-__version__ = "38.3.0"
+__version__ = "39.0.0"

cg/apps/cgstats/db/models/demux_sample.py

@@ -1,5 +1,5 @@
 import logging
-from pydantic import BaseModel, validator
+from pydantic.v1 import BaseModel, validator
 
 from cg.apps.cgstats.parsers.conversion_stats import SampleConversionResults
 from cg.apps.cgstats.parsers.demux_stats import SampleBarcodeStats

cg/apps/cgstats/db/models/dragen_demux_sample.py

@@ -3,7 +3,7 @@
 
 import logging
 
-from pydantic import BaseModel
+from pydantic.v1 import BaseModel
 
 LOG = logging.getLogger(__name__)
 

cg/apps/cgstats/parsers/conversion_stats.py

@@ -27,7 +27,7 @@
 from typing import Dict, List, Optional, Set
 from xml.etree.ElementTree import Element, iterparse
 
-from pydantic import BaseModel
+from pydantic.v1 import BaseModel
 
 LOG = logging.getLogger(__name__)
 

cg/apps/cgstats/parsers/demux_stats.py

@@ -4,7 +4,7 @@
 from typing import Dict, Optional, Set
 from xml.etree.ElementTree import Element, iterparse
 
-from pydantic import BaseModel
+from pydantic.v1 import BaseModel
 
 LOG = logging.getLogger(__name__)
 

cg/apps/cgstats/parsers/dragen_demultiplexing_stats.py

@@ -4,7 +4,7 @@
 from pathlib import Path
 from typing import Dict
 
-from pydantic import BaseModel
+from pydantic.v1 import BaseModel
 
 LOG = logging.getLogger(__name__)
 

cg/apps/crunchy/crunchy.py

@@ -10,9 +10,8 @@
 from pathlib import Path
 from typing import Dict, Optional
 
-from cgmodels.crunchy.metadata import CrunchyFile, CrunchyMetadata
-
 from cg.apps.crunchy import files
+from cg.apps.crunchy.models import CrunchyFile, CrunchyMetadata
 from cg.apps.slurm.slurm_api import SlurmAPI
 from cg.constants import FASTQ_DELTA
 from cg.models import CompressionData

cg/apps/crunchy/files.py

@@ -5,10 +5,10 @@
 from pathlib import Path
 from typing import Dict, List, Optional
 
+from cg.apps.crunchy.models import CrunchyFile, CrunchyMetadata
 from cg.constants.constants import FileFormat
 from cg.io.controller import ReadFile, ReadStream, WriteFile
 from cg.utils.date import get_date
-from cgmodels.crunchy.metadata import CrunchyFile, CrunchyMetadata
 
 LOG = logging.getLogger(__name__)
 

cg/apps/crunchy/models.py

@@ -0,0 +1,17 @@
+from datetime import date
+from typing import Optional
+
+from pydantic.v1 import BaseModel, conlist
+from typing_extensions import Literal
+
+
+class CrunchyFile(BaseModel):
+    path: str
+    file: Literal["first_read", "second_read", "spring"]
+    checksum: Optional[str]
+    algorithm: Literal["sha1", "md5", "sha256"] = None
+    updated: date = None
+
+
+class CrunchyMetadata(BaseModel):
+    files: conlist(CrunchyFile, max_items=3, min_items=3)

cg/apps/demultiplex/demultiplex_api.py

@@ -139,7 +139,7 @@ def is_demultiplexing_completed(self, flow_cell: FlowCellDirectoryData) -> bool:
         LOG.info(f"Check if demultiplexing is ready for {flow_cell.path}")
         logfile: Path = self.get_stderr_logfile(flow_cell)
         if not logfile.exists():
-            LOG.warning("Could not find logfile!")
+            LOG.warning(f"Could not find logfile: {logfile}!")
             return False
         return self.demultiplexing_completed_path(flow_cell).exists()
 

cg/apps/demultiplex/demux_report.py

@@ -6,7 +6,7 @@
     SampleConversionResults,
     UnknownBarcode,
 )
-from pydantic import BaseModel
+from pydantic.v1 import BaseModel
 from typing_extensions import Literal
 
 LOG = logging.getLogger(__name__)

cg/apps/demultiplex/sample_sheet/index.py

@@ -10,7 +10,7 @@
 from cg.resources import VALID_INDEXES_PATH
 from cg.utils.utils import get_hamming_distance
 from packaging import version
-from pydantic import BaseModel
+from pydantic.v1 import BaseModel
 
 LOG = logging.getLogger(__name__)
 DNA_COMPLEMENTS: Dict[str, str] = {"A": "T", "C": "G", "G": "C", "T": "A"}

cg/apps/demultiplex/sample_sheet/models.py

@@ -1,5 +1,5 @@
 import logging
-from pydantic import BaseModel, Extra, Field
+from pydantic.v1 import BaseModel, Extra, Field
 from typing import List
 
 from cg.constants.constants import GenomeVersion

cg/apps/demultiplex/sample_sheet/read_sample_sheet.py

@@ -1,7 +1,7 @@
 import logging
 from pathlib import Path
 from typing import Dict, List, Type
-from pydantic import parse_obj_as
+from pydantic.v1 import parse_obj_as
 
 from cg.apps.demultiplex.sample_sheet.models import FlowCellSample, SampleSheet
 from cg.constants.constants import FileFormat

cg/apps/hermes/models.py

@@ -3,7 +3,7 @@
 from pathlib import Path
 from typing import List, Optional
 
-from pydantic import BaseModel, validator
+from pydantic.v1 import BaseModel, validator
 
 LOG = logging.getLogger(__name__)
 

cg/apps/housekeeper/models.py

@@ -2,7 +2,7 @@
 from datetime import datetime
 from typing import List, Optional
 
-from pydantic import BaseModel
+from pydantic.v1 import BaseModel
 
 
 class InputFile(BaseModel):

cg/apps/orderform/excel_orderform_parser.py

@@ -6,7 +6,7 @@
 from openpyxl.cell.cell import Cell
 from openpyxl.workbook import Workbook
 from openpyxl.worksheet.worksheet import Worksheet
-from pydantic import parse_obj_as
+from pydantic.v1 import parse_obj_as
 
 from cg.apps.orderform.orderform_parser import OrderformParser
 from cg.constants import DataDelivery

cg/apps/orderform/orderform_parser.py

@@ -2,7 +2,7 @@
 from pathlib import Path
 from typing import Dict, List, Optional, Set, Hashable, Iterable
 
-from pydantic import constr, BaseModel
+from pydantic.v1 import constr, BaseModel
 
 from cg.constants import DataDelivery
 from cg.exc import OrderFormError

cg/apps/scout/scout_export.py

@@ -3,7 +3,7 @@
 from datetime import datetime
 from typing import List, Optional, Dict
 
-from pydantic import BaseModel, Field, validator
+from pydantic.v1 import BaseModel, Field, validator
 from typing_extensions import Literal
 
 from cg.constants.gene_panel import GENOME_BUILD_37

cg/apps/sequencing_metrics_parser/models/bcl2fastq_metrics.py

@@ -1,5 +1,5 @@
 from typing import List, Dict
-from pydantic import BaseModel, Field, validator
+from pydantic.v1 import BaseModel, Field, validator
 
 
 class IndexMetric(BaseModel):

cg/apps/sequencing_metrics_parser/models/bcl_convert.py

@@ -1,5 +1,4 @@
-from pydantic import BaseModel, BaseConfig, Field
-import xml.etree.ElementTree as ET
+from pydantic.v1 import BaseModel, BaseConfig, Field
 from cg.constants.bcl_convert_metrics import (
     BclConvertQualityMetricsColumnNames,
     BclConvertDemuxMetricsColumnNames,

cg/apps/sequencing_metrics_parser/parsers/bcl_convert_to_sequencing_statistics.py

@@ -25,7 +25,7 @@ def create_sample_lane_sequencing_metrics_from_bcl_convert_metrics_for_flow_cell
                     sample_internal_id=sample_internal_id,
                     flow_cell_name=FlowCellDirectoryData(
                         flow_cell_path=flow_cell_dir, bcl_converter=BclConverter.BCLCONVERT
-                    ).flow_cell_name,
+                    ).id,
                     flow_cell_lane_number=lane,
                     sample_total_reads_in_lane=metrics_parser.calculate_total_reads_for_sample_in_lane(
                         sample_internal_id=sample_internal_id, lane=lane

cg/apps/tb/models.py

@@ -3,7 +3,7 @@
 from typing import Optional
 
 from dateutil.parser import parse as parse_datestr
-from pydantic import BaseModel, validator
+from pydantic.v1 import BaseModel, validator
 
 
 class TrailblazerAnalysis(BaseModel):

cg/cli/demultiplex/finish.py

@@ -32,6 +32,11 @@ def finish_all_cmd(context: CGConfig, bcl_converter: str, dry_run: bool) -> None
     demux_post_processing_api.set_dry_run(dry_run=dry_run)
     demux_post_processing_api.finish_all_flow_cells(bcl_converter=bcl_converter)
 
+    # Temporary finish flow cell logic will replace logic above when validated
+    demux_post_processing_api_temp: DemuxPostProcessingAPI = DemuxPostProcessingAPI(config=context)
+    demux_post_processing_api_temp.set_dry_run(dry_run=dry_run)
+    demux_post_processing_api_temp.finish_all_flow_cells_temp()
+
 
 @finish_group.command(name="flow-cell")
 @click.argument("flow-cell-name")
@@ -54,6 +59,10 @@ def finish_flow_cell(
     demux_post_processing_api.finish_flow_cell(
         flow_cell_name=flow_cell_name, force=force, bcl_converter=bcl_converter
     )
+    # Temporary finish flow cell logic will replace logic above when validated
+    demux_post_processing_api_temp: DemuxPostProcessingAPI = DemuxPostProcessingAPI(config=context)
+    demux_post_processing_api_temp.set_dry_run(dry_run)
+    demux_post_processing_api_temp.finish_flow_cell_temp(flow_cell_directory_name=flow_cell_name)
 
 
 @finish_group.command(name="temporary")
@@ -66,6 +75,16 @@ def finish_flow_cell_temporary(context: CGConfig, flow_cell_directory_name: str)
     )
 
 
+@finish_group.command(name="temporary-all")
+@click.pass_obj
+@DRY_RUN
+def finish_flow_cell_temporary_all(context: CGConfig, dry_run: bool):
+    # Temporary finish flow cell logic will replace logic above when validated
+    demux_post_processing_api_temp: DemuxPostProcessingAPI = DemuxPostProcessingAPI(config=context)
+    demux_post_processing_api_temp.set_dry_run(dry_run=dry_run)
+    demux_post_processing_api_temp.finish_all_flow_cells_temp()
+
+
 @finish_group.command(name="all-hiseq-x")
 @DRY_RUN
 @click.pass_obj

cg/cli/demultiplex/sample_sheet.py

@@ -14,7 +14,7 @@
 from cg.io.controller import WriteFile, WriteStream
 from cg.models.cg_config import CGConfig
 from cg.models.demultiplex.flow_cell import FlowCellDirectoryData
-from pydantic import ValidationError
+from pydantic.v1 import ValidationError
 
 LOG = logging.getLogger(__name__)
 

cg/cli/generate/report/base.py

@@ -1,33 +1,30 @@
-"""Commands to generate delivery reports"""
-
+"""Commands to generate delivery reports."""
 import logging
 import sys
 from datetime import datetime
 from pathlib import Path
-from typing import TextIO, Optional
+from typing import Optional
 
 import click
+from housekeeper.store.models import Version
 
-from cg.store.models import Family
-from cgmodels.cg.constants import Pipeline
-from housekeeper.store import models as hk_models
-
-from cg.cli.generate.report.utils import (
-    get_report_case,
-    get_report_api,
-    get_report_analysis_started,
-    get_report_api_pipeline,
-)
 from cg.cli.generate.report.options import (
     ARGUMENT_CASE_ID,
     OPTION_FORCE_REPORT,
     OPTION_DRY_RUN,
     OPTION_STARTED_AT,
     OPTION_PIPELINE,
 )
-from cg.constants import EXIT_SUCCESS, EXIT_FAIL
+from cg.cli.generate.report.utils import (
+    get_report_case,
+    get_report_api,
+    get_report_analysis_started,
+    get_report_api_pipeline,
+)
+from cg.constants import EXIT_SUCCESS, EXIT_FAIL, Pipeline
 from cg.exc import CgError
 from cg.meta.report.report_api import ReportAPI
+from cg.store.models import Family
 
 LOG = logging.getLogger(__name__)
 
@@ -44,11 +41,9 @@ def generate_delivery_report(
     force_report: bool,
     dry_run: bool,
     analysis_started_at: str = None,
-):
+) -> None:
     """Creates a delivery report for the provided case."""
-
     click.echo(click.style("--------------- DELIVERY REPORT ---------------"))
-
     case: Family = get_report_case(context, case_id)
     report_api: ReportAPI = get_report_api(context, case)
     analysis_date: datetime = get_report_analysis_started(case, report_api, analysis_started_at)
@@ -61,22 +56,32 @@ def generate_delivery_report(
         click.echo(delivery_report_html)
         return
 
-    delivery_report_file: TextIO = report_api.create_delivery_report_file(
-        case_id,
-        file_path=Path(report_api.analysis_api.root, case_id),
+    version: Version = report_api.housekeeper_api.version(bundle=case_id, date=analysis_date)
+    delivery_report: Optional[str] = report_api.get_delivery_report_from_hk(
+        case_id=case_id, version=version
+    )
+    if delivery_report:
+        click.echo(
+            click.style(f"Delivery report already in housekeeper: {delivery_report}", fg="yellow")
+        )
+        return
+
+    created_delivery_report: Path = report_api.create_delivery_report_file(
+        case_id=case_id,
+        directory=Path(report_api.analysis_api.root, case_id),
         analysis_date=analysis_date,
         force_report=force_report,
     )
-
-    hk_report_file: Optional[hk_models.File] = report_api.add_delivery_report_to_hk(
-        delivery_report_file, case_id, analysis_date
+    report_api.add_delivery_report_to_hk(
+        case_id=case_id, delivery_report_file=created_delivery_report, version=version
     )
-
-    if hk_report_file:
-        click.echo(click.style("Uploaded delivery report to housekeeper", fg="green"))
-        report_api.update_delivery_report_date(case, analysis_date)
-    else:
-        click.echo(click.style("Delivery report already uploaded to housekeeper", fg="yellow"))
+    click.echo(
+        click.style(
+            f"Uploaded delivery report to housekeeper: {created_delivery_report.as_posix()}",
+            fg="green",
+        )
+    )
+    report_api.update_delivery_report_date(case=case, analysis_date=analysis_date)
 
 
 @click.command("available-delivery-reports")
@@ -86,7 +91,7 @@ def generate_delivery_report(
 @click.pass_context
 def generate_available_delivery_reports(
     context: click.Context, pipeline: Pipeline, force_report: bool, dry_run: bool
-):
+) -> None:
     """Generates delivery reports for all cases that need one and stores them in housekeeper."""
 
     click.echo(click.style("--------------- AVAILABLE DELIVERY REPORTS ---------------"))