Add hdxms-datasets

pyhdx/__init__.py

@@ -4,9 +4,8 @@
     Coverage,
     HDXMeasurementSet,
 )
-from pyhdx.fileIO import read_dynamx
+from pyhdx.datasets import read_dynamx
 from pyhdx.fitting_torch import TorchFitResult, TorchFitResultSet
-from pyhdx.batch_processing import StateParser
 from pyhdx.__version__ import __version__
 
 VERSION_STRING = f"PyHDX {__version__}"

pyhdx/batch_processing.py

@@ -6,6 +6,7 @@
 from io import StringIO
 from pathlib import Path
 from typing import Union, Literal, Optional
+import warnings
 
 import pandas as pd
 
@@ -27,6 +28,9 @@ class DataFile(object):
 
     filepath_or_buffer: Union[Path, StringIO]
 
+    def __post_init__(self):
+        warnings.warn("Will be removed in favour of the `hdxms-datasets` package ", DeprecationWarning)
+
     @cached_property
     def data(self) -> pd.DataFrame:
         if self.format == "DynamX":
@@ -58,6 +62,8 @@ def __init__(
         # filter_kwargs: Optional[dict[str, Any]] = None,
         # correction_kwargs: Optional[dict[str, Any]] = None,
     ) -> None:
+
+        warnings.warn("Will be removed in favour of the `hdxms-datasets` package ", DeprecationWarning)
         self.hdx_spec = hdx_spec
         self.data_files: dict[str, DataFile] = {}
 
@@ -177,6 +183,7 @@ def batch_filter_peptides(
     Returns:
         Filtered dataframe.
     """
+    warnings.warn("Will be removed in favour of the `hdxms-datasets` package ", DeprecationWarning)
 
     if state is not None:
         df = df[df["state"] == state]
@@ -212,6 +219,7 @@ def batch_convert_time(
         Converted time value(s).
     """
 
+    warnings.warn("Will be removed in favour of the `hdxms-datasets` package ", DeprecationWarning)
     src_unit = time_dict["unit"]
 
     time_factor = time_factors[src_unit] / time_factors[target_unit]

pyhdx/config.py

@@ -1,8 +1,9 @@
 from __future__ import annotations
 
+from contextlib import contextmanager
 from os import PathLike
 from pathlib import Path
-from typing import Union, Dict, Any, Optional
+from typing import Union, Dict, Any, Optional, Generator
 
 import torch
 from omegaconf import OmegaConf, DictConfig, DictKeyType
@@ -107,6 +108,14 @@ def log_dir(self) -> Path:
 
         return log_dir
 
+    @property
+    def database_dir(self) -> Path:
+        """HDXMS-datasets database directory"""
+        spec_path = self.conf.server.database_dir
+        database_dir = Path(spec_path.replace("~", str(Path().home())))
+
+        return database_dir
+
     @property
     def TORCH_DTYPE(self) -> Union[torch.float64, torch.float32]:
         """PyTorch dtype used for ΔG calculations"""
@@ -124,6 +133,18 @@ def TORCH_DEVICE(self) -> torch.device:
         device = self.conf.fitting.device
         return torch.device(device)
 
+    @contextmanager
+    def context(self, settings: dict) -> Generator[PyHDXConfig, None, None]:
+        from pyhdx.support import rsetattr
+
+        original_config = self.conf.copy()
+
+        try:
+            for attr, value in settings.items():
+                rsetattr(cfg, attr, value)
+            yield cfg
+        finally:
+            cfg.conf = original_config
 
 def valid_config() -> bool:
     """Checks if the current config file in the user home directory is a valid config

pyhdx/config.yaml

@@ -5,6 +5,7 @@ cluster:
 server:
   assets_dir: ~/.pyhdx/assets
   log_dir: ~/.pyhdx/logs
+  database_dir : ~/.hdxms_datasets/datasets
 
 fitting:
   dtype: float64

pyhdx/datasets.py

@@ -0,0 +1 @@
+from hdxms_datasets import *

pyhdx/fileIO.py

@@ -6,14 +6,15 @@
 import re
 import shutil
 from datetime import datetime
-from io import StringIO
+from io import StringIO, BytesIO
 from pathlib import Path
-from typing import Union, Literal, Tuple, List, TextIO, Optional, TYPE_CHECKING, Any
+from typing import Union, Literal, Tuple, List, TextIO, Optional, TYPE_CHECKING, Any, BinaryIO
 from importlib import import_module
 import torch.nn as nn
 import torch as t
 import pandas as pd
 import yaml
+import warnings
 
 import pyhdx
 
@@ -43,6 +44,8 @@ def read_dynamx(
         Peptide table as a pandas DataFrame.
     """
 
+    warnings.warn("Will be removed in favour of the `hdxms-datasets` package ", DeprecationWarning)
+
     if isinstance(filepath_or_buffer, StringIO):
         hdr = filepath_or_buffer.readline().strip("# \n\t")
         filepath_or_buffer.seek(0)
@@ -64,7 +67,7 @@ def read_dynamx(
     return df
 
 
-def read_header(file_obj: TextIO, comment: str = "#") -> List[str]:
+def read_header(file_obj: Union[TextIO, BinaryIO], comment: str = "#") -> List[str]:
     header = []
 
     while True:
@@ -77,7 +80,7 @@ def read_header(file_obj: TextIO, comment: str = "#") -> List[str]:
     return header
 
 
-def parse_header(filepath_or_buffer: Union[Path[str], str, StringIO], comment: str = "#") -> dict:
+def parse_header(filepath_or_buffer: Union[Path[str], str, StringIO, BytesIO], comment: str = "#") -> dict:
     """
     Reads the header from a file and returns JSON metadata from header lines marked as comment.
 
@@ -89,7 +92,7 @@ def parse_header(filepath_or_buffer: Union[Path[str], str, StringIO], comment: s
         Dictionary of read metadata.
     """
 
-    if isinstance(filepath_or_buffer, StringIO):
+    if isinstance(filepath_or_buffer, (StringIO, BytesIO)):
         header = read_header(filepath_or_buffer, comment=comment)
         filepath_or_buffer.seek(0)
     else:

pyhdx/fitting.py

@@ -641,7 +641,7 @@ def fit_gibbs_global(
     optimizer="SGD",
     callbacks=None,
     **optimizer_kwargs,
-):
+) -> TorchFitResult:
     """
     Fit Gibbs free energies globally to all D-uptake data in the supplied hdxm
 

pyhdx/models.py

@@ -5,7 +5,7 @@
 import warnings
 from functools import partial
 from numbers import Number
-from typing import Optional, Any, Union
+from typing import Optional, Any, Union, TYPE_CHECKING
 
 import numpy as np
 import numpy.typing as npt
@@ -18,10 +18,13 @@
 
 from pyhdx.alignment import align_dataframes
 from pyhdx.fileIO import dataframe_to_file
-from pyhdx.process import verify_sequence, parse_temperature
+from pyhdx.process import verify_sequence, parse_temperature, correct_d_uptake, apply_control
 from pyhdx.support import reduce_inter, dataframe_intersection, array_intersection
 from pyhdx.config import cfg
 
+if TYPE_CHECKING:
+    from hdxms_datasets import HDXDataSet
+
 
 class Coverage:
     """
@@ -283,6 +286,46 @@ def __init__(self, data: pd.DataFrame, **metadata: Any):
             .sort_index(axis=1, level=0, sort_remaining=False)
         )
 
+    @classmethod
+    def from_dataset(cls, dataset: HDXDataSet, state: str | int, **metadata) -> HDXMeasurement:
+        """Create an HDXMeasurement object from a HDXDataSet object.
+
+        Args:
+            dataset: HDXDataSet object
+            state: State label or index for measurement in the dataset
+
+
+        Returns:
+            HDXMeasurement object.
+
+        """
+
+        state = dataset.states[state] if isinstance(state, int) else state
+        peptide_spec = dataset.hdx_spec["states"][state]["peptides"]
+
+        peptides = dataset.load_peptides(state, "experiment")
+        fd_peptides = (
+            dataset.load_peptides(state, "FD_control") if "FD_control" in peptide_spec else None
+        )
+        nd_peptides = (
+            dataset.load_peptides(state, "ND_control") if "ND_control" in peptide_spec else None
+        )
+
+        # take globally defined metadata and update with state specific metadata
+        spec_metadata = dataset.hdx_spec.get("metadata", {})
+        spec_metadata.update(dataset.hdx_spec["states"][state]["metadata"])
+
+        metadata = {**spec_metadata, **metadata}
+
+        peptides = apply_control(peptides, fd_peptides, nd_peptides)
+        peptides = correct_d_uptake(
+            peptides,
+            drop_first=cfg.analysis.drop_first,
+            d_percentage=metadata.get("d_percentage", 100.0),
+        )
+
+        return HDXMeasurement(peptides, name=state, **metadata)
+
     def __str__(self) -> str:
         """String representation of this HDX measurement object.
 
@@ -329,9 +372,7 @@ def temperature(self) -> Optional[float]:
             return temperature
         elif isinstance(temperature, dict):
             return parse_temperature(**temperature)
-
-        return self.metadata.get("temperature", None)
-
+
     @property
     def pH(self) -> Optional[float]:
         """pH of the H/D exchange reaction."""
@@ -765,6 +806,12 @@ def __iter__(self):
     def __getitem__(self, item: int) -> HDXMeasurement:
         return self.hdxm_list.__getitem__(item)
 
+    @classmethod
+    def from_dataset(self, dataset: HDXDataSet, **metadata) -> HDXMeasurementSet:
+        hdxm_list = [HDXMeasurement.from_dataset(dataset, state, **metadata) for state in dataset.states]
+
+        return HDXMeasurementSet(hdxm_list)
+
     def get(self, name: str) -> HDXMeasurement:
         """
         Get HDXMeasurement object by name.

pyhdx/plot.py

@@ -615,13 +615,19 @@ def linear_bars(
     sort=False,
     **figure_kwargs,
 ):
+
+    # input data should always be 3 levels
+    # grouping is done by the first level
+    # second level gives each bar
+    # third level should have columns with the specified 'field'
     if data.columns.nlevels == 2:
         data = data.copy()
         columns = pd.MultiIndex.from_tuples(
             [("", *tup) for tup in data.columns], names=["group"] + data.columns.names
         )
         data.columns = columns
 
+    # todo this should be done by the 'user'
     data = data.xs(level=-1, key=field, drop_level=False, axis=1)
 
     groupby = groupby or data.columns.names[0]

pyhdx/process.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from functools import reduce
+import warnings
 from typing import Optional, Literal, Union
 
 import pandas as pd
@@ -254,6 +255,7 @@ def filter_peptides(
 
     """
 
+    warnings.warn("`filter_peptides` will be moved to the `hdxms-datasets` package", DeprecationWarning)
     if state:
         df = df[df["state"] == state]
 

pyhdx/support.py

@@ -847,3 +847,15 @@ def array_intersection(arrays: Iterable[np.ndarray], fields: Iterable[str]) -> l
     selected = [elem[np.isin(fields_view(elem, fields), intersection)] for elem in arrays]
 
     return selected
+
+# https://stackoverflow.com/questions/31174295/getattr-and-setattr-on-nested-subobjects-chained-properties
+def rsetattr(obj, attr, val):
+    pre, _, post = attr.rpartition(".")
+    return setattr(rgetattr(obj, pre) if pre else obj, post, val)
+
+
+def rgetattr(obj, attr, *args):
+    def _getattr(obj, attr):
+        return getattr(obj, attr, *args)
+
+    return reduce(_getattr, [obj] + attr.split("."))