Parser for Gatan DigitalMicrograph DM3/DM4, fixes #12

.github/workflows/pylint.yml

@@ -7,23 +7,24 @@ env:
 jobs:
   linting:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python_version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python 3.10
+        with:
+          fetch-depth: 0
+          submodules: recursive
+      - name: Set up Python ${{ matrix.python_version }}
         uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: ${{ matrix.python_version }}
       - name: Install dependencies
         run: |
-          git submodule sync --recursive
-          git submodule update --init --recursive --jobs=4
           curl -LsSf https://astral.sh/uv/install.sh | sh
-      - name: Install package
-        run: |
-          uv pip install --no-deps .
-      - name: Install dev requirements
-        run: |
-          uv pip install -r dev-requirements.txt
+          uv pip install ".[dev,docs]"
       - name: ruff check
         run: |
           ruff check src/pynxtools_em tests

src/pynxtools_em/concepts/mapping_functors_pint.py

@@ -503,7 +503,6 @@ def add_specific_metadata_pint(
                 )
             if functor_key.startswith("map_to_"):
                 dtype_key = functor_key.replace("map_to_", "")
-                print(f"dtype_key >>>> {dtype_key}")
                 if dtype_key in MAP_TO_DTYPES:
                     map_functor(
                         cfg[functor_key],

src/pynxtools_em/configurations/gatan_cfg.py

@@ -0,0 +1,110 @@
+#
+# Copyright The NOMAD Authors.
+#
+# This file is part of NOMAD. See https://nomad-lab.eu for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Dict mapping Gatan DigitalMicrograph custom schema instances on concepts in NXem."""
+
+from typing import Any, Dict
+
+from pynxtools_em.utils.pint_custom_unit_registry import ureg
+
+# be careful compared to Nion and other tech partners data for may have reversed order!
+# TODO:: confirming that this implementation is correct demands examples with dissimilar sized
+# rectangular, cubodial, and hypercuboidal stacks!
+GATAN_WHICH_SPECTRUM = {
+    "eV": ("spectrum_0d", ["axis_energy"]),
+    "eV_m": ("spectrum_1d", ["axis_energy", "axis_i"]),
+    "eV_m_m": ("spectrum_2d", ["axis_energy", "axis_i", "axis_j"]),
+}
+GATAN_WHICH_IMAGE = {
+    "m": ("image_1d", ["axis_i"]),
+    "1/m": ("image_1d", ["axis_i"]),
+    "m_m": ("image_2d", ["axis_i", "axis_j"]),
+    "1/m_1/m": ("image_2d", ["axis_i", "axis_j"]),
+}
+
+
+GATAN_DYNAMIC_VARIOUS_TO_NX_EM: Dict[str, Any] = {
+    "prefix_trg": "/ENTRY[entry*]/measurement/event_data_em_set/EVENT_DATA_EM[event_data_em*]/em_lab",
+    "prefix_src": "ImageList/TagGroup0/ImageTags/Microscope Info/",
+    "map_to_f8": [
+        (
+            "ebeam_column/electron_source/voltage",
+            ureg.kilovolt,
+            "Voltage",
+            ureg.volt,
+        ),  # volt?
+        (
+            "ebeam_column/electron_source/emission_current",
+            ureg.ampere,
+            "Emission Current (µA)",
+            ureg.microampere,
+        ),
+        # Formatted Voltage, HT Extrapolated
+        (
+            "ebeam_column/BEAM[beam]/diameter",
+            ureg.meter,
+            "Probe Size (nm)",
+            ureg.nanometer,
+        ),  # diameter? radius ?
+        (
+            "OPTICAL_SETUP_EM[optical_setup]/probe_current",
+            ureg.ampere,
+            "Probe Current (nA)",
+            ureg.nanoampere,
+        ),
+        (
+            "OPTICAL_SETUP_EM[optical_setup]/field_of_view",
+            ureg.meter,
+            "Field of View (µm)",
+            ureg.micrometer,
+        ),
+        ("OPTICAL_SETUP_EM[optical_setup]/magnification", "Actual Magnification"),
+        (
+            "OPTICAL_SETUP_EM[optical_setup]/camera_length",
+            ureg.meter,
+            "STEM Camera Length",
+            ureg.meter,
+        ),
+        # Cs(mm), Indicated Magnification, Magnification Interpolated, Formatted Actual Mag, Formatted Indicated Mag
+    ],
+    "map": [
+        ("OPTICAL_SETUP_EM[optical_setup]/illumination_mode", "Illumination Mode"),
+        (
+            "OPTICAL_SETUP_EM[optical_setup]/illumination_submode",
+            "Illumination Sub-mode",
+        ),
+        ("OPTICAL_SETUP_EM[optical_setup]/imaging_mode", "Imaging Mode"),
+        ("OPTICAL_SETUP_EM[optical_setup]/name", "Name"),
+        ("OPTICAL_SETUP_EM[optical_setup]/operation_mode", "Operation Mode"),
+        ("OPTICAL_SETUP_EM[optical_setup]/operation_mode_type", "Operation Mode Type"),
+    ],
+}
+
+GATAN_DYNAMIC_STAGE_TO_NX_EM: Dict[str, Any] = {
+    "prefix_trg": "/ENTRY[entry*]/measurement/event_data_em_set/EVENT_DATA_EM[event_data_em*]/em_lab/STAGE_LAB[stage]",
+    "prefix_src": "ImageList/TagGroup0/ImageTags/Microscope Info/Stage Position/",
+    "map_to_f8": [
+        ("tilt1", ureg.radian, "Stage Alpha", ureg.radian),
+        ("tilt2", ureg.radian, "Stage Beta", ureg.radian),
+        (
+            "position",
+            ureg.picometer,
+            ["Stage X", "Stage Y", "Stage Z"],
+            ureg.meter,
+        ),
+    ],
+}

src/pynxtools_em/configurations/nion_cfg.py

@@ -21,7 +21,7 @@
 
 from pynxtools_em.utils.pint_custom_unit_registry import ureg
 
-WHICH_SPECTRUM = {
+NION_WHICH_SPECTRUM = {
     "eV": ("spectrum_0d", ["axis_energy"]),
     "nm_eV": ("spectrum_1d", ["axis_i", "axis_energy"]),
     "nm_nm_eV": ("spectrum_2d", ["axis_j", "axis_i", "axis_energy"]),
@@ -37,7 +37,7 @@
         ["spectrum_identifier", "axis_k", "axis_j", "axis_i", "axis_energy"],
     ),
 }
-WHICH_IMAGE = {
+NION_WHICH_IMAGE = {
     "nm": ("image_1d", ["axis_i"]),
     "nm_nm": ("image_2d", ["axis_j", "axis_i"]),
     "nm_nm_nm": ("image_3d", ["axis_k", "axis_j", "axis_i"]),
@@ -48,6 +48,7 @@
         ["image_identifier", "axis_k", "axis_j", "axis_i"],
     ),
 }
+# TODO::use mapping to base_units like exemplified for the gatan parser
 
 
 MAG = "magnitude"

src/pynxtools_em/parsers/nxs_nion.py

@@ -41,15 +41,15 @@
     NION_PINPOINT_EVENT_TIME,
     NION_STATIC_DETECTOR_TO_NX_EM,
     NION_STATIC_LENS_TO_NX_EM,
-    WHICH_IMAGE,
-    WHICH_SPECTRUM,
+    NION_WHICH_IMAGE,
+    NION_WHICH_SPECTRUM,
 )
 from pynxtools_em.utils.get_file_checksum import (
     DEFAULT_CHECKSUM_ALGORITHM,
     get_sha256_of_file_content,
 )
 from pynxtools_em.utils.nion_utils import (
-    image_spectrum_or_generic_nxdata,
+    nion_image_spectrum_or_generic_nxdata,
     uuid_to_file_name,
 )
 from pynxtools_em.utils.pint_custom_unit_registry import ureg
@@ -88,7 +88,7 @@ def __init__(
 
     def check_if_nionswift_project(self):
         """Inspect the content of the compressed project file to check if supported."""
-        self.supported = False  # try to falsify
+        self.supported = False
         if self.file_path.endswith(".zip"):
             self.is_zipped = True
         elif self.file_path.endswith(".nsproj"):
@@ -425,7 +425,7 @@ def process_event_data_em_data(
     ) -> dict:
         """Map Nion-specifically formatted data arrays on NeXus NXdata/NXimage/NXspectrum."""
         axes = flat_metadata["dimensional_calibrations"]
-        unit_combination = image_spectrum_or_generic_nxdata(axes)
+        unit_combination = nion_image_spectrum_or_generic_nxdata(axes)
         print(f"{unit_combination}, {np.shape(nparr)}")
         print(axes)
         print(f"entry_id {self.entry_id}, event_id {self.event_id}")
@@ -439,18 +439,20 @@ def process_event_data_em_data(
         # return template
 
         axis_names = None
-        if unit_combination in WHICH_SPECTRUM:
-            trg = f"{prfx}/SPECTRUM_SET[spectrum_set1]/{WHICH_SPECTRUM[unit_combination][0]}"
+        if unit_combination in NION_WHICH_SPECTRUM:
+            trg = f"{prfx}/SPECTRUM_SET[spectrum_set1]/{NION_WHICH_SPECTRUM[unit_combination][0]}"
             template[f"{trg}/title"] = f"{flat_metadata['title']}"
             template[f"{trg}/@signal"] = f"intensity"
             template[f"{trg}/intensity"] = {"compress": nparr, "strength": 1}
-            axis_names = WHICH_SPECTRUM[unit_combination][1]
-        elif unit_combination in WHICH_IMAGE:
-            trg = f"{prfx}/IMAGE_SET[image_set1]/{WHICH_IMAGE[unit_combination][0]}"
+            axis_names = NION_WHICH_SPECTRUM[unit_combination][1]
+        elif unit_combination in NION_WHICH_IMAGE:
+            trg = (
+                f"{prfx}/IMAGE_SET[image_set1]/{NION_WHICH_IMAGE[unit_combination][0]}"
+            )
             template[f"{trg}/title"] = f"{flat_metadata['title']}"
             template[f"{trg}/@signal"] = f"real"  # TODO::unless COMPLEX
             template[f"{trg}/real"] = {"compress": nparr, "strength": 1}
-            axis_names = WHICH_IMAGE[unit_combination][1]
+            axis_names = NION_WHICH_IMAGE[unit_combination][1]
         elif not any(
             (value in ["1/", "iteration"]) for value in unit_combination.split(";")
         ):
@@ -491,11 +493,11 @@ def process_event_data_em_data(
                             np.float32,
                         )
                     )
-                    if unit_combination in WHICH_SPECTRUM:
+                    if unit_combination in NION_WHICH_SPECTRUM:
                         template[f"{trg}/AXISNAME[{axis_name}]/@long_name"] = (
                             f"Spectrum identifier"
                         )
-                    elif unit_combination in WHICH_IMAGE:
+                    elif unit_combination in NION_WHICH_IMAGE:
                         template[f"{trg}/AXISNAME[{axis_name}]/@long_name"] = (
                             f"Image identifier"
                         )
@@ -518,6 +520,8 @@ def process_event_data_em_data(
                         f"{ureg.Unit(units)}"
                     )
                     if units == "eV":
+                        # TODO::this is only robust if Nion reports always as eV and not with other prefix like kilo etc.
+                        # in such case the solution from the gatan parser is required, i.e. conversion to base units
                         template[f"{trg}/AXISNAME[{axis_name}]/@long_name"] = (
                             f"Energy ({ureg.Unit(units)})"  # eV
                         )