FAIRmat-NFDI · mkuehbach · Aug 26, 2024 · Aug 8, 2024 · Aug 8, 2024 · Aug 9, 2024
diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml
@@ -7,23 +7,24 @@ env:
 jobs:
   linting:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python_version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python 3.10
+        with:
+          fetch-depth: 0
+          submodules: recursive
+      - name: Set up Python ${{ matrix.python_version }}
         uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: ${{ matrix.python_version }}
       - name: Install dependencies
         run: |
-          git submodule sync --recursive
-          git submodule update --init --recursive --jobs=4
           curl -LsSf https://astral.sh/uv/install.sh | sh
-      - name: Install package
-        run: |
-          uv pip install --no-deps .
-      - name: Install dev requirements
-        run: |
-          uv pip install -r dev-requirements.txt
+          uv pip install ".[dev,docs]"
       - name: ruff check
         run: |
           ruff check src/pynxtools_em tests

diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -11,14 +11,51 @@
             "cwd": "${workspaceFolder}",
             "program": "../.py3.12.4/bin/dataconverter",
             "args": ["convert",
-                     // "examples/eln_data.yaml",
-                     // "examples/em.oasis.specific.yaml",
-                     "../tescan/CZ04-2_102_Pic_2.tif",
+                     // eln_data/oasis_specific
+                     //"examples/em.conventions.yaml",
+                     //"examples/eln_data.yaml",
+                     //"examples/em.oasis.specific.yaml",
+                     //"../axon/ReductionOfFeOxSmall.zip",
+                     //"../digitalmicrograph/218.ebdf722abcf63000e2fa71fc6a72cd4b4747991c702a2dc65e400196f990cbad.dm3",
+                     //"../digitalmicrograph/241.a2c338fd458e6b7023ec946a5e3ce8c85bd2befcb5d17dae7ae5f44b2dede81b.dm4",
+                     //"../hitachi/360.tif",
+                     //"../hitachi/360.txt",
+                     //"../hitachi/TemHitachiImageScExample.txt",
+                     //"../jeol/20240227_A1_2m_0_FA3_1.tif",
+                     //"../jeol/20240227_A1_2m_0_FA3_1.txt",
+                     //"../nion-parsing/2022-02-18_Metadata_Kuehbach.zip",
+                     //"../point/Defekt1.tif",
+                     //"../tescan/Ti3C2VacDriedFilm19.tif",
+                     //"../tescan/Ti3C2VacDriedFilm19.hdr",
+                     //"../tescan/CZ04-2_102_Pic_2.tif",
+                     //"zeiss/SE2.tif",
+                     //"tfs/0c8nA_3deg_003_AplusB_test.tif",
+                     //"tfs/ALN_baoh_021.tif",
+                     //"tfs/ETD_image.tif",
+                     //"tfs/NavCam_normal_vis_light_ccd.tif",
+                     //"tfs/T3_image.tif",
+                     //"velox/CG71113 1138 Ceta 660 mm Camera.emd",
+                     //"../apex/InGaN_nanowires_linescan.edaxh5",
+                     //"../apex/InGaN_nanowires_map.edaxh5",
+                     //"../apex/InGaN_nanowires_spectra.edaxh5",
+                     //"../apex/2023-08-16_Ni_NFDI.edaxh5",
                      "--reader",
                      "em",
                      "--nxdl",
                      "NXem",
-                     "--output=dbg/dbg.nxs"],
+                     //"--output=dbg/conv.nxs",
+                     //"--output=dbg/eln.nxs",
+                     //"--output=dbg/axon.nxs",
+                     //"--output=dbg/gatan_dm3.nxs",
+                     //"--output=dbg/gatan_dm4.nxs",
+                     //"--output=dbg/hitachi.nxs",
+                     //"--output=dbg/jeol.nxs",
+                     //"--output=dbg/nion.nxs",
+                     //"--output=dbg/ebic.nxs",
+                     //"--output=dbg/tescan.nxs",
+                     //"--output=pynxtools_em/dbg/zeiss.nxs",
+                     //"--output=pynxtools_em/dbg/tfs.nxs",
+                     //"--output=dbg/apex.nxs"],
         }
     ]
 }
diff --git a/dev-requirements.txt b/dev-requirements.txt
@@ -564,7 +564,7 @@ pymdown-extensions==10.9
     # via
     #   pynxtools-em (pyproject.toml)
     #   mkdocs-material
-pynxtools @ git+https://github.com/FAIRmat-NFDI/pynxtools@622c580c75f9bcc4b5c2de57884f389864614818
+pynxtools @ git+https://github.com/FAIRmat-NFDI/pynxtools@9c46cc99a86736dcba137ed2ed9b19078372f567
     # via pynxtools-em (pyproject.toml)
 pyparsing==3.1.2
     # via matplotlib

diff --git a/dev/test_vars.ipynb b/dev/test_vars.ipynb
@@ -0,0 +1,127 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "815e2bd0-2ff4-424c-9512-a5cc60bf320a",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "| Concept | NeXus/HDF5 |\n",
+      "| --------------- | --------------  |\n",
+      "| Device | :heavy_check_mark: |\n",
+      "| EmissionCurrent | :heavy_check_mark: |\n",
+      "| HV | :heavy_check_mark: |\n",
+      "| Magnification | :heavy_check_mark: |\n",
+      "| PredictedBeamCurrent | :heavy_check_mark: |\n",
+      "| SerialNumber | :heavy_check_mark: |\n",
+      "| SpecimenCurrent | :heavy_check_mark: |\n",
+      "| SpotSize | :heavy_check_mark: |\n",
+      "| StageRotation | :heavy_check_mark: |\n",
+      "| StageTilt | :heavy_check_mark: |\n",
+      "| StageX | :heavy_check_mark: |\n",
+      "| StageY | :heavy_check_mark: |\n",
+      "| StageZ | :heavy_check_mark: |\n",
+      "| StigmatorX | :heavy_check_mark: |\n",
+      "| StigmatorY | :heavy_check_mark: |\n",
+      "| WD | :heavy_check_mark: |\n"
+     ]
+    }
+   ],
+   "source": [
+    "import pynxtools_em.configurations.image_tiff_tescan_cfg as tescan_cfg\n",
+    "import pynxtools_em.configurations.nion_cfg as nion_cfg\n",
+    "from pynxtools_em.concepts.mapping_functors_pint import get_case\n",
+    "thisone = {\"NION\": nion_cfg,\n",
+    "            \"TESCAN\": tescan_cfg}\n",
+    "\n",
+    "tech_partner = \"TESCAN\"\n",
+    "a = vars(thisone[tech_partner])\n",
+    "src_concepts = set()\n",
+    "verbose = False\n",
+    "for key, obj in a.items():\n",
+    "    if key.startswith(tech_partner):\n",
+    "        if verbose:\n",
+    "            print(key)\n",
+    "        prefix_src = []\n",
+    "        concepts = []\n",
+    "        for k, v in obj.items():\n",
+    "            if not k.startswith(\"prefix\"):\n",
+    "                if k != \"use\":\n",
+    "                    if isinstance(v, list):\n",
+    "                        for entry in v:\n",
+    "                            case = get_case(entry)\n",
+    "                            if verbose:\n",
+    "                                print(f\"{entry}\")\n",
+    "                                print(f\">>>>{get_case(entry)}\")\n",
+    "                            if case == \"case_one\":\n",
+    "                                concepts.append(entry)\n",
+    "                            if case == \"case_two_str\":     \n",
+    "                                concepts.append(entry[1])\n",
+    "                            elif case == \"case_five_str\":\n",
+    "                                concepts.append(entry[2])\n",
+    "                            # TODO::add remaining cases\n",
+    "                            elif case == \"case_five_list\":\n",
+    "                                for val in entry[2]:\n",
+    "                                    concepts.append(val)\n",
+    "            elif k == \"prefix_src\":\n",
+    "                if isinstance(v, str):\n",
+    "                    prefix_src.append(v)\n",
+    "                elif isinstance(v, list):\n",
+    "                    for val in v:\n",
+    "                        prefix_src.append(val)\n",
+    "        for prefix in prefix_src:\n",
+    "            for concept in concepts:\n",
+    "                src_concepts.add(f\"{prefix}{concept}\")\n",
+    "\n",
+    "print(\"| Concept | NeXus/HDF5 |\")\n",
+    "print(\"| --------------- | --------------  |\")\n",
+    "for src_concept in sorted(src_concepts):\n",
+    "    print(f\"| {src_concept} | :heavy_check_mark: |\")\n",
+    "        # print(obj)\n",
+    "        # get case\n",
+    "        # fish all src concepts to build sorted list of {prefix_src}/src for all {prefix_src}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b753eec4-0bea-4e39-80c6-520f5e00bd9f",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e0962859-5aef-49d8-8832-f9fb7a30921e",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/docs/how-tos/kikuchi.md b/docs/how-tos/kikuchi.md
@@ -5,7 +5,7 @@ The following diagram shows a comprehensive example how diverse datasets from Ki
 
 <img src="media/kikuchi.png" />
 
-[Further details to the parsing route via MTex](mtex.md)
-[Further details to the parsing route via pyxem](pyxem.md)
-[Further details to the NXem_ebsd base class](https://fairmat-nfdi.github.io/nexus_definitions/classes/contributed_definitions/NXem_ebsd.html#nxem-ebsd)
-[Further details to  the NXem application definition](https://fairmat-nfdi.github.io/nexus_definitions/classes/contributed_definitions/NXem.html#nxem)
+- [Further details to the parsing route via MTex](mtex.md)
+- [Further details to the parsing route via pyxem](pyxem.md)
+- [Further details to the NXem_ebsd base class](https://fairmat-nfdi.github.io/nexus_definitions/classes/contributed_definitions/NXem_ebsd.html#nxem-ebsd)
+- [Further details to  the NXem application definition](https://fairmat-nfdi.github.io/nexus_definitions/classes/contributed_definitions/NXem.html#nxem)
diff --git a/docs/index.md b/docs/index.md
@@ -55,11 +55,23 @@ Here you can learn which specific pieces of information and concepts pynxtools-e
 for the respective file formats of technology partners of the electron microscopy community.
 
 - [How to map pieces of information to NeXus](reference/contextualization.md)
-- [Tagged Image File Format (TIFF)](reference/tiff.md)
-- [Portable Network Graphics (PNG)](reference/png.md)
-- [Velox EMD](reference/velox.md)
+
+- [Conventions collected with a text file or ELN](reference/conventions.md)
+- [Metadata collected with an ELN and RDM-specific configurations](reference/eln_and_cfg.md)
+
+- [AXON Protochips Portable Network Graphics PNG](reference/png_axon.md)
 - [EDAX APEX](reference/apex.md)
-- [Nion Co. projects](reference/nion.md)
+- [Gatan DigitalMicrograph DM3/DM4](reference/gatan.md)
+- [Hitachi Tagged Image File Format TIFF](reference/tiff_hitachi.md)
+- [JEOL Tagged Image File Format TIFF](reference/tiff_jeol.md)
+- [Nion Co. projects with NDATA and HDF5 files](reference/nion.md)
+- [Point Electronic DISS Tagged Image File Format TIFF](reference/tiff_point.md)
+- [TESCAN Tagged Image File Format TIFF](reference/tiff_tescan.md)
+- [ThermoFisher Tagged Image File Format TIFF](reference/tiff_tfs.md)
+- [ThermoFisher Velox EMD](reference/velox.md)
+- [Zeiss Tagged Image File Format TIFF](reference/tiff_zeiss.md)
+- [EBSD-centric content for the parsing route via MTex](how-tos/mtex.md)
+- [EBSD-centric content for the parsing route via pyxem](how-tos/pyxem.md)
 
 </div>
 </div>

diff --git a/docs/reference/conventions.md b/docs/reference/conventions.md
@@ -0,0 +1,3 @@
+# Conventions collected with a text file or ELN
+
+<!--The pynxtools-em parser and normalizer reads the following content and maps them on respective NeXus concepts that are defined in the NXem application definition:-->
diff --git a/docs/reference/eln_and_cfg.md b/docs/reference/eln_and_cfg.md
@@ -0,0 +1,3 @@
+# Metadata collected with an ELN and RDM-specific configurations
+
+<!--The pynxtools-em parser and normalizer reads the following content and maps them on respective NeXus concepts that are defined in the NXem application definition:-->
diff --git a/docs/reference/gatan.md b/docs/reference/gatan.md
@@ -0,0 +1,9 @@
+# Gatan DigitalMicrograph DM3/DM4
+
+The pynxtools-em parser and normalizer reads the following content and maps them on respective NeXus concepts that are defined in the NXem application definition:
+
+<!--| Velox | NeXus/HDF5 |
+| --------------- | --------------  |
+| Reconstructed positions (x, y, z) | :heavy_check_mark: |
+| Mass-to-charge-state-ratio values (m/q) | :heavy_check_mark: |-->
+