Add native SolutionArray / HDF support to C++ OneDim

.github/workflows/main.yml

@@ -28,6 +28,9 @@ jobs:
         python-version: ['3.8', '3.10', '3.11']
         os: ['ubuntu-20.04', 'ubuntu-22.04']
       fail-fast: false
+    env:
+      HDF5_LIBDIR: /usr/lib/x86_64-linux-gnu/hdf5/serial
+      HDF5_INCLUDEDIR: /usr/include/hdf5/serial
     steps:
     - uses: actions/checkout@v2
       name: Checkout the repository
@@ -42,15 +45,19 @@ jobs:
       run: |
         sudo apt update
         sudo apt install libboost-dev gfortran libopenmpi-dev libpython3-dev \
-        libblas-dev liblapack-dev
+        libblas-dev liblapack-dev libhdf5-dev
     - name: Upgrade pip
       run: python3 -m pip install -U pip setuptools wheel
     - name: Install Python dependencies
+      # h5py is optional; some versions don't have binaries (yet)
       run: |
-        python3 -m pip install ruamel.yaml scons==3.1.2 numpy cython pandas pytest pytest-github-actions-annotate-failures
-        python3 -m pip install h5py || true
+        python3 -m pip install ruamel.yaml scons==3.1.2 numpy cython pandas pytest \
+        pytest-github-actions-annotate-failures
+        python3 -m pip install h5py
     - name: Build Cantera
-      run: python3 `which scons` build env_vars=all -j2 debug=n --debug=time
+      run: |
+        python3 `which scons` build env_vars=all -j2 debug=n --debug=time \
+        hdf_libdir=$HDF5_LIBDIR hdf_include=$HDF5_INCLUDEDIR
     - name: Upload shared library
       uses: actions/upload-artifact@v3
       if: matrix.python-version == '3.10' && matrix.os == 'ubuntu-22.04'
@@ -66,6 +73,9 @@ jobs:
     name: LLVM/Clang with Python 3.8
     runs-on: ubuntu-22.04
     timeout-minutes: 60
+    env:
+      HDF5_LIBDIR: /usr/lib/x86_64-linux-gnu/hdf5/serial
+      HDF5_INCLUDEDIR: /usr/include/hdf5/serial
     steps:
     - uses: actions/checkout@v2
       name: Checkout the repository
@@ -79,16 +89,16 @@ jobs:
     - name: Install Apt dependencies
       run: |
         sudo apt update
-        sudo apt install libboost-dev gfortran libomp-dev libomp5 libopenblas-dev
+        sudo apt install libboost-dev gfortran libomp-dev libomp5 libopenblas-dev libhdf5-dev
     - name: Upgrade pip
       run: python3 -m pip install -U pip setuptools wheel
     - name: Install Python dependencies
-      run: python3 -m pip install ruamel.yaml scons numpy cython h5py pandas pytest
-        pytest-github-actions-annotate-failures
+      run: |
+        python3 -m pip install ruamel.yaml scons numpy cython pandas h5py pytest pytest-github-actions-annotate-failures
     - name: Build Cantera
       run: python3 `which scons` build env_vars=all
         CXX=clang++-12 CC=clang-12 f90_interface=n extra_lib_dirs=/usr/lib/llvm/lib
-        -j2 debug=n --debug=time
+        -j2 debug=n --debug=time hdf_libdir=$HDF5_LIBDIR hdf_include=$HDF5_INCLUDEDIR
     - name: Test Cantera
       run:
         python3 `which scons` test show_long_tests=yes verbose_tests=yes --debug=time
@@ -122,7 +132,7 @@ jobs:
         python-version: 3.11
       if: matrix.python-version == '3.11'
     - name: Install Brew dependencies
-      run: brew install boost libomp
+      run: brew install boost libomp hdf5
     - name: Setup Homebrew Python
       # This path should work for future Python versions as well
       if: matrix.python-version != '3.11'
@@ -162,6 +172,9 @@ jobs:
     name: Coverage
     runs-on: ubuntu-latest
     timeout-minutes: 90
+    env:
+      HDF5_LIBDIR: /usr/lib/x86_64-linux-gnu/hdf5/serial
+      HDF5_INCLUDEDIR: /usr/include/hdf5/serial
     steps:
     - uses: actions/checkout@v2
       name: Checkout the repository
@@ -175,11 +188,12 @@ jobs:
     - name: Install Apt dependencies
       run: |
         sudo apt update
-        sudo apt install libboost-dev gfortran liblapack-dev libblas-dev libsundials-dev
+        sudo apt install libboost-dev gfortran liblapack-dev libblas-dev libsundials-dev libhdf5-dev
     - name: Upgrade pip
       run: python3 -m pip install -U pip setuptools wheel
     - name: Install Python dependencies
-      run: python3 -m pip install ruamel.yaml scons numpy cython h5py pandas scipy pytest
+      run: |
+        python3 -m pip install ruamel.yaml scons numpy cython pandas scipy pytest h5py \
         pytest-github-actions-annotate-failures pytest-cov gcovr
     - name: Setup .NET Core SDK
       uses: actions/setup-dotnet@v2
@@ -189,7 +203,8 @@ jobs:
       run: |
         python3 `which scons` build blas_lapack_libs=lapack,blas coverage=y \
         optimize=n skip_slow_tests=y no_optimize_flags='-DNDEBUG -O0' \
-        FORTRANFLAGS='-O0' env_vars=all -j2 --debug=time
+        FORTRANFLAGS='-O0' env_vars=all -j2 --debug=time \
+        hdf_libdir=$HDF5_LIBDIR hdf_include=$HDF5_INCLUDEDIR
     - name: Test Cantera
       run:
         python3 `which scons` test show_long_tests=yes verbose_tests=yes --debug=time
@@ -327,6 +342,9 @@ jobs:
       matrix:
         python-version: ['3.8', '3.10', '3.11']
       fail-fast: false
+    env:
+      HDF5_LIBDIR: /usr/lib/x86_64-linux-gnu/hdf5/serial
+      HDF5_INCLUDEDIR: /usr/include/hdf5/serial
     steps:
       - uses: actions/checkout@v2
         name: Checkout the repository
@@ -340,17 +358,27 @@ jobs:
       - name: Install Apt dependencies
         run: |
           sudo apt update
-          sudo apt install libboost-dev gfortran graphviz liblapack-dev libblas-dev gcc-9 g++-9
+          sudo apt install libboost-dev gfortran graphviz liblapack-dev libblas-dev \
+          gcc-9 g++-9 libhdf5-dev
       - name: Upgrade pip
         run: python3 -m pip install -U pip setuptools wheel
       - name: Install Python dependencies
         run: |
-          python3 -m pip install ruamel.yaml scons numpy cython pandas matplotlib scipy
-          python3 -m pip install h5py || true
+          python3 -m pip install ruamel.yaml scons numpy cython pandas matplotlib scipy h5py
       - name: Build Cantera
         # compile with GCC 9.4.0 on ubuntu-20.04 as an alternative to the default
         # (GCC 7.5.0 is both default and oldest supported version)
+        # compile without native HDF5 support
         run: python3 `which scons` build -j2 debug=n CC=gcc-9 CXX=g++-9
+        if: matrix.python-version != '3.10'
+      - name: Build Cantera (Python 3.10 with HDF)
+        # compile with GCC 9.4.0 on ubuntu-20.04 as an alternative to the default
+        # (GCC 7.5.0 is both default and oldest supported version)
+        # compile with native HDF5 support
+        run: |
+          python3 `which scons` build -j2 debug=n CC=gcc-9 CXX=g++-9 \
+          hdf_libdir=$HDF5_LIBDIR hdf_include=$HDF5_INCLUDEDIR
+        if: matrix.python-version == '3.10'
       - name: Run the examples
         # See https://unix.stackexchange.com/a/392973 for an explanation of the -exec part
         run: |
@@ -401,11 +429,12 @@ jobs:
         # use boost-cpp rather than boost from conda-forge
         run: |
           conda install -q sundials=${{ matrix.sundials-ver }} scons numpy ruamel.yaml \
-          cython boost-cpp fmt eigen yaml-cpp h5py pandas libgomp openblas pytest
+          cython boost-cpp fmt eigen yaml-cpp h5py pandas libgomp openblas pytest \
+          highfive
       - name: Build Cantera
         run: |
           scons build system_fmt=y system_eigen=y system_yamlcpp=y system_sundials=y \
-          blas_lapack_libs='lapack,blas' -j2 logging=debug debug=n \
+          system_highfive=y blas_lapack_libs='lapack,blas' -j2 logging=debug debug=n \
           optimize_flags='-O3 -ffast-math -fno-finite-math-only'
       - name: Test Cantera
         run: scons test show_long_tests=yes verbose_tests=yes
@@ -463,7 +492,7 @@ jobs:
       # use boost-cpp rather than boost from conda-forge
       # Install SCons >=4.4.0 to make sure that MSVC_TOOLSET_VERSION variable is present
       run: |
-        mamba install -q '"scons>=4.4.0"' numpy cython ruamel.yaml boost-cpp eigen yaml-cpp h5py pandas pytest
+        mamba install -q '"scons>=4.4.0"' numpy cython ruamel.yaml boost-cpp eigen yaml-cpp h5py pandas pytest highfive
       shell: pwsh
     - name: Build Cantera
       run: scons build system_eigen=y system_yamlcpp=y logging=debug
@@ -560,6 +589,8 @@ jobs:
     env:
       INTEL_REPO: https://apt.repos.intel.com
       INTEL_KEY: GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
+      HDF5_LIBDIR: /usr/lib/x86_64-linux-gnu/hdf5/serial
+      HDF5_INCLUDEDIR: /usr/include/hdf5/serial
     steps:
     - name: Intel Apt repository
       timeout-minutes: 1
@@ -573,7 +604,7 @@ jobs:
       timeout-minutes: 5
       run: |
         sudo apt-get install intel-oneapi-compiler-fortran intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic \
-        intel-oneapi-mpi intel-oneapi-mpi-devel intel-oneapi-mkl ninja-build libboost-dev
+        intel-oneapi-mpi intel-oneapi-mpi-devel intel-oneapi-mkl ninja-build libboost-dev libhdf5-dev
     - uses: actions/checkout@v2
       name: Checkout the repository
       with:
@@ -586,17 +617,19 @@ jobs:
     - name: Upgrade pip
       run: python3 -m pip install -U pip setuptools wheel
     - name: Install Python dependencies
-      run: python3 -m pip install ruamel.yaml scons numpy cython h5py pandas pytest
+      run: |
+        python3 -m pip install ruamel.yaml scons numpy cython pandas h5py pytest \
         pytest-github-actions-annotate-failures
     - name: Setup Intel oneAPI environment
       run: |
         source /opt/intel/oneapi/setvars.sh
         printenv >> $GITHUB_ENV
     - name: Build Cantera
       run: python3 `which scons` build env_vars=all CC=icx CXX=icpx -j2 debug=n
+        hdf_libdir=$HDF5_LIBDIR hdf_include=$HDF5_INCLUDEDIR
         --debug=time f90_interface=n # FORTRAN=ifx
     - name: Test Cantera
-      run:
+      run: |
         python3 `which scons` test show_long_tests=yes verbose_tests=yes --debug=time
 
   linux-intel-oneapi-classic:
@@ -718,9 +751,15 @@ jobs:
     - name: Install Python dependencies
       run: python3 -m pip install ruamel.yaml
     - name: Install library dependencies with Conda (Windows)
-      run: mamba install -q yaml-cpp mkl
+      run: mamba install -q yaml-cpp mkl highfive
       shell: pwsh
       if: matrix.os == 'windows-2022'
+    - name: Install Brew dependencies (macOS)
+      run: brew install hdf5
+      if: matrix.os == 'macos-11'
+    - name: Install Apt dependencies (Ubuntu)
+      run: sudo apt install libhdf5-dev
+      if: matrix.os == 'ubuntu-22.04'
     - name: Setup .NET Core SDK
       uses: actions/setup-dotnet@v2
       with:

.gitmodules

@@ -13,3 +13,6 @@
 [submodule "ext/yaml-cpp"]
 	path = ext/yaml-cpp
 	url = https://github.com/jbeder/yaml-cpp.git
+[submodule "ext/HighFive"]
+	path = ext/HighFive
+	url = https://github.com/BlueBrain/HighFive.git

SConstruct

@@ -356,6 +356,34 @@ config_options = [
            must include the shared version of the library, for example,
            'libfmt.so'.""",
         "default", ("default", "y", "n")),
+    EnumOption(
+        "hdf_support",
+        """Select whether to support HDF5 container files natively ('y'), disable HDF5
+           support ('n'), or to decide automatically based on the system configuration
+           ('default'). Native HDF5 support uses the HDF5 library as well as the
+           header-only HighFive C++ wrapper (see option 'system_highfive'). Specifying
+           'hdf_include' or 'hdf_libdir' changes the default to 'y'.""",
+        "default", ("default", "y", "n")),
+    PathOption(
+        "hdf_include",
+        """The directory where the HDF5 header files are installed. This should be the
+           directory that contains files 'H5Version.h' and 'H5Public.h', amongst others.
+           Not needed if the headers are installed in a standard location, for example,
+           '/usr/include'.""",
+        "", PathVariable.PathAccept),
+    PathOption(
+        "hdf_libdir",
+        """The directory where the HDF5 libraries are installed. Not needed if the
+           libraries are installed in a standard location, for example, '/usr/lib'.""",
+        "", PathVariable.PathAccept),
+    EnumOption(
+        "system_highfive",
+        """Select whether to use HighFive from a system installation ('y'), from a
+           Git submodule ('n'), or to decide automatically ('default'). If HighFive
+           is not installed directly into a system include directory, for example, it
+           is installed in '/opt/include/HighFive', then you will need to add
+           '/opt/include/HighFive' to 'extra_inc_dirs'.""",
+        "default", ("default", "y", "n")),
     EnumOption(
         "system_yamlcpp",
         """Select whether to use the yaml-cpp library from a system installation
@@ -1497,6 +1525,76 @@ else: # env['system_sundials'] == 'n'
     env['sundials_version'] = '5.3'
     env['has_sundials_lapack'] = int(env['use_lapack'])
 
+if env["hdf_include"]:
+    env["hdf_include"] = Path(env["hdf_include"]).as_posix()
+    env.Append(CPPPATH=[env["hdf_include"]])
+    env["hdf_support"] = "y"
+if env["hdf_libdir"]:
+    env["hdf_libdir"] = Path(env["hdf_libdir"]).as_posix()
+    env.Append(LIBPATH=[env["hdf_libdir"]])
+    env["hdf_support"] = "y"
+    if env["use_rpath_linkage"]:
+        env.Append(RPATH=env["hdf_libdir"])
+
+if env["hdf_support"] == "n":
+    env["use_hdf5"] = False
+else:
+    env["use_hdf5"] = conf.CheckLib("hdf5", autoadd=False)
+    if not env["use_hdf5"] and env["hdf_support"] == "y":
+        config_error("HDF5 support has been specified but libraries were not found.")
+
+if env["use_hdf5"] and env["system_highfive"] in ("n", "default"):
+    env["system_highfive"] = False
+    if not os.path.exists("ext/eigen/HighFive/include"):
+        if not os.path.exists(".git"):
+            config_error("HighFive is missing. Install HighFive in ext/HighFive.")
+
+        try:
+            code = subprocess.call(["git", "submodule", "update", "--init",
+                                    "--recursive", "ext/HighFive"])
+        except Exception:
+            code = -1
+        if code:
+            config_error("HighFive not found and submodule checkout failed.\n"
+                        "Try manually checking out the submodule with:\n\n"
+                        "    git submodule update --init --recursive ext/HighFive\n")
+
+    env["use_hdf5"] = conf.CheckLibWithHeader(
+        "hdf5", "../ext/HighFive/include/highfive/H5File.hpp",
+        language="C++", autoadd=False)
+
+    if env["use_hdf5"]:
+        logger.info("Using private installation of HighFive.")
+    elif env["hdf_support"] == "y":
+        config_error("HDF5 support has been specified but HighFive configuration failed.")
+    else:
+        logger.warning("HighFive is not configured correctly; skipping.")
+        env["use_hdf5"] = False
+
+elif env["use_hdf5"]:
+    env["system_highfive"] = True
+    env["use_hdf5"] = conf.CheckLibWithHeader(
+        "hdf5", "highfive/H5File.hpp", language="C++", autoadd=False)
+    if env["use_hdf5"]:
+        logger.info("Using system installation of HighFive.")
+    else:
+        config_error("Unable to locate system HighFive installation.")
+
+if env["use_hdf5"]:
+    hdf_version = textwrap.dedent("""\
+        #include <iostream>
+        #include "H5public.h"
+        int main(int argc, char** argv) {
+            std::cout << H5_VERS_MAJOR << "." << H5_VERS_MINOR << "." << H5_VERS_RELEASE;
+            return 0;
+        }
+    """)
+    retcode, hdf_version = conf.TryRun(hdf_version, ".cpp")
+    if retcode:
+        logger.info(f"Compiling against HDF5 version {hdf_version}")
+    else:
+        logger.warning("Failed to determine HDF5 version.")
+
 def set_fortran(pattern, value):
     # Set compiler / flags for all Fortran versions to be the same
     for version in ("FORTRAN", "F77", "F90", "F95", "F03", "F08"):
@@ -2023,6 +2121,8 @@ cdefine('LAPACK_FTN_TRAILING_UNDERSCORE', 'lapack_ftn_trailing_underscore')
 cdefine('FTN_TRAILING_UNDERSCORE', 'lapack_ftn_trailing_underscore')
 cdefine('LAPACK_NAMES_LOWERCASE', 'lapack_names', 'lower')
 cdefine('CT_USE_LAPACK', 'use_lapack')
+cdefine("CT_USE_HDF5", "use_hdf5")
+cdefine("CT_USE_SYSTEM_HIGHFIVE", "system_highfive")
 cdefine("CT_USE_SYSTEM_EIGEN", "system_eigen")
 cdefine("CT_USE_SYSTEM_EIGEN_PREFIXED", "system_eigen_prefixed")
 cdefine('CT_USE_SYSTEM_FMT', 'system_fmt')
@@ -2112,6 +2212,9 @@ else:
 env["external_libs"] = []
 env["external_libs"].extend(env["sundials_libs"])
 
+if env["use_hdf5"]:
+    env["external_libs"].append("hdf5")
+
 if env["system_fmt"]:
     env["external_libs"].append("fmt")
 

ext/SConscript

@@ -125,6 +125,14 @@ if not env['system_eigen']:
     copyenv.Depends(copyenv['config_h_target'], h)
     ext_copies.extend(h)
 
+if not env["system_highfive"]:
+    localenv = prep_default(env)
+    license_files["HighFive"] = File("#ext/HighFive/LICENSE")
+    h = build(copyenv.Command('#include/cantera/ext/HighFive', '#ext/HighFive/include/highfive',
+                              Copy('$TARGET', '$SOURCE')))
+    copyenv.Depends(copyenv['config_h_target'], h)
+    ext_copies.extend(h)
+
 # Google Test: Used internally for Cantera unit tests.
 if env['googletest'] == 'submodule':
     localenv = prep_gtest(env)