Add MATLAB support

.devcontainer/Dockerfile

@@ -13,6 +13,7 @@ FROM ${DEVCONTAINER_BASE} as file-normalizer
 
 COPY environment.yml \
      .devcontainer/devcontainer.bashrc  \
+     .devcontainer/matlab-r2023b.ubuntu-22.04.dependencies.txt \
      /data/
 
 RUN chmod -R 555 /data/
@@ -116,3 +117,61 @@ RUN . /opt/conda/etc/profile.d/conda.sh \
     && mkdir -p /home/vscode/.local/share/CMakeTools \
     && echo "[{\"name\":\"Conda\",\"compilers\":{\"C\":\"$GCC\",\"CXX\":\"$GXX\"}}]" > /home/vscode/.local/share/CMakeTools/cmake-tools-kits.json \
     && chown vscode:conda /home/vscode/.local/share/CMakeTools/cmake-tools-kits.json
+
+
+#########################################################
+# Install Matlab
+#   Based on mathworks/{matlab-deps:r2023b,matlab:r2023b}
+#
+# Configure the Matlab License Server to use Matlab within the devcontainer (including VSCode extensions):
+#   In your HOST environment, export the environment variable MATLAB_LICENSE_FILE, e.g.
+#       export MATLAB_LICENSE_FILE=/mnt/c/Users/username/Documents/MATLAB/license.lic
+#########################################################
+
+ENV DEBIAN_FRONTEND="noninteractive" TZ="Etc/UTC"
+
+COPY --from=file-normalizer /data/matlab-r2023b.ubuntu-22.04.dependencies.txt /tmp/matlab-dependencies.txt
+
+RUN export DEBIAN_FRONTEND=noninteractive \
+    && apt-get update \
+    && apt-get install --no-install-recommends --yes \
+    `cat /tmp/matlab-dependencies.txt` \
+    wget \
+    unzip \
+    ca-certificates \
+    && apt-get clean \
+    && apt-get -y autoremove \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN [ -d /usr/share/X11/xkb ] || mkdir -p /usr/share/X11/xkb
+
+ARG MATLAB_RELEASE=r2023b
+ARG MATLAB_PRODUCT_LIST="MATLAB"
+ARG MATLAB_INSTALL_LOCATION="/opt/matlab/${MATLAB_RELEASE}"
+ARG LICENSE_SERVER=
+
+RUN adduser --shell /bin/bash --disabled-password --gecos "" matlab \
+    && echo "matlab ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/matlab \
+    && chmod 0440 /etc/sudoers.d/matlab
+
+USER matlab
+WORKDIR /home/matlab
+
+ENV MLM_LICENSE_FILE=${LICENSE_SERVER}
+
+RUN wget -q https://www.mathworks.com/mpm/glnxa64/mpm \
+    && chmod +x mpm \
+    && sudo HOME=${HOME} ./mpm install \
+    --release=${MATLAB_RELEASE} \
+    --destination=${MATLAB_INSTALL_LOCATION} \
+    --products ${MATLAB_PRODUCT_LIST} \
+    || (echo "MPM Installation Failure. See below for more information:" && cat /tmp/mathworks_root.log && false) \
+    && sudo rm -f mpm /tmp/mathworks_root.log \
+    && sudo ln -s ${MATLAB_INSTALL_LOCATION}/bin/matlab /usr/local/bin/matlab \
+    && sudo ln -s ${MATLAB_INSTALL_LOCATION} /opt/matlab/latest \
+    && sudo mkdir -p ${MATLAB_INSTALL_LOCATION}/licenses
+
+# Install workaround run-matlab-command script to unify local and CI invocations of `matlab -batch`
+# See https://github.com/matlab-actions/run-command/issues/53
+RUN sudo wget -O /usr/local/bin/run-matlab-command https://ssd.mathworks.com/supportfiles/ci/run-matlab-command/v2/glnxa64/run-matlab-command \
+    && sudo chmod +x /usr/local/bin/run-matlab-command

.devcontainer/devcontainer-on-create.sh

@@ -0,0 +1,11 @@
+#! /usr/bin/env bash
+
+set -euo pipefail
+
+host_license_file="/tmp/matlab.host.lic"
+
+if [ -s "${host_license_file}" ]; then
+    # We don't bind mount directly to $MATLAB_LICENSE_FILE so that we can write
+    # to it later without affecting the host.
+    sudo cp "${host_license_file}" "${MATLAB_LICENSE_FILE}"
+fi

.devcontainer/devcontainer.json

@@ -6,12 +6,19 @@
     "dockerfile": "Dockerfile",
     "context": ".."
   },
-  "runArgs": ["--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined"],
+  "runArgs": [
+    "--cap-add=SYS_PTRACE",
+    "--security-opt",
+    "seccomp=unconfined",
+    "--network=host"
+  ],
   "overrideCommand": false,
   "mounts": [
     // Bind mount docker socket under an alias to support docker-from-docker
-    "source=/var/run/docker.sock,target=/var/run/docker-host.sock,type=bind"
+    "source=/var/run/docker.sock,target=/var/run/docker-host.sock,type=bind",
+    "source=${localEnv:MATLAB_LICENSE_FILE:/dev/null},target=/tmp/matlab.host.lic,type=bind,readonly"
   ],
+  "onCreateCommand": ".devcontainer/devcontainer-on-create.sh",
   "remoteUser": "vscode",
 
   // Configure tool-specific properties.
@@ -102,6 +109,7 @@
         "JacquesLucke.gcov-viewer",
         "jinliming2.vscode-go-template",
         "matepek.vscode-catch2-test-adapter",
+        "mathworks.language-matlab",
         "mhutchie.git-graph",
         "ms-python.black-formatter",
         "ms-python.python",
@@ -118,6 +126,7 @@
 
   "containerEnv": {
     "PYTHONPATH": "/workspaces/yardl/python",
-    "CGO_ENABLED": "0"
+    "CGO_ENABLED": "0",
+    "MATLAB_LICENSE_FILE": "/opt/matlab/latest/licenses/license.lic"
   }
 }

.devcontainer/matlab-r2023b.ubuntu-22.04.dependencies.txt

@@ -0,0 +1,44 @@
+ca-certificates
+libasound2
+libc6
+libcairo-gobject2
+libcairo2
+libcap2
+libcups2
+libdrm2
+libgbm1
+libgdk-pixbuf-2.0-0
+libgl1
+libglib2.0-0
+libgstreamer-plugins-base1.0-0
+libgstreamer1.0-0
+libgtk-3-0
+libice6
+libltdl7
+libnspr4
+libnss3
+libpam0g
+libpango-1.0-0
+libpangocairo-1.0-0
+libpangoft2-1.0-0
+libsndfile1
+libuuid1
+libwayland-client0
+libxcomposite1
+libxcursor1
+libxdamage1
+libxfixes3
+libxft2
+libxinerama1
+libxrandr2
+libxt6
+libxtst6
+libxxf86vm1
+locales
+locales-all
+make
+net-tools
+procps
+sudo
+unzip
+zlib1g

.github/actions/configure-environment/action.yml

@@ -44,7 +44,22 @@ runs:
       if: steps.cache-conda.outputs.cache-hit != 'true'
 
 
-    - uses: actions/setup-go@v4
+    - name: Setup Go
+      uses: actions/setup-go@v4
       with:
         go-version: '1.21.3'
         cache-dependency-path: tooling/go.sum
+
+    - name: Setup Matlab
+      uses: matlab-actions/setup-matlab@v2
+      with:
+        release: R2022b
+        cache: true
+
+    # Workaround to running `matlab -batch` from within justfile and Python
+    # See: https://github.com/matlab-actions/run-command/issues/53
+    - name: Get run-matlab-command
+      shell: bash
+      run: |
+        wget -O /usr/local/bin/run-matlab-command https://ssd.mathworks.com/supportfiles/ci/run-matlab-command/v2/glnxa64/run-matlab-command
+        chmod +x /usr/local/bin/run-matlab-command

.github/workflows/ci.yml

@@ -35,7 +35,7 @@ jobs:
         uses: ./.github/actions/configure-environment
 
       - name: Build and test
-        run: just cpp_version=${{ matrix.cppVersion }} validate-with-no-changes
+        run: just cpp_version=${{ matrix.cppVersion }} matlab=enabled validate-with-no-changes
 
   buildBinaries:
     name: Build yardl binaries

.gitignore

@@ -24,3 +24,5 @@ cpp/build/
 cpp/sandbox/generated/
 
 **/*.bin
+
+matlab_workspace.mat

README.md

@@ -123,6 +123,16 @@ $ just
 
 from the repo root.
 
+To enable support for Matlab, you must provide a Matlab license file to the devcontainer.
+In your HOST environment, export the environment variable `MATLAB_LICENSE_FILE`,
+e.g. in `$HOME/.profile`
+
+```bash
+export MATLAB_LICENSE_FILE=/mnt/c/Users/username/Documents/MATLAB/license.lic
+```
+
+Then invoke `just matlab=enabled ...`.
+
 ## Contributing
 
 This project welcomes contributions and suggestions.  Most contributions require

cpp/test/benchmark.cc

@@ -310,7 +310,7 @@ std::optional<std::function<Result(Format)>> GetScenarioFunction(std::string sce
 
 int main(int argc, char* argv[]) {
   if (argc != 3) {
-    std::cerr << "Incorrect number of arguments. Usage: banchmark <scenario> <hdf5 | binary | ndjson>" << std::endl;
+    std::cerr << "Incorrect number of arguments. Usage: benchmark <scenario> <hdf5 | binary | ndjson>" << std::endl;
     return 1;
   }
 

cpp/test/generated/binary/protocols.cc

@@ -547,13 +547,14 @@ struct IsTriviallySerializable<test_model::RecordWithComputedFields> {
     IsTriviallySerializable<decltype(__T__::vector_field)>::value &&
     IsTriviallySerializable<decltype(__T__::vector_of_vectors_field)>::value &&
     IsTriviallySerializable<decltype(__T__::fixed_vector_field)>::value &&
+    IsTriviallySerializable<decltype(__T__::fixed_vector_of_vectors_field)>::value &&
     IsTriviallySerializable<decltype(__T__::optional_named_array)>::value &&
     IsTriviallySerializable<decltype(__T__::int_float_union)>::value &&
     IsTriviallySerializable<decltype(__T__::nullable_int_float_union)>::value &&
     IsTriviallySerializable<decltype(__T__::union_with_nested_generic_union)>::value &&
     IsTriviallySerializable<decltype(__T__::map_field)>::value &&
-    (sizeof(__T__) == (sizeof(__T__::array_field) + sizeof(__T__::array_field_map_dimensions) + sizeof(__T__::dynamic_array_field) + sizeof(__T__::fixed_array_field) + sizeof(__T__::int_field) + sizeof(__T__::int8_field) + sizeof(__T__::uint8_field) + sizeof(__T__::int16_field) + sizeof(__T__::uint16_field) + sizeof(__T__::uint32_field) + sizeof(__T__::int64_field) + sizeof(__T__::uint64_field) + sizeof(__T__::size_field) + sizeof(__T__::float32_field) + sizeof(__T__::float64_field) + sizeof(__T__::complexfloat32_field) + sizeof(__T__::complexfloat64_field) + sizeof(__T__::string_field) + sizeof(__T__::tuple_field) + sizeof(__T__::vector_field) + sizeof(__T__::vector_of_vectors_field) + sizeof(__T__::fixed_vector_field) + sizeof(__T__::optional_named_array) + sizeof(__T__::int_float_union) + sizeof(__T__::nullable_int_float_union) + sizeof(__T__::union_with_nested_generic_union) + sizeof(__T__::map_field))) &&
-    offsetof(__T__, array_field) < offsetof(__T__, array_field_map_dimensions) && offsetof(__T__, array_field_map_dimensions) < offsetof(__T__, dynamic_array_field) && offsetof(__T__, dynamic_array_field) < offsetof(__T__, fixed_array_field) && offsetof(__T__, fixed_array_field) < offsetof(__T__, int_field) && offsetof(__T__, int_field) < offsetof(__T__, int8_field) && offsetof(__T__, int8_field) < offsetof(__T__, uint8_field) && offsetof(__T__, uint8_field) < offsetof(__T__, int16_field) && offsetof(__T__, int16_field) < offsetof(__T__, uint16_field) && offsetof(__T__, uint16_field) < offsetof(__T__, uint32_field) && offsetof(__T__, uint32_field) < offsetof(__T__, int64_field) && offsetof(__T__, int64_field) < offsetof(__T__, uint64_field) && offsetof(__T__, uint64_field) < offsetof(__T__, size_field) && offsetof(__T__, size_field) < offsetof(__T__, float32_field) && offsetof(__T__, float32_field) < offsetof(__T__, float64_field) && offsetof(__T__, float64_field) < offsetof(__T__, complexfloat32_field) && offsetof(__T__, complexfloat32_field) < offsetof(__T__, complexfloat64_field) && offsetof(__T__, complexfloat64_field) < offsetof(__T__, string_field) && offsetof(__T__, string_field) < offsetof(__T__, tuple_field) && offsetof(__T__, tuple_field) < offsetof(__T__, vector_field) && offsetof(__T__, vector_field) < offsetof(__T__, vector_of_vectors_field) && offsetof(__T__, vector_of_vectors_field) < offsetof(__T__, fixed_vector_field) && offsetof(__T__, fixed_vector_field) < offsetof(__T__, optional_named_array) && offsetof(__T__, optional_named_array) < offsetof(__T__, int_float_union) && offsetof(__T__, int_float_union) < offsetof(__T__, nullable_int_float_union) && offsetof(__T__, nullable_int_float_union) < offsetof(__T__, union_with_nested_generic_union) && offsetof(__T__, union_with_nested_generic_union) < offsetof(__T__, map_field);
+    (sizeof(__T__) == (sizeof(__T__::array_field) + sizeof(__T__::array_field_map_dimensions) + sizeof(__T__::dynamic_array_field) + sizeof(__T__::fixed_array_field) + sizeof(__T__::int_field) + sizeof(__T__::int8_field) + sizeof(__T__::uint8_field) + sizeof(__T__::int16_field) + sizeof(__T__::uint16_field) + sizeof(__T__::uint32_field) + sizeof(__T__::int64_field) + sizeof(__T__::uint64_field) + sizeof(__T__::size_field) + sizeof(__T__::float32_field) + sizeof(__T__::float64_field) + sizeof(__T__::complexfloat32_field) + sizeof(__T__::complexfloat64_field) + sizeof(__T__::string_field) + sizeof(__T__::tuple_field) + sizeof(__T__::vector_field) + sizeof(__T__::vector_of_vectors_field) + sizeof(__T__::fixed_vector_field) + sizeof(__T__::fixed_vector_of_vectors_field) + sizeof(__T__::optional_named_array) + sizeof(__T__::int_float_union) + sizeof(__T__::nullable_int_float_union) + sizeof(__T__::union_with_nested_generic_union) + sizeof(__T__::map_field))) &&
+    offsetof(__T__, array_field) < offsetof(__T__, array_field_map_dimensions) && offsetof(__T__, array_field_map_dimensions) < offsetof(__T__, dynamic_array_field) && offsetof(__T__, dynamic_array_field) < offsetof(__T__, fixed_array_field) && offsetof(__T__, fixed_array_field) < offsetof(__T__, int_field) && offsetof(__T__, int_field) < offsetof(__T__, int8_field) && offsetof(__T__, int8_field) < offsetof(__T__, uint8_field) && offsetof(__T__, uint8_field) < offsetof(__T__, int16_field) && offsetof(__T__, int16_field) < offsetof(__T__, uint16_field) && offsetof(__T__, uint16_field) < offsetof(__T__, uint32_field) && offsetof(__T__, uint32_field) < offsetof(__T__, int64_field) && offsetof(__T__, int64_field) < offsetof(__T__, uint64_field) && offsetof(__T__, uint64_field) < offsetof(__T__, size_field) && offsetof(__T__, size_field) < offsetof(__T__, float32_field) && offsetof(__T__, float32_field) < offsetof(__T__, float64_field) && offsetof(__T__, float64_field) < offsetof(__T__, complexfloat32_field) && offsetof(__T__, complexfloat32_field) < offsetof(__T__, complexfloat64_field) && offsetof(__T__, complexfloat64_field) < offsetof(__T__, string_field) && offsetof(__T__, string_field) < offsetof(__T__, tuple_field) && offsetof(__T__, tuple_field) < offsetof(__T__, vector_field) && offsetof(__T__, vector_field) < offsetof(__T__, vector_of_vectors_field) && offsetof(__T__, vector_of_vectors_field) < offsetof(__T__, fixed_vector_field) && offsetof(__T__, fixed_vector_field) < offsetof(__T__, fixed_vector_of_vectors_field) && offsetof(__T__, fixed_vector_of_vectors_field) < offsetof(__T__, optional_named_array) && offsetof(__T__, optional_named_array) < offsetof(__T__, int_float_union) && offsetof(__T__, int_float_union) < offsetof(__T__, nullable_int_float_union) && offsetof(__T__, nullable_int_float_union) < offsetof(__T__, union_with_nested_generic_union) && offsetof(__T__, union_with_nested_generic_union) < offsetof(__T__, map_field);
 };
 
 template <>
@@ -2439,6 +2440,7 @@ template<typename A, yardl::binary::Reader<A> ReadA, typename B, yardl::binary::
   yardl::binary::WriteVector<int32_t, yardl::binary::WriteInteger>(stream, value.vector_field);
   yardl::binary::WriteVector<std::vector<int32_t>, yardl::binary::WriteVector<int32_t, yardl::binary::WriteInteger>>(stream, value.vector_of_vectors_field);
   yardl::binary::WriteArray<int32_t, yardl::binary::WriteInteger, 3>(stream, value.fixed_vector_field);
+  yardl::binary::WriteArray<std::array<int32_t, 3>, yardl::binary::WriteArray<int32_t, yardl::binary::WriteInteger, 3>, 2>(stream, value.fixed_vector_of_vectors_field);
   yardl::binary::WriteOptional<test_model::NamedNDArray, test_model::binary::WriteNamedNDArray>(stream, value.optional_named_array);
   WriteUnion<int32_t, yardl::binary::WriteInteger, float, yardl::binary::WriteFloatingPoint>(stream, value.int_float_union);
   WriteUnion<std::monostate, yardl::binary::WriteMonostate, int32_t, yardl::binary::WriteInteger, float, yardl::binary::WriteFloatingPoint>(stream, value.nullable_int_float_union);
@@ -2474,6 +2476,7 @@ template<typename A, yardl::binary::Reader<A> ReadA, typename B, yardl::binary::
   yardl::binary::ReadVector<int32_t, yardl::binary::ReadInteger>(stream, value.vector_field);
   yardl::binary::ReadVector<std::vector<int32_t>, yardl::binary::ReadVector<int32_t, yardl::binary::ReadInteger>>(stream, value.vector_of_vectors_field);
   yardl::binary::ReadArray<int32_t, yardl::binary::ReadInteger, 3>(stream, value.fixed_vector_field);
+  yardl::binary::ReadArray<std::array<int32_t, 3>, yardl::binary::ReadArray<int32_t, yardl::binary::ReadInteger, 3>, 2>(stream, value.fixed_vector_of_vectors_field);
   yardl::binary::ReadOptional<test_model::NamedNDArray, test_model::binary::ReadNamedNDArray>(stream, value.optional_named_array);
   ReadUnion<int32_t, yardl::binary::ReadInteger, float, yardl::binary::ReadFloatingPoint>(stream, value.int_float_union);
   ReadUnion<std::monostate, yardl::binary::ReadMonostate, int32_t, yardl::binary::ReadInteger, float, yardl::binary::ReadFloatingPoint>(stream, value.nullable_int_float_union);
@@ -3469,6 +3472,68 @@ void DynamicNDArraysReader::CloseImpl() {
   stream_.VerifyFinished();
 }
 
+void MultiDArraysWriter::WriteImagesImpl(yardl::NDArray<float, 4> const& value) {
+  yardl::binary::WriteBlock<yardl::NDArray<float, 4>, yardl::binary::WriteNDArray<float, yardl::binary::WriteFloatingPoint, 4>>(stream_, value);
+}
+
+void MultiDArraysWriter::WriteImagesImpl(std::vector<yardl::NDArray<float, 4>> const& values) {
+  if (!values.empty()) {
+    yardl::binary::WriteVector<yardl::NDArray<float, 4>, yardl::binary::WriteNDArray<float, yardl::binary::WriteFloatingPoint, 4>>(stream_, values);
+  }
+}
+
+void MultiDArraysWriter::EndImagesImpl() {
+  yardl::binary::WriteInteger(stream_, 0U);
+}
+
+void MultiDArraysWriter::WriteFramesImpl(yardl::FixedNDArray<float, 1, 1, 64, 32> const& value) {
+  yardl::binary::WriteBlock<yardl::FixedNDArray<float, 1, 1, 64, 32>, yardl::binary::WriteFixedNDArray<float, yardl::binary::WriteFloatingPoint, 1, 1, 64, 32>>(stream_, value);
+}
+
+void MultiDArraysWriter::WriteFramesImpl(std::vector<yardl::FixedNDArray<float, 1, 1, 64, 32>> const& values) {
+  if (!values.empty()) {
+    yardl::binary::WriteVector<yardl::FixedNDArray<float, 1, 1, 64, 32>, yardl::binary::WriteFixedNDArray<float, yardl::binary::WriteFloatingPoint, 1, 1, 64, 32>>(stream_, values);
+  }
+}
+
+void MultiDArraysWriter::EndFramesImpl() {
+  yardl::binary::WriteInteger(stream_, 0U);
+}
+
+void MultiDArraysWriter::Flush() {
+  stream_.Flush();
+}
+
+void MultiDArraysWriter::CloseImpl() {
+  stream_.Flush();
+}
+
+bool MultiDArraysReader::ReadImagesImpl(yardl::NDArray<float, 4>& value) {
+  bool read_block_successful = false;
+  read_block_successful = yardl::binary::ReadBlock<yardl::NDArray<float, 4>, yardl::binary::ReadNDArray<float, yardl::binary::ReadFloatingPoint, 4>>(stream_, current_block_remaining_, value);
+  return read_block_successful;
+}
+
+bool MultiDArraysReader::ReadImagesImpl(std::vector<yardl::NDArray<float, 4>>& values) {
+  yardl::binary::ReadBlocksIntoVector<yardl::NDArray<float, 4>, yardl::binary::ReadNDArray<float, yardl::binary::ReadFloatingPoint, 4>>(stream_, current_block_remaining_, values);
+  return current_block_remaining_ != 0;
+}
+
+bool MultiDArraysReader::ReadFramesImpl(yardl::FixedNDArray<float, 1, 1, 64, 32>& value) {
+  bool read_block_successful = false;
+  read_block_successful = yardl::binary::ReadBlock<yardl::FixedNDArray<float, 1, 1, 64, 32>, yardl::binary::ReadFixedNDArray<float, yardl::binary::ReadFloatingPoint, 1, 1, 64, 32>>(stream_, current_block_remaining_, value);
+  return read_block_successful;
+}
+
+bool MultiDArraysReader::ReadFramesImpl(std::vector<yardl::FixedNDArray<float, 1, 1, 64, 32>>& values) {
+  yardl::binary::ReadBlocksIntoVector<yardl::FixedNDArray<float, 1, 1, 64, 32>, yardl::binary::ReadFixedNDArray<float, yardl::binary::ReadFloatingPoint, 1, 1, 64, 32>>(stream_, current_block_remaining_, values);
+  return current_block_remaining_ != 0;
+}
+
+void MultiDArraysReader::CloseImpl() {
+  stream_.VerifyFinished();
+}
+
 void MapsWriter::WriteStringToIntImpl(std::unordered_map<std::string, int32_t> const& value) {
   yardl::binary::WriteMap<std::string, int32_t, yardl::binary::WriteString, yardl::binary::WriteInteger>(stream_, value);
 }