test: add minimal tests to all official examples

examples/Makefile

@@ -11,5 +11,8 @@ build/%.tgz: official/%/
 	mkdir -p $$(dirname "$@")
 	tar -czf "$@" -C $$(dirname "$<") $$(basename "$<")
 
+test:
+	pytest -vv -s --durations=0 test_examples.py
+
 .PHONY: build
 build: $(EXAMPLES_DIRS)

examples/test_examples.py

@@ -0,0 +1,29 @@
+import os
+import pathlib
+import pytest
+import subprocess
+from ruamel import yaml
+
+from determined import experimental, load
+
+
+official_examples = [
+    ("official/cifar10_cnn_pytorch", "official/cifar10_cnn_pytorch/const.yaml"),
+    ("official/cifar10_cnn_tf_keras", "official/cifar10_cnn_tf_keras/const.yaml"),
+    ("official/fashion_mnist_tf_keras", "official/fashion_mnist_tf_keras/const.yaml"),
+    ("official/iris_tf_keras", "official/iris_tf_keras/const.yaml"),
+    ("official/mnist_estimator", "official/mnist_estimator/const.yaml"),
+    ("official/mnist_pytorch", "official/mnist_pytorch/const.yaml"),
+    ("official/multiple_lr_schedulers_pytorch", "official/multiple_lr_schedulers_pytorch/const.yaml"),
+    # ("official/object_detection_pytorch", "official/object_detection_pytorch/const.yaml"),
+]
+
+
+@pytest.mark.parametrize("model_def,config_file", official_examples)
+def test_official(model_def: str, config_file: str) -> None:
+    examples_dir = pathlib.Path(__file__).parent
+    model_def_absolute = examples_dir.joinpath(model_def)
+    config_file_absolute = examples_dir.joinpath(config_file)
+
+    subprocess.check_output(
+        ("det", "experiment", "create", "--local", "--test", str(config_file_absolute), str(model_def_absolute)))