ensure deterministic order

mila-iqia · Jul 9, 2024 · 2c442f1 · 2c442f1
1 parent bba4d3f
commit 2c442f1
Show file tree

Hide file tree

Showing 5 changed files with 32 additions and 27 deletions.
diff --git a/benchmate/benchmate/datagen.py b/benchmate/benchmate/datagen.py
@@ -93,6 +93,7 @@ def generate_fakeimagenet():
     parser.add_argument("--batch-size", default=512, type=int)
     parser.add_argument("--batch-count", default=60, type=int)
     parser.add_argument("--device-count", default=device_count(), type=int)
+    parser.add_argument("--device", default=None, type=str)
     parser.add_argument("--image-size", default=[3, 384, 384], type=int, nargs="+")
     parser.add_argument("--val", default=0.1, type=float, nargs="+")
     parser.add_argument("--test", default=0.1, type=float, nargs="+")
@@ -111,8 +112,6 @@ def generate_fakeimagenet():
         f"test": int(total_images * args.test),
     }
 
-
-
     generate_sets(dest, size_spec, args.image_size)
     print("Done!")
 

diff --git a/milabench/utils.py b/milabench/utils.py
@@ -200,7 +200,10 @@ def result(self):
     def report(self, **kwargs):
         """Generate a full report containing warnings from all loggers"""
         summary = Summary()
-        for _, layer in self.funs.items():
+
+        items = sorted(self.funs.items(), key=lambda item: str(item[0]))
+
+        for _, layer in items:
             if hasattr(layer, "report"):
                 layer.report(summary, **kwargs)
         summary.show()
@@ -211,6 +214,8 @@ def multilogger(*logs, **kwargs):
     """Combine loggers into a single context manager"""
     results = dict()
 
+    logs = sorted(logs, key=lambda l: str(type(l)))
+
     with ExitStack() as stack:
         for log in logs:
             results[type(log)] = stack.enter_context(log)

diff --git a/tests/test_mock.py b/tests/test_mock.py
@@ -100,8 +100,11 @@ def test_milabench(monkeypatch, bench, module_tmp_dir, standard_config):
 
 
 ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
-def cleanpath(out):
-    return out.replace(str(ROOT), "$SRC")
+def cleanpath(out, tmppath):
+    return (out
+        .replace(str(ROOT), "$SRC")
+        .replace(str(tmppath), "$TMP")
+    )
 
 
 
@@ -119,7 +122,7 @@ def test_milabench_bad_install(monkeypatch, tmp_path, config, capsys, file_regre
 
     # Check that the error was extracted
     all = capsys.readouterr()
-    stdout = cleanpath(all.out)
+    stdout = cleanpath(all.out, tmp_path)
     stdout = "\n".join(stdout.split("\n")[-15:-2])
     file_regression.check(stdout)
 
@@ -137,7 +140,7 @@ def test_milabench_bad_prepare(monkeypatch, tmp_path, config, capsys, file_regre
 
     # Check that the error was extracted
     all = capsys.readouterr()
-    stdout = cleanpath(all.out)
+    stdout = cleanpath(all.out, tmp_path)
     stdout = "\n".join(stdout.split("\n")[-12:-2])
     file_regression.check(stdout)
 
@@ -171,15 +174,13 @@ def test_milabench_bad_run(monkeypatch, tmp_path, config, capsys, file_regressio
 
     # Check that the error was extracted
     all = capsys.readouterr()
-    stdout = cleanpath(all.out)
+    stdout = cleanpath(all.out, tmp_path)
     stdout = "\n".join(stdout.split("\n")[-53:-2])
     file_regression.check(stdout)
 
 
 
 def test_milabench_bad_run_before_install(monkeypatch, tmp_path, config, capsys, file_regression):
-    from milabench.cli.dry import assume_gpu
-
     #
     # Do a bad run
     #
@@ -201,6 +202,6 @@ def test_milabench_bad_run_before_install(monkeypatch, tmp_path, config, capsys,
 
     # Check that the error was extracted
     all = capsys.readouterr()
-    stdout = cleanpath(all.out)
+    stdout = cleanpath(all.out, tmp_path)
     stdout = "\n".join(stdout.split("\n")[-53:-2])
     file_regression.check(stdout)
diff --git a/tests/test_mock/test_milabench_bad_run.txt b/tests/test_mock/test_milabench_bad_run.txt
@@ -5,17 +5,17 @@ benchio.0
   * 1 exceptions found
     * 1 x RuntimeError
         | Traceback (most recent call last):
-        |   File "/tmp/pytest-of-delaunap/pytest-35/test_milabench_bad_run0/venv/benchio/bin/voir", line 8, in <module>
+        |   File "$TMP/venv/benchio/bin/voir", line 8, in <module>
         |     sys.exit(main())
         |              ^^^^^^
-        |   File "/tmp/pytest-of-delaunap/pytest-35/test_milabench_bad_run0/venv/benchio/lib/python3.11/site-packages/voir/cli.py", line 124, in main
+        |   File "$TMP/venv/benchio/lib/python3.11/site-packages/voir/cli.py", line 124, in main
         |     ov(sys.argv[1:] if argv is None else argv)
-        |   File "/tmp/pytest-of-delaunap/pytest-35/test_milabench_bad_run0/venv/benchio/lib/python3.11/site-packages/voir/phase.py", line 331, in __call__
+        |   File "$TMP/venv/benchio/lib/python3.11/site-packages/voir/phase.py", line 331, in __call__
         |     self._run(*args, **kwargs)
-        |   File "/tmp/pytest-of-delaunap/pytest-35/test_milabench_bad_run0/venv/benchio/lib/python3.11/site-packages/voir/overseer.py", line 242, in _run
+        |   File "$TMP/venv/benchio/lib/python3.11/site-packages/voir/overseer.py", line 242, in _run
         |     set_value(func())
         |               ^^^^^^
-        |   File "/tmp/pytest-of-delaunap/pytest-35/test_milabench_bad_run0/venv/benchio/lib/python3.11/site-packages/voir/scriptutils.py", line 37, in <lambda>
+        |   File "$TMP/venv/benchio/lib/python3.11/site-packages/voir/scriptutils.py", line 37, in <lambda>
         |     return lambda: exec(mainsection, glb, glb)
         |                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^
         |   File "$SRC/milabench/tests/yoshua-benchio/main.py", line 46, in <module>
@@ -30,17 +30,17 @@ benchio.1
   * 1 exceptions found
     * 1 x RuntimeError
         | Traceback (most recent call last):
-        |   File "/tmp/pytest-of-delaunap/pytest-35/test_milabench_bad_run0/venv/benchio/bin/voir", line 8, in <module>
+        |   File "$TMP/venv/benchio/bin/voir", line 8, in <module>
         |     sys.exit(main())
         |              ^^^^^^
-        |   File "/tmp/pytest-of-delaunap/pytest-35/test_milabench_bad_run0/venv/benchio/lib/python3.11/site-packages/voir/cli.py", line 124, in main
+        |   File "$TMP/venv/benchio/lib/python3.11/site-packages/voir/cli.py", line 124, in main
         |     ov(sys.argv[1:] if argv is None else argv)
-        |   File "/tmp/pytest-of-delaunap/pytest-35/test_milabench_bad_run0/venv/benchio/lib/python3.11/site-packages/voir/phase.py", line 331, in __call__
+        |   File "$TMP/venv/benchio/lib/python3.11/site-packages/voir/phase.py", line 331, in __call__
         |     self._run(*args, **kwargs)
-        |   File "/tmp/pytest-of-delaunap/pytest-35/test_milabench_bad_run0/venv/benchio/lib/python3.11/site-packages/voir/overseer.py", line 242, in _run
+        |   File "$TMP/venv/benchio/lib/python3.11/site-packages/voir/overseer.py", line 242, in _run
         |     set_value(func())
         |               ^^^^^^
-        |   File "/tmp/pytest-of-delaunap/pytest-35/test_milabench_bad_run0/venv/benchio/lib/python3.11/site-packages/voir/scriptutils.py", line 37, in <lambda>
+        |   File "$TMP/venv/benchio/lib/python3.11/site-packages/voir/scriptutils.py", line 37, in <lambda>
         |     return lambda: exec(mainsection, glb, glb)
         |                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^
         |   File "$SRC/milabench/tests/yoshua-benchio/main.py", line 46, in <module>

diff --git a/tests/test_mock/test_milabench_bad_run_before_install.txt b/tests/test_mock/test_milabench_bad_run_before_install.txt
@@ -1,8 +1,8 @@
-benchio.0
+benchio.1
 =========
   * Error codes = 1
   * 1 exceptions found
-    * 1 x FileNotFoundError: [Errno 2] No such file or directory: '/tmp/pytest-of-delaunap/pytest-52/test_milabench_bad_run_before_0/venv/benchio/bin/voir'
+    * 1 x FileNotFoundError: [Errno 2] No such file or directory: '$TMP/venv/benchio/bin/voir'
         | Traceback (most recent call last):
         |   File "$SRC/milabench/milabench/commands/executors.py", line 35, in execute
         |     return await run(
@@ -22,12 +22,12 @@ benchio.0
         |     self._execute_child(args, executable, preexec_fn, close_fds,
         |   File "$SRC/conda/lib/python3.11/subprocess.py", line 1950, in _execute_child
         |     raise child_exception_type(errno_num, err_msg, err_filename)
-        | FileNotFoundError: [Errno 2] No such file or directory: '/tmp/pytest-of-delaunap/pytest-52/test_milabench_bad_run_before_0/venv/benchio/bin/voir'
-benchio.1
+        | FileNotFoundError: [Errno 2] No such file or directory: '$TMP/venv/benchio/bin/voir'
+benchio.0
 =========
   * Error codes = 1
   * 1 exceptions found
-    * 1 x FileNotFoundError: [Errno 2] No such file or directory: '/tmp/pytest-of-delaunap/pytest-52/test_milabench_bad_run_before_0/venv/benchio/bin/voir'
+    * 1 x FileNotFoundError: [Errno 2] No such file or directory: '$TMP/venv/benchio/bin/voir'
         | Traceback (most recent call last):
         |   File "$SRC/milabench/milabench/commands/executors.py", line 35, in execute
         |     return await run(
@@ -47,4 +47,4 @@ benchio.1
         |     self._execute_child(args, executable, preexec_fn, close_fds,
         |   File "$SRC/conda/lib/python3.11/subprocess.py", line 1950, in _execute_child
         |     raise child_exception_type(errno_num, err_msg, err_filename)
-        | FileNotFoundError: [Errno 2] No such file or directory: '/tmp/pytest-of-delaunap/pytest-52/test_milabench_bad_run_before_0/venv/benchio/bin/voir'
+        | FileNotFoundError: [Errno 2] No such file or directory: '$TMP/venv/benchio/bin/voir'