Add covalent cloud aws ec2 infra and report --push

covalent is not compatible with milabench as it requires sqlalchemy<2.0.0

benchmarks/_template/requirements.cpu.txt

@@ -0,0 +1,46 @@
+#
+# This file is autogenerated by pip-compile with Python 3.10
+# by the following command:
+#
+#    pip-compile --output-file=benchmarks/_template/requirements.cpu.txt benchmarks/_template/requirements.in
+#
+antlr4-python3-runtime==4.9.3
+    # via omegaconf
+asttokens==2.4.1
+    # via giving
+codefind==0.1.3
+    # via ptera
+executing==1.2.0
+    # via varname
+giving==0.4.2
+    # via
+    #   ptera
+    #   voir
+markdown-it-py==3.0.0
+    # via rich
+mdurl==0.1.2
+    # via markdown-it-py
+omegaconf==2.3.0
+    # via voir
+ovld==0.3.2
+    # via voir
+ptera==1.4.1
+    # via voir
+pygments==2.17.2
+    # via rich
+pynvml==11.5.0
+    # via voir
+pyyaml==6.0.1
+    # via omegaconf
+reactivex==4.0.4
+    # via giving
+rich==13.7.0
+    # via voir
+six==1.16.0
+    # via asttokens
+typing-extensions==4.10.0
+    # via reactivex
+varname==0.10.0
+    # via giving
+voir==0.2.12
+    # via -r benchmarks/_template/requirements.in

config/examples/ec2-system.yaml

@@ -0,0 +1,20 @@
+system:
+  # Nodes list
+  nodes:
+      # Alias used to reference the node
+    - name: manager
+      # Use 1.1.1.1 as an ip placeholder
+      ip: 1.1.1.1
+      # Use this node as the master node or not
+      main: true
+      # User to use in remote milabench operations
+      user: user
+
+  # Cloud instances profiles
+  cloud_profiles:
+    ec2:
+      profile: mb_test_sog_3
+      username: ubuntu
+      instance_type: t2.micro
+      volume_size: 8
+      region: us-east-2

config/test.yaml

@@ -0,0 +1,24 @@
+_defaults:
+  max_duration: 600
+  voir:
+    options:
+      stop: 60
+      interval: "1s"
+
+test:
+  inherits: _defaults
+  group: test_remote
+  install_group: test_remote
+  definition: ../benchmarks/_template
+  plan:
+    method: njobs
+    n: 1
+
+testing:
+  inherits: _defaults
+  definition: ../benchmarks/_template
+  group: test_remote_2
+  install_group: test_remote_2
+  plan:
+    method: njobs
+    n: 1

milabench/__init__.py

@@ -0,0 +1,5 @@
+import pathlib
+
+ROOT_FOLDER = pathlib.Path(__file__).resolve().parent.parent
+CONFIG_FOLDER = ROOT_FOLDER / "config"
+BENCHMARK_FOLDER = ROOT_FOLDER / "benchmarks"

milabench/cli/__init__.py

@@ -3,6 +3,7 @@
 
 from coleo import run_cli
 
+from .cloud import cli_cloud
 from .compare import cli_compare
 from .dev import cli_dev
 from .install import cli_install
@@ -12,6 +13,7 @@
 from .pr import cli_write_report_to_pr
 from .prepare import cli_prepare
 from .publish import cli_publish
+from .purge_cloud import cli_purge_cloud
 from .report import cli_report
 from .run import cli_run
 from .schedule import cli_schedule
@@ -37,6 +39,14 @@ def pin():
         """Pin the benchmarks' dependencies."""
         cli_pin()
 
+    def cloud():
+        """Setup cloud instances."""
+        cli_cloud()
+
+    def purge_cloud():
+        """Purge running cloud instannces the benchmarks' dependencies."""
+        cli_purge_cloud()
+
     def dev():
         """Create a shell in a benchmark's environment for development."""
         cli_dev()

milabench/cli/badges/__main__.py

@@ -0,0 +1,45 @@
+import pathlib
+import subprocess
+import sys
+
+
+def main(argv=None):
+    if argv is None:
+        argv = sys.argv[1:]
+
+    try:
+        import pybadges as _
+    except ImportError:
+        module = pathlib.Path(__file__).resolve().parent
+        cache_dir = pathlib.Path(f"/tmp/milabench/{module.name}_venv")
+        python3 = str(cache_dir / "bin/python3")
+        check_module = "import pybadges"
+        try:
+            subprocess.run([python3, "-c", check_module], check=True)
+        except (FileNotFoundError, subprocess.CalledProcessError):
+            cache_dir.mkdir(parents=True, exist_ok=True)
+            subprocess.run([sys.executable, "-m", "virtualenv", str(cache_dir)], check=True)
+            subprocess.run([python3, "-m", "pip", "install", "-U", "pip"], check=True)
+            subprocess.run([
+                python3,
+                "-m",
+                "pip",
+                "install",
+                "-r",
+                str(module / "requirements.txt")
+            ], check=True)
+            subprocess.run([python3, "-c", check_module], check=True)
+        return subprocess.call(
+            [python3, __file__, *argv],
+        )
+
+    return subprocess.run([
+        sys.executable,
+        "-m",
+        "pybadges",
+        *argv
+    ], check=True).returncode
+
+
+if __name__ == "__main__":
+    sys.exit(main())

milabench/cli/badges/requirements.txt

@@ -0,0 +1 @@
+pybadges

milabench/cli/cloud.py

@@ -0,0 +1,153 @@
+from copy import deepcopy
+import os
+import subprocess
+import sys
+
+from coleo import Option, tooled
+import yaml
+
+# import milabench as mb
+from ..common import get_multipack
+
+
+_SETUP = "setup"
+_TEARDOWN = "teardown"
+_LIST = "list"
+_ACTIONS = (_SETUP, _TEARDOWN, _LIST)
+
+
+def manage_cloud(pack, packs, run_on, action="setup"):
+    assert run_on in pack.config["system"]["cloud_profiles"]
+
+    key_map = {
+        "hostname":(lambda v: ("ip",v)),
+        "username":(lambda v: ("user",v)),
+        "ssh_key_file":(lambda v: ("key",v)),
+        "env":(lambda v: ("env",[".", v, ";", "conda", "activate", "milabench", "&&"])),
+    }
+    plan_params = deepcopy(pack.config["system"]["cloud_profiles"][run_on])
+
+    nodes = iter(enumerate(pack.config["system"]["nodes"]))
+
+    state_prefix = []
+    for p in packs.values():
+        state_prefix.append(p.config["name"])
+        state_prefix.append(p.config["install_variant"])
+
+    while True:
+        try:
+            i, n = next(nodes)
+            if n["ip"] != "1.1.1.1":
+                continue
+        except StopIteration:
+            break
+
+        plan_params["state_prefix"] = plan_params.get("state_prefix", None) or "-".join([str(i), *state_prefix])
+        plan_params["state_id"] = plan_params.get("state_id", None) or pack.config["hash"]
+
+        import milabench.cli.covalent as cv
+
+        subprocess.run(
+            [
+                sys.executable,
+                "-m", cv.__name__,
+                "serve", "start"
+            ]
+            , stdout=sys.stderr
+            , check=True
+        )
+
+        cmd = [
+            sys.executable,
+            "-m", cv.__name__,
+            run_on,
+            f"--{action}",
+            *[
+                f"--{k.replace('_', '-')}={v}"
+                for k, v in plan_params.items()
+            ],
+        ]
+        p = subprocess.Popen(
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+
+        stdout_chunks = []
+        while True:
+            line = p.stdout.readline()
+            if not line:
+                break
+            line_str = line.decode("utf-8").strip()
+            stdout_chunks.append(line_str)
+            print(line_str, file=sys.stderr)
+
+            if not line_str:
+                continue
+            try:
+                k, v = line_str.split("::>")
+                k, v = key_map[k](v)
+                if k == "ip" and n[k] != "1.1.1.1":
+                    i, n = next(nodes)
+                n[k] = v
+            except ValueError:
+                pass
+
+        _, stderr = p.communicate()
+        stderr = stderr.decode("utf-8").strip()
+        print(stderr, file=sys.stderr)
+
+        if p.returncode != 0:
+            stdout = os.linesep.join(stdout_chunks)
+            raise subprocess.CalledProcessError(
+                p.returncode,
+                cmd,
+                stdout,
+                stderr
+            )
+
+    return pack.config["system"]
+
+
+@tooled
+def _setup():
+    """Setup a cloud infrastructure"""
+
+    # Setup cloud on target infra
+    run_on: Option & str
+
+    mp = get_multipack()
+    setup_pack = mp.setup_pack()
+    system_config = manage_cloud(setup_pack, mp.packs, run_on, action=_SETUP)
+
+    print(f"# hash::>{setup_pack.config['hash']}")
+    print(yaml.dump({"system": system_config}))
+
+
+@tooled
+def _teardown():
+    """Teardown a cloud infrastructure"""
+
+    # Setup cloud on target infra
+    run_on: Option & str
+
+    mp = get_multipack()
+    setup_pack = mp.setup_pack()
+    manage_cloud(setup_pack, mp.packs, run_on, action=_TEARDOWN)
+
+
+@tooled
+def cli_cloud():
+    """Manage cloud instances."""
+
+    # Setup a cloud infrastructure
+    setup: Option & bool = False
+    # Teardown a cloud infrastructure
+    teardown: Option & bool = False
+
+    assert any((setup, teardown)) and not all((setup, teardown))
+
+    if setup:
+        _setup()
+    elif teardown:
+        _teardown()