From 4b5f192ce139d32dd46ba22c988faf905a86f2dc Mon Sep 17 00:00:00 2001 From: Setepenre Date: Tue, 5 Dec 2023 12:51:56 -0500 Subject: [PATCH] Refactor (#179) * Refactor * Ruff * Add missing updates --------- Co-authored-by: pierre.delaunay --- benchmarks/accelerate_opt/benchfile.py | 22 +- benchmarks/accelerate_opt/main.py | 6 +- benchmarks/flops/benchfile.py | 11 +- benchmarks/llama/benchfile.py | 7 +- benchmarks/rwkv/rwkv-v4neo/chat.py | 2 +- benchmarks/rwkv/rwkv-v4neo/img_demoAE.py | 8 +- benchmarks/rwkv/rwkv-v4neo/run.py | 4 +- benchmarks/rwkv/rwkv-v4neo/src/binidx.py | 2 - benchmarks/rwkv/rwkv-v4neo/src/dataset.py | 2 +- benchmarks/rwkv/rwkv-v4neo/src/model.py | 3 +- benchmarks/rwkv/rwkv-v4neo/src/model_img.py | 6 +- benchmarks/rwkv/rwkv-v4neo/src/model_run.py | 3 +- benchmarks/rwkv/rwkv-v4neo/src/trainer.py | 5 +- benchmarks/rwkv/rwkv-v4neo/train.py | 4 +- benchmarks/stargan/stargan/model.py | 1 - benchmarks/stargan/stargan/solver.py | 1 - benchmarks/stargan/stargan/synth.py | 1 - benchmarks/super-slomo/slomo/model.py | 3 - benchmarks/super-slomo/slomo/train.py | 2 - benchmarks/timm/benchfile.py | 5 +- milabench/cli.py | 2 - .../{executors.py => commands/__init__.py} | 212 ++++++------------ milabench/commands/executors.py | 74 ++++++ milabench/datasets/fake_images.py | 1 - milabench/metrics/sqlalchemy.py | 3 +- milabench/multi.py | 2 +- milabench/pack.py | 30 +-- milabench/remote.py | 54 ++--- milabench/report.py | 4 +- milabench/structs.py | 1 - milabench/utils.py | 1 - milabench/validation/error.py | 4 +- milabench/validation/loss.py | 4 +- milabench/validation/validation.py | 1 - tests/test_executors.py | 56 ++--- tests/test_metrics.py | 1 - tests/test_summary.py | 6 - tests/test_validation.py | 2 +- tests/yoshua-benchio/prepare.py | 1 - 39 files changed, 266 insertions(+), 291 deletions(-) rename milabench/{executors.py => commands/__init__.py} (76%) create mode 100644 milabench/commands/executors.py diff --git a/benchmarks/accelerate_opt/benchfile.py b/benchmarks/accelerate_opt/benchfile.py index c886e8f5f..56bd98bb9 100644 --- a/benchmarks/accelerate_opt/benchfile.py +++ b/benchmarks/accelerate_opt/benchfile.py @@ -1,9 +1,9 @@ -from milabench.executors import ( - AccelerateLaunchExecutor, - CmdExecutor, - DockerRunExecutor, - ListExecutor, - SSHExecutor, +from milabench.commands import ( + AccelerateLaunchCommand, + CmdCommand, + DockerRunCommand, + ListCommand, + SSHCommand, ) from milabench.pack import Package from milabench.utils import select_nodes @@ -18,7 +18,7 @@ def make_env(self): return env def build_prepare_plan(self): - return CmdExecutor( + return CmdCommand( self, "accelerate", "launch", @@ -59,19 +59,19 @@ def build_run_plan(self): tags.append("nolog") pack = self.copy({"tag": tags}) - worker = SSHExecutor( + worker = SSHCommand( host=host, user=user, key=key, - executor=DockerRunExecutor( - AccelerateLaunchExecutor(pack, rank=rank), + executor=DockerRunCommand( + AccelerateLaunchCommand(pack, rank=rank), self.config["system"].get("docker_image"), ), **options ) plans.append(worker) - return ListExecutor(*plans) + return ListCommand(*plans) __pack__ = AccelerateBenchmark diff --git a/benchmarks/accelerate_opt/main.py b/benchmarks/accelerate_opt/main.py index 65d749f0c..7d1a33e61 100644 --- a/benchmarks/accelerate_opt/main.py +++ b/benchmarks/accelerate_opt/main.py @@ -66,20 +66,18 @@ def arguments(): from dataclasses import dataclass from datetime import timedelta from itertools import chain -from pathlib import Path from typing import Optional import datasets import rich.logging import torch import transformers -from accelerate import Accelerator, DistributedType +from accelerate import Accelerator from accelerate.logging import get_logger from accelerate.utils import DummyOptim, DummyScheduler from accelerate.utils.dataclasses import InitProcessGroupKwargs from datasets import load_dataset from torch.utils.data import DataLoader -from tqdm.auto import tqdm from transformers import ( AutoConfig, AutoModelForCausalLM, @@ -88,7 +86,6 @@ def arguments(): default_data_collator, get_scheduler, ) -from transformers.utils import get_full_repo_name from voir.smuggle import SmuggleWriter from voir.instruments.gpu import get_gpu_info from voir.instruments.utils import Monitor @@ -374,7 +371,6 @@ def group_texts(examples): completed_steps = 0 starting_epoch = 0 - best_metric = None last_log_time = time.time() for epoch in range(starting_epoch, num_train_epochs): diff --git a/benchmarks/flops/benchfile.py b/benchmarks/flops/benchfile.py index 0bb601d67..aa82e6e38 100644 --- a/benchmarks/flops/benchfile.py +++ b/benchmarks/flops/benchfile.py @@ -6,13 +6,12 @@ class FlopsBenchmarch(Package): prepare_script = "prepare.py" main_script = "main.py" - def build_run_plan(self) -> "execs.Executor": - import milabench.executors as execs + def build_run_plan(self) -> "Command": + import milabench.commands as cmd - main = self.dirs.code / self.main_script - pack = execs.PackExecutor(self, *self.argv, lazy=True) - # pack = execs.VoirExecutor(pack, cwd=main.parent) - pack = execs.ActivatorExecutor(pack, use_stdout=True) + pack = cmd.PackCommand(self, *self.argv, lazy=True) + # pack = cmd.VoirCommand(pack, cwd=main.parent) + pack = cmd.ActivatorCommand(pack, use_stdout=True) return pack diff --git a/benchmarks/llama/benchfile.py b/benchmarks/llama/benchfile.py index 8b253bc92..541c6288f 100644 --- a/benchmarks/llama/benchfile.py +++ b/benchmarks/llama/benchfile.py @@ -1,6 +1,5 @@ -import uuid -from milabench.executors import CmdExecutor +from milabench.commands import CmdCommand from milabench.pack import Package @@ -18,7 +17,7 @@ async def install(self): await super().install() def build_prepare_plan(self): - return CmdExecutor( + return CmdCommand( self, "python", str(self.dirs.code / "main.py"), @@ -29,7 +28,7 @@ def build_prepare_plan(self): ) def build_run_plan(self): - return CmdExecutor( + return CmdCommand( self, "python", str(self.dirs.code / "main.py"), diff --git a/benchmarks/rwkv/rwkv-v4neo/chat.py b/benchmarks/rwkv/rwkv-v4neo/chat.py index 19e2b36f9..00f66085f 100644 --- a/benchmarks/rwkv/rwkv-v4neo/chat.py +++ b/benchmarks/rwkv/rwkv-v4neo/chat.py @@ -188,7 +188,7 @@ def load_all_stat(srv, name): ######################################################################################################## # Run inference -print(f"\nRun prompt...") +print("\nRun prompt...") out = run_rnn(tokenizer.tokenizer.encode(init_prompt)) gc.collect() diff --git a/benchmarks/rwkv/rwkv-v4neo/img_demoAE.py b/benchmarks/rwkv/rwkv-v4neo/img_demoAE.py index 43c0c3cf3..cadde9569 100644 --- a/benchmarks/rwkv/rwkv-v4neo/img_demoAE.py +++ b/benchmarks/rwkv/rwkv-v4neo/img_demoAE.py @@ -2,7 +2,7 @@ # The RWKV Language Model - https://github.com/BlinkDL/RWKV-LM ######################################################################################################## -import torch, types, os +import torch, types import numpy as np from PIL import Image import torch.nn as nn @@ -11,7 +11,7 @@ import torchvision.transforms as transforms np.set_printoptions(precision=4, suppress=True, linewidth=200) -print(f"loading...") +print("loading...") ######################################################################################################## @@ -138,7 +138,7 @@ def forward(self, code): ######################################################################################################## -print(f"building model...") +print("building model...") args = types.SimpleNamespace() args.my_img_bit = 13 encoder = R_ENCODER(args).eval().cuda() @@ -151,7 +151,7 @@ def forward(self, code): ######################################################################################################## -print(f"test image...") +print("test image...") img_transform = transforms.Compose( [ transforms.PILToTensor(), diff --git a/benchmarks/rwkv/rwkv-v4neo/run.py b/benchmarks/rwkv/rwkv-v4neo/run.py index eb7109cb6..70707933d 100644 --- a/benchmarks/rwkv/rwkv-v4neo/run.py +++ b/benchmarks/rwkv/rwkv-v4neo/run.py @@ -3,7 +3,7 @@ ######################################################################################################## import numpy as np -import math, os, sys, types, time, gc +import os, sys, types, time, gc import torch from src.utils import TOKENIZER @@ -137,7 +137,7 @@ model = RWKV_RNN(args) -print(f"\nOptimizing speed...") +print("\nOptimizing speed...") out, _ = model.forward([187], None) # print(out) gc.collect() diff --git a/benchmarks/rwkv/rwkv-v4neo/src/binidx.py b/benchmarks/rwkv/rwkv-v4neo/src/binidx.py index 8d5b40bfe..8093f1346 100644 --- a/benchmarks/rwkv/rwkv-v4neo/src/binidx.py +++ b/benchmarks/rwkv/rwkv-v4neo/src/binidx.py @@ -1,8 +1,6 @@ -from lib2to3.pgen2 import token import os import torch import numpy as np -import shutil import struct from functools import lru_cache from itertools import accumulate diff --git a/benchmarks/rwkv/rwkv-v4neo/src/dataset.py b/benchmarks/rwkv/rwkv-v4neo/src/dataset.py index 6ddafb90b..28e9b3519 100644 --- a/benchmarks/rwkv/rwkv-v4neo/src/dataset.py +++ b/benchmarks/rwkv/rwkv-v4neo/src/dataset.py @@ -2,7 +2,7 @@ # The RWKV Language Model - https://github.com/BlinkDL/RWKV-LM ######################################################################################################## -import json, math, random, os, sys +import json, math, random import numpy as np import torch from torch.utils.data import Dataset diff --git a/benchmarks/rwkv/rwkv-v4neo/src/model.py b/benchmarks/rwkv/rwkv-v4neo/src/model.py index 0914c160e..25492885c 100644 --- a/benchmarks/rwkv/rwkv-v4neo/src/model.py +++ b/benchmarks/rwkv/rwkv-v4neo/src/model.py @@ -10,7 +10,6 @@ import torch.nn as nn from torch.nn import functional as F import pytorch_lightning as pl -from pytorch_lightning.utilities import rank_zero_info, rank_zero_only from pytorch_lightning.strategies import DeepSpeedStrategy if importlib.util.find_spec("deepspeed"): @@ -713,7 +712,7 @@ def training_step_end(self, batch_parts): def generate_init_weight(self): print( - f""" + """ ############################################################################ # # Init model weight (slow for large models)... diff --git a/benchmarks/rwkv/rwkv-v4neo/src/model_img.py b/benchmarks/rwkv/rwkv-v4neo/src/model_img.py index 3a9bceb4e..923b72605 100644 --- a/benchmarks/rwkv/rwkv-v4neo/src/model_img.py +++ b/benchmarks/rwkv/rwkv-v4neo/src/model_img.py @@ -3,15 +3,13 @@ ######################################################################################################## import numpy as np -import os, math, gc +import os, gc import torch import torch.nn as nn import torch.nn.functional as F import torchvision as vision import pytorch_lightning as pl -from pytorch_lightning.utilities import rank_zero_info, rank_zero_only from pytorch_lightning.strategies import DeepSpeedStrategy -import deepspeed from deepspeed.ops.adam import DeepSpeedCPUAdam, FusedAdam # from pytorch_msssim import MS_SSIM @@ -425,7 +423,7 @@ def training_step_end(self, batch_parts): def generate_init_weight(self): print( - f""" + """ ############################################################################ # # Init model weight (slow for large models)... diff --git a/benchmarks/rwkv/rwkv-v4neo/src/model_run.py b/benchmarks/rwkv/rwkv-v4neo/src/model_run.py index 184a35cfa..598ceb0e5 100644 --- a/benchmarks/rwkv/rwkv-v4neo/src/model_run.py +++ b/benchmarks/rwkv/rwkv-v4neo/src/model_run.py @@ -4,10 +4,9 @@ import types import torch -import math, os, gc +import os, gc from torch.nn import functional as F import torch.nn as nn -from typing import List, Dict MyModule = nn.Module diff --git a/benchmarks/rwkv/rwkv-v4neo/src/trainer.py b/benchmarks/rwkv/rwkv-v4neo/src/trainer.py index 98f229c40..833ed9f81 100644 --- a/benchmarks/rwkv/rwkv-v4neo/src/trainer.py +++ b/benchmarks/rwkv/rwkv-v4neo/src/trainer.py @@ -1,8 +1,7 @@ -import os, math, time, datetime, subprocess +import math, time, datetime import torch -from torch.utils.data import DataLoader import pytorch_lightning as pl -from pytorch_lightning.utilities import rank_zero_info, rank_zero_only +from pytorch_lightning.utilities import rank_zero_only from giving import give diff --git a/benchmarks/rwkv/rwkv-v4neo/train.py b/benchmarks/rwkv/rwkv-v4neo/train.py index 875d9c4eb..7ab1301f1 100644 --- a/benchmarks/rwkv/rwkv-v4neo/train.py +++ b/benchmarks/rwkv/rwkv-v4neo/train.py @@ -7,7 +7,7 @@ import os from argparse import ArgumentParser from pytorch_lightning import Trainer - from pytorch_lightning.utilities import rank_zero_info, rank_zero_only + from pytorch_lightning.utilities import rank_zero_info rank_zero_info("########## work in progress ##########") @@ -152,7 +152,7 @@ ######################################################################################################## - import os, warnings, math, datetime, sys, time, importlib + import os, warnings, datetime, importlib import numpy as np import torch from torch.utils.data import DataLoader diff --git a/benchmarks/stargan/stargan/model.py b/benchmarks/stargan/stargan/model.py index a9ecb43e3..2083770c4 100644 --- a/benchmarks/stargan/stargan/model.py +++ b/benchmarks/stargan/stargan/model.py @@ -1,6 +1,5 @@ import torch import torch.nn as nn -import torch.nn.functional as F import numpy as np diff --git a/benchmarks/stargan/stargan/solver.py b/benchmarks/stargan/stargan/solver.py index d45bb6f9e..7bce3f8fb 100644 --- a/benchmarks/stargan/stargan/solver.py +++ b/benchmarks/stargan/stargan/solver.py @@ -1,6 +1,5 @@ from model import Generator from model import Discriminator -from torch.autograd import Variable from torchvision.utils import save_image import torch import torch.nn.functional as F diff --git a/benchmarks/stargan/stargan/synth.py b/benchmarks/stargan/stargan/synth.py index 7add08be5..a6d4a4400 100644 --- a/benchmarks/stargan/stargan/synth.py +++ b/benchmarks/stargan/stargan/synth.py @@ -1,4 +1,3 @@ -import torch class SyntheticData: diff --git a/benchmarks/super-slomo/slomo/model.py b/benchmarks/super-slomo/slomo/model.py index ef706a2a2..c160d0839 100644 --- a/benchmarks/super-slomo/slomo/model.py +++ b/benchmarks/super-slomo/slomo/model.py @@ -1,7 +1,4 @@ import torch -import torchvision -import torchvision.transforms as transforms -import torch.optim as optim import torch.nn as nn import torch.nn.functional as F import numpy as np diff --git a/benchmarks/super-slomo/slomo/train.py b/benchmarks/super-slomo/slomo/train.py index 7fea1a045..ccc3e6da1 100644 --- a/benchmarks/super-slomo/slomo/train.py +++ b/benchmarks/super-slomo/slomo/train.py @@ -4,12 +4,10 @@ import argparse import torch import torchvision -import torchvision.transforms as transforms import torch.optim as optim import torch.nn as nn import torch.nn.functional as F import model -import dataloader from giving import give import voir from synth import SyntheticData diff --git a/benchmarks/timm/benchfile.py b/benchmarks/timm/benchfile.py index f8d0652e0..484a255a6 100644 --- a/benchmarks/timm/benchfile.py +++ b/benchmarks/timm/benchfile.py @@ -1,6 +1,5 @@ -import uuid -from milabench.executors import TorchRunExecutor +from milabench.commands import TorchRunCommand from milabench.pack import Package @@ -43,7 +42,7 @@ async def install(self): def build_run_plan(self): # self.config is not the right config for this plan = super().build_run_plan() - return TorchRunExecutor(plan, use_stdout=True) + return TorchRunCommand(plan, use_stdout=True) __pack__ = TimmBenchmarkPack diff --git a/milabench/cli.py b/milabench/cli.py index 5b6a7599e..6c60d71a8 100644 --- a/milabench/cli.py +++ b/milabench/cli.py @@ -3,10 +3,8 @@ import re import io import runpy -import shutil import subprocess import sys -import tempfile import traceback from datetime import datetime import getpass diff --git a/milabench/executors.py b/milabench/commands/__init__.py similarity index 76% rename from milabench/executors.py rename to milabench/commands/__init__.py index 1b991e23f..a5decd8d2 100644 --- a/milabench/executors.py +++ b/milabench/commands/__init__.py @@ -4,90 +4,47 @@ import os import json from copy import deepcopy -import warnings from hashlib import md5 from typing import Dict, Generator, List, Tuple from voir.instruments.gpu import get_gpu_info -from . import pack -from .alt_async import destroy, run -from .fs import XPath -from .merge import merge -from .metadata import machine_metadata -from .structs import BenchLogEntry -from .utils import select_nodes - +from .. import pack +from ..fs import XPath +from ..merge import merge +from ..utils import select_nodes +from .executors import execute_command def clone_with(cfg, new_cfg): return merge(deepcopy(cfg), new_cfg) -async def force_terminate(pack, delay): - await asyncio.sleep(delay) - for proc in pack.processes: - ret = proc.poll() - if ret is None: - await pack.message( - f"Terminating process because it ran for longer than {delay} seconds." - ) - destroy(proc) - - -async def execute(pack, *args, cwd=None, env={}, external=False, **kwargs): - """Run a command in the virtual environment. - - Unless specified otherwise, the command is run with - ``self.dirs.code`` as the cwd. - - Arguments: - args: The arguments to the command - cwd: The cwd to use (defaults to ``self.dirs.code``) - """ - args = [str(x) for x in args] - if cwd is None: - cwd = pack.dirs.code - - exec_env = pack.full_env(env) if not external else {**os.environ, **env} - - return await run( - args, - **kwargs, - info={"pack": pack}, - env=exec_env, - constructor=BenchLogEntry, - cwd=cwd, - process_accumulator=pack.processes, - ) - - -# TODO Move this to command -class Executor: +class Command: """Base class for an execution plan - Will recursively go through its embedded `Executor` to build a command - line's arguments list to be passed to the leaf `Executor`'s - `BasePackage.execute()` + Will recursively go through its embedded `Command` to build a command + line's arguments list to be passed to the leaf `Command`'s + `pack.BasePackage.execute()` Arguments: - pack_or_exec: `Executor` or `BasePackage`. If a `BasePackage`, the + pack_or_exec: `Command` or `pack.BasePackage`. If a `pack.BasePackage`, the instance's `execute()` will be used to perform the commands calls **kwargs: kwargs to be passed to the `pack_or_exec.execute()`, if a - `BasePackage` + `pack.BasePackage` Notes: All dynamic operations need to happen inside the argv/_argv methods. Those methods are guaranteed to be called with the final configuration. """ - def __init__(self, pack_or_exec: Executor | pack.BasePackage, **kwargs) -> None: + def __init__(self, pack_or_exec: Command | pack.BasePackage, **kwargs) -> None: self._pack = None self.exec = None # used to know if the command is executed through SSH or locally self.remote = False - if isinstance(pack_or_exec, Executor): + if isinstance(pack_or_exec, Command): self.exec = pack_or_exec self._pack = None elif isinstance(pack_or_exec, pack.BasePackage): @@ -127,10 +84,10 @@ def copy(self, pack): return copy def kwargs(self) -> Dict: - """Return the `Executor`'s kwargs to send to `BasePackage.execute()` - merged with the embeded `Executor`, if any + """Return the `Command`'s kwargs to send to `pack.BasePackage.execute()` + merged with the embeded `Command`, if any - The `Executor`'s kwargs will take priority over the embeded `Executor`'s + The `Command`'s kwargs will take priority over the embeded `Command`'s kwargs """ kwargs = self._kwargs @@ -139,47 +96,24 @@ def kwargs(self) -> Dict: return kwargs def commands(self) -> Generator[Tuple[pack.BasePackage, List, Dict], None, None]: - """Return a tuple of the leaf's `BasePackage`, the `Executor`'s list of - command line's arguments and the `Executor`'s kwargs to send to - `BasePackage.execute()` + """Return a tuple of the leaf's `pack.BasePackage`, the `Command`'s list of + command line's arguments and the `Command`'s kwargs to send to + `pack.BasePackage.execute()` """ yield self.pack, [], self.kwargs() async def execute(self, phase="run", timeout=False, timeout_delay=600, **kwargs): """Execute all the commands and return the aggregated results""" - coro = [] - - for pack in self.packs(): - pack.phase = phase - - timeout_tasks = [] - for pack, argv, _kwargs in self.commands(): - await pack.send(event="config", data=pack.config) - await pack.send(event="meta", data=machine_metadata(pack)) - - fut = execute(pack, *argv, **{**_kwargs, **kwargs}) - coro.append(fut) - - if timeout: - delay = pack.config.get("max_duration", timeout_delay) - timeout_task = asyncio.create_task(force_terminate(pack, delay)) - timeout_tasks.append(timeout_task) - - results = await asyncio.gather(*coro) - - if timeout: - for task in timeout_tasks: - task.cancel() - return results + return await execute_command(self, phase, timeout, timeout_delay, **kwargs) -class SingleCmdExecutor(Executor): +class SingleCmdCommand(Command): def argv(self, **kwargs) -> List: - """Return the list of command line's arguments for this `Executor` - followed by its embedded `Executor`'s list of command line's arguments + """Return the list of command line's arguments for this `Command` + followed by its embedded `Command`'s list of command line's arguments Arguments: - **kwargs: some `Executor` might need an argument to dynamically + **kwargs: some `Command` might need an argument to dynamically generate the list of command line's arguments """ if self.exec: @@ -194,15 +128,15 @@ def _argv(self, **kwargs) -> List: return [] -class ListExecutor(Executor): - """Execute a list of `Executor`s in parallel +class ListCommand(Command): + """Execute a list of `Command`s in parallel Arguments: - executors: `Executor`s to be executed + executors: `Command`s to be executed **kwargs: kwargs to be passed to the `pack.execute()` """ - def __init__(self, *executors: Tuple[Executor], **kwargs) -> None: + def __init__(self, *executors: Tuple[Command], **kwargs) -> None: super().__init__(None, **kwargs) self.executors = executors @@ -219,21 +153,21 @@ def packs(self): yield from exec.packs() -class CmdExecutor(SingleCmdExecutor): +class CmdCommand(SingleCmdCommand): """Execute a command Arguments: - pack: `BasePackage`'s instance from which `execute()` will be used to + pack: `pack.BasePackage`'s instance from which `execute()` will be used to perform the command *cmd_argv: command line arguments list to execute **kwargs: kwargs to be passed to the `pack.execute()` """ def __init__(self, pack: pack.BasePackage, *cmd_argv, **kwargs) -> None: - if isinstance(pack, Executor): + if isinstance(pack, Command): raise TypeError( f"{self.__class__.__name__} does not accept nested" - f" {Executor.__name__}" + f" {Command.__name__}" ) super().__init__(pack, **kwargs) self.cmd_argv = cmd_argv @@ -242,7 +176,7 @@ def _argv(self, **_) -> List: return [*self.cmd_argv] -class PackExecutor(CmdExecutor): +class PackCommand(CmdCommand): """Execute a `Package`'s script. If not specified, the `Package`'s main_script will be used @@ -293,23 +227,23 @@ def _argv(self, **kwargs) -> List: return script + self.command_arguments() -class VoidExecutor(CmdExecutor): +class VoidCommand(CmdCommand): """Execute nothing""" def __init__(self, pack: pack.BasePackage, *argv, **kwargs) -> None: super().__init__(pack, "true", *argv, **kwargs) -class WrapperExecutor(SingleCmdExecutor): - """Wrap an `Executor` with any command +class WrapperCommand(SingleCmdCommand): + """Wrap an `Command` with any command Arguments: - executor: `Executor` to be executed + executor: `Command` to be executed *wrapper_argv: command line arguments list **kwargs: kwargs to be passed to the `pack.execute()` """ - def __init__(self, executor: SingleCmdExecutor, *wrapper_argv, **kwargs) -> None: + def __init__(self, executor: SingleCmdCommand, *wrapper_argv, **kwargs) -> None: super().__init__(executor, **kwargs) self.wrapper_argv = wrapper_argv @@ -318,18 +252,18 @@ def _argv(self, **kwargs) -> List: return [*self.wrapper_argv] -class DockerRunExecutor(WrapperExecutor): - """Execute an `Executor` through Docker +class DockerRunCommand(WrapperCommand): + """Execute an `Command` through Docker Arguments: - executor: `Executor` to be executed through Docker + executor: `Command` to be executed through Docker image: the Docker image to use *docker_argv: Docker command line arguments list **kwargs: kwargs to be passed to the `pack.execute()` """ def __init__( - self, executor: SingleCmdExecutor, image: str, *docker_argv, **kwargs + self, executor: SingleCmdCommand, image: str, *docker_argv, **kwargs ) -> None: super().__init__( executor, @@ -359,11 +293,11 @@ def as_container_path(self, path): return path def argv(self, **kwargs) -> List: - """Return the list of command line's arguments for this `Executor` - followed by its embedded `Executor`'s list of command line's arguments + """Return the list of command line's arguments for this `Command` + followed by its embedded `Command`'s list of command line's arguments Arguments: - **kwargs: some `Executor` might need an argument to dynamically + **kwargs: some `Command` might need an argument to dynamically generate the list of command line's arguments """ script_args = self.exec.argv(**kwargs) @@ -403,11 +337,11 @@ def _argv(self, **kwargs) -> List: return argv -class SSHExecutor(WrapperExecutor): - """Execute an `Executor` through ssh +class SSHCommand(WrapperCommand): + """Execute an `Command` through ssh Arguments: - executor: `Executor` to be executed through ssh + executor: `Command` to be executed through ssh host: host's address *ssh_argv: ssh command line arguments list user: username to use to connect to the host. By default, `pack.config` @@ -422,7 +356,7 @@ class SSHExecutor(WrapperExecutor): def __init__( self, - executor: SingleCmdExecutor, + executor: SingleCmdCommand, host: str, *ssh_argv, user: str = None, @@ -485,7 +419,7 @@ def _argv(self, **kwargs) -> List: return argv -class SCPExecutor(SSHExecutor, CmdExecutor): +class SCPCommand(SSHCommand, CmdCommand): _BIN = "scp" def __init__( @@ -511,8 +445,8 @@ def _argv(self, **kwargs) -> List: return argv -class TorchRunExecutor(WrapperExecutor): - def __init__(self, executor: SingleCmdExecutor, *torchrun_argv, **kwargs) -> None: +class TorchRunCommand(WrapperCommand): + def __init__(self, executor: SingleCmdCommand, *torchrun_argv, **kwargs) -> None: super().__init__(executor, "torchrun", *torchrun_argv, **kwargs) def _argv(self, **kwargs): @@ -529,16 +463,16 @@ def _argv(self, **kwargs): return [] -class VoirExecutor(WrapperExecutor): - """Execute an `Executor` through voir +class VoirCommand(WrapperCommand): + """Execute an `Command` through voir Arguments: - executor: `Executor` to be executed + executor: `Command` to be executed *voir_argv: voir command line arguments list **kwargs: kwargs to be passed to the `pack.execute()` """ - def __init__(self, executor: SingleCmdExecutor, *voir_argv, **kwargs) -> None: + def __init__(self, executor: SingleCmdCommand, *voir_argv, **kwargs) -> None: super().__init__(executor, "voir", **{"setsid": True, **kwargs}) self.voir_argv = voir_argv @@ -564,16 +498,16 @@ def _argv(self, **kwargs) -> List: ] -class NJobs(ListExecutor): - """Execute n instances of the same `Executor` in parallel +class NJobs(ListCommand): + """Execute n instances of the same `Command` in parallel Arguments: - executor: `Executor` to be executed + executor: `Command` to be executed n: number of times `executor` should be executed **kwargs: kwargs to be passed to the `pack.execute()` """ - def __init__(self, executor: Executor, n: int, gpus: list = None, **kwargs) -> None: + def __init__(self, executor: Command, n: int, gpus: list = None, **kwargs) -> None: self.n = n if gpus is None: gpus = [] @@ -594,10 +528,10 @@ def __init__(self, executor: Executor, n: int, gpus: list = None, **kwargs) -> N super().__init__(*executors, **kwargs) -class SequenceExecutor(ListExecutor): - """Execute a list of `Executor`s in sequence +class SequenceCommand(ListCommand): + """Execute a list of `Command`s in sequence Arguments: - executors: `Executor`s to be executed + executors: `Command`s to be executed **kwargs: kwargs to be passed to the `pack.execute()` """ @@ -626,15 +560,15 @@ def on_message(msg): return error_count -class PerGPU(ListExecutor): - """Execute one instance of an `Executor` on each gpu +class PerGPU(ListCommand): + """Execute one instance of an `Command` on each gpu Arguments: - executor: `Executor` to be executed + executor: `Command` to be executed **kwargs: kwargs to be passed to the `pack.execute()` """ - def __init__(self, executor: Executor, gpus: list = None, **kwargs) -> None: + def __init__(self, executor: Command, gpus: list = None, **kwargs) -> None: if gpus is None: gpus = [{"device": 0, "selection_variable": "CPU_VISIBLE_DEVICE"}] @@ -663,7 +597,7 @@ def __init__(self, executor: Executor, gpus: list = None, **kwargs) -> None: # I think if we use python script.py it will load # the right env and we do not need the activator # -class ActivatorExecutor(SingleCmdExecutor): +class ActivatorCommand(SingleCmdCommand): def __init__(self, pack: pack.BasePackage, **kwargs): super().__init__(pack, **kwargs) @@ -672,11 +606,11 @@ def _argv(self, **_) -> List: # Accelerate -class AccelerateLaunchExecutor(SingleCmdExecutor): - """Execute a `BasePackage` with Accelerate +class AccelerateLaunchCommand(SingleCmdCommand): + """Execute a `pack.BasePackage` with Accelerate Arguments: - pack: `BasePackage`'s instance from which `execute()` will be used to + pack: `pack.BasePackage`'s instance from which `execute()` will be used to perform the command *accelerate_argv: Accelerate's command line arguments list **kwargs: kwargs to be passed to the `pack.execute()` @@ -715,9 +649,9 @@ def _argv(self, **_) -> List: return [ # -- Run the command in the right venv - # This could be inside the SSH Executor + # This could be inside the SSH Command # but it would need to be repeated for Docker - # could be its own Executor like VenvExecutor that execute code + # could be its own Command like VenvCommand that execute code # inside a specifc venv f"{self.pack.dirs.code / 'activator'}", f"{self.pack.dirs.venv}", diff --git a/milabench/commands/executors.py b/milabench/commands/executors.py new file mode 100644 index 000000000..7974395b2 --- /dev/null +++ b/milabench/commands/executors.py @@ -0,0 +1,74 @@ + +import asyncio +import os + + +from ..alt_async import run, destroy +from ..structs import BenchLogEntry +from ..metadata import machine_metadata + + +async def execute(pack, *args, cwd=None, env={}, external=False, **kwargs): + """Run a command in the virtual environment. + + Unless specified otherwise, the command is run with + ``self.dirs.code`` as the cwd. + + Arguments: + args: The arguments to the command + cwd: The cwd to use (defaults to ``self.dirs.code``) + """ + args = [str(x) for x in args] + if cwd is None: + cwd = pack.dirs.code + + exec_env = pack.full_env(env) if not external else {**os.environ, **env} + + return await run( + args, + **kwargs, + info={"pack": pack}, + env=exec_env, + constructor=BenchLogEntry, + cwd=cwd, + process_accumulator=pack.processes, + ) + + +async def force_terminate(pack, delay): + await asyncio.sleep(delay) + for proc in pack.processes: + ret = proc.poll() + if ret is None: + await pack.message( + f"Terminating process because it ran for longer than {delay} seconds." + ) + destroy(proc) + + +async def execute_command(command, phase="run", timeout=False, timeout_delay=600, **kwargs): + """Execute all the commands and return the aggregated results""" + coro = [] + + for pack in command.packs(): + pack.phase = phase + + timeout_tasks = [] + for pack, argv, _kwargs in command.commands(): + await pack.send(event="config", data=pack.config) + await pack.send(event="meta", data=machine_metadata(pack)) + + fut = execute(pack, *argv, **{**_kwargs, **kwargs}) + coro.append(fut) + + if timeout: + delay = pack.config.get("max_duration", timeout_delay) + timeout_task = asyncio.create_task(force_terminate(pack, delay)) + timeout_tasks.append(timeout_task) + + results = await asyncio.gather(*coro) + + if timeout: + for task in timeout_tasks: + task.cancel() + return results \ No newline at end of file diff --git a/milabench/datasets/fake_images.py b/milabench/datasets/fake_images.py index 6e1f411fb..6e7c14de6 100644 --- a/milabench/datasets/fake_images.py +++ b/milabench/datasets/fake_images.py @@ -26,7 +26,6 @@ def write(args): def generate(image_size, n, outdir): p_count = min(multiprocessing.cpu_count(), 8) - count = n // p_count pool = multiprocessing.Pool(p_count) for _ in tqdm( pool.imap_unordered(write, ((image_size, i, n, outdir) for i in range(n))), diff --git a/milabench/metrics/sqlalchemy.py b/milabench/metrics/sqlalchemy.py index f311ca36d..3866e7d86 100644 --- a/milabench/metrics/sqlalchemy.py +++ b/milabench/metrics/sqlalchemy.py @@ -6,7 +6,6 @@ import time from ..structs import BenchLogEntry -from ..metadata import machine_metadata from bson.json_util import dumps as to_json from bson.json_util import loads as from_json @@ -140,7 +139,7 @@ def create_database(uri): try: Base.metadata.create_all(engine) - except DBAPIError as err: + except DBAPIError: print("could not create database schema because of {err}") diff --git a/milabench/multi.py b/milabench/multi.py index 5ff20036d..8b721bf31 100644 --- a/milabench/multi.py +++ b/milabench/multi.py @@ -5,7 +5,7 @@ from voir.instruments.gpu import get_gpu_info -from .executors import NJobs, PerGPU +from .commands import NJobs, PerGPU from .fs import XPath from .utils import make_constraints_file from .pack import Package diff --git a/milabench/pack.py b/milabench/pack.py index ec7a8dac3..f911683bf 100644 --- a/milabench/pack.py +++ b/milabench/pack.py @@ -14,7 +14,7 @@ class defines good default behavior. from nox.sessions import Session, SessionRunner -from . import executors as execs +from . import commands as cmd from .alt_async import run, send from .fs import XPath from .merge import merge @@ -206,7 +206,7 @@ async def execute(self, *args, cwd=None, env={}, external=False, **kwargs): """ from .executors import execute - return execute(pack, *args, cwd=cwd, env=env, external=external, **kwargs) + return execute(self, *args, cwd=cwd, env=env, external=external, **kwargs) async def python(self, *args, **kwargs): """Run a Python script. @@ -241,15 +241,17 @@ async def voir(self, script, args=(), wrapper=[], cwd=None, **kwargs): Returns: A subprocess.Popen instance representing the running process. """ + from . import commands as cmd + if isinstance(script, list): - executor = execs.CmdExecutor(self, *script, *args) + executor = cmd.CmdCommand(self, *script, *args) else: if not XPath(script).is_absolute(): script = str(self.dirs.code / script) - executor = execs.PackExecutor(self, script, *args) + executor = cmd.PackCommand(self, script, *args) - voir = execs.VoirExecutor(executor, cwd=cwd, **kwargs) - wrapper = execs.WrapperExecutor(voir, *wrapper) + voir = cmd.VoirCommand(executor, cwd=cwd, **kwargs) + wrapper = cmd.WrapperCommand(voir, *wrapper) return await wrapper.execute() @@ -398,7 +400,9 @@ async def exec_pip_compile( self, requirements_file: XPath, input_files: XPath, argv=[] ): input_files = [relativize(inp) for inp in input_files] - return await execs.CmdExecutor( + from . import commands as cmd + + return await cmd.CmdCommand( self, "python3", "-m", @@ -429,14 +433,14 @@ async def prepare(self): assert self.phase == "prepare" return await self.build_prepare_plan().execute() - def build_prepare_plan(self) -> "execs.Executor": + def build_prepare_plan(self) -> "cmd.Command": if self.prepare_script is not None: prep = self.dirs.code / self.prepare_script if prep.exists(): - return execs.PackExecutor( + return cmd.PackCommand( self, prep, *self.argv, env=self.make_env(), cwd=prep.parent ) - return execs.VoidExecutor(self) + return cmd.VoidCommand(self) async def run(self): """Start the benchmark and return the running process. @@ -461,7 +465,7 @@ async def run(self): assert self.phase == "run" return await self.build_run_plan().execute() - def build_run_plan(self) -> "execs.Executor": + def build_run_plan(self) -> "cmd.Command": main = self.dirs.code / self.main_script - pack = execs.PackExecutor(self, *self.argv, lazy=True) - return execs.VoirExecutor(pack, cwd=main.parent) + pack = cmd.PackCommand(self, *self.argv, lazy=True) + return cmd.VoirCommand(pack, cwd=main.parent) diff --git a/milabench/remote.py b/milabench/remote.py index e2d03fe43..f399d7556 100644 --- a/milabench/remote.py +++ b/milabench/remote.py @@ -1,13 +1,13 @@ import os import sys -from .executors import ( - CmdExecutor, - SSHExecutor, - ListExecutor, - SequenceExecutor, - VoidExecutor, - Executor, +from .commands import ( + CmdCommand, + SSHCommand, + ListCommand, + SequenceCommand, + VoidCommand, + Command, ) @@ -43,19 +43,19 @@ def rsync(node, folder, dest=None) -> list: "rsync", "-av", "-e", - f"ssh -oCheckHostIP=no -oStrictHostKeyChecking=no", + "ssh -oCheckHostIP=no -oStrictHostKeyChecking=no", folder, f"{user}@{host}:{dest}", ] -def pip_install_milabench(pack, node, folder) -> SSHExecutor: +def pip_install_milabench(pack, node, folder) -> SSHCommand: host = node["ip"] user = node["user"] cmd = ["pip", "install", "-e", folder] - plan = CmdExecutor(pack, *cmd) - return SSHExecutor(plan, host=host, user=user) + plan = CmdCommand(pack, *cmd) + return SSHCommand(plan, host=host, user=user) def milabench_remote_sync(pack, worker): @@ -76,7 +76,7 @@ def should_run_for(worker, setup_for): return worker["main"] -def milabench_remote_setup_plan(pack, setup_for="worker") -> SequenceExecutor: +def milabench_remote_setup_plan(pack, setup_for="worker") -> SequenceCommand: """Copy milabench source files to remote Notes @@ -93,7 +93,7 @@ def milabench_remote_setup_plan(pack, setup_for="worker") -> SequenceExecutor: if should_run_for(node, setup_for): node_pack = worker_pack(pack, node) - copy.append(CmdExecutor(node_pack, *rsync(node, INSTALL_FOLDER))) + copy.append(CmdCommand(node_pack, *rsync(node, INSTALL_FOLDER))) node_packs.append(node_pack) @@ -102,9 +102,9 @@ def milabench_remote_setup_plan(pack, setup_for="worker") -> SequenceExecutor: if should_run_for(node, setup_for): install.append(pip_install_milabench(node_packs[i], node, INSTALL_FOLDER)) - return SequenceExecutor( - ListExecutor(*copy), - ListExecutor(*install), + return SequenceCommand( + ListCommand(*copy), + ListCommand(*install), ) @@ -120,7 +120,7 @@ def worker_pack(pack, worker): ) -def milabench_remote_command(pack, *command, run_for="worker") -> ListExecutor: +def milabench_remote_command(pack, *command, run_for="worker") -> ListCommand: nodes = pack.config["system"]["nodes"] key = pack.config["system"].get("sshkey") cmds = [] @@ -131,15 +131,15 @@ def milabench_remote_command(pack, *command, run_for="worker") -> ListExecutor: user = worker["user"] cmds.append( - SSHExecutor( - CmdExecutor(worker_pack(pack, worker), f"milabench", *command), + SSHCommand( + CmdCommand(worker_pack(pack, worker), "milabench", *command), host=host, user=user, key=key, ) ) - return ListExecutor(*cmds) + return ListCommand(*cmds) def is_multinode(pack): @@ -176,39 +176,39 @@ def _sanity(pack, setup_for): assert is_remote(pack), "Only a remote node can setup the main node" -def milabench_remote_install(pack, setup_for="worker") -> SequenceExecutor: +def milabench_remote_install(pack, setup_for="worker") -> SequenceCommand: """Copy milabench code, install milabench, execute milabench install""" _sanity(pack, setup_for) if is_worker(pack): - return VoidExecutor(pack) + return VoidCommand(pack) argv = sys.argv[2:] - return SequenceExecutor( + return SequenceCommand( milabench_remote_setup_plan(pack, setup_for), milabench_remote_command(pack, "install", *argv, run_for=setup_for), ) -def milabench_remote_prepare(pack, run_for="worker") -> Executor: +def milabench_remote_prepare(pack, run_for="worker") -> Command: """Execute milabench prepare""" _sanity(pack, run_for) if is_worker(pack): - return VoidExecutor(pack) + return VoidCommand(pack) argv = sys.argv[2:] return milabench_remote_command(pack, "prepare", *argv, run_for=run_for) -def milabench_remote_run(pack) -> Executor: +def milabench_remote_run(pack) -> Command: """Execute milabench run""" # already on the main node, the regular flow # will be executed if is_main_local(pack): - return VoidExecutor(pack) + return VoidCommand(pack) argv = sys.argv[2:] return milabench_remote_command(pack, "run", *argv) diff --git a/milabench/report.py b/milabench/report.py index 51a59d162..5d7d32057 100644 --- a/milabench/report.py +++ b/milabench/report.py @@ -139,7 +139,7 @@ def print(self, x): def section(self, title): self.text("") self.text(title) - self.text(f"-" * len(title)) + self.text("-" * len(title)) self.html(H.h2(title)) def title(self, title): @@ -276,7 +276,7 @@ def _score(column): out.print(df) if errdata: - out.section(f"Errors") + out.section("Errors") out._text(f"{len(errdata)} errors, details in HTML report.") boxid = 0 for filename, err in sorted(list(errdata.items())): diff --git a/milabench/structs.py b/milabench/structs.py index 7b3482180..0c262c1e0 100644 --- a/milabench/structs.py +++ b/milabench/structs.py @@ -1,4 +1,3 @@ -import asyncio from contextlib import asynccontextmanager from dataclasses import dataclass from itertools import count diff --git a/milabench/utils.py b/milabench/utils.py index cb058bec0..699d373d8 100644 --- a/milabench/utils.py +++ b/milabench/utils.py @@ -3,7 +3,6 @@ from contextlib import contextmanager, ExitStack import random import sys -import tempfile import importlib import pkgutil import traceback diff --git a/milabench/validation/error.py b/milabench/validation/error.py index 4ec408d4c..7dd08aeeb 100644 --- a/milabench/validation/error.py +++ b/milabench/validation/error.py @@ -2,7 +2,7 @@ from dataclasses import dataclass, field from typing import List -from .validation import ValidationLayer, BenchLogEntry +from .validation import ValidationLayer @dataclass @@ -102,7 +102,7 @@ def report(self, summary, short=True, **kwargs): for line in tracebacks[1:]: summary.add(" " + line) else: - summary.add(f"* No traceback info about the error") + summary.add("* No traceback info about the error") self.set_error_code(failures) return failures diff --git a/milabench/validation/loss.py b/milabench/validation/loss.py index 948f4cba0..3de29e280 100644 --- a/milabench/validation/loss.py +++ b/milabench/validation/loss.py @@ -1,5 +1,5 @@ from collections import defaultdict -from dataclasses import dataclass, field +from dataclasses import dataclass import math @@ -69,7 +69,7 @@ def report(self, summary, **kwargs): loss_count = warnings.loss_count if loss_count == 0: - summary.add(f"* No loss was found") + summary.add("* No loss was found") if nan_counts > 0: summary.add(f"* Loss was Nan {nan_counts} times") diff --git a/milabench/validation/validation.py b/milabench/validation/validation.py index 22536cfcd..e02e98cee 100644 --- a/milabench/validation/validation.py +++ b/milabench/validation/validation.py @@ -1,4 +1,3 @@ -import json from contextlib import contextmanager from dataclasses import dataclass, field diff --git a/tests/test_executors.py b/tests/test_executors.py index 2201e141b..dc432c861 100644 --- a/tests/test_executors.py +++ b/tests/test_executors.py @@ -3,19 +3,19 @@ import pytest -from milabench.executors import ( +from milabench.commands import ( PerGPU, - PackExecutor, - SingleCmdExecutor, - VoirExecutor, + PackCommand, + SingleCmdCommand, + VoirCommand, NJobs, - TorchRunExecutor, + TorchRunCommand, ) from milabench.cli import _get_multipack from milabench.alt_async import proceed -class ExecMock1(SingleCmdExecutor): +class ExecMock1(SingleCmdCommand): def __init__(self, pack_or_exec, *exec_argv, **kwargs) -> None: super().__init__(pack_or_exec, **kwargs) self.exec_argv = exec_argv @@ -119,7 +119,7 @@ def test_executor_execute(): def test_pack_executor(): # voir is not setup so we are not receiving anything - executor = PackExecutor(benchio(), "--start", "2", "--end", "20") + executor = PackCommand(benchio(), "--start", "2", "--end", "20") acc = 0 for r in proceed(executor.execute()): @@ -130,8 +130,8 @@ def test_pack_executor(): def test_voir_executor(): - executor = PackExecutor(benchio(), "--start", "2", "--end", "20") - voir = VoirExecutor(executor) + executor = PackCommand(benchio(), "--start", "2", "--end", "20") + voir = VoirCommand(executor) acc = 0 for r in proceed(voir.execute()): @@ -142,8 +142,8 @@ def test_voir_executor(): def test_timeout(): - executor = PackExecutor(benchio(), "--start", "2", "--end", "20", "--sleep", 20) - voir = VoirExecutor(executor) + executor = PackCommand(benchio(), "--start", "2", "--end", "20", "--sleep", 20) + voir = VoirCommand(executor) acc = 0 for r in proceed(voir.execute(timeout=True, timeout_delay=1)): @@ -154,8 +154,8 @@ def test_timeout(): def test_njobs_executor(): - executor = PackExecutor(benchio(), "--start", "2", "--end", "20") - voir = VoirExecutor(executor) + executor = PackCommand(benchio(), "--start", "2", "--end", "20") + voir = VoirCommand(executor) njobs = NJobs(voir, 5) acc = 0 @@ -170,15 +170,15 @@ def test_njobs_gpus_executor(): """Two GPUs so torch run IS used""" devices = mock_gpu_list() - try: - import torch - except ImportError: + from importlib.util import find_spec + + if find_spec("torch") is None: pytest.skip("Pytorch is not installed") - executor = PackExecutor(benchio(), "--start", "2", "--end", "20") - voir = VoirExecutor(executor) - torch = TorchRunExecutor(voir, use_stdout=True) - njobs = NJobs(torch, 1, devices) + executor = PackCommand(benchio(), "--start", "2", "--end", "20") + voir = VoirCommand(executor) + torchcmd = TorchRunCommand(voir, use_stdout=True) + njobs = NJobs(torchcmd, 1, devices) acc = 0 for r in proceed(njobs.execute()): @@ -194,9 +194,9 @@ def test_njobs_gpu_executor(): """One GPU, so torch run is not used""" devices = [mock_gpu_list()[0]] - executor = PackExecutor(benchio(), "--start", "2", "--end", "20") - voir = VoirExecutor(executor) - torch = TorchRunExecutor(voir, use_stdout=True) + executor = PackCommand(benchio(), "--start", "2", "--end", "20") + voir = VoirCommand(executor) + torch = TorchRunCommand(voir, use_stdout=True) njobs = NJobs(torch, 1, devices) acc = 0 @@ -212,7 +212,7 @@ def test_njobs_gpu_executor(): def test_njobs_novoir_executor(): - executor = PackExecutor(benchio(), "--start", "2", "--end", "20") + executor = PackCommand(benchio(), "--start", "2", "--end", "20") njobs = NJobs(executor, 5) acc = 0 @@ -233,8 +233,8 @@ def mock_gpu_list(): def test_per_gpu_executor(): devices = mock_gpu_list() - executor = PackExecutor(benchio(), "--start", "2", "--end", "20") - voir = VoirExecutor(executor) + executor = PackCommand(benchio(), "--start", "2", "--end", "20") + voir = VoirCommand(executor) plan = PerGPU(voir, devices) acc = 0 @@ -246,9 +246,9 @@ def test_per_gpu_executor(): def test_void_executor(): - from milabench.executors import VoidExecutor + from milabench.commands import VoidCommand - plan = VoirExecutor(VoidExecutor(benchio())) + plan = VoirCommand(VoidCommand(benchio())) for _ in proceed(plan.execute()): pass diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 270759193..805bb14b6 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -1,4 +1,3 @@ -from copy import deepcopy from milabench.utils import multilogger from milabench.testing import replay_run, show_diff diff --git a/tests/test_summary.py b/tests/test_summary.py index 702cf82d8..b717918ad 100644 --- a/tests/test_summary.py +++ b/tests/test_summary.py @@ -45,13 +45,7 @@ def get_output(data): def test_empty_summary(): - points = [ - "1. Errors stuff happened", - "2. Errors stuff happened", - "3. Errors stuff happened", - ] report = Summary() - with report.section("Errors"): with report.section("Bench"): pass diff --git a/tests/test_validation.py b/tests/test_validation.py index eb765508c..03be4342b 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -126,7 +126,7 @@ def update_ctx(): config("scaling"), ) sizer_global.set(sizer) - system = system_global.set({"gpu": {"capacity": "41920 MiB"}}) + system_global.set({"gpu": {"capacity": "41920 MiB"}}) ctx.run(update_ctx) layer = MemoryUsageExtractor() diff --git a/tests/yoshua-benchio/prepare.py b/tests/yoshua-benchio/prepare.py index 8e16ed2cd..3be72cb48 100755 --- a/tests/yoshua-benchio/prepare.py +++ b/tests/yoshua-benchio/prepare.py @@ -1,6 +1,5 @@ #!/usr/bin/env python -import os if __name__ == "__main__": print("Hello I am doing some data stuff!")