From fc4a7f23184dfb9c0bcb59398ab1156ee3817b75 Mon Sep 17 00:00:00 2001 From: imanjra Date: Tue, 9 May 2023 12:46:39 -0400 Subject: [PATCH 1/2] add timeout to operator execution --- fiftyone/core/config.py | 6 +++++ fiftyone/operators/decorators.py | 42 ++++++++++++++++++++++++++++++++ fiftyone/operators/executor.py | 10 ++++++-- fiftyone/operators/server.py | 2 +- 4 files changed, 57 insertions(+), 3 deletions(-) create mode 100644 fiftyone/operators/decorators.py diff --git a/fiftyone/core/config.py b/fiftyone/core/config.py index fc492a40bf..5e871194d3 100644 --- a/fiftyone/core/config.py +++ b/fiftyone/core/config.py @@ -112,6 +112,12 @@ def __init__(self, d=None): self.plugins_dir = self.parse_string( d, "plugins_dir", env_var="FIFTYONE_PLUGINS_DIR", default=None ) + self.operator_timeout = self.parse_int( + d, + "operator_timeout", + env_var="FIFTYONE_OPERATOR_TIMEOUT", + default=600, # 600 seconds (10 minutes) + ) self.dataset_zoo_manifest_paths = self.parse_path_array( d, "dataset_zoo_manifest_paths", diff --git a/fiftyone/operators/decorators.py b/fiftyone/operators/decorators.py new file mode 100644 index 0000000000..d025155b84 --- /dev/null +++ b/fiftyone/operators/decorators.py @@ -0,0 +1,42 @@ +import asyncio +import signal +from contextlib import contextmanager +from functools import wraps + + +def coroutine_timeout(seconds): + def decorator(func): + @wraps(func) + async def wrapper(*args, **kwargs): + try: + if asyncio.iscoroutinefunction(func): + return await asyncio.wait_for( + func(*args, **kwargs), timeout=seconds + ) + else: + raise TypeError( + f"Function {func.__name__} is not a coroutine function" + ) + except asyncio.TimeoutError: + raise_timeout_error(seconds) + + return wrapper + + return decorator + + +@contextmanager +def timeout(seconds: int): + signal.signal( + signal.SIGALRM, lambda signum, frame: raise_timeout_error(seconds) + ) + signal.alarm(seconds) + + try: + yield + finally: + signal.signal(signal.SIGALRM, signal.SIG_IGN) + + +def raise_timeout_error(seconds): + raise TimeoutError(f"Timeout occurred after {seconds} seconds") from None diff --git a/fiftyone/operators/executor.py b/fiftyone/operators/executor.py index 8129bf6b8d..303b607ea0 100644 --- a/fiftyone/operators/executor.py +++ b/fiftyone/operators/executor.py @@ -13,6 +13,8 @@ from .message import GeneratedMessage, MessageType import types as python_types import traceback +from .decorators import coroutine_timeout +import asyncio class InvocationRequest: @@ -71,7 +73,8 @@ def to_json(self): } -def execute_operator(operator_name, request_params): +@coroutine_timeout(seconds=fo.config.operator_timeout) +async def execute_operator(operator_name, request_params): """Executes the operator with the given name. Args: operator_name: the name of the operator @@ -92,7 +95,10 @@ def execute_operator(operator_name, request_params): if validation_ctx.invalid: return ExecutionResult(None, None, "Validation Error", validation_ctx) try: - raw_result = operator.execute(ctx) + if asyncio.iscoroutinefunction(operator.execute): + raw_result = await operator.execute(ctx) + else: + raw_result = operator.execute(ctx) except Exception as e: return ExecutionResult(None, executor, str(e)) diff --git a/fiftyone/operators/server.py b/fiftyone/operators/server.py index a09703d8e4..960d1dc395 100644 --- a/fiftyone/operators/server.py +++ b/fiftyone/operators/server.py @@ -66,7 +66,7 @@ async def post(self, request: Request, data: dict) -> dict: "loading_errors": registry.list_errors(), } raise HTTPException(status_code=404, detail=erroDetail) - result = execute_operator(operator_uri, data) + result = await execute_operator(operator_uri, data) json = result.to_json() if result.error is not None: print(result.error) From 5bb340665998c2372c534343d10c1f8cb0198062 Mon Sep 17 00:00:00 2001 From: imanjra Date: Tue, 9 May 2023 15:42:04 -0400 Subject: [PATCH 2/2] add operator timeout config to docs --- docs/source/user_guide/config.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/source/user_guide/config.rst b/docs/source/user_guide/config.rst index 0e8d4dabc9..56fd9e75fa 100644 --- a/docs/source/user_guide/config.rst +++ b/docs/source/user_guide/config.rst @@ -88,6 +88,9 @@ FiftyOne supports the configuration options described below: | `module_path` | `FIFTYONE_MODULE_PATH` | `None` | A list of modules that should be automatically imported whenever FiftyOne is imported. | | | | | See :ref:`this page ` for an example usage. | +-------------------------------+-------------------------------------+-------------------------------+----------------------------------------------------------------------------------------+ +| `operator_timeout` | `FIFTYONE_OPERATOR_TIMEOUT` | `600` | The timeout for execution of an operator. See :ref:`this page ` for more | +| | | | information. | ++-------------------------------+-------------------------------------+-------------------------------+----------------------------------------------------------------------------------------+ | `plugins_dir` | `FIFTYONE_PLUGINS_DIR` | `None` | A directory containing custom App plugins. See :ref:`this page ` for more | | | | | information. | +-------------------------------+-------------------------------------+-------------------------------+----------------------------------------------------------------------------------------+ @@ -148,6 +151,7 @@ and the CLI: "model_zoo_dir": "~/fiftyone/__models__", "model_zoo_manifest_paths": null, "module_path": null, + "operator_timeout": 600, "plugins_dir": null, "requirement_error_level": 0, "show_progress_bars": true, @@ -191,6 +195,7 @@ and the CLI: "model_zoo_dir": "~/fiftyone/__models__", "model_zoo_manifest_paths": null, "module_path": null, + "operator_timeout": 600, "plugins_dir": null, "requirement_error_level": 0, "show_progress_bars": true,