Skip to content

Commit

Permalink
[Core] Introduce DictConfigProvider interface for logging config (#45736
Browse files Browse the repository at this point in the history
)

Signed-off-by: Jiajun Yao <[email protected]>
  • Loading branch information
jjyao authored Jun 5, 2024
1 parent 99108ae commit d02c416
Show file tree
Hide file tree
Showing 5 changed files with 92 additions and 51 deletions.
1 change: 1 addition & 0 deletions doc/source/ray-core/api/core.rst
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ Core API
ray.shutdown
ray.is_initialized
ray.job_config.JobConfig
ray.LoggingConfig

Tasks
-----
Expand Down
30 changes: 0 additions & 30 deletions python/ray/_private/ray_logging/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,33 +45,3 @@ class LogKey(str, Enum):
FILENAME = "filename"
LINENO = "lineno"
EXC_TEXT = "exc_text"


LOG_MODE_DICT = {
"TEXT": lambda log_level: {
"version": 1,
"disable_existing_loggers": False,
"formatters": {
"text": {
"()": "ray._private.ray_logging.formatters.TextFormatter",
},
},
"filters": {
"core_context": {
"()": "ray._private.ray_logging.filters.CoreContextFilter",
},
},
"handlers": {
"console": {
"level": log_level,
"class": "logging.StreamHandler",
"formatter": "text",
"filters": ["core_context"],
},
},
"root": {
"level": log_level,
"handlers": ["console"],
},
},
}
4 changes: 4 additions & 0 deletions python/ray/_private/ray_logging/default_impl.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
def get_dict_config_provider():
from ray._private.ray_logging.logging_config import DefaultDictConfigProvider

return DefaultDictConfigProvider()
106 changes: 86 additions & 20 deletions python/ray/_private/ray_logging/logging_config.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,95 @@
from ray._private.ray_logging.constants import LOG_MODE_DICT
from abc import ABC, abstractmethod
from typing import Set

from ray._private.ray_logging import default_impl
from ray._private.ray_logging.formatters import TextFormatter
from ray._private.ray_logging.filters import CoreContextFilter
from ray.util.annotations import PublicAPI

from dataclasses import dataclass


class DictConfigProvider(ABC):
@abstractmethod
def get_supported_encodings(self) -> Set[str]:
raise NotImplementedError

@abstractmethod
def get_dict_config(self, encoding: str, log_level: str) -> dict:
raise NotImplementedError


class DefaultDictConfigProvider(DictConfigProvider):
def __init__(self):
self._dict_configs = {
"TEXT": lambda log_level: {
"version": 1,
"disable_existing_loggers": False,
"formatters": {
"text": {
"()": (
f"{TextFormatter.__module__}."
f"{TextFormatter.__qualname__}"
),
},
},
"filters": {
"core_context": {
"()": (
f"{CoreContextFilter.__module__}."
f"{CoreContextFilter.__qualname__}"
),
},
},
"handlers": {
"console": {
"level": log_level,
"class": "logging.StreamHandler",
"formatter": "text",
"filters": ["core_context"],
},
},
"root": {
"level": log_level,
"handlers": ["console"],
},
}
}

def get_supported_encodings(self) -> Set[str]:
return self._dict_configs.keys()

def get_dict_config(self, encoding: str, log_level: str) -> dict:
return self._dict_configs[encoding](log_level)


_dict_config_provider: DictConfigProvider = default_impl.get_dict_config_provider()


@PublicAPI(stability="alpha")
@dataclass
class LoggingConfig:
"""

encoding: str = "TEXT"
log_level: str = "INFO"

def __post_init__(self):
if self.encoding not in _dict_config_provider.get_supported_encodings():
raise ValueError(
f"Invalid encoding type: {self.encoding}. "
"Valid encoding types are: "
f"{list(_dict_config_provider.get_supported_encodings())}"
)

def _get_dict_config(self) -> dict:
"""Get the logging configuration based on the encoding type.
Returns:
dict: The logging configuration.
"""
return _dict_config_provider.get_dict_config(self.encoding, self.log_level)


LoggingConfig.__doc__ = f"""
Logging configuration for a Ray job. These configurations are used to set up the
root logger of the driver process and all Ray tasks and actor processes that belong
to the job.
Expand Down Expand Up @@ -35,24 +117,8 @@ def f():
2024-06-03 07:53:50,815 INFO test.py:11 -- This is a Ray task job_id=01000000 worker_id=0dbbbd0f17d5343bbeee8228fa5ff675fe442445a1bc06ec899120a8 node_id=577706f1040ea8ebd76f7cf5a32338d79fe442e01455b9e7110cddfc task_id=c8ef45ccd0112571ffffffffffffffffffffffff01000000
Args:
encoding: Encoding type for the logs. The valid value is 'TEXT'
encoding: Encoding type for the logs. The valid values are
{list(_dict_config_provider.get_supported_encodings())}
log_level: Log level for the logs. Defaults to 'INFO'. You can set
it to 'DEBUG' to receive more detailed debug logs.
""" # noqa: E501

encoding: str = "TEXT"
log_level: str = "INFO"

def __post_init__(self):
if self.encoding not in LOG_MODE_DICT:
raise ValueError(
f"Invalid encoding type: {self.encoding}. "
f"Valid encoding types are: {list(LOG_MODE_DICT.keys())}"
)

def _get_dict_config(self) -> dict:
"""Get the logging configuration based on the encoding type.
Returns:
dict: The logging configuration.
"""
return LOG_MODE_DICT[self.encoding](self.log_level)
2 changes: 1 addition & 1 deletion python/ray/_private/worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -1337,7 +1337,7 @@ def init(
"configure_logging" is true.
logging_config: [Experimental] Logging configuration will be applied to the
root loggers for both the driver process and all worker processes belonging
to the current job. See :class:`~LoggingConfig` for details.
to the current job. See :class:`~ray.LoggingConfig` for details.
log_to_driver: If true, the output from all of the worker
processes on all nodes will be directed to the driver.
namespace: A namespace is a logical grouping of jobs and named actors.
Expand Down

0 comments on commit d02c416

Please sign in to comment.