Organise modules by domain (#197)

This simplifies code and moves modules and classes by their respective
domains/components, roughly mapped on folders. This is a change from the
previous diamond-shaped cross-module dependency layout, which is harder
to evolve.

Main tenets be:

- any component can depend on `mixins` (staging are for SDK) and
`framework` (common modules).
- tight coupling is allowed within a single component. An example is
local group migration, which exposes a single high-level migration
toolkit (`from databricks.labs.ucx.workspace_access import
GroupMigrationToolkit`) and leaves implementation details private.
Another example is a toolkit to migrate from HMS to UC DDL.
- all execution is either triggered from `install` (which also handles
update) or `runtime` (as the entrypoint for jobs). Multiple toolkits may
be combined into a single Databricks Workflows (aka Jobs) - e.g. pulling
permissions and tables inventory is necessary for the assessment step,
but later the Workflows may branch out.
- if two classes are used together in 90% of cases, they have to be
defined in the single file (aka Python Module). We don't define `types`
and `managers` modules/packages, as this is an example of a
diamond-shaped dependencies, which are more difficult to evolve over
time.
- methods and fields have to be private by default. make them public
only by necessity.
- classes must have the following order - __init__, overridden methods,
public methods, other methods. the most important logic has to be first.

notebooks/toolkit.py

@@ -17,8 +17,8 @@
     MigrationConfig,
     TaclConfig,
 )
-from databricks.labs.ucx.toolkits.group_migration import GroupMigrationToolkit
-from databricks.labs.ucx.toolkits.table_acls import TaclToolkit
+from databricks.labs.ucx.workspace_access import GroupMigrationToolkit
+from databricks.labs.ucx.hive_metastore import TaclToolkit
 
 # COMMAND ----------
 

pyproject.toml

@@ -163,14 +163,14 @@ ban-relative-imports = "all"
     "ARG001" # tests may not use the provided fixtures
 ]
 
-"src/databricks/labs/ucx/providers/mixins/redash.py" = ["A002", "A003", "N815"]
+"src/databricks/labs/ucx/mixins/redash.py" = ["A002", "A003", "N815"]
 
 [tool.coverage.run]
 branch = true
 parallel = true
 
 [tool.coverage.report]
-omit = ["src/databricks/labs/ucx/providers/mixins/*"]
+omit = ["src/databricks/labs/ucx/mixins/*"]
 exclude_lines = [
     "no cov",
     "if __name__ == .__main__.:",

src/databricks/labs/ucx/__init__.py

@@ -1,3 +1,3 @@
-from databricks.labs.ucx.logger import _install
+from databricks.labs.ucx.framework.logger import _install
 
 _install()

src/databricks/labs/ucx/toolkits/assessment.py → src/databricks/labs/ucx/assessment/assessment.py

@@ -6,7 +6,7 @@
 from databricks.sdk.service.compute import Language
 
 from databricks.labs.ucx.assessment import commands
-from databricks.labs.ucx.providers.mixins.compute import CommandExecutor
+from databricks.labs.ucx.mixins.compute import CommandExecutor
 
 logger = logging.getLogger(__name__)
 

src/databricks/labs/ucx/tacl/_internal.py → src/databricks/labs/ucx/framework/crawlers.py

@@ -6,7 +6,7 @@
 
 from databricks.sdk import WorkspaceClient
 
-from databricks.labs.ucx.providers.mixins.sql import StatementExecutionExt
+from databricks.labs.ucx.mixins.sql import StatementExecutionExt
 
 logger = logging.getLogger(__name__)
 

src/databricks/labs/ucx/utils.py → src/databricks/labs/ucx/framework/parallel.py

@@ -5,8 +5,6 @@
 from concurrent.futures import ALL_COMPLETED, ThreadPoolExecutor
 from typing import Generic, TypeVar
 
-from databricks.labs.ucx.generic import StrEnum
-
 ExecutableResult = TypeVar("ExecutableResult")
 ExecutableFunction = Callable[..., ExecutableResult]
 logger = logging.getLogger(__name__)
@@ -66,20 +64,5 @@ def run(self) -> list[ExecutableResult]:
         return collected
 
 
-class Request:
-    def __init__(self, req: dict):
-        self.request = req
-
-    def as_dict(self) -> dict:
-        return self.request
-
-
-class WorkspaceLevelEntitlement(StrEnum):
-    WORKSPACE_ACCESS = "workspace-access"
-    DATABRICKS_SQL_ACCESS = "databricks-sql-access"
-    ALLOW_CLUSTER_CREATE = "allow-cluster-create"
-    ALLOW_INSTANCE_POOL_CREATE = "allow-instance-pool-create"
-
-
 def noop():
     pass

src/databricks/labs/ucx/tasks.py → src/databricks/labs/ucx/framework/tasks.py

@@ -5,7 +5,7 @@
 from pathlib import Path
 
 from databricks.labs.ucx.config import MigrationConfig
-from databricks.labs.ucx.logger import _install
+from databricks.labs.ucx.framework.logger import _install
 
 _TASKS: dict[str, "Task"] = {}
 

src/databricks/labs/ucx/hive_metastore/__init__.py

@@ -0,0 +1,3 @@
+from databricks.labs.ucx.hive_metastore.table_acls import TaclToolkit
+
+__all__ = ["TaclToolkit"]

src/databricks/labs/ucx/tacl/grants.py → src/databricks/labs/ucx/hive_metastore/grants.py

@@ -2,9 +2,9 @@
 from dataclasses import dataclass
 from functools import partial
 
-from databricks.labs.ucx.tacl._internal import CrawlerBase
-from databricks.labs.ucx.tacl.tables import TablesCrawler
-from databricks.labs.ucx.utils import ThreadedExecution
+from databricks.labs.ucx.framework.crawlers import CrawlerBase
+from databricks.labs.ucx.framework.parallel import ThreadedExecution
+from databricks.labs.ucx.hive_metastore.tables import TablesCrawler
 
 
 @dataclass(frozen=True)

src/databricks/labs/ucx/toolkits/table_acls.py → src/databricks/labs/ucx/hive_metastore/table_acls.py

@@ -2,13 +2,13 @@
 
 from databricks.sdk import WorkspaceClient
 
-from databricks.labs.ucx.tacl._internal import (
+from databricks.labs.ucx.framework.crawlers import (
     RuntimeBackend,
     SqlBackend,
     StatementExecutionBackend,
 )
-from databricks.labs.ucx.tacl.grants import GrantsCrawler
-from databricks.labs.ucx.tacl.tables import TablesCrawler
+from databricks.labs.ucx.hive_metastore.grants import GrantsCrawler
+from databricks.labs.ucx.hive_metastore.tables import TablesCrawler
 
 logger = logging.getLogger(__name__)
 

src/databricks/labs/ucx/tacl/tables.py → src/databricks/labs/ucx/hive_metastore/tables.py

@@ -3,9 +3,9 @@
 from dataclasses import dataclass
 from functools import partial
 
-from databricks.labs.ucx.providers.mixins.sql import Row
-from databricks.labs.ucx.tacl._internal import CrawlerBase, SqlBackend
-from databricks.labs.ucx.utils import ThreadedExecution
+from databricks.labs.ucx.framework.crawlers import CrawlerBase, SqlBackend
+from databricks.labs.ucx.framework.parallel import ThreadedExecution
+from databricks.labs.ucx.mixins.sql import Row
 
 logger = logging.getLogger(__name__)
 
@@ -75,8 +75,7 @@ def __init__(self, backend: SqlBackend, catalog, schema):
         Initializes a TablesCrawler instance.
 
         Args:
-            ws (WorkspaceClient): The WorkspaceClient instance.
-            warehouse_id: The warehouse ID.
+            backend (SqlBackend): The SQL Execution Backend abstraction (either REST API or Spark)
             catalog (str): The catalog name for the inventory persistence.
             schema: The schema name for the inventory persistence.
         """

src/databricks/labs/ucx/install.py

@@ -16,8 +16,8 @@
 
 from databricks.labs.ucx.__about__ import __version__
 from databricks.labs.ucx.config import GroupsConfig, MigrationConfig, TaclConfig
+from databricks.labs.ucx.framework.tasks import _TASKS
 from databricks.labs.ucx.runtime import main
-from databricks.labs.ucx.tasks import _TASKS
 
 TAG_STEP = "step"
 TAG_APP = "App"

src/databricks/labs/ucx/runtime.py

@@ -5,9 +5,9 @@
 from databricks.sdk import WorkspaceClient
 
 from databricks.labs.ucx.config import MigrationConfig
-from databricks.labs.ucx.tasks import task, trigger
-from databricks.labs.ucx.toolkits.group_migration import GroupMigrationToolkit
-from databricks.labs.ucx.toolkits.table_acls import TaclToolkit
+from databricks.labs.ucx.framework.tasks import task, trigger
+from databricks.labs.ucx.hive_metastore import TaclToolkit
+from databricks.labs.ucx.workspace_access import GroupMigrationToolkit
 
 logger = logging.getLogger(__name__)