From 0f78570ed8fe94381b297fef6d440ffb4df08d7a Mon Sep 17 00:00:00 2001 From: Serge Smertin Date: Tue, 5 Sep 2023 00:39:34 +0200 Subject: [PATCH 1/4] Run integration test with a single group To remove nightly test flakiness, this PR replaces `_create_groups` utilities with `make_ucx_group`, which relies on the common fixture factories. --- .../labs/ucx/providers/mixins/fixtures.py | 35 ++++++-- tests/integration/conftest.py | 87 +++++++------------ tests/integration/utils.py | 59 +------------ 3 files changed, 62 insertions(+), 119 deletions(-) diff --git a/src/databricks/labs/ucx/providers/mixins/fixtures.py b/src/databricks/labs/ucx/providers/mixins/fixtures.py index b60d145a7f..5ca4916f88 100644 --- a/src/databricks/labs/ucx/providers/mixins/fixtures.py +++ b/src/databricks/labs/ucx/providers/mixins/fixtures.py @@ -8,7 +8,7 @@ from typing import BinaryIO import pytest -from databricks.sdk import WorkspaceClient +from databricks.sdk import AccountClient, WorkspaceClient from databricks.sdk.core import DatabricksError from databricks.sdk.service import compute, iam, jobs, pipelines, workspace @@ -55,6 +55,13 @@ def ws() -> WorkspaceClient: return WorkspaceClient() +@pytest.fixture(scope="session") +def acc() -> AccountClient: + # Use variables from Unified Auth + # See https://databricks-sdk-py.readthedocs.io/en/latest/authentication.html + return AccountClient() + + @pytest.fixture def make_secret_scope(ws, make_random): def create(**kwargs): @@ -125,19 +132,35 @@ def _scim_values(ids: list[str]) -> list[iam.ComplexValue]: return [iam.ComplexValue(value=x) for x in ids] -@pytest.fixture -def make_group(ws, make_random): +def _make_group(name, interface, make_random): def create( - *, members: list[str] | None = None, roles: list[str] | None = None, display_name: str | None = None, **kwargs + *, + members: list[str] | None = None, + roles: list[str] | None = None, + entitlements: list[str] | None = None, + display_name: str | None = None, + **kwargs, ): kwargs["display_name"] = f"sdk-{make_random(4)}" if display_name is None else display_name if members is not None: kwargs["members"] = _scim_values(members) if roles is not None: kwargs["roles"] = _scim_values(roles) - return ws.groups.create(**kwargs) + if entitlements is not None: + kwargs["entitlements"] = _scim_values(entitlements) + return interface.create(**kwargs) + + yield from factory(name, create, lambda item: interface.delete(item.id)) + + +@pytest.fixture +def make_group(ws, make_random): + yield from _make_group("workspace group", ws.groups, make_random) - yield from factory("workspace group", create, lambda item: ws.groups.delete(item.id)) + +@pytest.fixture +def make_acc_group(acc, make_random): + yield from _make_group("account group", acc.groups, make_random) @pytest.fixture diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 3da1451401..2cb1d80b25 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -3,12 +3,10 @@ import logging import os import random -import uuid from functools import partial import databricks.sdk.core import pytest -from _pytest.fixtures import SubRequest from databricks.sdk import AccountClient, WorkspaceClient from databricks.sdk.core import Config, DatabricksError from databricks.sdk.service.compute import ( @@ -46,8 +44,6 @@ EnvironmentInfo, InstanceProfile, WorkspaceObjects, - _cleanup_groups, - _create_groups, _get_basic_job_cluster, _get_basic_task, _set_random_permissions, @@ -220,48 +216,29 @@ def test_table_fixture(make_table): logger.info(f'Created new view in new schema: {make_table(view=True, ctas="SELECT 2+2 AS four")}') -@pytest.fixture(scope="session") -def env(ws: WorkspaceClient, acc: AccountClient, request: SubRequest) -> EnvironmentInfo: - # prepare environment - test_uid = f"{UCX_TESTING_PREFIX}_{str(uuid.uuid4())[:8]}" - logger.debug(f"Creating environment with uid {test_uid}") - groups = _create_groups(ws, acc, test_uid, NUM_TEST_GROUPS, Threader) - - def post_cleanup(): - print("\n") - logger.debug("Cleaning up the environment") - logger.debug("Deleting test groups") - cleanups = [partial(_cleanup_groups, ws, acc, g) for g in groups] - - def error_silencer(func): - def _wrapped(*args, **kwargs): - try: - func(*args, **kwargs) - except Exception as e: - logger.warning(f"Cannot delete temp group, skipping it. Original exception {e}") - - return _wrapped - - silent_delete = error_silencer(ws.groups.delete) - - temp_cleanups = [ - # TODO: this is too heavy for SCIM API, refactor to ID lookup - partial(silent_delete, g.id) - for g in ws.groups.list(filter=f"displayName sw 'db-temp-{test_uid}'") - ] - new_ws_groups_cleanups = [ - partial(silent_delete, g.id) for g in ws.groups.list(filter=f"displayName sw '{test_uid}'") - ] +@pytest.fixture +def user_pool(ws): + return list(ws.users.list(filter="displayName sw 'test-user-'", attributes="id, userName, displayName")) - all_cleanups = cleanups + temp_cleanups + new_ws_groups_cleanups - Threader(all_cleanups).run() - logger.debug(f"Finished cleanup for the environment {test_uid}") - request.addfinalizer(post_cleanup) - yield EnvironmentInfo(test_uid=test_uid, groups=groups) +@pytest.fixture +def make_ucx_group(make_random, make_group, make_acc_group, user_pool): + def inner(): + display_name = f"ucx_{make_random(4)}" + members = [_.id for _ in random.choices(user_pool, k=random.randint(1, 40))] + ws_group = make_group(display_name=display_name, members=members, entitlements=["allow-cluster-create"]) + acc_group = make_acc_group(display_name=display_name, members=members) + return ws_group, acc_group + return inner -@pytest.fixture(scope="session") +@pytest.fixture +def env(make_ucx_group, make_random) -> EnvironmentInfo: + test_uid = f"ucx_{make_random(4)}" + yield EnvironmentInfo(test_uid=test_uid, groups=[make_ucx_group()]) + + +@pytest.fixture def instance_profiles(env: EnvironmentInfo, ws: WorkspaceClient) -> list[InstanceProfile]: logger.debug("Adding test instance profiles") profiles: list[InstanceProfile] = [] @@ -295,7 +272,7 @@ def instance_profiles(env: EnvironmentInfo, ws: WorkspaceClient) -> list[Instanc logger.debug("Test instance profiles deleted") -@pytest.fixture(scope="session") +@pytest.fixture def instance_pools(env: EnvironmentInfo, ws: WorkspaceClient) -> list[CreateInstancePoolResponse]: logger.debug("Creating test instance pools") @@ -320,7 +297,7 @@ def instance_pools(env: EnvironmentInfo, ws: WorkspaceClient) -> list[CreateInst Threader(executables).run() -@pytest.fixture(scope="session") +@pytest.fixture def pipelines(env: EnvironmentInfo, ws: WorkspaceClient) -> list[CreatePipelineResponse]: logger.debug("Creating test DLT pipelines") @@ -350,7 +327,7 @@ def pipelines(env: EnvironmentInfo, ws: WorkspaceClient) -> list[CreatePipelineR Threader(executables).run() -@pytest.fixture(scope="session") +@pytest.fixture def jobs(env: EnvironmentInfo, ws: WorkspaceClient) -> list[CreateResponse]: logger.debug("Creating test jobs") @@ -377,7 +354,7 @@ def jobs(env: EnvironmentInfo, ws: WorkspaceClient) -> list[CreateResponse]: Threader(executables).run() -@pytest.fixture(scope="session") +@pytest.fixture def cluster_policies(env: EnvironmentInfo, ws: WorkspaceClient) -> list[CreatePolicyResponse]: logger.debug("Creating test cluster policies") @@ -412,7 +389,7 @@ def cluster_policies(env: EnvironmentInfo, ws: WorkspaceClient) -> list[CreatePo Threader(executables).run() -@pytest.fixture(scope="session") +@pytest.fixture def clusters(env: EnvironmentInfo, ws: WorkspaceClient) -> list[ClusterDetails]: logger.debug("Creating test clusters") @@ -447,7 +424,7 @@ def clusters(env: EnvironmentInfo, ws: WorkspaceClient) -> list[ClusterDetails]: logger.debug("Test clusters deleted") -@pytest.fixture(scope="session") +@pytest.fixture def experiments(ws: WorkspaceClient, env: EnvironmentInfo) -> list[CreateExperimentResponse]: logger.debug("Creating test experiments") @@ -480,7 +457,7 @@ def experiments(ws: WorkspaceClient, env: EnvironmentInfo) -> list[CreateExperim logger.debug("Test experiments deleted") -@pytest.fixture(scope="session") +@pytest.fixture def models(ws: WorkspaceClient, env: EnvironmentInfo) -> list[ModelDatabricks]: logger.debug("Creating models") @@ -513,7 +490,7 @@ def models(ws: WorkspaceClient, env: EnvironmentInfo) -> list[ModelDatabricks]: logger.debug("Test models deleted") -@pytest.fixture(scope="session") +@pytest.fixture def warehouses(ws: WorkspaceClient, env: EnvironmentInfo) -> list[GetWarehouseResponse]: logger.debug("Creating warehouses") @@ -548,13 +525,13 @@ def warehouses(ws: WorkspaceClient, env: EnvironmentInfo) -> list[GetWarehouseRe logger.debug("Test warehouses deleted") -@pytest.fixture(scope="session") +@pytest.fixture def tokens(ws: WorkspaceClient, env: EnvironmentInfo) -> list[AccessControlRequest]: logger.debug("Adding token-level permissions to groups") token_permissions = [ AccessControlRequest(group_name=ws_group.display_name, permission_level=PermissionLevel.CAN_USE) - for ws_group, _ in random.sample(env.groups, k=NUM_TEST_TOKENS) + for ws_group, _ in random.sample(env.groups, k=min(len(env.groups), NUM_TEST_TOKENS)) ] ws.permissions.update( @@ -566,7 +543,7 @@ def tokens(ws: WorkspaceClient, env: EnvironmentInfo) -> list[AccessControlReque yield token_permissions -@pytest.fixture(scope="session") +@pytest.fixture def secret_scopes(ws: WorkspaceClient, env: EnvironmentInfo) -> list[SecretScope]: logger.debug("Creating test secret scopes") @@ -587,7 +564,7 @@ def secret_scopes(ws: WorkspaceClient, env: EnvironmentInfo) -> list[SecretScope Threader(executables).run() -@pytest.fixture(scope="session") +@pytest.fixture def workspace_objects(ws: WorkspaceClient, env: EnvironmentInfo) -> WorkspaceObjects: logger.info(f"Creating test workspace objects under /{env.test_uid}") ws.workspace.mkdirs(f"/{env.test_uid}") @@ -642,7 +619,7 @@ def workspace_objects(ws: WorkspaceClient, env: EnvironmentInfo) -> WorkspaceObj logger.debug("Test workspace objects deleted") -@pytest.fixture(scope="session") +@pytest.fixture def verifiable_objects( clusters, instance_pools, diff --git a/tests/integration/utils.py b/tests/integration/utils.py index 7d886a6aa1..d153a48ce6 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -1,23 +1,19 @@ import logging import random from dataclasses import dataclass -from functools import partial from typing import Any -from databricks.sdk import AccountClient, WorkspaceClient +from databricks.sdk import WorkspaceClient from databricks.sdk.service.compute import ClusterSpec, DataSecurityMode from databricks.sdk.service.iam import ( AccessControlRequest, - ComplexValue, Group, PermissionLevel, - User, ) from databricks.sdk.service.jobs import JobCluster, PythonWheelTask, Task from databricks.sdk.service.workspace import ObjectInfo from databricks.labs.ucx.inventory.types import RequestObjectType -from databricks.labs.ucx.utils import WorkspaceLevelEntitlement logger = logging.getLogger(__name__) @@ -34,59 +30,6 @@ class EnvironmentInfo: groups: list[tuple[Group, Group]] -def generate_group_by_id( - _ws: WorkspaceClient, _acc: AccountClient, group_name: str, users_sample: list[User] -) -> tuple[Group, Group]: - entities = [ComplexValue(display=user.display_name, value=user.id) for user in users_sample] - logger.debug(f"Creating group with name {group_name}") - - def get_random_entitlements(): - chosen: list[WorkspaceLevelEntitlement] = random.choices( - list(WorkspaceLevelEntitlement), - k=random.randint(1, 3), - ) - entitlements = [ComplexValue(display=None, primary=None, type=None, value=value) for value in chosen] - return entitlements - - ws_group = _ws.groups.create(display_name=group_name, members=entities, entitlements=get_random_entitlements()) - acc_group = _acc.groups.create(display_name=group_name, members=entities) - return ws_group, acc_group - - -def _create_groups( - _ws: WorkspaceClient, _acc: AccountClient, prefix: str, num_test_groups: int, threader: callable -) -> list[tuple[Group, Group]]: - logger.debug("Listing users to create sample groups") - test_users = list(_ws.users.list(filter="displayName sw 'test-user-'", attributes="id, userName, displayName")) - logger.debug(f"Total of test users {len(test_users)}") - user_samples: dict[str, list[User]] = { - f"{prefix}-test-group-{gid}": random.choices(test_users, k=random.randint(1, 40)) - for gid in range(num_test_groups) - } - executables = [ - partial(generate_group_by_id, _ws, _acc, group_name, users_sample) - for group_name, users_sample in user_samples.items() - ] - return threader(executables).run() - - -def _cleanup_groups(_ws: WorkspaceClient, _acc: AccountClient, _groups: tuple[Group, Group]): - ws_g, acc_g = _groups - logger.debug(f"Deleting groups {ws_g.display_name} [ws-level] and {acc_g.display_name} [acc-level]") - - try: - _ws.groups.delete(ws_g.id) - except Exception as e: - logger.warning(f"Cannot delete ws-level group {ws_g.display_name}, skipping it. Original exception {e}") - - try: - g = next(iter(_acc.groups.list(filter=f"displayName eq '{acc_g.display_name}'")), None) - if g: - _acc.groups.delete(g.id) - except Exception as e: - logger.warning(f"Cannot delete acc-level group {acc_g.display_name}, skipping it. Original exception {e}") - - def _set_random_permissions( objects: list[Any], id_attribute: str, From 91279481b068bd207bee4413cd61951cf99c840c Mon Sep 17 00:00:00 2001 From: Serge Smertin Date: Tue, 5 Sep 2023 01:02:05 +0200 Subject: [PATCH 2/4] Run integration test with a single group To remove nightly test flakiness, this PR replaces `_create_groups` utilities with `make_ucx_group`, which relies on the common fixture factories. --- tests/integration/test_e2e.py | 14 ----- tests/integration/test_jobs.py | 103 --------------------------------- 2 files changed, 117 deletions(-) delete mode 100644 tests/integration/test_jobs.py diff --git a/tests/integration/test_e2e.py b/tests/integration/test_e2e.py index 7557a5d57c..459a24a104 100644 --- a/tests/integration/test_e2e.py +++ b/tests/integration/test_e2e.py @@ -181,20 +181,6 @@ def test_e2e( toolkit = GroupMigrationToolkit(config) toolkit.prepare_environment() - logger.debug("Verifying that the groups were created") - - assert len(ws.groups.list(filter=f"displayName sw '{config.groups.backup_group_prefix}{env.test_uid}'")) == len( - toolkit.group_manager.migration_groups_provider.groups - ) - - assert len(ws.groups.list(filter=f"displayName sw '{env.test_uid}'")) == len( - toolkit.group_manager.migration_groups_provider.groups - ) - - assert len(toolkit.group_manager._list_account_level_groups(filter=f"displayName sw '{env.test_uid}'")) == len( - toolkit.group_manager.migration_groups_provider.groups - ) - for _info in toolkit.group_manager.migration_groups_provider.groups: _ws = ws.groups.get(id=_info.workspace.id) _backup = ws.groups.get(id=_info.backup.id) diff --git a/tests/integration/test_jobs.py b/tests/integration/test_jobs.py deleted file mode 100644 index 7f1f071485..0000000000 --- a/tests/integration/test_jobs.py +++ /dev/null @@ -1,103 +0,0 @@ -import logging - -import pytest -from databricks.sdk import WorkspaceClient -from pyspark.errors import AnalysisException - -from databricks.labs.ucx.config import ( - ConnectConfig, - GroupsConfig, - InventoryConfig, - InventoryTable, - MigrationConfig, -) -from databricks.labs.ucx.inventory.types import RequestObjectType -from databricks.labs.ucx.toolkits.group_migration import GroupMigrationToolkit - -from .test_e2e import _verify_group_permissions, _verify_roles_and_entitlements -from .utils import EnvironmentInfo - -logger = logging.getLogger(__name__) - - -def test_jobs( - env: EnvironmentInfo, - inventory_table: InventoryTable, - ws: WorkspaceClient, - jobs, -): - logger.debug(f"Test environment: {env.test_uid}") - - config = MigrationConfig( - connect=ConnectConfig.from_databricks_config(ws.config), - with_table_acls=False, - inventory=InventoryConfig(table=inventory_table), - groups=GroupsConfig(selected=[g[0].display_name for g in env.groups]), - log_level="DEBUG", - ) - toolkit = GroupMigrationToolkit(config) - toolkit.prepare_environment() - - logger.debug("Verifying that the groups were created") - - assert len(ws.groups.list(filter=f"displayName sw '{config.groups.backup_group_prefix}{env.test_uid}'")) == len( - toolkit.group_manager.migration_groups_provider.groups - ) - - assert len(ws.groups.list(filter=f"displayName sw '{env.test_uid}'")) == len( - toolkit.group_manager.migration_groups_provider.groups - ) - - assert len(toolkit.group_manager._list_account_level_groups(filter=f"displayName sw '{env.test_uid}'")) == len( - toolkit.group_manager.migration_groups_provider.groups - ) - - for _info in toolkit.group_manager.migration_groups_provider.groups: - _ws = ws.groups.get(id=_info.workspace.id) - _backup = ws.groups.get(id=_info.backup.id) - _ws_members = sorted([m.value for m in _ws.members]) - _backup_members = sorted([m.value for m in _backup.members]) - assert _ws_members == _backup_members - - logger.debug("Verifying that the groups were created - done") - - toolkit.cleanup_inventory_table() - - with pytest.raises(AnalysisException): - toolkit.table_manager.spark.catalog.getTable(toolkit.table_manager.config.table.to_spark()) - - toolkit.inventorize_permissions() - - toolkit.apply_permissions_to_backup_groups() - - verifiable_objects = [ - (jobs, "job_id", RequestObjectType.JOBS), - ] - for _objects, id_attribute, request_object_type in verifiable_objects: - _verify_group_permissions(_objects, id_attribute, request_object_type, ws, toolkit, "backup") - - _verify_roles_and_entitlements(toolkit.group_manager.migration_groups_provider, ws, "backup") - - toolkit.replace_workspace_groups_with_account_groups() - - new_groups = list(ws.groups.list(filter=f"displayName sw '{env.test_uid}'", attributes="displayName,meta")) - assert len(new_groups) == len(toolkit.group_manager.migration_groups_provider.groups) - assert all(g.meta.resource_type == "Group" for g in new_groups) - - toolkit.apply_permissions_to_account_groups() - - for _objects, id_attribute, request_object_type in verifiable_objects: - _verify_group_permissions(_objects, id_attribute, request_object_type, ws, toolkit, "account") - - _verify_roles_and_entitlements(toolkit.group_manager.migration_groups_provider, ws, "account") - - toolkit.delete_backup_groups() - - backup_groups = list( - ws.groups.list( - filter=f"displayName sw '{config.groups.backup_group_prefix}{env.test_uid}'", attributes="displayName,meta" - ) - ) - assert len(backup_groups) == 0 - - toolkit.cleanup_inventory_table() From 6280d73fb444a1e7e855c4c76e05bc43beecbee0 Mon Sep 17 00:00:00 2001 From: Serge Smertin Date: Tue, 5 Sep 2023 02:11:25 +0200 Subject: [PATCH 3/4] more fixes --- .../labs/ucx/providers/groups_info.py | 6 ++++++ tests/integration/test_e2e.py | 17 +++++++---------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/databricks/labs/ucx/providers/groups_info.py b/src/databricks/labs/ucx/providers/groups_info.py index 9c1e54cc46..5f3dbb048f 100644 --- a/src/databricks/labs/ucx/providers/groups_info.py +++ b/src/databricks/labs/ucx/providers/groups_info.py @@ -19,6 +19,12 @@ def __init__(self): def add(self, group: MigrationGroupInfo): self.groups.append(group) + def is_in_scope(self, attr: str, group: Group) -> bool: + for info in self.groups: + if getattr(info, attr).id == group.id: + return True + return False + def get_by_workspace_group_name(self, workspace_group_name: str) -> MigrationGroupInfo | None: found = [g for g in self.groups if g.workspace.display_name == workspace_group_name] if len(found) == 0: diff --git a/tests/integration/test_e2e.py b/tests/integration/test_e2e.py index 459a24a104..997e83687e 100644 --- a/tests/integration/test_e2e.py +++ b/tests/integration/test_e2e.py @@ -181,7 +181,8 @@ def test_e2e( toolkit = GroupMigrationToolkit(config) toolkit.prepare_environment() - for _info in toolkit.group_manager.migration_groups_provider.groups: + group_migration_state = toolkit.group_manager.migration_groups_provider + for _info in group_migration_state.groups: _ws = ws.groups.get(id=_info.workspace.id) _backup = ws.groups.get(id=_info.backup.id) _ws_members = sorted([m.value for m in _ws.members]) @@ -202,12 +203,12 @@ def test_e2e( for _objects, id_attribute, request_object_type in verifiable_objects: _verify_group_permissions(_objects, id_attribute, request_object_type, ws, toolkit, "backup") - _verify_roles_and_entitlements(toolkit.group_manager.migration_groups_provider, ws, "backup") + _verify_roles_and_entitlements(group_migration_state, ws, "backup") toolkit.replace_workspace_groups_with_account_groups() - new_groups = list(ws.groups.list(filter=f"displayName sw '{env.test_uid}'", attributes="displayName,meta")) - assert len(new_groups) == len(toolkit.group_manager.migration_groups_provider.groups) + new_groups = [_ for _ in ws.groups.list(attributes="displayName,meta") if group_migration_state.is_in_scope('account', _)] + assert len(new_groups) == len(group_migration_state.groups) assert all(g.meta.resource_type == "Group" for g in new_groups) toolkit.apply_permissions_to_account_groups() @@ -215,15 +216,11 @@ def test_e2e( for _objects, id_attribute, request_object_type in verifiable_objects: _verify_group_permissions(_objects, id_attribute, request_object_type, ws, toolkit, "account") - _verify_roles_and_entitlements(toolkit.group_manager.migration_groups_provider, ws, "account") + _verify_roles_and_entitlements(group_migration_state, ws, "account") toolkit.delete_backup_groups() - backup_groups = list( - ws.groups.list( - filter=f"displayName sw '{config.groups.backup_group_prefix}{env.test_uid}'", attributes="displayName,meta" - ) - ) + backup_groups = [_ for _ in ws.groups.list(attributes="displayName,meta") if group_migration_state.is_in_scope('backup', _)] assert len(backup_groups) == 0 toolkit.cleanup_inventory_table() From fde20e267eed5eb61972b91e69649ceee2d9e7e2 Mon Sep 17 00:00:00 2001 From: Serge Smertin Date: Tue, 5 Sep 2023 02:26:21 +0200 Subject: [PATCH 4/4] more fixes --- src/databricks/labs/ucx/providers/mixins/fixtures.py | 6 ++---- tests/integration/conftest.py | 1 + tests/integration/test_e2e.py | 8 ++++++-- tests/integration/utils.py | 6 +----- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/databricks/labs/ucx/providers/mixins/fixtures.py b/src/databricks/labs/ucx/providers/mixins/fixtures.py index 5ca4916f88..693d07f1f5 100644 --- a/src/databricks/labs/ucx/providers/mixins/fixtures.py +++ b/src/databricks/labs/ucx/providers/mixins/fixtures.py @@ -31,10 +31,8 @@ def inner(**kwargs): _LOG.debug(f"removing {name} fixture: {x}") remove(x) except DatabricksError as e: - if e.error_code in ("RESOURCE_DOES_NOT_EXIST",): - _LOG.debug(f"ignoring error while {name} {x} teardown: {e}") - continue - raise e + # TODO: fix on the databricks-labs-pytester level + _LOG.debug(f"ignoring error while {name} {x} teardown: {e}") @pytest.fixture diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 2cb1d80b25..436d1bd2ba 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -229,6 +229,7 @@ def inner(): ws_group = make_group(display_name=display_name, members=members, entitlements=["allow-cluster-create"]) acc_group = make_acc_group(display_name=display_name, members=members) return ws_group, acc_group + return inner diff --git a/tests/integration/test_e2e.py b/tests/integration/test_e2e.py index 997e83687e..f26c81f8f9 100644 --- a/tests/integration/test_e2e.py +++ b/tests/integration/test_e2e.py @@ -207,7 +207,9 @@ def test_e2e( toolkit.replace_workspace_groups_with_account_groups() - new_groups = [_ for _ in ws.groups.list(attributes="displayName,meta") if group_migration_state.is_in_scope('account', _)] + new_groups = [ + _ for _ in ws.groups.list(attributes="displayName,meta") if group_migration_state.is_in_scope("account", _) + ] assert len(new_groups) == len(group_migration_state.groups) assert all(g.meta.resource_type == "Group" for g in new_groups) @@ -220,7 +222,9 @@ def test_e2e( toolkit.delete_backup_groups() - backup_groups = [_ for _ in ws.groups.list(attributes="displayName,meta") if group_migration_state.is_in_scope('backup', _)] + backup_groups = [ + _ for _ in ws.groups.list(attributes="displayName,meta") if group_migration_state.is_in_scope("backup", _) + ] assert len(backup_groups) == 0 toolkit.cleanup_inventory_table() diff --git a/tests/integration/utils.py b/tests/integration/utils.py index d153a48ce6..13872ef1f1 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -5,11 +5,7 @@ from databricks.sdk import WorkspaceClient from databricks.sdk.service.compute import ClusterSpec, DataSecurityMode -from databricks.sdk.service.iam import ( - AccessControlRequest, - Group, - PermissionLevel, -) +from databricks.sdk.service.iam import AccessControlRequest, Group, PermissionLevel from databricks.sdk.service.jobs import JobCluster, PythonWheelTask, Task from databricks.sdk.service.workspace import ObjectInfo