From 78b07f7d490aed3bd373cc4673ed8e8751da5428 Mon Sep 17 00:00:00 2001 From: shabbywu Date: Fri, 20 Aug 2021 17:13:19 +0800 Subject: [PATCH 1/2] =?UTF-8?q?refactor:=20=E9=87=8D=E6=9E=84=20LDAP=20?= =?UTF-8?q?=E5=8D=8F=E8=AE=AE=E7=9A=84=E7=94=A8=E6=88=B7=E5=90=8C=E6=AD=A5?= =?UTF-8?q?=E9=80=BB=E8=BE=91,=20=E4=BD=BF=E5=85=B6=E7=BB=9F=E8=AE=A1?= =?UTF-8?q?=E5=90=8C=E6=AD=A5=E6=88=90=E5=8A=9F=E6=88=96=E5=A4=B1=E8=B4=A5?= =?UTF-8?q?=E7=9A=84=E5=AF=B9=E8=B1=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkuser_core/categories/plugins/base.py | 43 +- .../categories/plugins/custom/client.py | 5 +- .../categories/plugins/custom/helpers.py | 2 + .../categories/plugins/ldap/adaptor.py | 141 +++++++ .../categories/plugins/ldap/client.py | 4 +- .../categories/plugins/ldap/helper.py | 240 +++++++++++ .../categories/plugins/ldap/login.py | 7 +- .../categories/plugins/ldap/metas.py | 22 ++ .../categories/plugins/ldap/models.py | 41 ++ .../categories/plugins/ldap/syncer.py | 372 ++++++------------ .../bkuser_core/categories/plugins/utils.py | 23 ++ src/api/bkuser_core/categories/views.py | 2 +- .../tests/categories/plugins/conftest.py | 57 +++ .../categories/plugins/custom/test_helper.py | 12 +- .../categories/plugins/ldap/test_adaptor.py | 285 ++++++++++++++ .../categories/plugins/ldap/test_client.py | 10 +- .../categories/plugins/ldap/test_helper.py | 208 ++++++++++ .../categories/plugins/ldap/test_syncer.py | 88 +---- src/api/bkuser_core/tests/conftest.py | 3 + 19 files changed, 1197 insertions(+), 368 deletions(-) create mode 100644 src/api/bkuser_core/categories/plugins/ldap/adaptor.py create mode 100644 src/api/bkuser_core/categories/plugins/ldap/helper.py create mode 100644 src/api/bkuser_core/categories/plugins/ldap/metas.py create mode 100644 src/api/bkuser_core/categories/plugins/ldap/models.py create mode 100644 src/api/bkuser_core/tests/categories/plugins/conftest.py create mode 100644 src/api/bkuser_core/tests/categories/plugins/ldap/test_adaptor.py create mode 100644 src/api/bkuser_core/tests/categories/plugins/ldap/test_helper.py diff --git a/src/api/bkuser_core/categories/plugins/base.py b/src/api/bkuser_core/categories/plugins/base.py index ac10a5843..2c01a8f92 100644 --- a/src/api/bkuser_core/categories/plugins/base.py +++ b/src/api/bkuser_core/categories/plugins/base.py @@ -11,11 +11,11 @@ import datetime import logging from abc import abstractmethod -from collections import defaultdict +from collections import UserDict, defaultdict from contextlib import contextmanager from dataclasses import dataclass, field from threading import RLock -from typing import Any, ClassVar, Dict, List, Optional, Type +from typing import Any, ClassVar, Dict, List, MutableMapping, Optional, Type, TypeVar from bkuser_core.categories.models import ProfileCategory from bkuser_core.categories.plugins.constants import SYNC_LOG_TEMPLATE_MAP, SyncStep @@ -25,6 +25,7 @@ from bkuser_core.profiles.models import LeaderThroughModel, Profile from bkuser_core.user_settings.loader import ConfigProvider from django.db.models import Model +from typing_extensions import Protocol logger = logging.getLogger(__name__) @@ -298,3 +299,41 @@ class LoginHandler: @abstractmethod def check(self, *args, **kwargs): raise NotImplementedError + + +class TypeProtocol(Protocol): + @property + def key_field(self) -> str: + """The Key Field to make obj unique.""" + + @property + def display_str(self) -> str: + """The Display str for obj.""" + + +M = TypeVar("M") + + +class TypeList(UserDict, MutableMapping[str, M]): + @classmethod + def from_list(cls, items: List[TypeProtocol]): + items_map = {i.key_field: i for i in items} + return cls(items_map) + + @classmethod + def get_type(cls) -> Type[M]: + # As of Python 3.6. there is a public __args__ and (__parameters__) field for Generic + return cls.__args__[0] # type: ignore + + +class DBSyncHelper(Protocol): + """将 TypeList 塞入到 DBSyncManager 中的协议""" + + category: ProfileCategory + db_sync_manager: DBSyncManager + target_obj_list: TypeList + context: SyncContext + + def load_to_memory(self): + """将数据对象加载到内存""" + raise NotImplementedError diff --git a/src/api/bkuser_core/categories/plugins/custom/client.py b/src/api/bkuser_core/categories/plugins/custom/client.py index 1b34d220a..0bdbbc365 100644 --- a/src/api/bkuser_core/categories/plugins/custom/client.py +++ b/src/api/bkuser_core/categories/plugins/custom/client.py @@ -14,11 +14,10 @@ import curlify import requests +from bkuser_core.categories.plugins.custom.exceptions import CustomAPIRequestFailed +from bkuser_core.categories.plugins.custom.models import CustomDepartment, CustomProfile, CustomTypeList from bkuser_core.user_settings.loader import ConfigProvider -from .exceptions import CustomAPIRequestFailed -from .models import CustomDepartment, CustomProfile, CustomTypeList - logger = logging.getLogger(__name__) diff --git a/src/api/bkuser_core/categories/plugins/custom/helpers.py b/src/api/bkuser_core/categories/plugins/custom/helpers.py index 0d95c0683..cb736d114 100644 --- a/src/api/bkuser_core/categories/plugins/custom/helpers.py +++ b/src/api/bkuser_core/categories/plugins/custom/helpers.py @@ -21,6 +21,7 @@ from bkuser_core.categories.plugins.custom.utils import handle_with_progress_info from bkuser_core.common.db_sync import SyncOperation from bkuser_core.departments.models import Department, DepartmentThroughModel +from bkuser_core.profiles.constants import ProfileStatus from bkuser_core.profiles.models import LeaderThroughModel, Profile from bkuser_core.profiles.validators import validate_username from django.db.models import Model @@ -172,6 +173,7 @@ def _load_base_info(self): "telephone": info.telephone, "position": info.position, "extras": info.extras, + "status": ProfileStatus.NORMAL.value, } # 2. 更新或创建 Profile 对象 diff --git a/src/api/bkuser_core/categories/plugins/ldap/adaptor.py b/src/api/bkuser_core/categories/plugins/ldap/adaptor.py new file mode 100644 index 000000000..b5493ef93 --- /dev/null +++ b/src/api/bkuser_core/categories/plugins/ldap/adaptor.py @@ -0,0 +1,141 @@ +# -*- coding: utf-8 -*- +from dataclasses import dataclass +from typing import Any, Dict, List, NamedTuple, Optional + +from bkuser_core.categories.plugins.ldap.models import DepartmentProfile, UserProfile +from bkuser_core.user_settings.loader import ConfigProvider +from django.utils.encoding import force_str +from ldap3.utils import dn as dn_utils + + +@dataclass +class ProfileFieldMapper: + """从 ldap 对象属性中获取用户字段""" + + config_loader: ConfigProvider + setting_field_map: dict + + def get_field(self, user_meta: Dict[str, List[bytes]], field_name: str, raise_exception: bool = False) -> str: + """根据字段映射关系, 从 ldap 中获取 `field_name` 的值""" + try: + setting_name = self.setting_field_map[field_name] + except KeyError: + if raise_exception: + raise ValueError("该用户字段没有在配置中有对应项，无法同步") + return "" + + try: + ldap_field_name = self.config_loader[setting_name] + except KeyError: + if raise_exception: + raise ValueError(f"用户目录配置中缺失字段 {setting_name}") + return "" + + try: + if user_meta[ldap_field_name]: + return force_str(user_meta[ldap_field_name][0]) + + return "" + except KeyError: + if raise_exception: + raise ValueError(f"搜索数据中没有对应的字段 {ldap_field_name}") + return "" + + def get_user_attributes(self) -> list: + """获取远端属性名列表""" + return [self.config_loader[x] for x in self.setting_field_map.values() if self.config_loader[x]] + + +def user_adaptor( + code: str, user_meta: Dict[str, Any], field_mapper: ProfileFieldMapper, restrict_types: List[str] +) -> UserProfile: + groups = user_meta["attributes"][field_mapper.config_loader["user_member_of"]] + + return UserProfile( + username=field_mapper.get_field(user_meta=user_meta["raw_attributes"], field_name="username"), + email=field_mapper.get_field(user_meta=user_meta["raw_attributes"], field_name="email"), + telephone=field_mapper.get_field(user_meta=user_meta["raw_attributes"], field_name="telephone"), + display_name=field_mapper.get_field(user_meta=user_meta["raw_attributes"], field_name="display_name"), + code=code, + # TODO: 完成转换 departments 的逻辑 + departments=[ + # 根据约定, dn 中除去第一个成分以外的部分即为用户所在的部门, 因此需要取 [1:] + list(reversed(parse_dn_value_list(user_meta["dn"], restrict_types)[1:])), + # 用户与用户组之间的关系 + *[list(reversed(parse_dn_value_list(group, restrict_types))) for group in groups], + ], + ) + + +def department_adaptor(code: str, dept_meta: Dict, is_group: bool, restrict_types: List[str]) -> DepartmentProfile: + dn = dept_meta["dn"] + dn_values = parse_dn_value_list(dn, restrict_types=restrict_types) + + parent_dept: Optional[DepartmentProfile] = None + for dept_name in reversed(dn_values): + parent_dept = DepartmentProfile( + name=dept_name, + parent=parent_dept, + is_group=is_group, + ) + + assert parent_dept is not None + parent_dept.code = code + return parent_dept + + +class RDN(NamedTuple): + """RelativeDistinguishedName""" + + type: str + value: str + separator: str + + +def parse_dn_tree(dn, restrict_types: List[str] = None) -> List[RDN]: + """A DN is a sequence of relative distinguished names (RDN) connected by commas, For examples: + + we have a dn = "CN=Jeff Smith,OU=Sales,DC=Fabrikam,DC=COM", this method will parse the dn to: + >>> parse_dn_tree("CN=Jeff Smith,OU=Sales,DC=Fabrikam,DC=COM") + [RDN(type='CN', value='Jeff Smith', separator=','), + RDN(type='OU', value='Sales', separator=','), + RDN(type='DC', value='Fabrikam', separator=','), + RDN(type='DC', value='COM', separator='')] + + if provide restrict_types, this method will ignore the attribute not in restrict_types, For examples: + >>> parse_dn_tree("CN=Jeff Smith,OU=Sales,DC=Fabrikam,DC=COM", restrict_types=["DC"]) + [RDN(type='DC', value='Fabrikam', separator=','), RDN(type='DC', value='COM', separator='')] + + Furthermore, restrict_types is Case-insensitive, the ["DC"], ["dc"], ["Dc"] are Exactly equal. + >>> parse_dn_tree("CN=Jeff Smith,OU=Sales,DC=Fabrikam,DC=COM", restrict_types=["dc"]) + [RDN(type='DC', value='Fabrikam', separator=','), RDN(type='DC', value='COM', separator='')] + + See Also: https://docs.microsoft.com/en-us/previous-versions/windows/desktop/ldap/distinguished-names + """ + restrict_types = [_type.upper() for _type in (restrict_types or [])] + items = dn_utils.parse_dn(dn, escape=True) + + if restrict_types: + parts = [RDN(*i) for i in items if i[0].upper() in restrict_types] + else: + parts = [RDN(*i) for i in items] + + return parts + + +def parse_dn_value_list(dn, restrict_types: List[str] = None) -> List[str]: + """this method work like parse_dn_tree, be only return values of those attributes, For examples: + + >>> parse_dn_value_list("CN=Jeff Smith,OU=Sales,DC=Fabrikam,DC=COM") + ['Jeff Smith', 'Sales', 'Fabrikam', 'COM'] + + if provide restrict_types, this method will ignore the attribute not in restrict_types, For examples: + >>> parse_dn_value_list("CN=Jeff Smith,OU=Sales,DC=Fabrikam,DC=COM", restrict_types=["DC"]) + ['Fabrikam', 'COM'] + + """ + tree = parse_dn_tree(dn, restrict_types) + parts = [] + for part in tree: + parts.append(part.value) + return parts diff --git a/src/api/bkuser_core/categories/plugins/ldap/client.py b/src/api/bkuser_core/categories/plugins/ldap/client.py index 6e7ded464..fb95bbf3f 100644 --- a/src/api/bkuser_core/categories/plugins/ldap/client.py +++ b/src/api/bkuser_core/categories/plugins/ldap/client.py @@ -10,7 +10,7 @@ """ import logging from dataclasses import dataclass -from typing import TYPE_CHECKING, Dict +from typing import TYPE_CHECKING, Dict, List import ldap3 from bkuser_core.categories.loader import get_plugin_by_name @@ -81,7 +81,7 @@ def search( force_filter_str: str = "", start_root: str = None, attributes: list = None, - ) -> Dict: + ) -> List[Dict]: """搜索""" if not start_root: start_root = self.start_root diff --git a/src/api/bkuser_core/categories/plugins/ldap/helper.py b/src/api/bkuser_core/categories/plugins/ldap/helper.py new file mode 100644 index 000000000..2d70108ce --- /dev/null +++ b/src/api/bkuser_core/categories/plugins/ldap/helper.py @@ -0,0 +1,240 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available. +Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import logging +from dataclasses import dataclass +from typing import Dict, List, Optional, Type + +from bkuser_core.categories.models import ProfileCategory +from bkuser_core.categories.plugins.base import DBSyncManager, SyncContext, SyncStep, TypeList +from bkuser_core.categories.plugins.ldap.metas import LdapDepartmentMeta, LdapProfileMeta +from bkuser_core.categories.plugins.ldap.models import DepartmentProfile, UserProfile +from bkuser_core.categories.plugins.utils import handle_with_progress_info +from bkuser_core.common.db_sync import SyncOperation +from bkuser_core.departments.models import Department, DepartmentThroughModel +from bkuser_core.profiles.constants import ProfileStatus +from bkuser_core.profiles.models import Profile +from bkuser_core.profiles.validators import validate_username +from bkuser_core.user_settings.loader import ConfigProvider +from django.db.models import Model +from django.utils.functional import cached_property +from django.utils.translation import gettext_lazy as _ +from rest_framework.exceptions import ValidationError + +logger = logging.getLogger(__name__) + + +@dataclass +class DepartmentSyncHelper: + category: ProfileCategory + db_sync_manager: DBSyncManager + target_obj_list: TypeList[DepartmentProfile] + context: SyncContext + config_loader: ConfigProvider + + _MPTT_INIT_PARAMS = { + "tree_id": 0, + "lft": 0, + "rght": 0, + "level": 0, + } + + @cached_property + def db_departments(self) -> Dict[str, Department]: + # 由于 bulk_update 需要从数据库查询完整的 Department 信息, 为提高查询效率, 统一执行查询操作, 减轻数据库负担 + all_departments: List[Department] = list( + Department.objects.filter(category_id=self.category.pk, enabled=True).all() + ) + + def make_key(dept: Department): + names = [] + while dept: + names.append(dept.name) + dept = dept.parent + return "/".join(reversed(names)) + + return {make_key(dept): dept for dept in all_departments} + + def load_to_memory(self): + for dept in handle_with_progress_info( + self.target_obj_list, progress_title="handle department" + ): # type: DepartmentProfile + self._handle_department(dept) + + def _handle_department(self, dept_info: DepartmentProfile) -> Optional[Department]: + """将 DepartmentProfile 转换成 Department, 并递归处理其父节点 + + 如果父节点存在, 则递归处理父节点, 并绑定部门上下级关系, 再将部门对象(Department)插入缓存层 + 如果父节点不存在, 则直接将部门对象(Department)插入缓存层 + """ + if dept_info.parent: + parent_dept = self._handle_department(dept_info.parent) + else: + parent_dept = None + + defaults = { + "code": dept_info.key_field, + "category_id": self.category.pk, + "name": dept_info.name, + "enabled": True, + "parent_id": getattr(parent_dept, "pk", None), + "extras": { + "type": self.config_loader["user_group_class"] + if dept_info.is_group + else self.config_loader["organization_class"] + }, + **self._MPTT_INIT_PARAMS, + } + if dept_info.code: + defaults["code"] = dept_info.code + + dept = self._insert_dept(dept_info=dept_info, defaults=defaults) + return dept + + def _insert_dept(self, dept_info: DepartmentProfile, defaults: Dict) -> Department: + dept: Department = self.db_sync_manager.magic_get( + unique_key=dept_info.key_field, target_meta=LdapDepartmentMeta + ) + if dept and dept_info.code: + dept.code = dept_info.code + return dept + + if dept_info.key_field in self.db_departments: + dept = self.db_departments[dept_info.key_field] + for key, value in defaults.items(): + setattr(dept, key, value) + self.db_sync_manager.magic_add(dept, SyncOperation.UPDATE.value) + else: + defaults["pk"] = self.db_sync_manager.register_id(LdapDepartmentMeta) + dept = Department(**defaults) + self.db_sync_manager.magic_add(dept, SyncOperation.ADD.value) + + self.context.add_record(step=SyncStep.DEPARTMENTS, success=True, department=dept_info.key_field) + return dept + + +@dataclass +class ProfileSyncHelper: + category: ProfileCategory + db_sync_manager: DBSyncManager + target_obj_list: TypeList[UserProfile] + context: SyncContext + + @cached_property + def db_profiles(self) -> Dict[str, Profile]: + # 由于 bulk_update 需要从数据库查询完整的 Profile 信息, 为提高查询效率, 统一执行查询操作, 减轻数据库负担 + return {profile.username: profile for profile in Profile.objects.filter(category_id=self.category.pk).all()} + + @cached_property + def db_departments(self) -> Dict[str, Department]: + # 由于 bulk_update 需要从数据库查询完整的 Department 信息, 为提高查询效率, 统一执行查询操作, 减轻数据库负担 + all_departments: List[Department] = list( + Department.objects.filter(category_id=self.category.pk, enabled=True).all() + ) + + def make_key(dept: Department): + names = [] + while dept: + names.append(dept.name) + dept = dept.parent + return "/".join(reversed(names)) + + return {make_key(dept): dept for dept in all_departments} + + def _load_base_info(self): + for info in handle_with_progress_info(self.target_obj_list, progress_title="handle profile"): + try: + validate_username(value=info.username) + except ValidationError as e: + self.context.add_record( + step=SyncStep.USERS, + success=False, + username=info.username, + error=str(e), + ) + logger.warning("username<%s> does not meet format", info.username) + continue + + # 1. 先更新 profile 本身 + profile_params = { + "category_id": self.category.pk, + "domain": self.category.domain, + "enabled": True, + "username": info.username, + "display_name": info.display_name, + "email": info.email, + "code": info.code, + "telephone": info.telephone, + "status": ProfileStatus.NORMAL.value, + } + + # 2. 更新或创建 Profile 对象 + if info.username in self.db_profiles: + profile = self.db_profiles[info.username] + for key, value in profile_params.items(): + setattr(profile, key, value) + self.db_sync_manager.magic_add(profile, SyncOperation.UPDATE.value) + else: + profile = Profile(**profile_params) + if self.db_sync_manager.magic_exists(profile): + # 如果增加用户的行为已经添加过了, 则使用内存中的 Profile + logger.debug( + "profile<%s> already add into db sync manager, only get", + profile, + ) + profile = self.db_sync_manager.magic_get(info.code, LdapProfileMeta) + else: + profile.id = self.db_sync_manager.register_id(LdapProfileMeta) + self.db_sync_manager.magic_add(profile, SyncOperation.ADD.value) + + # 3. 维护关联关系 + for full_department_name_list in info.departments: + department_key = "/".join(full_department_name_list) + department = self.db_departments.get(department_key, None) + if not department: + self.context.add_record( + step=SyncStep.DEPT_USER_RELATIONSHIP, + success=False, + username=info.username, + department=department_key, + error=_("部门不存在"), + ) + logger.warning( + "the department<%s> of profile<%s> is missing", + department_key, + info.username, + ) + continue + + self.try_add_relation( + params={"profile_id": profile.pk, "department_id": department.pk}, + target_model=DepartmentThroughModel, + ) + self.context.add_record( + step=SyncStep.DEPT_USER_RELATIONSHIP, + success=True, + username=info.username, + department=department.name, + ) + self.context.add_record(step=SyncStep.USERS, success=True, username=info.username) + + def _load_leader_info(self): + raise NotImplementedError + + def load_to_memory(self): + self._load_base_info() + # TODO: 支持处理上下级关系 + # self._load_leader_info() + + def try_add_relation(self, params: dict, target_model: Type[Model]): + """增加关联关系""" + logger.debug("trying to add relation: %s", params) + relation = target_model(**params) + self.db_sync_manager.magic_add(relation) diff --git a/src/api/bkuser_core/categories/plugins/ldap/login.py b/src/api/bkuser_core/categories/plugins/ldap/login.py index 4b30acd59..1e48384de 100644 --- a/src/api/bkuser_core/categories/plugins/ldap/login.py +++ b/src/api/bkuser_core/categories/plugins/ldap/login.py @@ -8,14 +8,13 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ +from bkuser_core.categories.plugins.ldap.adaptor import ProfileFieldMapper from bkuser_core.categories.plugins.ldap.client import LDAPClient -from bkuser_core.categories.plugins.ldap.syncer import ProfileFieldMapper +from bkuser_core.categories.plugins.ldap.exceptions import FetchUserMetaInfoFailed +from bkuser_core.categories.plugins.ldap.syncer import SETTING_FIELD_MAP from bkuser_core.user_settings.loader import ConfigProvider from django.utils.encoding import force_str -from .exceptions import FetchUserMetaInfoFailed -from .syncer import SETTING_FIELD_MAP - class LoginHandler: @staticmethod diff --git a/src/api/bkuser_core/categories/plugins/ldap/metas.py b/src/api/bkuser_core/categories/plugins/ldap/metas.py new file mode 100644 index 000000000..553d013d0 --- /dev/null +++ b/src/api/bkuser_core/categories/plugins/ldap/metas.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available. +Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from bkuser_core.categories.plugins.base import DepartmentMeta, ProfileMeta + + +class LdapDepartmentMeta(DepartmentMeta): + update_exclude_fields = ["category_id", "code"] + unique_key_field = "code" + use_bulk = True + + +class LdapProfileMeta(ProfileMeta): + unique_key_field = "username" + update_exclude_fields = ["code", "username"] diff --git a/src/api/bkuser_core/categories/plugins/ldap/models.py b/src/api/bkuser_core/categories/plugins/ldap/models.py new file mode 100644 index 000000000..f1fb24da1 --- /dev/null +++ b/src/api/bkuser_core/categories/plugins/ldap/models.py @@ -0,0 +1,41 @@ +from dataclasses import dataclass +from typing import List, Optional + +from django.utils.functional import cached_property + + +@dataclass +class UserProfile: + username: str + display_name: str + email: str + telephone: str + code: str + + departments: List[List[str]] + + @property + def key_field(self): + return self.username + + @property + def display_str(self): + return self.display_name + + +@dataclass +class DepartmentProfile: + name: str + parent: Optional['DepartmentProfile'] = None + code: Optional[str] = None + is_group: bool = False + + @property + def key_field(self): + return self.display_str + + @cached_property + def display_str(self): + if self.parent: + return self.parent.display_str + "/" + self.name + return self.name diff --git a/src/api/bkuser_core/categories/plugins/ldap/syncer.py b/src/api/bkuser_core/categories/plugins/ldap/syncer.py index 36c523712..986edce17 100644 --- a/src/api/bkuser_core/categories/plugins/ldap/syncer.py +++ b/src/api/bkuser_core/categories/plugins/ldap/syncer.py @@ -12,22 +12,20 @@ import logging import re from dataclasses import dataclass -from typing import Callable, List, Optional +from itertools import chain, product +from typing import List, Optional, Tuple from bkuser_core.categories.exceptions import FetchDataFromRemoteFailed -from bkuser_core.categories.plugins.base import Fetcher, ProfileMeta, Syncer +from bkuser_core.categories.plugins.base import DBSyncManager, Fetcher, SyncContext, Syncer, SyncStep, TypeList +from bkuser_core.categories.plugins.ldap.adaptor import ProfileFieldMapper, department_adaptor, user_adaptor from bkuser_core.categories.plugins.ldap.client import LDAPClient -from bkuser_core.common.db_sync import SyncOperation -from bkuser_core.common.progress import progress -from bkuser_core.departments.models import Department, Profile -from bkuser_core.profiles.constants import ProfileStatus -from bkuser_core.profiles.validators import validate_username -from bkuser_core.user_settings.loader import ConfigProvider -from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist +from bkuser_core.categories.plugins.ldap.helper import DepartmentSyncHelper, ProfileSyncHelper +from bkuser_core.categories.plugins.ldap.metas import LdapDepartmentMeta, LdapProfileMeta +from bkuser_core.categories.plugins.ldap.models import DepartmentProfile, UserProfile +from bkuser_core.departments.models import Department, DepartmentThroughModel +from bkuser_core.profiles.models import LeaderThroughModel, Profile from django.db import transaction -from django.utils.encoding import force_bytes, force_str -from ldap3.utils import dn as dn_utils -from rest_framework.exceptions import ValidationError +from django.utils.encoding import force_bytes logger = logging.getLogger(__name__) @@ -49,6 +47,7 @@ class LDAPFetcher(Fetcher): def __post_init__(self): self.client = LDAPClient(self.config_loader) self.field_mapper = ProfileFieldMapper(config_loader=self.config_loader, setting_field_map=SETTING_FIELD_MAP) + self._data: Tuple[List, List, List] = None def fetch(self): """fetch data from remote ldap server""" @@ -62,13 +61,12 @@ def fetch(self): def test_fetch_data(self, configs: dict): """测试获取数据""" - self._fetch_data( + return self._fetch_data( basic_pull_node=configs["basic_pull_node"], user_filter=configs["user_filter"], organization_class=configs["organization_class"], user_group_filter=configs["user_group_filter"], ) - return def _fetch_data( self, @@ -102,6 +100,64 @@ def _fetch_data( return groups, departments, users + def _get_code(self, raw_obj: dict) -> str: + """如果不存在 uuid 则用 dn(sha) 作为唯一标示""" + entry_uuid = raw_obj.get("raw_attributes", {}).get("entryUUID", []) + if isinstance(entry_uuid, list) and entry_uuid: + logger.debug("uuid in raw_attributes: return %s", entry_uuid[0]) + return entry_uuid[0] + else: + # 由于其他目录也可能会出现这样的 code，所以添加 category_id 进行转换 + dn = f"{self.category_id}-{raw_obj.get('dn')}" + + sha = hashlib.sha256(force_bytes(dn)).hexdigest() + logger.debug("no uuid in raw_attributes, use dn instead: %s -> %s", dn, sha) + return sha + + def _load(self): + # TODO: 将 Fetcher 拆成两个对象, 或者不再遵循原来的 Fetcher 协议 + if self._data is None: + self._data = self.fetch() + return self._data + + def fetch_profiles(self, restrict_types: List[str]): + """获取 profile 对象列表""" + _, _, users = self._load() + profiles = [] + for user in users: + if not user.get("dn"): + logger.info("no dn field, skipping for %s", user) + continue + + profiles.append( + user_adaptor( + code=self._get_code(user), + user_meta=user, + field_mapper=self.field_mapper, + restrict_types=restrict_types, + ) + ) + return profiles + + def fetch_departments(self, restrict_types: List[str]): + """获取 department 对象列表""" + groups, departments, _ = self._load() + results = [] + for is_group, dept_meta in chain.from_iterable(iter([product([False], departments), product([True], groups)])): + if not dept_meta.get("dn"): + logger.info("no dn field, skipping for %s", dept_meta) + continue + + results.append( + department_adaptor( + code=self._get_code(dept_meta), + dept_meta=dept_meta, + is_group=is_group, + restrict_types=restrict_types, + ) + ) + return results + @dataclass class LDAPSyncer(Syncer): @@ -115,259 +171,51 @@ def __post_init__(self): self.fetcher: LDAPFetcher = self.get_fetcher() self._field_mapper = self.fetcher.field_mapper + self.db_sync_manager = DBSyncManager({"department": LdapDepartmentMeta, "profile": LdapProfileMeta}) + self.context = SyncContext() def sync(self): - groups, departments, users = self.fetcher.fetch() with transaction.atomic(): - self.disable_departments_before_sync() - self._sync_departments(departments) - self._sync_departments(groups, True) - logger.info("all departments synced.") + self._sync_department() with transaction.atomic(): + self._sync_profile() + + def _sync_department(self): + DepartmentSyncHelper( + category=self.category, + db_sync_manager=self.db_sync_manager, + target_obj_list=(TypeList[DepartmentProfile]).from_list( + self.fetcher.fetch_departments([self.OU_KEY, self.CN_KEY]) + ), + context=self.context, + config_loader=self.config_loader, + ).load_to_memory() + + with Department.tree_objects.disable_mptt_updates(), self.context([SyncStep.DEPARTMENTS]): + # 禁用所有 Department, 在同步时会重新激活仍然有效的 Department + self.disable_departments_before_sync() + self.db_sync_manager.sync_type(target_type=Department) + + # 由于使用 bulk_update 无法第一时间更新树信息，所以在保存完之后强制确保树信息全部正确 + logger.info("make sure tree sane...") + # 由于插入时并没有更新 tree_id，所以这里需要全量更新 + Department.tree_objects.rebuild() + + def _sync_profile(self): + ProfileSyncHelper( + category=self.category, + db_sync_manager=self.db_sync_manager, + target_obj_list=(TypeList[UserProfile]).from_list(self.fetcher.fetch_profiles([self.OU_KEY, self.CN_KEY])), + context=self.context, + ).load_to_memory() + + with self.context([SyncStep.USERS, SyncStep.DEPT_USER_RELATIONSHIP, SyncStep.USERS_RELATIONSHIP]): + # 禁用所有 Profiles, 在同步时会重新激活仍然有效的 Profiles self.disable_profiles_before_sync() - self._sync_users(users) - self.db_sync_manager.sync_all() - logger.info("all profiles & relations synced.") - def _sync_departments(self, raw_departments: list, is_user_group=False): - """序列化部门""" - logger.debug("going to sync raw departments: %s", raw_departments) + self.db_sync_manager.sync_type(target_type=Profile) + self.db_sync_manager.sync_type(target_type=DepartmentThroughModel) + self.db_sync_manager.sync_type(target_type=LeaderThroughModel) - _total = len(raw_departments) - for index, raw_department in enumerate(raw_departments): - if not raw_department.get("dn"): - # 没有 dn 字段忽略 - logger.info("no dn field, skipping for %s", raw_department) - continue - - dn = raw_department["dn"] - tree = self._parse_tree(dn, [self.OU_KEY, self.CN_KEY]) - # 通过 dn 拿到目标组织和组织整条链路 - leaf, route = tree[0], tree[1:] - - leaf_name = list(leaf.values())[0] - parent_department = None - if route: - logger.debug("%s has parents %s", leaf, route) - # 从根开始逐级增加组织 - route.reverse() - for dep in route: - # 暂时不需要区分 ou 或者 cn, parse tree 时已经限定了 - dep_name = list(dep.values())[0] - - # TODO: 使用 sync manager 批量同步? - try: - parent_department, _ = Department.objects.update_or_create( - name=dep_name, - parent=parent_department, - category_id=self.category_id, - defaults={"extras": self._make_department_extras(is_user_group), "enabled": True}, - ) - except MultipleObjectsReturned: - # 删除后创建的同名组织 - departments = Department.objects.filter(name=dep_name, parent=parent_department).order_by( - "-create_time" - ) - for department in departments[1:]: - department.hard_delete() - - parent_department = departments[0] - - # 上级建立完毕之后，建立自己 - _, _ = Department.objects.update_or_create( - name=leaf_name, - code=self._get_code(raw_department), - category_id=self.category_id, - defaults={ - "parent": parent_department, - "extras": self._make_department_extras(is_user_group), - "enabled": True, - }, - ) - progress( - index, - _total, - f"adding {'department' if not is_user_group else 'group'}" - f"<{leaf_name}>, dn<{dn}> ({index}/{_total})", - ) - - def _sync_users(self, raw_users): - _total = len(raw_users) - for index, user in enumerate(raw_users): - if not user.get("dn"): - logger.info("no dn field, skipping for %s", user) - continue - - # TODO: 使用 dataclass 优化 user 数据结构 - username = self._field_mapper.get_field(user_meta=user["raw_attributes"], field_name="username") - try: - validate_username(value=username) - except ValidationError: - logger.warning("username<%s> does not meet format", username) - continue - - progress( - index, - _total, - f"adding profile<{username}> ({index}/{_total})", - ) - - # 1. 先更新 profile 本身 - profile_params = { - "username": username, - "code": self._get_code(user), - "display_name": self._field_mapper.get_field( - user_meta=user["raw_attributes"], field_name="display_name" - ), - "email": self._field_mapper.get_field(user_meta=user["raw_attributes"], field_name="email"), - "telephone": self._field_mapper.get_field(user_meta=user["raw_attributes"], field_name="telephone"), - "enabled": True, - "category_id": self.category_id, - "domain": self.category.domain, - "status": ProfileStatus.NORMAL.value, - } - - try: - profile = Profile.objects.get(category_id=self.category_id, username=username) - for key, value in profile_params.items(): - setattr(profile, key, value) - - self.db_sync_manager.magic_add(profile, SyncOperation.UPDATE.value) - except ObjectDoesNotExist: - profile = Profile(**profile_params) - profile.id = self.db_sync_manager.register_id(ProfileMeta) - self.db_sync_manager.magic_add(profile) - - # 2. 更新 department 关系 - # 这里我们默认用户只能挂载在 用户组(cn) 和 组织(ou) 下 - def get_full_route(parse_method: Callable, parse_params: dict, raw_route: str) -> list: - return parse_method(raw_route, **parse_params) - - def get_target_department(category_id: int, dep_route: list) -> Department: - departments = [x for x in dep_route if list(x.keys())[0] in [self.OU_KEY, self.CN_KEY]] - # 由于所有路径都是到根的，所以从根开始找寻 - departments.reverse() - target_department = None - for dep in departments: - dep_name = list(dep.values())[0] - try: - target_department = Department.objects.filter(category_id=category_id).get( - name=dep_name, parent_id=target_department - ) - except ObjectDoesNotExist: - logger.warning( - "cannot find target department<%s>, parent dep<%s>", - dep_name, - target_department, - ) - except Exception: # pylint: disable=broad-except - logger.warning( - "cannot find target department<%s>, parent dep<%s>, break, please check", - dep_name, - target_department, - ) - - return target_department - - # 同一个人员只能属于一个单位组织，但是可以属于多个用户组 - # 通常我们从 dn 里解析的，有两种可能： - # 对于用户a: cn=a,ou=b,ou=c 或 cn=a,cn=b,cn=c, 前者表明了组织单位链路，后者说明用户只存在于某个用户组 - # 所以第一 cn=a 需要从整个链路中剔除 - full_ou = get_full_route( - self._parse_tree, - {"restrict_types": [self.OU_KEY, self.CN_KEY]}, - user["dn"], - )[1:] - full_groups = [ - get_full_route(self._parse_tree, {"restrict_types": [self.OU_KEY, self.CN_KEY]}, x) - for x in user["attributes"][self.config_loader["user_member_of"]] - ] - - binding_departments = set() - target_ou = get_target_department(self.category_id, full_ou) - if target_ou is not None: - binding_departments.add(target_ou) - - # 原数据可能有多个用户组绑定关系 - for full_group in full_groups: - d = get_target_department(self.category_id, full_group) - if d is None: - logger.warning("can not find %s(group) from saved departments", full_group) - continue - - binding_departments.add(d) - - for d in binding_departments: - self.try_to_add_profile_department_relation(profile=profile, department=d) - - @staticmethod - def _parse_tree(dn, restrict_types: List[str] = None) -> List: - """解析树路径""" - restrict_types = restrict_types or [] - items = dn_utils.parse_dn(dn, escape=True) - - if restrict_types: - parts = [{i[0]: i[1]} for i in items if i[0] in restrict_types] - else: - parts = [{i[0]: i[1]} for i in items] - - return parts - - def _make_department_extras(self, is_user_group): - if is_user_group: - return {"type": self.config_loader["user_group_class"]} - else: - return {"type": self.config_loader["organization_class"]} - - def _get_code(self, raw_obj: dict) -> str: - """如果不存在 uuid 则用 dn(sha) 作为唯一标示""" - entry_uuid = raw_obj.get("raw_attributes", {}).get("entryUUID", []) - if isinstance(entry_uuid, list) and entry_uuid: - logger.debug("uuid in raw_attributes: return %s", entry_uuid[0]) - return entry_uuid[0] - else: - # 由于其他目录也可能会出现这样的 code，所以添加 category_id 进行转换 - dn = f"{self.category_id}-{raw_obj.get('dn')}" - - sha = hashlib.sha256(force_bytes(dn)).hexdigest() - logger.debug("no uuid in raw_attributes, use dn instead: %s -> %s", dn, sha) - return sha - - -@dataclass -class ProfileFieldMapper: - """从 ldap 对象属性中获取用户字段""" - - config_loader: ConfigProvider - setting_field_map: dict - - def get_field(self, user_meta, field_name, raise_exception=False) -> str: - """通过字段名从 ldap 配置中获取内容""" - try: - setting_name = self.setting_field_map[field_name] - except KeyError: - if raise_exception: - raise ValueError("该用户字段没有在配置中有对应项，无法同步") - return "" - - try: - ldap_field_name = self.config_loader[setting_name] - except KeyError: - if raise_exception: - raise ValueError(f"用户目录配置中缺失字段 {setting_name}") - return "" - - try: - if user_meta[ldap_field_name]: - return force_str(user_meta[ldap_field_name][0]) - - return "" - except KeyError: - if raise_exception: - raise ValueError(f"搜索数据中没有对应的字段 {ldap_field_name}") - return "" - - def get_user_attributes(self) -> list: - """获取远端属性名列表""" - return [self.config_loader[x] for x in self.setting_field_map.values() if self.config_loader[x]] + logger.info("all profiles & relations synced.") diff --git a/src/api/bkuser_core/categories/plugins/utils.py b/src/api/bkuser_core/categories/plugins/utils.py index 86e99c313..a0728fcfd 100644 --- a/src/api/bkuser_core/categories/plugins/utils.py +++ b/src/api/bkuser_core/categories/plugins/utils.py @@ -11,6 +11,8 @@ import json import logging +from bkuser_core.categories.plugins.base import TypeList, TypeProtocol +from bkuser_core.common.progress import progress from django_celery_beat.models import IntervalSchedule, PeriodicTask logger = logging.getLogger(__name__) @@ -66,3 +68,24 @@ def delete_periodic_sync_task(category_id: int): except PeriodicTask.DoesNotExist: logger.warning("PeriodicTask %s has been deleted, skip it...", str(category_id)) return + + +def handle_with_progress_info( + item_list: TypeList[TypeProtocol], progress_title: str, continue_if_exception: bool = True +): + """控制进度""" + total = len(item_list) + for index, (key, item) in enumerate(item_list.items()): # type: int, (str, TypeProtocol) + try: + progress( + index + 1, + total, + f"{progress_title}: {item.display_str}<{key}> ({index + 1}/{total})", + ) + yield item + except Exception: + logger.exception("%s failed", progress_title) + if continue_if_exception: + continue + + raise diff --git a/src/api/bkuser_core/categories/views.py b/src/api/bkuser_core/categories/views.py index 6e0bb9540..0c7e7bb46 100644 --- a/src/api/bkuser_core/categories/views.py +++ b/src/api/bkuser_core/categories/views.py @@ -246,7 +246,7 @@ def test_fetch_data(self, request, lookup_value): raise error_codes.TEST_CONNECTION_FAILED.f("请确保连接设置正确") try: - syncer.test_fetch_data(serializer.validated_data) + syncer.fetcher.test_fetch_data(serializer.validated_data) except FetchDataFromRemoteFailed as e: raise error_codes.TEST_FETCH_DATA_FAILED.f(f"{e}") except Exception: # pylint: disable=broad-except diff --git a/src/api/bkuser_core/tests/categories/plugins/conftest.py b/src/api/bkuser_core/tests/categories/plugins/conftest.py new file mode 100644 index 000000000..57a466b99 --- /dev/null +++ b/src/api/bkuser_core/tests/categories/plugins/conftest.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available. +Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import pytest +from bkuser_core.categories.plugins.base import DBSyncManager, SyncContext +from bkuser_core.categories.plugins.ldap.adaptor import ProfileFieldMapper +from bkuser_core.categories.plugins.ldap.syncer import SETTING_FIELD_MAP + + +@pytest.fixture() +def ldap_config(): + return { + "user_member_of": "memberOf", + "basic_pull_node": "DC=center,DC=com", + "user_group_description": "description", + "user_group_name": "cn", + "user_group_class": "groupOfUniqueNames", + "mad_fields": [], + "bk_fields": "", + "telephone": "", + "email": "mail", + "display_name": "displayName", + "username": "sAMAccountName", + "organization_class": "organizationalUnit", + "user_class": "user", + "user_group_filter": "(objectclass=groupOfUniqueNames)", + "user_filter": "(&(objectCategory=Person)(sAMAccountName=*))", + "password": "password of Administrator", + "user": "CN=Administrator,CN=admin,DC=corp,DC=Fabrikam,DC=COM", + "base_dn": "DC=center,DC=com", + "pull_cycle": 60, + "timeout_setting": 120, + "connection_url": "ldap://127.0.0.1:389", + "ssl_encryption": "无", + } + + +@pytest.fixture() +def profile_field_mapper(ldap_config): + return ProfileFieldMapper(config_loader=ldap_config, setting_field_map=SETTING_FIELD_MAP) + + +@pytest.fixture +def sync_context(): + return SyncContext() + + +@pytest.fixture +def db_sync_manager(): + return DBSyncManager() diff --git a/src/api/bkuser_core/tests/categories/plugins/custom/test_helper.py b/src/api/bkuser_core/tests/categories/plugins/custom/test_helper.py index 8bf083188..8583c3756 100644 --- a/src/api/bkuser_core/tests/categories/plugins/custom/test_helper.py +++ b/src/api/bkuser_core/tests/categories/plugins/custom/test_helper.py @@ -9,7 +9,6 @@ specific language governing permissions and limitations under the License. """ import pytest -from bkuser_core.categories.plugins.base import DBSyncManager, SyncContext from bkuser_core.categories.plugins.custom.helpers import DepSyncHelper, ProSyncHelper from bkuser_core.categories.plugins.custom.metas import CustomDepartmentMeta, CustomProfileMeta from bkuser_core.categories.plugins.custom.models import CustomDepartment, CustomProfile, CustomTypeList @@ -20,14 +19,8 @@ @pytest.fixture -def sync_context(): - return SyncContext() - - -@pytest.fixture -def make_pro_sync_helper(test_custom_category, sync_context): +def make_pro_sync_helper(test_custom_category, db_sync_manager, sync_context): def helper(target_obj_list: CustomTypeList) -> ProSyncHelper: - db_sync_manager = DBSyncManager() db_sync_manager.update_model_meta({"department": CustomDepartmentMeta, "profile": CustomProfileMeta}) return ProSyncHelper(test_custom_category, db_sync_manager, target_obj_list, context=sync_context) @@ -35,9 +28,8 @@ def helper(target_obj_list: CustomTypeList) -> ProSyncHelper: @pytest.fixture -def make_dep_sync_helper(test_custom_category, sync_context): +def make_dep_sync_helper(test_custom_category, db_sync_manager, sync_context): def helper(target_obj_list: CustomTypeList) -> DepSyncHelper: - db_sync_manager = DBSyncManager() db_sync_manager.update_model_meta({"department": CustomDepartmentMeta, "profile": CustomProfileMeta}) return DepSyncHelper(test_custom_category, db_sync_manager, target_obj_list, context=sync_context) diff --git a/src/api/bkuser_core/tests/categories/plugins/ldap/test_adaptor.py b/src/api/bkuser_core/tests/categories/plugins/ldap/test_adaptor.py new file mode 100644 index 000000000..22f4648d0 --- /dev/null +++ b/src/api/bkuser_core/tests/categories/plugins/ldap/test_adaptor.py @@ -0,0 +1,285 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available. +Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import pytest +from bkuser_core.categories.plugins.ldap.adaptor import ( + RDN, + department_adaptor, + parse_dn_tree, + parse_dn_value_list, + user_adaptor, +) +from bkuser_core.categories.plugins.ldap.models import DepartmentProfile, UserProfile + + +@pytest.mark.parametrize( + "dn, restrict_types, expected", + [ + ( + "CN=Jeff Smith,OU=Sales,DC=Fabrikam,DC=COM", + [], + ["Jeff Smith", "Sales", "Fabrikam", "COM"], + ), + ( + "CN=Jeff Smith,OU=Sales,DC=Fabrikam,DC=COM", + None, + ["Jeff Smith", "Sales", "Fabrikam", "COM"], + ), + ("CN=Jeff Smith,OU=Sales,DC=Fabrikam,DC=COM", ["DC"], ["Fabrikam", "COM"]), + ( + "CN=Karen Berge,CN=admin,DC=corp,DC=Fabrikam,DC=COM", + ["CN"], + ["Karen Berge", "admin"], + ), + ( + "CN=Karen Berge,CN=admin,DC=corp,DC=Fabrikam,DC=COM", + ["CN", "DC"], + ["Karen Berge", "admin", "corp", "Fabrikam", "COM"], + ), + ( + "CN=Karen Berge,CN=admin,DC=corp,DC=Fabrikam,DC=COM", + [], + ["Karen Berge", "admin", "corp", "Fabrikam", "COM"], + ), + ], +) +def test_parse_dn_value_list(dn, restrict_types, expected): + assert parse_dn_value_list(dn, restrict_types) == expected + + +@pytest.mark.parametrize( + "dn, restrict_types, expected", + [ + ( + "CN=Jeff Smith,OU=Sales,DC=Fabrikam,DC=COM", + [], + [ + RDN(type="CN", value="Jeff Smith", separator=","), + RDN(type="OU", value="Sales", separator=","), + RDN(type="DC", value="Fabrikam", separator=","), + RDN(type="DC", value="COM", separator=""), + ], + ), + ( + "CN=Jeff Smith,OU=Sales,DC=Fabrikam,DC=COM", + None, + [ + RDN(type="CN", value="Jeff Smith", separator=","), + RDN(type="OU", value="Sales", separator=","), + RDN(type="DC", value="Fabrikam", separator=","), + RDN(type="DC", value="COM", separator=""), + ], + ), + ( + "CN=Jeff Smith,OU=Sales,DC=Fabrikam,DC=COM", + ["DC"], + [ + RDN(type="DC", value="Fabrikam", separator=","), + RDN(type="DC", value="COM", separator=""), + ], + ), + ( + "CN=Karen Berge,CN=admin,DC=corp,DC=Fabrikam,DC=COM", + [], + [ + RDN(type="CN", value="Karen Berge", separator=","), + RDN(type="CN", value="admin", separator=","), + RDN(type="DC", value="corp", separator=","), + RDN(type="DC", value="Fabrikam", separator=","), + RDN(type="DC", value="COM", separator=""), + ], + ), + ( + "CN=Karen Berge,CN=admin,DC=corp,DC=Fabrikam,DC=COM", + ["CN"], + [ + RDN(type="CN", value="Karen Berge", separator=","), + RDN(type="CN", value="admin", separator=","), + ], + ), + ( + "CN=Karen Berge,CN=admin,DC=corp,DC=Fabrikam,DC=COM", + ["CN", "DC"], + [ + RDN(type="CN", value="Karen Berge", separator=","), + RDN(type="CN", value="admin", separator=","), + RDN(type="DC", value="corp", separator=","), + RDN(type="DC", value="Fabrikam", separator=","), + RDN(type="DC", value="COM", separator=""), + ], + ), + ( + "cn=xx x,cn=《梦工厂大冒险》攻坚组,ou=4+1,dc=test,dc=or g", + [], + [ + RDN(type="cn", value="xx x", separator=","), + RDN(type="cn", value="《梦工厂大冒险》攻坚组", separator=","), + RDN(type="ou", value="4\\+1", separator=","), + RDN(type="dc", value="test", separator=","), + RDN(type="dc", value="or g", separator=""), + ], + ), + ( + "cn=xx x,cn=qq q,ou=4+1,dc=test,dc=or g", + [], + [ + RDN(type="cn", value="xx x", separator=","), + RDN(type="cn", value="qq q", separator=","), + RDN(type="ou", value="4\\+1", separator=","), + RDN(type="dc", value="test", separator=","), + RDN(type="dc", value="or g", separator=""), + ], + ), + ], +) +def test_parse_dn_tree(dn, restrict_types, expected): + assert parse_dn_tree(dn, restrict_types) == expected + + +@pytest.mark.parametrize( + "user_meta, restrict_types, expected", + [ + ( + { + "raw_dn": b"CN=Administrator,CN=Users,DC=center,DC=com", + "dn": "CN=Administrator,CN=Users,DC=center,DC=com", + "raw_attributes": { + "memberOf": [ + b"CN=Group Policy Creator Owners,CN=Users,DC=center,DC=com", + b"CN=Domain Admins,CN=Users,DC=center,DC=com", + b"CN=Enterprise Admins,CN=Users,DC=center,DC=com", + b"CN=Schema Admins,CN=Users,DC=center,DC=com", + b"CN=Administrators,CN=Builtin,DC=center,DC=com", + ], + "sAMAccountName": [b"Administrator"], + "mail": [b"asdf@asdf.com"], + "displayName": [], + }, + "attributes": { + "memberOf": [ + "CN=Group Policy Creator Owners,CN=Users,DC=center,DC=com", + "CN=Domain Admins,CN=Users,DC=center,DC=com", + "CN=Enterprise Admins,CN=Users,DC=center,DC=com", + "CN=Schema Admins,CN=Users,DC=center,DC=com", + "CN=Administrators,CN=Builtin,DC=center,DC=com", + ], + "sAMAccountName": "Administrator", + "mail": "asdf@asdf.com", + "displayName": [], + }, + "type": "searchResEntry", + }, + [], + UserProfile( + username="Administrator", + email="asdf@asdf.com", + telephone="", + display_name="", + code="dummy", + departments=[ + ["com", "center", "Users"], + ["com", "center", "Users", "Group Policy Creator Owners"], + ["com", "center", "Users", "Domain Admins"], + ["com", "center", "Users", "Enterprise Admins"], + ["com", "center", "Users", "Schema Admins"], + ["com", "center", "Builtin", "Administrators"], + ], + ), + ), + ( + { + "raw_dn": b"CN=Guest,CN=Users,DC=center,DC=com", + "dn": "CN=Guest,CN=Users,DC=center,DC=com", + "raw_attributes": { + "memberOf": [b"CN=Guests,OU=Builtin,DC=center,DC=com"], + "sAMAccountName": [b"Guest"], + "displayName": [], + "mail": [], + }, + "attributes": { + "memberOf": ["CN=Guests,CN=Builtin,DC=center,DC=com"], + "sAMAccountName": "Guest", + "displayName": [], + "mail": [], + }, + "type": "searchResEntry", + }, + ["OU", "CN"], + UserProfile( + username="Guest", + email="", + telephone="", + display_name="", + code="dummy", + departments=[ + ["Users"], + ["Builtin", "Guests"], + ], + ), + ), + ], +) +def test_user_adaptor(profile_field_mapper, user_meta, restrict_types, expected): + assert ( + user_adaptor( + code="dummy", + user_meta=user_meta, + field_mapper=profile_field_mapper, + restrict_types=restrict_types, + ) + == expected + ) + + +@pytest.mark.parametrize( + "dept_meta, restrict_types, expected", + [ + ( + { + "raw_dn": b"OU=shenzhen,OU=guangdong,DC=center,DC=com", + "dn": "OU=shenzhen,OU=guangdong,DC=center,DC=com", + "raw_attributes": {}, + "attributes": {}, + "type": "searchResEntry", + }, + ["OU", "CN"], + DepartmentProfile( + name="shenzhen", + parent=DepartmentProfile(name="guangdong"), + code="dummy", + ), + ), + ( + { + "raw_dn": b"OU=shenzhen,OU=guangdong,OU=china,DC=center,DC=com", + "dn": "OU=shenzhen,OU=guangdong,OU=china,DC=center,DC=com", + "raw_attributes": {}, + "attributes": {}, + "type": "searchResEntry", + }, + ["OU", "CN"], + DepartmentProfile( + name="shenzhen", + parent=DepartmentProfile(name="guangdong", parent=DepartmentProfile(name="china")), + code="dummy", + ), + ), + ], +) +def test_department_adaptor(dept_meta, restrict_types, expected): + assert ( + department_adaptor( + code="dummy", + dept_meta=dept_meta, + is_group=False, + restrict_types=restrict_types, + ) + == expected + ) diff --git a/src/api/bkuser_core/tests/categories/plugins/ldap/test_client.py b/src/api/bkuser_core/tests/categories/plugins/ldap/test_client.py index fbaed3471..b337bacce 100644 --- a/src/api/bkuser_core/tests/categories/plugins/ldap/test_client.py +++ b/src/api/bkuser_core/tests/categories/plugins/ldap/test_client.py @@ -21,21 +21,16 @@ class TestClient: def test_error_server_load(self, test_ldap_config_provider): """测试无法正常连接 Ldap""" - test_ldap_config_provider["connection_url"] = "ldap://localhost:389" + test_ldap_config_provider["connection_url"] = "ldap://localhost:3891" with pytest.raises(LdapCannotBeInitialized): LDAPClient(test_ldap_config_provider) def test_correct_server_load(self, test_ldap_config_provider): """测试正常连接 Ldap(仅当存在可用 Ldap 服务器时可用)""" - if not settings.TEST_LDAP: - return LDAPClient(test_ldap_config_provider) def test_search(self, test_ldap_config_provider): """测试正常搜索(仅当存在可用 Ldap 服务器时可用)""" - if not settings.TEST_LDAP: - return - client = LDAPClient(test_ldap_config_provider) client.search( start_root=test_ldap_config_provider["basic_pull_node"], @@ -47,9 +42,6 @@ def test_search(self, test_ldap_config_provider): def test_check(self, test_ldap_config_provider): """测试登陆""" - if not settings.TEST_LDAP: - return - client = LDAPClient(test_ldap_config_provider) with pytest.raises(ldap3.core.exceptions.LDAPBindError): diff --git a/src/api/bkuser_core/tests/categories/plugins/ldap/test_helper.py b/src/api/bkuser_core/tests/categories/plugins/ldap/test_helper.py new file mode 100644 index 000000000..0a002a870 --- /dev/null +++ b/src/api/bkuser_core/tests/categories/plugins/ldap/test_helper.py @@ -0,0 +1,208 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available. +Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import pytest +from bkuser_core.categories.plugins.base import TypeList +from bkuser_core.categories.plugins.ldap.adaptor import department_adaptor, user_adaptor +from bkuser_core.categories.plugins.ldap.helper import DepartmentSyncHelper, ProfileSyncHelper +from bkuser_core.categories.plugins.ldap.metas import LdapDepartmentMeta, LdapProfileMeta +from bkuser_core.categories.plugins.ldap.models import DepartmentProfile, UserProfile +from bkuser_core.departments.models import Department, DepartmentThroughModel, Profile + +pytestmark = pytest.mark.django_db + + +class TestDepartmentSyncHelper: + @pytest.fixture(autouse=True) + def setup(self, db_sync_manager): + db_sync_manager.update_model_meta({"department": LdapDepartmentMeta, "profile": LdapProfileMeta}) + + @pytest.mark.parametrize( + "dept_info, expected_count", + [(DepartmentProfile(name="c", parent=DepartmentProfile(name="b", parent=DepartmentProfile(name="a"))), 3)], + ) + def test_handle_department( + self, test_ldap_category, test_ldap_config_provider, db_sync_manager, sync_context, dept_info, expected_count + ): + helper = DepartmentSyncHelper( + test_ldap_category, db_sync_manager, TypeList[DepartmentProfile](), sync_context, test_ldap_config_provider + ) + helper._handle_department(dept_info) + assert len(db_sync_manager[Department].adding_items) == expected_count + + dept = dept_info + while dept: + if dept.parent: + assert ( + db_sync_manager.magic_get(dept.key_field, target_meta=LdapDepartmentMeta).parent_id + 1 + == db_sync_manager.magic_get(dept.key_field, target_meta=LdapDepartmentMeta).pk + ) + dept = dept.parent + + @pytest.mark.parametrize( + "departments, expected_logs, expected_count, expected_groups", + [ + ( + [ + { + "raw_dn": b"ou=shenzhen,ou=guangdong,dc=center,dc=com", + "dn": "ou=shenzhen,ou=guangdong,dc=center,dc=com", + "raw_attributes": {}, + "attributes": {}, + "type": "searchResEntry", + }, + { + "raw_dn": b"ou=beijing,dc=center,dc=com", + "dn": "ou=beijing,dc=center,dc=com", + "raw_attributes": {}, + "attributes": {}, + "type": "searchResEntry", + }, + { + "raw_dn": b"ou=Domain Controllers,dc=center,dc=com", + "dn": "ou=Domain Controllers,dc=center,dc=com", + "raw_attributes": {}, + "attributes": {}, + "type": "searchResEntry", + }, + ], + [ + "handle department: guangdong/shenzhen (1/3)", + "handle department: beijing (2/3)", + "handle department: Domain Controllers (3/3)", + ], + 4, + [["guangdong", "shenzhen"], ["beijing"], ["Domain Controllers"]], + ) + ], + ) + def test_load_then_sync( + self, + test_ldap_category, + db_sync_manager, + sync_context, + test_ldap_config_provider, + caplog, + departments, + expected_logs, + expected_count, + expected_groups, + ): + target_objs = [ + department_adaptor(code="", dept_meta=dept_meta, is_group=False, restrict_types=["ou", "cn"]) + for dept_meta in departments + ] + helper = DepartmentSyncHelper( + test_ldap_category, + db_sync_manager, + TypeList[DepartmentProfile].from_list(target_objs), + sync_context, + test_ldap_config_provider, + ) + helper.load_to_memory() + + for log in expected_logs: + assert log in caplog.text + + helper.db_sync_manager.sync_type(target_type=Department) + assert Department.objects.filter(category_id=test_ldap_category.id).count() == expected_count + + for group in expected_groups: + parent = None + for member in group: + parent = Department.objects.get(category_id=test_ldap_category.id, name=member, parent=parent) + + +class TestProfileSyncHelper: + @pytest.mark.parametrize( + "users, departments, expected_logs, expect_groups", + [ + ( + [ + { + "raw_dn": b"CN=Administrator,CN=Users,DC=center,DC=com", + "dn": "CN=Administrator,CN=Users,DC=center,DC=com", + "raw_attributes": { + "memberOf": [ + b"CN=Group Policy Creator Owners,CN=Users,DC=center,DC=com", + ], + "sAMAccountName": [b"Administrator"], + "mail": [b"asdf@asdf.com"], + "displayName": [b"fakeman"], + }, + "attributes": {"memberOf": ["CN=Group Policy Creator Owners,CN=Users,DC=center,DC=com"]}, + "type": "searchResEntry", + }, + { + "raw_dn": b"CN=Guest,CN=Users,DC=center,DC=com", + "dn": "CN=Guest,CN=Users,DC=center,DC=com", + "raw_attributes": { + "memberOf": [], + "sAMAccountName": [b"Guest"], + "mail": [b"asdf@asdf.com"], + "displayName": [b"fakeman"], + }, + "attributes": {"memberOf": []}, + "type": "searchResEntry", + }, + ], + [["Users", "Administrator"], ["Users", "Group Policy Creator Owners"], ["Users", "Guest"]], + ["handle profile: fakeman (1/2)", "handle profile: fakeman (2/2)"], + {"Guest": ['Users'], "Administrator": ['Users', 'Group Policy Creator Owners']}, + ) + ], + ) + def test_load_then_sync( + self, + test_ldap_category, + db_sync_manager, + sync_context, + profile_field_mapper, + caplog, + users, + departments, + expected_logs, + expect_groups, + ): + for group in departments: + parent = None + for member in group: + parent, _ = Department.objects.update_or_create( + name=member, parent=parent, defaults={"enabled": True}, category_id=test_ldap_category.pk + ) + + target_objs = [ + user_adaptor( + code=None, user_meta=user_meta, field_mapper=profile_field_mapper, restrict_types=["ou", "cn"] + ) + for user_meta in users + ] + helper = ProfileSyncHelper( + test_ldap_category, + db_sync_manager, + TypeList[UserProfile].from_list(target_objs), + sync_context, + ) + helper.load_to_memory() + + for log in expected_logs: + assert log in caplog.text + + helper.db_sync_manager.sync_type(target_type=Profile) + helper.db_sync_manager.sync_type(target_type=DepartmentThroughModel) + assert Profile.objects.filter(category_id=test_ldap_category.id).filter( + username__in=expect_groups.keys() + ).count() == len(expect_groups) + for username, departments in expect_groups.items(): + assert sorted( + DepartmentThroughModel.objects.filter(profile__username=username).values_list( + "department__name", flat=True + ) + ) == sorted(departments) diff --git a/src/api/bkuser_core/tests/categories/plugins/ldap/test_syncer.py b/src/api/bkuser_core/tests/categories/plugins/ldap/test_syncer.py index 737798429..d1b35b4e2 100644 --- a/src/api/bkuser_core/tests/categories/plugins/ldap/test_syncer.py +++ b/src/api/bkuser_core/tests/categories/plugins/ldap/test_syncer.py @@ -8,6 +8,8 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ +from unittest import mock + import pytest from bkuser_core.categories.plugins.metas import ProfileMeta from bkuser_core.departments.models import Department @@ -18,73 +20,7 @@ class TestSyncer: @pytest.mark.parametrize( - "dn,first,second", - [ - ("cn=xxx,cn=qqq,ou=vvv,dc=test,dc=org", {"cn": "xxx"}, {"dc": "org"}), - # 带空格 - ("cn=xx x,cn=qq q,ou=v vv,dc=test,dc=or g", {"cn": "xx x"}, {"dc": "or g"}), - ], - ) - def test_parse_tree(self, test_ldap_syncer, dn, first, second): - """测试解析 dn 树""" - b = test_ldap_syncer._parse_tree(dn) - assert b[0] == first - assert b[-1] == second - - @pytest.mark.parametrize( - "dn,target,first,second", - [ - ( - "cn=xxx,cn=qqq,ou=vvv,dc=test,dc=org", - ["cn"], - {"cn": "xxx"}, - {"cn": "qqq"}, - ), - ( - "cn=xxx,ou=ddd,ou=vvv,dc=test,dc=org", - ["ou"], - {"ou": "ddd"}, - {"ou": "vvv"}, - ), - ], - ) - def test_parse_tree_restrict(self, test_ldap_syncer, dn, target, first, second): - b = test_ldap_syncer._parse_tree(dn, target) - - assert b[0] == first - assert b[1] == second - - @pytest.mark.parametrize( - "dn,results", - [ - ( - "cn=xx x,cn=qq q,ou=4+1,dc=test,dc=or g", - [ - {"cn": "xx x"}, - {"cn": "qq q"}, - {"ou": "4\\+1"}, - {"dc": "test"}, - {"dc": "or g"}, - ], - ), - ( - "cn=xx x,cn=《梦工厂大冒险》攻坚组,ou=4+1,dc=test,dc=or g", - [ - {"cn": "xx x"}, - {"cn": "《梦工厂大冒险》攻坚组"}, - {"ou": "4\\+1"}, - {"dc": "test"}, - {"dc": "or g"}, - ], - ), - ], - ) - def test_parse_tree_with_special(self, test_ldap_syncer, dn, results): - """测试解析 dn 树，特殊字符""" - assert test_ldap_syncer._parse_tree(dn) == results - - @pytest.mark.parametrize( - "pre_created,users,expected_adding, expected_updating", + "pre_created, users, expected_adding, expected_updating", [ ( [], @@ -219,7 +155,9 @@ def test_sync_users(self, pre_created, test_ldap_syncer, users, expected_adding, for p in pre_created: make_simple_profile(p, force_create_params={"category_id": test_ldap_syncer.category_id}) - test_ldap_syncer._sync_users(users) + with mock.patch.object(test_ldap_syncer.fetcher, "fetch") as fetch: + fetch.return_value = [], [], users + test_ldap_syncer._sync_profile() for k in expected_adding: assert ( @@ -234,7 +172,7 @@ def test_sync_users(self, pre_created, test_ldap_syncer, users, expected_adding, ) @pytest.mark.parametrize( - "departments,expected", + "departments, expected", [ ( [ @@ -278,7 +216,9 @@ def test_sync_users(self, pre_created, test_ldap_syncer, users, expected_adding, ) def test_sync_departments(self, test_ldap_syncer, departments, expected): """测试同步部门""" - test_ldap_syncer._sync_departments(departments) + with mock.patch.object(test_ldap_syncer.fetcher, "fetch") as fetch: + fetch.return_value = [], departments, [] + test_ldap_syncer._sync_department() for route in expected: parent = None @@ -335,13 +275,11 @@ def test_sync_departments(self, test_ldap_syncer, departments, expected): ) def test_sync_groups(self, test_ldap_syncer, groups, expected): """测试同步组织""" - test_ldap_syncer._sync_departments(groups, True) + with mock.patch.object(test_ldap_syncer.fetcher, "fetch") as fetch: + fetch.return_value = groups, [], [] + test_ldap_syncer._sync_department() for route in expected: parent = None for d in route: parent = Department.objects.get(name=d, parent=parent, category_id=test_ldap_syncer.category_id) - - -class TestFetcher: - """Test Fetcher""" diff --git a/src/api/bkuser_core/tests/conftest.py b/src/api/bkuser_core/tests/conftest.py index f3aeedcf5..c3447dab0 100644 --- a/src/api/bkuser_core/tests/conftest.py +++ b/src/api/bkuser_core/tests/conftest.py @@ -46,6 +46,9 @@ def test_ldap_category() -> ProfileCategory: @pytest.fixture def test_ldap_config_provider(test_ldap_category) -> ConfigProvider: + if not settings.TEST_LDAP: + return pytest.skip("未配置测试的 Ldap 服务器") + c = ConfigProvider(test_ldap_category.id) c["connection_url"] = settings.TEST_LDAP["url"] c["user"] = settings.TEST_LDAP["user"] From 0e44f3177fa6b6caaccd940e6bc8617c5a312d05 Mon Sep 17 00:00:00 2001 From: shabbywu Date: Tue, 24 Aug 2021 14:39:46 +0800 Subject: [PATCH 2/2] refactor with discussion --- src/api/bkuser_core/bkiam/constants.py | 4 ++-- .../management/commands/test_category_sync.py | 6 +++++- src/api/bkuser_core/categories/plugins/base.py | 10 +++++----- .../categories/plugins/custom/helpers.py | 6 +++--- .../bkuser_core/categories/plugins/ldap/adaptor.py | 12 ++++++------ .../bkuser_core/categories/plugins/ldap/helper.py | 8 ++------ .../bkuser_core/categories/plugins/ldap/syncer.py | 6 +++--- src/api/bkuser_core/categories/tasks.py | 4 ++-- src/api/bkuser_core/categories/views.py | 14 +++++++------- .../tests/categories/plugins/ldap/test_adaptor.py | 8 ++++---- .../tests/categories/plugins/ldap/test_helper.py | 6 +++--- 11 files changed, 42 insertions(+), 42 deletions(-) diff --git a/src/api/bkuser_core/bkiam/constants.py b/src/api/bkuser_core/bkiam/constants.py index ae1d72069..1a5750218 100644 --- a/src/api/bkuser_core/bkiam/constants.py +++ b/src/api/bkuser_core/bkiam/constants.py @@ -147,8 +147,8 @@ def get_by_model(cls, instance) -> "ResourceType": @classmethod def get_attr_by_model(cls, instance, index: int) -> str: """通过 model instance 获取""" - _type = cls.get_by_model(instance) - id_name_pair = cls.get_id_name_pair(_type) + type_ = cls.get_by_model(instance) + id_name_pair = cls.get_id_name_pair(type_) return getattr(instance, id_name_pair[index]) @classmethod diff --git a/src/api/bkuser_core/categories/management/commands/test_category_sync.py b/src/api/bkuser_core/categories/management/commands/test_category_sync.py index 2c9edadae..1e0fb4a68 100644 --- a/src/api/bkuser_core/categories/management/commands/test_category_sync.py +++ b/src/api/bkuser_core/categories/management/commands/test_category_sync.py @@ -9,6 +9,7 @@ specific language governing permissions and limitations under the License. """ import logging +import uuid from bkuser_core.categories.models import ProfileCategory from bkuser_core.categories.tasks import adapter_sync @@ -29,11 +30,14 @@ def add_arguments(self, parser): def handle(self, *args, **options): category_type = options["category_type"] excel_file = options["excel_file"] + task_id = uuid.uuid4() + self.stdout.write(f"Your Task ID: {str(task_id)}") if excel_file: try: adapter_sync( ProfileCategory.objects.filter(type=category_type)[0].pk, + task_id=task_id, raw_data_file=excel_file, ) except Exception: # pylint: disable=broad-except @@ -41,6 +45,6 @@ def handle(self, *args, **options): return try: - adapter_sync(ProfileCategory.objects.filter(type=category_type)[0].pk) + adapter_sync(ProfileCategory.objects.filter(type=category_type)[0].pk, task_id=task_id) except Exception: # pylint: disable=broad-except logger.exception("can not find category by type<%s>", category_type) diff --git a/src/api/bkuser_core/categories/plugins/base.py b/src/api/bkuser_core/categories/plugins/base.py index 2c01a8f92..fd2babf27 100644 --- a/src/api/bkuser_core/categories/plugins/base.py +++ b/src/api/bkuser_core/categories/plugins/base.py @@ -131,9 +131,9 @@ def __init__(self, meta_map: dict = None): def __getitem__(self, item): return self._sets[item] - def register_id(self, _type: Type[SyncModelMeta]): + def register_id(self, type_: Type[SyncModelMeta]): """注册自增ID""" - return next(self.id_generators[_type.target_model]) + return next(self.id_generators[type_.target_model]) def sync_type(self, target_type: Type[Model]): """针对某种类型同步""" @@ -146,10 +146,10 @@ def sync_all(self): def detect_model_manager(self, model_type: Type[Model]) -> SyncModelManager: """根据传递的 Model 类型获取对应的 SyncModelManager""" - for _type in list(self.meta_map.values()): - if issubclass(model_type, _type.target_model): + for type_ in list(self.meta_map.values()): + if issubclass(model_type, type_.target_model): return self._sets[model_type] - supported_types = [_type.target_model for _type in self.meta_map.values()] + supported_types = [type_.target_model for type_ in self.meta_map.values()] raise ValueError(f"Unsupported Type<{model_type}>, item should be within types: {supported_types}") def magic_add(self, item: Model, operation: SyncOperation = None): diff --git a/src/api/bkuser_core/categories/plugins/custom/helpers.py b/src/api/bkuser_core/categories/plugins/custom/helpers.py index cb736d114..228d68a02 100644 --- a/src/api/bkuser_core/categories/plugins/custom/helpers.py +++ b/src/api/bkuser_core/categories/plugins/custom/helpers.py @@ -65,7 +65,7 @@ class DepSyncHelper(DBSyncHelper): @cached_property def db_departments(self) -> Dict[str, Department]: # 由于 bulk_update 需要从数据库查询完整的 Department 信息, 为提高查询效率, 统一执行查询操作, 减轻数据库负担 - return {dep.code: dep for dep in Department.objects.filter(category_id=self.category.pk).all()} + return {dep.code: dep for dep in Department.objects.filter(category_id=self.category.pk)} def load_to_memory(self): for dept in handle_with_progress_info(self.target_obj_list, progress_title="handle department"): @@ -144,12 +144,12 @@ class ProSyncHelper(DBSyncHelper): @cached_property def db_profiles(self) -> Dict[str, Profile]: # 由于 bulk_update 需要从数据库查询完整的 Profile 信息, 为提高查询效率, 统一执行查询操作, 减轻数据库负担 - return {profile.username: profile for profile in Profile.objects.filter(category_id=self.category.pk).all()} + return {profile.username: profile for profile in Profile.objects.filter(category_id=self.category.pk)} @cached_property def db_departments(self) -> Dict[str, Department]: # 由于 bulk_update 需要从数据库查询完整的 Department 信息, 为提高查询效率, 统一执行查询操作, 减轻数据库负担 - return {dep.code: dep for dep in Department.objects.filter(category_id=self.category.pk, enabled=True).all()} + return {dep.code: dep for dep in Department.objects.filter(category_id=self.category.pk, enabled=True)} def _load_base_info(self): for info in handle_with_progress_info(self.target_obj_list, progress_title="handle profile"): diff --git a/src/api/bkuser_core/categories/plugins/ldap/adaptor.py b/src/api/bkuser_core/categories/plugins/ldap/adaptor.py index b5493ef93..1d56bd085 100644 --- a/src/api/bkuser_core/categories/plugins/ldap/adaptor.py +++ b/src/api/bkuser_core/categories/plugins/ldap/adaptor.py @@ -46,7 +46,7 @@ def get_user_attributes(self) -> list: return [self.config_loader[x] for x in self.setting_field_map.values() if self.config_loader[x]] -def user_adaptor( +def user_adapter( code: str, user_meta: Dict[str, Any], field_mapper: ProfileFieldMapper, restrict_types: List[str] ) -> UserProfile: groups = user_meta["attributes"][field_mapper.config_loader["user_member_of"]] @@ -67,7 +67,7 @@ def user_adaptor( ) -def department_adaptor(code: str, dept_meta: Dict, is_group: bool, restrict_types: List[str]) -> DepartmentProfile: +def department_adapter(code: str, dept_meta: Dict, is_group: bool, restrict_types: List[str]) -> DepartmentProfile: dn = dept_meta["dn"] dn_values = parse_dn_value_list(dn, restrict_types=restrict_types) @@ -79,7 +79,7 @@ def department_adaptor(code: str, dept_meta: Dict, is_group: bool, restrict_type is_group=is_group, ) - assert parent_dept is not None + assert parent_dept is not None, "未从 dn 中提取到任何部门信息" parent_dept.code = code return parent_dept @@ -92,7 +92,7 @@ class RDN(NamedTuple): separator: str -def parse_dn_tree(dn, restrict_types: List[str] = None) -> List[RDN]: +def parse_dn_tree(dn: str, restrict_types: List[str] = None) -> List[RDN]: """A DN is a sequence of relative distinguished names (RDN) connected by commas, For examples: we have a dn = "CN=Jeff Smith,OU=Sales,DC=Fabrikam,DC=COM", this method will parse the dn to: @@ -112,7 +112,7 @@ def parse_dn_tree(dn, restrict_types: List[str] = None) -> List[RDN]: See Also: https://docs.microsoft.com/en-us/previous-versions/windows/desktop/ldap/distinguished-names """ - restrict_types = [_type.upper() for _type in (restrict_types or [])] + restrict_types = [type_.upper() for type_ in (restrict_types or [])] items = dn_utils.parse_dn(dn, escape=True) if restrict_types: @@ -123,7 +123,7 @@ def parse_dn_tree(dn, restrict_types: List[str] = None) -> List[RDN]: return parts -def parse_dn_value_list(dn, restrict_types: List[str] = None) -> List[str]: +def parse_dn_value_list(dn: str, restrict_types: List[str] = None) -> List[str]: """this method work like parse_dn_tree, be only return values of those attributes, For examples: >>> parse_dn_value_list("CN=Jeff Smith,OU=Sales,DC=Fabrikam,DC=COM") diff --git a/src/api/bkuser_core/categories/plugins/ldap/helper.py b/src/api/bkuser_core/categories/plugins/ldap/helper.py index 2d70108ce..2b9a60e1c 100644 --- a/src/api/bkuser_core/categories/plugins/ldap/helper.py +++ b/src/api/bkuser_core/categories/plugins/ldap/helper.py @@ -49,9 +49,7 @@ class DepartmentSyncHelper: @cached_property def db_departments(self) -> Dict[str, Department]: # 由于 bulk_update 需要从数据库查询完整的 Department 信息, 为提高查询效率, 统一执行查询操作, 减轻数据库负担 - all_departments: List[Department] = list( - Department.objects.filter(category_id=self.category.pk, enabled=True).all() - ) + all_departments: List[Department] = list(Department.objects.filter(category_id=self.category.pk, enabled=True)) def make_key(dept: Department): names = [] @@ -135,9 +133,7 @@ def db_profiles(self) -> Dict[str, Profile]: @cached_property def db_departments(self) -> Dict[str, Department]: # 由于 bulk_update 需要从数据库查询完整的 Department 信息, 为提高查询效率, 统一执行查询操作, 减轻数据库负担 - all_departments: List[Department] = list( - Department.objects.filter(category_id=self.category.pk, enabled=True).all() - ) + all_departments: List[Department] = list(Department.objects.filter(category_id=self.category.pk, enabled=True)) def make_key(dept: Department): names = [] diff --git a/src/api/bkuser_core/categories/plugins/ldap/syncer.py b/src/api/bkuser_core/categories/plugins/ldap/syncer.py index 986edce17..2201ad17c 100644 --- a/src/api/bkuser_core/categories/plugins/ldap/syncer.py +++ b/src/api/bkuser_core/categories/plugins/ldap/syncer.py @@ -17,7 +17,7 @@ from bkuser_core.categories.exceptions import FetchDataFromRemoteFailed from bkuser_core.categories.plugins.base import DBSyncManager, Fetcher, SyncContext, Syncer, SyncStep, TypeList -from bkuser_core.categories.plugins.ldap.adaptor import ProfileFieldMapper, department_adaptor, user_adaptor +from bkuser_core.categories.plugins.ldap.adaptor import ProfileFieldMapper, department_adapter, user_adapter from bkuser_core.categories.plugins.ldap.client import LDAPClient from bkuser_core.categories.plugins.ldap.helper import DepartmentSyncHelper, ProfileSyncHelper from bkuser_core.categories.plugins.ldap.metas import LdapDepartmentMeta, LdapProfileMeta @@ -130,7 +130,7 @@ def fetch_profiles(self, restrict_types: List[str]): continue profiles.append( - user_adaptor( + user_adapter( code=self._get_code(user), user_meta=user, field_mapper=self.field_mapper, @@ -149,7 +149,7 @@ def fetch_departments(self, restrict_types: List[str]): continue results.append( - department_adaptor( + department_adapter( code=self._get_code(dept_meta), dept_meta=dept_meta, is_group=is_group, diff --git a/src/api/bkuser_core/categories/tasks.py b/src/api/bkuser_core/categories/tasks.py index bea4849a6..135426a65 100644 --- a/src/api/bkuser_core/categories/tasks.py +++ b/src/api/bkuser_core/categories/tasks.py @@ -22,7 +22,7 @@ @app.task -def adapter_sync(instance_id: int, *args, **kwargs): +def adapter_sync(instance_id: int, task_id: uuid.UUID, args, **kwargs): logger.info("going to sync Category<%s>", instance_id) instance = ProfileCategory.objects.get(pk=instance_id) @@ -41,7 +41,7 @@ def adapter_sync(instance_id: int, *args, **kwargs): raise error_codes.LOAD_DATA_ADAPTER_FAILED with catch_time() as context: - plugin.sync(instance_id=instance_id, task_id=uuid.uuid4(), *args, **kwargs) + plugin.sync(instance_id=instance_id, task_id=task_id, *args, **kwargs) logger.info(f"同步总耗时: {context.time_delta}s, 消耗总CPU时间: {context.clock_delta}s.") # 标记同步 diff --git a/src/api/bkuser_core/categories/views.py b/src/api/bkuser_core/categories/views.py index 0c7e7bb46..d3dbea48a 100644 --- a/src/api/bkuser_core/categories/views.py +++ b/src/api/bkuser_core/categories/views.py @@ -66,23 +66,23 @@ def list_metas(self, request): """ helper = IAMHelper() - def make_meta(_type: CategoryType): + def make_meta(type_: CategoryType): return { - "type": _type, - "description": CategoryType.get_description(_type), - "name": CategoryType.get_choice_label(_type), + "type": type_, + "description": CategoryType.get_description(type_), + "name": CategoryType.get_choice_label(type_), } metas = [] - for _type in CategoryType.all(): + for type_ in CategoryType.all(): # 这里目前只返回创建目录类型的权限操作，后期应该可扩展 try: - action_id = IAMAction.get_action_by_category_type(_type) + action_id = IAMAction.get_action_by_category_type(type_) except KeyError: # tof 属于隐藏目录，这里直接忽略掉 continue - _meta = make_meta(_type) + _meta = make_meta(type_) # Q：为什么这里需要手动判断权限，而不是通用 permission_classes？ # A：因为这里的资源（目录类型）是没有对应实体，同时也没有在权限中心注册 if need_iam(request) and not helper.action_allow(request.operator, action_id): diff --git a/src/api/bkuser_core/tests/categories/plugins/ldap/test_adaptor.py b/src/api/bkuser_core/tests/categories/plugins/ldap/test_adaptor.py index 22f4648d0..f676049d7 100644 --- a/src/api/bkuser_core/tests/categories/plugins/ldap/test_adaptor.py +++ b/src/api/bkuser_core/tests/categories/plugins/ldap/test_adaptor.py @@ -11,10 +11,10 @@ import pytest from bkuser_core.categories.plugins.ldap.adaptor import ( RDN, - department_adaptor, + department_adapter, parse_dn_tree, parse_dn_value_list, - user_adaptor, + user_adapter, ) from bkuser_core.categories.plugins.ldap.models import DepartmentProfile, UserProfile @@ -228,7 +228,7 @@ def test_parse_dn_tree(dn, restrict_types, expected): ) def test_user_adaptor(profile_field_mapper, user_meta, restrict_types, expected): assert ( - user_adaptor( + user_adapter( code="dummy", user_meta=user_meta, field_mapper=profile_field_mapper, @@ -275,7 +275,7 @@ def test_user_adaptor(profile_field_mapper, user_meta, restrict_types, expected) ) def test_department_adaptor(dept_meta, restrict_types, expected): assert ( - department_adaptor( + department_adapter( code="dummy", dept_meta=dept_meta, is_group=False, diff --git a/src/api/bkuser_core/tests/categories/plugins/ldap/test_helper.py b/src/api/bkuser_core/tests/categories/plugins/ldap/test_helper.py index 0a002a870..32a1052ed 100644 --- a/src/api/bkuser_core/tests/categories/plugins/ldap/test_helper.py +++ b/src/api/bkuser_core/tests/categories/plugins/ldap/test_helper.py @@ -10,7 +10,7 @@ """ import pytest from bkuser_core.categories.plugins.base import TypeList -from bkuser_core.categories.plugins.ldap.adaptor import department_adaptor, user_adaptor +from bkuser_core.categories.plugins.ldap.adaptor import department_adapter, user_adapter from bkuser_core.categories.plugins.ldap.helper import DepartmentSyncHelper, ProfileSyncHelper from bkuser_core.categories.plugins.ldap.metas import LdapDepartmentMeta, LdapProfileMeta from bkuser_core.categories.plugins.ldap.models import DepartmentProfile, UserProfile @@ -96,7 +96,7 @@ def test_load_then_sync( expected_groups, ): target_objs = [ - department_adaptor(code="", dept_meta=dept_meta, is_group=False, restrict_types=["ou", "cn"]) + department_adapter(code="", dept_meta=dept_meta, is_group=False, restrict_types=["ou", "cn"]) for dept_meta in departments ] helper = DepartmentSyncHelper( @@ -179,7 +179,7 @@ def test_load_then_sync( ) target_objs = [ - user_adaptor( + user_adapter( code=None, user_meta=user_meta, field_mapper=profile_field_mapper, restrict_types=["ou", "cn"] ) for user_meta in users