Merge pull request #28 from shabbywu/master

refactor: 重构 LDAP 协议的用户同步逻辑, 使其统计同步成功或失败的对象

src/api/bkuser_core/bkiam/constants.py

@@ -147,8 +147,8 @@ def get_by_model(cls, instance) -> "ResourceType":
     @classmethod
     def get_attr_by_model(cls, instance, index: int) -> str:
         """通过 model instance 获取"""
-        _type = cls.get_by_model(instance)
-        id_name_pair = cls.get_id_name_pair(_type)
+        type_ = cls.get_by_model(instance)
+        id_name_pair = cls.get_id_name_pair(type_)
         return getattr(instance, id_name_pair[index])
 
     @classmethod

src/api/bkuser_core/categories/management/commands/test_category_sync.py

@@ -9,6 +9,7 @@
 specific language governing permissions and limitations under the License.
 """
 import logging
+import uuid
 
 from bkuser_core.categories.models import ProfileCategory
 from bkuser_core.categories.tasks import adapter_sync
@@ -29,18 +30,21 @@ def add_arguments(self, parser):
     def handle(self, *args, **options):
         category_type = options["category_type"]
         excel_file = options["excel_file"]
+        task_id = uuid.uuid4()
+        self.stdout.write(f"Your Task ID: {str(task_id)}")
 
         if excel_file:
             try:
                 adapter_sync(
                     ProfileCategory.objects.filter(type=category_type)[0].pk,
+                    task_id=task_id,
                     raw_data_file=excel_file,
                 )
             except Exception:  # pylint: disable=broad-except
                 logger.exception("can not find category by type<%s>", category_type)
             return
 
         try:
-            adapter_sync(ProfileCategory.objects.filter(type=category_type)[0].pk)
+            adapter_sync(ProfileCategory.objects.filter(type=category_type)[0].pk, task_id=task_id)
         except Exception:  # pylint: disable=broad-except
             logger.exception("can not find category by type<%s>", category_type)

src/api/bkuser_core/categories/plugins/base.py

@@ -11,11 +11,11 @@
 import datetime
 import logging
 from abc import abstractmethod
-from collections import defaultdict
+from collections import UserDict, defaultdict
 from contextlib import contextmanager
 from dataclasses import dataclass, field
 from threading import RLock
-from typing import Any, ClassVar, Dict, List, Optional, Type
+from typing import Any, ClassVar, Dict, List, MutableMapping, Optional, Type, TypeVar
 
 from bkuser_core.categories.models import ProfileCategory
 from bkuser_core.categories.plugins.constants import SYNC_LOG_TEMPLATE_MAP, SyncStep
@@ -25,6 +25,7 @@
 from bkuser_core.profiles.models import LeaderThroughModel, Profile
 from bkuser_core.user_settings.loader import ConfigProvider
 from django.db.models import Model
+from typing_extensions import Protocol
 
 logger = logging.getLogger(__name__)
 
@@ -130,9 +131,9 @@ def __init__(self, meta_map: dict = None):
     def __getitem__(self, item):
         return self._sets[item]
 
-    def register_id(self, _type: Type[SyncModelMeta]):
+    def register_id(self, type_: Type[SyncModelMeta]):
         """注册自增ID"""
-        return next(self.id_generators[_type.target_model])
+        return next(self.id_generators[type_.target_model])
 
     def sync_type(self, target_type: Type[Model]):
         """针对某种类型同步"""
@@ -145,10 +146,10 @@ def sync_all(self):
 
     def detect_model_manager(self, model_type: Type[Model]) -> SyncModelManager:
         """根据传递的 Model 类型获取对应的 SyncModelManager"""
-        for _type in list(self.meta_map.values()):
-            if issubclass(model_type, _type.target_model):
+        for type_ in list(self.meta_map.values()):
+            if issubclass(model_type, type_.target_model):
                 return self._sets[model_type]
-        supported_types = [_type.target_model for _type in self.meta_map.values()]
+        supported_types = [type_.target_model for type_ in self.meta_map.values()]
         raise ValueError(f"Unsupported Type<{model_type}>, item should be within types: {supported_types}")
 
     def magic_add(self, item: Model, operation: SyncOperation = None):
@@ -298,3 +299,41 @@ class LoginHandler:
     @abstractmethod
     def check(self, *args, **kwargs):
         raise NotImplementedError
+
+
+class TypeProtocol(Protocol):
+    @property
+    def key_field(self) -> str:
+        """The Key Field to make obj unique."""
+
+    @property
+    def display_str(self) -> str:
+        """The Display str for obj."""
+
+
+M = TypeVar("M")
+
+
+class TypeList(UserDict, MutableMapping[str, M]):
+    @classmethod
+    def from_list(cls, items: List[TypeProtocol]):
+        items_map = {i.key_field: i for i in items}
+        return cls(items_map)
+
+    @classmethod
+    def get_type(cls) -> Type[M]:
+        # As of Python 3.6. there is a public __args__ and (__parameters__) field for Generic
+        return cls.__args__[0]  # type: ignore
+
+
+class DBSyncHelper(Protocol):
+    """将 TypeList 塞入到 DBSyncManager 中的协议"""
+
+    category: ProfileCategory
+    db_sync_manager: DBSyncManager
+    target_obj_list: TypeList
+    context: SyncContext
+
+    def load_to_memory(self):
+        """将数据对象加载到内存"""
+        raise NotImplementedError

src/api/bkuser_core/categories/plugins/custom/client.py

@@ -14,11 +14,10 @@
 
 import curlify
 import requests
+from bkuser_core.categories.plugins.custom.exceptions import CustomAPIRequestFailed
+from bkuser_core.categories.plugins.custom.models import CustomDepartment, CustomProfile, CustomTypeList
 from bkuser_core.user_settings.loader import ConfigProvider
 
-from .exceptions import CustomAPIRequestFailed
-from .models import CustomDepartment, CustomProfile, CustomTypeList
-
 logger = logging.getLogger(__name__)
 
 

src/api/bkuser_core/categories/plugins/custom/helpers.py

@@ -21,6 +21,7 @@
 from bkuser_core.categories.plugins.custom.utils import handle_with_progress_info
 from bkuser_core.common.db_sync import SyncOperation
 from bkuser_core.departments.models import Department, DepartmentThroughModel
+from bkuser_core.profiles.constants import ProfileStatus
 from bkuser_core.profiles.models import LeaderThroughModel, Profile
 from bkuser_core.profiles.validators import validate_username
 from django.db.models import Model
@@ -64,7 +65,7 @@ class DepSyncHelper(DBSyncHelper):
     @cached_property
     def db_departments(self) -> Dict[str, Department]:
         # 由于 bulk_update 需要从数据库查询完整的 Department 信息, 为提高查询效率, 统一执行查询操作, 减轻数据库负担
-        return {dep.code: dep for dep in Department.objects.filter(category_id=self.category.pk).all()}
+        return {dep.code: dep for dep in Department.objects.filter(category_id=self.category.pk)}
 
     def load_to_memory(self):
         for dept in handle_with_progress_info(self.target_obj_list, progress_title="handle department"):
@@ -143,12 +144,12 @@ class ProSyncHelper(DBSyncHelper):
     @cached_property
     def db_profiles(self) -> Dict[str, Profile]:
         # 由于 bulk_update 需要从数据库查询完整的 Profile 信息, 为提高查询效率, 统一执行查询操作, 减轻数据库负担
-        return {profile.username: profile for profile in Profile.objects.filter(category_id=self.category.pk).all()}
+        return {profile.username: profile for profile in Profile.objects.filter(category_id=self.category.pk)}
 
     @cached_property
     def db_departments(self) -> Dict[str, Department]:
         # 由于 bulk_update 需要从数据库查询完整的 Department 信息, 为提高查询效率, 统一执行查询操作, 减轻数据库负担
-        return {dep.code: dep for dep in Department.objects.filter(category_id=self.category.pk, enabled=True).all()}
+        return {dep.code: dep for dep in Department.objects.filter(category_id=self.category.pk, enabled=True)}
 
     def _load_base_info(self):
         for info in handle_with_progress_info(self.target_obj_list, progress_title="handle profile"):
@@ -172,6 +173,7 @@ def _load_base_info(self):
                 "telephone": info.telephone,
                 "position": info.position,
                 "extras": info.extras,
+                "status": ProfileStatus.NORMAL.value,
             }
 
             # 2. 更新或创建 Profile 对象

src/api/bkuser_core/categories/plugins/ldap/adaptor.py

@@ -0,0 +1,141 @@
+# -*- coding: utf-8 -*-
+from dataclasses import dataclass
+from typing import Any, Dict, List, NamedTuple, Optional
+
+from bkuser_core.categories.plugins.ldap.models import DepartmentProfile, UserProfile
+from bkuser_core.user_settings.loader import ConfigProvider
+from django.utils.encoding import force_str
+from ldap3.utils import dn as dn_utils
+
+
+@dataclass
+class ProfileFieldMapper:
+    """从 ldap 对象属性中获取用户字段"""
+
+    config_loader: ConfigProvider
+    setting_field_map: dict
+
+    def get_field(self, user_meta: Dict[str, List[bytes]], field_name: str, raise_exception: bool = False) -> str:
+        """根据字段映射关系, 从 ldap 中获取 `field_name` 的值"""
+        try:
+            setting_name = self.setting_field_map[field_name]
+        except KeyError:
+            if raise_exception:
+                raise ValueError("该用户字段没有在配置中有对应项，无法同步")
+            return ""
+
+        try:
+            ldap_field_name = self.config_loader[setting_name]
+        except KeyError:
+            if raise_exception:
+                raise ValueError(f"用户目录配置中缺失字段 {setting_name}")
+            return ""
+
+        try:
+            if user_meta[ldap_field_name]:
+                return force_str(user_meta[ldap_field_name][0])
+
+            return ""
+        except KeyError:
+            if raise_exception:
+                raise ValueError(f"搜索数据中没有对应的字段 {ldap_field_name}")
+            return ""
+
+    def get_user_attributes(self) -> list:
+        """获取远端属性名列表"""
+        return [self.config_loader[x] for x in self.setting_field_map.values() if self.config_loader[x]]
+
+
+def user_adapter(
+    code: str, user_meta: Dict[str, Any], field_mapper: ProfileFieldMapper, restrict_types: List[str]
+) -> UserProfile:
+    groups = user_meta["attributes"][field_mapper.config_loader["user_member_of"]]
+
+    return UserProfile(
+        username=field_mapper.get_field(user_meta=user_meta["raw_attributes"], field_name="username"),
+        email=field_mapper.get_field(user_meta=user_meta["raw_attributes"], field_name="email"),
+        telephone=field_mapper.get_field(user_meta=user_meta["raw_attributes"], field_name="telephone"),
+        display_name=field_mapper.get_field(user_meta=user_meta["raw_attributes"], field_name="display_name"),
+        code=code,
+        # TODO: 完成转换 departments 的逻辑
+        departments=[
+            # 根据约定, dn 中除去第一个成分以外的部分即为用户所在的部门, 因此需要取 [1:]
+            list(reversed(parse_dn_value_list(user_meta["dn"], restrict_types)[1:])),
+            # 用户与用户组之间的关系
+            *[list(reversed(parse_dn_value_list(group, restrict_types))) for group in groups],
+        ],
+    )
+
+
+def department_adapter(code: str, dept_meta: Dict, is_group: bool, restrict_types: List[str]) -> DepartmentProfile:
+    dn = dept_meta["dn"]
+    dn_values = parse_dn_value_list(dn, restrict_types=restrict_types)
+
+    parent_dept: Optional[DepartmentProfile] = None
+    for dept_name in reversed(dn_values):
+        parent_dept = DepartmentProfile(
+            name=dept_name,
+            parent=parent_dept,
+            is_group=is_group,
+        )
+
+    assert parent_dept is not None, "未从 dn 中提取到任何部门信息"
+    parent_dept.code = code
+    return parent_dept
+
+
+class RDN(NamedTuple):
+    """RelativeDistinguishedName"""
+
+    type: str
+    value: str
+    separator: str
+
+
+def parse_dn_tree(dn: str, restrict_types: List[str] = None) -> List[RDN]:
+    """A DN is a sequence of relative distinguished names (RDN) connected by commas, For examples:
+
+    we have a dn = "CN=Jeff Smith,OU=Sales,DC=Fabrikam,DC=COM", this method will parse the dn to:
+    >>> parse_dn_tree("CN=Jeff Smith,OU=Sales,DC=Fabrikam,DC=COM")
+    [RDN(type='CN', value='Jeff Smith', separator=','),
+     RDN(type='OU', value='Sales', separator=','),
+     RDN(type='DC', value='Fabrikam', separator=','),
+     RDN(type='DC', value='COM', separator='')]
+
+    if provide restrict_types, this method will ignore the attribute not in restrict_types, For examples:
+    >>> parse_dn_tree("CN=Jeff Smith,OU=Sales,DC=Fabrikam,DC=COM", restrict_types=["DC"])
+    [RDN(type='DC', value='Fabrikam', separator=','), RDN(type='DC', value='COM', separator='')]
+
+    Furthermore, restrict_types is Case-insensitive, the ["DC"], ["dc"], ["Dc"] are Exactly equal.
+    >>> parse_dn_tree("CN=Jeff Smith,OU=Sales,DC=Fabrikam,DC=COM", restrict_types=["dc"])
+    [RDN(type='DC', value='Fabrikam', separator=','), RDN(type='DC', value='COM', separator='')]
+
+    See Also: https://docs.microsoft.com/en-us/previous-versions/windows/desktop/ldap/distinguished-names
+    """
+    restrict_types = [type_.upper() for type_ in (restrict_types or [])]
+    items = dn_utils.parse_dn(dn, escape=True)
+
+    if restrict_types:
+        parts = [RDN(*i) for i in items if i[0].upper() in restrict_types]
+    else:
+        parts = [RDN(*i) for i in items]
+
+    return parts
+
+
+def parse_dn_value_list(dn: str, restrict_types: List[str] = None) -> List[str]:
+    """this method work like parse_dn_tree, be only return values of those attributes, For examples:
+
+    >>> parse_dn_value_list("CN=Jeff Smith,OU=Sales,DC=Fabrikam,DC=COM")
+    ['Jeff Smith', 'Sales', 'Fabrikam', 'COM']
+
+    if provide restrict_types, this method will ignore the attribute not in restrict_types, For examples:
+    >>> parse_dn_value_list("CN=Jeff Smith,OU=Sales,DC=Fabrikam,DC=COM", restrict_types=["DC"])
+    ['Fabrikam', 'COM']
+
+    """
+    tree = parse_dn_tree(dn, restrict_types)
+    parts = []
+    for part in tree:
+        parts.append(part.value)
+    return parts

src/api/bkuser_core/categories/plugins/ldap/client.py

@@ -10,7 +10,7 @@
 """
 import logging
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, Dict
+from typing import TYPE_CHECKING, Dict, List
 
 import ldap3
 from bkuser_core.categories.loader import get_plugin_by_name
@@ -81,7 +81,7 @@ def search(
         force_filter_str: str = "",
         start_root: str = None,
         attributes: list = None,
-    ) -> Dict:
+    ) -> List[Dict]:
         """搜索"""
         if not start_root:
             start_root = self.start_root