feature: PaaS容器部署适配 (closed #19)

.gitignore

@@ -96,6 +96,8 @@ celerybeat-schedule
 env/
 venv/
 ENV/
+!env/
+
 
 # Spyder project settings
 .spyderproject

app_desc.yaml

@@ -0,0 +1,65 @@
+spec_version: 2
+app_version: "{{APP_VERSION}}"
+app:
+    region: default
+    bk_app_code: "bk_gsekit"
+    bk_app_name: 进程配置管理
+    market:
+        category: 运维工具
+        introduction: 进程配置管理
+        display_options:
+            width: 1300
+            height: 720
+            is_win_maximize: True
+            open_mode: "new_tab"
+modules:
+    default:
+        is_default: True
+        source_dir: src
+        language: Python
+        services:
+            - name: mysql
+            - name: redis
+            - name: rabbitmq
+        env_variables:
+            - key: BKAPP_IS_V3_CONTAINER
+              value: "True"
+              description: 是否部署在V3PaaS容器
+            - key: PIP_VERSION
+              value: "20.2.3"
+              description: 固化pip版本
+
+        svc_discovery:
+            bk_saas:
+                - bk_app_code: "bk_iam"
+                - bk_app_code: "bk_gsekit"
+
+        processes:
+            web:
+                command: gunicorn --timeout 600 --max-requests 200 --max-requests-jitter 20 wsgi -w 8 -b :$PORT --access-logfile - --error-logfile - --access-logformat '[%(h)s] %({request_id}i)s %(u)s %(t)s "%(r)s" %(s)s %(D)s %(b)s "%(f)s" "%(a)s"'
+                plan: 4C2G5R
+                replicas: 5
+            dworker:
+                command: celery -A blueapps.core.celery worker -Q default -n default@%h -c 5 -l info --maxtasksperchild=50
+                plan: 4C2G5R
+                replicas: 5
+            cworker:
+                command: celery -A blueapps.core.celery worker -Q pipeline_additional_task,pipeline_additional_task_priority -n common_worker@%h -c 5 -l info --maxtasksperchild=50
+                plan: 4C2G5R
+                replicas: 5
+            ereworker:
+                command: celery -A blueapps.core.celery worker -Q er_execute -n ri_worker@%h -l info -c 10 -l info --maxtasksperchild=100
+                plan: 4C2G5R
+                replicas: 5
+            ersworker:
+                command: celery -A blueapps.core.celery worker -Q er_schedule -n ri_worker@%h -l info -c 10 -l info --maxtasksperchild=200
+                plan: 4C2G5R
+                replicas: 5
+            beat:
+                command: celery -A blueapps.core.celery beat -l info
+                plan: 4C2G5R
+                replicas: 1
+            pwatch:
+                command: python manage.py watch_process
+                plan: 4C2G5R
+                replicas: 1

apps/api/domains.py

@@ -13,7 +13,7 @@
 
 ESB_PREFIX = "/api/c/compapi/v2/"
 
-APIGATEWAY_ROOT_PREFIX_FORMAT = "{}{}{{}}/".format(settings.BK_PAAS_INNER_HOST, ESB_PREFIX)
+APIGATEWAY_ROOT_PREFIX_FORMAT = "{}{}{{}}/".format(settings.BK_COMPONENT_API_URL, ESB_PREFIX)
 
 CC_APIGATEWAY_ROOT = os.getenv("BKAPP_CC_APIGATEWAY_ROOT") or APIGATEWAY_ROOT_PREFIX_FORMAT.format("cc")
 JOB_APIGATEWAY_ROOT_V3 = os.getenv("BKAPP_JOB_APIGATEWAY_ROOT") or APIGATEWAY_ROOT_PREFIX_FORMAT.format("jobv3")

apps/exceptions.py

@@ -68,7 +68,7 @@ def __init__(self, context=None, data=None, **kwargs):
             if kwargs.get("permission"):
                 permission = kwargs.get("permission")
                 client = IAM(
-                    settings.APP_ID, settings.APP_TOKEN, settings.BK_IAM_INNER_HOST, settings.BK_PAAS_INNER_HOST
+                    settings.APP_ID, settings.APP_TOKEN, settings.BK_IAM_INNER_HOST, settings.BK_COMPONENT_API_URL
                 )
                 for action in permission.get("actions") or []:
                     action["name"] = action.get("name") or action.get("id")

apps/iam/handlers/permission.py

@@ -56,9 +56,10 @@ def __init__(self, username: str = "", request=None):
     @classmethod
     def get_iam_client(cls):
         if settings.BK_IAM_SKIP:
-            return DummyIAM(settings.APP_ID, settings.APP_TOKEN,
-                            settings.BK_IAM_INNER_HOST, settings.BK_PAAS_INNER_HOST)
-        return IAM(settings.APP_ID, settings.APP_TOKEN, settings.BK_IAM_INNER_HOST, settings.BK_PAAS_INNER_HOST)
+            return DummyIAM(
+                settings.APP_ID, settings.APP_TOKEN, settings.BK_IAM_INNER_HOST, settings.BK_COMPONENT_API_URL
+            )
+        return IAM(settings.APP_ID, settings.APP_TOKEN, settings.BK_IAM_INNER_HOST, settings.BK_COMPONENT_API_URL)
 
     def make_request(self, action: Union[ActionMeta, str], resources: List[Resource] = None) -> Request:
         """
@@ -125,7 +126,9 @@ def _make_application(
 
                     related_resources.append(
                         RelatedResourceType(
-                            system_id=related_resource.system_id, type=related_resource.id, instances=instances,
+                            system_id=related_resource.system_id,
+                            type=related_resource.id,
+                            instances=instances,
                         )
                     )
 
@@ -207,7 +210,9 @@ def is_allowed(
         if not result and raise_exception:
             apply_data, apply_url = self.get_apply_data([action], resources)
             raise PermissionDeniedError(
-                action_name=action.name, apply_url=apply_url, permission=apply_data,
+                action_name=action.name,
+                apply_url=apply_url,
+                permission=apply_data,
             )
 
         return result

apps/utils/cache.py

@@ -8,7 +8,31 @@
 an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and limitations under the License.
 """
-from dogpile.cache import make_region
 
-cache_dict = {}
-region = make_region().configure("dogpile.cache.memory", arguments={"cache_dict": cache_dict})
+from functools import wraps
+from typing import Callable, Optional
+
+
+def class_member_cache(name: Optional[str] = None):
+    """
+    类成员缓存
+    :param name: 缓存名称，为空则使用 class_func.__name__
+    :return:
+    """
+
+    def class_member_cache_inner(class_func: Callable) -> Callable:
+        @wraps(class_func)
+        def wrapper(self, *args, **kwargs):
+
+            cache_field = f"_{name or class_func.__name__}"
+
+            cache_member = getattr(self, cache_field, None)
+            if cache_member:
+                return cache_member
+            cache_member = class_func(self, *args, **kwargs)
+            setattr(self, cache_field, cache_member)
+            return cache_member
+
+        return wrapper
+
+    return class_member_cache_inner

apps/utils/enum.py

@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+"""
+Tencent is pleased to support the open source community by making 蓝鲸 (Blueking) available.
+Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and limitations under the License.
+"""
+import abc
+from enum import Enum
+from typing import Any, Dict, List, Tuple
+
+from ..utils.cache import class_member_cache
+
+
+class EnhanceEnum(Enum):
+    """增强枚举类，提供常用的枚举值列举方法"""
+
+    @classmethod
+    @abc.abstractmethod
+    def _get_member__alias_map(cls) -> Dict[Enum, str]:
+        """
+        获取枚举成员与释义的映射关系
+        :return:
+        """
+        raise NotImplementedError
+
+    @classmethod
+    @class_member_cache()
+    def list_member_values(cls) -> List[Any]:
+        """
+        获取所有的枚举成员值
+        :return:
+        """
+        member_values = []
+        for member in cls._member_names_:
+            member_values.append(cls._value2member_map_[member].value)
+        return member_values
+
+    @classmethod
+    @class_member_cache()
+    def get_member_value__alias_map(cls) -> Dict[Any, str]:
+        """
+        获取枚举成员值与释义的映射关系，缓存计算结果
+        :return:
+        """
+        member_value__alias_map = {}
+        member__alias_map = cls._get_member__alias_map()
+
+        for member, alias in member__alias_map.items():
+            if type(member) is not cls:
+                raise ValueError(f"except member type -> {cls}, but got -> {type(member)}")
+            member_value__alias_map[member.value] = alias
+
+        return member_value__alias_map
+
+    @classmethod
+    @class_member_cache()
+    def list_choices(cls) -> List[Tuple[Any, Any]]:
+        """
+        获取可选项列表，一般用于序列化器、model的choices选项
+        :return:
+        """
+        return list(cls.get_member_value__alias_map().items())

apps/utils/env.py

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+"""
+Tencent is pleased to support the open source community by making 蓝鲸 (Blueking) available.
+Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and limitations under the License.
+"""
+
+import os
+from typing import Any
+
+from apps.utils.string import str2bool
+
+
+def get_type_env(key: str, default: Any = None, _type: type = str, exempt_empty_str: bool = False) -> Any:
+    """
+    获取环境变量并转为目标类型
+    :param key: 变量名
+    :param default: 默认值，若获取不到环境变量会默认使用该值
+    :param _type: 环境变量需要转换的类型，不会转 default
+    :param exempt_empty_str: 是否豁免空串
+    :return:
+    """
+    value = os.getenv(key) or default
+    if value == default:
+        return value
+
+    # 豁免空串
+    if isinstance(value, str) and not value and exempt_empty_str:
+        return value
+
+    if _type == bool:
+        return str2bool(value)
+
+    try:
+        value = _type(value)
+    except TypeError:
+        raise TypeError(f"can not convert env value -> {value} to type -> {_type}")
+
+    return value

apps/utils/string.py

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+"""
+Tencent is pleased to support the open source community by making 蓝鲸 (Blueking) available.
+Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and limitations under the License.
+"""
+
+from typing import Optional
+
+
+def str2bool(string: Optional[str], strict: bool = True) -> bool:
+    """
+    字符串转布尔值
+    对于bool(str) 仅在len(str) == 0 or str is None 的情况下为False，为了适配bool("False") 等环境变量取值情况，定义该函数
+    参考：https://stackoverflow.com/questions/21732123/convert-true-false-value-read-from-file-to-boolean
+    :param string:
+    :param strict: 严格校验，非 False / True / false / true  时抛出异常，用于环境变量的转换
+    :return:
+    """
+    if string in ["False", "false"]:
+        return False
+    if string in ["True", "true"]:
+        return True
+
+    if strict:
+        raise ValueError(f"{string} can not convert to bool")
+    return bool(string)

blueking/component/conf.py

@@ -8,7 +8,7 @@
 
     APP_CODE = settings.APP_ID
     SECRET_KEY = settings.APP_TOKEN
-    COMPONENT_SYSTEM_HOST = getattr(settings, "BK_PAAS_INNER_HOST", settings.BK_PAAS_HOST)
+    COMPONENT_SYSTEM_HOST = settings.BK_COMPONENT_API_URL
     DEFAULT_BK_API_VER = getattr(settings, "DEFAULT_BK_API_VER", "v2")
 except Exception:
     APP_CODE = ""

config/__init__.py

@@ -13,7 +13,7 @@
 
 __author__ = "蓝鲸智云"
 __copyright__ = "Copyright © 2012-2019 Tencent BlueKing. All Rights Reserved."
-__all__ = ["celery_app", "ENVIRONMENT", "RUN_VER", "BK_URL", "BASE_DIR"]
+__all__ = ["celery_app", "RUN_VER", "BK_URL", "BASE_DIR"]
 
 # This will make sure the app is always imported when
 # Django starts so that shared_task will use this app.
@@ -32,16 +32,11 @@ def get_env_or_raise(key):
     return value
 
 
-# V3判断环境的环境变量为BKPAAS_ENVIRONMENT
-if "BKPAAS_ENVIRONMENT" in os.environ:
-    ENVIRONMENT = os.getenv("BKPAAS_ENVIRONMENT", "dev")
-# V2判断环境的环境变量为BK_ENV
-else:
-    PAAS_V2_ENVIRONMENT = os.environ.get("BK_ENV", "development")
-    ENVIRONMENT = {"development": "dev", "testing": "stag", "production": "prod"}.get(PAAS_V2_ENVIRONMENT)
-
 # SaaS运行版本，如非必要请勿修改
 RUN_VER = os.environ.get("BKPAAS_ENGINE_REGION", "open")
+# 兼容 V3 取值差异
+if RUN_VER == "default":
+    RUN_VER = "open"
 
 APP_ID = APP_CODE = os.environ.get("APP_ID", "bk_gsekit")
 APP_TOKEN = SECRET_KEY = os.environ.get("APP_TOKEN", "28b7b410-c7b7-4537-9a65-8ce55738170e")