feature: 同步 CMDB 主机 cpu 架构及操作系统信息(closed TencentBlueKing#1454)

# Conflicts:
#	apps/node_man/models.py

Author: neko12583

apps/node_man/constants.py

@@ -117,6 +117,12 @@ def get_optional_items(cls) -> List[str]:
 OsType = choices_to_namedtuple(OS_CHOICES)
 OS_CHN = {os_type: os_type if os_type == OsType.AIX else os_type.capitalize() for os_type in OS_TUPLE}
 BK_OS_TYPE = {"LINUX": "1", "WINDOWS": "2", "AIX": "3", "SOLARIS": "5"}
+# 操作系统匹配关键词
+OS_KEYWORDS = {
+    OsType.LINUX: ["linux", "ubuntu", "centos", "redhat", "suse", "debian", "fedora"],
+    OsType.WINDOWS: ["windows", "xserver"],
+    OsType.AIX: ["aix"]
+}
 
 # 操作系统->系统账户映射表
 ACCOUNT_MAP = {
@@ -477,6 +483,10 @@ def _get_member__alias_map(cls) -> Dict[Enum, str]:
     OsType.AIX: CpuType.powerpc,
     OsType.SOLARIS: CpuType.sparc,
 }
+CMDB_CPU_MAP = {
+    "x86": CpuType.x86,
+    "arm": CpuType.aarch64
+}
 
 PACKAGE_PATH_RE = re.compile(
     f"(?P<is_external>external_)?plugins_(?P<os>({'|'.join(map(str, PLUGIN_OS_TUPLE))}))"
@@ -622,6 +632,7 @@ class BkappRunEnvType(Enum):
     "bk_state",
     "bk_state_name",
     "bk_supplier_account",
+    "bk_cpu_architecture",
 ]
 
 # 限流窗口配置，用于控制CMDB订阅触发的变更频率

apps/node_man/models.py

@@ -114,6 +114,15 @@ class KeyEnum(Enum):
         ENABLE_PUSH_HOST_IDENTIFIER = "ENABLE_PUSH_HOST_IDENTIFIER"
         # 插件配置公共常量支持 issue: https://github.com/TencentBlueKing/bk-nodeman/issues/1500
         PLUGIN_COMMON_CONSTANTS = "PLUGIN_COMMON_CONSTANTS"
+        # GSE 2.0 灰度列表
+        GSE2_GRAY_SCOPE_LIST = "GSE2_GRAY_SCOPE_LIST"
+        # 禁用订阅业务列表
+        DISABLE_SUBSCRIPTION_SCOPE_LIST = "DISABLE_SUBSCRIPTION_SCOPE_LIST"
+        GSE2_GRAY_AP_MAP = "GSE2_GRAY_AP_MAP"
+        # 是否同步 cc 的CPU架构
+        SYNC_CMDB_HOST_APPLY_CPU_ARCH = "SYNC_CMDB_HOST_APPLY_CPU_ARCH"
+        # 同步 cc 主机操作系统时，是否 bk_os_type 匹配优先与 bk_os_name
+        SYNC_CMDB_HOST_OS_TYPE_PRIORITY = "SYNC_CMDB_HOST_OS_TYPE_PRIORITY"
 
     key = models.CharField(_("键"), max_length=255, db_index=True, primary_key=True)
     v_json = JSONField(_("值"))

apps/node_man/periodic_tasks/sync_cmdb_host.py

@@ -99,6 +99,7 @@ def _list_resource_pool_hosts(start):
 def _bulk_update_host(hosts, extra_fields):
     update_fields = [
         "node_type",
+        "bk_biz_id",
         "bk_cloud_id",
         "bk_host_name",
         "bk_addressing",
@@ -107,14 +108,16 @@ def _bulk_update_host(hosts, extra_fields):
         "inner_ipv6",
         "outer_ipv6",
         "bk_agent_id",
+        "os_type",
     ] + extra_fields
     if hosts:
         models.Host.objects.bulk_update(hosts, fields=update_fields)
 
 
-def _generate_host(biz_id, host, ap_id):
-    os_type = tools.HostV2Tools.get_os_type(host)
-    cpu_arch = tools.HostV2Tools.get_cpu_arch(host)
+def _generate_host(biz_id, host, ap_id, is_os_type_priority=False, is_sync_cmdb_host_apply_cpu_arch=False):
+    os_type = tools.HostV2Tools.get_os_type(host, is_os_type_priority)
+    cpu_arch = tools.HostV2Tools.get_cpu_arch(host, is_sync_cmdb_host_apply_cpu_arch, os_type=os_type)
+
     host_data = models.Host(
         bk_host_id=host["bk_host_id"],
         bk_agent_id=host.get("bk_agent_id"),
@@ -192,16 +195,16 @@ def update_or_create_host_base(biz_id, task_id, cmdb_host_data):
         ).values_list("bk_host_id", flat=True)
     )
 
-    need_delete_host_ids: typing.Set[int] = set()
-    need_create_host_without_biz: typing.List[typing.Dict] = []
+    is_sync_cmdb_host_apply_cpu_arch = tools.HostV2Tools.is_sync_cmdb_host_apply_cpu_arch()
+    is_os_type_priority = tools.HostV2Tools.is_os_type_priority()
+
     need_update_hosts: typing.List[models.Host] = []
-    need_update_hosts_without_biz: typing.List[models.Host] = []
-    need_update_hosts_without_os: typing.List[models.Host] = []
-    need_update_hosts_without_biz_os: typing.List[models.Host] = []
+    need_update_hosts_with_arch: typing.List[models.Host] = []
+
     need_create_hosts: typing.List[models.Host] = []
-    need_update_host_identity_objs: typing.List[models.IdentityData] = []
     need_create_host_identity_objs: typing.List[models.IdentityData] = []
     need_create_process_status_objs: typing.List[models.ProcessStatus] = []
+    need_update_host_identity_objs: typing.List[models.IdentityData] = []
 
     ap_id = constants.DEFAULT_AP_ID if models.AccessPoint.objects.count() > 1 else models.AccessPoint.objects.first().id
 
@@ -218,6 +221,7 @@ def update_or_create_host_base(biz_id, task_id, cmdb_host_data):
         host_params = {
             "bk_host_id": host["bk_host_id"],
             "bk_agent_id": host.get("bk_agent_id"),
+            "bk_biz_id": biz_id,
             "bk_cloud_id": host["bk_cloud_id"],
             "bk_host_name": host.get("bk_host_name"),
             "bk_addressing": host.get("bk_addressing") or constants.CmdbAddressingType.STATIC.value,
@@ -226,74 +230,47 @@ def update_or_create_host_base(biz_id, task_id, cmdb_host_data):
             "inner_ipv6": (host.get("bk_host_innerip_v6") or "").split(",")[0],
             "outer_ipv6": (host.get("bk_host_outerip_v6") or "").split(",")[0],
         }
-        if host["bk_host_id"] in exist_agent_host_ids:
 
-            os_type = tools.HostV2Tools.get_os_type(host)
+        if host["bk_host_id"] in exist_agent_host_ids:
+            host_params["os_type"] = tools.HostV2Tools.get_os_type(host, is_os_type_priority)
             host_params["node_type"] = (constants.NodeType.PAGENT, constants.NodeType.AGENT)[
                 host["bk_cloud_id"] == constants.DEFAULT_CLOUD
             ]
-
-            if os_type and biz_id:
-                host_params["bk_biz_id"] = biz_id
-                host_params["os_type"] = os_type
-                need_update_hosts.append(models.Host(**host_params))
-            elif biz_id:
-                host_params["bk_biz_id"] = biz_id
-                need_update_hosts_without_os.append(models.Host(**host_params))
-            elif os_type:
-                host_params["os_type"] = os_type
-                need_update_hosts_without_biz.append(models.Host(**host_params))
-            else:
-                need_update_hosts_without_biz_os.append(models.Host(**host_params))
         elif host["bk_host_id"] in exist_proxy_host_ids:
             host_params["os_type"] = constants.OsType.LINUX
             host_params["node_type"] = constants.NodeType.PROXY
-            if biz_id:
-                host_params["bk_biz_id"] = biz_id
-                need_update_hosts.append(models.Host(**host_params))
-            else:
-                need_update_hosts_without_biz.append(models.Host(**host_params))
         else:
-            # 不是agent不是proxy的主机需要创建
-            if not biz_id:
-                need_create_host_without_biz.append(host)
+            host_data, identify_data, process_status_data = _generate_host(
+                biz_id,
+                host,
+                ap_id,
+                is_os_type_priority,
+                is_sync_cmdb_host_apply_cpu_arch,
+            )
+            need_create_hosts.append(host_data)
+            if identify_data.bk_host_id not in host_ids_in_exist_identity_data:
+                need_create_host_identity_objs.append(identify_data)
             else:
-                host_data, identify_data, process_status_data = _generate_host(biz_id, host, ap_id)
-                need_create_hosts.append(host_data)
-                if identify_data.bk_host_id not in host_ids_in_exist_identity_data:
-                    need_create_host_identity_objs.append(identify_data)
-                else:
-                    need_update_host_identity_objs.append(identify_data)
-                if process_status_data.bk_host_id not in host_ids_in_exist_proc_statuses:
-                    need_create_process_status_objs.append(process_status_data)
-
-    if need_create_host_without_biz:
-        # 查询业务主机数据进行创建
-        find_host_biz_ids = [_host["bk_host_id"] for _host in need_create_host_without_biz]
-        host_biz_relation = find_host_biz_relations(find_host_biz_ids)
-        # 查询不到业务需要删除
-        need_delete_host_ids = set(find_host_biz_ids) - set(host_biz_relation.keys())
-
-        for need_create_host in need_create_host_without_biz:
-            if need_create_host["bk_host_id"] not in need_delete_host_ids:
-                host_data, identify_data, process_status_data = _generate_host(
-                    host_biz_relation[need_create_host["bk_host_id"]],
-                    need_create_host,
-                    ap_id,
-                )
-                need_create_hosts.append(host_data)
-                if identify_data.bk_host_id not in host_ids_in_exist_identity_data:
-                    need_create_host_identity_objs.append(identify_data)
-                else:
-                    need_update_host_identity_objs.append(identify_data)
-                if process_status_data.bk_host_id not in host_ids_in_exist_proc_statuses:
-                    need_create_process_status_objs.append(process_status_data)
+                need_update_host_identity_objs.append(identify_data)
+            if process_status_data.bk_host_id not in host_ids_in_exist_proc_statuses:
+                need_create_process_status_objs.append(process_status_data)
+            continue
+
+        cpu_arch = tools.HostV2Tools.get_cpu_arch(
+            host,
+            is_sync_cmdb_host_apply_cpu_arch,
+            get_default=False,
+        )
+        if is_sync_cmdb_host_apply_cpu_arch and cpu_arch:
+            host_params["cpu_arch"] = cpu_arch
+            need_update_hosts_with_arch.append(models.Host(**host_params))
+            continue
+
+        need_update_hosts.append(models.Host(**host_params))
 
     with transaction.atomic():
-        _bulk_update_host(need_update_hosts, ["bk_biz_id", "os_type"])
-        _bulk_update_host(need_update_hosts_without_biz, ["os_type"])
-        _bulk_update_host(need_update_hosts_without_os, ["bk_biz_id"])
-        _bulk_update_host(need_update_hosts_without_biz_os, [])
+        _bulk_update_host(need_update_hosts, [])
+        _bulk_update_host(need_update_hosts_with_arch, ["cpu_arch"])
 
         if need_create_hosts:
             models.Host.objects.bulk_create(need_create_hosts, batch_size=500)
@@ -306,7 +283,7 @@ def update_or_create_host_base(biz_id, task_id, cmdb_host_data):
         if need_create_process_status_objs:
             models.ProcessStatus.objects.bulk_create(need_create_process_status_objs, batch_size=500)
 
-    return bk_host_ids, list(need_delete_host_ids)
+    return bk_host_ids
 
 
 def sync_biz_incremental_hosts(bk_biz_id: int, expected_bk_host_ids: typing.Iterable[int]):
@@ -362,7 +339,7 @@ def _update_or_create_host(biz_id, start=0, task_id=None):
         f"host_count -> {host_count}, range -> {start}-{start + constants.QUERY_CMDB_LIMIT}"
     )
 
-    bk_host_ids, _ = update_or_create_host_base(biz_id, task_id, host_data)
+    bk_host_ids = update_or_create_host_base(biz_id, task_id, host_data)
 
     # 递归
     if host_count > start + constants.QUERY_CMDB_LIMIT:

apps/node_man/tools/host_v2.py

@@ -17,39 +17,71 @@
 
 class HostV2Tools:
     @classmethod
-    def get_os_type(cls, host: Dict) -> str:
-        """根据CC的主机属性，得到可能的操作系统"""
-        bk_os_name = host.get("bk_os_name") or "unknown"
-        bk_os_type = host.get("bk_os_type")
-        os_name = bk_os_name.lower()
+    def get_os_type_by_os_type(cls, bk_os_type: str) -> Optional[str]:
+        """根据 os_type 获取主机操作系统"""
+        return constants.OS_TYPE.get(bk_os_type)
+
+    @classmethod
+    def get_os_type_by_os_name(cls, bk_os_name: str) -> Optional[str]:
+        """根据 os_name 获取主机操作系统"""
+        bk_os_name = bk_os_name.lower()
+
+        for os_type, keywords in constants.OS_KEYWORDS.items():
+            for keyword in keywords:
+                if keyword in bk_os_name:
+                    return os_type
 
-        linux_keywords = ["linux", "ubuntu", "centos", "redhat", "suse", "debian", "fedora"]
-        windows_keywords = ["windows", "xserver"]
-        aix_keywords = ["aix"]
+        return None
 
-        for linux_keyword in linux_keywords:
-            if linux_keyword in os_name:
-                return constants.OsType.LINUX
+    @classmethod
+    def is_os_type_priority(cls) -> bool:
+        return bool(
+            models.GlobalSettings.get_config(models.GlobalSettings.KeyEnum.SYNC_CMDB_HOST_OS_TYPE_PRIORITY.value)
+        )
 
-        for windows_keyword in windows_keywords:
-            if windows_keyword in os_name:
-                return constants.OsType.WINDOWS
+    @classmethod
+    def get_os_type(cls, host: Dict, is_os_type_priority: bool = False) -> str:
+        """根据CC的主机属性，得到可能的操作系统"""
+        bk_os_name = host.get("bk_os_name") or "unknown"
+        bk_os_type = host.get("bk_os_type")
 
-        for aix_keyword in aix_keywords:
-            if aix_keyword in os_name:
-                return constants.OsType.AIX
+        os_type = cls.get_os_type_by_os_name(bk_os_name)
 
-        if bk_os_type in constants.OS_TYPE:
-            return constants.OS_TYPE[bk_os_type]
+        if is_os_type_priority:
+            os_type = cls.get_os_type_by_os_type(bk_os_type) or os_type
+        else:
+            os_type = os_type or cls.get_os_type_by_os_type(bk_os_type)
 
         # 若CMDB中区分不出操作系统，则默认是LINUX
-        return constants.OsType.LINUX
+        return os_type or constants.OsType.LINUX
 
     @classmethod
-    def get_cpu_arch(cls, host: Dict) -> str:
-        os_type = cls.get_os_type(host)
-        # 暂时通过操作系统区分CPU架构，后续通过CMDB的CPU架构字段区分
-        return constants.DEFAULT_OS_CPU_MAP.get(os_type, constants.CpuType.x86_64)
+    def is_sync_cmdb_host_apply_cpu_arch(cls) -> bool:
+        sync_cmdb_host_apply_cpu_arch = models.GlobalSettings.get_config(
+            models.GlobalSettings.KeyEnum.SYNC_CMDB_HOST_APPLY_CPU_ARCH.value
+        )
+        return bool(sync_cmdb_host_apply_cpu_arch)
+
+    @classmethod
+    def get_cpu_arch(
+        cls,
+        host: Dict,
+        is_sync_cmdb_host_apply_cpu_arch: bool = False,
+        get_default: bool = True,
+        os_type: str = constants.OsType.LINUX
+    ) -> Optional[str]:
+        if is_sync_cmdb_host_apply_cpu_arch:
+            cpu_arch = constants.CMDB_CPU_MAP.get(host.get("bk_cpu_architecture"))
+            if cpu_arch == constants.CpuType.x86 and host.get("bk_os_bit"):
+                bk_os_bit = host.get("bk_os_bit")
+                bit_suffix = "" if "32" in bk_os_bit else "_64"
+                return cpu_arch + bit_suffix
+            elif cpu_arch:
+                return cpu_arch
+        if get_default:
+            return constants.DEFAULT_OS_CPU_MAP.get(os_type, constants.CpuType.x86_64)
+
+        return None
 
     @classmethod
     def list_scope_host_ids(cls, scope: Dict) -> List[int]: