Merge pull request #361 from yejianquan/jianquanye/202405-msft-gcu-perf-opt

Backport GCU performance optimizations from PRs #4476 and #4478 to 202405

Cherry-pick of key optimizations from sonic-net/sonic-utilities master:

PR #4476: MD5 hash cache for validate_config_db_config, loadData dedup via shared _loaded_sy across validator and find_ref_paths, eliminate redundant copy.deepcopy in validation path, _validate_replace reorder (validate added_paths first to leverage already-loaded config)
PR #4478: BulkLeafListMoveGenerator - batches N leaf-list REMOVE ops into a single REPLACE move, reducing DFS search space
Performance results on Cisco 8800 VOQ chassis (str3-7800-lc3-1, asic0):
Baseline (stock 202405 + VOQ_QUEUE leafref yang fix): 8m50s
With this patch: 5m59s (32% improvement, 2m51s saved)
Tested with 45-op all-add port provisioning patch (2 ports + full
networking stack: PORT, QUEUE, BUFFER_PG, PFC_WD, PORTCHANNEL, BGP)
Second run confirmed: 5m59.8s (consistent)

Functional verification:

add operations: PASS (45-op patch applied correctly)
replace operations: PASS (MTU change, 7-8s)
remove operations: PASS (DEVICE_NEIGHBOR removal, 6.6s)
End state verified: ports at correct speed, all dependent tables present
Originally merged to master as commits 5d54e44 (#4476) and bfc67f5 (#4478). Adapted for 202405 method signatures and stock sonic-yang-mgmt (no sonic_yang_path.py dependency, no must_size SWIG patch required).

Signed-off-by: jianquanye@microsoft.com

Author: yejianquan

generic_config_updater/gu_common.py

@@ -9,6 +9,7 @@
 import copy
 import re
 import os
+import hashlib
 from sonic_py_common import logger, multi_asic
 from enum import Enum
 
@@ -80,6 +81,9 @@ def __init__(self, yang_dir=YANG_DIR, scope=multi_asic.DEFAULT_NAMESPACE):
         self.scope = scope
         self.yang_dir = YANG_DIR
         self.sonic_yang_with_loaded_models = None
+        self._validate_config_cache = {}
+        self._currently_loaded_hash = None
+        self._loaded_sy = None
 
     def get_config_db_as_json(self):
         return get_config_db_as_json(self.scope)
@@ -138,27 +142,41 @@ def validate_sonic_yang_config(self, sonic_yang_as_json):
             return False, ex
 
     def validate_config_db_config(self, config_db_as_json):
+        _cache_key = hashlib.md5(
+            json.dumps(config_db_as_json, sort_keys=True).encode()
+        ).hexdigest()
+        if _cache_key in self._validate_config_cache:
+            return self._validate_config_cache[_cache_key]
+
         sy = self.create_sonic_yang_with_loaded_models()
 
         # TODO: Move these validators to YANG models
         supplemental_yang_validators = [self.validate_bgp_peer_group,
                                         self.validate_lanes]
 
         try:
-            tmp_config_db_as_json = copy.deepcopy(config_db_as_json)
-
-            sy.loadData(tmp_config_db_as_json)
+            sy.loadData(config_db_as_json)
+            self._currently_loaded_hash = _cache_key
+            self._loaded_sy = sy
 
             sy.validate_data_tree()
 
             for supplemental_yang_validator in supplemental_yang_validators:
                 success, error = supplemental_yang_validator(config_db_as_json)
                 if not success:
-                    return success, error
+                    result = (success, error)
+                    self._validate_config_cache[_cache_key] = result
+                    return result
         except sonic_yang.SonicYangException as ex:
-            return False, ex
+            self._currently_loaded_hash = None
+            self._loaded_sy = None
+            result = (False, ex)
+            self._validate_config_cache[_cache_key] = result
+            return result
 
-        return True, None
+        result = (True, None)
+        self._validate_config_cache[_cache_key] = result
+        return result
 
     def validate_field_operation(self, old_config, target_config):
         """
@@ -522,7 +540,7 @@ def create_xpath(self, tokens):
     def _create_sonic_yang_with_loaded_models(self):
         return self.config_wrapper.create_sonic_yang_with_loaded_models()
 
-    def find_ref_paths(self, path, config):
+    def find_ref_paths(self, path, config, reload_config=True):
         """
         Finds the paths referencing any line under the given 'path' within the given 'config'.
         Example:
@@ -560,22 +578,39 @@ def find_ref_paths(self, path, config):
             /ACL_TABLE/EVERFLOW6/ports/1
         """
         # TODO: Also fetch references by must statement (check similar statements)
-        return self._find_leafref_paths(path, config)
-
-    def _find_leafref_paths(self, path, config):
-        sy = self._create_sonic_yang_with_loaded_models()
-
-        tmp_config = copy.deepcopy(config)
-
-        sy.loadData(tmp_config)
-
-        xpath = self.convert_path_to_xpath(path, config, sy)
+        return self._find_leafref_paths(path, config, reload_config=reload_config)
+
+    def _find_leafref_paths(self, path, config, reload_config=True):
+        if reload_config:
+            _config_hash = hashlib.md5(
+                json.dumps(config, sort_keys=True).encode()
+            ).hexdigest()
+            already_loaded = (
+                self.config_wrapper is not None and
+                self.config_wrapper._currently_loaded_hash == _config_hash and
+                self.config_wrapper._loaded_sy is not None
+            )
+            if not already_loaded:
+                sy = self._create_sonic_yang_with_loaded_models()
+                sy.loadData(config)
+                if self.config_wrapper is not None:
+                    self.config_wrapper._currently_loaded_hash = _config_hash
+                    self.config_wrapper._loaded_sy = sy
+            else:
+                sy = self.config_wrapper._loaded_sy
+        else:
+            sy = self._create_sonic_yang_with_loaded_models()
+            sy.loadData(config)
 
-        leaf_xpaths = self._get_inner_leaf_xpaths(xpath, sy)
+        if not isinstance(path, list):
+            path = [path]
 
         ref_xpaths = []
-        for xpath in leaf_xpaths:
-            ref_xpaths.extend(sy.find_data_dependencies(xpath))
+        for inner_path in path:
+            xpath = self.convert_path_to_xpath(inner_path, config, sy)
+            leaf_xpaths = self._get_inner_leaf_xpaths(xpath, sy)
+            for leaf_xpath in leaf_xpaths:
+                ref_xpaths.extend(sy.find_data_dependencies(leaf_xpath))
 
         ref_paths = []
         ref_paths_set = set()

generic_config_updater/patch_sorter.py

@@ -850,14 +850,14 @@ def _validate_replace(self, move, diff):
         simulated_config = move.apply(diff.current_config)
         deleted_paths, added_paths = self._get_paths(diff.current_config, simulated_config, [])
 
-        # For deleted paths, we check the current config has no dependencies between nodes under the removed path
-        if not self._validate_paths_config(deleted_paths, diff.current_config):
-            return False
-
         # For added paths, we check the simulated config has no dependencies between nodes under the added path
         if not self._validate_paths_config(added_paths, simulated_config):
             return False
 
+        # For deleted paths, we check the current config has no dependencies between nodes under the removed path
+        if not self._validate_paths_config(deleted_paths, diff.current_config):
+            return False
+
         return True
 
     def _get_paths(self, current_ptr, target_ptr, tokens):
@@ -935,10 +935,7 @@ def _validate_paths_config(self, paths, config):
         return True
 
     def _find_ref_paths(self, paths, config):
-        refs = []
-        for path in paths:
-            refs.extend(self.path_addressing.find_ref_paths(path, config))
-        return refs
+        return self.path_addressing.find_ref_paths(paths, config)
 
 class NoEmptyTableMoveValidator:
     """
@@ -1055,6 +1052,43 @@ def _get_non_existing_tables_tokens(self, config1, config2):
             if not(table in config2):
                 yield [table]
 
+
+class BulkLeafListMoveGenerator:
+    """
+    A class that generates bulk REPLACE moves for leaf-lists (lists of primitive
+    values) that differ between current and target configs.
+    """
+    def generate(self, diff):
+        for move in self._traverse(diff, diff.current_config, diff.target_config, []):
+            yield move
+
+    def _traverse(self, diff, current_ptr, target_ptr, tokens):
+        if not isinstance(current_ptr, dict) or not isinstance(target_ptr, dict):
+            return
+
+        for key in current_ptr:
+            if key not in target_ptr:
+                continue
+
+            current_val = current_ptr[key]
+            target_val = target_ptr[key]
+            tokens.append(key)
+
+            if isinstance(current_val, list) and isinstance(target_val, list):
+                if (current_val != target_val and
+                        self._is_leaf_list(current_val) and
+                        self._is_leaf_list(target_val)):
+                    yield JsonMove(diff, OperationType.REPLACE, list(tokens), list(tokens))
+            elif isinstance(current_val, dict) and isinstance(target_val, dict):
+                for move in self._traverse(diff, current_val, target_val, tokens):
+                    yield move
+
+            tokens.pop()
+
+    @staticmethod
+    def _is_leaf_list(lst):
+        return all(isinstance(item, (str, int, float, bool)) for item in lst)
+
 class KeyLevelMoveGenerator:
     """
     A class that key level moves. The item name at the root level of ConfigDB is called 'Table', the item
@@ -1652,7 +1686,8 @@ def create(self, algorithm=Algorithm.DFS):
         move_generators = [RemoveCreateOnlyDependencyMoveGenerator(self.path_addressing),
                            LowLevelMoveGenerator(self.path_addressing)]
         # TODO: Enable TableLevelMoveGenerator once it is confirmed whole table can be updated at the same time
-        move_non_extendable_generators = [KeyLevelMoveGenerator()]
+        move_non_extendable_generators = [BulkLeafListMoveGenerator(),
+                                          KeyLevelMoveGenerator()]
         move_extenders = [RequiredValueMoveExtender(self.path_addressing, self.operation_wrapper),
                           UpperLevelMoveExtender(),
                           DeleteInsteadOfReplaceMoveExtender(),