feature: adjust helpindex logic to draw from overlay for extensions

Author: DanielMicrosoft

src/azure-cli-core/azure/cli/core/__init__.py

@@ -73,7 +73,7 @@ def __init__(self, **kwargs):
             register_ids_argument, register_global_subscription_argument, register_global_policy_argument)
         from azure.cli.core.cloud import get_active_cloud
         from azure.cli.core.commands.transform import register_global_transforms
-        from azure.cli.core._session import ACCOUNT, CONFIG, SESSION, INDEX, EXTENSION_INDEX, HELP_INDEX, VERSIONS
+        from azure.cli.core._session import ACCOUNT, CONFIG, SESSION, INDEX, EXTENSION_INDEX, HELP_INDEX, EXTENSION_HELP_INDEX, VERSIONS
         from azure.cli.core.util import handle_version_update
 
         from knack.util import ensure_dir
@@ -92,6 +92,7 @@ def __init__(self, **kwargs):
         INDEX.load(os.path.join(azure_folder, 'commandIndex.json'))
         EXTENSION_INDEX.load(os.path.join(azure_folder, 'extensionIndex.json'))
         HELP_INDEX.load(os.path.join(azure_folder, 'helpIndex.json'))
+        EXTENSION_HELP_INDEX.load(os.path.join(azure_folder, 'extensionHelpIndex.json'))
         VERSIONS.load(os.path.join(azure_folder, 'versionCheck.json'))
         handle_version_update()
 
@@ -744,10 +745,11 @@ def __init__(self, cli_ctx=None):
 
         :param cli_ctx: Only needed when `get` or `update` is called.
         """
-        from azure.cli.core._session import INDEX, EXTENSION_INDEX, HELP_INDEX
+        from azure.cli.core._session import INDEX, EXTENSION_INDEX, HELP_INDEX, EXTENSION_HELP_INDEX
         self.INDEX = INDEX
         self.EXTENSION_INDEX = EXTENSION_INDEX
         self.HELP_INDEX = HELP_INDEX
+        self.EXTENSION_HELP_INDEX = EXTENSION_HELP_INDEX
         if cli_ctx:
             self.version = __version__
             self.cloud_profile = cli_ctx.cloud.profile
@@ -782,6 +784,13 @@ def _is_extension_index_valid(self):
         return (index_version and index_version == self.version and
                 cloud_profile and cloud_profile == self.cloud_profile)
 
+    def _is_extension_help_index_valid(self):
+        """Check if the extension help index version and cloud profile are valid."""
+        index_version = self.EXTENSION_HELP_INDEX.get(self._COMMAND_INDEX_VERSION)
+        cloud_profile = self.EXTENSION_HELP_INDEX.get(self._COMMAND_INDEX_CLOUD_PROFILE)
+        return (index_version and index_version == self.version and
+                cloud_profile and cloud_profile == self.cloud_profile)
+
     def _get_top_level_completion_commands(self, index=None):
         """Get top-level command names for tab completion optimization.
 
@@ -918,6 +927,34 @@ def _blend_command_indices(core_index, extension_index):
                     blended[cmd].append(mod)
         return blended
 
+    @staticmethod
+    def _blend_help_indices(base_help_index, extension_help_index):
+        """Blend packaged core help with extension-only help overlay."""
+        blended = {
+            'groups': dict((base_help_index or {}).get('groups') or {}),
+            'commands': dict((base_help_index or {}).get('commands') or {})
+        }
+        ext_help_index = extension_help_index or {}
+        for section in ('groups', 'commands'):
+            blended_section = blended[section]
+            for key, value in (ext_help_index.get(section) or {}).items():
+                blended_section[key] = value
+        return blended
+
+    @staticmethod
+    def _build_extension_help_overlay(base_help_index, full_help_index):
+        """Build extension-only help overlay by diffing full help against packaged core help."""
+        overlay = {'groups': {}, 'commands': {}}
+        base_help_index = base_help_index or {}
+        full_help_index = full_help_index or {}
+        for section in ('groups', 'commands'):
+            base_section = base_help_index.get(section) or {}
+            full_section = full_help_index.get(section) or {}
+            for key, value in full_section.items():
+                if key not in base_section or base_section[key] != value:
+                    overlay[section][key] = value
+        return overlay
+
     def _get_blended_latest_index(self):
         """Get effective index for latest profile by blending core and extension indices."""
         if self.cloud_profile != 'latest':
@@ -1095,20 +1132,39 @@ def get_help_index(self):
         :return: Dictionary mapping top-level commands to their short summaries, or None if not available
         """
         if self.cloud_profile == 'latest':
-            # Prefer local cache if available and valid, as it may include extension-specific help entries.
-            if self._is_index_valid():
-                help_index = self.HELP_INDEX.get(self._HELP_INDEX, {})
-                if not help_index:
-                    help_index = self._migrate_legacy_help_index() or {}
-                if help_index:
-                    logger.debug("Using cached local help index with %d entries", len(help_index))
-                    return help_index
-
+            # Packaged help is the base for latest profile.
             packaged_help_index = self._load_packaged_help_index()
-            if packaged_help_index:
-                logger.debug("Using packaged help index with %d entries", len(packaged_help_index))
-                return packaged_help_index
-            return None
+            if not packaged_help_index:
+                # Defensive fallback to local cache if packaged asset is unavailable.
+                if self._is_index_valid():
+                    help_index = self.HELP_INDEX.get(self._HELP_INDEX, {})
+                    if not help_index:
+                        help_index = self._migrate_legacy_help_index() or {}
+                    if help_index:
+                        logger.debug("Using cached local help index with %d entries", len(help_index))
+                        return help_index
+                return None
+
+            if self._is_extension_help_index_valid():
+                extension_help_index = self.EXTENSION_HELP_INDEX.get(self._HELP_INDEX, {})
+                if extension_help_index:
+                    logger.debug("Blending packaged help index with extension help overlay (%d groups, %d commands).",
+                                 len(extension_help_index.get('groups') or {}),
+                                 len(extension_help_index.get('commands') or {}))
+                return self._blend_help_indices(packaged_help_index, extension_help_index)
+
+            # Clear stale overlay cache if schema exists but metadata is invalid.
+            if self.EXTENSION_HELP_INDEX.get(self._HELP_INDEX):
+                self.EXTENSION_HELP_INDEX[self._COMMAND_INDEX_VERSION] = ""
+                self.EXTENSION_HELP_INDEX[self._COMMAND_INDEX_CLOUD_PROFILE] = ""
+                self.EXTENSION_HELP_INDEX[self._HELP_INDEX] = {}
+
+            if self._has_non_always_loaded_extensions():
+                logger.debug("Extension help overlay unavailable on latest profile. Triggering refresh via full load.")
+                return None
+
+            logger.debug("Using packaged help index with %d entries", len(packaged_help_index))
+            return packaged_help_index
 
         if not self._is_index_valid():
             return None
@@ -1127,6 +1183,20 @@ def set_help_index(self, help_data):
 
         :param help_data: Help index data structure containing groups and commands
         """
+        if self.cloud_profile == 'latest':
+            packaged_help_index = self._load_packaged_help_index() or {'groups': {}, 'commands': {}}
+            extension_help_overlay = self._build_extension_help_overlay(packaged_help_index, help_data)
+
+            self.EXTENSION_HELP_INDEX[self._COMMAND_INDEX_VERSION] = __version__
+            self.EXTENSION_HELP_INDEX[self._COMMAND_INDEX_CLOUD_PROFILE] = self.cloud_profile
+            self.EXTENSION_HELP_INDEX[self._HELP_INDEX] = extension_help_overlay
+
+            # Keep local full help cache empty for latest; packaged base + extension overlay are authoritative.
+            self.HELP_INDEX[self._HELP_INDEX] = {}
+            if self.INDEX.get(self._HELP_INDEX):
+                self.INDEX[self._HELP_INDEX] = {}
+            return
+
         self.HELP_INDEX[self._HELP_INDEX] = help_data
         # Clear legacy key if it exists in commandIndex.json.
         if self.INDEX.get(self._HELP_INDEX):
@@ -1187,6 +1257,9 @@ def invalidate(self):
         self.EXTENSION_INDEX[self._COMMAND_INDEX_VERSION] = ""
         self.EXTENSION_INDEX[self._COMMAND_INDEX_CLOUD_PROFILE] = ""
         self.EXTENSION_INDEX[self._COMMAND_INDEX] = {}
+        self.EXTENSION_HELP_INDEX[self._COMMAND_INDEX_VERSION] = ""
+        self.EXTENSION_HELP_INDEX[self._COMMAND_INDEX_CLOUD_PROFILE] = ""
+        self.EXTENSION_HELP_INDEX[self._HELP_INDEX] = {}
         self.HELP_INDEX[self._HELP_INDEX] = {}
         # Clear legacy key if it exists in commandIndex.json.
         if self.INDEX.get(self._HELP_INDEX):

src/azure-cli-core/azure/cli/core/_session.py

@@ -103,6 +103,9 @@ def __len__(self):
 # HELP_INDEX contains cached help summaries for top-level help display
 HELP_INDEX = Session()
 
+# EXTENSION_HELP_INDEX contains extension-only help overlay for top-level help display
+EXTENSION_HELP_INDEX = Session()
+
 # VERSIONS provides local versions and pypi versions.
 # DO NOT USE it to get the current version of azure-cli,
 # it could be lagged behind and can be used to check whether

src/azure-cli-core/azure/cli/core/mock.py

@@ -33,7 +33,7 @@ def __init__(self, commands_loader_cls=None, random_config_dir=False, **kwargs):
             self.env_patch.start()
 
             # Always copy command index to avoid initializing it again
-            files_to_copy = ['commandIndex.json', 'extensionIndex.json', 'helpIndex.json']
+            files_to_copy = ['commandIndex.json', 'extensionIndex.json', 'helpIndex.json', 'extensionHelpIndex.json']
             # In recording mode, copy login credentials from global config dir to the dummy config dir
             if os.getenv(ENV_VAR_TEST_LIVE, '').lower() == 'true':
                 files_to_copy.extend([

src/azure-cli-core/azure/cli/core/tests/test_help.py

@@ -164,7 +164,13 @@ def tearDown(self):
         shutil.rmtree(self._tempdirName)
         self.helps.clear()
         # Invalidate help cache to prevent test data from polluting production cache
-        from azure.cli.core._session import HELP_INDEX, INDEX
+        from azure.cli.core._session import EXTENSION_HELP_INDEX, HELP_INDEX, INDEX
+        if 'helpIndex' in EXTENSION_HELP_INDEX:
+            del EXTENSION_HELP_INDEX['helpIndex']
+        if 'version' in EXTENSION_HELP_INDEX:
+            del EXTENSION_HELP_INDEX['version']
+        if 'cloudProfile' in EXTENSION_HELP_INDEX:
+            del EXTENSION_HELP_INDEX['cloudProfile']
         if 'helpIndex' in HELP_INDEX:
             del HELP_INDEX['helpIndex']
         if 'helpIndex' in INDEX:
@@ -543,8 +549,8 @@ def test_help_cache_extraction(self):
         self.assertEqual(commands['login']['summary'], 'Log in to Azure')
 
     def test_help_cache_storage_and_retrieval(self):
-        """Test that help cache is stored and can be retrieved."""
-        from azure.cli.core import CommandIndex
+        """Test non-latest help cache remains local and retrievable."""
+        from azure.cli.core import CommandIndex, __version__
         from azure.cli.core._session import HELP_INDEX
 
         test_help_data = {
@@ -556,8 +562,11 @@ def test_help_cache_storage_and_retrieval(self):
             }
         }
 
-        command_index = CommandIndex(self.test_cli)
-        command_index.set_help_index(test_help_data)
+        with mock.patch.object(self.test_cli.cloud, 'profile', '2019-03-01-hybrid'):
+            command_index = CommandIndex(self.test_cli)
+            command_index.version = __version__
+            command_index.cloud_profile = '2019-03-01-hybrid'
+            command_index.set_help_index(test_help_data)
 
         retrieved = HELP_INDEX.get('helpIndex')
 
@@ -570,7 +579,7 @@ def test_help_cache_storage_and_retrieval(self):
     def test_help_cache_invalidation(self):
         """Test that cache is invalidated correctly."""
         from azure.cli.core import CommandIndex
-        from azure.cli.core._session import HELP_INDEX
+        from azure.cli.core._session import EXTENSION_HELP_INDEX, HELP_INDEX
 
         test_help_data = {'root': {'groups': {}, 'commands': {}}}
         command_index = CommandIndex(self.test_cli)
@@ -581,9 +590,10 @@ def test_help_cache_invalidation(self):
         command_index.invalidate()
 
         self.assertEqual(HELP_INDEX.get('helpIndex'), {})
+        self.assertEqual(EXTENSION_HELP_INDEX.get('helpIndex'), {})
 
     def test_help_cache_legacy_migration(self):
-        """Test legacy helpIndex migration from commandIndex.json to helpIndex.json."""
+        """Test legacy helpIndex migration from commandIndex.json to helpIndex.json for non-latest."""
         from azure.cli.core import CommandIndex, __version__
         from azure.cli.core._session import HELP_INDEX, INDEX
 
@@ -592,12 +602,13 @@ def test_help_cache_legacy_migration(self):
             'commands': {'legacy-cmd': {'summary': 'Legacy command', 'tags': ''}}
         }
 
-        command_index = CommandIndex(self.test_cli)
-        INDEX[CommandIndex._COMMAND_INDEX_VERSION] = __version__
-        INDEX[CommandIndex._COMMAND_INDEX_CLOUD_PROFILE] = self.test_cli.cloud.profile
-        INDEX['helpIndex'] = test_help_data
+        with mock.patch.object(self.test_cli.cloud, 'profile', '2019-03-01-hybrid'):
+            command_index = CommandIndex(self.test_cli)
+            INDEX[CommandIndex._COMMAND_INDEX_VERSION] = __version__
+            INDEX[CommandIndex._COMMAND_INDEX_CLOUD_PROFILE] = '2019-03-01-hybrid'
+            INDEX['helpIndex'] = test_help_data
 
-        migrated = command_index.get_help_index()
+            migrated = command_index.get_help_index()
 
         self.assertEqual(migrated, test_help_data)
         self.assertEqual(HELP_INDEX.get('helpIndex'), test_help_data)
@@ -634,11 +645,65 @@ def test_help_index_uses_packaged_latest_without_local_index(self):
             'commands': {'version': {'summary': 'Show version.', 'tags': ''}}
         }
 
-        with mock.patch.object(CommandIndex, '_load_packaged_help_index', return_value=packaged_help_data):
+        with mock.patch.object(CommandIndex, '_load_packaged_help_index', return_value=packaged_help_data), \
+                mock.patch.object(CommandIndex, '_has_non_always_loaded_extensions', return_value=False):
             help_index = command_index.get_help_index()
 
         self.assertEqual(help_index, packaged_help_data)
 
+    def test_help_index_latest_missing_overlay_with_extensions_triggers_refresh(self):
+        """Test latest profile returns None to force refresh when extension help overlay is unavailable."""
+        from azure.cli.core import CommandIndex
+        from azure.cli.core._session import EXTENSION_HELP_INDEX, HELP_INDEX, INDEX
+
+        command_index = CommandIndex(self.test_cli)
+
+        INDEX[CommandIndex._COMMAND_INDEX_VERSION] = ""
+        INDEX[CommandIndex._COMMAND_INDEX_CLOUD_PROFILE] = ""
+        INDEX[CommandIndex._COMMAND_INDEX] = {}
+        HELP_INDEX[CommandIndex._HELP_INDEX] = {}
+        EXTENSION_HELP_INDEX[CommandIndex._COMMAND_INDEX_VERSION] = ""
+        EXTENSION_HELP_INDEX[CommandIndex._COMMAND_INDEX_CLOUD_PROFILE] = ""
+        EXTENSION_HELP_INDEX[CommandIndex._HELP_INDEX] = {}
+
+        packaged_help_data = {
+            'groups': {'vm': {'summary': 'Manage VMs.', 'tags': ''}},
+            'commands': {'version': {'summary': 'Show version.', 'tags': ''}}
+        }
+
+        with mock.patch.object(CommandIndex, '_load_packaged_help_index', return_value=packaged_help_data), \
+                mock.patch.object(CommandIndex, '_has_non_always_loaded_extensions', return_value=True):
+            help_index = command_index.get_help_index()
+
+        self.assertIsNone(help_index)
+
+    def test_help_index_latest_blends_packaged_with_extension_overlay(self):
+        """Test latest profile blends packaged help with extension help overlay."""
+        from azure.cli.core import CommandIndex, __version__
+        from azure.cli.core._session import EXTENSION_HELP_INDEX
+
+        command_index = CommandIndex(self.test_cli)
+
+        EXTENSION_HELP_INDEX[CommandIndex._COMMAND_INDEX_VERSION] = __version__
+        EXTENSION_HELP_INDEX[CommandIndex._COMMAND_INDEX_CLOUD_PROFILE] = self.test_cli.cloud.profile
+        EXTENSION_HELP_INDEX[CommandIndex._HELP_INDEX] = {
+            'groups': {'ext-group': {'summary': 'Extension group summary.', 'tags': ''}},
+            'commands': {'ext-cmd': {'summary': 'Extension command summary.', 'tags': ''}}
+        }
+
+        packaged_help_data = {
+            'groups': {'vm': {'summary': 'Manage VMs.', 'tags': ''}},
+            'commands': {'version': {'summary': 'Show version.', 'tags': ''}}
+        }
+
+        with mock.patch.object(CommandIndex, '_load_packaged_help_index', return_value=packaged_help_data):
+            help_index = command_index.get_help_index()
+
+        self.assertIn('vm', help_index['groups'])
+        self.assertIn('ext-group', help_index['groups'])
+        self.assertIn('version', help_index['commands'])
+        self.assertIn('ext-cmd', help_index['commands'])
+
     def test_show_cached_help_output(self):
         """Test that cached help is displayed correctly."""
         from azure.cli.core._help import AzCliHelp