diff --git a/.gitattributes b/.gitattributes index 391370e95ee..8ae308ad2a7 100644 --- a/.gitattributes +++ b/.gitattributes @@ -15,3 +15,7 @@ build_scripts/windows/scripts/az eol=lf # sh scripts should be LF *.sh eol=lf + +# Generated latest index assets should always use LF to avoid cross-platform churn +src/azure-cli-core/azure/cli/core/commandIndex.latest.json text eol=lf +src/azure-cli-core/azure/cli/core/helpIndex.latest.json text eol=lf diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 20dd1dbd762..d76f5531208 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1037,6 +1037,23 @@ jobs: docker pull ${DISTRO_BASE_IMAGE} docker run --rm -e DISTRO=${DISTRO} -e CLI_VERSION=$CLI_VERSION -v $SYSTEM_ARTIFACTSDIRECTORY/debian:/mnt/artifacts -v $(pwd):/azure-cli ${DISTRO_BASE_IMAGE} /bin/bash "/azure-cli/scripts/release/debian/test_deb_in_docker.sh" +- job: VerifyLatestIndices + displayName: "Verify latest index assets" + timeoutInMinutes: 20 + pool: + name: ${{ variables.ubuntu_pool }} + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.13' + inputs: + versionSpec: 3.13 + - template: .azure-pipelines/templates/azdev_setup.yml + - bash: | + set -ev + . env/bin/activate + python scripts/generate_latest_indices.py verify + displayName: 'Verify generated latest indices' + - job: CheckStyle displayName: "Check CLI Style" timeoutInMinutes: 120 diff --git a/scripts/generate_latest_indices.py b/scripts/generate_latest_indices.py new file mode 100644 index 00000000000..927df74fee8 --- /dev/null +++ b/scripts/generate_latest_indices.py @@ -0,0 +1,317 @@ +#!/usr/bin/env python + +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +"""Generate or verify packaged latest command/help index assets. + +This script updates or validates: +- src/azure-cli-core/azure/cli/core/commandIndex.latest.json +- src/azure-cli-core/azure/cli/core/helpIndex.latest.json + +The script runs in an isolated temp AZURE_CONFIG_DIR and with extension directories +redirected to empty folders to avoid local machine state affecting output. +""" + +import argparse +import json +import os +import sys +import tempfile +from contextlib import contextmanager +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[1] +CORE_DIR = REPO_ROOT / 'src' / 'azure-cli-core' / 'azure' / 'cli' / 'core' +COMMAND_INDEX_PATH = CORE_DIR / 'commandIndex.latest.json' +HELP_INDEX_PATH = CORE_DIR / 'helpIndex.latest.json' +CORE_COMMAND_MODULE_PREFIX = 'azure.cli.command_modules.' + + +def _bootstrap_repo_paths(): + """Ensure local source trees are importable when running from repo root.""" + source_roots = [ + REPO_ROOT / 'src' / 'azure-cli-core', + REPO_ROOT / 'src' / 'azure-cli', + REPO_ROOT / 'src' / 'azure-cli-telemetry', + REPO_ROOT / 'src' / 'azure-cli-testsdk', + ] + + for source_root in source_roots: + source_root_str = str(source_root) + if source_root_str not in sys.path: + sys.path.insert(0, source_root_str) + + +@contextmanager +def _isolated_cli_environment(): + """Temporarily isolate config/extension directories for deterministic output.""" + tracked_vars = ['AZURE_CONFIG_DIR', 'AZURE_EXTENSION_DIR'] + previous = {name: os.environ.get(name) for name in tracked_vars} + + with tempfile.TemporaryDirectory(prefix='az-index-gen-') as temp_config_dir: + extension_dir = os.path.join(temp_config_dir, 'cliextensions') + os.makedirs(extension_dir, exist_ok=True) + + os.environ['AZURE_CONFIG_DIR'] = temp_config_dir + os.environ['AZURE_EXTENSION_DIR'] = extension_dir + + try: + yield temp_config_dir, extension_dir + finally: + for name, value in previous.items(): + if value is None: + os.environ.pop(name, None) + else: + os.environ[name] = value + + +def _read_json(path): + if not path.is_file(): + return None + with path.open('r', encoding='utf-8-sig') as handle: + return json.load(handle) + + +def _order_keys_like_template(generated, template): + """Preserve existing key order when possible, append new keys in sorted order.""" + if not isinstance(generated, dict): + return generated + + if not isinstance(template, dict): + return {key: generated[key] for key in sorted(generated)} + + ordered = {} + for key in template: + if key in generated: + ordered[key] = generated[key] + + for key in sorted(generated): + if key not in ordered: + ordered[key] = generated[key] + + return ordered + + +def _extract_builtin_module_name(command): + """Return built-in module name for a command table entry, or None for extension entries.""" + command_source = getattr(command, 'command_source', None) + if isinstance(command_source, str) and command_source.startswith(CORE_COMMAND_MODULE_PREFIX): + return command_source + + command_loader = getattr(command, 'loader', None) + loader_module = getattr(command_loader, '__module__', None) + if isinstance(loader_module, str) and loader_module.startswith(CORE_COMMAND_MODULE_PREFIX): + return loader_module + + return None + + +def _build_command_index_map(command_table): + command_index = {} + for command_name, command in command_table.items(): + top_command = command_name.split()[0] + module_name = _extract_builtin_module_name(command) + if not module_name: + continue + + modules = command_index.setdefault(top_command, []) + if module_name not in modules: + modules.append(module_name) + + for top_command, modules in command_index.items(): + command_index[top_command] = sorted(modules) + + return command_index + + +def _build_help_index_map(cli_ctx, commands_loader): + from azure.cli.core._help import CliGroupHelpFile, extract_help_index_data + from azure.cli.core.parser import AzCliCommandParser + + parser = AzCliCommandParser(cli_ctx) + parser.load_command_table(commands_loader) + + root_subparser = parser.subparsers.get(tuple()) + if not root_subparser: + return {'groups': {}, 'commands': {}} + + help_obj = cli_ctx.help_cls(cli_ctx) + root_help = CliGroupHelpFile(help_obj, '', root_subparser) + root_help.load(root_subparser) + + groups, commands = extract_help_index_data(root_help) + + normalized_groups = { + group_name: { + 'summary': group_data.get('summary', ''), + 'tags': group_data.get('tags', '') + } + for group_name, group_data in groups.items() + } + normalized_commands = { + command_name: { + 'summary': command_data.get('summary', ''), + 'tags': command_data.get('tags', '') + } + for command_name, command_data in commands.items() + } + + return { + 'groups': {key: normalized_groups[key] for key in sorted(normalized_groups)}, + 'commands': {key: normalized_commands[key] for key in sorted(normalized_commands)} + } + + +def _generate_documents(): + _bootstrap_repo_paths() + + with _isolated_cli_environment() as (temp_config_dir, extension_dir): + from azure.cli.core import CommandIndex, __version__, get_default_cli + import azure.cli.core.extension as extension_module + + # Hard pin extension discovery directories so local/global installed extensions do not leak in. + extension_module.EXTENSIONS_DIR = extension_dir + extension_module.EXTENSIONS_SYS_DIR = os.path.join(temp_config_dir, 'empty-system-extensions') + extension_module.DEV_EXTENSION_SOURCES = [] + os.makedirs(extension_module.EXTENSIONS_SYS_DIR, exist_ok=True) + + cli = get_default_cli() + cli.cloud.profile = 'latest' + cli.data['completer_active'] = False + + invoker = cli.invocation_cls( + cli_ctx=cli, + commands_loader_cls=cli.commands_loader_cls, + parser_cls=cli.parser_cls, + help_cls=cli.help_cls + ) + cli.invocation = invoker + commands_loader = invoker.commands_loader + command_table = commands_loader.load_command_table(None) + + current_command_doc = _read_json(COMMAND_INDEX_PATH) or {} + current_help_doc = _read_json(HELP_INDEX_PATH) or {} + + generated_command_index = _build_command_index_map(command_table) + generated_help_index = _build_help_index_map(cli, commands_loader) + + ordered_command_index = _order_keys_like_template( + generated_command_index, + current_command_doc.get(CommandIndex._COMMAND_INDEX) # pylint: disable=protected-access + ) + + help_template = current_help_doc.get(CommandIndex._HELP_INDEX, {}) # pylint: disable=protected-access + ordered_help_groups = _order_keys_like_template(generated_help_index['groups'], help_template.get('groups')) + ordered_help_commands = _order_keys_like_template(generated_help_index['commands'], help_template.get('commands')) + + command_doc = { + CommandIndex._COMMAND_INDEX_VERSION: __version__, # pylint: disable=protected-access + CommandIndex._COMMAND_INDEX_CLOUD_PROFILE: 'latest', # pylint: disable=protected-access + CommandIndex._COMMAND_INDEX: ordered_command_index # pylint: disable=protected-access + } + + help_doc = { + CommandIndex._COMMAND_INDEX_VERSION: __version__, # pylint: disable=protected-access + CommandIndex._COMMAND_INDEX_CLOUD_PROFILE: 'latest', # pylint: disable=protected-access + CommandIndex._HELP_INDEX: { # pylint: disable=protected-access + 'groups': ordered_help_groups, + 'commands': ordered_help_commands + } + } + + return command_doc, help_doc + + +def _serialize_json(document): + return json.dumps(document, indent=2) + '\n' + + +def _write_file(path, content): + path.parent.mkdir(parents=True, exist_ok=True) + with path.open('w', encoding='utf-8', newline='\n') as handle: + handle.write(content) + + +def _load_text(path): + if not path.is_file(): + return None + return path.read_text(encoding='utf-8-sig') + + +def _run_generate(command_text, help_text): + current_command_text = _load_text(COMMAND_INDEX_PATH) + current_help_text = _load_text(HELP_INDEX_PATH) + + updated_files = [] + + if current_command_text != command_text: + _write_file(COMMAND_INDEX_PATH, command_text) + updated_files.append(COMMAND_INDEX_PATH) + + if current_help_text != help_text: + _write_file(HELP_INDEX_PATH, help_text) + updated_files.append(HELP_INDEX_PATH) + + if updated_files: + print('Updated generated latest index files:') + for path in updated_files: + print(f' - {path.relative_to(REPO_ROOT)}') + else: + print('Latest index files are already up-to-date.') + + return 0 + + +def _run_verify(command_text, help_text): + mismatched = [] + + if _load_text(COMMAND_INDEX_PATH) != command_text: + mismatched.append(COMMAND_INDEX_PATH) + if _load_text(HELP_INDEX_PATH) != help_text: + mismatched.append(HELP_INDEX_PATH) + + if mismatched: + print('Generated latest index files are out of date:') + for path in mismatched: + print(f' - {path.relative_to(REPO_ROOT)}') + print('Run:') + print(' python scripts/generate_latest_indices.py generate') + return 1 + + print('Verified: latest index files are up-to-date.') + return 0 + + +def _parse_args(): + parser = argparse.ArgumentParser( + description='Generate or verify packaged latest command and help index JSON files.' + ) + parser.add_argument( + 'mode', + nargs='?', + choices=['generate', 'verify'], + default='generate', + help='Mode to run. generate writes files; verify checks drift and exits non-zero on mismatch.' + ) + return parser.parse_args() + + +def main(): + args = _parse_args() + + command_doc, help_doc = _generate_documents() + command_text = _serialize_json(command_doc) + help_text = _serialize_json(help_doc) + + if args.mode == 'verify': + return _run_verify(command_text, help_text) + + return _run_generate(command_text, help_text) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/azure-cli-core/azure/cli/core/__init__.py b/src/azure-cli-core/azure/cli/core/__init__.py index 35777f7264d..fc4dcae82c1 100644 --- a/src/azure-cli-core/azure/cli/core/__init__.py +++ b/src/azure-cli-core/azure/cli/core/__init__.py @@ -8,6 +8,7 @@ import os import sys +import json import timeit import concurrent.futures from concurrent.futures import ThreadPoolExecutor @@ -30,6 +31,8 @@ EVENT_FAILED_EXTENSION_LOAD = 'MainLoader.OnFailedExtensionLoad' # Marker used by CommandIndex.get() to signal top-level tab completion optimization TOP_LEVEL_COMPLETION_MARKER = '__top_level_completion__' +# Internal sentinel used to trigger latest-profile extension help overlay refresh path. +REFRESH_EXTENSION_HELP_OVERLAY_SENTINEL = '__refresh_extension_help_overlay__' # [Reserved, in case of future usage] # Modules that will always be loaded. They don't expose commands but hook into CLI core. @@ -42,6 +45,18 @@ MAX_WORKER_THREAD_COUNT = 4 +def _get_top_level_command(args): + """Return normalized top-level command token or None when unavailable.""" + if not args: + return None + + top_command = args[0] + if not top_command or top_command.startswith('-'): + return None + + return top_command.lower() + + def _configure_knack(): """Override consts defined in knack to make them Azure CLI-specific.""" @@ -72,7 +87,7 @@ def __init__(self, **kwargs): register_ids_argument, register_global_subscription_argument, register_global_policy_argument) from azure.cli.core.cloud import get_active_cloud from azure.cli.core.commands.transform import register_global_transforms - from azure.cli.core._session import ACCOUNT, CONFIG, SESSION, INDEX, VERSIONS + from azure.cli.core._session import ACCOUNT, CONFIG, SESSION, INDEX, EXTENSION_INDEX, HELP_INDEX, EXTENSION_HELP_INDEX, VERSIONS from azure.cli.core.util import handle_version_update from knack.util import ensure_dir @@ -89,6 +104,9 @@ def __init__(self, **kwargs): CONFIG.load(os.path.join(azure_folder, 'az.json')) SESSION.load(os.path.join(azure_folder, 'az.sess'), max_age=3600) INDEX.load(os.path.join(azure_folder, 'commandIndex.json')) + EXTENSION_INDEX.load(os.path.join(azure_folder, 'extensionIndex.json')) + HELP_INDEX.load(os.path.join(azure_folder, 'helpIndex.json')) + EXTENSION_HELP_INDEX.load(os.path.join(azure_folder, 'extensionHelpIndex.json')) VERSIONS.load(os.path.join(azure_folder, 'versionCheck.json')) handle_version_update() @@ -252,7 +270,21 @@ def _update_command_definitions(self): loader.command_table = self.command_table loader._update_command_definitions() # pylint: disable=protected-access - # pylint: disable=too-many-statements, too-many-locals + @staticmethod + def _should_update_extension_index(index_extensions, command_index): + """Return True when latest-profile extension overlays should be refreshed.""" + return (index_extensions is None and + command_index is not None and + command_index.cloud_profile == 'latest') + + def _is_latest_non_completion_invocation(self, command_index, args): + """Return True for real latest-profile invocations (not shell completion).""" + return (command_index is not None and + command_index.cloud_profile == 'latest' and + bool(args) and + not self.cli_ctx.data['completer_active']) + + # pylint: disable=too-many-statements, too-many-locals, too-many-return-statements def load_command_table(self, args): from importlib import import_module import pkgutil @@ -471,6 +503,10 @@ def _get_extension_suppressions(mod_loaders): # The index won't contain suppressed extensions _update_command_table_from_extensions([], index_extensions) + if self._should_update_extension_index(index_extensions, command_index): + command_index.update_extension_index(self.command_table) + self._cache_help_index(command_index) + logger.debug("Loaded %d groups, %d commands.", len(self.command_group_table), len(self.command_table)) from azure.cli.core.util import roughly_parse_command # The index may be outdated. Make sure the command appears in the loaded command table @@ -512,6 +548,14 @@ def _get_extension_suppressions(mod_loaders): logger.debug("Could not find a match in the command or command group table for '%s'. " "The index may be outdated.", raw_cmd) + + if self._is_latest_non_completion_invocation(command_index, args): + top_command = _get_top_level_command(args) + packaged_core_index = command_index.get_packaged_core_index() or {} + if top_command and top_command != 'help' and top_command not in packaged_core_index: + logger.debug("Top-level command '%s' is not in packaged core index. " + "Skipping full core module reload.", top_command) + return self.command_table else: logger.debug("No module found from index for '%s'", args) @@ -730,14 +774,19 @@ class CommandIndex: _COMMAND_INDEX_VERSION = 'version' _COMMAND_INDEX_CLOUD_PROFILE = 'cloudProfile' _HELP_INDEX = 'helpIndex' + _PACKAGED_COMMAND_INDEX_LATEST = 'commandIndex.latest.json' + _PACKAGED_HELP_INDEX_LATEST = 'helpIndex.latest.json' def __init__(self, cli_ctx=None): """Class to manage command index. :param cli_ctx: Only needed when `get` or `update` is called. """ - from azure.cli.core._session import INDEX + from azure.cli.core._session import INDEX, EXTENSION_INDEX, HELP_INDEX, EXTENSION_HELP_INDEX self.INDEX = INDEX + self.EXTENSION_INDEX = EXTENSION_INDEX + self.HELP_INDEX = HELP_INDEX + self.EXTENSION_HELP_INDEX = EXTENSION_HELP_INDEX if cli_ctx: self.version = __version__ self.cloud_profile = cli_ctx.cloud.profile @@ -753,7 +802,33 @@ def _is_index_valid(self): return (index_version and index_version == self.version and cloud_profile and cloud_profile == self.cloud_profile) - def _get_top_level_completion_commands(self): + def _is_extension_index_valid(self): + """Check if the extension index version and cloud profile are valid.""" + index_version = self.EXTENSION_INDEX.get(self._COMMAND_INDEX_VERSION) + cloud_profile = self.EXTENSION_INDEX.get(self._COMMAND_INDEX_CLOUD_PROFILE) + return (index_version and index_version == self.version and + cloud_profile and cloud_profile == self.cloud_profile) + + def _is_extension_help_index_valid(self): + """Check if the extension help index version and cloud profile are valid.""" + index_version = self.EXTENSION_HELP_INDEX.get(self._COMMAND_INDEX_VERSION) + cloud_profile = self.EXTENSION_HELP_INDEX.get(self._COMMAND_INDEX_CLOUD_PROFILE) + return (index_version and index_version == self.version and + cloud_profile and cloud_profile == self.cloud_profile) + + def _clear_extension_index_cache(self): + """Clear extension command index cache metadata and payload.""" + self.EXTENSION_INDEX[self._COMMAND_INDEX_VERSION] = "" + self.EXTENSION_INDEX[self._COMMAND_INDEX_CLOUD_PROFILE] = "" + self.EXTENSION_INDEX[self._COMMAND_INDEX] = {} + + def _clear_extension_help_overlay_cache(self): + """Clear extension help overlay cache metadata and payload.""" + self.EXTENSION_HELP_INDEX[self._COMMAND_INDEX_VERSION] = "" + self.EXTENSION_HELP_INDEX[self._COMMAND_INDEX_CLOUD_PROFILE] = "" + self.EXTENSION_HELP_INDEX[self._HELP_INDEX] = {} + + def _get_top_level_completion_commands(self, index=None): """Get top-level command names for tab completion optimization. Returns marker and list of top-level commands (e.g., 'network', 'vm') for creating @@ -762,7 +837,7 @@ def _get_top_level_completion_commands(self): :return: tuple of (TOP_LEVEL_COMPLETION_MARKER, list of top-level command names) or None """ - index = self.INDEX.get(self._COMMAND_INDEX) or {} + index = index or self.INDEX.get(self._COMMAND_INDEX) or {} if not index: logger.debug("Command index is empty, will fall back to loading all modules") return None @@ -770,31 +845,258 @@ def _get_top_level_completion_commands(self): logger.debug("Top-level completion: %d commands available", len(top_level_commands)) return TOP_LEVEL_COMPLETION_MARKER, top_level_commands + def _can_use_packaged_command_index(self, ignore_extensions=False): + """Whether packaged command index can be used safely for this invocation.""" + if self.cloud_profile != 'latest': + return False + + if ignore_extensions: + return True + + # If non-always-loaded extensions are installed, we need a full rebuild to include overrides/extensions. + if self._has_non_always_loaded_extensions(): + return False + + return True + + def _load_packaged_command_index(self): + """Load packaged command index for latest profile if present.""" + file_path = os.path.join(os.path.dirname(__file__), self._PACKAGED_COMMAND_INDEX_LATEST) + if not os.path.isfile(file_path): + return None + + try: + with open(file_path, 'r', encoding='utf-8-sig') as f: + data = json.load(f) + except (OSError, json.JSONDecodeError) as ex: + logger.debug("Failed to load packaged command index file '%s': %s", file_path, ex) + return None + + if not isinstance(data, dict): + logger.debug("Packaged command index file '%s' has invalid schema.", file_path) + return None + + return data + + def _load_packaged_help_index(self): + """Load packaged help index for latest profile if present.""" + file_path = os.path.join(os.path.dirname(__file__), self._PACKAGED_HELP_INDEX_LATEST) + if not os.path.isfile(file_path): + return None + + try: + with open(file_path, 'r', encoding='utf-8-sig') as f: + data = json.load(f) + except (OSError, json.JSONDecodeError) as ex: + logger.debug("Failed to load packaged help index file '%s': %s", file_path, ex) + return None + + if not isinstance(data, dict): + logger.debug("Packaged help index file '%s' has invalid schema.", file_path) + return None + + version_matches = data.get(self._COMMAND_INDEX_VERSION) == self.version + profile_matches = data.get(self._COMMAND_INDEX_CLOUD_PROFILE) == self.cloud_profile + if not version_matches or not profile_matches: + if not version_matches: + logger.debug("Packaged help index version doesn't match current CLI version.") + if not profile_matches: + logger.debug("Packaged help index cloud profile doesn't match current cloud profile.") + return None + + help_index = data.get(self._HELP_INDEX) + if not isinstance(help_index, dict) or not help_index: + logger.debug("Packaged help index mapping is missing or empty.") + return None + + return help_index + + @staticmethod + def _has_non_always_loaded_extensions(): + """Return True if a non-always-loaded extension is installed.""" + from azure.cli.core.extension import get_extensions, get_extension_modname + + try: + for ext in get_extensions() or []: + ext_mod = get_extension_modname(ext.name, ext.path) + if ext_mod not in ALWAYS_LOADED_EXTENSIONS: + logger.debug("Found installed extension '%s' (%s).", ext.name, ext_mod) + return True + except Exception as ex: # pylint: disable=broad-except + logger.debug("Failed to evaluate installed extensions: %s", ex) + return True + + return False + + def _get_packaged_command_index(self, ignore_extensions=False): + """Get packaged command index mapping if valid for current profile/version.""" + if not self._can_use_packaged_command_index(ignore_extensions=ignore_extensions): + return None + + packaged_index = self._load_packaged_command_index() + if not packaged_index: + return None + + if packaged_index.get(self._COMMAND_INDEX_VERSION) != self.version: + logger.debug("Packaged command index version doesn't match current CLI version.") + return None + + if packaged_index.get(self._COMMAND_INDEX_CLOUD_PROFILE) != self.cloud_profile: + logger.debug("Packaged command index cloud profile doesn't match current cloud profile.") + return None + + index = packaged_index.get(self._COMMAND_INDEX) + if not isinstance(index, dict) or not index: + logger.debug("Packaged command index mapping is missing or empty.") + return None + + logger.debug("Using packaged command index for profile '%s'.", self.cloud_profile) + return index + + def get_packaged_core_index(self): + """Get packaged core command index mapping, ignoring extension presence checks.""" + return self._get_packaged_command_index(ignore_extensions=True) + + @staticmethod + def _blend_command_indices(core_index, extension_index): + """Blend packaged core index with local extension overlay index.""" + blended = {cmd: list(mods) for cmd, mods in (core_index or {}).items()} + for cmd, mods in (extension_index or {}).items(): + if cmd not in blended: + blended[cmd] = [] + for mod in mods: + if mod not in blended[cmd]: + blended[cmd].append(mod) + return blended + + @staticmethod + def _blend_help_indices(base_help_index, extension_help_index): + """Blend packaged core help with extension-only help overlay.""" + blended = { + 'groups': dict((base_help_index or {}).get('groups') or {}), + 'commands': dict((base_help_index or {}).get('commands') or {}) + } + ext_help_index = extension_help_index or {} + for section in ('groups', 'commands'): + blended_section = blended[section] + for key, value in (ext_help_index.get(section) or {}).items(): + blended_section[key] = value + return blended + + @staticmethod + def _build_extension_help_overlay(base_help_index, full_help_index): + """Build extension-only help overlay by diffing full help against packaged core help.""" + overlay = {'groups': {}, 'commands': {}} + base_help_index = base_help_index or {} + full_help_index = full_help_index or {} + for section in ('groups', 'commands'): + base_section = base_help_index.get(section) or {} + full_section = full_help_index.get(section) or {} + for key, value in full_section.items(): + if key not in base_section or base_section[key] != value: + overlay[section][key] = value + return overlay + + def _get_blended_latest_index(self): + """Get effective index for latest profile by blending core and extension indices.""" + if self.cloud_profile != 'latest': + return None, False, False + + core_index = self._get_packaged_command_index(ignore_extensions=True) + if not core_index: + return None, False, False + + extension_index = {} + extension_index_available = False + has_non_always_loaded_extensions = self._has_non_always_loaded_extensions() + if self._is_extension_index_valid(): + extension_index = self.EXTENSION_INDEX.get(self._COMMAND_INDEX) or {} + extension_index_available = True + else: + if self.EXTENSION_INDEX.get(self._COMMAND_INDEX): + logger.debug("Extension index version or cloud profile is invalid, clearing local extension index.") + self._clear_extension_index_cache() + + if extension_index: + logger.debug("Blending packaged core index with local extension index.") + return self._blend_command_indices(core_index, extension_index), extension_index_available, has_non_always_loaded_extensions + + def _resolve_latest_index_lookup(self, args, top_command): + """Resolve command lookup for latest profile using blended packaged and extension indices.""" + force_packaged_for_version = bool(top_command == 'version') + index, extension_index_available, has_non_always_loaded_extensions = self._get_blended_latest_index() + if index is None: + return None + + force_load_all_extensions = (has_non_always_loaded_extensions and + not extension_index_available and + not force_packaged_for_version) + result = self._lookup_command_in_index(index, args, + force_load_all_extensions=force_load_all_extensions) + if result: + return result + + if (args and not args[0].startswith('-') and + not self.cli_ctx.data['completer_active'] and + not force_packaged_for_version and + top_command != 'help'): + # Unknown top-level command on latest should prefer extension-only retry and avoid + # full core module rebuild to preserve packaged-index startup benefit. + if has_non_always_loaded_extensions: + logger.debug("No match found in blended latest index for '%s'. Loading all extensions.", + args[0]) + return [], None + + logger.debug("No match found in latest index for '%s' and no dynamic extensions are installed. " + "Skipping core module rebuild.", args[0]) + return [], [] + + logger.debug("No match found in blended latest index. Falling back to local command index.") + return None + def get(self, args): """Get the corresponding module and extension list of a command. :param args: command arguments, like ['network', 'vnet', 'create', '-h'] :return: a tuple containing a list of modules and a list of extensions. """ - # If the command index version or cloud profile doesn't match those of the current command, - # invalidate the command index. - if not self._is_index_valid(): - logger.debug("Command index version or cloud profile is invalid or doesn't match the current command.") - self.invalidate() - return None + + top_command = _get_top_level_command(args) + + if self.cloud_profile == 'latest': + latest_result = self._resolve_latest_index_lookup(args, top_command) + if latest_result is not None: + return latest_result + + # For non-latest, use local command index and fallback logic. + index = None + if self._is_index_valid(): + index = self.INDEX.get(self._COMMAND_INDEX) or {} + else: + # `az version` should stay fast even when extensions are installed. + force_packaged_for_version = bool(top_command == 'version' and self.cloud_profile == 'latest') + index = self._get_packaged_command_index(ignore_extensions=force_packaged_for_version) + if index is None: + logger.debug("Command index version or cloud profile is invalid or doesn't match the current command.") + self.invalidate() + return None + + return self._lookup_command_in_index(index, args) + + def _lookup_command_in_index(self, index, args, force_load_all_extensions=False): + """Lookup command modules/extensions from a resolved index mapping.""" # Make sure the top-level command is provided, like `az version`. # Skip command index for `az` or `az --help`. - if not args or args[0].startswith('-'): + top_command = _get_top_level_command(args) + if not top_command: # For top-level completion (az [tab]) if not args and self.cli_ctx.data.get('completer_active'): - return self._get_top_level_completion_commands() + return self._get_top_level_completion_commands(index=index) return None - # Get the top-level command, like `network` in `network vnet create -h` - # Normalize top-level command for index lookup so mixed-case commands hit key - top_command = args[0].lower() - index = self.INDEX[self._COMMAND_INDEX] + index = index or {} + # Check the command index for (command: [module]) mapping, like # "network": ["azure.cli.command_modules.natgateway", "azure.cli.command_modules.network", "azext_firewall"] index_modules_extensions = index.get(top_command) @@ -822,31 +1124,103 @@ def get(self, args): index_extensions.append(m) else: logger.warning("Unrecognized module: %s", m) + + if force_load_all_extensions: + logger.debug("Extension index is unavailable. Loading all installed extensions for safety.") + index_extensions = None return index_builtin_modules, index_extensions + if force_load_all_extensions and not self.cli_ctx.data['completer_active']: + logger.debug("Top-level command '%s' not found in blended index. Loading all extensions.", top_command) + return [], None + return None + def _get_help_index_cached_local(self, latest_fallback=False): + """Return cached local help index when available and index metadata is valid.""" + if not self._is_index_valid(): + return None + + help_index = self.HELP_INDEX.get(self._HELP_INDEX, {}) + if not help_index: + return None + + if latest_fallback: + logger.debug("Using cached local help index with %d entries", len(help_index)) + else: + logger.debug("Using cached help index with %d entries", len(help_index)) + return help_index + + def _get_help_index_latest(self): + """Return help index for latest profile using packaged help and extension overlay.""" + # Packaged help is the base for latest profile. + packaged_help_index = self._load_packaged_help_index() + if not packaged_help_index: + # Defensive fallback to local cache if packaged asset is unavailable. + return self._get_help_index_cached_local(latest_fallback=True) + + if self._is_extension_help_index_valid(): + extension_help_index = self.EXTENSION_HELP_INDEX.get(self._HELP_INDEX, {}) + if extension_help_index: + logger.debug("Blending packaged help index with extension help overlay (%d groups, %d commands).", + len(extension_help_index.get('groups') or {}), + len(extension_help_index.get('commands') or {})) + return self._blend_help_indices(packaged_help_index, extension_help_index) + + # Clear stale overlay cache if schema exists but metadata is invalid. + if self.EXTENSION_HELP_INDEX.get(self._HELP_INDEX): + self._clear_extension_help_overlay_cache() + + if self._has_non_always_loaded_extensions(): + logger.debug("Extension help overlay unavailable on latest profile. Help index will be refreshed when overlay becomes available.") + return None + + logger.debug("Using packaged help index with %d entries", len(packaged_help_index)) + return packaged_help_index + def get_help_index(self): """Get the help index for top-level help display. :return: Dictionary mapping top-level commands to their short summaries, or None if not available """ - if not self._is_index_valid(): - return None + if self.cloud_profile == 'latest': + return self._get_help_index_latest() - help_index = self.INDEX.get(self._HELP_INDEX, {}) - if help_index: - logger.debug("Using cached help index with %d entries", len(help_index)) - return help_index + return self._get_help_index_cached_local() - return None + def needs_latest_extension_help_overlay_refresh(self): + """Return True when latest-profile top-level help should refresh extension help overlay.""" + if self.cloud_profile != 'latest': + return False + + if self._is_extension_help_index_valid(): + return False + + return self._has_non_always_loaded_extensions() def set_help_index(self, help_data): """Set the help index data. :param help_data: Help index data structure containing groups and commands """ - self.INDEX[self._HELP_INDEX] = help_data + if self.cloud_profile == 'latest': + packaged_help_index = self._load_packaged_help_index() or {'groups': {}, 'commands': {}} + extension_help_overlay = self._build_extension_help_overlay(packaged_help_index, help_data) + + self.EXTENSION_HELP_INDEX[self._COMMAND_INDEX_VERSION] = __version__ + self.EXTENSION_HELP_INDEX[self._COMMAND_INDEX_CLOUD_PROFILE] = self.cloud_profile + self.EXTENSION_HELP_INDEX[self._HELP_INDEX] = extension_help_overlay + + # Keep local full help cache empty for latest; packaged base + extension overlay are authoritative. + self.HELP_INDEX[self._HELP_INDEX] = {} + if self.INDEX.get(self._HELP_INDEX): + self.INDEX[self._HELP_INDEX] = {} + return + + self.HELP_INDEX[self._HELP_INDEX] = help_data + # Clear legacy key if it exists in commandIndex.json. + if self.INDEX.get(self._HELP_INDEX): + self.INDEX[self._HELP_INDEX] = {} def update(self, command_table): """Update the command index according to the given command table. @@ -870,8 +1244,28 @@ def update(self, command_table): elapsed_time = timeit.default_timer() - start_time self.INDEX[self._COMMAND_INDEX] = index + + self.update_extension_index(command_table) + logger.debug("Updated command index in %.3f seconds.", elapsed_time) + def update_extension_index(self, command_table): + """Update extension-only overlay index from a command table (latest profile only).""" + if self.cloud_profile != 'latest': + return + + from collections import defaultdict + extension_index = defaultdict(list) + for command_name, command in command_table.items(): + top_command = command_name.split()[0] + module_name = command.loader.__module__ + if module_name.startswith('azext_') and module_name not in extension_index[top_command]: + extension_index[top_command].append(module_name) + + self.EXTENSION_INDEX[self._COMMAND_INDEX_VERSION] = __version__ + self.EXTENSION_INDEX[self._COMMAND_INDEX_CLOUD_PROFILE] = self.cloud_profile + self.EXTENSION_INDEX[self._COMMAND_INDEX] = extension_index + def invalidate(self): """Invalidate the command index. @@ -886,7 +1280,12 @@ def invalidate(self): self.INDEX[self._COMMAND_INDEX_VERSION] = "" self.INDEX[self._COMMAND_INDEX_CLOUD_PROFILE] = "" self.INDEX[self._COMMAND_INDEX] = {} - self.INDEX[self._HELP_INDEX] = {} + self._clear_extension_index_cache() + self._clear_extension_help_overlay_cache() + self.HELP_INDEX[self._HELP_INDEX] = {} + # Clear legacy key if it exists in commandIndex.json. + if self.INDEX.get(self._HELP_INDEX): + self.INDEX[self._HELP_INDEX] = {} logger.debug("Command index has been invalidated.") diff --git a/src/azure-cli-core/azure/cli/core/_session.py b/src/azure-cli-core/azure/cli/core/_session.py index 463174218fc..294cc923f74 100644 --- a/src/azure-cli-core/azure/cli/core/_session.py +++ b/src/azure-cli-core/azure/cli/core/_session.py @@ -97,6 +97,15 @@ def __len__(self): # INDEX contains {top-level command: [command_modules and extensions]} mapping index INDEX = Session() +# EXTENSION_INDEX contains top-level command mappings for installed extensions +EXTENSION_INDEX = Session() + +# HELP_INDEX contains cached help summaries for top-level help display +HELP_INDEX = Session() + +# EXTENSION_HELP_INDEX contains extension-only help overlay for top-level help display +EXTENSION_HELP_INDEX = Session() + # VERSIONS provides local versions and pypi versions. # DO NOT USE it to get the current version of azure-cli, # it could be lagged behind and can be used to check whether diff --git a/src/azure-cli-core/azure/cli/core/commandIndex.latest.json b/src/azure-cli-core/azure/cli/core/commandIndex.latest.json new file mode 100644 index 00000000000..51a8da53a2d --- /dev/null +++ b/src/azure-cli-core/azure/cli/core/commandIndex.latest.json @@ -0,0 +1,327 @@ +{ + "version": "2.84.0", + "cloudProfile": "latest", + "commandIndex": { + "account": [ + "azure.cli.command_modules.profile", + "azure.cli.command_modules.resource" + ], + "acr": [ + "azure.cli.command_modules.acr" + ], + "ad": [ + "azure.cli.command_modules.role" + ], + "advisor": [ + "azure.cli.command_modules.advisor" + ], + "afd": [ + "azure.cli.command_modules.cdn" + ], + "aks": [ + "azure.cli.command_modules.acs", + "azure.cli.command_modules.serviceconnector" + ], + "ams": [ + "azure.cli.command_modules.ams" + ], + "apim": [ + "azure.cli.command_modules.apim" + ], + "appconfig": [ + "azure.cli.command_modules.appconfig" + ], + "appservice": [ + "azure.cli.command_modules.appservice" + ], + "aro": [ + "azure.cli.command_modules.aro" + ], + "backup": [ + "azure.cli.command_modules.backup" + ], + "batch": [ + "azure.cli.command_modules.batch" + ], + "batchai": [ + "azure.cli.command_modules.batchai" + ], + "bicep": [ + "azure.cli.command_modules.resource" + ], + "billing": [ + "azure.cli.command_modules.billing" + ], + "bot": [ + "azure.cli.command_modules.botservice" + ], + "cache": [ + "azure.cli.command_modules.configure" + ], + "capacity": [ + "azure.cli.command_modules.vm" + ], + "cdn": [ + "azure.cli.command_modules.cdn" + ], + "cloud": [ + "azure.cli.command_modules.cloud" + ], + "cognitiveservices": [ + "azure.cli.command_modules.cognitiveservices" + ], + "compute-fleet": [ + "azure.cli.command_modules.computefleet" + ], + "compute-recommender": [ + "azure.cli.command_modules.compute_recommender" + ], + "config": [ + "azure.cli.command_modules.config" + ], + "configure": [ + "azure.cli.command_modules.configure" + ], + "connection": [ + "azure.cli.command_modules.serviceconnector" + ], + "consumption": [ + "azure.cli.command_modules.consumption" + ], + "container": [ + "azure.cli.command_modules.container" + ], + "containerapp": [ + "azure.cli.command_modules.containerapp", + "azure.cli.command_modules.serviceconnector" + ], + "cosmosdb": [ + "azure.cli.command_modules.cosmosdb" + ], + "data-boundary": [ + "azure.cli.command_modules.resource" + ], + "databoxedge": [ + "azure.cli.command_modules.databoxedge" + ], + "demo": [ + "azure.cli.command_modules.util" + ], + "deployment": [ + "azure.cli.command_modules.resource" + ], + "deployment-scripts": [ + "azure.cli.command_modules.resource" + ], + "disk": [ + "azure.cli.command_modules.vm" + ], + "disk-access": [ + "azure.cli.command_modules.vm" + ], + "disk-encryption-set": [ + "azure.cli.command_modules.vm" + ], + "dls": [ + "azure.cli.command_modules.dls" + ], + "dms": [ + "azure.cli.command_modules.dms" + ], + "eventgrid": [ + "azure.cli.command_modules.eventgrid" + ], + "eventhubs": [ + "azure.cli.command_modules.eventhubs" + ], + "extension": [ + "azure.cli.command_modules.extension" + ], + "feature": [ + "azure.cli.command_modules.resource" + ], + "feedback": [ + "azure.cli.command_modules.feedback" + ], + "find": [ + "azure.cli.command_modules.find" + ], + "functionapp": [ + "azure.cli.command_modules.appservice", + "azure.cli.command_modules.serviceconnector" + ], + "group": [ + "azure.cli.command_modules.resource" + ], + "hdinsight": [ + "azure.cli.command_modules.hdinsight" + ], + "identity": [ + "azure.cli.command_modules.identity" + ], + "image": [ + "azure.cli.command_modules.vm" + ], + "interactive": [ + "azure.cli.command_modules.interactive" + ], + "iot": [ + "azure.cli.command_modules.iot" + ], + "keyvault": [ + "azure.cli.command_modules.keyvault" + ], + "lab": [ + "azure.cli.command_modules.lab" + ], + "lock": [ + "azure.cli.command_modules.resource" + ], + "logicapp": [ + "azure.cli.command_modules.appservice" + ], + "login": [ + "azure.cli.command_modules.profile" + ], + "logout": [ + "azure.cli.command_modules.profile" + ], + "managed-cassandra": [ + "azure.cli.command_modules.cosmosdb" + ], + "managedapp": [ + "azure.cli.command_modules.resource" + ], + "managedservices": [ + "azure.cli.command_modules.managedservices" + ], + "maps": [ + "azure.cli.command_modules.maps" + ], + "mariadb": [ + "azure.cli.command_modules.rdbms" + ], + "monitor": [ + "azure.cli.command_modules.monitor" + ], + "mysql": [ + "azure.cli.command_modules.mysql", + "azure.cli.command_modules.rdbms" + ], + "netappfiles": [ + "azure.cli.command_modules.netappfiles" + ], + "network": [ + "azure.cli.command_modules.network", + "azure.cli.command_modules.privatedns" + ], + "policy": [ + "azure.cli.command_modules.policyinsights", + "azure.cli.command_modules.resource" + ], + "postgres": [ + "azure.cli.command_modules.postgresql" + ], + "ppg": [ + "azure.cli.command_modules.vm" + ], + "private-link": [ + "azure.cli.command_modules.resource" + ], + "provider": [ + "azure.cli.command_modules.resource" + ], + "redis": [ + "azure.cli.command_modules.redis" + ], + "relay": [ + "azure.cli.command_modules.relay" + ], + "resource": [ + "azure.cli.command_modules.resource" + ], + "resourcemanagement": [ + "azure.cli.command_modules.resource" + ], + "rest": [ + "azure.cli.command_modules.util" + ], + "restore-point": [ + "azure.cli.command_modules.vm" + ], + "role": [ + "azure.cli.command_modules.role" + ], + "search": [ + "azure.cli.command_modules.search" + ], + "security": [ + "azure.cli.command_modules.security" + ], + "self-test": [ + "azure.cli.command_modules.profile" + ], + "servicebus": [ + "azure.cli.command_modules.servicebus" + ], + "sf": [ + "azure.cli.command_modules.servicefabric" + ], + "sig": [ + "azure.cli.command_modules.vm" + ], + "signalr": [ + "azure.cli.command_modules.signalr" + ], + "snapshot": [ + "azure.cli.command_modules.vm" + ], + "sql": [ + "azure.cli.command_modules.sql", + "azure.cli.command_modules.sqlvm" + ], + "sshkey": [ + "azure.cli.command_modules.vm" + ], + "stack": [ + "azure.cli.command_modules.resource" + ], + "staticwebapp": [ + "azure.cli.command_modules.appservice" + ], + "storage": [ + "azure.cli.command_modules.storage" + ], + "survey": [ + "azure.cli.command_modules.feedback" + ], + "synapse": [ + "azure.cli.command_modules.synapse" + ], + "tag": [ + "azure.cli.command_modules.resource" + ], + "term": [ + "azure.cli.command_modules.marketplaceordering" + ], + "ts": [ + "azure.cli.command_modules.resource" + ], + "upgrade": [ + "azure.cli.command_modules.util" + ], + "version": [ + "azure.cli.command_modules.util" + ], + "vm": [ + "azure.cli.command_modules.vm" + ], + "vmss": [ + "azure.cli.command_modules.vm" + ], + "webapp": [ + "azure.cli.command_modules.appservice", + "azure.cli.command_modules.serviceconnector" + ] + } +} diff --git a/src/azure-cli-core/azure/cli/core/commands/__init__.py b/src/azure-cli-core/azure/cli/core/commands/__init__.py index 9ab97f29b6c..a9dc9cfbc43 100644 --- a/src/azure-cli-core/azure/cli/core/commands/__init__.py +++ b/src/azure-cli-core/azure/cli/core/commands/__init__.py @@ -741,10 +741,23 @@ def _try_show_cached_help(self, args): Returns CommandResultItem if cached help was shown, None otherwise. """ - from azure.cli.core import CommandIndex + from azure.cli.core import CommandIndex, REFRESH_EXTENSION_HELP_OVERLAY_SENTINEL command_index = CommandIndex(self.cli_ctx) help_index = command_index.get_help_index() + if not help_index and command_index.needs_latest_extension_help_overlay_refresh(): + logger.debug("Top-level cached help is unavailable on latest profile. " + "Refreshing extension help overlay without full core module load.") + try: + if self.cli_ctx.invocation.data.get('command_string') is None: + self.cli_ctx.invocation.data['command_string'] = '' + # Unknown top-level command forces extension-only load path on latest profile. + self.commands_loader.load_command_table([REFRESH_EXTENSION_HELP_OVERLAY_SENTINEL]) + help_index = command_index.get_help_index() + except Exception as ex: # pylint: disable=broad-except + # Keep cached-help refresh best-effort; normal invocation flow can still continue. + logger.debug("Failed to refresh latest extension help overlay: %s", ex) + if help_index: # Display cached help using the help system self.help.show_cached_help(help_index, args) diff --git a/src/azure-cli-core/azure/cli/core/helpIndex.latest.json b/src/azure-cli-core/azure/cli/core/helpIndex.latest.json new file mode 100644 index 00000000000..616fcee61d5 --- /dev/null +++ b/src/azure-cli-core/azure/cli/core/helpIndex.latest.json @@ -0,0 +1,414 @@ +{ + "version": "2.84.0", + "cloudProfile": "latest", + "helpIndex": { + "groups": { + "account": { + "summary": "Manage Azure subscription information.", + "tags": "" + }, + "acr": { + "summary": "Manage private registries with Azure Container Registries.", + "tags": "" + }, + "ad": { + "summary": "Manage Microsoft Entra ID (formerly known as Azure Active Directory, Azure AD, AAD) entities needed for Azure role-based access control (Azure RBAC) through Microsoft Graph API.", + "tags": "" + }, + "advisor": { + "summary": "Manage Azure Advisor.", + "tags": "" + }, + "afd": { + "summary": "Manage Azure Front Door Standard/Premium.", + "tags": "" + }, + "aks": { + "summary": "Azure Kubernetes Service.", + "tags": "" + }, + "ams": { + "summary": "Manage Azure Media Services resources.", + "tags": "" + }, + "apim": { + "summary": "Manage Azure API Management services.", + "tags": "" + }, + "appconfig": { + "summary": "Manage App Configurations.", + "tags": "" + }, + "appservice": { + "summary": "Manage Appservice.", + "tags": "" + }, + "aro": { + "summary": "Manage Azure Red Hat OpenShift clusters.", + "tags": "" + }, + "backup": { + "summary": "Manage Azure Backups.", + "tags": "" + }, + "batch": { + "summary": "Manage Azure Batch.", + "tags": "" + }, + "bicep": { + "summary": "Bicep CLI command group.", + "tags": "" + }, + "billing": { + "summary": "Manage Azure Billing.", + "tags": "" + }, + "bot": { + "summary": "Manage Microsoft Azure Bot Service.", + "tags": "" + }, + "cache": { + "summary": "Commands to manage CLI objects cached using the `--defer` argument.", + "tags": "" + }, + "capacity": { + "summary": "Manage capacity.", + "tags": "" + }, + "cdn": { + "summary": "Manage Azure Content Delivery Networks (CDNs).", + "tags": "" + }, + "cloud": { + "summary": "Manage registered Azure clouds.", + "tags": "" + }, + "cognitiveservices": { + "summary": "Manage Azure Cognitive Services accounts.", + "tags": "" + }, + "compute-fleet": { + "summary": "Manage for Azure Compute Fleet.", + "tags": "[Preview]" + }, + "compute-recommender": { + "summary": "Manage sku/zone/region recommender info for compute resources.", + "tags": "" + }, + "config": { + "summary": "Manage Azure CLI configuration.", + "tags": "[Experimental]" + }, + "connection": { + "summary": "Commands to manage Service Connector local connections which allow local environment to connect Azure Resource. If you want to manage connection for compute service, please run 'az webapp/containerapp/spring connection'.", + "tags": "" + }, + "consumption": { + "summary": "Manage consumption of Azure resources.", + "tags": "[Preview]" + }, + "container": { + "summary": "Manage Azure Container Instances.", + "tags": "" + }, + "containerapp": { + "summary": "Manage Azure Container Apps.", + "tags": "" + }, + "cosmosdb": { + "summary": "Manage Azure Cosmos DB database accounts.", + "tags": "" + }, + "data-boundary": { + "summary": "Data boundary operations.", + "tags": "" + }, + "databoxedge": { + "summary": "Manage device with databoxedge.", + "tags": "" + }, + "deployment": { + "summary": "Manage Azure Resource Manager template deployment at subscription scope.", + "tags": "" + }, + "deployment-scripts": { + "summary": "Manage deployment scripts at subscription or resource group scope.", + "tags": "" + }, + "disk": { + "summary": "Manage Azure Managed Disks.", + "tags": "" + }, + "disk-access": { + "summary": "Manage disk access resources.", + "tags": "" + }, + "disk-encryption-set": { + "summary": "Disk Encryption Set resource.", + "tags": "" + }, + "dls": { + "summary": "Manage Data Lake Store accounts and filesystems.", + "tags": "[Preview]" + }, + "dms": { + "summary": "Manage Azure Data Migration Service (classic) instances.", + "tags": "" + }, + "eventgrid": { + "summary": "Manage Azure Event Grid topics, domains, domain topics, system topics partner topics, event subscriptions, system topic event subscriptions and partner topic event subscriptions.", + "tags": "" + }, + "eventhubs": { + "summary": "Eventhubs.", + "tags": "" + }, + "extension": { + "summary": "Manage and update CLI extensions.", + "tags": "" + }, + "feature": { + "summary": "Manage resource provider features.", + "tags": "" + }, + "functionapp": { + "summary": "Manage function apps. To install the Azure Functions Core tools see https://github.com/Azure/azure-functions-core-tools.", + "tags": "" + }, + "group": { + "summary": "Manage resource groups and template deployments.", + "tags": "" + }, + "hdinsight": { + "summary": "Manage HDInsight resources.", + "tags": "" + }, + "identity": { + "summary": "Manage Managed Identity.", + "tags": "" + }, + "image": { + "summary": "Manage custom virtual machine images.", + "tags": "" + }, + "iot": { + "summary": "Manage Internet of Things (IoT) assets.", + "tags": "" + }, + "keyvault": { + "summary": "Manage KeyVault keys, secrets, and certificates.", + "tags": "" + }, + "lab": { + "summary": "Manage azure devtest labs.", + "tags": "[Preview]" + }, + "lock": { + "summary": "Manage Azure locks.", + "tags": "" + }, + "logicapp": { + "summary": "Manage logic apps.", + "tags": "" + }, + "managed-cassandra": { + "summary": "Azure Managed Cassandra.", + "tags": "" + }, + "managedapp": { + "summary": "Manage template solutions provided and maintained by Independent Software Vendors (ISVs).", + "tags": "" + }, + "managedservices": { + "summary": "Manage the registration assignments and definitions in Azure.", + "tags": "" + }, + "maps": { + "summary": "Manage Azure Maps.", + "tags": "" + }, + "mariadb": { + "summary": "Manage Azure Database for MariaDB servers.", + "tags": "" + }, + "monitor": { + "summary": "Manage the Azure Monitor Service.", + "tags": "" + }, + "mysql": { + "summary": "Manage Azure Database for MySQL servers.", + "tags": "" + }, + "netappfiles": { + "summary": "Manage Azure NetApp Files (ANF) Resources.", + "tags": "" + }, + "network": { + "summary": "Manage Azure Network resources.", + "tags": "" + }, + "policy": { + "summary": "Manage resources defined and used by the Azure Policy service.", + "tags": "" + }, + "postgres": { + "summary": "Manage Azure Database for PostgreSQL.", + "tags": "" + }, + "ppg": { + "summary": "Manage Proximity Placement Groups.", + "tags": "" + }, + "private-link": { + "summary": "Private-link association CLI command group.", + "tags": "" + }, + "provider": { + "summary": "Manage resource providers.", + "tags": "" + }, + "redis": { + "summary": "Manage dedicated Redis caches for your Azure applications.", + "tags": "" + }, + "relay": { + "summary": "Manage Azure Relay Service namespaces, WCF relays, hybrid connections, and rules.", + "tags": "" + }, + "resource": { + "summary": "Manage Azure resources.", + "tags": "" + }, + "resourcemanagement": { + "summary": "Resourcemanagement CLI command group.", + "tags": "" + }, + "restore-point": { + "summary": "Manage restore point with res.", + "tags": "" + }, + "role": { + "summary": "Manage Azure role-based access control (Azure RBAC).", + "tags": "" + }, + "search": { + "summary": "Manage Search.", + "tags": "" + }, + "security": { + "summary": "Manage your security posture with Microsoft Defender for Cloud.", + "tags": "" + }, + "servicebus": { + "summary": "Servicebus.", + "tags": "" + }, + "sf": { + "summary": "Manage and administer Azure Service Fabric clusters.", + "tags": "" + }, + "sig": { + "summary": "Manage shared image gallery.", + "tags": "" + }, + "signalr": { + "summary": "Manage Azure SignalR Service.", + "tags": "" + }, + "snapshot": { + "summary": "Manage point-in-time copies of managed disks, native blobs, or other snapshots.", + "tags": "" + }, + "sql": { + "summary": "Manage Azure SQL Databases and Data Warehouses.", + "tags": "" + }, + "sshkey": { + "summary": "Manage ssh public key with vm.", + "tags": "" + }, + "stack": { + "summary": "A deployment stack is a native Azure resource type that enables you to perform operations on a resource collection as an atomic unit.", + "tags": "" + }, + "staticwebapp": { + "summary": "Manage static apps.", + "tags": "" + }, + "storage": { + "summary": "Manage Azure Cloud Storage resources.", + "tags": "" + }, + "synapse": { + "summary": "Manage and operate Synapse Workspace, Spark Pool, SQL Pool.", + "tags": "" + }, + "tag": { + "summary": "Tag Management on a resource.", + "tags": "" + }, + "term": { + "summary": "Manage marketplace agreement with marketplaceordering.", + "tags": "[Experimental]" + }, + "ts": { + "summary": "Manage template specs at subscription or resource group scope.", + "tags": "" + }, + "vm": { + "summary": "Manage Linux or Windows virtual machines.", + "tags": "" + }, + "vmss": { + "summary": "Manage groupings of virtual machines in an Azure Virtual Machine Scale Set (VMSS).", + "tags": "" + }, + "webapp": { + "summary": "Manage web apps.", + "tags": "" + } + }, + "commands": { + "configure": { + "summary": "Manage Azure CLI configuration. This command is interactive.", + "tags": "" + }, + "feedback": { + "summary": "Send feedback to the Azure CLI Team.", + "tags": "" + }, + "find": { + "summary": "I'm an AI robot, my advice is based on our Azure documentation as well as the usage patterns of Azure CLI and Azure ARM users. Using me improves Azure products and documentation.", + "tags": "" + }, + "interactive": { + "summary": "Start interactive mode. Installs the Interactive extension if not installed already.", + "tags": "[Preview]" + }, + "login": { + "summary": "Log in to Azure.", + "tags": "" + }, + "logout": { + "summary": "Log out to remove access to Azure subscriptions.", + "tags": "" + }, + "rest": { + "summary": "Invoke a custom request.", + "tags": "" + }, + "survey": { + "summary": "Take Azure CLI survey.", + "tags": "" + }, + "upgrade": { + "summary": "Upgrade Azure CLI and extensions.", + "tags": "[Preview]" + }, + "version": { + "summary": "Show the versions of Azure CLI modules and extensions in JSON format by default or format configured by --output.", + "tags": "" + } + } + } +} diff --git a/src/azure-cli-core/azure/cli/core/mock.py b/src/azure-cli-core/azure/cli/core/mock.py index 4f125a1b7ae..9bf6ca2032c 100644 --- a/src/azure-cli-core/azure/cli/core/mock.py +++ b/src/azure-cli-core/azure/cli/core/mock.py @@ -32,8 +32,8 @@ def __init__(self, commands_loader_cls=None, random_config_dir=False, **kwargs): self.env_patch = patch.dict(os.environ, {'AZURE_CONFIG_DIR': config_dir}) self.env_patch.start() - # Always copy command index to avoid initializing it again - files_to_copy = ['commandIndex.json'] + # Always copy CLI index/cache files (command, extension, help) to avoid initializing them again + files_to_copy = ['commandIndex.json', 'extensionIndex.json', 'helpIndex.json', 'extensionHelpIndex.json'] # In recording mode, copy login credentials from global config dir to the dummy config dir if os.getenv(ENV_VAR_TEST_LIVE, '').lower() == 'true': files_to_copy.extend([ diff --git a/src/azure-cli-core/azure/cli/core/tests/test_command_registration.py b/src/azure-cli-core/azure/cli/core/tests/test_command_registration.py index 0e3ec03b70a..b8361ecf30b 100644 --- a/src/azure-cli-core/azure/cli/core/tests/test_command_registration.py +++ b/src/azure-cli-core/azure/cli/core/tests/test_command_registration.py @@ -165,6 +165,9 @@ def _mock_get_extensions(): MockExtension(name=__name__ + '.Ext2CommandsLoader', preview=False, experimental=False, path=None, get_metadata=lambda: {}), MockExtension(name=__name__ + '.ExtAlwaysLoadedCommandsLoader', preview=False, experimental=False, path=None, get_metadata=lambda: {})] + def _mock_no_extensions(): + return [] + def _mock_load_command_loader(loader, args, name, prefix): class TestCommandsLoader(AzCommandsLoader): @@ -186,6 +189,13 @@ def load_command_table(self, args): self.__module__ = "azure.cli.command_modules.extra" return self.command_table + class UtilCommandsLoader(AzCommandsLoader): + def load_command_table(self, args): + with self.command_group('version', operations_tmpl='{}#TestCommandRegistration.{{}}'.format(__name__)) as g: + g.command('', 'sample_vm_get') + self.__module__ = "azure.cli.command_modules.util" + return self.command_table + # Extend existing group by adding a new command class ExtCommandsLoader(AzCommandsLoader): @@ -214,7 +224,7 @@ def load_command_table(self, args): return self.command_table if prefix == 'azure.cli.command_modules.': - command_loaders = {'hello': TestCommandsLoader, 'extra': Test2CommandsLoader} + command_loaders = {'hello': TestCommandsLoader, 'extra': Test2CommandsLoader, 'util': UtilCommandsLoader} else: command_loaders = {'azext_hello1': ExtCommandsLoader, 'azext_hello2': Ext2CommandsLoader, @@ -305,6 +315,8 @@ def test_command_index(self): from azure.cli.core import CommandIndex, __version__ cli = DummyCli() + # This test validates legacy local index rebuild behavior, not latest packaged-index shortcuts. + cli.cloud.profile = "2019-03-01-hybrid" loader = cli.commands_loader command_index = CommandIndex(cli) @@ -429,6 +441,267 @@ def update_and_check_index(): del INDEX[CommandIndex._COMMAND_INDEX_CLOUD_PROFILE] del INDEX[CommandIndex._COMMAND_INDEX] + @mock.patch('importlib.import_module', _mock_import_lib) + @mock.patch('pkgutil.iter_modules', _mock_iter_modules) + @mock.patch('azure.cli.core.commands._load_command_loader', _mock_load_command_loader) + @mock.patch('azure.cli.core.extension.get_extensions', _mock_no_extensions) + def test_command_index_uses_packaged_latest_without_seeding(self): + from azure.cli.core._session import INDEX + from azure.cli.core import CommandIndex, __version__ + + cli = DummyCli() + loader = cli.commands_loader + + # Simulate no local index metadata. This is the no-seeding path. + INDEX[CommandIndex._COMMAND_INDEX_VERSION] = "" + INDEX[CommandIndex._COMMAND_INDEX_CLOUD_PROFILE] = "" + INDEX[CommandIndex._COMMAND_INDEX] = {} + + packaged_index = { + CommandIndex._COMMAND_INDEX_VERSION: __version__, + CommandIndex._COMMAND_INDEX_CLOUD_PROFILE: cli.cloud.profile, + CommandIndex._COMMAND_INDEX: { + 'hello': ['azure.cli.command_modules.hello'], + 'extra': ['azure.cli.command_modules.extra'] + } + } + + with mock.patch.object(CommandIndex, '_load_packaged_command_index', return_value=packaged_index): + cmd_tbl = loader.load_command_table(["hello", "mod-only"]) + + # Uses packaged index directly and doesn't require local index seeding. + self.assertEqual(list(cmd_tbl), ['hello mod-only', 'hello overridden']) + self.assertEqual(INDEX[CommandIndex._COMMAND_INDEX], {}) + + @mock.patch('importlib.import_module', _mock_import_lib) + @mock.patch('pkgutil.iter_modules', _mock_iter_modules) + @mock.patch('azure.cli.core.commands._load_command_loader', _mock_load_command_loader) + @mock.patch('azure.cli.core.extension.get_extension_modname', _mock_get_extension_modname) + @mock.patch('azure.cli.core.extension.get_extensions', _mock_get_extensions) + def test_command_index_loads_all_extensions_when_overlay_missing(self): + from azure.cli.core._session import INDEX, EXTENSION_INDEX + from azure.cli.core import CommandIndex, __version__ + + cli = DummyCli() + loader = cli.commands_loader + cli.invocation = cli.invocation_cls(cli_ctx=cli, + commands_loader_cls=cli.commands_loader_cls, + parser_cls=cli.parser_cls, + help_cls=cli.help_cls) + + INDEX[CommandIndex._COMMAND_INDEX_VERSION] = "" + INDEX[CommandIndex._COMMAND_INDEX_CLOUD_PROFILE] = "" + INDEX[CommandIndex._COMMAND_INDEX] = {} + + packaged_index = { + CommandIndex._COMMAND_INDEX_VERSION: __version__, + CommandIndex._COMMAND_INDEX_CLOUD_PROFILE: cli.cloud.profile, + CommandIndex._COMMAND_INDEX: { + 'hello': ['azure.cli.command_modules.hello'], + 'extra': ['azure.cli.command_modules.extra'] + } + } + + with mock.patch.object(CommandIndex, '_load_packaged_command_index', return_value=packaged_index): + cmd_tbl = loader.load_command_table(["hello", "mod-only"]) + + # Missing overlay triggers loading all extensions, but avoids full module rebuild. + self.assertEqual(list(cmd_tbl), ['hello mod-only', 'hello overridden', 'hello ext-only']) + self.assertEqual(INDEX[CommandIndex._COMMAND_INDEX], {}) + self.assertEqual(EXTENSION_INDEX[CommandIndex._COMMAND_INDEX_VERSION], __version__) + self.assertEqual(EXTENSION_INDEX[CommandIndex._COMMAND_INDEX_CLOUD_PROFILE], cli.cloud.profile) + self.assertIn('hello', EXTENSION_INDEX[CommandIndex._COMMAND_INDEX]) + self.assertIn('azext_hello1', EXTENSION_INDEX[CommandIndex._COMMAND_INDEX]['hello']) + self.assertIn('azext_hello2', EXTENSION_INDEX[CommandIndex._COMMAND_INDEX]['hello']) + + @mock.patch('importlib.import_module', _mock_import_lib) + @mock.patch('pkgutil.iter_modules', _mock_iter_modules) + @mock.patch('azure.cli.core.commands._load_command_loader', _mock_load_command_loader) + @mock.patch('azure.cli.core.extension.get_extension_modname', _mock_get_extension_modname) + @mock.patch('azure.cli.core.extension.get_extensions', _mock_get_extensions) + def test_command_index_latest_uppercase_help_triggers_full_core_reload(self): + from azure.cli.core._session import INDEX, EXTENSION_INDEX + from azure.cli.core import CommandIndex, __version__ + + cli = DummyCli() + loader = cli.commands_loader + + INDEX[CommandIndex._COMMAND_INDEX_VERSION] = "" + INDEX[CommandIndex._COMMAND_INDEX_CLOUD_PROFILE] = "" + INDEX[CommandIndex._COMMAND_INDEX] = {} + EXTENSION_INDEX[CommandIndex._COMMAND_INDEX_VERSION] = "" + EXTENSION_INDEX[CommandIndex._COMMAND_INDEX_CLOUD_PROFILE] = "" + EXTENSION_INDEX[CommandIndex._COMMAND_INDEX] = {} + + packaged_index = { + CommandIndex._COMMAND_INDEX_VERSION: __version__, + CommandIndex._COMMAND_INDEX_CLOUD_PROFILE: cli.cloud.profile, + CommandIndex._COMMAND_INDEX: { + 'hello': ['azure.cli.command_modules.hello'], + 'extra': ['azure.cli.command_modules.extra'] + } + } + + with mock.patch.object(CommandIndex, '_load_packaged_command_index', return_value=packaged_index): + cmd_tbl = loader.load_command_table(["HELP"]) + + # HELP should be treated the same as help and must not short-circuit core module reload. + self.assertIn('hello mod-only', cmd_tbl) + self.assertIn('extra final', cmd_tbl) + self.assertIn('hello', INDEX[CommandIndex._COMMAND_INDEX]) + self.assertIn('extra', INDEX[CommandIndex._COMMAND_INDEX]) + + @mock.patch('importlib.import_module', _mock_import_lib) + @mock.patch('pkgutil.iter_modules', _mock_iter_modules) + @mock.patch('azure.cli.core.commands._load_command_loader', _mock_load_command_loader) + @mock.patch('azure.cli.core.extension.get_extension_modname', _mock_get_extension_modname) + @mock.patch('azure.cli.core.extension.get_extensions', _mock_get_extensions) + def test_command_index_latest_unknown_non_core_skips_full_core_reload(self): + from azure.cli.core._session import INDEX, EXTENSION_INDEX + from azure.cli.core import CommandIndex, __version__ + + cli = DummyCli() + loader = cli.commands_loader + + INDEX[CommandIndex._COMMAND_INDEX_VERSION] = "" + INDEX[CommandIndex._COMMAND_INDEX_CLOUD_PROFILE] = "" + INDEX[CommandIndex._COMMAND_INDEX] = {} + EXTENSION_INDEX[CommandIndex._COMMAND_INDEX_VERSION] = "" + EXTENSION_INDEX[CommandIndex._COMMAND_INDEX_CLOUD_PROFILE] = "" + EXTENSION_INDEX[CommandIndex._COMMAND_INDEX] = {} + + packaged_index = { + CommandIndex._COMMAND_INDEX_VERSION: __version__, + CommandIndex._COMMAND_INDEX_CLOUD_PROFILE: cli.cloud.profile, + CommandIndex._COMMAND_INDEX: { + 'hello': ['azure.cli.command_modules.hello'], + 'extra': ['azure.cli.command_modules.extra'] + } + } + + with mock.patch.object(CommandIndex, '_load_packaged_command_index', return_value=packaged_index): + cmd_tbl = loader.load_command_table(["foobar", "list"]) + + # Unknown non-core top-level command should try extensions without rebuilding all core modules. + self.assertNotIn('hello mod-only', cmd_tbl) + self.assertNotIn('extra final', cmd_tbl) + self.assertEqual(INDEX[CommandIndex._COMMAND_INDEX], {}) + self.assertEqual(EXTENSION_INDEX[CommandIndex._COMMAND_INDEX_VERSION], __version__) + self.assertEqual(EXTENSION_INDEX[CommandIndex._COMMAND_INDEX_CLOUD_PROFILE], cli.cloud.profile) + self.assertIn('hello', EXTENSION_INDEX[CommandIndex._COMMAND_INDEX]) + + @mock.patch('importlib.import_module', _mock_import_lib) + @mock.patch('pkgutil.iter_modules', _mock_iter_modules) + @mock.patch('azure.cli.core.commands._load_command_loader', _mock_load_command_loader) + @mock.patch('azure.cli.core.extension.get_extension_modname', _mock_get_extension_modname) + @mock.patch('azure.cli.core.extension.get_extensions', _mock_get_extensions) + def test_command_index_blends_packaged_with_extension_overlay(self): + from azure.cli.core._session import INDEX, EXTENSION_INDEX + from azure.cli.core import CommandIndex, __version__ + + cli = DummyCli() + loader = cli.commands_loader + + # Local command index is empty; packaged core index + extensionIndex overlay should be blended. + INDEX[CommandIndex._COMMAND_INDEX_VERSION] = "" + INDEX[CommandIndex._COMMAND_INDEX_CLOUD_PROFILE] = "" + INDEX[CommandIndex._COMMAND_INDEX] = {} + + EXTENSION_INDEX[CommandIndex._COMMAND_INDEX_VERSION] = __version__ + EXTENSION_INDEX[CommandIndex._COMMAND_INDEX_CLOUD_PROFILE] = cli.cloud.profile + EXTENSION_INDEX[CommandIndex._COMMAND_INDEX] = { + 'hello': ['azext_hello1', 'azext_hello2'] + } + + packaged_index = { + CommandIndex._COMMAND_INDEX_VERSION: __version__, + CommandIndex._COMMAND_INDEX_CLOUD_PROFILE: cli.cloud.profile, + CommandIndex._COMMAND_INDEX: { + 'hello': ['azure.cli.command_modules.hello'], + 'extra': ['azure.cli.command_modules.extra'] + } + } + + with mock.patch.object(CommandIndex, '_load_packaged_command_index', return_value=packaged_index): + cmd_tbl = loader.load_command_table(["hello", "mod-only"]) + + # Extension commands are loaded through the overlay without rebuilding all modules. + self.assertEqual(list(cmd_tbl), ['hello mod-only', 'hello overridden', 'hello ext-only']) + self.assertEqual(INDEX[CommandIndex._COMMAND_INDEX], {}) + + EXTENSION_INDEX[CommandIndex._COMMAND_INDEX_VERSION] = "" + EXTENSION_INDEX[CommandIndex._COMMAND_INDEX_CLOUD_PROFILE] = "" + EXTENSION_INDEX[CommandIndex._COMMAND_INDEX] = {} + + def test_packaged_command_index_file_schema(self): + from azure.cli.core import CommandIndex, __version__ + + cli = DummyCli() + command_index = CommandIndex(cli) + packaged_index = command_index._load_packaged_command_index() # pylint: disable=protected-access + + self.assertIsNotNone(packaged_index) + self.assertEqual(packaged_index.get(CommandIndex._COMMAND_INDEX_VERSION), __version__) + self.assertEqual(packaged_index.get(CommandIndex._COMMAND_INDEX_CLOUD_PROFILE), 'latest') + self.assertTrue(packaged_index.get(CommandIndex._COMMAND_INDEX)) + + @mock.patch('importlib.import_module', _mock_import_lib) + @mock.patch('pkgutil.iter_modules', _mock_iter_modules) + @mock.patch('azure.cli.core.commands._load_command_loader', _mock_load_command_loader) + @mock.patch('azure.cli.core.extension.get_extension_modname', _mock_get_extension_modname) + @mock.patch('azure.cli.core.extension.get_extensions', _mock_get_extensions) + def test_command_index_uses_packaged_for_version_with_extensions(self): + from azure.cli.core._session import INDEX + from azure.cli.core import CommandIndex, __version__ + + cli = DummyCli() + loader = cli.commands_loader + + INDEX[CommandIndex._COMMAND_INDEX_VERSION] = "" + INDEX[CommandIndex._COMMAND_INDEX_CLOUD_PROFILE] = "" + INDEX[CommandIndex._COMMAND_INDEX] = {} + + packaged_index = { + CommandIndex._COMMAND_INDEX_VERSION: __version__, + CommandIndex._COMMAND_INDEX_CLOUD_PROFILE: cli.cloud.profile, + CommandIndex._COMMAND_INDEX: { + 'version': ['azure.cli.command_modules.util'] + } + } + + with mock.patch.object(CommandIndex, '_load_packaged_command_index', return_value=packaged_index): + cmd_tbl = loader.load_command_table(["version"]) + + self.assertIn('version', cmd_tbl) + # No full rebuild should happen for this fast path. + self.assertEqual(INDEX[CommandIndex._COMMAND_INDEX], {}) + + @mock.patch('importlib.import_module', _mock_import_lib) + @mock.patch('pkgutil.iter_modules', _mock_iter_modules) + @mock.patch('azure.cli.core.commands._load_command_loader', _mock_load_command_loader) + @mock.patch('azure.cli.core.extension.get_extension_modname', _mock_get_extension_modname) + @mock.patch('azure.cli.core.extension.get_extensions', _mock_get_extensions) + def test_command_index_non_latest_uses_local_mechanism(self): + from azure.cli.core._session import INDEX + from azure.cli.core import CommandIndex, __version__ + + cli = DummyCli() + loader = cli.commands_loader + + with mock.patch.object(cli.cloud, "profile", "2019-03-01-hybrid"): + # Valid local index for non-latest profile should be used directly. + INDEX[CommandIndex._COMMAND_INDEX_VERSION] = __version__ + INDEX[CommandIndex._COMMAND_INDEX_CLOUD_PROFILE] = "2019-03-01-hybrid" + INDEX[CommandIndex._COMMAND_INDEX] = { + 'hello': ['azure.cli.command_modules.hello'] + } + + with mock.patch.object(CommandIndex, '_load_packaged_command_index', + side_effect=AssertionError('Packaged index should not be used for non-latest')): + cmd_tbl = loader.load_command_table(["hello", "mod-only"]) + + self.assertEqual(list(cmd_tbl), ['hello mod-only', 'hello overridden']) + @mock.patch('importlib.import_module', _mock_import_lib) @mock.patch('pkgutil.iter_modules', _mock_iter_modules) @mock.patch('azure.cli.core.commands._load_command_loader', _mock_load_command_loader) @@ -465,6 +738,8 @@ def test_command_index_positional_argument(self): from azure.cli.core import CommandIndex cli = DummyCli() + # Use a non-latest profile so command index rebuild/usage follows local index semantics. + cli.cloud.profile = "2019-03-01-hybrid" loader = cli.commands_loader index = CommandIndex() index.invalidate() diff --git a/src/azure-cli-core/azure/cli/core/tests/test_help.py b/src/azure-cli-core/azure/cli/core/tests/test_help.py index 15528e71440..3ec55758e90 100644 --- a/src/azure-cli-core/azure/cli/core/tests/test_help.py +++ b/src/azure-cli-core/azure/cli/core/tests/test_help.py @@ -164,7 +164,15 @@ def tearDown(self): shutil.rmtree(self._tempdirName) self.helps.clear() # Invalidate help cache to prevent test data from polluting production cache - from azure.cli.core._session import INDEX + from azure.cli.core._session import EXTENSION_HELP_INDEX, HELP_INDEX, INDEX + if 'helpIndex' in EXTENSION_HELP_INDEX: + del EXTENSION_HELP_INDEX['helpIndex'] + if 'version' in EXTENSION_HELP_INDEX: + del EXTENSION_HELP_INDEX['version'] + if 'cloudProfile' in EXTENSION_HELP_INDEX: + del EXTENSION_HELP_INDEX['cloudProfile'] + if 'helpIndex' in HELP_INDEX: + del HELP_INDEX['helpIndex'] if 'helpIndex' in INDEX: del INDEX['helpIndex'] @@ -541,9 +549,9 @@ def test_help_cache_extraction(self): self.assertEqual(commands['login']['summary'], 'Log in to Azure') def test_help_cache_storage_and_retrieval(self): - """Test that help cache is stored and can be retrieved.""" - from azure.cli.core import CommandIndex - from azure.cli.core._session import INDEX + """Test non-latest help cache remains local and retrievable.""" + from azure.cli.core import CommandIndex, __version__ + from azure.cli.core._session import HELP_INDEX test_help_data = { 'groups': { @@ -554,10 +562,13 @@ def test_help_cache_storage_and_retrieval(self): } } - command_index = CommandIndex(self.test_cli) - command_index.set_help_index(test_help_data) + with mock.patch.object(self.test_cli.cloud, 'profile', '2019-03-01-hybrid'): + command_index = CommandIndex(self.test_cli) + command_index.version = __version__ + command_index.cloud_profile = '2019-03-01-hybrid' + command_index.set_help_index(test_help_data) - retrieved = INDEX.get('helpIndex') + retrieved = HELP_INDEX.get('helpIndex') self.assertIsNotNone(retrieved) self.assertIn('groups', retrieved) @@ -568,17 +579,130 @@ def test_help_cache_storage_and_retrieval(self): def test_help_cache_invalidation(self): """Test that cache is invalidated correctly.""" from azure.cli.core import CommandIndex - from azure.cli.core._session import INDEX + from azure.cli.core._session import EXTENSION_HELP_INDEX, HELP_INDEX test_help_data = {'root': {'groups': {}, 'commands': {}}} command_index = CommandIndex(self.test_cli) command_index.set_help_index(test_help_data) - self.assertIn('helpIndex', INDEX) + self.assertIn('helpIndex', HELP_INDEX) command_index.invalidate() - self.assertEqual(INDEX.get('helpIndex'), {}) + self.assertEqual(HELP_INDEX.get('helpIndex'), {}) + self.assertEqual(EXTENSION_HELP_INDEX.get('helpIndex'), {}) + + def test_help_cache_legacy_command_index_is_ignored(self): + """Test legacy helpIndex payload in commandIndex.json is not migrated for non-latest.""" + from azure.cli.core import CommandIndex, __version__ + from azure.cli.core._session import HELP_INDEX, INDEX + + test_help_data = { + 'groups': {'legacy-group': {'summary': 'Legacy summary', 'tags': ''}}, + 'commands': {'legacy-cmd': {'summary': 'Legacy command', 'tags': ''}} + } + + with mock.patch.object(self.test_cli.cloud, 'profile', '2019-03-01-hybrid'): + command_index = CommandIndex(self.test_cli) + INDEX[CommandIndex._COMMAND_INDEX_VERSION] = __version__ + INDEX[CommandIndex._COMMAND_INDEX_CLOUD_PROFILE] = '2019-03-01-hybrid' + INDEX['helpIndex'] = test_help_data + + cached_help = command_index.get_help_index() + + self.assertIsNone(cached_help) + self.assertNotEqual(HELP_INDEX.get('helpIndex'), test_help_data) + self.assertEqual(INDEX.get('helpIndex'), test_help_data) + + def test_packaged_help_index_file_schema(self): + """Test packaged helpIndex.latest.json schema and metadata.""" + from azure.cli.core import CommandIndex, __version__ + + command_index = CommandIndex(self.test_cli) + packaged_help_index = command_index._load_packaged_help_index() # pylint: disable=protected-access + + self.assertIsNotNone(packaged_help_index) + self.assertIsInstance(packaged_help_index, dict) + self.assertIn('groups', packaged_help_index) + self.assertIn('commands', packaged_help_index) + self.assertEqual(command_index.version, __version__) + + def test_help_index_uses_packaged_latest_without_local_index(self): + """Test latest profile uses packaged help index when local command/help index is invalid.""" + from azure.cli.core import CommandIndex + from azure.cli.core._session import HELP_INDEX, INDEX + + command_index = CommandIndex(self.test_cli) + + # Simulate missing local command/help cache metadata. + INDEX[CommandIndex._COMMAND_INDEX_VERSION] = "" + INDEX[CommandIndex._COMMAND_INDEX_CLOUD_PROFILE] = "" + INDEX[CommandIndex._COMMAND_INDEX] = {} + HELP_INDEX[CommandIndex._HELP_INDEX] = {} + + packaged_help_data = { + 'groups': {'vm': {'summary': 'Manage VMs.', 'tags': ''}}, + 'commands': {'version': {'summary': 'Show version.', 'tags': ''}} + } + + with mock.patch.object(CommandIndex, '_load_packaged_help_index', return_value=packaged_help_data), \ + mock.patch.object(CommandIndex, '_has_non_always_loaded_extensions', return_value=False): + help_index = command_index.get_help_index() + + self.assertEqual(help_index, packaged_help_data) + + def test_help_index_latest_missing_overlay_with_extensions_triggers_refresh(self): + """Test latest profile returns None to force refresh when extension help overlay is unavailable.""" + from azure.cli.core import CommandIndex + from azure.cli.core._session import EXTENSION_HELP_INDEX, HELP_INDEX, INDEX + + command_index = CommandIndex(self.test_cli) + + INDEX[CommandIndex._COMMAND_INDEX_VERSION] = "" + INDEX[CommandIndex._COMMAND_INDEX_CLOUD_PROFILE] = "" + INDEX[CommandIndex._COMMAND_INDEX] = {} + HELP_INDEX[CommandIndex._HELP_INDEX] = {} + EXTENSION_HELP_INDEX[CommandIndex._COMMAND_INDEX_VERSION] = "" + EXTENSION_HELP_INDEX[CommandIndex._COMMAND_INDEX_CLOUD_PROFILE] = "" + EXTENSION_HELP_INDEX[CommandIndex._HELP_INDEX] = {} + + packaged_help_data = { + 'groups': {'vm': {'summary': 'Manage VMs.', 'tags': ''}}, + 'commands': {'version': {'summary': 'Show version.', 'tags': ''}} + } + + with mock.patch.object(CommandIndex, '_load_packaged_help_index', return_value=packaged_help_data), \ + mock.patch.object(CommandIndex, '_has_non_always_loaded_extensions', return_value=True): + help_index = command_index.get_help_index() + + self.assertIsNone(help_index) + + def test_help_index_latest_blends_packaged_with_extension_overlay(self): + """Test latest profile blends packaged help with extension help overlay.""" + from azure.cli.core import CommandIndex, __version__ + from azure.cli.core._session import EXTENSION_HELP_INDEX + + command_index = CommandIndex(self.test_cli) + + EXTENSION_HELP_INDEX[CommandIndex._COMMAND_INDEX_VERSION] = __version__ + EXTENSION_HELP_INDEX[CommandIndex._COMMAND_INDEX_CLOUD_PROFILE] = self.test_cli.cloud.profile + EXTENSION_HELP_INDEX[CommandIndex._HELP_INDEX] = { + 'groups': {'ext-group': {'summary': 'Extension group summary.', 'tags': ''}}, + 'commands': {'ext-cmd': {'summary': 'Extension command summary.', 'tags': ''}} + } + + packaged_help_data = { + 'groups': {'vm': {'summary': 'Manage VMs.', 'tags': ''}}, + 'commands': {'version': {'summary': 'Show version.', 'tags': ''}} + } + + with mock.patch.object(CommandIndex, '_load_packaged_help_index', return_value=packaged_help_data): + help_index = command_index.get_help_index() + + self.assertIn('vm', help_index['groups']) + self.assertIn('ext-group', help_index['groups']) + self.assertIn('version', help_index['commands']) + self.assertIn('ext-cmd', help_index['commands']) def test_show_cached_help_output(self): """Test that cached help is displayed correctly.""" @@ -619,6 +743,32 @@ def test_show_cached_help_output(self): finally: sys.stdout = sys.__stdout__ + def test_try_show_cached_help_refreshes_latest_extension_overlay(self): + """Test top-level cached help retries after refreshing latest extension help overlay.""" + from azure.cli.core import CommandIndex, REFRESH_EXTENSION_HELP_OVERLAY_SENTINEL + + invoker = self.test_cli.invocation_cls( + cli_ctx=self.test_cli, + commands_loader_cls=self.test_cli.commands_loader_cls, + parser_cls=self.test_cli.parser_cls, + help_cls=self.test_cli.help_cls) + self.test_cli.invocation = invoker + + refreshed_help_data = { + 'groups': {'vm': {'summary': 'Manage VMs.', 'tags': ''}}, + 'commands': {'version': {'summary': 'Show version.', 'tags': ''}} + } + + with mock.patch.object(CommandIndex, 'get_help_index', side_effect=[None, refreshed_help_data]), \ + mock.patch.object(CommandIndex, 'needs_latest_extension_help_overlay_refresh', return_value=True), \ + mock.patch.object(invoker.commands_loader, 'load_command_table') as mock_load_cmd_table, \ + mock.patch.object(invoker.help, 'show_cached_help') as mock_show_cached_help: + result = invoker._try_show_cached_help(['--help', '--debug']) + + self.assertIsNotNone(result) + mock_load_cmd_table.assert_called_once_with([REFRESH_EXTENSION_HELP_OVERLAY_SENTINEL]) + mock_show_cached_help.assert_called_once_with(refreshed_help_data, ['--help', '--debug']) + # create a temporary file in the temp dir. Return the path of the file. def _create_new_temp_file(self, data, suffix=""): with tempfile.NamedTemporaryFile(mode='w', dir=self._tempdirName, delete=False, suffix=suffix) as f: diff --git a/src/azure-cli-core/setup.py b/src/azure-cli-core/setup.py index 90bec1c5f19..2c037a399bb 100644 --- a/src/azure-cli-core/setup.py +++ b/src/azure-cli-core/setup.py @@ -84,5 +84,5 @@ packages=find_packages(exclude=["*.tests", "*.tests.*", "tests.*", "tests", "azure", "azure.cli"]), install_requires=DEPENDENCIES, python_requires='>=3.10.0', - package_data={'azure.cli.core': ['auth/landing_pages/*.html']} + package_data={'azure.cli.core': ['auth/landing_pages/*.html', 'commandIndex.latest.json', 'helpIndex.latest.json']} )