[New Command] add azdev generate-docs (#109)

* WIP - implement sphinx logic for doc gen.

* More work. azdev generate-docs -h runs successfully. However currently having issues retrieving doc source map.

* Add missing comma to setup.py

* Add logic to generate docs from extensions. Doc source map now included in core cli docs. Sphinx commands now always reads in files, doesn't cache input info. Todo: split command into two. Ensure that output is the same between command and existing approach.

* Add logic to use slightly different configuration for extensions.

* Split generate-docs command into cli and extension command.

* Remove unnecessary Argument. Updated help text and displayed information.

* Update license header to correct license header.

* Remove operations/help.py in lieu of operations/__init__.py. Move recent changes over.

* Add newlines to pass license_verify.py

* Style fixes

Style fix.

* Fix style issues. Primarily flake8.

* Minor changes

* azdev extension generate-docs overrides smartquotes and sets it to False to be consistent with the existing extension doc gen script.

Author: adewaleo

azdev/commands.py

@@ -57,10 +57,11 @@ def operation_group(name):
         g.command('remove', 'remove_extension_repo')
         g.command('list', 'list_extension_repos')
 
-    # TODO: implement
-    # with CommandGroup(self, operation_group('help')) as g:
-    #     g.command('generate', 'generate_help_xml')
-    #     g.command('convert', 'convert_help_to_yaml')
+    with CommandGroup(self, 'cli', operation_group('help')) as g:
+        g.command('generate-docs', 'generate_cli_ref_docs')
+
+    with CommandGroup(self, 'extension', operation_group('help')) as g:
+        g.command('generate-docs', 'generate_extension_ref_docs')
 
     # TODO: implement
     # with CommandGroup(self, 'coverage', command_path) as g:

azdev/help.py

@@ -54,6 +54,12 @@
 """
 
 
+helps['cli generate-docs'] = """
+    short-summary: >
+       Generate reference docs for CLI commands.
+"""
+
+
 helps['configure'] = """
     short-summary: Configure azdev for use without installing anything.
 """
@@ -241,3 +247,10 @@
     short-summary: >
         List the repositories that will be searched for in-development extensions.
 """
+
+helps['extension generate-docs'] = """
+    short-summary: >
+       Generate reference docs for CLI extensions commands.
+    long-summary: >
+        This commands installs the extensions in a temporary directory and sets it as the extensions dir when generating reference docs.
+"""

azdev/operations/help.py

@@ -0,0 +1,241 @@
+# -----------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# -----------------------------------------------------------------------------
+
+from __future__ import print_function
+
+import os
+import sys
+import copy
+import json
+import shutil
+import tempfile
+
+from subprocess import check_call, CalledProcessError
+
+from knack.util import CLIError
+from knack.log import get_logger
+
+from azdev.utilities import (
+    display, heading, subheading,
+    get_cli_repo_path, get_path_table
+)
+
+from azdev.utilities.tools import require_azure_cli
+from azure.cli.core.extension.operations import list_available_extensions  # pylint: disable=import-error
+
+DOC_MAP_NAME = 'doc_source_map.json'
+HELP_FILE_NAME = '_help.py'
+DOC_SOURCE_MAP_PATH = os.path.join('doc', 'sphinx', 'azhelpgen', DOC_MAP_NAME)
+
+_logger = get_logger(__name__)
+
+
+def check_document_map():
+
+    heading('Verify Document Map')
+
+    cli_repo = get_cli_repo_path()
+
+    map_path = os.path.join(cli_repo, DOC_SOURCE_MAP_PATH)
+    help_files_in_map = _get_help_files_in_map(map_path)
+    help_files_not_found = _map_help_files_not_found(cli_repo, help_files_in_map)
+    help_files_to_add_to_map = _help_files_not_in_map(cli_repo, help_files_in_map)
+
+    subheading('Results')
+    if help_files_not_found or help_files_to_add_to_map:
+        error_lines = []
+        error_lines.append('Errors whilst verifying {}!'.format(DOC_MAP_NAME))
+        if help_files_not_found:
+            error_lines.append('The following files are in {} but do not exist:'.format(DOC_MAP_NAME))
+            error_lines += help_files_not_found
+        if help_files_to_add_to_map:
+            error_lines.append('The following files should be added to {}:'.format(DOC_MAP_NAME))
+            error_lines += help_files_to_add_to_map
+        error_msg = '\n'.join(error_lines)
+        raise CLIError(error_msg)
+    display('Verified {} OK.'.format(DOC_MAP_NAME))
+
+
+def _get_help_files_in_map(map_path):
+    with open(map_path) as json_file:
+        json_data = json.load(json_file)
+        return [os.path.normpath(x) for x in list(json_data.values())]
+
+
+def _map_help_files_not_found(cli_repo, help_files_in_map):
+    missing_files = []
+    for path in help_files_in_map:
+        if not os.path.isfile(os.path.normpath(os.path.join(cli_repo, path))):
+            missing_files.append(path)
+    return missing_files
+
+
+def _help_files_not_in_map(cli_repo, help_files_in_map):
+    not_in_map = []
+    for _, path in get_path_table()['mod'].items():
+        help_path = os.path.join(path, HELP_FILE_NAME)
+        help_path = help_path.replace(cli_repo.lower() + os.sep, '')
+        if help_path in help_files_in_map or not os.path.isfile(help_path):
+            continue
+        not_in_map.append(help_path)
+    return not_in_map
+
+
+def generate_cli_ref_docs(output_dir=None, output_type=None):
+    # require that azure cli installed
+    require_azure_cli()
+    output_dir = _process_ref_doc_output_dir(output_dir)
+
+    heading('Generate CLI Reference Docs')
+    display("Docs will be placed in {}.".format(output_dir))
+
+    # Generate documentation for all comamnds
+    _call_sphinx_build(output_type, output_dir)
+
+    display("\nThe {} files are in {}".format(output_type, output_dir))
+
+
+def generate_extension_ref_docs(output_dir=None, output_type=None):
+    # require that azure cli installed
+    require_azure_cli()
+    output_dir = _process_ref_doc_output_dir(output_dir)
+
+    heading('Generate CLI Extensions Reference Docs')
+    display("Docs will be placed in {}.".format(output_dir))
+
+    display("Generating Docs for public extensions. Installed extensions will not be affected...")
+    _generate_ref_docs_for_public_exts(output_type, output_dir)
+
+    display("\nThe {} files are in {}".format(output_type, output_dir))
+
+
+def _process_ref_doc_output_dir(output_dir):
+    # handle output_dir
+    # if non specified, store in "_build" in the current working directory
+    if not output_dir:
+        output_dir = tempfile.mkdtemp(prefix="doc_output_")
+    # ensure output_dir exists otherwise create it
+    output_dir = os.path.abspath(output_dir)
+    if not os.path.exists(output_dir):
+        existing_path = os.path.dirname(output_dir)
+        base_dir = os.path.basename(output_dir)
+        if not os.path.exists(existing_path):
+            raise CLIError("Cannot create output directory {} in non-existent path {}."
+                           .format(base_dir, existing_path))
+
+        os.mkdir(output_dir)
+    return output_dir
+
+
+def _generate_ref_docs_for_public_exts(output_type, base_output_dir):
+    # TODO: this shouldn't define the env key, but should reference it from a central place in the cli repo.
+    ENV_KEY_AZURE_EXTENSION_DIR = 'AZURE_EXTENSION_DIR'
+
+    extensions_url_tups = _get_available_extension_urls()
+    if not extensions_url_tups:
+        raise CLIError("Failed to retrieve public extensions.")
+
+    temp_dir = tempfile.mkdtemp(prefix="temp_whl_ext_dir")
+    _logger.debug("Created temp directory to store downloaded whl files: %s", temp_dir)
+
+    try:
+        for name, file_name, download_url in extensions_url_tups:
+            # for every compatible public extensions
+            # download the whl file
+            whl_file_name = _get_whl_from_url(download_url, file_name, temp_dir)
+
+            # install the whl file in a new temp directory
+            installed_ext_dir = tempfile.mkdtemp(prefix="temp_extension_dir_", dir=temp_dir)
+            _logger.debug("Created temp directory %s to use as the extension installation dir for %s extension.",
+                          installed_ext_dir, name)
+            pip_cmd = [sys.executable, '-m', 'pip', 'install', '--target',
+                       os.path.join(installed_ext_dir, 'extension'),
+                       whl_file_name, '--disable-pip-version-check', '--no-cache-dir']
+            display('Executing "{}"'.format(' '.join(pip_cmd)))
+            check_call(pip_cmd)
+
+            # set the directory as the extension directory in the environment used to call sphinx-build
+            env = os.environ.copy()
+            env[ENV_KEY_AZURE_EXTENSION_DIR] = installed_ext_dir
+            # generate documentation for installed extensions
+
+            ext_output_dir = os.path.join(base_output_dir, name)
+            os.makedirs(ext_output_dir)
+            _call_sphinx_build(output_type, ext_output_dir, for_extensions_alone=True, call_env=env,
+                               msg="\nGenerating ref docs for {}".format(name))
+    finally:
+        # finally delete the temp dir
+        shutil.rmtree(temp_dir)
+        _logger.debug("Deleted temp whl extension directory: %s", temp_dir)
+
+
+def _call_sphinx_build(builder_name, output_dir, for_extensions_alone=False, call_env=None, msg=""):
+    conf_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'refdoc')
+
+    if for_extensions_alone:
+        source_dir = os.path.abspath(os.path.join(conf_dir, 'extension_docs'))
+    else:
+        source_dir = os.path.abspath(os.path.join(conf_dir, 'cli_docs'))
+
+    try:
+        opts = ['-E', '-b', builder_name, '-c', conf_dir]
+        args = [source_dir, output_dir]
+        if for_extensions_alone:
+            # apparently the configuration in extensions and core CLI differed in this way. This is only cosmetic
+            # set smartquotes to false. Due to a bug, one has to use "0" instead "False"
+            opts.extend(["-D", "smartquotes=0"])
+
+        sphinx_cmd = ['sphinx-build'] + opts + args
+        display("sphinx cmd: {}".format(" ".join(sphinx_cmd)))
+        display(msg)
+        # call sphinx-build
+        check_call(sphinx_cmd, stdout=sys.stdout, stderr=sys.stderr, env=call_env)
+
+    except CalledProcessError:
+        raise CLIError("Doc generation failed.")
+
+
+# Todo, this would be unnecessary if list_available_extensions has a switch for including download urls....
+def _get_available_extension_urls():
+    """ Get download urls for all the CLI extensions compatible with the installed development CLI.
+
+    :return: list of 3-tuples in the form of '(extension_name, extension_file_name, extensions_download_url)'
+    """
+    all_pub_extensions = list_available_extensions(show_details=True)
+    compatible_extensions = list_available_extensions()
+
+    name_url_tups = []
+
+    for ext in compatible_extensions:
+        old_length = len(name_url_tups)
+        ext_name, ext_version = ext["name"], ext["version"]
+
+        for ext_info in all_pub_extensions[ext_name]:
+            if ext_version == ext_info["metadata"]["version"]:
+                name_url_tups.append((ext_name, ext_info["filename"], ext_info["downloadUrl"]))
+                break
+
+        if old_length == len(name_url_tups):
+            _logger.warning("'%s' has no versions compatible with the installed CLI's version", ext_name)
+
+    return name_url_tups
+
+
+def _get_whl_from_url(url, filename, tmp_dir, whl_cache=None):
+    if not whl_cache:
+        whl_cache = {}
+    if url in whl_cache:
+        return whl_cache[url]
+    import requests
+    r = requests.get(url, stream=True)
+    assert r.status_code == 200, "Request to {} failed with {}".format(url, r.status_code)
+    ext_file = os.path.join(tmp_dir, filename)
+    with open(ext_file, 'wb') as f:
+        for chunk in r.iter_content(chunk_size=1024):
+            if chunk:  # ignore keep-alive new chunks
+                f.write(chunk)
+    whl_cache[url] = ext_file
+    return ext_file

azdev/operations/help/__init__.py

@@ -0,0 +1,5 @@
+# -----------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# -----------------------------------------------------------------------------