diff --git a/.azure-pipelines/templates/azdev_setup.yml b/.azure-pipelines/templates/azdev_setup.yml index 1ce70a65237..0bf1143c985 100644 --- a/.azure-pipelines/templates/azdev_setup.yml +++ b/.azure-pipelines/templates/azdev_setup.yml @@ -14,12 +14,10 @@ steps: # clone azure-cli git clone -q --single-branch -b dev https://github.com/Azure/azure-cli.git ../azure-cli python -m pip install -U pip - pip install azdev - azdev --version + pip install --trusted-host azurecliprod.blob.core.windows.net "https://azurecliprod.blob.core.windows.net/beta/azdev-0.2.3b1-py3-none-any.whl" azdev setup -c $CLI_REPO_PATH -r $CLI_EXT_REPO_PATH --debug - # Installing setuptools with a version higher than 70.0.0 will not generate metadata.json - pip install setuptools==70.0.0 pip list -v + azdev --version az --version displayName: 'azdev setup' env: diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 20fb5bb194f..c78a0de9f90 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -76,10 +76,11 @@ jobs: displayName: 'Use Python 3.12' inputs: versionSpec: 3.12 + - template: .azure-pipelines/templates/azdev_setup.yml - bash: | #!/usr/bin/env bash set -ev - pip install wheel==0.30.0 requests packaging setuptools + source ./env/bin/activate export CI="ADO" python ./scripts/ci/test_index.py -v displayName: "Verify Extensions Index" @@ -104,8 +105,6 @@ jobs: inputs: versionSpec: '$(python.version)' - template: .azure-pipelines/templates/azdev_setup.yml - - bash: pip install wheel==0.30.0 - displayName: 'Install wheel==0.30.0' - bash: | #!/usr/bin/env bash set -ev @@ -254,5 +253,9 @@ jobs: displayName: 'Use Python 3.x' inputs: versionSpec: 3.x + - template: .azure-pipelines/templates/azdev_setup.yml - bash: | + #!/usr/bin/env bash + set -ev + source ./env/bin/activate python scripts/ci/test_init.py -v diff --git a/scripts/ci/test_index.py b/scripts/ci/test_index.py index 093ebaa7388..237bb9cc2ac 100755 --- a/scripts/ci/test_index.py +++ b/scripts/ci/test_index.py @@ -14,13 +14,13 @@ import json import logging import os +import re import shutil import tempfile import unittest from packaging import version from util import SRC_PATH -from wheel.install import WHEEL_INFO_RE from util import get_ext_metadata, get_whl_from_url, get_index_data @@ -32,6 +32,14 @@ logger.addHandler(ch) +# copy from wheel==0.30.0 +WHEEL_INFO_RE = re.compile( + r"""^(?P(?P.+?)(-(?P\d.+?))?) + ((-(?P\d.*?))?-(?P.+?)-(?P.+?)-(?P.+?) + \.whl|\.dist-info)$""", + re.VERBOSE).match + + def get_sha256sum(a_file): sha256 = hashlib.sha256() with open(a_file, 'rb') as f: @@ -85,8 +93,6 @@ def test_extension_filenames(self): "Extension name mismatch in extensions['{}']. " "Found an extension in the list with name " "{}".format(ext_name, item['metadata']['name'])) - # Due to https://github.com/pypa/wheel/issues/235 we prevent whls built with 0.31.0 or greater. - # 0.29.0, 0.30.0 are the two previous versions before that release. parsed_filename = WHEEL_INFO_RE(item['filename']) p = parsed_filename.groupdict() self.assertTrue(p.get('name'), "Can't get name for {}".format(item['filename'])) @@ -196,15 +202,27 @@ def test_metadata(self): else: raise ex - # Due to https://github.com/pypa/wheel/issues/195 we prevent whls built with 0.31.0 or greater. - # 0.29.0, 0.30.0 are the two previous versions before that release. supported_generators = ['bdist_wheel (0.29.0)', 'bdist_wheel (0.30.0)'] self.assertIn(metadata.get('generator'), supported_generators, "{}: 'generator' should be one of {}. " - "Build the extension with a different version of the 'wheel' package " - "(e.g. `pip install wheel==0.30.0`). " - "This is due to https://github.com/pypa/wheel/issues/195".format(ext_name, + "Please install the latest azdev." + "(e.g. `pip install azdev==0.2.3b1 && `)." + "And update the extension index with the latest azdev." + "(e.g. `azdev extension update-index xxx.whl`).".format(ext_name, supported_generators)) + + # Ignore generator which is hardcoded in azdev/operations/extensions/metadata.py. + metadata.pop('generator', None) + item['metadata'].pop('generator', None) + + # Ignore test_requires which is defined in setup.py. + # e.g: https://hciarcvmsstorage.z13.web.core.windows.net/cli-extensions/stack_hci_vm-1.7.8-py3-none-any.whl + metadata['extensions']['python.details'].pop('document_names', None) + item['metadata']['extensions']['python.details'].pop('document_names', None) + + # Ignore test_requires which is defined in setup.py. + item['metadata'].pop('test_requires', None) + self.assertDictEqual(metadata, item['metadata'], "Metadata for {} in index doesn't match the expected of: \n" "{}".format(item['filename'], json.dumps(metadata, indent=2, sort_keys=True, diff --git a/scripts/ci/util.py b/scripts/ci/util.py index ffc7d54797b..e3f76cec366 100644 --- a/scripts/ci/util.py +++ b/scripts/ci/util.py @@ -10,6 +10,7 @@ import json import zipfile +from azdev.operations.extensions.metadata import pkginfo_to_dict from subprocess import check_output logger = logging.getLogger(__name__) @@ -53,8 +54,7 @@ def _get_azext_metadata(ext_dir): def get_ext_metadata(ext_dir, ext_file, ext_name): - # Modification of https://github.com/Azure/azure-cli/blob/dev/src/azure-cli-core/azure/cli/core/extension.py#L89 - WHL_METADATA_FILENAME = 'metadata.json' + generated_metadata = pkginfo_to_dict(ext_file) zip_ref = zipfile.ZipFile(ext_file, 'r') zip_ref.extractall(ext_dir) zip_ref.close() @@ -71,10 +71,7 @@ def get_ext_metadata(ext_dir, ext_file, ext_name): for dist_info_dirname in dist_info_dirs: parsed_dist_info_dir = WHEEL_INFO_RE(dist_info_dirname) if parsed_dist_info_dir and parsed_dist_info_dir.groupdict().get('name') == ext_name.replace('-', '_'): - whl_metadata_filepath = os.path.join(ext_dir, dist_info_dirname, WHL_METADATA_FILENAME) - if os.path.isfile(whl_metadata_filepath): - with open(whl_metadata_filepath) as f: - metadata.update(json.load(f)) + metadata.update(generated_metadata) return metadata