From a4829d9441e906ab2eb967a16fa8f1ebb3e097c6 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Mon, 7 Apr 2025 15:26:20 +0800 Subject: [PATCH 1/9] upgrade setuptools --- .azure-pipelines/templates/azdev_setup.yml | 4 +--- azure-pipelines.yml | 5 ++--- scripts/ci/test_index.py | 21 +++++++++++++-------- scripts/ci/util.py | 9 +++------ 4 files changed, 19 insertions(+), 20 deletions(-) diff --git a/.azure-pipelines/templates/azdev_setup.yml b/.azure-pipelines/templates/azdev_setup.yml index 1ce70a65237..7e8915f05b8 100644 --- a/.azure-pipelines/templates/azdev_setup.yml +++ b/.azure-pipelines/templates/azdev_setup.yml @@ -14,11 +14,9 @@ steps: # clone azure-cli git clone -q --single-branch -b dev https://github.com/Azure/azure-cli.git ../azure-cli python -m pip install -U pip - pip install azdev + pip install --trusted-host azurecliprod.blob.core.windows.net "https://azurecliprod.blob.core.windows.net/beta/azdev-0.2.3b1-py3-none-any.whl" azdev --version azdev setup -c $CLI_REPO_PATH -r $CLI_EXT_REPO_PATH --debug - # Installing setuptools with a version higher than 70.0.0 will not generate metadata.json - pip install setuptools==70.0.0 pip list -v az --version displayName: 'azdev setup' diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 20fb5bb194f..8ce4621b672 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -76,10 +76,11 @@ jobs: displayName: 'Use Python 3.12' inputs: versionSpec: 3.12 + - template: .azure-pipelines/templates/azdev_setup.yml - bash: | #!/usr/bin/env bash set -ev - pip install wheel==0.30.0 requests packaging setuptools + pip install requests packaging setuptools export CI="ADO" python ./scripts/ci/test_index.py -v displayName: "Verify Extensions Index" @@ -104,8 +105,6 @@ jobs: inputs: versionSpec: '$(python.version)' - template: .azure-pipelines/templates/azdev_setup.yml - - bash: pip install wheel==0.30.0 - displayName: 'Install wheel==0.30.0' - bash: | #!/usr/bin/env bash set -ev diff --git a/scripts/ci/test_index.py b/scripts/ci/test_index.py index 093ebaa7388..2bff550f58e 100755 --- a/scripts/ci/test_index.py +++ b/scripts/ci/test_index.py @@ -14,13 +14,13 @@ import json import logging import os +import re import shutil import tempfile import unittest from packaging import version from util import SRC_PATH -from wheel.install import WHEEL_INFO_RE from util import get_ext_metadata, get_whl_from_url, get_index_data @@ -32,6 +32,14 @@ logger.addHandler(ch) +# copy from wheel==0.30.0 +WHEEL_INFO_RE = re.compile( + r"""^(?P(?P.+?)(-(?P\d.+?))?) + ((-(?P\d.*?))?-(?P.+?)-(?P.+?)-(?P.+?) + \.whl|\.dist-info)$""", + re.VERBOSE).match + + def get_sha256sum(a_file): sha256 = hashlib.sha256() with open(a_file, 'rb') as f: @@ -85,8 +93,6 @@ def test_extension_filenames(self): "Extension name mismatch in extensions['{}']. " "Found an extension in the list with name " "{}".format(ext_name, item['metadata']['name'])) - # Due to https://github.com/pypa/wheel/issues/235 we prevent whls built with 0.31.0 or greater. - # 0.29.0, 0.30.0 are the two previous versions before that release. parsed_filename = WHEEL_INFO_RE(item['filename']) p = parsed_filename.groupdict() self.assertTrue(p.get('name'), "Can't get name for {}".format(item['filename'])) @@ -196,14 +202,13 @@ def test_metadata(self): else: raise ex - # Due to https://github.com/pypa/wheel/issues/195 we prevent whls built with 0.31.0 or greater. - # 0.29.0, 0.30.0 are the two previous versions before that release. supported_generators = ['bdist_wheel (0.29.0)', 'bdist_wheel (0.30.0)'] self.assertIn(metadata.get('generator'), supported_generators, "{}: 'generator' should be one of {}. " - "Build the extension with a different version of the 'wheel' package " - "(e.g. `pip install wheel==0.30.0`). " - "This is due to https://github.com/pypa/wheel/issues/195".format(ext_name, + "Please install the latest azdev." + "(e.g. `pip install azdev==0.2.3b1 && `)." + "And update the extension index with the latest azdev." + "(e.g. `azdev extension update-index xxx.whl`).".format(ext_name, supported_generators)) self.assertDictEqual(metadata, item['metadata'], "Metadata for {} in index doesn't match the expected of: \n" diff --git a/scripts/ci/util.py b/scripts/ci/util.py index ffc7d54797b..e3f76cec366 100644 --- a/scripts/ci/util.py +++ b/scripts/ci/util.py @@ -10,6 +10,7 @@ import json import zipfile +from azdev.operations.extensions.metadata import pkginfo_to_dict from subprocess import check_output logger = logging.getLogger(__name__) @@ -53,8 +54,7 @@ def _get_azext_metadata(ext_dir): def get_ext_metadata(ext_dir, ext_file, ext_name): - # Modification of https://github.com/Azure/azure-cli/blob/dev/src/azure-cli-core/azure/cli/core/extension.py#L89 - WHL_METADATA_FILENAME = 'metadata.json' + generated_metadata = pkginfo_to_dict(ext_file) zip_ref = zipfile.ZipFile(ext_file, 'r') zip_ref.extractall(ext_dir) zip_ref.close() @@ -71,10 +71,7 @@ def get_ext_metadata(ext_dir, ext_file, ext_name): for dist_info_dirname in dist_info_dirs: parsed_dist_info_dir = WHEEL_INFO_RE(dist_info_dirname) if parsed_dist_info_dir and parsed_dist_info_dir.groupdict().get('name') == ext_name.replace('-', '_'): - whl_metadata_filepath = os.path.join(ext_dir, dist_info_dirname, WHL_METADATA_FILENAME) - if os.path.isfile(whl_metadata_filepath): - with open(whl_metadata_filepath) as f: - metadata.update(json.load(f)) + metadata.update(generated_metadata) return metadata From 43a4f6adf8578fd47f561c6b814a7a884ce0f28e Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Mon, 7 Apr 2025 16:36:23 +0800 Subject: [PATCH 2/9] minor fix --- azure-pipelines.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 8ce4621b672..0e42587c224 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -253,5 +253,9 @@ jobs: displayName: 'Use Python 3.x' inputs: versionSpec: 3.x + - template: .azure-pipelines/templates/azdev_setup.yml - bash: | + #!/usr/bin/env bash + set -ev + source ./env/bin/activate python scripts/ci/test_init.py -v From 631a9122fcd88cec7dd5aa568c8726f793c3f7d2 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Tue, 8 Apr 2025 17:26:18 +0800 Subject: [PATCH 3/9] minor fix --- .azure-pipelines/templates/azdev_setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/templates/azdev_setup.yml b/.azure-pipelines/templates/azdev_setup.yml index 7e8915f05b8..0bf1143c985 100644 --- a/.azure-pipelines/templates/azdev_setup.yml +++ b/.azure-pipelines/templates/azdev_setup.yml @@ -15,9 +15,9 @@ steps: git clone -q --single-branch -b dev https://github.com/Azure/azure-cli.git ../azure-cli python -m pip install -U pip pip install --trusted-host azurecliprod.blob.core.windows.net "https://azurecliprod.blob.core.windows.net/beta/azdev-0.2.3b1-py3-none-any.whl" - azdev --version azdev setup -c $CLI_REPO_PATH -r $CLI_EXT_REPO_PATH --debug pip list -v + azdev --version az --version displayName: 'azdev setup' env: From 1258a67658fc5a1d48cecb7a8bb34e57e705454b Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Tue, 8 Apr 2025 18:27:40 +0800 Subject: [PATCH 4/9] minor fix --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 0e42587c224..c78a0de9f90 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -80,7 +80,7 @@ jobs: - bash: | #!/usr/bin/env bash set -ev - pip install requests packaging setuptools + source ./env/bin/activate export CI="ADO" python ./scripts/ci/test_index.py -v displayName: "Verify Extensions Index" From 3c9657a52509fbf4cb43195e834d46e95ec9324c Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Wed, 9 Apr 2025 10:46:22 +0800 Subject: [PATCH 5/9] minor fix --- scripts/ci/test_index.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/ci/test_index.py b/scripts/ci/test_index.py index 2bff550f58e..aeed0daaa9f 100755 --- a/scripts/ci/test_index.py +++ b/scripts/ci/test_index.py @@ -210,6 +210,9 @@ def test_metadata(self): "And update the extension index with the latest azdev." "(e.g. `azdev extension update-index xxx.whl`).".format(ext_name, supported_generators)) + # Ignore test_requires which is defined in setup.py, + # as this information cannot be extracted from the whl pkg. + item['metadata'].pop('test_requires', None) self.assertDictEqual(metadata, item['metadata'], "Metadata for {} in index doesn't match the expected of: \n" "{}".format(item['filename'], json.dumps(metadata, indent=2, sort_keys=True, From 15f37e8609fd495a607a6a4352730e2341d31684 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Wed, 9 Apr 2025 12:50:44 +0800 Subject: [PATCH 6/9] minor fix --- scripts/ci/test_index.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/ci/test_index.py b/scripts/ci/test_index.py index aeed0daaa9f..cc2ec68b0b0 100755 --- a/scripts/ci/test_index.py +++ b/scripts/ci/test_index.py @@ -210,6 +210,10 @@ def test_metadata(self): "And update the extension index with the latest azdev." "(e.g. `azdev extension update-index xxx.whl`).".format(ext_name, supported_generators)) + # Ignore generator which is hardcoded in azdev/operations/extensions/metadata.py. + metadata.pop('generator', None) + item['metadata'].pop('generator', None) + # Ignore test_requires which is defined in setup.py, # as this information cannot be extracted from the whl pkg. item['metadata'].pop('test_requires', None) From 3e4816ff65b0c3f30a382c900045afff01e662cf Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Wed, 9 Apr 2025 13:33:30 +0800 Subject: [PATCH 7/9] minor fix --- scripts/ci/test_index.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/ci/test_index.py b/scripts/ci/test_index.py index cc2ec68b0b0..8c73092a69e 100755 --- a/scripts/ci/test_index.py +++ b/scripts/ci/test_index.py @@ -214,6 +214,11 @@ def test_metadata(self): metadata.pop('generator', None) item['metadata'].pop('generator', None) + # Ignore document_names which is inconsistent with whl pkg. + # e.g: https://hciarcvmsstorage.z13.web.core.windows.net/cli-extensions/stack_hci_vm-1.7.8-py3-none-any.whl + metadata['extensions'].pop('document_names', None) + item['metadata']['extensions'].pop('document_names', None) + # Ignore test_requires which is defined in setup.py, # as this information cannot be extracted from the whl pkg. item['metadata'].pop('test_requires', None) From ff03e0df76be20e98ab49862e650f05d2cdcb6f5 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Wed, 9 Apr 2025 15:36:44 +0800 Subject: [PATCH 8/9] minor fix --- scripts/ci/test_index.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/ci/test_index.py b/scripts/ci/test_index.py index 8c73092a69e..994c6c7b6f0 100755 --- a/scripts/ci/test_index.py +++ b/scripts/ci/test_index.py @@ -210,18 +210,20 @@ def test_metadata(self): "And update the extension index with the latest azdev." "(e.g. `azdev extension update-index xxx.whl`).".format(ext_name, supported_generators)) + # Ignore generator which is hardcoded in azdev/operations/extensions/metadata.py. metadata.pop('generator', None) item['metadata'].pop('generator', None) # Ignore document_names which is inconsistent with whl pkg. # e.g: https://hciarcvmsstorage.z13.web.core.windows.net/cli-extensions/stack_hci_vm-1.7.8-py3-none-any.whl - metadata['extensions'].pop('document_names', None) - item['metadata']['extensions'].pop('document_names', None) + metadata['extensions']['python.details'].pop('document_names', None) + item['metadata']['extensions']['python.details'].pop('document_names', None) # Ignore test_requires which is defined in setup.py, # as this information cannot be extracted from the whl pkg. item['metadata'].pop('test_requires', None) + self.assertDictEqual(metadata, item['metadata'], "Metadata for {} in index doesn't match the expected of: \n" "{}".format(item['filename'], json.dumps(metadata, indent=2, sort_keys=True, From 4a06ffd282b39caaea9943d04c3bcebf43caea62 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Wed, 9 Apr 2025 21:58:21 +0800 Subject: [PATCH 9/9] minor fix --- scripts/ci/test_index.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/ci/test_index.py b/scripts/ci/test_index.py index 994c6c7b6f0..237bb9cc2ac 100755 --- a/scripts/ci/test_index.py +++ b/scripts/ci/test_index.py @@ -215,13 +215,12 @@ def test_metadata(self): metadata.pop('generator', None) item['metadata'].pop('generator', None) - # Ignore document_names which is inconsistent with whl pkg. + # Ignore test_requires which is defined in setup.py. # e.g: https://hciarcvmsstorage.z13.web.core.windows.net/cli-extensions/stack_hci_vm-1.7.8-py3-none-any.whl metadata['extensions']['python.details'].pop('document_names', None) item['metadata']['extensions']['python.details'].pop('document_names', None) - # Ignore test_requires which is defined in setup.py, - # as this information cannot be extracted from the whl pkg. + # Ignore test_requires which is defined in setup.py. item['metadata'].pop('test_requires', None) self.assertDictEqual(metadata, item['metadata'],