diff --git a/.travis.yml b/.travis.yml index 0b9eda9fd4c..d5713d69d4c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ language: python services: - docker install: - - pip install pylint==1.9.2 flake8==3.5.0 requests wheel==0.30.0 -q + - pip install pylint==1.9.2 flake8==3.5.0 requests wheel==0.30.0 azdev -q jobs: include: - stage: precheck diff --git a/scripts/ci/test_static.sh b/scripts/ci/test_static.sh index c35d30b2486..22fbf57e1b8 100755 --- a/scripts/ci/test_static.sh +++ b/scripts/ci/test_static.sh @@ -1,25 +1,17 @@ #!/usr/bin/env bash set -e -proc_number=`python -c 'import multiprocessing; print(multiprocessing.cpu_count())'` +export AZDEV_CLI_REPO_PATH='_NONE_' +export AZDEV_EXT_REPO_PATHS=$(pwd) -AZURE_SDK_AUTOGEN_FILES=$(egrep -lir --exclude-dir site-packages "# Code generated by Microsoft \(R\) AutoRest Code Generator." ./src) -FLAKE8_EXCLUDES=$(echo "$AZURE_SDK_AUTOGEN_FILES" | sed -e s/$/,/g | tr -d '\n') -PYLINT_EXCLUDES=$(echo "$AZURE_SDK_AUTOGEN_FILES" | sed -e s=\./src/=src/=g -e 's%/[^/]*\ %\ %g' -e 's%/[^/]*$%%g' | sort -u | sed -e s/$/,/g | tr -d '\n') - -# Run pylint/flake8 on extensions -echo "Running pylint on extensions..." -pylint ./src/*/azext_*/ --ignore=$PYLINT_EXCLUDES,vendored_sdks --ignore-patterns=test_* --rcfile=./pylintrc -j $proc_number - -echo "Pylint OK." -echo "Running flake8 on extensions..." -flake8 --statistics --append-config=./.flake8 ./src/*/azext_*/ -echo "Flake8 OK." +azdev setup -r $AZDEV_CLI_REPO_PATH +azdev style --pylint # Run pylint/flake8 on CI files pylint ./scripts/ci/*.py --rcfile=./pylintrc flake8 --append-config=./.flake8 ./scripts/ci/*.py # Other static checks +azdev verify license python ./scripts/ci/verify_codeowners.py -python ./scripts/ci/verify_license.py + diff --git a/scripts/ci/update_index.py b/scripts/ci/update_index.py deleted file mode 100644 index 28f10f99177..00000000000 --- a/scripts/ci/update_index.py +++ /dev/null @@ -1,68 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -import hashlib -import json -import re -import sys -import tempfile - -from util import get_ext_metadata, get_whl_from_url - -NAME_REGEX = r'.*/([^/]*)-\d+.\d+.\d+' - - -def get_sha256sum(a_file): - sha256 = hashlib.sha256() - with open(a_file, 'rb') as f: - sha256.update(f.read()) - return sha256.hexdigest() - - -def main(): - - # Get extension WHL from URL - whl_path = None - try: - whl_path = sys.argv[1] - except IndexError: - pass - if not whl_path or not whl_path.endswith('.whl') or not whl_path.startswith('https:'): - raise ValueError('incorrect usage: update_script ') - - # Extract the extension name - try: - extension_name = re.findall(NAME_REGEX, whl_path)[0] - extension_name = extension_name.replace('_', '-') - except IndexError: - raise ValueError('unable to parse extension name') - - extensions_dir = tempfile.mkdtemp() - ext_dir = tempfile.mkdtemp(dir=extensions_dir) - whl_cache_dir = tempfile.mkdtemp() - whl_cache = {} - ext_file = get_whl_from_url(whl_path, extension_name, whl_cache_dir, whl_cache) - - with open('./src/index.json', 'r') as infile: - curr_index = json.loads(infile.read()) - - try: - entry = curr_index['extensions'][extension_name] - except IndexError: - raise ValueError('{} not found in index.json'.format(extension_name)) - - entry[0]['downloadUrl'] = whl_path - entry[0]['sha256Digest'] = get_sha256sum(ext_file) - entry[0]['filename'] = whl_path.split('/')[-1] - entry[0]['metadata'] = get_ext_metadata(ext_dir, ext_file, extension_name) - - # update index and write back to file - curr_index['extensions'][extension_name] = entry - with open('./src/index.json', 'w') as outfile: - outfile.write(json.dumps(curr_index, indent=4, sort_keys=True)) - - -if __name__ == '__main__': - main() diff --git a/scripts/ci/verify_license.py b/scripts/ci/verify_license.py deleted file mode 100644 index 363f80584a7..00000000000 --- a/scripts/ci/verify_license.py +++ /dev/null @@ -1,47 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -from __future__ import print_function - -import os -import sys -import argparse - -from util import get_repo_root - -REPO_ROOT = get_repo_root() -SRC_DIR = os.path.join(REPO_ROOT, 'src') - -LICENSE_HEADER = ("Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT License. " - "See License.txt in the project root for license information.") - - -def main(args): - excluded_paths = args.excluded_paths - excluded_paths.append('env') - excluded_paths = tuple([os.path.join(REPO_ROOT, relative_path) for relative_path in excluded_paths]) - - files_without_header = [] - for current_dir, _, files in os.walk(get_repo_root()): - if current_dir.startswith(excluded_paths): - continue - file_itr = (os.path.join(current_dir, p) for p in files if p.endswith('.py')) - for python_file in file_itr: - with open(python_file, 'r') as f: - file_text = f.read().replace('\r\n', '\n') - file_text = file_text.replace('\n#', '') - if file_text and (LICENSE_HEADER not in file_text): - files_without_header.append(os.path.join(current_dir, python_file)) - - if files_without_header: - print("Error: The following files don't have the required license headers: \n{}".format( - '\n'.join(files_without_header)), file=sys.stderr) - sys.exit(1) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('excluded_paths', nargs='*') - main(parser.parse_args()) diff --git a/scripts/ci/verify_modified_index.sh b/scripts/ci/verify_modified_index.sh index 06fdd60c2e1..6b624fc5fcd 100755 --- a/scripts/ci/verify_modified_index.sh +++ b/scripts/ci/verify_modified_index.sh @@ -2,9 +2,8 @@ set -ex # Install CLI & Dev Tools -echo "Installing azure-cli-dev-tools and azure-cli..." +echo "Installing azure-cli..." pip install --pre azure-cli --extra-index-url https://azurecliprod.blob.core.windows.net/edge -pip install -e "git+https://github.com/Azure/azure-cli@dev#egg=azure-cli-dev-tools&subdirectory=tools" -q echo "Installed." az --version set +x @@ -32,9 +31,9 @@ while read line; do fi set -e echo "Load all commands" - azdev verify load-all + az self-test echo "Running linter" - azdev cli-lint --ci --extensions $ext + azdev linter $ext az extension remove -n $ext echo $ext "extension has been removed." done <<< "$modified_extensions"