diff --git a/scripts/live_test/CLITest.yml b/scripts/live_test/CLITest.yml index 8bd19444b56..ccf93a3097f 100644 --- a/scripts/live_test/CLITest.yml +++ b/scripts/live_test/CLITest.yml @@ -4,7 +4,7 @@ # extensions: ALL extensions # vm ext-account: certain targets -name: CLI TEST RUN $(USER_TARGET) $(USER_LIVE) $(USER_USERNAME) $(Date:yyyyMMdd)$(Rev:.r) +name: CLI TEST RUN ${{ parameters.userTarget }} ${{ parameters.userLive }} ${{ parameters.userUsername }} $(Date:yyyyMMdd)$(Rev:.r) trigger: branches: @@ -14,6 +14,56 @@ trigger: variables: - template: ${{ variables.Pipeline.Workspace }}/.azure-pipelines/templates/variables.yml +parameters: +- name: maxParallel + displayName: Max Parallel Count + type: number + default: 5 +- name: pythonVersion + displayName: Python Version + type: string + default: '3.12' +- name: staticWebUrl + displayName: Static Web URL + type: string + default: 'https://clilivetestsa.z13.web.core.windows.net/' +- name: userBranch + displayName: User Branch + type: string + default: 'live-test' +- name: userBranchExt + displayName: User Branch for Extensions + type: string + default: 'main' +- name: userLive + displayName: User Live Test Flag + type: string + default: '--live' +- name: userParallelism + displayName: User Parallelism + type: number + default: 8 +- name: userRepo + displayName: User Repo + type: string + default: 'https://github.com/Azure/azure-cli.git' +- name: userRepoExt + displayName: User Repo for Extensions + type: string + default: 'https://github.com/Azure/azure-cli-extensions.git' +- name: userTarget + displayName: User Target + type: string + default: 'all' +- name: userToken + displayName: User Token + type: string + default: ' ' +- name: userUsername + displayName: User Username + type: string + default: ' ' + #schedules: #- cron: "0 18 * * 5" # displayName: Weekly Friday test @@ -31,10 +81,10 @@ stages: - job: LiveTest displayName: Live Test # Sometimes the live test will be stuck forever, so the timeout cannot be set to 0, it is recommended to set it to twice the maximum time - timeoutInMinutes: 600 + timeoutInMinutes: 1200 strategy: # If the maxParallel is too large, the resource group will reach the limit of 980, so it is recommended that the maxParallel should not exceed 30. - maxParallel: 30 + maxParallel: ${{ parameters.maxParallel }} matrix: acr: Target: acr @@ -452,7 +502,7 @@ stages: name: ${{ variables.ubuntu_pool }} steps: - bash: | - if [[ "$(USER_TARGET)" =~ "$(Target)" || ("$(USER_TARGET)" == "main" && ! "$(Target)" =~ ^ext-.*) || ("$(USER_TARGET)" == "extensions" && "$(Target)" =~ ^ext-.*) || ("$(USER_TARGET)" == "ALL" || "$(USER_TARGET)" == "all" || "$(USER_TARGET)" == "") ]]; then + if [[ "${{ parameters.userTarget }}" =~ "$(Target)" || ("${{ parameters.userTarget }}" == "main" && ! "$(Target)" =~ ^ext-.*) || ("${{ parameters.userTarget }}" == "extensions" && "$(Target)" =~ ^ext-.*) || ("${{ parameters.userTarget }}" == "ALL" || "${{ parameters.userTarget }}" == "all" || "${{ parameters.userTarget }}" == "") ]]; then echo "Match!" echo "##vso[task.setvariable variable=Match]1" echo "##vso[task.setvariable variable=platform]ubuntulatest" @@ -463,13 +513,13 @@ stages: - task: UsePythonVersion@0 condition: and(succeededOrFailed(), eq(variables.Match, '1')) inputs: - versionSpec: '$(PYTHON_VERSION)' + versionSpec: '${{ parameters.pythonVersion }}' addToPath: true architecture: 'x64' - task: AzureCLI@2 displayName: 'Run live test' inputs: - azureSubscription: $(AZURE_CLI_LIVE_TEST_CONNECTED_SERVICE) + connectedServiceNameARM: $(azure-cli-live-test-bami-connected-service) scriptType: bash scriptLocation: inlineScript inlineScript: | @@ -484,21 +534,22 @@ stages: # print variables echo "PLATFORM: $(PLATFORM)" - echo "USER_TARGET: $(USER_TARGET)" - echo "USER_REPO: $(USER_REPO)" - echo "USER_REPO_EXT: $(USER_REPO_EXT)" - echo "USER_BRANCH: $(USER_BRANCH)" - echo "USER_BRANCH_EXT: $(USER_BRANCH_EXT)" + echo "USER_TARGET: ${{ parameters.userTarget }}" + echo "USER_REPO: ${{ parameters.userRepo }}" + echo "USER_REPO_EXT: ${{ parameters.userRepoExt }}" + echo "USER_BRANCH: ${{ parameters.userBranch }}" + echo "USER_BRANCH_EXT: ${{ parameters.userBranchExt }}" + echo "USER_USERNAME: ${{ parameters.userUsername }}" # Whether commit changes - if [[ "$(USER_USERNAME)" != "" && "$(USER_TOKEN)" != "" ]]; then - REPO="$(USER_REPO)" - EXT_REPO="$(USER_REPO_EXT)" + if [[ "${{ parameters.userUsername }}" != " " && "${{ parameters.userToken }}" != " " ]]; then + REPO="${{ parameters.userRepo }}" + EXT_REPO="${{ parameters.userRepoExt }}" # Pass username and token so that we can commit changes - git clone -b $(USER_BRANCH) ${REPO:0:8}$(USER_USERNAME):$(USER_TOKEN)@${REPO:8} - git clone -b $(USER_BRANCH_EXT) ${EXT_REPO:0:8}$(USER_USERNAME):$(USER_TOKEN)@${EXT_REPO:8} + git clone -b ${{ parameters.userBranch }} ${REPO:0:8}${{ parameters.userUsername }}:${{ parameters.userToken }}@${REPO:8} + git clone -b ${{ parameters.userBranchExt }} ${EXT_REPO:0:8}${{ parameters.userUsername }}:${{ parameters.userToken }}@${EXT_REPO:8} else - git clone -b $(USER_BRANCH) $(USER_REPO) - git clone -b $(USER_BRANCH_EXT) $(USER_REPO_EXT) + git clone -b ${{ parameters.userBranch }} ${{ parameters.userRepo }} + git clone -b ${{ parameters.userBranchExt }} ${{ parameters.userRepoExt }} fi python -m venv env @@ -528,44 +579,43 @@ stages: python $workDir/s/scripts/compact_aaz.py az -v - # az login --service-principal --username $(AZURECLITESTUSER) --password $(AZURECLITESTPASSWORD) --tenant $(AZURECLITESTTENANTID) - az account set -s $(AZURECLITESTSUBSCRIPTIONID) + az account set -s $(azure-cli-live-test-bami-sub-id) # Clean policy python $workDir/s/scripts/live_test/clean_policy.py echo "Run tests" # Whether commit changes - if [[ "$(USER_USERNAME)" != "" && "$(USER_TOKEN)" != "" ]]; then + if [[ "${{ parameters.userUsername }}" != " " && "${{ parameters.userToken }}" != " " ]]; then echo "Commit mode" - azdev test ${FinalTarget} --no-exitfirst -a "-n $(USER_PARALLELISM)" - azdev test ${FinalTarget} --live --lf --xml-path test_results.parallel.xml --no-exitfirst -a "-n $(USER_PARALLELISM) --json-report --json-report-summary --json-report-file=$(Target).$(platform).report.parallel.json --html=$(Target).$(platform).report.parallel.html --self-contained-html --capture=sys" + azdev test ${FinalTarget} --no-exitfirst -a "-n ${{ parameters.userParallelism }}" + azdev test ${FinalTarget} --live --lf --xml-path test_results.parallel.xml --no-exitfirst -a "-n ${{ parameters.userParallelism }} --json-report --json-report-summary --json-report-file=$(Target).$(platform).report.parallel.json --html=$(Target).$(platform).report.parallel.html --self-contained-html --capture=sys" elif [[ "$serial_modules" =~ "$FinalTarget" ]]; then echo "Series mode" - azdev test ${FinalTarget} $(USER_LIVE) --xml-path test_results.sequential.xml --no-exitfirst -a "-n 1 --json-report --json-report-summary --json-report-file=$(Target).$(platform).report.sequential.json --html=$(Target).$(platform).report.sequential.html --self-contained-html --capture=sys" + azdev test ${FinalTarget} ${{ parameters.userLive }} --xml-path test_results.sequential.xml --no-exitfirst -a "-n 1 --json-report --json-report-summary --json-report-file=$(Target).$(platform).report.sequential.json --html=$(Target).$(platform).report.sequential.html --self-contained-html --capture=sys" else echo "Normal mode" # Sequential - azdev test ${FinalTarget} $(USER_LIVE) --mark serial --xml-path test_results.sequential.xml --no-exitfirst -a "-n 1 --json-report --json-report-summary --json-report-file=$(Target).$(platform).report.sequential.json --html=$(Target).$(platform).report.sequential.html --self-contained-html --capture=sys" + azdev test ${FinalTarget} ${{ parameters.userLive }} --mark serial --xml-path test_results.sequential.xml --no-exitfirst -a "-n 1 --json-report --json-report-summary --json-report-file=$(Target).$(platform).report.sequential.json --html=$(Target).$(platform).report.sequential.html --self-contained-html --capture=sys" # Parallel - azdev test ${FinalTarget} $(USER_LIVE) --mark "not serial" --xml-path test_results.parallel.xml --no-exitfirst -a "-n $(USER_PARALLELISM) --json-report --json-report-summary --json-report-file=$(Target).$(platform).report.parallel.json --html=$(Target).$(platform).report.parallel.html --self-contained-html --capture=sys" + azdev test ${FinalTarget} ${{ parameters.userLive }} --mark "not serial" --xml-path test_results.parallel.xml --no-exitfirst -a "-n ${{ parameters.userParallelism }} --json-report --json-report-summary --json-report-file=$(Target).$(platform).report.parallel.json --html=$(Target).$(platform).report.parallel.html --self-contained-html --capture=sys" fi pwd ls # Whether commit changes - if [[ "$(USER_USERNAME)" != "" && "$(USER_TOKEN)" != "" && ! "$(USER_TARGET)" =~ ^ext-.* && "$(USER_TARGET)" != "extensions" && "$(USER_TARGET)" != "" && "$(USER_TARGET)" != "all" && "$(USER_TARGET)" != "ALL" ]]; then + if [[ "${{ parameters.userUsername }}" != " " && "${{ parameters.userToken }}" != " " && ! "${{ parameters.userTarget }}" =~ ^ext-.* && "${{ parameters.userTarget }}" != "extensions" && "${{ parameters.userTarget }}" != "" && "${{ parameters.userTarget }}" != "all" && "${{ parameters.userTarget }}" != "ALL" ]]; then cd azure-cli git status git add . git commit -m "Upload recording files" - git push origin $(USER_BRANCH) - elif [[ "$(USER_USERNAME)" != "" && "$(USER_TOKEN)" != "" && "$(USER_TARGET)" =~ ^ext-.* ]]; then + git push origin ${{ parameters.userBranch }} + elif [[ "${{ parameters.userUsername }}" != " " && "${{ parameters.userToken }}" != " " && "${{ parameters.userTarget }}" =~ ^ext-.* ]]; then cd azure-cli-extensions git status git add . git commit -m "Upload recording files" - git push origin $(USER_BRANCH_EXT) + git push origin ${{ parameters.userBranchExt }} fi condition: and(succeededOrFailed(), eq(variables.Match, '1')) - task: PublishTestResults@2 @@ -607,10 +657,10 @@ stages: displayName: Live Test # Sometimes the live test will be stuck forever, so the timeout cannot be set to 0, it is recommended to set it to twice the maximum time # For 360 minutes (6 hours) on Microsoft-hosted agents with a public project and public repository - timeoutInMinutes: 600 + timeoutInMinutes: 1200 strategy: # If the maxParallel is too large, the resource group will reach the limit of 980, so it is recommended that the maxParallel should not exceed 30. - maxParallel: 30 + maxParallel: ${{ parameters.maxParallel }} matrix: acr: Target: acr @@ -1028,7 +1078,7 @@ stages: vmImage: ${{ variables.macos_pool }} steps: - bash: | - if [[ "$(USER_TARGET)" =~ "$(Target)" || ("$(USER_TARGET)" == "main" && ! "$(Target)" =~ ^ext-.*) || ("$(USER_TARGET)" == "extensions" && "$(Target)" =~ ^ext-.*) || ("$(USER_TARGET)" == "ALL" || "$(USER_TARGET)" == "all" || "$(USER_TARGET)" == "") ]]; then + if [[ "${{ parameters.userTarget }}" =~ "$(Target)" || ("${{ parameters.userTarget }}" == "main" && ! "$(Target)" =~ ^ext-.*) || ("${{ parameters.userTarget }}" == "extensions" && "$(Target)" =~ ^ext-.*) || ("${{ parameters.userTarget }}" == "ALL" || "${{ parameters.userTarget }}" == "all" || "${{ parameters.userTarget }}" == "") ]]; then echo "Match!" echo "##vso[task.setvariable variable=Match]1" echo "##vso[task.setvariable variable=platform]macos12" @@ -1039,13 +1089,13 @@ stages: - task: UsePythonVersion@0 condition: and(succeededOrFailed(), eq(variables.Match, '1')) inputs: - versionSpec: '$(PYTHON_VERSION)' + versionSpec: '${{ parameters.pythonVersion }}' addToPath: true architecture: 'x64' - task: AzureCLI@2 displayName: 'Run live test on Macos' inputs: - azureSubscription: $(AZURE_CLI_LIVE_TEST_CONNECTED_SERVICE) + connectedServiceNameARM: $(azure-cli-live-test-bami-connected-service) scriptType: bash scriptLocation: inlineScript inlineScript: | @@ -1060,21 +1110,21 @@ stages: # print variables echo "PLATFORM: $(PLATFORM)" - echo "USER_TARGET: $(USER_TARGET)" - echo "USER_REPO: $(USER_REPO)" - echo "USER_REPO_EXT: $(USER_REPO_EXT)" - echo "USER_BRANCH: $(USER_BRANCH)" - echo "USER_BRANCH_EXT: $(USER_BRANCH_EXT)" + echo "USER_TARGET: ${{ parameters.userTarget }}" + echo "USER_REPO: ${{ parameters.userRepo }}" + echo "USER_REPO_EXT: ${{ parameters.userRepoExt }}" + echo "USER_BRANCH: ${{ parameters.userBranch }}" + echo "USER_BRANCH_EXT: ${{ parameters.userBranchExt }}" # Whether commit changes - if [[ "$(USER_USERNAME)" != "" && "$(USER_TOKEN)" != "" ]]; then - REPO="$(USER_REPO)" - EXT_REPO="$(USER_REPO_EXT)" + if [[ "${{ parameters.userUsername }}" != " " && "${{ parameters.userToken }}" != " " ]]; then + REPO="${{ parameters.userRepo }}" + EXT_REPO="${{ parameters.userRepoExt }}" # Pass username and token so that we can commit changes - git clone -b $(USER_BRANCH) ${REPO:0:8}$(USER_USERNAME):$(USER_TOKEN)@${REPO:8} - git clone -b $(USER_BRANCH_EXT) ${EXT_REPO:0:8}$(USER_USERNAME):$(USER_TOKEN)@${EXT_REPO:8} + git clone -b ${{ parameters.userBranch }} ${REPO:0:8}${{ parameters.userUsername }}:${{ parameters.userToken }}@${REPO:8} + git clone -b ${{ parameters.userBranchExt }} ${EXT_REPO:0:8}${{ parameters.userUsername }}:${{ parameters.userToken }}@${EXT_REPO:8} else - git clone -b $(USER_BRANCH) $(USER_REPO) - git clone -b $(USER_BRANCH_EXT) $(USER_REPO_EXT) + git clone -b ${{ parameters.userBranch }} ${{ parameters.userRepo }} + git clone -b ${{ parameters.userBranchExt }} ${{ parameters.userRepoExt }} fi python -m venv env @@ -1105,44 +1155,43 @@ stages: python $workDir/s/scripts/compact_aaz.py az -v - # az login --service-principal --username $(AZURECLITESTUSER) --password $(AZURECLITESTPASSWORD) --tenant $(AZURECLITESTTENANTID) - az account set -s $(AZURECLITESTSUBSCRIPTIONID) + az account set -s $(azure-cli-live-test-bami-sub-id) # Clean policy python $workDir/s/scripts/live_test/clean_policy.py echo "Run tests" # Whether commit changes - if [[ "$(USER_USERNAME)" != "" && "$(USER_TOKEN)" != "" ]]; then + if [[ "${{ parameters.userUsername }}" != " " && "${{ parameters.userToken }}" != " " ]]; then echo "Commit mode" - azdev test ${FinalTarget} --no-exitfirst -a "-n $(USER_PARALLELISM)" - azdev test ${FinalTarget} --live --lf --xml-path test_results.parallel.xml --no-exitfirst -a "-n $(USER_PARALLELISM) --json-report --json-report-summary --json-report-file=$(Target).$(platform).report.parallel.json --html=$(Target).$(platform).report.parallel.html --self-contained-html --capture=sys" + azdev test ${FinalTarget} --no-exitfirst -a "-n ${{ parameters.userParallelism }}" + azdev test ${FinalTarget} --live --lf --xml-path test_results.parallel.xml --no-exitfirst -a "-n ${{ parameters.userParallelism }} --json-report --json-report-summary --json-report-file=$(Target).$(platform).report.parallel.json --html=$(Target).$(platform).report.parallel.html --self-contained-html --capture=sys" elif [[ "$serial_modules" =~ "$FinalTarget" ]]; then echo "Series mode" - azdev test ${FinalTarget} $(USER_LIVE) --xml-path test_results.sequential.xml --no-exitfirst -a "-n 1 --json-report --json-report-summary --json-report-file=$(Target).$(platform).report.sequential.json --html=$(Target).$(platform).report.sequential.html --self-contained-html --capture=sys" + azdev test ${FinalTarget} ${{ parameters.userLive }} --xml-path test_results.sequential.xml --no-exitfirst -a "-n 1 --json-report --json-report-summary --json-report-file=$(Target).$(platform).report.sequential.json --html=$(Target).$(platform).report.sequential.html --self-contained-html --capture=sys" else echo "Normal mode" # Sequential - azdev test ${FinalTarget} $(USER_LIVE) --mark serial --xml-path test_results.sequential.xml --no-exitfirst -a "-n 1 --json-report --json-report-summary --json-report-file=$(Target).$(platform).report.sequential.json --html=$(Target).$(platform).report.sequential.html --self-contained-html --capture=sys" + azdev test ${FinalTarget} ${{ parameters.userLive }} --mark serial --xml-path test_results.sequential.xml --no-exitfirst -a "-n 1 --json-report --json-report-summary --json-report-file=$(Target).$(platform).report.sequential.json --html=$(Target).$(platform).report.sequential.html --self-contained-html --capture=sys" # Parallel - azdev test ${FinalTarget} $(USER_LIVE) --mark "not serial" --xml-path test_results.parallel.xml --no-exitfirst -a "-n $(USER_PARALLELISM) --json-report --json-report-summary --json-report-file=$(Target).$(platform).report.parallel.json --html=$(Target).$(platform).report.parallel.html --self-contained-html --capture=sys" + azdev test ${FinalTarget} ${{ parameters.userLive }} --mark "not serial" --xml-path test_results.parallel.xml --no-exitfirst -a "-n ${{ parameters.userParallelism }} --json-report --json-report-summary --json-report-file=$(Target).$(platform).report.parallel.json --html=$(Target).$(platform).report.parallel.html --self-contained-html --capture=sys" fi pwd ls # Whether commit changes - if [[ "$(USER_USERNAME)" != "" && "$(USER_TOKEN)" != "" && ! "$(USER_TARGET)" =~ ^ext-.* && "$(USER_TARGET)" != "extensions" && "$(USER_TARGET)" != "" && "$(USER_TARGET)" != "all" && "$(USER_TARGET)" != "ALL" ]]; then + if [[ "${{ parameters.userUsername }}" != " " && "${{ parameters.userToken }}" != " " && ! "${{ parameters.userTarget }}" =~ ^ext-.* && "${{ parameters.userTarget }}" != "extensions" && "${{ parameters.userTarget }}" != "" && "${{ parameters.userTarget }}" != "all" && "${{ parameters.userTarget }}" != "ALL" ]]; then cd azure-cli git status git add . git commit -m "Upload recording files" - git push origin $(USER_BRANCH) - elif [[ "$(USER_USERNAME)" != "" && "$(USER_TOKEN)" != "" && "$(USER_TARGET)" =~ ^ext-.* ]]; then + git push origin ${{ parameters.userBranch }} + elif [[ "${{ parameters.userUsername }}" != " " && "${{ parameters.userToken }}" != " " && "${{ parameters.userTarget }}" =~ ^ext-.* ]]; then cd azure-cli-extensions git status git add . git commit -m "Upload recording files" - git push origin $(USER_BRANCH_EXT) + git push origin ${{ parameters.userBranchExt }} fi condition: and(succeededOrFailed(), eq(variables.Match, '1')) - task: PublishTestResults@2 @@ -1183,10 +1232,10 @@ stages: - job: LiveTest displayName: Live Test # Sometimes the live test will be stuck forever, so the timeout cannot be set to 0, it is recommended to set it to twice the maximum time - timeoutInMinutes: 600 + timeoutInMinutes: 1200 strategy: # If the maxParallel is too large, the resource group will reach the limit of 980, so it is recommended that the maxParallel should not exceed 30. - maxParallel: 30 + maxParallel: ${{ parameters.maxParallel }} matrix: acr: Target: acr @@ -1604,7 +1653,7 @@ stages: name: ${{ variables.windows_pool }} steps: - bash: | - if [[ "$(USER_TARGET)" =~ "$(Target)" || ("$(USER_TARGET)" == "main" && ! "$(Target)" =~ ^ext-.*) || ("$(USER_TARGET)" == "extensions" && "$(Target)" =~ ^ext-.*) || ("$(USER_TARGET)" == "ALL" || "$(USER_TARGET)" == "all" || "$(USER_TARGET)" == "") ]]; then + if [[ "${{ parameters.userTarget }}" =~ "$(Target)" || ("${{ parameters.userTarget }}" == "main" && ! "$(Target)" =~ ^ext-.*) || ("${{ parameters.userTarget }}" == "extensions" && "$(Target)" =~ ^ext-.*) || ("${{ parameters.userTarget }}" == "ALL" || "${{ parameters.userTarget }}" == "all" || "${{ parameters.userTarget }}" == "") ]]; then echo "Match!" echo "##vso[task.setvariable variable=Match]1" echo "##vso[task.setvariable variable=platform]windows2019" @@ -1615,13 +1664,13 @@ stages: - task: UsePythonVersion@0 condition: and(succeededOrFailed(), eq(variables.Match, '1')) inputs: - versionSpec: '$(PYTHON_VERSION)' + versionSpec: '${{ parameters.pythonVersion }}' addToPath: true architecture: 'x64' - task: AzureCLI@2 displayName: 'Run live test on Windows' inputs: - azureSubscription: $(AZURE_CLI_LIVE_TEST_CONNECTED_SERVICE) + connectedServiceNameARM: $(azure-cli-live-test-bami-connected-service) scriptType: bash scriptLocation: inlineScript inlineScript: | @@ -1636,21 +1685,21 @@ stages: # print variables echo "PLATFORM: $(PLATFORM)" - echo "USER_TARGET: $(USER_TARGET)" - echo "USER_REPO: $(USER_REPO)" - echo "USER_REPO_EXT: $(USER_REPO_EXT)" - echo "USER_BRANCH: $(USER_BRANCH)" - echo "USER_BRANCH_EXT: $(USER_BRANCH_EXT)" + echo "USER_TARGET: ${{ parameters.userTarget }}" + echo "USER_REPO: ${{ parameters.userRepo }}" + echo "USER_REPO_EXT: ${{ parameters.userRepoExt }}" + echo "USER_BRANCH: ${{ parameters.userBranch }}" + echo "USER_BRANCH_EXT: ${{ parameters.userBranchExt }}" # Whether commit changes - if [[ "$(USER_USERNAME)" != "" && "$(USER_TOKEN)" != "" ]]; then - REPO="$(USER_REPO)" - EXT_REPO="$(USER_REPO_EXT)" + if [[ "${{ parameters.userUsername }}" != " " && "${{ parameters.userToken }}" != " " ]]; then + REPO="${{ parameters.userRepo }}" + EXT_REPO="${{ parameters.userRepoExt }}" # Pass username and token so that we can commit changes - git clone -b $(USER_BRANCH) ${REPO:0:8}$(USER_USERNAME):$(USER_TOKEN)@${REPO:8} - git clone -b $(USER_BRANCH_EXT) ${EXT_REPO:0:8}$(USER_USERNAME):$(USER_TOKEN)@${EXT_REPO:8} + git clone -b ${{ parameters.userBranch }} ${REPO:0:8}${{ parameters.userUsername }}:${{ parameters.userToken }}@${REPO:8} + git clone -b ${{ parameters.userBranchExt }} ${EXT_REPO:0:8}${{ parameters.userUsername }}:${{ parameters.userToken }}@${EXT_REPO:8} else - git clone -b $(USER_BRANCH) $(USER_REPO) - git clone -b $(USER_BRANCH_EXT) $(USER_REPO_EXT) + git clone -b ${{ parameters.userBranch }} ${{ parameters.userRepo }} + git clone -b ${{ parameters.userBranchExt }} ${{ parameters.userRepoExt }} fi python -m venv env @@ -1680,44 +1729,43 @@ stages: python $workDir/s/scripts/compact_aaz.py az -v - # az login --service-principal --username $(AZURECLITESTUSER) --password $(AZURECLITESTPASSWORD) --tenant $(AZURECLITESTTENANTID) - az account set -s $(AZURECLITESTSUBSCRIPTIONID) + az account set -s $(azure-cli-live-test-bami-sub-id) # Clean policy python $workDir/s/scripts/live_test/clean_policy.py echo "Run tests" # Whether commit changes - if [[ "$(USER_USERNAME)" != "" && "$(USER_TOKEN)" != "" ]]; then + if [[ "${{ parameters.userUsername }}" != " " && "${{ parameters.userToken }}" != " " ]]; then echo "Commit mode" - azdev test ${FinalTarget} --no-exitfirst -a "-n $(USER_PARALLELISM)" - azdev test ${FinalTarget} --live --lf --xml-path test_results.parallel.xml --no-exitfirst -a "-n $(USER_PARALLELISM) --json-report --json-report-summary --json-report-file=$(Target).$(platform).report.parallel.json --html=$(Target).$(platform).report.parallel.html --self-contained-html --capture=sys" + azdev test ${FinalTarget} --no-exitfirst -a "-n ${{ parameters.userParallelism }}" + azdev test ${FinalTarget} --live --lf --xml-path test_results.parallel.xml --no-exitfirst -a "-n ${{ parameters.userParallelism }} --json-report --json-report-summary --json-report-file=$(Target).$(platform).report.parallel.json --html=$(Target).$(platform).report.parallel.html --self-contained-html --capture=sys" elif [[ "$serial_modules" =~ "$FinalTarget" ]]; then echo "Series mode" - azdev test ${FinalTarget} $(USER_LIVE) --xml-path test_results.sequential.xml --no-exitfirst -a "-n 1 --json-report --json-report-summary --json-report-file=$(Target).$(platform).report.sequential.json --html=$(Target).$(platform).report.sequential.html --self-contained-html --capture=sys" + azdev test ${FinalTarget} ${{ parameters.userLive }} --xml-path test_results.sequential.xml --no-exitfirst -a "-n 1 --json-report --json-report-summary --json-report-file=$(Target).$(platform).report.sequential.json --html=$(Target).$(platform).report.sequential.html --self-contained-html --capture=sys" else echo "Normal mode" # Sequential - azdev test ${FinalTarget} $(USER_LIVE) --mark serial --xml-path test_results.sequential.xml --no-exitfirst -a "-n 1 --json-report --json-report-summary --json-report-file=$(Target).$(platform).report.sequential.json --html=$(Target).$(platform).report.sequential.html --self-contained-html --capture=sys" + azdev test ${FinalTarget} ${{ parameters.userLive }} --mark serial --xml-path test_results.sequential.xml --no-exitfirst -a "-n 1 --json-report --json-report-summary --json-report-file=$(Target).$(platform).report.sequential.json --html=$(Target).$(platform).report.sequential.html --self-contained-html --capture=sys" # Parallel - azdev test ${FinalTarget} $(USER_LIVE) --mark "not serial" --xml-path test_results.parallel.xml --no-exitfirst -a "-n $(USER_PARALLELISM) --json-report --json-report-summary --json-report-file=$(Target).$(platform).report.parallel.json --html=$(Target).$(platform).report.parallel.html --self-contained-html --capture=sys" + azdev test ${FinalTarget} ${{ parameters.userLive }} --mark "not serial" --xml-path test_results.parallel.xml --no-exitfirst -a "-n ${{ parameters.userParallelism }} --json-report --json-report-summary --json-report-file=$(Target).$(platform).report.parallel.json --html=$(Target).$(platform).report.parallel.html --self-contained-html --capture=sys" fi pwd ls # Whether commit changes - if [[ "$(USER_USERNAME)" != "" && "$(USER_TOKEN)" != "" && ! "$(USER_TARGET)" =~ ^ext-.* && "$(USER_TARGET)" != "extensions" && "$(USER_TARGET)" != "" && "$(USER_TARGET)" != "all" && "$(USER_TARGET)" != "ALL" ]]; then + if [[ "${{ parameters.userUsername }}" != " " && "${{ parameters.userToken }}" != " " && ! "${{ parameters.userTarget }}" =~ ^ext-.* && "${{ parameters.userTarget }}" != "extensions" && "${{ parameters.userTarget }}" != "" && "${{ parameters.userTarget }}" != "all" && "${{ parameters.userTarget }}" != "ALL" ]]; then cd azure-cli git status git add . git commit -m "Upload recording files" - git push origin $(USER_BRANCH) - elif [[ "$(USER_USERNAME)" != "" && "$(USER_TOKEN)" != "" && "$(USER_TARGET)" =~ ^ext-.* ]]; then + git push origin ${{ parameters.userBranch }} + elif [[ "${{ parameters.userUsername }}" != " " && "${{ parameters.userToken }}" != " " && "${{ parameters.userTarget }}" =~ ^ext-.* ]]; then cd azure-cli-extensions git status git add . git commit -m "Upload recording files" - git push origin $(USER_BRANCH_EXT) + git push origin ${{ parameters.userBranchExt }} fi condition: and(succeededOrFailed(), eq(variables.Match, '1')) - task: PublishTestResults@2 @@ -1764,7 +1812,7 @@ stages: steps: - task: UsePythonVersion@0 inputs: - versionSpec: '$(PYTHON_VERSION)' + versionSpec: '${{ parameters.pythonVersion }}' addToPath: true architecture: 'x64' - task: DownloadBuildArtifacts@0 @@ -1775,7 +1823,7 @@ stages: - task: AzureCLI@2 displayName: 'Send live test email' inputs: - azureSubscription: $(AZURE_CLI_LIVE_TEST_CONNECTED_SERVICE) + connectedServiceNameARM: $(azure-cli-live-test-msft-connected-service) scriptType: bash scriptLocation: inlineScript inlineScript: | @@ -1793,34 +1841,32 @@ stages: # Send notification az -v echo "login to user azureclilivetest" - # az login --service-principal --username $(AZURECLITESTUSER) --password $(AZURECLITESTPASSWORD) --tenant $(AZURECLITESTTENANTID) - az account set -s $(AZURECLITESTSUBSCRIPTIONID) + az account set -s $(azure-cli-msft-test-sub-id) echo "set account" echo "send email to grid" python $workDir/scripts/live_test/sendemail.py "$commit_id" env: - ACCOUNT_KEY: $(ACCOUNT_KEY) ARTIFACTS_DIR: $(System.ArtifactsDirectory) BUILD_ID: $(Build.BuildId) + EMAIL_KEY: $(email-key) EMAIL_ADDRESS: $(Build.RequestedForEmail) - EMAIL_KEY: $(SENDGRID_KEY) - KUSTO_CLIENT_ID: $(KUSTO_CLIENT_ID) - KUSTO_CLIENT_SECRET: $(KUSTO_CLIENT_SECRET) - KUSTO_CLUSTER: $(KUSTO_CLUSTER) - KUSTO_DATABASE: $(KUSTO_DATABASE) - KUSTO_TABLE: $(KUSTO_TABLE) - KUSTO_TENANT_ID: $(KUSTO_TENANT_ID) - PYTHON_VERSION: $(PYTHON_VERSION) - USER_BRANCH: $(USER_BRANCH) - USER_BRANCH_EXT: $(USER_BRANCH_EXT) - USER_LIVE: $(USER_LIVE) - USER_REPO: $(USER_REPO) - USER_REPO_EXT: $(USER_REPO_EXT) - USER_TARGET: $(USER_TARGET) + ACCOUNT_NAME: $(account-name) + KUSTO_CLUSTER: $(kusto-cluster) + KUSTO_DATABASE: $(kusto-database) + KUSTO_TABLE: $(kusto-table) + IDENTITY_CLIENT_ID: $(identity-client-id) + PYTHON_VERSION: ${{ parameters.pythonVersion }} + USER_BRANCH: ${{ parameters.userBranch }} + USER_BRANCH_EXT: ${{ parameters.userBranchExt }} + USER_LIVE: ${{ parameters.userLive }} + USER_REPO: ${{ parameters.userRepo }} + USER_REPO_EXT: ${{ parameters.userRepoExt }} + USER_TARGET: ${{ parameters.userTarget }} + STATIC_WEB_URL: ${{ parameters.staticWebUrl }} - job: CleanResource displayName: Clean Resource Job - timeoutInMinutes: 360 + timeoutInMinutes: 1200 condition: succeededOrFailed() continueOnError: true pool: @@ -1828,13 +1874,13 @@ stages: steps: - task: UsePythonVersion@0 inputs: - versionSpec: '$(PYTHON_VERSION)' + versionSpec: '${{ parameters.pythonVersion }}' addToPath: true architecture: 'x64' - task: AzureCLI@2 displayName: 'Clean resource' inputs: - azureSubscription: $(AZURE_CLI_LIVE_TEST_CONNECTED_SERVICE) + connectedServiceNameARM: $(azure-cli-live-test-bami-connected-service) scriptType: bash scriptLocation: inlineScript inlineScript: | @@ -1843,8 +1889,7 @@ stages: pip install tqdm az -v echo "login to user azureclilivetest" - # az login --service-principal --username $(AZURECLITESTUSER) --password $(AZURECLITESTPASSWORD) --tenant $(AZURECLITESTTENANTID) - az account set -s $(AZURECLITESTSUBSCRIPTIONID) + az account set -s $(azure-cli-live-test-bami-sub-id) echo "set account" echo "clean resource" python $workDir/scripts/live_test/clean.py diff --git a/scripts/live_test/clean.py b/scripts/live_test/clean.py index 0a6d3e6423a..d11ea4715cd 100644 --- a/scripts/live_test/clean.py +++ b/scripts/live_test/clean.py @@ -16,12 +16,13 @@ def main(): print('Azure cli resource clean up: version 1.0') + clean_resource_group() clean_lock() - clean_sig() - clean_storage() - clean_servicebus() - clean_backup() - clean_deleted_keyvault() + # clean_sig() + # clean_storage() + # clean_servicebus() + # clean_backup() + # clean_deleted_keyvault() def clean_lock(): @@ -277,6 +278,7 @@ def clean_backup(): def clean_deleted_keyvault(): + print('Clean keyvault') cmd = ['az', 'keyvault', 'list-deleted', '--query', '[][name, properties.scheduledPurgeDate, type]'] print(cmd) out = subprocess.run(cmd, capture_output=True) @@ -298,6 +300,7 @@ def clean_deleted_keyvault(): def clean_resource_group(): + print('Clean resource group') skip_grous = [] cmd = ['az', 'group', 'list', '--query', '[].name'] print(cmd) diff --git a/scripts/live_test/generate_index.py b/scripts/live_test/generate_index.py index 97b47f1fa74..c149091c59b 100644 --- a/scripts/live_test/generate_index.py +++ b/scripts/live_test/generate_index.py @@ -13,56 +13,31 @@ import requests import xml.etree.ElementTree as ET import logging - +STATIC_WEB_URL = os.environ.get('STATIC_WEB_URL') logger = logging.getLogger(__name__) -def generate(container, container_url, testdata, USER_REPO, USER_BRANCH, COMMIT_ID, USER_LIVE, USER_TARGET, ACCOUNT_KEY, USER_REPO_EXT, USER_BRANCH_EXT): +def generate(ACCOUNT_NAME, container, testdata, USER_REPO, USER_BRANCH, COMMIT_ID, USER_LIVE, USER_REPO_EXT, USER_BRANCH_EXT): """ Generate index.html. Upload it to storage account :param container: - :param container_url: :return: a HTML string """ logger.warning('Enter generate()') - # [{'name': name, 'url': url}] - data = [] - url = container_url + '?restype=container&comp=list' - content = requests.get(url).content - logger.warning(content) - root = ET.fromstring(content) - for blobs in root: - for blob in blobs: - name = url = '' - for e in blob: - if e.tag == 'Name': - name = e.text - if e.tag == 'Url': - url = e.text - if name == '' or url == '': - logger.warning('[Warning] Blob\'s name or url is empty, name: {}, url: {}'.format(name, url)) - if name.endswith('.html'): - data.append({'name': name, 'url': url}) - break - logger.warning(data) - html = render(data, container, container_url, testdata, USER_REPO, USER_BRANCH, COMMIT_ID, USER_LIVE, USER_REPO_EXT, USER_BRANCH_EXT) + html = render(testdata, USER_REPO, USER_BRANCH, COMMIT_ID, USER_LIVE, USER_REPO_EXT, USER_BRANCH_EXT) with open('index.html', 'w') as f: f.write(html) # Upload to storage account - cmd = 'az storage blob upload -f index.html -c {} -n index.html --account-name clitestresultstac --auth-mode login --overwrite'.format(container) + cmd = f'az storage blob upload -f index.html -c {container} -n index.html --account-name {ACCOUNT_NAME} --auth-mode login --overwrite' logger.warning('Running: ' + cmd) os.system(cmd) - # Upload to latest container if it is a full live test of official repo dev branch - if USER_TARGET.lower() in ['all', ''] \ - and USER_REPO == 'https://github.com/Azure/azure-cli.git' \ - and USER_REPO_EXT == 'https://github.com/Azure/azure-cli-extensions.git' \ - and USER_BRANCH == 'dev' and USER_BRANCH_EXT == 'main' and USER_LIVE == '--live': - cmd = 'az storage blob upload -f index.html -c latest -n index.html --account-name clitestresultstac --auth-mode login --overwrite' - logger.warning('Running: ' + cmd) - os.system(cmd) + # Upload to $web container + cmd = f"az storage blob upload -f index.html -c '$web' -n index.html --account-name {ACCOUNT_NAME} --auth-mode login --overwrite" + logger.warning('Running: ' + cmd) + os.system(cmd) logger.warning('Exit generate()') return html @@ -77,12 +52,9 @@ def sort_by_module_name(item): return 0, item[0] # sort with lower priority -def render(data, container, container_url, testdata, USER_REPO, USER_BRANCH, COMMIT_ID, USER_LIVE, USER_REPO_EXT, USER_BRANCH_EXT): +def render(testdata, USER_REPO, USER_BRANCH, COMMIT_ID, USER_LIVE, USER_REPO_EXT, USER_BRANCH_EXT): """ Return a HTML string - :param data: - :param container: - :param container_url: :param testdata: :param USER_REPO: :param USER_BRANCH: @@ -149,7 +121,6 @@ def render(data, container, container_url, testdata, USER_REPO, USER_BRANCH, COM Pass Fail Pass rate - Reports """ @@ -159,42 +130,21 @@ def render(data, container, container_url, testdata, USER_REPO, USER_BRANCH, COM {} {} {} - N/A """.format(testdata.total[1], testdata.total[2], testdata.total[3]) sorted_modules = sorted(testdata.modules, key=sort_by_module_name) for module, passed, failed, rate in sorted_modules: - reports = '' - for x in data: - name = x['name'] - url = x['url'] - if name.startswith(module + '.'): - display_name = 'report' - # if 'parallel' in name: - # display_name = 'parallel' - # elif 'sequential' in name: - # display_name = 'sequential' - try: - html = requests.get(url).text - pattern = re.compile('\\d+ tests ran in') - match = pattern.search(html) - number = match.group().split()[0] - if number.isdigit(): - display_name += '(' + number + ')' - except: - logger.exception(traceback.print_exc()) - reports += '{} '.format(url, display_name) table += """ {} {} {} {} - {} - """.format(module, passed, failed, rate, reports) + """.format('{} '.format(STATIC_WEB_URL+module+'.report.html', module), + passed, failed, rate) table += """ diff --git a/scripts/live_test/sendemail.py b/scripts/live_test/sendemail.py index 1fbf06b9df6..f3e55c368dd 100644 --- a/scripts/live_test/sendemail.py +++ b/scripts/live_test/sendemail.py @@ -31,19 +31,17 @@ logger.addHandler(ch) COMMIT_ID = sys.argv[1] -ACCOUNT_KEY = os.environ.get('ACCOUNT_KEY') # not used ARTIFACT_DIR = os.environ.get('ARTIFACTS_DIR') BUILD_ID = os.environ.get('BUILD_ID') EMAIL_ADDRESS = os.environ.get('EMAIL_ADDRESS') EMAIL_KEY = os.environ.get('EMAIL_KEY') -# authenticate with AAD application. -KUSTO_CLIENT_ID = os.environ.get('KUSTO_CLIENT_ID') -KUSTO_CLIENT_SECRET = os.environ.get('KUSTO_CLIENT_SECRET') +ACCOUNT_NAME = os.environ.get('ACCOUNT_NAME') +# authenticate with AAD token. KUSTO_CLUSTER = os.environ.get('KUSTO_CLUSTER') KUSTO_DATABASE = os.environ.get('KUSTO_DATABASE') KUSTO_TABLE = os.environ.get('KUSTO_TABLE') +IDENTITY_CLIENT_ID = os.environ.get('IDENTITY_CLIENT_ID') # get tenant id from https://learn.microsoft.com/en-us/onedrive/find-your-office-365-tenant-id -KUSTO_TENANT_ID = os.environ.get('KUSTO_TENANT_ID') PYTHON_VERSION = os.environ.get('PYTHON_VERSION') USER_BRANCH = os.environ.get('USER_BRANCH') USER_BRANCH_EXT = os.environ.get('USER_BRANCH_EXT') @@ -326,8 +324,7 @@ def main(): # Generate index.html, send email try: # Generate index.html - container_url = 'https://clitestresultstac.blob.core.windows.net/' + container - html_content = generate_index.generate(container, container_url, testdata, USER_REPO, USER_BRANCH, COMMIT_ID, USER_LIVE, USER_TARGET, ACCOUNT_KEY, USER_REPO_EXT, USER_BRANCH_EXT) + html_content = generate_index.generate(ACCOUNT_NAME, container, testdata, USER_REPO, USER_BRANCH, COMMIT_ID, USER_LIVE, USER_REPO_EXT, USER_BRANCH_EXT) # Send email send_email(html_content) except Exception: @@ -372,7 +369,7 @@ def get_remaining_tests(): with open('resource.html', 'w') as f: f.write(str(soup)) logger.info('resource.html: ' + str(soup)) - cmd = 'az storage blob upload -f resource.html -c {} -n resource.html --account-name clitestresultstac --auth-mode login'.format(BUILD_ID) + cmd = f'az storage blob upload -f resource.html -c {BUILD_ID} -n resource.html --account-name {ACCOUNT_NAME} --auth-mode login --overwrite' logger.info('Running: ' + cmd) os.system(cmd) @@ -466,7 +463,7 @@ def summary_data(testdata): if USER_TARGET.lower() in ['all', ''] \ and USER_REPO == 'https://github.com/Azure/azure-cli.git' \ and USER_REPO_EXT == 'https://github.com/Azure/azure-cli-extensions.git' \ - and USER_BRANCH == 'dev' and USER_BRANCH_EXT == 'main' \ + and USER_BRANCH in ['dev', 'live-test'] and USER_BRANCH_EXT in ['main', 'live-test'] \ and USER_LIVE == '--live' and data: send_to_kusto(data) @@ -516,13 +513,17 @@ def html_to_csv(html_file, module, platform): def send_to_kusto(data): logger.info('Start send csv data to kusto db') - + try: + from azure.identity import DefaultAzureCredential, get_bearer_token_provider + except ImportError: + subprocess.check_call([sys.executable, "-m", "pip", "install", "azure-identity"]) + from azure.identity import DefaultAzureCredential, get_bearer_token_provider + token_provider = get_bearer_token_provider(DefaultAzureCredential(), "https://kusto.kusto.windows.net/.default") with open(f'{ARTIFACT_DIR}/livetest.csv', mode='w', newline='') as file: writer = csv.writer(file) writer.writerows(data) logger.info('Finish generate csv file for live test.') - - kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication(KUSTO_CLUSTER, KUSTO_CLIENT_ID, KUSTO_CLIENT_SECRET, KUSTO_TENANT_ID) + kcsb = KustoConnectionStringBuilder.with_token_provider(KUSTO_CLUSTER, token_provider) # The authentication method will be taken from the chosen KustoConnectionStringBuilder. client = QueuedIngestClient(kcsb) @@ -561,7 +562,7 @@ def upload_files(container): logger.info('Enter upload_files()') # Create container - cmd = 'az storage container create -n {} --account-name clitestresultstac --public-access container --auth-mode login'.format(container) + cmd = f'az storage container create -n {container} --account-name {ACCOUNT_NAME} --public-access off --auth-mode login' os.system(cmd) # Upload files @@ -569,7 +570,9 @@ def upload_files(container): for name in files: if name.endswith('html') or name.endswith('json'): fullpath = os.path.join(root, name) - cmd = 'az storage blob upload -f {} -c {} -n {} --account-name clitestresultstac --auth-mode login'.format(fullpath, container, name) + cmd = f'az storage blob upload -f {fullpath} -c {container} -n {name} --account-name {ACCOUNT_NAME} --auth-mode login' + os.system(cmd) + cmd = f"az storage blob upload -f {fullpath} -c '$web' -n {name} --account-name {ACCOUNT_NAME} --auth-mode login --overwrite" os.system(cmd) logger.info('Exit upload_files()') @@ -598,7 +601,7 @@ def send_email(html_content): elif USER_TARGET.lower() in ['all', ''] \ and USER_REPO == 'https://github.com/Azure/azure-cli.git' \ and USER_REPO_EXT == 'https://github.com/Azure/azure-cli-extensions.git' \ - and USER_BRANCH == 'dev' and USER_BRANCH_EXT == 'main' \ + and USER_BRANCH in ['dev', 'live-test'] and USER_BRANCH_EXT in ['main', 'live-test'] \ and USER_LIVE == '--live' and EMAIL_ADDRESS == '': recipients = { "to": [