Merge branch 'main' of https://github.com/Azure/azure-sdk-for-python into drop-python-3.8

Author: msyyc

.github/CODEOWNERS

@@ -299,7 +299,7 @@
 
 # ServiceLabel: %Maps
 # PRLabel: %Maps
-/sdk/maps/                                                           @dubiety
+/sdk/maps/                                                           @dubiety @andykao1213
 
 # ServiceLabel: %Mixed Reality
 # PRLabel: %Mixed Reality

eng/common/pipelines/templates/archetype-typespec-emitter.yml

@@ -67,6 +67,15 @@ parameters:
   type: object
   default: []
 
+# The path to the emitter package json file.
+- name: EmitterPackageJsonOutputPath
+  type: string
+  default: "eng/emitter-package.json"
+
+# The relative path to the emitter package.
+- name: EmitterPackagePath
+  type: string
+
 extends:
   template: /eng/pipelines/templates/stages/1es-redirect.yml
   parameters:
@@ -94,7 +103,17 @@ extends:
           inputs:
             pwsh: true
             filePath: $(Build.SourcesDirectory)/eng/scripts/typespec/Initialize-WorkingDirectory.ps1
-            arguments: -UseTypeSpecNext:$${{ parameters.UseTypeSpecNext }}
+            ${{ if parameters.BuildPrereleaseVersion }}:
+              arguments: >
+                -PrereleaseSuffix "-alpha.$(Build.BuildNumber)"
+                -OutputDirectory "$(Build.ArtifactStagingDirectory)"
+                -UseTypeSpecNext:$${{ parameters.UseTypeSpecNext }}
+                -EmitterPackagePath:${{ parameters.EmitterPackagePath }}
+            ${{ else }}:
+              arguments: >
+                -OutputDirectory "$(Build.ArtifactStagingDirectory)"
+                -UseTypeSpecNext:$${{ parameters.UseTypeSpecNext }}
+                -EmitterPackagePath:${{ parameters.EmitterPackagePath }}
 
         - task: PowerShell@2
           displayName: 'Run build script'
@@ -103,10 +122,9 @@ extends:
             pwsh: true
             filePath: $(Build.SourcesDirectory)/eng/scripts/typespec/Build-Emitter.ps1
             arguments: >
-              -BuildNumber "$(Build.BuildNumber)"
-              -OutputDirectory "$(Build.ArtifactStagingDirectory)"
+              -OutputDirectory "$(Build.ArtifactStagingDirectory)/packages"
               -TargetNpmJsFeed:$${{ parameters.PublishPublic }}
-              -Prerelease:$${{ parameters.BuildPrereleaseVersion }}
+              -EmitterPackagePath:${{ parameters.EmitterPackagePath }}
 
         - pwsh: |
             $sourceBranch = '$(Build.SourceBranch)'
@@ -134,7 +152,7 @@ extends:
     # Publish stage
     # Responsible for publishing the packages in `build_artifacts/packages` and producing `emitter-package-lock.json`
     # Produces the artifact `publish_artifacts` which contains the following:
-    #   emitter-package.json: Created using the package json from the build step. 
+    #   emitter-package.json: Created using the package json from the build step.
     #   emitter-package-lock.json: Created by calling `npm install` using `emitter-package.json`
     - ${{ if parameters.ShouldPublish }}:
       - stage: Publish
@@ -189,26 +207,30 @@ extends:
                 ServiceEndpointUrl: https://api.esrp.microsoft.com
                 MainPublisher: ESRPRELPACMANTEST
 
-          - task: PowerShell@2
-            displayName: Create emitter-package.json
-            inputs:
-              pwsh: true
-              filePath: ./eng/common/scripts/typespec/New-EmitterPackageJson.ps1
-              arguments: >
-                -PackageJsonPath '$(buildArtifactsPath)/lock-files/package.json'
-                -OverridesPath '$(buildArtifactsPath)/overrides.json'
-                -OutputDirectory '$(Build.ArtifactStagingDirectory)'
-              workingDirectory: $(Build.SourcesDirectory)
+          - pwsh: |
+              npm install -g @azure-tools/typespec-client-generator-cli@latest
+            displayName: Install tsp-client
 
-          - task: PowerShell@2
-            displayName: Create emitter-package-lock.json
-            inputs:
-              pwsh: true
-              filePath: ./eng/common/scripts/typespec/New-EmitterPackageLock.ps1
-              arguments: >
-                -EmitterPackageJsonPath '$(Build.ArtifactStagingDirectory)/emitter-package.json'
-                -OutputDirectory '$(Build.ArtifactStagingDirectory)'
-              workingDirectory: $(Build.SourcesDirectory)
+          - pwsh: |
+              Write-Host "Overrides location: $(buildArtifactsPath)/packages/overrides.json"
+
+              if (Test-Path -Path '$(buildArtifactsPath)/packages/overrides.json') {
+                Write-Host "Using overrides.json to generate emitter-package.json"
+                tsp-client generate-config-files `
+                --package-json '$(buildArtifactsPath)/lock-files/package.json' `
+                --output-dir '$(Build.SourcesDirectory)' `
+                --emitter-package-json-path '${{ parameters.EmitterPackageJsonOutputPath }}' `
+                --overrides '$(buildArtifactsPath)/packages/overrides.json'
+              } else {
+                Write-Host "No overrides.json found. Running tsp-client without overrides."
+                
+                tsp-client generate-config-files `
+                --package-json '$(buildArtifactsPath)/lock-files/package.json' `
+                --output-dir '$(Build.SourcesDirectory)' `
+                --emitter-package-json-path '${{ parameters.EmitterPackageJsonOutputPath }}'
+              }
+            displayName: Generate emitter-package.json and emitter-package-lock files
+            workingDirectory: $(Build.SourcesDirectory)
 
           - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml
             parameters:
@@ -240,11 +262,8 @@ extends:
             displayName: Download pipeline artifacts
 
           - pwsh: |
-              Write-Host "Copying emitter-package.json to $(Build.SourcesDirectory)/eng"
-              Copy-Item $(publishArtifactsPath)/emitter-package.json $(Build.SourcesDirectory)/eng/ -Force
-
-              Write-Host "Copying emitter-package-lock.json to $(Build.SourcesDirectory)/eng"
-              Copy-Item $(publishArtifactsPath)/emitter-package-lock.json $(Build.SourcesDirectory)/eng/ -Force
+              Write-Host "Copying *emitter-package*.json to $(Build.SourcesDirectory)/eng"
+              Copy-Item $(publishArtifactsPath)/*emitter-package*.json $(Build.SourcesDirectory)/eng/ -Force
             displayName: Copy emitter-package json files
 
           - ${{ parameters.InitializationSteps }}
@@ -291,7 +310,7 @@ extends:
               Paths:
               - "/*"
               - "!SessionRecords"
-              
+
           - download: current
             displayName: Download pipeline artifacts
 
@@ -432,7 +451,9 @@ extends:
             inputs:
               pwsh: true
               filePath: $(Build.SourcesDirectory)/eng/scripts/typespec/Initialize-WorkingDirectory.ps1
-              arguments: -BuildArtifactsPath '$(buildArtifactsPath)'
+              arguments: > 
+                -BuildArtifactsPath '$(buildArtifactsPath)/lock-files'
+                -EmitterPackagePath: ${{ parameters.EmitterPackagePath }}
 
           - task: PowerShell@2
             displayName: 'Run test script'
@@ -442,6 +463,7 @@ extends:
               arguments: >
                 $(TestArguments)
                 -OutputDirectory "$(Build.ArtifactStagingDirectory)"
+                -EmitterPackagePath: ${{ parameters.EmitterPackagePath }}
 
           - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml
             parameters:

eng/common/scripts/Package-Properties.ps1

@@ -115,11 +115,28 @@ class PackageProps {
         return $null
     }
 
-    [PSCustomObject]GetCIYmlForArtifact() {
+    [System.IO.FileInfo[]]ResolveCIFolderPath() {
         $RepoRoot = Resolve-Path (Join-Path $PSScriptRoot ".." ".." "..")
-
         $ciFolderPath = Join-Path -Path $RepoRoot -ChildPath (Join-Path "sdk" $this.ServiceDirectory)
-        $ciFiles = @(Get-ChildItem -Path $ciFolderPath -Filter "ci*.yml" -File)
+        $ciFiles = @()
+
+        # if this path exists, then we should look in it for the ci.yml files and return nothing if nothing is found
+        if (Test-Path $ciFolderPath){
+            $ciFiles = @(Get-ChildItem -Path $ciFolderPath -Filter "ci*.yml" -File)
+        }
+        # if not, we should at least try to resolve the eng/ folder to fall back and see if that's where the path exists
+        else {
+            $ciFolderPath = Join-Path -Path $RepoRoot -ChildPath (Join-Path "eng" $this.ServiceDirectory)
+            if (Test-Path $ciFolderPath) {
+                $ciFiles = @(Get-ChildItem -Path $ciFolderPath -Filter "ci*.yml" -File)
+            }
+        }
+
+        return $ciFiles
+    }
+
+    [PSCustomObject]GetCIYmlForArtifact() {
+        $ciFiles = @($this.ResolveCIFolderPath())
         $ciArtifactResult = $null
         $soleCIYml = ($ciFiles.Count -eq 1)
 

eng/common/scripts/job-matrix/Create-JobMatrix.ps1

@@ -15,6 +15,8 @@ param (
     [Parameter(Mandatory=$False)][array] $Filters,
     [Parameter(Mandatory=$False)][array] $Replace,
     [Parameter(Mandatory=$False)][array] $NonSparseParameters,
+    # Use for local generation/debugging when env: values are set in a matrix
+    [Parameter(Mandatory=$False)][switch] $SkipEnvironmentVariables,
     [Parameter()][switch] $CI = ($null -ne $env:SYSTEM_TEAMPROJECTID)
 )
 
@@ -34,7 +36,8 @@ $Filters = $Filters | Where-Object { $_ }
     -displayNameFilter $DisplayNameFilter `
     -filters $Filters `
     -replace $Replace `
-    -nonSparseParameters $NonSparseParameters
+    -nonSparseParameters $NonSparseParameters `
+    -skipEnvironmentVariables:$SkipEnvironmentVariables
 
 $serialized = SerializePipelineMatrix $matrix
 

eng/common/scripts/job-matrix/job-matrix-functions.ps1

@@ -101,8 +101,8 @@ function GenerateMatrix(
 ) {
     $result = ProcessImport $config.matrixParameters $selectFromMatrixType $nonSparseParameters $config.displayNamesLookup
 
-    $matrixParameters          = $result.Matrix
-    $importedMatrix            = $result.ImportedMatrix
+    $matrixParameters = $result.Matrix
+    $importedMatrix = $result.ImportedMatrix
     $combinedDisplayNameLookup = $result.DisplayNamesLookup
 
     if ($selectFromMatrixType -eq "sparse") {
@@ -148,7 +148,7 @@ function ProcessNonSparseParameters(
     $nonSparse = [MatrixParameter[]]@()
 
     foreach ($param in $parameters) {
-        if ($null -eq $param){
+        if ($null -eq $param) {
             continue
         }
         if ($param.Name -in $nonSparseParameters) {
@@ -430,9 +430,9 @@ function ProcessImport([MatrixParameter[]]$matrix, [String]$selection, [Array]$n
     }
     if ((!$matrix -and !$importPath) -or !$importPath) {
         return [PSCustomObject]@{
-            Matrix                = $matrix
-            ImportedMatrix        = @()
-            DisplayNamesLookup    = $displayNamesLookup
+            Matrix             = $matrix
+            ImportedMatrix     = @()
+            DisplayNamesLookup = $displayNamesLookup
         }
     }
 
@@ -456,9 +456,9 @@ function ProcessImport([MatrixParameter[]]$matrix, [String]$selection, [Array]$n
     }
 
     return [PSCustomObject]@{
-        Matrix                = $matrix ?? @()
-        ImportedMatrix        = $importedMatrix
-        DisplayNamesLookup    = $combinedDisplayNameLookup
+        Matrix             = $matrix ?? @()
+        ImportedMatrix     = $importedMatrix
+        DisplayNamesLookup = $combinedDisplayNameLookup
     }
 }
 
@@ -643,7 +643,7 @@ function InitializeMatrix {
 function GetMatrixDimensions([MatrixParameter[]]$parameters) {
     $dimensions = @()
     foreach ($param in $parameters) {
-        if ($null -eq $param){
+        if ($null -eq $param) {
             continue
         }
         $dimensions += $param.Length()
@@ -760,12 +760,12 @@ function Get4dMatrixIndex([int]$index, [Array]$dimensions) {
 
 function GenerateMatrixForConfig {
     param (
-      [Parameter(Mandatory = $true)][string] $ConfigPath,
-      [Parameter(Mandatory = $true)][string] $Selection,
-      [Parameter(Mandatory = $false)][string] $DisplayNameFilter,
-      [Parameter(Mandatory = $false)][array] $Filters,
-      [Parameter(Mandatory = $false)][array] $Replace,
-      [Parameter(Mandatory = $false)][Array] $NonSparseParameters = @()
+        [Parameter(Mandatory = $true)][string] $ConfigPath,
+        [Parameter(Mandatory = $true)][string] $Selection,
+        [Parameter(Mandatory = $false)][string] $DisplayNameFilter,
+        [Parameter(Mandatory = $false)][array] $Filters,
+        [Parameter(Mandatory = $false)][array] $Replace,
+        [Parameter(Mandatory = $false)][Array] $NonSparseParameters = @()
     )
     $matrixFile = Join-Path $PSScriptRoot ".." ".." ".." ".." $ConfigPath
 
@@ -776,12 +776,12 @@ function GenerateMatrixForConfig {
     $Filters = $Filters | Where-Object { $_ }
 
     [array]$matrix = GenerateMatrix `
-      -config $config `
-      -selectFromMatrixType $Selection `
-      -displayNameFilter $DisplayNameFilter `
-      -filters $Filters `
-      -replace $Replace `
-      -nonSparseParameters $NonSparseParameters
+        -config $config `
+        -selectFromMatrixType $Selection `
+        -displayNameFilter $DisplayNameFilter `
+        -filters $Filters `
+        -replace $Replace `
+        -nonSparseParameters $NonSparseParameters
 
     return , $matrix
 }

scripts/auto_release/requirement.txt

@@ -5,4 +5,6 @@ PyGithub==1.55
 packaging==24.1
 pytest==6.2.5
 fastcore==1.3.25
-tox==4.15.0
+tox==4.15.0
+wheel==0.46.1
+setuptools==78.1.0

scripts/automation_generate.sh

@@ -1,11 +1,6 @@
 #!/bin/bash
 
-VIRTUAL_ENV=$TMPDIR/venv-sdk
-export VIRTUAL_ENV
-PATH="$VIRTUAL_ENV/bin:$PATH"
-export PATH
-
-TEMP_FILE="$TMPDIR/venv-sdk/auto_temp.json"
+TEMP_FILE="$TMPDIR/auto_temp.json"
 # generate code
 python -m packaging_tools.sdk_generator "$1" "$TEMP_FILE" --debug 2>&1
 echo "[Generate] codegen done!!!"

scripts/automation_init.sh

@@ -1,24 +1,15 @@
 #!/bin/bash
 
-# install python3.10
-sudo add-apt-repository ppa:deadsnakes/ppa > /dev/null
-sudo apt-get install python3.10 > /dev/null
-sudo apt-get install python3.10-venv > /dev/null
-
 # init env
-rm -rf $TMPDIR/venv-sdk > /dev/null
-python3.10 -m venv $TMPDIR/venv-sdk > /dev/null
-VIRTUAL_ENV=$TMPDIR/venv-sdk
-export VIRTUAL_ENV
-PATH="$VIRTUAL_ENV/bin:$PATH"
-export PATH
 python -m pip install -U pip > /dev/null
 python scripts/dev_setup.py -p azure-core > /dev/null
 pip install tox==4.15.0 > /dev/null
+pip install wheel==0.46.1 > /dev/null
+pip install setuptools==78.1.0 > /dev/null
 
 # install tsp-client globally (local install may interfere with tooling)
 echo Install tsp-client
-sudo npm install -g @azure-tools/typespec-client-generator-cli > /dev/null
+sudo npm install -g @azure-tools/typespec-client-generator-cli@0.20.0 > /dev/null
 
 echo "{}" >> $2
 echo "[Generate] init success!!!"

sdk/appconfiguration/azure-appconfiguration-provider/azure/appconfiguration/provider/_azureappconfigurationprovider.py

@@ -366,7 +366,11 @@ def refresh(self, **kwargs) -> None:  # pylint: disable=too-many-statements
                     if self._feature_flag_refresh_enabled:
                         need_ff_refresh, refresh_on_feature_flags, feature_flags, filters_used = (
                             client.refresh_feature_flags(
-                                self._refresh_on_feature_flags, self._feature_flag_selectors, headers=headers, **kwargs
+                                self._refresh_on_feature_flags,
+                                self._feature_flag_selectors,
+                                headers,
+                                self._origin_endpoint,
+                                **kwargs,
                             )
                         )
                         if refresh_on_feature_flags:
@@ -425,7 +429,11 @@ def _load_all(self, **kwargs):
                 configuration_settings_processed = self._process_configurations(configuration_settings)
                 if self._feature_flag_enabled:
                     feature_flags, feature_flag_sentinel_keys, used_filters = client.load_feature_flags(
-                        self._feature_flag_selectors, self._feature_flag_refresh_enabled, headers=headers, **kwargs
+                        self._feature_flag_selectors,
+                        self._feature_flag_refresh_enabled,
+                        self._origin_endpoint,
+                        headers=headers,
+                        **kwargs,
                     )
                     self._feature_filter_usage = used_filters
                     configuration_settings_processed[FEATURE_MANAGEMENT_KEY] = {}