Task 41737: Merge feat/azure-split to main: Step 3 - Tie everything together

- Addressed PR comments/suggestions.

Author: paulmedynski

.gitignore

@@ -193,12 +193,10 @@ PublishScripts/
 *.nupkg
 # NuGet Symbol Packages
 *.snupkg
-# The packages folder can be ignored because of Package Restore
+# Most of the packages folder can be ignored.
 **/[Pp]ackages/*
-# except build/, which is used as an MSBuild target.
-!**/[Pp]ackages/build/
-# Uncomment if necessary however generally it will be regenerated when needed
-#!**/[Pp]ackages/repositories.config
+!**/[Pp]ackages/.gitkeep
+
 # NuGet v3's project.json files produces more ignorable files
 *.nuget.props
 *.nuget.targets

BUILDGUIDE.md

@@ -211,27 +211,13 @@ dependencies.
 Alternatively, the `ReferenceType` build property may be specified with a value
 of `Package`.  This will change inter-component dependencies to use
 `<PackageReference>` dependencies, and require that dependent components be
-built and packaged before building the depending component.  In this scenario,
-the root `NuGet.config` file must be updated to include the following entry
-under the `<packageSources>` element:
-
-```xml
-<configuration>
-  <packageSources>
-    ...
-    <add key="local" value="packages/" />
-  </packageSources>
-</configuration>
-```
-
-As a convenience, a `NuGet.config.local` file is supplied with the above
-package source already present.  You may simply copy it over `NuGet.config`
-when using `Package` references.
+built and packaged before building the depending component.  This will generate NuGet
+packages in the root packages/ directory, and will be automatically searched by NuGet
+(see our root `NuGet.config`).
 
 Then, you can specify `Package` references be used, for example:
 
 ```bash
-cp NuGet.config.local NuGet.config
 dotnet build -t:BuildAbstractions
 dotnet build -t:BuildAzure -p:ReferenceType=Package
 dotnet build -t:BuildAll -p:ReferenceType=Package

NuGet.config

@@ -1,18 +1,25 @@
 <?xml version="1.0" encoding="utf-8"?>
 <configuration>
   <packageSources>
-    
+
     <!-- Clear all inherited feeds. -->
     <clear />
 
     <!--
-      We do not use the public nuget.org feed.  Instead, we use a governed
-      feed maintained by our SqlClientDrivers ADO organization.  Packages must
-      be individually vetted and added to this feed manually before they will
-      be available for consumption by our builds.
+      We do not use the public nuget.org feed.  Instead, we use a governed feed maintained by our
+      SqlClientDrivers ADO organization.  Packages must be individually vetted and added to this
+      feed manually before they will be available for consumption by our builds.
 
       https://sqlclientdrivers.visualstudio.com/public/_artifacts/feed/sqlclient
     -->
     <add key="governed" value="https://sqlclientdrivers.pkgs.visualstudio.com/public/_packaging/sqlclient/nuget/v3/index.json" />
+
+    <!--
+      This package source is used by developers and our pipelines builds when we need to reference
+      packages that are not available on the public nuget.org feed, for example to test MDS that
+      depends on a version of Abstractions that was just built in the workspace.
+    -->
+    <add key="local" value="packages/" />
+
   </packageSources>
 </configuration>

NuGet.config.local

@@ -285,11 +285,14 @@
   <!-- Run all unit tests applicable to Windows. -->
   <Target Name="RunUnitTestsWindows" Condition="'$(IsEnabledWindows)' == 'true' AND $(ReferenceType.Contains('Project'))">
     <PropertyGroup>
+      <!--
+        The UnitTests project exclusively uses Project references, which is the default applied by
+        src/Directory.Build.props, so no need to specify the ReferenceType property here.
+      -->
       <TestCommand>
         $(DotnetPath)dotnet test "@(UnitTestsProj)"
         -f $(TF)
         -p:Configuration=$(Configuration)
-        -p:ReferenceType=Project
         $(CollectStatement)
         --results-directory $(ResultsDirectory)
         --filter "$(FilterStatement)"
@@ -312,7 +315,6 @@
         $(DotnetPath)dotnet test "@(UnitTestsProj)"
         -f $(TF)
         -p:Configuration=$(Configuration)
-        -p:ReferenceType=Project
         $(CollectStatement)
         --results-directory $(ResultsDirectory)
         --filter "$(FilterStatement)"
@@ -446,7 +448,12 @@
     <RemoveDir Directories='$([System.IO.Directory]::GetDirectories(".", "bin", SearchOption.AllDirectories))' />
     <RemoveDir Directories='$([System.IO.Directory]::GetDirectories(".", ".nuget", SearchOption.AllDirectories))' />
     <RemoveDir Directories='$([System.IO.Directory]::GetDirectories(".", "obj", SearchOption.AllDirectories))' />
-    <RemoveDir Directories='$([System.IO.Directory]::GetDirectories(".","packages", SearchOption.AllDirectories))' />
+    <!-- We keep packages/ but remove all of the generated files. -->
+    <ItemGroup>
+      <PackagesToDelete Include="$(RepoRoot)/packages/*.nupkg" />
+      <PackagesToDelete Include="$(RepoRoot)/packages/*.snupkg" />
+    </ItemGroup>
+    <Delete Files="@(PackagesToDelete)" />
   </Target>
 
   <!-- AKV Targets ========================================================= -->

build.proj

@@ -75,19 +75,8 @@ jobs:
      # configuration
     - ${{ parameters.prebuildSteps }}
 
-  # If we're testing in Package mode, we have a few additional steps.
+  # If we're testing in Package mode, then we must download the Abstractions package artifacts.
   - ${{ if eq(parameters.referenceType, 'Package') }}:
-
-    # Create the packages/ directory.
-    - pwsh: New-Item -Path "$(Build.SourcesDirectory)/packages" -ItemType Directory -Force
-      displayName: Create packages/ directory
-
-    # Setup the top-level NuGet.config to look in packages/ for local NuGet
-    # dependencies.
-    - pwsh: Copy-Item -Path "$(Build.SourcesDirectory)/NuGet.config.local" -Destination "$(Build.SourcesDirectory)/NuGet.config" -Force
-      displayName: Use local NuGet packages
-
-    # Download the Abstractions package artifacts.
     - task: DownloadPipelineArtifact@2
       displayName: Download Abstractions Package Artifacts
       inputs:
@@ -127,8 +116,8 @@ jobs:
       nuspecPath: 'tools/specs/Microsoft.Data.SqlClient.nuspec'
       outputDirectory: $(packagePath)
       packageVersion: ${{ parameters.mdsPackageVersion }}
-      referenceType: ${{ parameters.referenceType }}
       properties: 'AbstractionsPackageVersion=${{ parameters.abstractionsPackageVersion }}'
+      referenceType: ${{ parameters.referenceType }}
 
   - template: /eng/pipelines/common/templates/steps/ci-project-build-step.yml@self
     parameters:
@@ -148,8 +137,8 @@ jobs:
       nuspecPath: 'tools/specs/add-ons/Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider.nuspec'
       outputDirectory: $(packagePath)
       packageVersion: $(akvPackageVersion)
-      referenceType: ${{ parameters.referenceType }}
       properties: 'MdsPackageVersion=${{ parameters.mdsPackageVersion }}'
+      referenceType: ${{ parameters.referenceType }}
 
   - task: PublishPipelineArtifact@1
     displayName: 'Publish Pipeline Artifacts'

eng/pipelines/common/templates/jobs/ci-build-nugets-job.yml

@@ -44,9 +44,6 @@ jobs:
 
   - ${{parameters.downloadPackageStep }}
 
-  - pwsh: Copy-Item -Path "$(Build.SourcesDirectory)/NuGet.config.local" -Destination "$(Build.SourcesDirectory)/NuGet.config" -Force
-    displayName: Use local NuGet packages
-
   - template: /eng/pipelines/common/templates/steps/update-config-file-step.yml
     parameters:
       TCPConnectionString: $(SQL_TCP_CONN_STRING)

eng/pipelines/common/templates/jobs/run-tests-package-reference-job.yml

@@ -28,14 +28,6 @@ steps:
   # Install the .NET SDK.
   - template: /eng/pipelines/steps/install-dotnet.yml@self
 
-  # Configure NuGet to use the local packages/ directory where the Abstractions
-  # package will be.
-  - pwsh: New-Item -Path "$(packageFolderName)" -ItemType Directory -Force
-    displayName: Create packages/ directory
-
-  - pwsh: Copy-Item -Path "$(Build.SourcesDirectory)/NuGet.config.local" -Destination "$(Build.SourcesDirectory)/NuGet.config" -Force
-    displayName: Use local NuGet packages
-
   - task: MSBuild@1
     displayName: 'BuildAllConfigurations using build.proj'
     inputs:

eng/pipelines/common/templates/steps/build-all-configurations-signed-dlls-step.yml

@@ -22,33 +22,36 @@ steps:
       # define file to update
       $NugetCfg = Join-Path -Path '.' -ChildPath 'NuGet.config'
       type $NugetCfg
-            
+
       # load content of xml from file defined above
       $xml = New-Object XML
       $xml.Load($NugetCfg)
-              
+
       # define namespace used to read a node
       $nsm = New-Object Xml.XmlNamespaceManager($xml.NameTable)
       $nsm.AddNamespace('ns', $xml.DocumentElement.NamespaceURI)
-              
+
       # get the package sources node
       $packageSources = $xml.SelectSingleNode('//ns:packageSources', $nsm)
-              
+
       # define new package source
       $newSource = $xml.CreateElement("add")
       $newSource.SetAttribute("key","SNIValidation")
       $newSource.SetAttribute("value","${{parameters.SNIValidationFeed}}")
-              
+
       # add the new package source
       $packageSources.AppendChild($newSource)
-              
+
       # save the xml file
       $xml.Save($NugetCfg)
       type $NugetCfg
 - task: PowerShell@2
   displayName: Update SNI Version in Versions.props
   inputs:
     targetType: inline
+    # TODO(https://sqlclientdrivers.visualstudio.com/ADO.Net/_workitems/edit/42204):
+    # Package dependency versions have moved to Directory.Packages.props, so the below script no
+    # longer functions.
     script: |
       Write-Host "SNI Version to test = ${{parameters.SNIVersion}}"
 
@@ -60,7 +63,7 @@ steps:
       $xml = New-Object XML
       $xml.Load($PropsPath)
 
-      # define namespace used to read a node 
+      # define namespace used to read a node
       $nsm = New-Object Xml.XmlNamespaceManager($xml.NameTable)
       $nsm.AddNamespace('ns', $xml.DocumentElement.NamespaceURI)
 

eng/pipelines/common/templates/steps/override-sni-version.yml

@@ -226,12 +226,6 @@ stages:
           - pwsh: 'Get-ChildItem env: | Sort-Object Name'
             displayName: '[Debug] List Environment Variables'
 
-        - ${{if eq(parameters.referenceType, 'Package')}}:
-          - pwsh: New-Item -Path "$(Build.SourcesDirectory)/packages" -ItemType Directory -Force
-            displayName: Create packages/ directory
-          - pwsh: Copy-Item -Path "$(Build.SourcesDirectory)/NuGet.config.local" -Destination "$(Build.SourcesDirectory)/NuGet.config" -Force
-            displayName: Use local NuGet packages
-
       # Include the code coverage job if the build type is Project.
       ${{ if eq(parameters.referenceType, 'Project') }}:
          # Jobs to run as part of the tests stage, after the tests are done.