Official Build Cleanup (#4123)

* Cleanup step parameters

* Parameter work for build-signed-csproj-package-job.yml:
* Drop buildConfiguration parameter
* Introduce symbols parameters
* Signing parameters with prefix

* Forward parameter defaults from build-signed-csproj-package-job to build-stages.yml

* Forward default values for symbols/signing parameters in build-stages.yml to parameters.

* Delete debug param from build-stages.yml
Forward default values to parameters on sqlclient-non-official.yml and sqlclient-official.yml

* Don't nest variables files.

* Do not import variables templates outside of the pipeline root

* OneBranch variable libraries moved to onebranch-variables.yml
Replace old libraries with new libraries
Replace old library references with new library references

* Project for manintaining the pipelines folder

* Parameterize package versions from the pipeline to the stages

* Store "effective" package versions in new, parameterized variables file

* Parameterize the artifact names

* Parameterize the assembly versions

* Parameterize the mds validation job

* Sweep remaining $() references that are not to well-known variables.

* Remove unused variables and cleanup remaining variables

* Cleanup SDL parameters

* AKVProvider -> AkvProvider

* MDS -> SqlClient

Author: benrr101

eng/pipelines/Pipelines.csproj

@@ -0,0 +1,28 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <!--
+    This is an ultra-hack. We have a lot of files in the /eng/pipelines directory, and it would be
+    nice to manage these inside an IDE. However, MSBuild doesn't have a good solution for including
+    recursive folder trees of non-compilable files, with the only solution to be manually adding
+    each file as a linked file in the solution. This is a terrible situation because it means each
+    time a file is added or removed from this folder, it must be manually updated in the solution.
+    It is also the case that most engineers forget to do this step, and it comes down to more
+    diligent engineers to keep the pipelines folder up to date. Well, the diligent engineers have
+    had enough and are tired of doing it.
+
+    So, this project was added as a way to automatically keep the pipelines directory up-to-date in
+    the solution. It is *not built* in any configuration, and exists purely to collect all files
+    under the /eng/pipelines folder.
+
+    No changes should be made to this project, perhaps updating the TargetFramework property if
+    the framework specified goes out of support.
+  -->
+
+  <PropertyGroup>
+    <TargetFramework>net10.0</TargetFramework>
+    <NoBuild>true</NoBuild>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <None Include="**/*" />
+  </ItemGroup>
+</Project>

eng/pipelines/onebranch/jobs/build-signed-csproj-package-job.yml

@@ -15,6 +15,49 @@
 # signed DLLs, and finally ESRP signs the NuGet packages.
 
 parameters:
+
+  # Signing Parameters -----------------------------------------------------
+
+  - name: signingAppRegistrationClientId
+    type: string
+
+  - name: signingAppRegistrationTenantId
+    type: string
+
+  - name: signingAuthAkvName
+    type: string
+
+  - name: signingAuthSignCertName
+    type: string
+
+  - name: signingEsrpClientId
+    type: string
+
+  - name: signingEsrpConnectedServiceName
+    type: string
+
+  # Symbols Publishing Parameters ------------------------------------------
+
+  - name: symbolsAzureSubscription
+    type: string
+    default: 'Symbols publishing Workload Identity federation service-ADO.Net'
+
+  - name: symbolsPublishProjectName
+    type: string
+    default: 'Microsoft.Data.SqlClient.SNI'
+
+  - name: symbolsPublishServer
+    type: string
+
+  - name: symbolsPublishTokenUri
+    type: string
+
+  - name: symbolsUploadAccount
+    type: string
+    default: 'SqlClientDrivers'
+
+  # OTHERS +=====================================
+
   # Short package name used in the job name, display strings, filesystem paths, and as a suffix for
   # the default Build and Pack targets if those aren't specified.
   - name: packageName
@@ -42,20 +85,17 @@ parameters:
     type: string
     default: ""
 
+  # True to enable ESRP malware scanning and code signing steps, which should not be
+  # run on non-official pipelines as they access production resources.
+  - name: isOfficial
+    type: boolean
+
   # The MSBuild pack target in build.proj (e.g. PackLogging).  If not specified, defaults to
   # Pack<packageName>.
   - name: packTarget
     type: string
     default: ""
 
-  # The C# build configuration to build (e.g. Debug or Release).
-  - name: buildConfiguration
-    type: string
-    values:
-      - Debug
-      - Release
-    default: Release
-
   # Additional MSBuild -p: arguments for version properties.  These may include versions of
   # packages this package depends on, or versions for this package itself.
   - name: versionProperties
@@ -70,30 +110,6 @@ parameters:
   - name: publishSymbols
     type: boolean
 
-  # True to enable ESRP malware scanning and code signing steps, which should not be
-  # run on non-official pipelines as they access production resources.
-  - name: isOfficial
-    type: boolean
-
-  # Values required by ESRP tasks.
-  - name: esrpConnectedServiceName
-    type: string
-
-  - name: esrpClientId
-    type: string
-
-  - name: appRegistrationClientId
-    type: string
-
-  - name: appRegistrationTenantId
-    type: string
-
-  - name: authAkvName
-    type: string
-
-  - name: authSignCertName
-    type: string
-
   # Optional list of pipeline artifacts to download before building.  Each entry is an object
   # with 'artifactName' (the pipeline artifact name) and 'displayName' (used in the task label).
   # This replaces hard-coded packageName conditionals so callers declare their own dependencies.
@@ -108,15 +124,20 @@ jobs:
       type: windows
 
     variables:
+      # Inform OneBranch that files put in this directory should be uploaded as artifacts.
       ob_outputDirectory: $(PACK_OUTPUT)
+
       # APIScan configuration for this Extension package
       ob_sdl_apiscan_enabled: true
-      ob_sdl_apiscan_softwareFolder: $(Build.SourcesDirectory)/apiScan/${{ parameters.packageName }}/dlls
-      ob_sdl_apiscan_symbolsFolder: $(Build.SourcesDirectory)/apiScan/${{ parameters.packageName }}/pdbs
+      ob_sdl_apiscan_softwareFolder: $(REPO_ROOT)/apiScan/${{ parameters.packageName }}/dlls
+      ob_sdl_apiscan_symbolsFolder: $(REPO_ROOT)/apiScan/${{ parameters.packageName }}/pdbs
       ob_sdl_apiscan_softwarename: ${{ parameters.packageFullName }}
       ob_sdl_apiscan_versionNumber: ${{ parameters.assemblyFileVersion }}
 
+      # If parameters.buildTarget is not provided, default to "Build{parameters.packageName}"
       buildTarget: ${{ coalesce(parameters.buildTarget, format('Build{0}', parameters.packageName)) }}
+
+      # If parameters.packTarget is not provided, default to "Pack{parameters.packageName}"
       packTarget: ${{ coalesce(parameters.packTarget, format('Pack{0}', parameters.packageName)) }}
 
     steps:
@@ -128,46 +149,45 @@ jobs:
             displayName: Download ${{ artifact.displayName }}
             inputs:
               artifactName: ${{ artifact.artifactName }}
-              targetPath: $(Build.SourcesDirectory)/packages
+              targetPath: $(REPO_ROOT)/packages
 
       # Install the .NET SDK.
       - template: /eng/pipelines/steps/install-dotnet.yml@self
 
       # Perform Roslyn analysis before building, since this step will clobber build output.
       - template: /eng/pipelines/onebranch/steps/roslyn-analyzers-csproj-step.yml@self
         parameters:
+          buildTarget: '${{ variables.buildTarget }}'
           msBuildArguments: >-
-            -t:$(buildTarget)
-            -p:Configuration=${{ parameters.buildConfiguration }}
+            -p:Configuration=Release
             -p:ReferenceType=Package
             ${{ parameters.versionProperties }}
 
       # Build the package, producing DLLs only (no NuGet package yet).
       - template: /eng/pipelines/onebranch/steps/build-csproj-step.yml@self
         parameters:
-          buildTarget: $(buildTarget)
-          buildConfiguration: ${{ parameters.buildConfiguration }}
+          buildTarget: '${{ variables.buildTarget }}'
           versionProperties: ${{ parameters.versionProperties }}
 
       - ${{ if eq(parameters.isOfficial, true) }}:
           # ESRP sign the DLLs.
           - template: /eng/pipelines/onebranch/steps/esrp-dll-signing-step.yml@self
             parameters:
-              appRegistrationClientId: ${{ parameters.appRegistrationClientId }}
-              appRegistrationTenantId: ${{ parameters.appRegistrationTenantId }}
-              authAkvName: ${{ parameters.authAkvName }}
-              authSignCertName: ${{ parameters.authSignCertName }}
-              esrpClientId: ${{ parameters.esrpClientId }}
-              esrpConnectedServiceName: ${{ parameters.esrpConnectedServiceName }}
-              pattern: ${{ parameters.packageFullName }}.dll
+              appRegistrationClientId: '${{ parameters.signingAppRegistrationClientId }}'
+              appRegistrationTenantId: '${{ parameters.signingAppRegistrationTenantId }}'
+              authAkvName: '${{ parameters.signingAuthAkvName }}'
+              authSignCertName: '${{ parameters.signingAuthSignCertName }}'
+              esrpClientId: '${{ parameters.signingEsrpClientId }}'
+              esrpConnectedServiceName: '${{ parameters.signingEsrpConnectedServiceName }}'
+              pattern: '${{ parameters.packageFullName }}.dll'
 
       # Copy signed/unsigned DLLs and PDBs to APIScan folders.
       - task: CopyFiles@2
         displayName: Copy DLLs for APIScan
         inputs:
           SourceFolder: $(BUILD_OUTPUT)/Package/bin
           Contents: "**/${{ parameters.packageFullName }}.dll"
-          TargetFolder: $(ob_sdl_apiscan_softwareFolder)
+          TargetFolder: ${{ variables.ob_sdl_apiscan_softwareFolder }}
           # We must preserve the folder structure since our C# projects may produce multiple
           # identically named DLLs for different target frameworks (e.g. netstandard2.0, net5.0,
           # etc.), and we need to keep those separate for APIScan to work correctly.
@@ -178,42 +198,41 @@ jobs:
         inputs:
           SourceFolder: $(BUILD_OUTPUT)/Package/bin
           Contents: "**/${{ parameters.packageFullName }}.pdb"
-          TargetFolder: $(ob_sdl_apiscan_symbolsFolder)
+          TargetFolder: ${{ variables.ob_sdl_apiscan_symbolsFolder }}
           flattenFolders: false
 
       # Pack the signed DLLs into NuGet package (NoBuild=true).
       - template: /eng/pipelines/onebranch/steps/pack-csproj-step.yml@self
         parameters:
-          packTarget: $(packTarget)
-          buildConfiguration: ${{ parameters.buildConfiguration }}
+          packTarget: ${{ variables.packTarget }}
           versionProperties: ${{ parameters.versionProperties }}
 
       - ${{ if eq(parameters.isOfficial, true) }}:
           # ESRP sign the NuGet package.
           - template: /eng/pipelines/onebranch/steps/esrp-nuget-signing-step.yml@self
             parameters:
-              appRegistrationClientId: ${{ parameters.appRegistrationClientId }}
-              appRegistrationTenantId: ${{ parameters.appRegistrationTenantId }}
-              authAkvName: ${{ parameters.authAkvName }}
-              authSignCertName: ${{ parameters.authSignCertName }}
-              esrpClientId: ${{ parameters.esrpClientId }}
-              esrpConnectedServiceName: ${{ parameters.esrpConnectedServiceName }}
+              appRegistrationClientId: '${{ parameters.signingAppRegistrationClientId }}'
+              appRegistrationTenantId: '${{ parameters.signingAppRegistrationTenantId }}'
+              authAkvName: '${{ parameters.signingAuthAkvName }}'
+              authSignCertName: '${{ parameters.signingAuthSignCertName }}'
+              esrpClientId: '${{ parameters.signingEsrpClientId }}'
+              esrpConnectedServiceName: '${{ parameters.signingEsrpConnectedServiceName }}'
               searchPath: $(PACK_OUTPUT)
-              searchPattern: '${{ parameters.packageFullName }}.*nupkg'
+              searchPattern: '${{ parameters.packageFullName}}.*nupkg'
 
       # Publish symbols to servers
       # @TODO: Get these parameters from variables/libraries
       - ${{ if eq(parameters.publishSymbols, true) }}:
           - template: /eng/pipelines/onebranch/steps/publish-symbols-step.yml@self
             parameters:
               artifactName: '${{ parameters.packageFullName }}_symbols_$(System.TeamProject)_$(Build.Repository.Name)_$(Build.SourceBranchName)_${{ parameters.packageVersion }}_$(System.TimelineId)'
-              azureSubscription: 'Symbols publishing Workload Identity federation service-ADO.Net'
+              azureSubscription: '${{ parameters.symbolsAzureSubscription }}'
               packageName: '${{ parameters.packageFullName }}'
-              publishProjectName: 'Microsoft.Data.SqlClient.SNI' # This is used for all SqlClient packages. Don't know why, but it is.
-              publishServer: '$(SymbolServer)'
+              publishProjectName: '${{ parameters.symbolsPublishProjectName }}'
+              publishServer: '${{ parameters.symbolsPublishServer }}'
               publishToInternal: 'true'
               publishToPublic: 'true'
-              publishTokenUri: '$(SymbolTokenUri)'
+              publishTokenUri: '${{ parameters.symbolsPublishTokenUri }}'
               searchPattern: '**/${{ parameters.packageFullName }}*.pdb'
-              uploadAccount: 'SqlClientDrivers'
+              uploadAccount: '${{ parameters.symbolsUploadAccount }}'
               version: '${{ parameters.packageVersion }}'

…ch/jobs/build-signed-mds-package-job.yml …s/build-signed-sqlclient-package-job.ymleng/pipelines/onebranch/jobs/build-signed-mds-package-job.yml renamed to eng/pipelines/onebranch/jobs/build-signed-sqlclient-package-job.yml eng/pipelines/onebranch/jobs/build-signed-mds-package-job.yml renamed to eng/pipelines/onebranch/jobs/build-signed-sqlclient-package-job.yml

@@ -67,14 +67,14 @@ parameters:
   - name: loggingPackageVersion
     type: string
 
-  - name: mdsAssemblyFileVersion
+  - name: sqlClientAssemblyFileVersion
     type: string
 
-  - name: mdsPackageVersion
+  - name: sqlClientPackageVersion
     type: string
 
 jobs:
-  - job: build_package_Mds
+  - job: build_package_SqlClient
     displayName: 'Build Microsoft.Data.SqlClient'
     pool:
       type: windows
@@ -84,7 +84,7 @@ jobs:
       ob_sdl_apiscan_softwareFolder: ${{ parameters.apiScanDllPath }}
       ob_sdl_apiscan_symbolsFolder: ${{ parameters.apiScanPdbPath }}
       ob_sdl_apiscan_softwarename: 'Microsoft.Data.SqlClient'
-      ob_sdl_apiscan_versionNumber: ${{ parameters.mdsAssemblyFileVersion }}
+      ob_sdl_apiscan_versionNumber: ${{ parameters.sqlClientAssemblyFileVersion }}
 
     steps:
       # Dump environment and parameters
@@ -114,21 +114,21 @@ jobs:
       - template: /eng/pipelines/steps/install-dotnet.yml@self
 
       # Perform analysis before building, since this step will clobber build output
-      - template: /eng/pipelines/onebranch/steps/roslyn-analyzers-mds-step.yml@self
+      - template: /eng/pipelines/onebranch/steps/roslyn-analyzers-sqlclient-step.yml@self
         parameters:
           abstractionsPackageVersion: '${{ parameters.abstractionsPackageVersion }}'
           loggingPackageVersion: '${{ parameters.loggingPackageVersion }}'
-          mdsPackageVersion: '${{ parameters.mdsPackageVersion }}'
+          sqlClientPackageVersion: '${{ parameters.sqlClientPackageVersion }}'
 
       # Perform the actual build
-      - template: /eng/pipelines/onebranch/steps/build-mds-step.yml@self
+      - template: /eng/pipelines/onebranch/steps/build-sqlclient-step.yml@self
         parameters:
           abstractionsPackageVersion: '${{ parameters.abstractionsPackageVersion }}'
           loggingPackageVersion: '${{ parameters.loggingPackageVersion }}'
-          mdsPackageVersion: '${{ parameters.mdsPackageVersion }}'
+          sqlClientPackageVersion: '${{ parameters.sqlClientPackageVersion }}'
 
       # Copy the built DLLs and PDBs to the APIScan output folder for APIScanning post-build
-      - template: /eng/pipelines/onebranch/steps/copy-apiscan-files-mds-step.yml@self
+      - template: /eng/pipelines/onebranch/steps/copy-apiscan-files-sqlclient-step.yml@self
         parameters:
           dllPath: '${{ parameters.apiScanDllPath }}'
           pdbPath: '${{ parameters.apiScanPdbPath }}'
@@ -147,11 +147,11 @@ jobs:
               pattern: 'Microsoft.Data.SqlClient*.dll'
 
       # Package the build output into a NuGet package
-      - template: /eng/pipelines/onebranch/steps/pack-mds-step.yml
+      - template: /eng/pipelines/onebranch/steps/pack-sqlclient-step.yml
         parameters:
           abstractionsPackageVersion: '${{ parameters.abstractionsPackageVersion }}'
           loggingPackageVersion: '${{ parameters.loggingPackageVersion }}'
-          mdsPackageVersion: '${{ parameters.mdsPackageVersion }}'
+          sqlClientPackageVersion: '${{ parameters.sqlClientPackageVersion }}'
 
       # Sign the NuGet packages
       - ${{ if parameters.isOfficial }}:
@@ -169,7 +169,7 @@ jobs:
       - ${{ if parameters.publishSymbols }}:
           - template: /eng/pipelines/onebranch/steps/publish-symbols-step.yml@self
             parameters:
-              artifactName: 'Microsoft.Data.SqlClient_symbols_$(System.TeamProject)_$(Build.Repository.Name)_$(Build.SourceBranchName)_${{ parameters.mdsPackageVersion }}_$(System.TimelineId)'
+              artifactName: 'Microsoft.Data.SqlClient_symbols_$(System.TeamProject)_$(Build.Repository.Name)_$(Build.SourceBranchName)_${{ parameters.sqlClientPackageVersion }}_$(System.TimelineId)'
               azureSubscription: '${{ parameters.symbolsAzureSubscription }}'
               packageName: 'Microsoft.Data.SqlClient'
               publishProjectName: '${{ parameters.symbolsPublishProjectName }}'
@@ -179,4 +179,4 @@ jobs:
               publishTokenUri: '${{ parameters.symbolsPublishTokenUri }}'
               searchPattern: '**/Microsoft.Data.SqlClient*.pdb' # @TODO: This seems very heavy
               uploadAccount: '${{ parameters.symbolsUploadAccount }}'
-              version: '${{ parameters.mdsPackageVersion }}'
+              version: '${{ parameters.sqlClientPackageVersion }}'

eng/pipelines/onebranch/jobs/publish-nuget-package-job.yml

@@ -65,7 +65,7 @@ jobs:
 
     variables:
       - name: ob_outputDirectory
-        value: $(Build.SourcesDirectory)/output
+        value: $(PACK_OUTPUT)
 
       - name: artifactPath
         value: $(Pipeline.Workspace)/${{ parameters.artifactName }}