Address AutoREST byRef preposition truncation (#915)

* Tweak ByRef OperationId.

* Download OpenAPI docs.

* Move logic to tweak OpenAPI files to a new files.

* Update readme.graph.md

* Read files on our own to reduce read time.

* Fix debug formatting

* Add _top and _count placeholders if not present.

* Update group directives

* Move regex pattern outside the loop

Author: peombwa

src/DeviceManagement/DeviceManagement/readme.md

@@ -60,9 +60,18 @@ directive:
   - where:
       subject: ^(DeviceManagementUserExperienceAnalyticAppHealthApplicationPerformance)$
       variant: ^Create2$|^CreateExpanded2$|^Get2$|^GetViaIdentity2$|^List2$|^Delete2$|^DeleteViaIdentity2$|^Update2$|^UpdateExpanded2$|^UpdateViaIdentity2$|^UpdateViaIdentityExpanded2$
+    set:
+      subject: $1ByAppVersionDetail
+  - where:
+      subject: ^(DeviceManagementUserExperienceAnalyticAppHealthApplicationPerformance)$
+      variant: ^Create3$|^CreateExpanded3$|^Get3$|^GetViaIdentity3$|^List3$|^Delete3$|^DeleteViaIdentity3$|^Update3$|^UpdateExpanded3$|^UpdateViaIdentity3$|^UpdateViaIdentityExpanded3$
+    set:
+      subject: $1ByAppVersionDeviceId
+  - where:
+      subject: ^(DeviceManagementUserExperienceAnalyticAppHealthApplicationPerformance)$
+      variant: ^Create4$|^CreateExpanded4$|^Get4$|^GetViaIdentity4$|^List4$|^Delete4$|^DeleteViaIdentity4$|^Update4$|^UpdateExpanded4$|^UpdateViaIdentity4$|^UpdateViaIdentityExpanded4$
     set:
       subject: $1ByOSVersion
-
   - where:
       subject: ^(DeviceManagementAndroid)$
       variant: ^Create$|^CreateExpanded$|^Get$|^GetViaIdentity$|^List$|^Delete$|^DeleteViaIdentity$|^Update$|^UpdateExpanded$|^UpdateViaIdentity$|^UpdateViaIdentityExpanded$

src/readme.graph.md

@@ -405,11 +405,7 @@ directive:
       variant: ^(Check|Verify)(.*)
     set:
       verb: Confirm
-# Rename all /$ref cmdlets to *ByRef e.g. New-MgGroupOwnerByRef
-  - from: 'openapi-document'
-    where: $.paths..operationId
-    transform: |
-      return ($.endsWith("ByRef")) ? $.replace("_", "GraphRef_") : $
+# Add ByRef suffix to /$ref cmdlets
   - where:
       subject: ^(\w*[a-z])GraphRef([A-Z]\w*)$
     set:
@@ -423,19 +419,19 @@ directive:
   - where:
       verb: Get|New
       subject: ^GroupMemberByRef$
-      variant: ^List$|^Create$|^CreateExpanded$|^CreateViaIdentity$|^CreateViaIdentityExpanded$|^List3$|^Create3$|^CreateExpanded3$|^CreateViaIdentity3$|^CreateViaIdentityExpanded3$
+      variant: ^List2$|^Create2$|^CreateExpanded2$|^CreateViaIdentity2$|^CreateViaIdentityExpanded2$|^List5$|^Create5$|^CreateExpanded5$|^CreateViaIdentity5$|^CreateViaIdentityExpanded5$
     set:
       subject: GroupMemberOfByRef
   - where:
       verb: Get|New
       subject: ^GroupMemberByRef$
-      variant: ^List2$|^Create2$|^CreateExpanded2$|^CreateViaIdentity2$|^CreateViaIdentityExpanded2$|^List5$|^Create5$|^CreateExpanded5$|^CreateViaIdentity5$|^CreateViaIdentityExpanded5$
+      variant: ^List1$|^Create1$|^CreateExpanded1$|^CreateViaIdentity1$|^CreateViaIdentityExpanded1$|^List4$|^Create4$|^CreateExpanded4$|^CreateViaIdentity4$|^CreateViaIdentityExpanded4$
     set:
       subject: GroupMemberWithLicenseErrorByRef
   - where:
-      verb: Get
+      verb: Get|New
       subject: ^GroupTransitiveMemberByRef$
-      variant: ^List$|^List2$
+      variant: ^List$|^List2$|^Create$|^Create2$|^CreateExpanded$|^CreateExpanded2$|^CreateViaIdentity$|^CreateViaIdentity2$|^CreateViaIdentityExpanded$|^CreateViaIdentityExpanded2$
     set:
       subject: GroupTransitiveMemberOfByRef
 # Alias then rename cmdlets to avoid breaking change.
@@ -558,7 +554,9 @@ directive:
           $ = $.replace(psBaseClassImplementationRegex, '$1Microsoft.Graph.PowerShell.Cmdlets.Custom.ListCmdlet');
 
           let beginProcessingRegex = /(^\s*)(protected\s*override\s*void\s*BeginProcessing\(\)\s*{)/gmi
-          $ = $.replace(beginProcessingRegex, '$1$2\n$1  if (this.InvocationInformation?.BoundParameters != null){ InitializeCmdlet(ref this.__invocationInfo, ref this._top, ref this._count); }\n$1');
+          let topPlaceholder = (!$.includes("private int _top;")) ? 'int _top = default;': ''
+          let countPlaceholder = (!$.includes("SwitchParameter _count;")) ? 'global::System.Management.Automation.SwitchParameter _count;': ''
+          $ = $.replace(beginProcessingRegex, `$1$2\n$1 ${countPlaceholder} ${topPlaceholder} if (this.InvocationInformation?.BoundParameters != null){ InitializeCmdlet(ref this.__invocationInfo, ref _top, ref _count); }\n$1`);
 
           let odataNextLinkCallRegex = /(^\s*)(await\s*this\.Client\.UsersUserListUser_Call\(requestMessage\,\s*onOk\,\s*onDefault\,\s*this\,\s*Pipeline\)\;)/gmi
           $ = $.replace(odataNextLinkCallRegex, '$1requestMessage.RequestUri = GetOverflowItemsNextLinkUri(requestMessage.RequestUri);\n$1$2');

tools/DownloadOpenApiDoc.ps1

@@ -1,3 +1,4 @@
+[CmdletBinding()]
 Param(
     [string] $ModuleName,
     [string] $ModuleRegex,
@@ -10,13 +11,13 @@ Param(
 if (-not (Test-Path $OpenApiDocOutput)) {
     New-Item -Path $OpenApiDocOutput -Type Directory
 }
-  
+
 $OpenApiBaseUrl = "https://graphexplorerapi.azurewebsites.net"
 $OpenApiServiceUrl = ("$OpenApiBaseUrl/`$openapi?tags={0}&title=$ModuleName&openapiversion=3&style=Powershell&graphVersion=$GraphVersion" -f $ModuleRegex)
 if ($ForceRefresh.IsPresent) {
     $OpenApiServiceUrl = "$OpenApiServiceUrl&forceRefresh=true"
 }
-Write-Host -ForegroundColor Green "[$RequestCount] Downloading OpenAPI doc for '$ModuleName' module: $OpenApiServiceUrl"
+Write-Debug "[$RequestCount] Downloading OpenAPI doc for '$ModuleName' module: $OpenApiServiceUrl"
 $Retries = 3
 $Delay = 3
 $Retrycount = 0
@@ -45,7 +46,7 @@ while (-not $Completed) {
                         $mPow = [math]::Pow(2, $Retrycount)
                         $DelayInSeconds = $mPow * $Delay
                     }
-    
+
                     Write-Warning "Request to $OpenApiServiceUrl failed. Retrying in $DelayInSeconds seconds."
                     Start-Sleep $DelayInSeconds
                     $Retrycount++

tools/TweakOpenApi.ps1

@@ -0,0 +1,38 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+[CmdletBinding()]
+Param(
+    [Parameter(Mandatory = $true)]
+    [ValidateNotNullOrEmpty()]
+    [string] $OpenAPIFilesPath
+)
+
+$TargetWord = "ByRef"
+$ReplacementWord = "GraphRef"
+$actionPattern = [Regex]::new("(_[A-Z][a-z]+)", "Compiled")
+$Stopwatch = [system.diagnostics.stopwatch]::StartNew()
+Get-ChildItem -Path $OpenAPIFilesPath | ForEach-Object {
+    $filePath = $_.FullName
+    $modified = $false
+    $updatedContent = Get-Content $filePath | ForEach-Object {
+        # Tweak '/$ref' operationIds (*ByRef) to byPass -> https://github.com/Azure/autorest.powershell/issues/795.
+        if ($_.endsWith($TargetWord) -and $_.contains("operationId:")) {
+            $operationId = $_
+            # Matches '_{ActionName}' in operationIds formatted as '{entitySet}_{ActionName}{NavigationProperty}'. e.g.,
+            # For 'applications_GetCreatedOnBehalfOfByRef', we will match '_Get'.
+            $match = $actionPattern.Match($operationId)
+            if ($null -ne $match -and $match.Success) {
+                $operationId = $operationId.Replace($TargetWord, "")
+                # Suffix matched '_{ActionName}' with 'GraphRef'. GraphRef will be our safe unique word for identifying *ByRef commands in PowerShell e.g.,
+                # 'applications_GetCreatedOnBehalfOfByRef' will be renamed to 'applications_GetGraphRefCreatedOnBehalfOf'.
+                $operationId = ($operationId -replace $match.Value, "$($match.Value)$ReplacementWord")
+                $modified = $true
+            }
+            return $operationId
+        }
+        return $_
+    }
+    if ($modified) { $updatedContent | Out-File $filePath -Force }
+}
+$stopwatch.Stop()
+Write-Debug "Tweaked '$OpenAPIFilesPath' OpenAPI files in '$($Stopwatch.Elapsed.TotalMinutes)' minutes."

tools/UpdateOpenApi.ps1

@@ -1,5 +1,6 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
+[CmdletBinding()]
 Param(
     [string] $ModuleMappingConfigPath = (Join-Path $PSScriptRoot "..\config\ModulesMapping.jsonc"),
     [string] $OpenApiDocOutput = (Join-Path $PSScriptRoot "..\openApiDocs"),
@@ -11,8 +12,11 @@ $LASTEXITCODE = $null
 if ($PSEdition -ne 'Core') {
     Write-Error 'This script requires PowerShell Core to execute. [Note] Generated cmdlets will work in both PowerShell Core or Windows PowerShell.'
 }
-# Install Powershell-yaml
-Install-Module powershell-yaml -Force
+
+if (!(Get-Module powershell-yaml -ListAvailable)){
+    # Install Powershell-yaml
+    Install-Module powershell-yaml -Force
+}
 
 $GraphVersion = "v1.0"
 if ($BetaGraphVersion) {
@@ -21,15 +25,16 @@ if ($BetaGraphVersion) {
 
 $OpenApiDocOutput = Join-Path $OpenApiDocOutput $GraphVersion
 
-# PS Scripts
+# Load PS Scripts
 $DownloadOpenApiDocPS1 = Join-Path $PSScriptRoot ".\DownloadOpenApiDoc.ps1" -Resolve
+$TweakOpenApiPs1 = Join-Path $PSScriptRoot ".\TweakOpenApi.ps1" -Resolve
 
 if (-not (Test-Path $ModuleMappingConfigPath)) {
     Write-Error "Module mapping file not be found: $ModuleMappingConfigPath."
 }
-
+$Stopwatch = [system.diagnostics.stopwatch]::StartNew()
 [HashTable] $ModuleMapping = Get-Content $ModuleMappingConfigPath | ConvertFrom-Json -AsHashTable
-$ModuleMapping.Keys | ForEach-Object -Begin { $RequestCount = 0 } -End { Write-Host -ForeGroundColor Green "Requests: $RequestCount" } -Process {
+$ModuleMapping.Keys | ForEach-Object -Begin { $RequestCount = 0 } -End { Write-Debug "Requests: $RequestCount" } -Process {
     $ModuleName = $_
     $ForceRefresh = $false
     # Check whether ForceRefresh is required, Only required for the First Request.
@@ -46,5 +51,9 @@ $ModuleMapping.Keys | ForEach-Object -Begin { $RequestCount = 0 } -End { Write-H
     }
     $RequestCount++
 }
+$stopwatch.Stop()
+Write-Debug "Downloaded $GraphVersion OpenAPI files in '$($Stopwatch.Elapsed.TotalMinutes)` minutes."
 
+# Tweak OperationIds to address known AutoREST.PowerShell limitations.
+. $TweakOpenApiPs1 -OpenAPIFilesPath $OpenApiDocOutput
 Write-Host -ForegroundColor Green "-------------Done-------------"