Add snippet updater to azpysdk (#45120)

* initial migrated script

* minor

* run black before update, refactor black

* update pipeline step

* add tests to azpysdk

* minor

* update doc

* delete old script mentions

* fail in update if black fails

* minor fix

* fix link

* dont omit mgmt

* fix, dont error if no samples dir

* move test to proper location

Author: JennyPng

doc/dev/code_snippets.md

@@ -4,7 +4,7 @@ All code snippets in README should be up to date and runnable. And we want to ha
 
 - [Define snippet definitions in the samples folder](#create-samples)
 - [In README, refer to the code snippets from the samples](#refer-samples)
-- [Run `python_snippet_updater.py` to keep them in sync](#python_snippet_updater-tool)
+- [Run `azpysdk update_snippet .` from the package directory to keep them in sync](#run-update_snippet-tool)
 
 ## Create samples
 
@@ -40,10 +40,13 @@ If you have a file in `samples\text_example_async.py` with a snippet named `asyn
 
 > Make sure you include a Python code fence within the snippet reference!
 
-## Run python_snippet_updater tool
+## Run update_snippet tool 
+
+The update_snippet tool is included in the [azpysdk CLI](https://github.com/Azure/azure-sdk-for-python/blob/main/doc/tool_usage_guide.md), which is installed with `azure-sdk-tools`.
 
 ```powershell
-python <azure-sdk-for-python>/eng/tools/azure-sdk-tools/ci_tools/snippet_update/python_snippet_updater.py <path_to_the_service>
+cd <package_dir>
+azpysdk update_snippet .
 ```
 
 The script scans the snippets in samples folder and populates snippet references in README with the snippet definitions from samples folder.

doc/tool_usage_guide.md

@@ -33,6 +33,7 @@ The following checks are available via the `azpysdk` entrypoint.
 |`samples`| Runs the package's samples. | `azpysdk samples .` |
 |`optional`| Invokes 'optional' requirements for a given package. | `azpysdk optional .` |
 |`devtest`| Tests a package against dependencies installed from a dev index. | `azpysdk devtest .` |
+|`update_snippet`| Updates README code snippets from sample files. | `azpysdk update_snippet .` |
 
 ## Common arguments
 

eng/pipelines/templates/steps/analyze.yml

@@ -147,7 +147,8 @@ steps:
 
   - template: /eng/pipelines/templates/steps/update_snippet.yml
     parameters:
-      PackageInfoFolder: $(Build.ArtifactStagingDirectory)/PackageInfo
+      ServiceDirectory: ${{ parameters.ServiceDirectory }}
+      AdditionalTestArgs: ${{ parameters.AdditionalTestArgs }}
 
   - template: ../steps/run_breaking_changes.yml
     parameters:

eng/pipelines/templates/steps/update_snippet.yml

@@ -1,38 +1,18 @@
 parameters:
-  - name: PackageInfoFolder
-    type: string
-    default: ''
+  ServiceDirectory: ''
+  AdditionalTestArgs: ''
 
 steps:
   # this step should run on python 3.9.
   # a previous step should have installed eng/ci_tools.txt to active venv
-  - pwsh: |
-      $failed = $false
-
-      if ("$(TargetingString)" -ne "null") {
-        foreach($targetedPackage in "$(TargetingString)".split(',')) {
-          $pkgInfo = Get-ChildItem -Path "${{ parameters.PackageInfoFolder }}" -Recurse -Filter "$targetedPackage.json" `
-            | Get-Content -Raw `
-            | ConvertFrom-Json
-
-          if ($pkgInfo) {
-            python eng/tools/azure-sdk-tools/ci_tools/snippet_update/python_snippet_updater.py "$(Build.SourcesDirectory)/sdk/$($pkgInfo.ServiceDirectory)/$targetedPackage"
-          }
-          else {
-            Write-Error "Unable to obtain package info for $targetedPackage."
-            $failed = $true
-          }
-
-          if ($LASTEXITCODE -ne 0) {
-            Write-Error "Failed to update snippets for $targetedPackage."
-            $failed = $true
-          }
-        }
-
-        if ($failed) {
-          Write-Error "At least one snippet update failed. Check above output for details."
-          exit 1
-        }
-      }
-    displayName: Update Snippets
+  - task: PythonScript@0
+    displayName: 'Update Snippets'
+    inputs:
+      scriptPath: 'eng/scripts/dispatch_checks.py'
+      arguments: >-
+        "$(TargetingString)"
+        --service="${{ parameters.ServiceDirectory }}"
+        --checks="update_snippet"
+        --disable-compatibility-filter
+        ${{ parameters.AdditionalTestArgs }}
     condition: and(succeededOrFailed(), ne(variables['Skip.UpdateSnippet'],'true'))

eng/tools/azure-sdk-tools/azpysdk/black.py

@@ -50,50 +50,55 @@ def run(self, args: argparse.Namespace) -> int:
 
             self.install_dev_reqs(executable, args, package_dir)
 
-            # install black
-            try:
-                install_into_venv(executable, [f"black=={BLACK_VERSION}"], package_dir)
-            except CalledProcessError as e:
-                logger.error(f"Failed to install black: {e}")
-                return e.returncode
-
             logger.info(f"Running black against {package_name}")
 
-            config_file_location = os.path.join(REPO_ROOT, "eng/black-pyproject.toml")
-
             if in_ci():
                 if not is_check_enabled(package_dir, "black", default=False):
                     logger.info(f"Package {package_name} opts-out of black check.")
                     continue
-            try:
-                run_result = subprocess.run(
-                    [
-                        executable,
-                        "-m",
-                        "black",
-                        f"--config={config_file_location}",
-                        package_dir,
-                    ],
-                    stdout=subprocess.PIPE,
-                    stderr=subprocess.PIPE,
-                    check=True,
-                )
-
-                if run_result.stderr and "reformatted" in run_result.stderr.decode("utf-8"):
-                    if in_ci():
-                        logger.info(
-                            f"The package {package_name} needs reformat. Run `azpysdk black .` locally from the package root to reformat."
-                        )
-                        results.append(1)
-                    else:
-                        logger.info(f"The package {package_name} was reformatted.")
-                else:
-                    logger.info(f"The package {package_name} is properly formatted, no files changed.")
 
-            except subprocess.CalledProcessError as e:
-                logger.error(f"Unable to invoke black for {package_name}. Ran into exception {e}.")
-                if e.stderr:
-                    logger.error(e.stderr.decode("utf-8"))
-                results.append(e.returncode)
+            result = self.format_directory(executable, package_dir)
+            if result is None:
+                # install or formatting failed, already logged
+                results.append(1)
+                continue
+
+            if result.stderr and "reformatted" in result.stderr.decode("utf-8"):
+                if in_ci():
+                    logger.info(
+                        f"The package {package_name} needs reformat. Run `azpysdk black .` locally from the package root to reformat."
+                    )
+                    results.append(1)
+                else:
+                    logger.info(f"The package {package_name} was reformatted.")
+            else:
+                logger.info(f"The package {package_name} is properly formatted, no files changed.")
 
         return max(results) if results else 0
+
+    @staticmethod
+    def format_directory(executable: str, target_dir: str) -> Optional[subprocess.CompletedProcess]:
+        """Run black on *target_dir* using the repo-wide config.
+
+        Installs the pinned black version into the environment of *executable*,
+        then formats all Python files under *target_dir*.
+        """
+        try:
+            install_into_venv(executable, [f"black=={BLACK_VERSION}"], target_dir)
+        except CalledProcessError as e:
+            logger.error(f"Failed to install black, skipping formatting: {e}")
+            return None
+
+        config_file_location = os.path.join(REPO_ROOT, "eng/black-pyproject.toml")
+        try:
+            return subprocess.run(
+                [executable, "-m", "black", f"--config={config_file_location}", target_dir],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                check=True,
+            )
+        except subprocess.CalledProcessError as e:
+            logger.error(f"Black formatting failed for {target_dir}: {e}")
+            if e.stderr:
+                logger.error(e.stderr.decode("utf-8"))
+            return None

eng/tools/azure-sdk-tools/azpysdk/main.py

@@ -40,6 +40,7 @@
 from .samples import samples
 from .devtest import devtest
 from .optional import optional
+from .update_snippet import update_snippet
 
 from ci_tools.logging import configure_logging, logger
 
@@ -115,6 +116,7 @@ def build_parser() -> argparse.ArgumentParser:
     samples().register(subparsers, [common])
     devtest().register(subparsers, [common])
     optional().register(subparsers, [common])
+    update_snippet().register(subparsers, [common])
 
     return parser