[Pipeline] optimize for batch run in pipeline

scripts/automation_generate.sh

@@ -6,6 +6,10 @@ PATH="$VIRTUAL_ENV/bin:$PATH"
 export PATH
 
 TEMP_FILE="$TMPDIR/venv-sdk/auto_temp.json"
+
+# Add runInPipeline property to the JSON file
+jq '. + {"runInPipeline": "true"}' "$1" > "$1.tmp" && mv "$1.tmp" "$1"
+
 # generate code
 python -m packaging_tools.sdk_generator "$1" "$TEMP_FILE" --debug 2>&1
 echo "[Generate] codegen done!!!"

tools/azure-sdk-tools/packaging_tools/generate_utils.py

@@ -429,13 +429,28 @@ def generate_ci(template_path: Path, folder_path: Path, package_name: str) -> No
         file_out.writelines(content)
 
 
-def gen_typespec(typespec_relative_path: str, spec_folder: str, head_sha: str, rest_repo_url: str) -> Dict[str, Any]:
+def gen_typespec(
+    typespec_relative_path: str,
+    spec_folder: str,
+    head_sha: str,
+    rest_repo_url: str,
+    run_in_pipeline: bool,
+) -> Dict[str, Any]:
     typespec_python = "@azure-tools/typespec-python"
     # call scirpt to generate sdk
     try:
         tsp_dir = (Path(spec_folder) / typespec_relative_path).resolve()
         repo_url = rest_repo_url.replace("https://github.com/", "")
-        cmd = f"tsp-client init --tsp-config {tsp_dir} --local-spec-repo {tsp_dir} --commit {head_sha} --repo {repo_url} --debug"
+        cmd = (
+            f"tsp-client init --tsp-config {tsp_dir} --local-spec-repo {tsp_dir} --commit {head_sha} --repo {repo_url}"
+        )
+        if run_in_pipeline:
+            if not os.path.exists("node_modules/@azure-tools/typespec-python"):
+                _LOGGER.info("install dependencies only for the first run")
+                check_output("tsp-client install-dependencies", stderr=STDOUT, shell=True)
+            cmd += " --skip-install --debug"
+        else:
+            cmd += " --debug"
         _LOGGER.info(f"generation cmd: {cmd}")
         output = check_output(cmd, stderr=STDOUT, shell=True)
     except CalledProcessError as e:

tools/azure-sdk-tools/packaging_tools/sdk_generator.py

@@ -220,6 +220,7 @@ def main(generate_input, generate_output):
     python_tag = data.get("python_tag")
     package_total = set()
     readme_and_tsp = data.get("relatedReadmeMdFiles", []) + data.get("relatedTypeSpecProjectFolder", [])
+    run_in_pipeline = data.get("runInPipeline", False)
     for readme_or_tsp in readme_and_tsp:
         _LOGGER.info(f"[CODEGEN]({readme_or_tsp})codegen begin")
         try:
@@ -243,7 +244,9 @@ def main(generate_input, generate_output):
                 config = gen_dpg(readme_or_tsp, data.get("autorestConfig", ""), dpg_relative_folder(spec_folder))
             else:
                 del_outdated_generated_files(str(Path(spec_folder, readme_or_tsp)))
-                config = gen_typespec(readme_or_tsp, spec_folder, data["headSha"], data["repoHttpsUrl"])
+                config = gen_typespec(
+                    readme_or_tsp, spec_folder, data["headSha"], data["repoHttpsUrl"], run_in_pipeline
+                )
         except Exception as e:
             _LOGGER.error(f"fail to generate sdk for {readme_or_tsp}: {str(e)}")
             for hint_message in [