From 7b262fae45822e4d69abcb99d0e665577464b832 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Tue, 25 Mar 2025 17:20:37 +0800 Subject: [PATCH 1/5] optimize for batch run --- scripts/automation_generate.sh | 4 ++++ .../packaging_tools/generate_utils.py | 20 +++++++++++++++++-- .../packaging_tools/sdk_generator.py | 7 ++++++- 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/scripts/automation_generate.sh b/scripts/automation_generate.sh index a913503893b2..0b23922c9bd7 100644 --- a/scripts/automation_generate.sh +++ b/scripts/automation_generate.sh @@ -6,6 +6,10 @@ PATH="$VIRTUAL_ENV/bin:$PATH" export PATH TEMP_FILE="$TMPDIR/venv-sdk/auto_temp.json" + +# Add runInPipeline property to the JSON file +jq '. + {"runInPipeline": "true"}' "$1" > "$1.tmp" && mv "$1.tmp" "$1" + # generate code python -m packaging_tools.sdk_generator "$1" "$TEMP_FILE" --debug 2>&1 echo "[Generate] codegen done!!!" diff --git a/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/tools/azure-sdk-tools/packaging_tools/generate_utils.py index 4c24616d2730..61e989881060 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -429,13 +429,29 @@ def generate_ci(template_path: Path, folder_path: Path, package_name: str) -> No file_out.writelines(content) -def gen_typespec(typespec_relative_path: str, spec_folder: str, head_sha: str, rest_repo_url: str) -> Dict[str, Any]: +def gen_typespec( + typespec_relative_path: str, + spec_folder: str, + head_sha: str, + rest_repo_url: str, + run_in_pipeline: bool, + first_run: bool, +) -> Dict[str, Any]: typespec_python = "@azure-tools/typespec-python" # call scirpt to generate sdk try: tsp_dir = (Path(spec_folder) / typespec_relative_path).resolve() repo_url = rest_repo_url.replace("https://github.com/", "") - cmd = f"tsp-client init --tsp-config {tsp_dir} --local-spec-repo {tsp_dir} --commit {head_sha} --repo {repo_url} --debug" + cmd = ( + f"tsp-client init --tsp-config {tsp_dir} --local-spec-repo {tsp_dir} --commit {head_sha} --repo {repo_url}" + ) + if run_in_pipeline: + if first_run: + _LOGGER.info("install dependencies only for the first run") + check_output("tsp-client install-dependencies", stderr=STDOUT, shell=True) + cmd += " --skip-install --debug" + else: + cmd += " --debug" _LOGGER.info(f"generation cmd: {cmd}") output = check_output(cmd, stderr=STDOUT, shell=True) except CalledProcessError as e: diff --git a/tools/azure-sdk-tools/packaging_tools/sdk_generator.py b/tools/azure-sdk-tools/packaging_tools/sdk_generator.py index d66ddc2e3d67..673a2cc2d346 100644 --- a/tools/azure-sdk-tools/packaging_tools/sdk_generator.py +++ b/tools/azure-sdk-tools/packaging_tools/sdk_generator.py @@ -220,6 +220,8 @@ def main(generate_input, generate_output): python_tag = data.get("python_tag") package_total = set() readme_and_tsp = data.get("relatedReadmeMdFiles", []) + data.get("relatedTypeSpecProjectFolder", []) + run_in_pipeline = data.get("runInPipeline", False) + first_run = True for readme_or_tsp in readme_and_tsp: _LOGGER.info(f"[CODEGEN]({readme_or_tsp})codegen begin") try: @@ -243,7 +245,10 @@ def main(generate_input, generate_output): config = gen_dpg(readme_or_tsp, data.get("autorestConfig", ""), dpg_relative_folder(spec_folder)) else: del_outdated_generated_files(str(Path(spec_folder, readme_or_tsp))) - config = gen_typespec(readme_or_tsp, spec_folder, data["headSha"], data["repoHttpsUrl"]) + config = gen_typespec( + readme_or_tsp, spec_folder, data["headSha"], data["repoHttpsUrl"], run_in_pipeline, first_run + ) + first_run = False except Exception as e: _LOGGER.error(f"fail to generate sdk for {readme_or_tsp}: {str(e)}") for hint_message in [ From 39376586b9c8adc4593f155389e8a8ede8e44307 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Tue, 25 Mar 2025 17:31:22 +0800 Subject: [PATCH 2/5] update --- tools/azure-sdk-tools/packaging_tools/generate_utils.py | 3 +-- tools/azure-sdk-tools/packaging_tools/sdk_generator.py | 4 +--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/tools/azure-sdk-tools/packaging_tools/generate_utils.py index 61e989881060..1921f1063b70 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -435,7 +435,6 @@ def gen_typespec( head_sha: str, rest_repo_url: str, run_in_pipeline: bool, - first_run: bool, ) -> Dict[str, Any]: typespec_python = "@azure-tools/typespec-python" # call scirpt to generate sdk @@ -446,7 +445,7 @@ def gen_typespec( f"tsp-client init --tsp-config {tsp_dir} --local-spec-repo {tsp_dir} --commit {head_sha} --repo {repo_url}" ) if run_in_pipeline: - if first_run: + if not os.path.exists("node_modules/@azure-tools/typespec-python"): _LOGGER.info("install dependencies only for the first run") check_output("tsp-client install-dependencies", stderr=STDOUT, shell=True) cmd += " --skip-install --debug" diff --git a/tools/azure-sdk-tools/packaging_tools/sdk_generator.py b/tools/azure-sdk-tools/packaging_tools/sdk_generator.py index 673a2cc2d346..47eb1752a9a0 100644 --- a/tools/azure-sdk-tools/packaging_tools/sdk_generator.py +++ b/tools/azure-sdk-tools/packaging_tools/sdk_generator.py @@ -221,7 +221,6 @@ def main(generate_input, generate_output): package_total = set() readme_and_tsp = data.get("relatedReadmeMdFiles", []) + data.get("relatedTypeSpecProjectFolder", []) run_in_pipeline = data.get("runInPipeline", False) - first_run = True for readme_or_tsp in readme_and_tsp: _LOGGER.info(f"[CODEGEN]({readme_or_tsp})codegen begin") try: @@ -246,9 +245,8 @@ def main(generate_input, generate_output): else: del_outdated_generated_files(str(Path(spec_folder, readme_or_tsp))) config = gen_typespec( - readme_or_tsp, spec_folder, data["headSha"], data["repoHttpsUrl"], run_in_pipeline, first_run + readme_or_tsp, spec_folder, data["headSha"], data["repoHttpsUrl"], run_in_pipeline ) - first_run = False except Exception as e: _LOGGER.error(f"fail to generate sdk for {readme_or_tsp}: {str(e)}") for hint_message in [ From 3482fe3af9647094ba58d487a72b9aa017f48fcc Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 11 Apr 2025 05:40:34 +0000 Subject: [PATCH 3/5] update --- scripts/automation_init.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/automation_init.sh b/scripts/automation_init.sh index c1a161a5d801..259f8f46cc76 100644 --- a/scripts/automation_init.sh +++ b/scripts/automation_init.sh @@ -9,7 +9,7 @@ pip install setuptools==78.1.0 > /dev/null # install tsp-client globally (local install may interfere with tooling) echo Install tsp-client -sudo npm install -g @azure-tools/typespec-client-generator-cli > /dev/null +sudo npm install -g @azure-tools/typespec-client-generator-cli@0.20.0 > /dev/null echo "{}" >> $2 echo "[Generate] init success!!!" From b12216e602d1c0cec56721d1ce27a52eb15bba21 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 11 Apr 2025 05:46:58 +0000 Subject: [PATCH 4/5] add log --- tools/azure-sdk-tools/packaging_tools/generate_utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/tools/azure-sdk-tools/packaging_tools/generate_utils.py index 455c9e4161e4..fcffd5bf02df 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -432,9 +432,12 @@ def gen_typespec( f"tsp-client init --tsp-config {tsp_dir} --local-spec-repo {tsp_dir} --commit {head_sha} --repo {repo_url}" ) if run_in_pipeline: - if not os.path.exists("node_modules/@azure-tools/typespec-python"): + emitter_name = "@azure-tools/typespec-python" + if not os.path.exists(f"node_modules{emitter_name}"): _LOGGER.info("install dependencies only for the first run") check_output("tsp-client install-dependencies", stderr=STDOUT, shell=True) + else: + _LOGGER.info(f"skip install since {emitter_name} is already installed") cmd += " --skip-install --debug" else: cmd += " --debug" From e61c17982f42c35fb2e4a0e861526069b65bc286 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 11 Apr 2025 06:18:36 +0000 Subject: [PATCH 5/5] fix --- tools/azure-sdk-tools/packaging_tools/generate_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/tools/azure-sdk-tools/packaging_tools/generate_utils.py index fcffd5bf02df..b75d208b5a7d 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -433,7 +433,7 @@ def gen_typespec( ) if run_in_pipeline: emitter_name = "@azure-tools/typespec-python" - if not os.path.exists(f"node_modules{emitter_name}"): + if not os.path.exists(f"node_modules/{emitter_name}"): _LOGGER.info("install dependencies only for the first run") check_output("tsp-client install-dependencies", stderr=STDOUT, shell=True) else: