upgrade torch for rag llm embedding image (#4202)

runhli · web-flow · commit 2444ecf0f686 · 2025-05-21T14:17:22.000-07:00
* remove torch pin for rag image

* update

* remove azure.ai.ml sdk for data generation
diff --git a/assets/large_language_models/components_pipelines/data_ingestion_dataset_to_acs_basic/spec.yaml b/assets/large_language_models/components_pipelines/data_ingestion_dataset_to_acs_basic/spec.yaml
@@ -4,7 +4,7 @@ type: pipeline
 tags:
     Preview: ""
 
-version: 0.0.87
+version: 0.0.88
 name: llm_ingest_dataset_to_acs_basic
 display_name: LLM - Dataset to ACS Pipeline
 is_deterministic: false
@@ -110,7 +110,7 @@ jobs:
       properties:
         compute_specification:
           automatic: true
-    component: 'azureml:llm_rag_validate_deployments:0.0.83'
+    component: 'azureml:llm_rag_validate_deployments:0.0.84'
     identity:
       type: user_identity
     inputs:
@@ -136,7 +136,7 @@ jobs:
       properties:
         compute_specification:
           automatic: true
-    component: 'azureml:llm_rag_crack_and_chunk:0.0.80'
+    component: 'azureml:llm_rag_crack_and_chunk:0.0.81'
     inputs:
       input_data: ${{parent.inputs.input_data}}
       input_glob: ${{parent.inputs.input_glob}}
@@ -159,7 +159,7 @@ jobs:
       properties:
         compute_specification:
           automatic: true
-    component: 'azureml:llm_rag_generate_embeddings:0.0.73'
+    component: 'azureml:llm_rag_generate_embeddings:0.0.74'
     inputs:
       chunks_source:
         type: uri_folder
@@ -209,7 +209,7 @@ jobs:
       properties:
         compute_specification:
           automatic: true
-    component: 'azureml:llm_rag_update_acs_index:0.0.77'
+    component: 'azureml:llm_rag_update_acs_index:0.0.78'
     inputs:
       embeddings:
         type: uri_folder
@@ -229,7 +229,7 @@ jobs:
       properties:
         compute_specification:
           automatic: true
-    component: 'azureml:llm_rag_register_mlindex_asset:0.0.77'
+    component: 'azureml:llm_rag_register_mlindex_asset:0.0.78'
     inputs:
       storage_uri: ${{parent.jobs.create_acs_index_job.outputs.index}}
       asset_name: ${{parent.inputs.embeddings_dataset_name}}
diff --git a/assets/large_language_models/components_pipelines/data_ingestion_dataset_to_acs_user_id/spec.yaml b/assets/large_language_models/components_pipelines/data_ingestion_dataset_to_acs_user_id/spec.yaml
@@ -4,7 +4,7 @@ type: pipeline
 tags:
     Preview: ""
 
-version: 0.0.85
+version: 0.0.86
 name: llm_ingest_dataset_to_acs_user_id
 display_name: LLM - Dataset to ACS Pipeline
 is_deterministic: false
@@ -110,7 +110,7 @@ jobs:
       properties:
         compute_specification:
           automatic: true
-    component: 'azureml:llm_rag_validate_deployments:0.0.83'
+    component: 'azureml:llm_rag_validate_deployments:0.0.84'
     identity:
       type: user_identity
     inputs:
@@ -136,7 +136,7 @@ jobs:
       properties:
         compute_specification:
           automatic: true
-    component: 'azureml:llm_rag_crack_and_chunk:0.0.80'
+    component: 'azureml:llm_rag_crack_and_chunk:0.0.81'
     identity:
       type: user_identity
     inputs:
@@ -161,7 +161,7 @@ jobs:
       properties:
         compute_specification:
           automatic: true
-    component: 'azureml:llm_rag_generate_embeddings:0.0.73'
+    component: 'azureml:llm_rag_generate_embeddings:0.0.74'
     identity:
       type: user_identity
     inputs:
@@ -215,7 +215,7 @@ jobs:
       properties:
         compute_specification:
           automatic: true
-    component: 'azureml:llm_rag_update_acs_index:0.0.77'
+    component: 'azureml:llm_rag_update_acs_index:0.0.78'
     identity:
       type: user_identity
     inputs:
@@ -237,7 +237,7 @@ jobs:
       properties:
         compute_specification:
           automatic: true
-    component: 'azureml:llm_rag_register_mlindex_asset:0.0.77'
+    component: 'azureml:llm_rag_register_mlindex_asset:0.0.78'
     identity:
       type: user_identity
     inputs:
diff --git a/assets/large_language_models/components_pipelines/data_ingestion_dataset_to_faiss_basic/spec.yaml b/assets/large_language_models/components_pipelines/data_ingestion_dataset_to_faiss_basic/spec.yaml
@@ -4,7 +4,7 @@ type: pipeline
 tags:
     Preview: ""
 
-version: 0.0.86
+version: 0.0.87
 name: llm_ingest_dataset_to_faiss_basic
 display_name: LLM - Dataset to FAISS Pipeline
 is_deterministic: false
@@ -102,7 +102,7 @@ jobs:
       properties:
         compute_specification:
           automatic: true
-    component: 'azureml:llm_rag_validate_deployments:0.0.83'
+    component: 'azureml:llm_rag_validate_deployments:0.0.84'
     identity:
       type: user_identity
     inputs:
@@ -125,7 +125,7 @@ jobs:
       properties:
         compute_specification:
           automatic: true
-    component: 'azureml:llm_rag_crack_and_chunk:0.0.80'
+    component: 'azureml:llm_rag_crack_and_chunk:0.0.81'
     inputs:
       input_data: ${{parent.inputs.input_data}}
       input_glob: ${{parent.inputs.input_glob}}
@@ -148,7 +148,7 @@ jobs:
       properties:
         compute_specification:
           automatic: true
-    component: 'azureml:llm_rag_generate_embeddings:0.0.73'
+    component: 'azureml:llm_rag_generate_embeddings:0.0.74'
     inputs:
       chunks_source:
         type: uri_folder
@@ -198,7 +198,7 @@ jobs:
       properties:
         compute_specification:
           automatic: true
-    component: 'azureml:llm_rag_create_faiss_index:0.0.78'
+    component: 'azureml:llm_rag_create_faiss_index:0.0.79'
     inputs:
       embeddings:
         type: uri_folder
@@ -216,7 +216,7 @@ jobs:
       properties:
         compute_specification:
           automatic: true
-    component: 'azureml:llm_rag_register_mlindex_asset:0.0.77'
+    component: 'azureml:llm_rag_register_mlindex_asset:0.0.78'
     inputs:
       storage_uri: ${{parent.jobs.create_faiss_index_job.outputs.index}}
       asset_name: ${{parent.inputs.embeddings_dataset_name}}
diff --git a/assets/large_language_models/components_pipelines/data_ingestion_dataset_to_faiss_user_id/spec.yaml b/assets/large_language_models/components_pipelines/data_ingestion_dataset_to_faiss_user_id/spec.yaml
@@ -4,7 +4,7 @@ type: pipeline
 tags:
     Preview: ""
 
-version: 0.0.85
+version: 0.0.86
 name: llm_ingest_dataset_to_faiss_user_id
 display_name: LLM - Dataset to FAISS Pipeline
 is_deterministic: false
@@ -102,7 +102,7 @@ jobs:
       properties:
         compute_specification:
           automatic: true
-    component: 'azureml:llm_rag_validate_deployments:0.0.83'
+    component: 'azureml:llm_rag_validate_deployments:0.0.84'
     identity:
       type: user_identity
     inputs:
@@ -125,7 +125,7 @@ jobs:
       properties:
         compute_specification:
           automatic: true
-    component: 'azureml:llm_rag_crack_and_chunk:0.0.80'
+    component: 'azureml:llm_rag_crack_and_chunk:0.0.81'
     identity:
       type: user_identity
     inputs:
@@ -150,7 +150,7 @@ jobs:
       properties:
         compute_specification:
           automatic: true
-    component: 'azureml:llm_rag_generate_embeddings:0.0.73'
+    component: 'azureml:llm_rag_generate_embeddings:0.0.74'
     identity:
       type: user_identity
     inputs:
@@ -204,7 +204,7 @@ jobs:
       properties:
         compute_specification:
           automatic: true
-    component: 'azureml:llm_rag_create_faiss_index:0.0.78'
+    component: 'azureml:llm_rag_create_faiss_index:0.0.79'
     identity:
       type: user_identity
     inputs:
@@ -224,7 +224,7 @@ jobs:
       properties:
         compute_specification:
           automatic: true
-    component: 'azureml:llm_rag_register_mlindex_asset:0.0.77'
+    component: 'azureml:llm_rag_register_mlindex_asset:0.0.78'
     identity:
       type: user_identity
     inputs:
diff --git a/assets/large_language_models/rag/components/crack_and_chunk/spec.yaml b/assets/large_language_models/rag/components/crack_and_chunk/spec.yaml
@@ -4,7 +4,7 @@ type: command
 tags:
     Preview: ""
 
-version: 0.0.80
+version: 0.0.81
 name: llm_rag_crack_and_chunk
 display_name: LLM - Crack and Chunk Data
 is_deterministic: true
diff --git a/assets/large_language_models/rag/components/crack_and_chunk_and_embed/spec.yaml b/assets/large_language_models/rag/components/crack_and_chunk_and_embed/spec.yaml
@@ -4,7 +4,7 @@ type: command
 tags:
     Preview: ""
 
-version: 0.0.51
+version: 0.0.52
 name: llm_rag_crack_and_chunk_and_embed
 display_name: LLM - Crack, Chunk and Embed Data
 is_deterministic: true
diff --git a/assets/large_language_models/rag/components/crack_chunk_embed_index_and_register/spec.yaml b/assets/large_language_models/rag/components/crack_chunk_embed_index_and_register/spec.yaml
@@ -1,5 +1,5 @@
 name: llm_rag_crack_chunk_embed_index_and_register
-version: 0.0.39
+version: 0.0.40
 tags:
     Preview: ""
 
diff --git a/assets/large_language_models/rag/components/crawl_url/spec.yaml b/assets/large_language_models/rag/components/crawl_url/spec.yaml
@@ -4,7 +4,7 @@ type: command
 tags:
     Preview: ""
 
-version: 0.0.38
+version: 0.0.39
 name: llm_rag_crawl_url
 display_name: LLM - Crawl URL to Retrieve Data
 is_deterministic: true
diff --git a/assets/large_language_models/rag/components/create_faiss_index/spec.yaml b/assets/large_language_models/rag/components/create_faiss_index/spec.yaml
@@ -4,7 +4,7 @@ type: command
 tags:
     Preview: ""
 
-version: 0.0.78
+version: 0.0.79
 name: llm_rag_create_faiss_index
 display_name: LLM - Create FAISS Index
 is_deterministic: true
diff --git a/assets/large_language_models/rag/components/create_promptflow/spec.yaml b/assets/large_language_models/rag/components/create_promptflow/spec.yaml
@@ -4,7 +4,7 @@ type: command
 tags:
     Preview: ""
 
-version: 0.0.89
+version: 0.0.90
 name: llm_rag_create_promptflow
 display_name: LLM - Create Prompt Flow
 is_deterministic: true
diff --git a/assets/large_language_models/rag/components/data_import_acs/spec.yaml b/assets/large_language_models/rag/components/data_import_acs/spec.yaml
@@ -4,7 +4,7 @@ type: command
 tags:
     Preview: ""
 
-version: 0.0.74
+version: 0.0.75
 name: llm_rag_data_import_acs
 display_name: LLM - Import Data from ACS
 is_deterministic: false
diff --git a/assets/large_language_models/rag/components/generate_embeddings/spec.yaml b/assets/large_language_models/rag/components/generate_embeddings/spec.yaml
@@ -4,7 +4,7 @@ type: command
 tags:
     Preview: ""
 
-version: 0.0.73
+version: 0.0.74
 name: llm_rag_generate_embeddings
 display_name: LLM - Generate Embeddings
 is_deterministic: true
diff --git a/assets/large_language_models/rag/components/generate_embeddings_parallel/spec.yaml b/assets/large_language_models/rag/components/generate_embeddings_parallel/spec.yaml
@@ -4,7 +4,7 @@ type: parallel
 tags:
     Preview: ""
 
-version: 0.0.79
+version: 0.0.80
 name: llm_rag_generate_embeddings_parallel
 display_name: LLM - Generate Embeddings Parallel
 is_deterministic: true
diff --git a/assets/large_language_models/rag/components/git_clone/spec.yaml b/assets/large_language_models/rag/components/git_clone/spec.yaml
@@ -4,7 +4,7 @@ type: command
 tags:
     Preview: ""
 
-version: 0.0.77
+version: 0.0.78
 name: llm_rag_git_clone
 display_name: LLM - Clone Git Repo
 is_deterministic: true
diff --git a/assets/large_language_models/rag/components/image_embed_index/spec.yaml b/assets/large_language_models/rag/components/image_embed_index/spec.yaml
@@ -4,7 +4,7 @@ type: command
 tags:
     Preview: ""
 
-version: 0.0.31
+version: 0.0.32
 name: llm_rag_image_embed_index
 display_name: LLM - Embedding images with Florence 
 is_deterministic: true
diff --git a/assets/large_language_models/rag/components/qa_data_generation/spec.yaml b/assets/large_language_models/rag/components/qa_data_generation/spec.yaml
@@ -4,7 +4,7 @@ type: command
 tags:
     Preview: ""
 
-version: 0.0.78
+version: 0.0.79
 name: llm_rag_qa_data_generation
 display_name: LLM - Generate QnA Test Data
 is_deterministic: true
diff --git a/assets/large_language_models/rag/components/register_mlindex_asset/spec.yaml b/assets/large_language_models/rag/components/register_mlindex_asset/spec.yaml
@@ -4,7 +4,7 @@ type: command
 tags:
     Preview: ""
 
-version: 0.0.77
+version: 0.0.78
 name: llm_rag_register_mlindex_asset
 display_name: LLM - Register Vector Index Asset
 is_deterministic: true
diff --git a/assets/large_language_models/rag/components/register_qa_data_asset/spec.yaml b/assets/large_language_models/rag/components/register_qa_data_asset/spec.yaml
@@ -4,7 +4,7 @@ type: command
 tags:
     Preview: ""
 
-version: 0.0.70
+version: 0.0.71
 name: llm_rag_register_qa_data_asset
 display_name: LLM - Register QA Generation Data Asset
 is_deterministic: true
diff --git a/assets/large_language_models/rag/components/update_acs_index/spec.yaml b/assets/large_language_models/rag/components/update_acs_index/spec.yaml
@@ -4,7 +4,7 @@ type: command
 tags:
     Preview: ""
 
-version: 0.0.77
+version: 0.0.78
 name: llm_rag_update_acs_index
 display_name: LLM - Update ACS Index
 is_deterministic: true
diff --git a/assets/large_language_models/rag/components/update_azure_cosmos_mongo_vcore_index/spec.yaml b/assets/large_language_models/rag/components/update_azure_cosmos_mongo_vcore_index/spec.yaml
@@ -4,7 +4,7 @@ type: command
 tags:
     Preview: ""
 
-version: 0.0.33
+version: 0.0.34
 name: llm_rag_update_cosmos_mongo_vcore_index
 display_name: LLM - Update Azure Cosmos Mongo vCore Index
 is_deterministic: true
diff --git a/assets/large_language_models/rag/components/update_milvus_index/spec.yaml b/assets/large_language_models/rag/components/update_milvus_index/spec.yaml
@@ -4,7 +4,7 @@ type: command
 tags:
     Preview: ""
 
-version: 0.0.33
+version: 0.0.34
 name: llm_rag_update_milvus_index
 display_name: LLM - Update Milvus Index
 is_deterministic: true
diff --git a/assets/large_language_models/rag/components/update_pinecone_index/spec.yaml b/assets/large_language_models/rag/components/update_pinecone_index/spec.yaml
@@ -4,7 +4,7 @@ type: command
 tags:
     Preview: ""
 
-version: 0.0.45
+version: 0.0.46
 name: llm_rag_update_pinecone_index
 display_name: LLM - Update Pinecone Index
 is_deterministic: true
diff --git a/assets/large_language_models/rag/components/validate_deployments/spec.yaml b/assets/large_language_models/rag/components/validate_deployments/spec.yaml
@@ -4,7 +4,7 @@ type: command
 tags:
     Preview: ""
 
-version: 0.0.83
+version: 0.0.84
 name: llm_rag_validate_deployments
 display_name: LLM - Validate Deployments
 is_deterministic: false
diff --git a/assets/large_language_models/rag/environments/rag_embeddings/context/Dockerfile b/assets/large_language_models/rag/environments/rag_embeddings/context/Dockerfile
@@ -14,7 +14,7 @@ RUN conda env create -p $AZUREML_CONDA_ENVIRONMENT_PATH -f conda_dependencies.ya
     conda clean -a -y
 
 # Sentence Transformers CPU only install
-RUN pip install --no-cache-dir torch==2.3.1+cpu -f https://download.pytorch.org/whl/torch_stable.html && \
+RUN pip install --no-cache-dir torch==2.7.0+cpu -f https://download.pytorch.org/whl/torch && \
     pip install --no-cache-dir transformers && \
     pip install --no-cache-dir tqdm numpy scikit-learn scipy nltk sentencepiece pillow && \
     pip install --no-cache-dir sentence-transformers --no-deps
diff --git a/assets/large_language_models/rag/environments/rag_embeddings/context/conda_dependencies.yaml b/assets/large_language_models/rag/environments/rag_embeddings/context/conda_dependencies.yaml
