Merge branch 'main' into fix/servicebus-batch-size-cap-ICM51000000793879

Author: EldertGrootenboer

.vscode/cspell.json

@@ -111,6 +111,7 @@
     "sdk/cosmos/azure-cosmos-spark_4-1_2-13/**",
     "sdk/cosmos/azure-cosmos-spark-account-data-resolver-sample/**",
     "sdk/cosmos/fabric-cosmos-spark-auth_3/**",
+    "sdk/cosmos/fabric-cosmos-spark-auth_4-0_2-13/**",
     "sdk/cosmos/azure-cosmos-encryption/**",
     "sdk/cosmos/azure-cosmos-spark_3_2-12/**",
     "sdk/spring/azure-spring-data-cosmos/**",

eng/.docsettings.yml

@@ -83,6 +83,7 @@ known_content_issues:
   - ['sdk/cosmos/azure-cosmos-spark_4-1_2-13/README.md', '#3113']
   - ['sdk/cosmos/azure-cosmos-spark-account-data-resolver-sample/README.md', '#3113']
   - ['sdk/cosmos/fabric-cosmos-spark-auth_3/README.md', '#3113']
+  - ['sdk/cosmos/fabric-cosmos-spark-auth_4-0_2-13/README.md', '#3113']
   - ['sdk/cosmos/azure-cosmos-spark_3_2-12/dev/README.md', '#3113']
   - ['sdk/cosmos/azure-cosmos-spark_3_2-12/docs/catalog-api.md', '#3113']
   - ['sdk/cosmos/azure-cosmos-spark_3_2-12/docs/configuration-reference.md', '#3113']

eng/versioning/external_dependencies.txt

@@ -229,6 +229,7 @@ cosmos_com.google.guava:guava;33.0.0-jre
 cosmos_io.dropwizard.metrics:metrics-core;4.1.0
 cosmos_org.hdrhistogram:HdrHistogram;2.1.12
 cosmos_com.microsoft.azure.synapse:synapseutils_2.12;1.5.4
+cosmos_com.microsoft.azure.synapse:synapseutils_2.13;1.6.2
 
 ## Cosmos Spark connector under sdk\cosmos\azure-cosmos-spark_3-<version>_2-12\pom.xml
 # Cosmos Spark connector runtime dependencies - provided by Spark runtime/host

eng/versioning/version_client.txt

@@ -121,6 +121,7 @@ com.azure.cosmos.spark:azure-cosmos-spark_4;0.0.1-beta.1;0.0.1-beta.1
 com.azure.cosmos.spark:azure-cosmos-spark_4-0_2-13;4.48.0;4.49.0-beta.1
 com.azure.cosmos.spark:azure-cosmos-spark_4-1_2-13;4.48.0;4.49.0-beta.1
 com.azure.cosmos.spark:fabric-cosmos-spark-auth_3;1.1.0;1.2.0-beta.1
+com.azure.cosmos.spark:fabric-cosmos-spark-auth_4-0_2-13;1.0.0-beta.1;1.0.0-beta.1
 com.azure:azure-cosmos-tests;1.0.0-beta.1;1.0.0-beta.1
 com.azure:azure-data-appconfiguration;1.9.2;1.10.0-beta.1
 com.azure.cosmos.kafka:azure-cosmos-kafka-connect;2.10.0;2.11.0-beta.1

sdk/cosmos/ci.yml

@@ -23,6 +23,7 @@ trigger:
       - sdk/cosmos/azure-cosmos-spark_4/
       - sdk/cosmos/azure-cosmos-spark_4-1_2-13/
       - sdk/cosmos/fabric-cosmos-spark-auth_3/
+      - sdk/cosmos/fabric-cosmos-spark-auth_4-0_2-13/
       - sdk/cosmos/azure-cosmos-test/
       - sdk/cosmos/azure-cosmos-tests/
       - sdk/cosmos/azure-cosmos-kafka-connect/
@@ -43,6 +44,7 @@ trigger:
       - sdk/cosmos/azure-cosmos-spark_4/pom.xml
       - sdk/cosmos/azure-cosmos-spark_4-1_2-13/pom.xml
       - sdk/cosmos/fabric-cosmos-spark-auth_3/pom.xml
+      - sdk/cosmos/fabric-cosmos-spark-auth_4-0_2-13/pom.xml
       - sdk/cosmos/azure-cosmos-kafka-connect/pom.xml
 
 pr:
@@ -72,6 +74,7 @@ pr:
       - sdk/cosmos/azure-cosmos-spark_4/
       - sdk/cosmos/azure-cosmos-spark_4-1_2-13/
       - sdk/cosmos/fabric-cosmos-spark-auth_3/
+      - sdk/cosmos/fabric-cosmos-spark-auth_4-0_2-13/
       - sdk/cosmos/faq/
       - sdk/cosmos/azure-cosmos-kafka-connect/
     exclude:
@@ -89,6 +92,7 @@ pr:
       - sdk/cosmos/azure-cosmos-spark_4/pom.xml
       - sdk/cosmos/azure-cosmos-spark_4-1_2-13/pom.xml
       - sdk/cosmos/fabric-cosmos-spark-auth_3/pom.xml
+      - sdk/cosmos/fabric-cosmos-spark-auth_4-0_2-13/pom.xml
       - sdk/cosmos/azure-cosmos-test/pom.xml
       - sdk/cosmos/azure-cosmos-tests/pom.xml
       - sdk/cosmos/azure-cosmos-kafka-connect/pom.xml
@@ -129,6 +133,10 @@ parameters:
     displayName: 'fabric-cosmos-spark-auth_3'
     type: boolean
     default: true
+  - name: release_fabriccosmossparkauth4
+    displayName: 'fabric-cosmos-spark-auth_4-0_2-13'
+    type: boolean
+    default: true
   - name: release_azurecosmostest
     displayName: 'azure-cosmos-test'
     type: boolean
@@ -201,6 +209,13 @@ extends:
         skipPublishDocGithubIo: true
         skipPublishDocMs: true
         releaseInBatch: ${{ parameters.release_fabriccosmossparkauth3 }}
+      - name: fabric-cosmos-spark-auth_4-0_2-13
+        groupId: com.azure.cosmos.spark
+        safeName: fabriccosmossparkauth4
+        uberJar: true
+        skipPublishDocGithubIo: true
+        skipPublishDocMs: true
+        releaseInBatch: ${{ parameters.release_fabriccosmossparkauth4 }}
       - name: azure-cosmos-test
         groupId: com.azure
         safeName: azurecosmostest

sdk/cosmos/fabric-cosmos-spark-auth_4-0_2-13/CHANGELOG.md

@@ -0,0 +1,6 @@
+## Release History
+
+### 1.0.0-beta.1 (Unreleased)
+
+#### Features Added
+* Added Spark 4.0 / Scala 2.13 variant of the Fabric account data resolver for Azure Cosmos DB Spark Connector. This allows signed-in users in Fabric Runtime 2.0 to authenticate to CosmosDB using Microsoft Entra ID.

sdk/cosmos/fabric-cosmos-spark-auth_4-0_2-13/README.md

@@ -0,0 +1,19 @@
+## Azure Cosmos DB OLTP Spark 4 Connector Fabric Account Data Resolver
+
+This project provides an implementation of the `AccountDataResolver` interface for the Azure Cosmos DB Spark Connector, specifically designed to work with Azure Fabric accounts. 
+It allows a signed-in user to authenticate with Microsoft Entra ID within the Azure Fabric environment. To use this implementation, you have to upload the jar file for this project to your fabric environment. It is also necessary to grant the signed-in user's 
+identity with the correct permissions to perform data plane operations. Follow the instructions [here](https://learn.microsoft.com/azure/cosmos-db/nosql/how-to-grant-data-plane-access?tabs=built-in-definition%2Ccsharp&pivots=azure-interface-cli) to grant the relevant permissions. 
+To learn more about Microsoft Entra ID (AAD) authentication using the Azure Cosmos DB Spark Connector, see the 
+[AAD Auth Documentation](https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/cosmos/azure-cosmos-spark_3/docs/AAD-Auth.md). Fabric Native accounts only support gateway mode, so the `spark.cosmos.useGatewayMode` must be set to `true` in the Spark configuration for these accounts.
+
+### Configuration options
+
+| Config Property Name                          | Default                               | Description                                                                                                                                                |
+|:----------------------------------------------|:--------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `spark.cosmos.accountDataResolverServiceName` | None                                  | Set this value to `com.azure.cosmos.spark.fabric.FabricAccountDataResolver` to use this implementation of the `AccountDataResolver`                        |
+| `spark.cosmos.auth.aad.audience`              | `https://cosmos.azure.com/.default`   | Set this value to change the audience used to obtain the Entra Id token.                                                                                   |
+
+### Current Limitations
+
+* There is no support for the [catalog api](https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/cosmos/azure-cosmos-spark_3/docs/catalog-api.md) operations using this implementation of the `AccountDataResolver` with a CosmosDB Fabric Native Account.
+* Only Spark 4.0 is supported with this implementation of the `AccountDataResolver`.