Skip to content

Commit ee7bfd7

Browse files
tvaron3tvaron3
authored
Fix Cosmos GSI live test secret Plumbing (#48038)
* Fix GSI live test secret plumbing Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 52b18c74-019a-4205-bf95-74fe08576581 * Fix GSI live test partition key projection Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 52b18c74-019a-4205-bf95-74fe08576581 * Populate source RID for GSI containers Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 52b18c74-019a-4205-bf95-74fe08576581 * Fix GSI live test response assertions Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 52b18c74-019a-4205-bf95-74fe08576581 * Normalize GSI container response properties Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 52b18c74-019a-4205-bf95-74fe08576581
1 parent dddd414 commit ee7bfd7

8 files changed

Lines changed: 124 additions & 18 deletions

File tree

‎eng/pipelines/templates/stages/archetype-sdk-tests.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@ parameters:
1414
- name: EnvVars
1515
type: object
1616
default: {}
17+
- name: VariableGroups
18+
type: object
19+
default: []
1720
- name: MaxParallel
1821
type: number
1922
default: 0
@@ -108,6 +111,9 @@ extends:
108111
- ${{ if not(contains(parameters.UnsupportedClouds, cloud.key)) }}:
109112
- stage:
110113
displayName: ${{ format('{0} {1} {2}', cloud.key, parameters.JobName, package) }}
114+
variables:
115+
- ${{ each group in parameters.VariableGroups }}:
116+
- group: ${{ group }}
111117
dependsOn: []
112118
jobs:
113119
- template: /eng/common/pipelines/templates/jobs/generate-job-matrix.yml

‎sdk/cosmos/azure-cosmos/azure/cosmos/_global_secondary_index.py‎

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,42 @@
2121

2222
"""Global Secondary Index (GSI) container definition."""
2323

24-
from typing import Optional
24+
from typing import Any, Mapping, Optional, TypeVar
25+
26+
# Wire key used by the service for the GSI/materialized-view definition.
27+
_MATERIALIZED_VIEW_DEFINITION_KEY = "materializedViewDefinition"
28+
# Public-facing key the SDK surfaces for a GSI definition.
29+
_GLOBAL_SECONDARY_INDEX_DEFINITION_KEY = "globalSecondaryIndexDefinition"
30+
31+
_PropertiesT = TypeVar("_PropertiesT", bound=Optional[Mapping[str, Any]])
32+
33+
34+
def _normalize_gsi_container_properties(properties: _PropertiesT) -> _PropertiesT:
35+
"""Surface ``globalSecondaryIndexDefinition`` from container properties.
36+
37+
Some service versions return the GSI definition under the legacy
38+
``materializedViewDefinition`` key. Promote it to
39+
``globalSecondaryIndexDefinition`` and drop the legacy key so the public
40+
contract never exposes ``materializedViewDefinition``, regardless of the
41+
backend contract version. The mutation is done in place when possible and
42+
the same object is returned for convenience.
43+
44+
:param properties: The container properties returned by the service.
45+
:type properties: Mapping[str, Any] or None
46+
:returns: The container properties with ``globalSecondaryIndexDefinition`` populated.
47+
:rtype: Mapping[str, Any] or None
48+
"""
49+
if properties is None or _MATERIALIZED_VIEW_DEFINITION_KEY not in properties:
50+
return properties
51+
try:
52+
if _GLOBAL_SECONDARY_INDEX_DEFINITION_KEY not in properties:
53+
properties[_GLOBAL_SECONDARY_INDEX_DEFINITION_KEY] = ( # type: ignore[index]
54+
properties[_MATERIALIZED_VIEW_DEFINITION_KEY])
55+
del properties[_MATERIALIZED_VIEW_DEFINITION_KEY] # type: ignore[attr-defined]
56+
except TypeError:
57+
# Read-only mapping; nothing to normalize in place.
58+
pass
59+
return properties
2560

2661

2762
class GlobalSecondaryIndexDefinition:

‎sdk/cosmos/azure-cosmos/azure/cosmos/aio/_container.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
from .._change_feed.feed_range_internal import FeedRangeInternalEpk
4444

4545
from .._cosmos_responses import CosmosDict, CosmosList, CosmosAsyncItemPaged
46+
from .._global_secondary_index import _normalize_gsi_container_properties
4647
from .._constants import _Constants as Constants, TimeoutScope
4748
from .._routing.routing_range import Range
4849
from .._session_token_helpers import get_latest_session_token
@@ -209,6 +210,7 @@ async def read(
209210
if populate_quota_info is not None:
210211
request_options["populateQuotaInfo"] = populate_quota_info
211212
container = await self.client_connection.ReadContainer(self.container_link, options=request_options, **kwargs)
213+
_normalize_gsi_container_properties(container)
212214
# Only cache Container Properties that will not change in the lifetime of the container
213215
self.client_connection._set_container_properties_cache(self.container_link, # pylint: disable=protected-access
214216
_build_properties_cache(container, self.container_link))

‎sdk/cosmos/azure-cosmos/azure/cosmos/aio/_database.py‎

Lines changed: 31 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@
4040
from ..documents import IndexingMode
4141
from ..partition_key import PartitionKey
4242
from .._cosmos_responses import CosmosDict
43-
from .._global_secondary_index import GlobalSecondaryIndexDefinition
43+
from .._global_secondary_index import GlobalSecondaryIndexDefinition, _normalize_gsi_container_properties
4444

4545

4646
__all__ = ("DatabaseProxy",)
@@ -469,9 +469,7 @@ async def create_container( # pylint:disable=docstring-should-be-keyword, too-ma
469469
if full_text_policy is not None:
470470
definition["fullTextPolicy"] = full_text_policy
471471
if global_secondary_index_definition is not None:
472-
gsi_dict = (global_secondary_index_definition._to_dict()
473-
if hasattr(global_secondary_index_definition, '_to_dict')
474-
else global_secondary_index_definition)
472+
gsi_dict = await self._resolve_gsi_definition(global_secondary_index_definition)
475473
definition["globalSecondaryIndexDefinition"] = gsi_dict
476474
definition["materializedViewDefinition"] = gsi_dict
477475
request_options = _build_options(kwargs)
@@ -480,6 +478,7 @@ async def create_container( # pylint:disable=docstring-should-be-keyword, too-ma
480478
data = await self.client_connection.CreateContainer(
481479
database_link=self.database_link, collection=definition, options=request_options, **kwargs
482480
)
481+
_normalize_gsi_container_properties(data)
483482
if not return_properties:
484483
return ContainerProxy(self.client_connection, self.database_link, data["id"], properties=data)
485484
return ContainerProxy(self.client_connection, self.database_link, data["id"], properties=data), data
@@ -772,6 +771,32 @@ def get_container_client(self, container: Union[str, ContainerProxy, dict[str, A
772771
id_value = str(container['id'])
773772
return ContainerProxy(self.client_connection, self.database_link, id_value)
774773

774+
async def _resolve_gsi_definition(
775+
self,
776+
global_secondary_index_definition: Union["GlobalSecondaryIndexDefinition", dict[str, Any]],
777+
) -> dict[str, Any]:
778+
"""Serialize a GSI definition and populate the source container's resource id (_rid).
779+
780+
The service resolves the source container by its resource id (``sourceCollectionRid``).
781+
When the caller only provides the source container id, read the source container to
782+
obtain its ``_rid`` and populate ``sourceCollectionRid`` on the wire payload.
783+
784+
:param global_secondary_index_definition: The GSI definition object or raw dict.
785+
:type global_secondary_index_definition:
786+
~azure.cosmos.GlobalSecondaryIndexDefinition or dict[str, Any]
787+
:returns: The serialized GSI definition including ``sourceCollectionRid``.
788+
:rtype: dict[str, Any]
789+
"""
790+
gsi_dict = (global_secondary_index_definition._to_dict() # pylint: disable=protected-access
791+
if hasattr(global_secondary_index_definition, '_to_dict')
792+
else dict(global_secondary_index_definition))
793+
if not gsi_dict.get("sourceCollectionRid"):
794+
source_container_id = gsi_dict.get("sourceCollectionId")
795+
if source_container_id:
796+
source_properties = await self.get_container_client(source_container_id).read()
797+
gsi_dict["sourceCollectionRid"] = source_properties["_rid"]
798+
return gsi_dict
799+
775800
@distributed_trace
776801
def list_containers(
777802
self,
@@ -1109,15 +1134,14 @@ async def replace_container( # pylint:disable=docstring-should-be-keyword
11091134
if value is not None
11101135
}
11111136
if global_secondary_index_definition is not None:
1112-
gsi_dict = (global_secondary_index_definition._to_dict()
1113-
if hasattr(global_secondary_index_definition, '_to_dict')
1114-
else global_secondary_index_definition)
1137+
gsi_dict = await self._resolve_gsi_definition(global_secondary_index_definition)
11151138
parameters["globalSecondaryIndexDefinition"] = gsi_dict
11161139
parameters["materializedViewDefinition"] = gsi_dict
11171140

11181141
container_properties = await self.client_connection.ReplaceContainer(
11191142
container_link, collection=parameters, options=request_options, **kwargs
11201143
)
1144+
_normalize_gsi_container_properties(container_properties)
11211145

11221146
if not return_properties:
11231147
return ContainerProxy(

‎sdk/cosmos/azure-cosmos/azure/cosmos/container.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
from ._constants import _Constants as Constants, TimeoutScope
4141
from ._cosmos_client_connection import CosmosClientConnection
4242
from ._cosmos_responses import CosmosDict, CosmosList, CosmosItemPaged
43+
from ._global_secondary_index import _normalize_gsi_container_properties
4344
from ._routing.routing_range import Range
4445
from ._session_token_helpers import get_latest_session_token
4546
from .exceptions import CosmosHttpResponseError
@@ -208,6 +209,7 @@ def read( # pylint:disable=docstring-missing-param
208209
if populate_quota_info is not None:
209210
request_options["populateQuotaInfo"] = populate_quota_info
210211
container = self.client_connection.ReadContainer(self.container_link, options=request_options, **kwargs)
212+
_normalize_gsi_container_properties(container)
211213
# Only cache Container Properties that will not change in the lifetime of the container
212214
self.client_connection._set_container_properties_cache(self.container_link, # pylint: disable=protected-access
213215
_build_properties_cache(container, self.container_link))

‎sdk/cosmos/azure-cosmos/azure/cosmos/database.py‎

Lines changed: 31 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@
3838
from .user import UserProxy
3939
from .documents import IndexingMode
4040
from ._cosmos_responses import CosmosDict
41-
from ._global_secondary_index import GlobalSecondaryIndexDefinition
41+
from ._global_secondary_index import GlobalSecondaryIndexDefinition, _normalize_gsi_container_properties
4242

4343
__all__ = ("DatabaseProxy",)
4444

@@ -459,16 +459,15 @@ def create_container( # pylint:disable=docstring-missing-param, too-many-statem
459459
if full_text_policy is not None:
460460
definition["fullTextPolicy"] = full_text_policy
461461
if global_secondary_index_definition is not None:
462-
gsi_dict = (global_secondary_index_definition._to_dict()
463-
if hasattr(global_secondary_index_definition, '_to_dict')
464-
else global_secondary_index_definition)
462+
gsi_dict = self._resolve_gsi_definition(global_secondary_index_definition)
465463
definition["globalSecondaryIndexDefinition"] = gsi_dict
466464
definition["materializedViewDefinition"] = gsi_dict
467465
request_options = build_options(kwargs)
468466
_set_throughput_options(offer=offer_throughput, request_options=request_options)
469467
result = self.client_connection.CreateContainer(
470468
database_link=self.database_link, collection=definition, options=request_options, **kwargs
471469
)
470+
_normalize_gsi_container_properties(result)
472471

473472
if not return_properties:
474473
return ContainerProxy(self.client_connection, self.database_link, result["id"], properties=result)
@@ -812,6 +811,32 @@ def get_container_client(self, container: Union[str, ContainerProxy, Mapping[str
812811
id_value = container["id"]
813812
return ContainerProxy(self.client_connection, self.database_link, id_value)
814813

814+
def _resolve_gsi_definition(
815+
self,
816+
global_secondary_index_definition: Union["GlobalSecondaryIndexDefinition", dict[str, Any]],
817+
) -> dict[str, Any]:
818+
"""Serialize a GSI definition and populate the source container's resource id (_rid).
819+
820+
The service resolves the source container by its resource id (``sourceCollectionRid``).
821+
When the caller only provides the source container id, read the source container to
822+
obtain its ``_rid`` and populate ``sourceCollectionRid`` on the wire payload.
823+
824+
:param global_secondary_index_definition: The GSI definition object or raw dict.
825+
:type global_secondary_index_definition:
826+
~azure.cosmos.GlobalSecondaryIndexDefinition or dict[str, Any]
827+
:returns: The serialized GSI definition including ``sourceCollectionRid``.
828+
:rtype: dict[str, Any]
829+
"""
830+
gsi_dict = (global_secondary_index_definition._to_dict() # pylint: disable=protected-access
831+
if hasattr(global_secondary_index_definition, '_to_dict')
832+
else dict(global_secondary_index_definition))
833+
if not gsi_dict.get("sourceCollectionRid"):
834+
source_container_id = gsi_dict.get("sourceCollectionId")
835+
if source_container_id:
836+
source_properties = self.get_container_client(source_container_id).read()
837+
gsi_dict["sourceCollectionRid"] = source_properties["_rid"]
838+
return gsi_dict
839+
815840
@distributed_trace
816841
def list_containers( # pylint:disable=docstring-missing-param
817842
self,
@@ -1161,14 +1186,13 @@ def replace_container( # pylint:disable=docstring-missing-param, docstring-shou
11611186
if value is not None
11621187
}
11631188
if global_secondary_index_definition is not None:
1164-
gsi_dict = (global_secondary_index_definition._to_dict()
1165-
if hasattr(global_secondary_index_definition, '_to_dict')
1166-
else global_secondary_index_definition)
1189+
gsi_dict = self._resolve_gsi_definition(global_secondary_index_definition)
11671190
parameters["globalSecondaryIndexDefinition"] = gsi_dict
11681191
parameters["materializedViewDefinition"] = gsi_dict
11691192

11701193
container_properties = self.client_connection.ReplaceContainer(
11711194
container_link, collection=parameters, options=request_options, **kwargs)
1195+
_normalize_gsi_container_properties(container_properties)
11721196

11731197
if not return_properties:
11741198
return ContainerProxy(

‎sdk/cosmos/azure-cosmos/tests/test_global_secondary_index_live.py‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,19 +54,30 @@ def test_create_gsi_container(self):
5454
source_container_id=source_container.id,
5555
definition="SELECT c.id, c.email, c.name FROM c"
5656
)
57-
gsi_container = self.test_db.create_container(
57+
gsi_container, create_properties = self.test_db.create_container(
5858
id="gsi-container-" + str(uuid.uuid4())[:8],
5959
partition_key=PartitionKey(path="/id"),
60-
global_secondary_index_definition=gsi_definition
60+
global_secondary_index_definition=gsi_definition,
61+
return_properties=True
6162
)
6263

64+
# The create response must surface the public "globalSecondaryIndexDefinition"
65+
# key and must never expose the legacy "materializedViewDefinition" wire key.
66+
self.assertIn("globalSecondaryIndexDefinition", create_properties)
67+
self.assertNotIn("materializedViewDefinition", create_properties)
68+
6369
# Read back the container properties and verify GSI definition is present
6470
properties = gsi_container.read()
6571
self.assertIn("globalSecondaryIndexDefinition", properties)
72+
self.assertNotIn("materializedViewDefinition", properties)
6673
gsi_props = properties["globalSecondaryIndexDefinition"]
6774
self.assertEqual(gsi_props["sourceCollectionId"], source_container.id)
6875
self.assertEqual(gsi_props["definition"], "SELECT c.id, c.email, c.name FROM c")
69-
self.assertIn("status", gsi_props)
76+
# "status" is a read-only, server-populated field that may be absent from the
77+
# response (e.g. before the index build is tracked). Match the Java SDK contract,
78+
# which treats status as optional, and only validate it when the service returns it.
79+
if "status" in gsi_props:
80+
self.assertIsNotNone(gsi_props["status"])
7081

7182
# Clean up - delete GSI container first, then source
7283
self.test_db.delete_container(gsi_container.id)

‎sdk/cosmos/tests.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ extends:
1616
MaxParallel: 8
1717
BuildTargetingString: azure-cosmos
1818
ServiceDirectory: cosmos
19+
VariableGroups:
20+
- 'Test Secrets for Cosmos Live Tests - user administered'
1921
EnvVars:
2022
GSI_ACCOUNT_HOST: $(gsi-pipeline-uri)
2123
GSI_ACCOUNT_KEY: $(gsi-pipeline-key)

0 commit comments

Comments
 (0)