-
Notifications
You must be signed in to change notification settings - Fork 445
IcebergCatalog- updateNamespaceProperties on single commit #4013
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -239,13 +239,17 @@ public UpdateNamespacePropertiesResponse updateNamespaceProperties( | |
| Map<String, String> startProperties = catalog.loadNamespaceMetadata(namespace); | ||
| Set<String> missing = Sets.difference(removals, startProperties.keySet()); | ||
|
|
||
| if (!updates.isEmpty()) { | ||
| catalog.setProperties(namespace, updates); | ||
| } | ||
| if (catalog instanceof IcebergCatalog polarisCatalog) { | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Due to that we have a direct dependency with I think it should be cleaner to isolate this in a specific method (for clarity).
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes, having If we have to have custom handling for the internal catalog (which is ok), perhaps it is time to refactor this code to use proper polymorphic Federated catalogs will have to delegate to the appropriate API.
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. |
||
| polarisCatalog.updateProperties(namespace, removals, updates); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit: I think if both |
||
| } else { | ||
| if (!updates.isEmpty()) { | ||
| catalog.setProperties(namespace, updates); | ||
| } | ||
|
|
||
| if (!removals.isEmpty()) { | ||
| // remove the original set just in case there was an update just after loading properties | ||
| catalog.removeProperties(namespace, removals); | ||
| if (!removals.isEmpty()) { | ||
| // remove the original set just in case there was an update just after loading properties | ||
| catalog.removeProperties(namespace, removals); | ||
| } | ||
| } | ||
|
|
||
| return UpdateNamespacePropertiesResponse.builder() | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -735,6 +735,63 @@ public boolean setProperties(Namespace namespace, Map<String, String> properties | |
| return true; | ||
| } | ||
|
|
||
| /** | ||
| * Update properties for a namespace. | ||
| * | ||
| * @param namespace the namespace to update properties for | ||
| * @param removals the set of properties to remove | ||
| * @param updates the map of properties to update | ||
| * @return true if the properties were updated, false otherwise | ||
| */ | ||
| public boolean updateProperties( | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The returned boolean is never used / checked. |
||
| Namespace namespace, Set<String> removals, Map<String, String> updates) | ||
| throws NoSuchNamespaceException { | ||
| PolarisResolvedPathWrapper resolvedEntities = resolvedEntityView.getResolvedPath(namespace); | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Why not using |
||
| if (resolvedEntities == null) { | ||
| throw noSuchNamespaceException(namespace); | ||
| } | ||
| PolarisEntity entity = resolvedEntities.getRawLeafEntity(); | ||
| Map<String, String> newProperties = new HashMap<>(entity.getPropertiesAsMap()); | ||
|
|
||
| // Merge new properties into existing map. | ||
| newProperties.putAll(updates); | ||
| // Remove properties. | ||
| removals.forEach(newProperties::remove); | ||
|
|
||
| PolarisEntity updatedEntity = | ||
| new PolarisEntity.Builder(entity).setProperties(newProperties).build(); | ||
|
|
||
| if (!realmConfig.getConfig(FeatureConfiguration.ALLOW_NAMESPACE_LOCATION_OVERLAP)) { | ||
| LOGGER.debug("Validating no overlap with sibling tables or namespaces"); | ||
| validateNoLocationOverlap( | ||
| NamespaceEntity.of(updatedEntity), resolvedEntities.getRawParentPath()); | ||
| } else { | ||
| LOGGER.debug("Skipping location overlap validation for namespace '{}'", namespace); | ||
| } | ||
| if (!realmConfig.getConfig( | ||
| BehaviorChangeConfiguration.ALLOW_NAMESPACE_CUSTOM_LOCATION, catalogEntity)) { | ||
| if (updates.containsKey(PolarisEntityConstants.ENTITY_BASE_LOCATION)) { | ||
| validateNamespaceLocation(NamespaceEntity.of(entity), resolvedEntities); | ||
| } | ||
| } | ||
|
Comment on lines
+764
to
+776
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. These checks appear twice, can we extract a common utility method? |
||
|
|
||
| List<PolarisEntity> parentPath = resolvedEntities.getRawFullPath(); | ||
| PolarisEntity returnedEntity = | ||
| Optional.ofNullable( | ||
| getMetaStoreManager() | ||
| .updateEntityPropertiesIfNotChanged( | ||
| getCurrentPolarisContext(), | ||
| PolarisEntity.toCoreList(parentPath), | ||
| updatedEntity) | ||
| .getEntity()) | ||
| .map(PolarisEntity::new) | ||
| .orElse(null); | ||
| if (returnedEntity == null) { | ||
| throw new CommitConflictException("Concurrent modification of namespace: %s", namespace); | ||
| } | ||
| return true; | ||
| } | ||
|
|
||
| @Override | ||
| public boolean removeProperties(Namespace namespace, Set<String> properties) | ||
| throws NoSuchNamespaceException { | ||
|
|
||
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
|
|
@@ -35,6 +35,7 @@ | |||||
| import com.azure.core.exception.HttpResponseException; | ||||||
| import com.google.cloud.storage.StorageException; | ||||||
| import com.google.common.collect.ImmutableMap; | ||||||
| import com.google.common.collect.ImmutableSet; | ||||||
| import com.google.common.collect.Iterables; | ||||||
| import com.google.common.collect.Streams; | ||||||
| import io.quarkus.test.junit.QuarkusMock; | ||||||
|
|
@@ -2614,4 +2615,30 @@ public void testPaginatedListNamespaces() { | |||||
| } | ||||||
| } | ||||||
| } | ||||||
|
|
||||||
| @Test | ||||||
| public void testUpdateNamespacePropertiesAtomic() { | ||||||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It might be preferable to promote this test to
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. What in this test ensures that the operation is "atomic"? 🤔 |
||||||
| catalog.createNamespace( | ||||||
| NS, | ||||||
| ImmutableMap.of( | ||||||
| "prop1", "val1", | ||||||
| "prop2", "val2", | ||||||
| "prop3", "val3")); | ||||||
|
|
||||||
| catalog.updateProperties( | ||||||
| NS, | ||||||
| ImmutableSet.of("prop1", "prop2", "missing_prop"), | ||||||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Nit: let's keep one old prop to assert that it is preserved after the update:
Suggested change
|
||||||
| ImmutableMap.of( | ||||||
| "prop3", "new_val3", | ||||||
| "prop4", "val4", | ||||||
| "prop5", "val5")); | ||||||
|
|
||||||
| Map<String, String> properties = catalog.loadNamespaceMetadata(NS); | ||||||
| Assertions.assertThat(properties).containsEntry("prop3", "new_val3"); | ||||||
| Assertions.assertThat(properties).containsEntry("prop4", "val4"); | ||||||
| Assertions.assertThat(properties).containsEntry("prop5", "val5"); | ||||||
| Assertions.assertThat(properties).doesNotContainKey("prop1"); | ||||||
| Assertions.assertThat(properties).doesNotContainKey("prop2"); | ||||||
| } | ||||||
| } | ||||||
|
|
||||||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Does this mean Polaris propagates namespace property updates to Federated catalogs even if
CatalogEntity.isStaticFacade()istrue? @dennishuo WDYT?