From 2b11f1f30a0d5c7ae81fef3596775d97fddede34 Mon Sep 17 00:00:00 2001 From: Rory Date: Mon, 11 May 2026 21:31:00 +0800 Subject: [PATCH 01/44] feat(core): JDBC backend and entity store for nested namespace naming - RelationalSchemaNamingBridge and JDBCBackend entity/relation conversions - SupportsRelationOperations.batchInsertRelations; RelationalEntityStore cache invalidation - RelationalBackend; backend-focused tests Co-authored-by: Cursor --- .../gravitino/SupportsRelationOperations.java | 34 ++ .../storage/relational/JDBCBackend.java | 321 ++++++++++-- .../storage/relational/RelationalBackend.java | 8 +- .../relational/RelationalEntityStore.java | 20 + .../RelationalSchemaNamingBridge.java | 496 ++++++++++++++++++ .../relational/BackendTestExtension.java | 1 + .../TestRelationalSchemaNamingBridge.java | 368 +++++++++++++ 7 files changed, 1202 insertions(+), 46 deletions(-) create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/RelationalSchemaNamingBridge.java create mode 100644 core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalSchemaNamingBridge.java diff --git a/core/src/main/java/org/apache/gravitino/SupportsRelationOperations.java b/core/src/main/java/org/apache/gravitino/SupportsRelationOperations.java index 854760060d6..cdf6c28ce7e 100644 --- a/core/src/main/java/org/apache/gravitino/SupportsRelationOperations.java +++ b/core/src/main/java/org/apache/gravitino/SupportsRelationOperations.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.util.List; +import java.util.Objects; import org.apache.gravitino.exceptions.NoSuchEntityException; /** @@ -165,6 +166,39 @@ void insertRelation( boolean override) throws IOException; + /** + * Batch inserts the same relation from many source entities to one destination. Parameter order + * matches {@link #insertRelation}. All sources share one {@code srcType}. Semantics match + * repeated {@link #insertRelation} with the same {@code relType}; a relational backend may use + * fewer round-trips for some relation types (e.g. {@link Type#OWNER_REL}). + * + * @param relType the relation type (e.g. {@link Type#OWNER_REL}) + * @param srcIdentifiers identifiers of the source side for each relation row + * @param srcType entity type shared by every identifier in {@code srcIdentifiers} + * @param dstIdentifier destination entity identifier (shared by all rows) + * @param dstType destination entity type + * @param override if true, replace existing relations of each source entity first, per {@link + * #insertRelation} + * @throws IOException if the storage operation fails + */ + default void batchInsertRelations( + Type relType, + List srcIdentifiers, + Entity.EntityType srcType, + NameIdentifier dstIdentifier, + Entity.EntityType dstType, + boolean override) + throws IOException { + Objects.requireNonNull(relType, "relType must not be null"); + Objects.requireNonNull(srcIdentifiers, "srcIdentifiers must not be null"); + Objects.requireNonNull(srcType, "srcType must not be null"); + Objects.requireNonNull(dstIdentifier, "dstIdentifier must not be null"); + Objects.requireNonNull(dstType, "dstType must not be null"); + for (NameIdentifier srcIdent : srcIdentifiers) { + insertRelation(relType, srcIdent, srcType, dstIdentifier, dstType, override); + } + } + /** * Updates the relations for a given entity by adding a set of new relations and removing another * set of relations. diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java b/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java index 22c7c98330b..c1fc087ad09 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java @@ -26,6 +26,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import java.io.IOException; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.function.Function; @@ -92,6 +93,14 @@ * a database that supports the JDBC protocol as storage. If the specified database has special SQL * syntax, please implement the SQL statements and methods in MyBatis Mapper separately and switch * according to the {@link Configs#ENTITY_RELATIONAL_JDBC_BACKEND_URL_KEY} parameter. + * + *

Hierarchical schema names use the configured logical separator in {@link NameIdentifier}s and + * {@link Namespace}s seen by callers; this layer translates schema segments to the internal + * physical form ({@link org.apache.gravitino.catalog.HierarchicalSchemaUtil#physicalSeparator()}) + * before delegating to meta services for applicable entity types, and translates back on results + * (except {@code insertSchema}, which still accepts logical {@link SchemaEntity} names for + * nested-schema materialization). Fileset, Topic, Model, and Model version entities are not + * transformed here. */ public class JDBCBackend implements RelationalBackend { @@ -120,11 +129,26 @@ public List list( case CATALOG: return (List) CatalogMetaService.getInstance().listCatalogsByNamespace(namespace); case SCHEMA: - return (List) SchemaMetaService.getInstance().listSchemasByNamespace(namespace); + return (List) + SchemaMetaService.getInstance().listSchemasByNamespace(namespace).stream() + .map(RelationalSchemaNamingBridge::schemaEntityForApi) + .collect(Collectors.toList()); case TABLE: - return (List) TableMetaService.getInstance().listTablesByNamespace(namespace); + return (List) + TableMetaService.getInstance() + .listTablesByNamespace( + RelationalSchemaNamingBridge.embeddedNamespaceForStorage(namespace)) + .stream() + .map(RelationalSchemaNamingBridge::tableEntityForApi) + .collect(Collectors.toList()); case VIEW: - return (List) ViewMetaService.getInstance().listViewsByNamespace(namespace); + return (List) + ViewMetaService.getInstance() + .listViewsByNamespace( + RelationalSchemaNamingBridge.embeddedNamespaceForStorage(namespace)) + .stream() + .map(RelationalSchemaNamingBridge::viewEntityForApi) + .collect(Collectors.toList()); case FILESET: return (List) FilesetMetaService.getInstance().listFilesetsByNamespace(namespace); case TOPIC: @@ -134,7 +158,10 @@ public List list( case USER: return (List) UserMetaService.getInstance().listUsersByNamespace(namespace, allFields); case ROLE: - return (List) RoleMetaService.getInstance().listRolesByNamespace(namespace); + return (List) + RoleMetaService.getInstance().listRolesByNamespace(namespace).stream() + .map(RelationalSchemaNamingBridge::roleEntityForApi) + .collect(Collectors.toList()); case GROUP: return (List) GroupMetaService.getInstance().listGroupsByNamespace(namespace, allFields); case MODEL: @@ -143,7 +170,13 @@ public List list( return (List) ModelVersionMetaService.getInstance().listModelVersionsByNamespace(namespace); case FUNCTION: - return (List) FunctionMetaService.getInstance().listFunctionsByNamespace(namespace); + return (List) + FunctionMetaService.getInstance() + .listFunctionsByNamespace( + RelationalSchemaNamingBridge.embeddedNamespaceForStorage(namespace)) + .stream() + .map(RelationalSchemaNamingBridge::functionEntityForApi) + .collect(Collectors.toList()); case POLICY: return (List) PolicyMetaService.getInstance().listPoliciesByNamespace(namespace); case JOB_TEMPLATE: @@ -152,10 +185,14 @@ public List list( case JOB: return (List) JobMetaService.getInstance().listJobsByNamespace(namespace); case TABLE_STATISTIC: + Namespace statisticNs = RelationalSchemaNamingBridge.embeddedNamespaceForStorage(namespace); return (List) StatisticMetaService.getInstance() .listStatisticsByEntity( - NameIdentifier.parse(namespace.toString()), Entity.EntityType.TABLE); + NameIdentifier.parse(statisticNs.toString()), Entity.EntityType.TABLE) + .stream() + .map(RelationalSchemaNamingBridge::statisticEntityForApi) + .collect(Collectors.toList()); default: throw new UnsupportedEntityTypeException( "Unsupported entity type: %s for list operation", entityType); @@ -182,7 +219,9 @@ public void insert(E e, boolean overwritten) } else if (e instanceof SchemaEntity) { SchemaMetaService.getInstance().insertSchema((SchemaEntity) e, overwritten); } else if (e instanceof TableEntity) { - TableMetaService.getInstance().insertTable((TableEntity) e, overwritten); + TableMetaService.getInstance() + .insertTable( + RelationalSchemaNamingBridge.tableEntityForStorage((TableEntity) e), overwritten); } else if (e instanceof FilesetEntity) { FilesetMetaService.getInstance().insertFileset((FilesetEntity) e, overwritten); } else if (e instanceof TopicEntity) { @@ -190,7 +229,9 @@ public void insert(E e, boolean overwritten) } else if (e instanceof UserEntity) { UserMetaService.getInstance().insertUser((UserEntity) e, overwritten); } else if (e instanceof RoleEntity) { - RoleMetaService.getInstance().insertRole((RoleEntity) e, overwritten); + RoleMetaService.getInstance() + .insertRole( + RelationalSchemaNamingBridge.roleEntityForStorage((RoleEntity) e), overwritten); } else if (e instanceof GroupEntity) { GroupMetaService.getInstance().insertGroup((GroupEntity) e, overwritten); } else if (e instanceof TagEntity) { @@ -205,7 +246,10 @@ public void insert(E e, boolean overwritten) } ModelVersionMetaService.getInstance().insertModelVersion((ModelVersionEntity) e); } else if (e instanceof FunctionEntity) { - FunctionMetaService.getInstance().insertFunction((FunctionEntity) e, overwritten); + FunctionMetaService.getInstance() + .insertFunction( + RelationalSchemaNamingBridge.functionEntityForStorage((FunctionEntity) e), + overwritten); } else if (e instanceof PolicyEntity) { PolicyMetaService.getInstance().insertPolicy((PolicyEntity) e, overwritten); } else if (e instanceof JobTemplateEntity) { @@ -213,7 +257,9 @@ public void insert(E e, boolean overwritten) } else if (e instanceof JobEntity) { JobMetaService.getInstance().insertJob((JobEntity) e, overwritten); } else if (e instanceof ViewEntity) { - ViewMetaService.getInstance().insertView((ViewEntity) e, overwritten); + ViewMetaService.getInstance() + .insertView( + RelationalSchemaNamingBridge.viewEntityForStorage((ViewEntity) e), overwritten); } else if (e instanceof GenericEntity) { GenericEntity genericEntity = (GenericEntity) e; throw new UnsupportedEntityTypeException( @@ -234,9 +280,19 @@ public E update( case CATALOG: return (E) CatalogMetaService.getInstance().updateCatalog(ident, updater); case SCHEMA: - return (E) SchemaMetaService.getInstance().updateSchema(ident, updater); + return (E) + SchemaMetaService.getInstance() + .updateSchema( + RelationalSchemaNamingBridge.schemaIdentifierForStorage(ident), updater); case TABLE: - return (E) TableMetaService.getInstance().updateTable(ident, updater); + return (E) + RelationalSchemaNamingBridge.tableEntityForApi( + TableMetaService.getInstance() + .updateTable( + RelationalSchemaNamingBridge.nameIdentifierForStorage( + ident, Entity.EntityType.TABLE), + RelationalSchemaNamingBridge.wrapperUpdater( + Entity.EntityType.TABLE, updater))); case FILESET: return (E) FilesetMetaService.getInstance().updateFileset(ident, updater); case TOPIC: @@ -254,13 +310,27 @@ public E update( case MODEL_VERSION: return (E) ModelVersionMetaService.getInstance().updateModelVersion(ident, updater); case FUNCTION: - return (E) FunctionMetaService.getInstance().updateFunction(ident, updater); + return (E) + RelationalSchemaNamingBridge.functionEntityForApi( + FunctionMetaService.getInstance() + .updateFunction( + RelationalSchemaNamingBridge.nameIdentifierForStorage( + ident, Entity.EntityType.FUNCTION), + RelationalSchemaNamingBridge.wrapperUpdater( + Entity.EntityType.FUNCTION, updater))); case POLICY: return (E) PolicyMetaService.getInstance().updatePolicy(ident, updater); case JOB_TEMPLATE: return (E) JobTemplateMetaService.getInstance().updateJobTemplate(ident, updater); case VIEW: - return (E) ViewMetaService.getInstance().updateView(ident, updater); + return (E) + RelationalSchemaNamingBridge.viewEntityForApi( + ViewMetaService.getInstance() + .updateView( + RelationalSchemaNamingBridge.nameIdentifierForStorage( + ident, Entity.EntityType.VIEW), + RelationalSchemaNamingBridge.wrapperUpdater( + Entity.EntityType.VIEW, updater))); default: throw new UnsupportedEntityTypeException( "Unsupported entity type: %s for update operation", entityType); @@ -277,9 +347,18 @@ public E get( case CATALOG: return (E) CatalogMetaService.getInstance().getCatalogByIdentifier(ident); case SCHEMA: - return (E) SchemaMetaService.getInstance().getSchemaByIdentifier(ident); + return (E) + RelationalSchemaNamingBridge.schemaEntityForApi( + SchemaMetaService.getInstance() + .getSchemaByIdentifier( + RelationalSchemaNamingBridge.schemaIdentifierForStorage(ident))); case TABLE: - return (E) TableMetaService.getInstance().getTableByIdentifier(ident); + return (E) + RelationalSchemaNamingBridge.tableEntityForApi( + TableMetaService.getInstance() + .getTableByIdentifier( + RelationalSchemaNamingBridge.nameIdentifierForStorage( + ident, Entity.EntityType.TABLE))); case FILESET: return (E) FilesetMetaService.getInstance().getFilesetByIdentifier(ident); case TOPIC: @@ -289,7 +368,9 @@ public E get( case GROUP: return (E) GroupMetaService.getInstance().getGroupByIdentifier(ident); case ROLE: - return (E) RoleMetaService.getInstance().getRoleByIdentifier(ident); + return (E) + RelationalSchemaNamingBridge.roleEntityForApi( + RoleMetaService.getInstance().getRoleByIdentifier(ident)); case TAG: return (E) TagMetaService.getInstance().getTagByIdentifier(ident); case MODEL: @@ -297,7 +378,12 @@ public E get( case MODEL_VERSION: return (E) ModelVersionMetaService.getInstance().getModelVersionByIdentifier(ident); case FUNCTION: - return (E) FunctionMetaService.getInstance().getFunctionByIdentifier(ident); + return (E) + RelationalSchemaNamingBridge.functionEntityForApi( + FunctionMetaService.getInstance() + .getFunctionByIdentifier( + RelationalSchemaNamingBridge.nameIdentifierForStorage( + ident, Entity.EntityType.FUNCTION))); case POLICY: return (E) PolicyMetaService.getInstance().getPolicyByIdentifier(ident); case JOB_TEMPLATE: @@ -305,7 +391,12 @@ public E get( case JOB: return (E) JobMetaService.getInstance().getJobByIdentifier(ident); case VIEW: - return (E) ViewMetaService.getInstance().getViewByIdentifier(ident); + return (E) + RelationalSchemaNamingBridge.viewEntityForApi( + ViewMetaService.getInstance() + .getViewByIdentifier( + RelationalSchemaNamingBridge.nameIdentifierForStorage( + ident, Entity.EntityType.VIEW))); default: throw new UnsupportedEntityTypeException( "Unsupported entity type: %s for get operation", entityType); @@ -332,9 +423,25 @@ public List batchGet( case CATALOG: return (List) CatalogMetaService.getInstance().batchGetCatalogByIdentifier(identifiers); case SCHEMA: - return (List) SchemaMetaService.getInstance().batchGetSchemaByIdentifier(identifiers); + List schemaStorageIdents = + identifiers.stream() + .map(RelationalSchemaNamingBridge::schemaIdentifierForStorage) + .collect(Collectors.toList()); + return (List) + SchemaMetaService.getInstance().batchGetSchemaByIdentifier(schemaStorageIdents).stream() + .map(RelationalSchemaNamingBridge::schemaEntityForApi) + .collect(Collectors.toList()); case TABLE: - return (List) TableMetaService.getInstance().batchGetTableByIdentifier(identifiers); + Namespace tableStorageNs = + RelationalSchemaNamingBridge.embeddedNamespaceForStorage(firstNamespace); + List tableStorageIdents = + identifiers.stream() + .map(id -> NameIdentifier.of(tableStorageNs, id.name())) + .collect(Collectors.toList()); + return (List) + TableMetaService.getInstance().batchGetTableByIdentifier(tableStorageIdents).stream() + .map(RelationalSchemaNamingBridge::tableEntityForApi) + .collect(Collectors.toList()); case FILESET: return (List) FilesetMetaService.getInstance().batchGetFilesetByIdentifier(identifiers); case TOPIC: @@ -351,7 +458,8 @@ public List batchGet( return (List) JobTemplateMetaService.getInstance().batchGetJobTemplateByIdentifier(identifiers); case USER: - // TODO: Add true batch SQL operations for users, roles, and views + // No batch SQL yet; missing entities are silently omitted per batchGet contract. + // TODO: replace loops with true batch SQL operations for USER/GROUP/ROLE/VIEW. List users = Lists.newArrayList(); for (NameIdentifier identifier : identifiers) { try { @@ -367,7 +475,10 @@ public List batchGet( List roles = Lists.newArrayList(); for (NameIdentifier identifier : identifiers) { try { - roles.add((E) RoleMetaService.getInstance().getRoleByIdentifier(identifier)); + roles.add( + (E) + RelationalSchemaNamingBridge.roleEntityForApi( + RoleMetaService.getInstance().getRoleByIdentifier(identifier))); } catch (NoSuchEntityException e) { LOG.debug("Skipping missing role during batch get: {}", identifier.name()); } @@ -377,7 +488,13 @@ public List batchGet( List views = Lists.newArrayList(); for (NameIdentifier identifier : identifiers) { try { - views.add((E) ViewMetaService.getInstance().getViewByIdentifier(identifier)); + views.add( + (E) + RelationalSchemaNamingBridge.viewEntityForApi( + ViewMetaService.getInstance() + .getViewByIdentifier( + RelationalSchemaNamingBridge.nameIdentifierForStorage( + identifier, Entity.EntityType.VIEW)))); } catch (NoSuchEntityException e) { LOG.debug("Skipping missing view during batch get: {}", identifier.name()); } @@ -398,9 +515,13 @@ public boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolea case CATALOG: return CatalogMetaService.getInstance().deleteCatalog(ident, cascade); case SCHEMA: - return SchemaMetaService.getInstance().deleteSchema(ident, cascade); + return SchemaMetaService.getInstance() + .deleteSchema(RelationalSchemaNamingBridge.schemaIdentifierForStorage(ident), cascade); case TABLE: - return TableMetaService.getInstance().deleteTable(ident); + return TableMetaService.getInstance() + .deleteTable( + RelationalSchemaNamingBridge.nameIdentifierForStorage( + ident, Entity.EntityType.TABLE)); case FILESET: return FilesetMetaService.getInstance().deleteFileset(ident); case TOPIC: @@ -418,7 +539,10 @@ public boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolea case MODEL_VERSION: return ModelVersionMetaService.getInstance().deleteModelVersion(ident); case FUNCTION: - return FunctionMetaService.getInstance().deleteFunction(ident); + return FunctionMetaService.getInstance() + .deleteFunction( + RelationalSchemaNamingBridge.nameIdentifierForStorage( + ident, Entity.EntityType.FUNCTION)); case POLICY: return PolicyMetaService.getInstance().deletePolicy(ident); case JOB_TEMPLATE: @@ -426,7 +550,10 @@ public boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolea case JOB: return JobMetaService.getInstance().deleteJob(ident); case VIEW: - return ViewMetaService.getInstance().deleteView(ident); + return ViewMetaService.getInstance() + .deleteView( + RelationalSchemaNamingBridge.nameIdentifierForStorage( + ident, Entity.EntityType.VIEW)); default: throw new UnsupportedEntityTypeException( "Unsupported entity type: %s for delete operation", entityType); @@ -601,7 +728,9 @@ public int batchDelete( 1 == namespaceSize, "All entities must be in the same namespace for batch delete operation."); - Namespace namespace = deleteIdents.get(0).namespace(); + Namespace namespace = + RelationalSchemaNamingBridge.embeddedNamespaceForStorage( + deleteIdents.get(0).namespace()); return StatisticMetaService.getInstance() .batchDeleteStatisticPOs( NameIdentifier.parse(namespace.toString()), @@ -634,10 +763,17 @@ public void batchPut(List entities, boolea 1 == entities.stream().collect(Collectors.groupingBy(HasIdentifier::namespace)).size(), "All entities must be in the same namespace for batchPut operation."); + List storageStatisticEntities = + statisticEntities.stream() + .map(RelationalSchemaNamingBridge::statisticEntityForStorage) + .collect(Collectors.toList()); + Namespace statisticNamespace = + RelationalSchemaNamingBridge.embeddedNamespaceForStorage( + statisticEntities.get(0).namespace()); StatisticMetaService.getInstance() .batchInsertStatisticPOsOnDuplicateKeyUpdate( - statisticEntities, - NameIdentifier.parse(statisticEntities.get(0).namespace().toString()), + storageStatisticEntities, + NameIdentifier.parse(statisticNamespace.toString()), Entity.EntityType.TABLE); break; default: @@ -654,13 +790,22 @@ public List listEntitiesByRelation( case OWNER_REL: List list = Lists.newArrayList(); OwnerMetaService.getInstance() - .getOwner(nameIdentifier, identType) + .getOwner( + RelationalSchemaNamingBridge.nameIdentifierForStorage(nameIdentifier, identType), + identType) .ifPresent(e -> list.add((E) e)); return list; case METADATA_OBJECT_ROLE_REL: return (List) RoleMetaService.getInstance() - .listRolesByMetadataObject(nameIdentifier, identType, allFields); + .listRolesByMetadataObject( + RelationalSchemaNamingBridge.nameIdentifierForStorage( + nameIdentifier, identType), + identType, + allFields) + .stream() + .map(RelationalSchemaNamingBridge::roleEntityForApi) + .collect(Collectors.toList()); case ROLE_GROUP_REL: if (identType == Entity.EntityType.ROLE) { return (List) GroupMetaService.getInstance().listGroupsByRoleIdent(nameIdentifier); @@ -672,7 +817,10 @@ public List listEntitiesByRelation( if (identType == Entity.EntityType.ROLE) { return (List) UserMetaService.getInstance().listUsersByRoleIdent(nameIdentifier); } else if (identType == Entity.EntityType.USER) { - return (List) RoleMetaService.getInstance().listRolesByUserIdent(nameIdentifier); + return (List) + RoleMetaService.getInstance().listRolesByUserIdent(nameIdentifier).stream() + .map(RelationalSchemaNamingBridge::roleEntityForApi) + .collect(Collectors.toList()); } else { throw new IllegalArgumentException( String.format("ROLE_USER_REL doesn't support type %s", identType.name())); @@ -681,20 +829,35 @@ public List listEntitiesByRelation( case POLICY_METADATA_OBJECT_REL: if (identType == Entity.EntityType.POLICY) { return (List) - PolicyMetaService.getInstance().listAssociatedEntitiesForPolicy(nameIdentifier); + PolicyMetaService.getInstance() + .listAssociatedEntitiesForPolicy(nameIdentifier) + .stream() + .map(RelationalSchemaNamingBridge::genericEntityMetadataFullNameForApi) + .collect(Collectors.toList()); } else { return (List) PolicyMetaService.getInstance() - .listPoliciesForMetadataObject(nameIdentifier, identType); + .listPoliciesForMetadataObject( + RelationalSchemaNamingBridge.nameIdentifierForStorage( + nameIdentifier, identType), + identType); } case TAG_METADATA_OBJECT_REL: if (identType == Entity.EntityType.TAG) { return (List) - TagMetaService.getInstance().listAssociatedMetadataObjectsForTag(nameIdentifier); + TagMetaService.getInstance() + .listAssociatedMetadataObjectsForTag(nameIdentifier) + .stream() + .map(RelationalSchemaNamingBridge::genericEntityMetadataFullNameForApi) + .collect(Collectors.toList()); } else { return (List) - TagMetaService.getInstance().listTagsForMetadataObject(nameIdentifier, identType); + TagMetaService.getInstance() + .listTagsForMetadataObject( + RelationalSchemaNamingBridge.nameIdentifierForStorage( + nameIdentifier, identType), + identType); } default: throw new IllegalArgumentException( @@ -708,7 +871,11 @@ public List> batchListEntitiesByRelation( throws IOException { switch (relType) { case OWNER_REL: - return OwnerMetaService.getInstance().batchGetOwner(nameIdentifiers, identType); + List ownerRelStorageIdents = + nameIdentifiers.stream() + .map(id -> RelationalSchemaNamingBridge.nameIdentifierForStorage(id, identType)) + .collect(Collectors.toList()); + return OwnerMetaService.getInstance().batchGetOwner(ownerRelStorageIdents, identType); default: throw new IllegalArgumentException( String.format("Doesn't support the relation type %s", relType)); @@ -725,7 +892,12 @@ public void insertRelation( boolean override) { switch (relType) { case OWNER_REL: - OwnerMetaService.getInstance().setOwner(srcIdentifier, srcType, dstIdentifier, dstType); + OwnerMetaService.getInstance() + .setOwner( + RelationalSchemaNamingBridge.nameIdentifierForStorage(srcIdentifier, srcType), + srcType, + RelationalSchemaNamingBridge.nameIdentifierForStorage(dstIdentifier, dstType), + dstType); break; default: throw new IllegalArgumentException( @@ -733,6 +905,41 @@ public void insertRelation( } } + @Override + public void batchInsertRelations( + Type relType, + List srcIdentifiers, + Entity.EntityType srcType, + NameIdentifier dstIdentifier, + Entity.EntityType dstType, + boolean override) + throws IOException { + if (srcIdentifiers == null || srcIdentifiers.isEmpty()) { + return; + } + Preconditions.checkNotNull(relType, "relType must not be null"); + Preconditions.checkNotNull(srcType, "srcType must not be null"); + Preconditions.checkNotNull(dstIdentifier, "dstIdentifier must not be null"); + Preconditions.checkNotNull(dstType, "dstType must not be null"); + switch (relType) { + case OWNER_REL: + List srcStorage = + srcIdentifiers.stream() + .map(id -> RelationalSchemaNamingBridge.nameIdentifierForStorage(id, srcType)) + .collect(Collectors.toList()); + OwnerMetaService.getInstance() + .batchSetOwners( + srcStorage, + srcType, + RelationalSchemaNamingBridge.nameIdentifierForStorage(dstIdentifier, dstType), + dstType); + break; + default: + throw new IllegalArgumentException( + String.format("Doesn't support batch insert for the relation type %s", relType)); + } + } + @Override public List updateEntityRelations( Type relType, @@ -746,12 +953,20 @@ public List updateEntityRelations( return (List) PolicyMetaService.getInstance() .associatePoliciesWithMetadataObject( - srcEntityIdent, srcEntityType, destEntitiesToAdd, destEntitiesToRemove); + RelationalSchemaNamingBridge.nameIdentifierForStorage( + srcEntityIdent, srcEntityType), + srcEntityType, + storageIdents(destEntitiesToAdd, Entity.EntityType.POLICY), + storageIdents(destEntitiesToRemove, Entity.EntityType.POLICY)); case TAG_METADATA_OBJECT_REL: return (List) TagMetaService.getInstance() .associateTagsWithMetadataObject( - srcEntityIdent, srcEntityType, destEntitiesToAdd, destEntitiesToRemove); + RelationalSchemaNamingBridge.nameIdentifierForStorage( + srcEntityIdent, srcEntityType), + srcEntityType, + storageIdents(destEntitiesToAdd, Entity.EntityType.TAG), + storageIdents(destEntitiesToRemove, Entity.EntityType.TAG)); default: throw new IllegalArgumentException( String.format("Doesn't support the relation type %s", relType)); @@ -769,17 +984,35 @@ public E getEntityByRelation( case POLICY_METADATA_OBJECT_REL: return (E) PolicyMetaService.getInstance() - .getPolicyForMetadataObject(srcIdentifier, srcType, destEntityIdent); + .getPolicyForMetadataObject( + RelationalSchemaNamingBridge.nameIdentifierForStorage(srcIdentifier, srcType), + srcType, + RelationalSchemaNamingBridge.nameIdentifierForStorage( + destEntityIdent, Entity.EntityType.POLICY)); case TAG_METADATA_OBJECT_REL: return (E) TagMetaService.getInstance() - .getTagForMetadataObject(srcIdentifier, srcType, destEntityIdent); + .getTagForMetadataObject( + RelationalSchemaNamingBridge.nameIdentifierForStorage(srcIdentifier, srcType), + srcType, + RelationalSchemaNamingBridge.nameIdentifierForStorage( + destEntityIdent, Entity.EntityType.TAG)); default: throw new IllegalArgumentException( String.format("Doesn't support the relation type %s", relType)); } } + private static NameIdentifier[] storageIdents( + NameIdentifier[] idents, Entity.EntityType entityType) { + if (idents == null) { + return null; + } + return Arrays.stream(idents) + .map(id -> RelationalSchemaNamingBridge.nameIdentifierForStorage(id, entityType)) + .toArray(NameIdentifier[]::new); + } + public enum JDBCBackendType { H2(true), MYSQL(false), diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalBackend.java b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalBackend.java index 61eb9c435ba..3e4318bb02f 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalBackend.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalBackend.java @@ -116,11 +116,15 @@ E get(NameIdentifier ident, Entity.EntityType /** * Batch retrieves the entities associated with the identifiers and the entity type. * + *

Partial-result contract: for entity types that do not have a native batch-SQL + * implementation (currently {@code USER}, {@code GROUP}, {@code ROLE}, {@code VIEW}), missing + * entities are silently omitted from the result rather than causing an exception. Callers must + * therefore not assume that the returned list has the same size as {@code identifiers}. + * * @param The type of the entity returned. * @param identifiers The identifiers of the entities. * @param entityType The type of the entity. - * @return The entities associated with the identifiers and the entity type, or null if the key - * does not exist. + * @return The entities found; may be smaller than {@code identifiers} if some are missing. */ List batchGet( List identifiers, Entity.EntityType entityType); diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStore.java b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStore.java index d89c6c678d8..d531d5cc011 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStore.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStore.java @@ -327,6 +327,26 @@ public void insertRelation( cache.invalidate(dstIdentifier, dstType, relType); } + @Override + public void batchInsertRelations( + Type relType, + List srcIdentifiers, + Entity.EntityType srcType, + NameIdentifier dstIdentifier, + Entity.EntityType dstType, + boolean override) + throws IOException { + if (srcIdentifiers == null || srcIdentifiers.isEmpty()) { + return; + } + backend.batchInsertRelations( + relType, srcIdentifiers, srcType, dstIdentifier, dstType, override); + for (NameIdentifier ident : srcIdentifiers) { + cache.invalidate(ident, srcType, relType); + } + cache.invalidate(dstIdentifier, dstType, relType); + } + @Override public List updateEntityRelations( Type relType, diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalSchemaNamingBridge.java b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalSchemaNamingBridge.java new file mode 100644 index 00000000000..bfd394b8ceb --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalSchemaNamingBridge.java @@ -0,0 +1,496 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gravitino.storage.relational; + +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.apache.gravitino.Entity; +import org.apache.gravitino.HasIdentifier; +import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.Namespace; +import org.apache.gravitino.authorization.SecurableObject; +import org.apache.gravitino.authorization.SecurableObjects; +import org.apache.gravitino.meta.AuditInfo; +import org.apache.gravitino.meta.FunctionEntity; +import org.apache.gravitino.meta.GenericEntity; +import org.apache.gravitino.meta.RoleEntity; +import org.apache.gravitino.meta.SchemaEntity; +import org.apache.gravitino.meta.StatisticEntity; +import org.apache.gravitino.meta.TableEntity; +import org.apache.gravitino.meta.TableStatisticEntity; +import org.apache.gravitino.meta.ViewEntity; +import org.apache.gravitino.utils.HierarchicalSchemaUtil; + +/** + * Translates hierarchical schema naming between the logical representation used above {@link + * JDBCBackend} (configured separator in identifiers/namespaces) and the physical representation + * expected by relational meta services (ASCII-1 hierarchy separator in schema segments). + * + *

{@link Entity.EntityType#FILESET}, {@link Entity.EntityType#TOPIC}, {@link + * Entity.EntityType#MODEL}, and {@link Entity.EntityType#MODEL_VERSION} identifiers/namespaces are + * left unchanged by this bridge (delegated as-is to meta services). + */ +public final class RelationalSchemaNamingBridge { + + private static final Joiner DOT_JOINER = Joiner.on('.'); + + private RelationalSchemaNamingBridge() {} + + private static String separator() { + return HierarchicalSchemaUtil.schemaSeparator(); + } + + /** + * Converts the dotted metadata {@code fullName} carried by {@link SecurableObject} so the schema + * segment matches relational storage (physical hierarchical encoding). Types excluded from schema + * embedding ({@link Entity.EntityType#FILESET}, {@link Entity.EntityType#TOPIC}, {@link + * Entity.EntityType#MODEL}, {@link Entity.EntityType#MODEL_VERSION}) are unchanged. + */ + public static SecurableObject securableObjectForStorage(SecurableObject object) { + String converted = + convertMetadataObjectDottedFullName(object.fullName(), object.type(), /* toStorage */ true); + if (converted.equals(object.fullName())) { + return object; + } + return SecurableObjects.parse(converted, object.type(), object.privileges()); + } + + /** + * Converts the dotted metadata {@code fullName} on a {@link SecurableObject} to logical schema + * naming for API callers. Passthrough rules match {@link + * #securableObjectForStorage(SecurableObject)}. + */ + public static SecurableObject securableObjectForApi(SecurableObject object) { + String converted = + convertMetadataObjectDottedFullName( + object.fullName(), object.type(), /* toStorage */ false); + if (converted.equals(object.fullName())) { + return object; + } + return SecurableObjects.parse(converted, object.type(), object.privileges()); + } + + public static RoleEntity roleEntityForStorage(RoleEntity entity) { + List objects = entity.securableObjects(); + if (objects == null || objects.isEmpty()) { + return entity; + } + List mapped = + objects.stream() + .map(RelationalSchemaNamingBridge::securableObjectForStorage) + .collect(Collectors.toList()); + if (mapped.equals(objects)) { + return entity; + } + return RoleEntity.builder() + .withId(entity.id()) + .withName(entity.name()) + .withNamespace(entity.namespace()) + .withProperties(entity.properties()) + .withAuditInfo(entity.auditInfo()) + .withSecurableObjects(mapped) + .build(); + } + + public static RoleEntity roleEntityForApi(RoleEntity entity) { + List objects = entity.securableObjects(); + if (objects == null || objects.isEmpty()) { + return entity; + } + List mapped = + objects.stream() + .map(RelationalSchemaNamingBridge::securableObjectForApi) + .collect(Collectors.toList()); + if (mapped.equals(objects)) { + return entity; + } + return RoleEntity.builder() + .withId(entity.id()) + .withName(entity.name()) + .withNamespace(entity.namespace()) + .withProperties(entity.properties()) + .withAuditInfo(entity.auditInfo()) + .withSecurableObjects(mapped) + .build(); + } + + /** + * Normalizes {@link GenericEntity#name()} when it holds a dotted metadata-object path whose + * schema segment uses physical hierarchical encoding, for API callers above {@link JDBCBackend}. + * + *

Uses {@link MetadataObject.Type} derived from {@link Entity.EntityType} (same as relational + * policy/tag stubs that only set dotted {@link GenericEntity#name()}, not a full table/view + * namespace). + */ + public static GenericEntity genericEntityMetadataFullNameForApi(GenericEntity entity) { + String name = entity.name(); + if (name == null || name.isEmpty()) { + return entity; + } + final MetadataObject.Type moType; + try { + moType = MetadataObject.Type.valueOf(entity.type().name()); + } catch (IllegalArgumentException e) { + return entity; + } + String convertedName = convertMetadataObjectDottedFullName(name, moType, false); + + if (convertedName.equals(name)) { + return entity; + } + return GenericEntity.builder() + .withId(entity.id()) + .withEntityType(entity.type()) + .withName(convertedName) + .withNamespace(entity.namespace()) + .build(); + } + + /** + * Rewrites the schema segment inside a dotted metadata full name (catalog.schema[.rest]) between + * logical and physical hierarchical forms. Non-matching part counts are returned unchanged. + */ + static String convertMetadataObjectDottedFullName( + String fullName, MetadataObject.Type type, boolean toStorage) { + if (fullName == null || fullName.isEmpty()) { + return fullName; + } + switch (type) { + case SCHEMA: + return convertSchemaSegmentAt(fullName, /* expectedParts */ 2, toStorage); + case TABLE: + case VIEW: + case FUNCTION: + return convertSchemaSegmentAt(fullName, /* expectedParts */ 3, toStorage); + case COLUMN: + return convertSchemaSegmentAt(fullName, /* expectedParts */ 4, toStorage); + default: + return fullName; + } + } + + private static String convertSchemaSegmentAt( + String fullName, int expectedParts, boolean toStorage) { + List parts = SecurableObjects.DOT_SPLITTER.splitToList(fullName); + if (parts.size() != expectedParts) { + return fullName; + } + String schemaSegment = parts.get(1); + if (schemaSegment == null || schemaSegment.isEmpty()) { + return fullName; + } + String mapped = + toStorage + ? HierarchicalSchemaUtil.logicalToPhysical(schemaSegment, separator()) + : HierarchicalSchemaUtil.physicalToLogical(schemaSegment, separator()); + if (mapped.equals(schemaSegment)) { + return fullName; + } + List copy = new ArrayList<>(parts); + copy.set(1, mapped); + return DOT_JOINER.join(copy); + } + + /** Converts the schema segment embedded at namespace index 2 when length ≥ 3. */ + public static Namespace embeddedNamespaceForStorage(Namespace ns) { + if (ns.length() < 3) { + return ns; + } + String[] lv = ns.levels(); + String[] copy = Arrays.copyOf(lv, lv.length); + copy[2] = HierarchicalSchemaUtil.logicalToPhysical(copy[2], separator()); + return Namespace.of(copy); + } + + public static Namespace embeddedNamespaceForApi(Namespace ns) { + if (ns.length() < 3) { + return ns; + } + String[] lv = ns.levels(); + String[] copy = Arrays.copyOf(lv, lv.length); + copy[2] = HierarchicalSchemaUtil.physicalToLogical(copy[2], separator()); + return Namespace.of(copy); + } + + public static NameIdentifier schemaIdentifierForStorage(NameIdentifier ident) { + return NameIdentifier.of( + ident.namespace(), HierarchicalSchemaUtil.logicalToPhysical(ident.name(), separator())); + } + + public static NameIdentifier schemaIdentifierForApi(NameIdentifier ident) { + return NameIdentifier.of( + ident.namespace(), HierarchicalSchemaUtil.physicalToLogical(ident.name(), separator())); + } + + /** + * Converts identifiers carrying optional hierarchical schema segments before delegating to JDBC + * meta services (physical naming). + */ + public static NameIdentifier nameIdentifierForStorage( + NameIdentifier ident, Entity.EntityType entityType) { + switch (entityType) { + case SCHEMA: + return schemaIdentifierForStorage(ident); + case TABLE: + case VIEW: + case FUNCTION: + case COLUMN: + case TABLE_STATISTIC: + return NameIdentifier.of(embeddedNamespaceForStorage(ident.namespace()), ident.name()); + default: + return ident; + } + } + + public static NameIdentifier nameIdentifierForApi( + NameIdentifier ident, Entity.EntityType entityType) { + switch (entityType) { + case SCHEMA: + return schemaIdentifierForApi(ident); + case TABLE: + case VIEW: + case FUNCTION: + case COLUMN: + case TABLE_STATISTIC: + return NameIdentifier.of(embeddedNamespaceForApi(ident.namespace()), ident.name()); + default: + return ident; + } + } + + public static SchemaEntity schemaEntityForApi(SchemaEntity entity) { + String logicalName = HierarchicalSchemaUtil.physicalToLogical(entity.name(), separator()); + if (logicalName.equals(entity.name())) { + return entity; + } + return SchemaEntity.builder() + .withId(entity.id()) + .withName(logicalName) + .withNamespace(entity.namespace()) + .withComment(entity.comment()) + .withProperties(entity.properties()) + .withAuditInfo(entity.auditInfo()) + .build(); + } + + public static TableEntity tableEntityForApi(TableEntity e) { + Namespace apiNs = embeddedNamespaceForApi(e.namespace()); + if (apiNs.equals(e.namespace())) { + return e; + } + return TableEntity.builder() + .withId(e.id()) + .withName(e.name()) + .withNamespace(apiNs) + .withColumns(e.columns()) + .withAuditInfo(e.auditInfo()) + .withProperties(e.properties()) + .withPartitioning(e.partitioning()) + .withSortOrders(e.sortOrders()) + .withDistribution(e.distribution()) + .withIndexes(e.indexes()) + .withComment(e.comment()) + .build(); + } + + public static TableEntity tableEntityForStorage(TableEntity e) { + Namespace storageNs = embeddedNamespaceForStorage(e.namespace()); + if (storageNs.equals(e.namespace())) { + return e; + } + return TableEntity.builder() + .withId(e.id()) + .withName(e.name()) + .withNamespace(storageNs) + .withColumns(e.columns()) + .withAuditInfo(e.auditInfo()) + .withProperties(e.properties()) + .withPartitioning(e.partitioning()) + .withSortOrders(e.sortOrders()) + .withDistribution(e.distribution()) + .withIndexes(e.indexes()) + .withComment(e.comment()) + .build(); + } + + public static ViewEntity viewEntityForApi(ViewEntity e) { + Namespace apiNs = embeddedNamespaceForApi(e.namespace()); + if (apiNs.equals(e.namespace())) { + return e; + } + return ViewEntity.builder() + .withId(e.id()) + .withName(e.name()) + .withNamespace(apiNs) + .withColumns(e.columns()) + .withRepresentations(e.representations()) + .withDefaultCatalog(e.defaultCatalog()) + .withDefaultSchema(e.defaultSchema()) + .withComment(e.comment()) + .withProperties(e.properties()) + .withAuditInfo(e.auditInfo()) + .build(); + } + + public static ViewEntity viewEntityForStorage(ViewEntity e) { + Namespace storageNs = embeddedNamespaceForStorage(e.namespace()); + if (storageNs.equals(e.namespace())) { + return e; + } + return ViewEntity.builder() + .withId(e.id()) + .withName(e.name()) + .withNamespace(storageNs) + .withColumns(e.columns()) + .withRepresentations(e.representations()) + .withDefaultCatalog(e.defaultCatalog()) + .withDefaultSchema(e.defaultSchema()) + .withComment(e.comment()) + .withProperties(e.properties()) + .withAuditInfo(e.auditInfo()) + .build(); + } + + public static FunctionEntity functionEntityForApi(FunctionEntity e) { + Namespace apiNs = embeddedNamespaceForApi(e.namespace()); + if (apiNs.equals(e.namespace())) { + return e; + } + return FunctionEntity.builder() + .withId(e.id()) + .withName(e.name()) + .withNamespace(apiNs) + .withComment(e.comment()) + .withFunctionType(e.functionType()) + .withDeterministic(e.deterministic()) + .withDefinitions(e.definitions()) + .withAuditInfo(e.auditInfo()) + .build(); + } + + public static FunctionEntity functionEntityForStorage(FunctionEntity e) { + Namespace storageNs = embeddedNamespaceForStorage(e.namespace()); + if (storageNs.equals(e.namespace())) { + return e; + } + return FunctionEntity.builder() + .withId(e.id()) + .withName(e.name()) + .withNamespace(storageNs) + .withComment(e.comment()) + .withFunctionType(e.functionType()) + .withDeterministic(e.deterministic()) + .withDefinitions(e.definitions()) + .withAuditInfo(e.auditInfo()) + .build(); + } + + public static StatisticEntity statisticEntityForApi(StatisticEntity e) { + Namespace apiNs = embeddedNamespaceForApi(e.namespace()); + if (apiNs.equals(e.namespace())) { + return e; + } + return TableStatisticEntity.builder() + .withId(e.id()) + .withName(e.name()) + .withValue(e.value()) + .withAuditInfo((AuditInfo) e.auditInfo()) + .withNamespace(apiNs) + .build(); + } + + public static StatisticEntity statisticEntityForStorage(StatisticEntity e) { + Namespace storageNs = embeddedNamespaceForStorage(e.namespace()); + if (storageNs.equals(e.namespace())) { + return e; + } + return TableStatisticEntity.builder() + .withId(e.id()) + .withName(e.name()) + .withValue(e.value()) + .withAuditInfo((AuditInfo) e.auditInfo()) + .withNamespace(storageNs) + .build(); + } + + /** + * Wraps a meta-service {@code updater} used from {@link JDBCBackend}: converts entities from + * storage naming to API naming, applies {@code updater}, then converts back to storage for + * relational persistence. + * + * @param entityType bridged entity kind (e.g. {@link Entity.EntityType#TABLE}); passed to {@link + * #entityForApi(Entity, Entity.EntityType)} + */ + @SuppressWarnings("unchecked") + public static Function wrapperUpdater( + Entity.EntityType entityType, Function updater) { + Preconditions.checkArgument( + entityType != Entity.EntityType.SCHEMA, + "Schema entity updates must go through SchemaMetaService directly, not wrapperUpdater"); + return e -> { + E logicalOld = entityForApi(e, entityType); + E logicalNew = updater.apply(logicalOld); + return entityForStorage(logicalNew); + }; + } + + @SuppressWarnings("unchecked") + public static E entityForApi( + E entity, Entity.EntityType type) { + switch (type) { + case SCHEMA: + return (E) schemaEntityForApi((SchemaEntity) entity); + case TABLE: + return (E) tableEntityForApi((TableEntity) entity); + case VIEW: + return (E) viewEntityForApi((ViewEntity) entity); + case FUNCTION: + return (E) functionEntityForApi((FunctionEntity) entity); + case TABLE_STATISTIC: + return (E) statisticEntityForApi((StatisticEntity) entity); + default: + return entity; + } + } + + @SuppressWarnings("unchecked") + public static E entityForStorage(E entity) { + if (entity instanceof SchemaEntity) { + return entity; + } + if (entity instanceof TableEntity) { + return (E) tableEntityForStorage((TableEntity) entity); + } + if (entity instanceof ViewEntity) { + return (E) viewEntityForStorage((ViewEntity) entity); + } + if (entity instanceof FunctionEntity) { + return (E) functionEntityForStorage((FunctionEntity) entity); + } + if (entity instanceof StatisticEntity) { + return (E) statisticEntityForStorage((StatisticEntity) entity); + } + return entity; + } +} diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/BackendTestExtension.java b/core/src/test/java/org/apache/gravitino/storage/relational/BackendTestExtension.java index a9377c8b599..0920052ff70 100644 --- a/core/src/test/java/org/apache/gravitino/storage/relational/BackendTestExtension.java +++ b/core/src/test/java/org/apache/gravitino/storage/relational/BackendTestExtension.java @@ -177,6 +177,7 @@ private RelationalBackend startBackend(String type) throws Exception { .thenReturn(DEFAULT_RELATIONAL_JDBC_BACKEND_MAX_WAIT_MILLISECONDS); Mockito.when(config.get(CACHE_ENABLED)).thenReturn(true); + Mockito.when(config.get(Configs.SCHEMA_SEPARATOR)).thenReturn(":"); FieldUtils.writeField(GravitinoEnv.getInstance(), "config", config, true); FieldUtils.writeField( diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalSchemaNamingBridge.java b/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalSchemaNamingBridge.java new file mode 100644 index 00000000000..48a436f77fb --- /dev/null +++ b/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalSchemaNamingBridge.java @@ -0,0 +1,368 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.gravitino.Config; +import org.apache.gravitino.Configs; +import org.apache.gravitino.Entity; +import org.apache.gravitino.GravitinoEnv; +import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.Namespace; +import org.apache.gravitino.meta.GenericEntity; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +/** Unit tests for {@link RelationalSchemaNamingBridge}. */ +public class TestRelationalSchemaNamingBridge { + + /** ASCII-1 internal separator used in physical storage. */ + private static final String P = ""; + + @BeforeEach + public void setUp() throws Exception { + mockSeparator(":"); + } + + @AfterEach + public void tearDown() throws Exception { + FieldUtils.writeField(GravitinoEnv.getInstance(), "config", null, true); + } + + // ------------------------------------------------------------------------- + // convertMetadataObjectDottedFullName – each MetadataObject.Type + // ------------------------------------------------------------------------- + + @Test + public void schemaToStorage() { + // SCHEMA expects 2 dot-parts; schema segment "a:b:c" → "abc" + String logical = "catalog.a:b:c"; + String physical = + RelationalSchemaNamingBridge.convertMetadataObjectDottedFullName( + logical, MetadataObject.Type.SCHEMA, true); + assertEquals("catalog.a" + P + "b" + P + "c", physical); + } + + @Test + public void schemaToApi() { + String physical = "catalog.a" + P + "b" + P + "c"; + String logical = + RelationalSchemaNamingBridge.convertMetadataObjectDottedFullName( + physical, MetadataObject.Type.SCHEMA, false); + assertEquals("catalog.a:b:c", logical); + } + + @Test + public void tableToStorage() { + // TABLE expects 3 dot-parts; schema segment at index 1 + String logical = "catalog.a:b.mytable"; + String physical = + RelationalSchemaNamingBridge.convertMetadataObjectDottedFullName( + logical, MetadataObject.Type.TABLE, true); + assertEquals("catalog.a" + P + "b.mytable", physical); + } + + @Test + public void tableToApi() { + String physical = "catalog.a" + P + "b.mytable"; + String logical = + RelationalSchemaNamingBridge.convertMetadataObjectDottedFullName( + physical, MetadataObject.Type.TABLE, false); + assertEquals("catalog.a:b.mytable", logical); + } + + @Test + public void viewToStorage() { + String logical = "catalog.a:b.myview"; + String physical = + RelationalSchemaNamingBridge.convertMetadataObjectDottedFullName( + logical, MetadataObject.Type.VIEW, true); + assertEquals("catalog.a" + P + "b.myview", physical); + } + + @Test + public void functionToStorage() { + String logical = "catalog.a:b.myfunc"; + String physical = + RelationalSchemaNamingBridge.convertMetadataObjectDottedFullName( + logical, MetadataObject.Type.FUNCTION, true); + assertEquals("catalog.a" + P + "b.myfunc", physical); + } + + @Test + public void columnToStorage() { + // COLUMN expects 4 dot-parts + String logical = "catalog.a:b.mytable.mycol"; + String physical = + RelationalSchemaNamingBridge.convertMetadataObjectDottedFullName( + logical, MetadataObject.Type.COLUMN, true); + assertEquals("catalog.a" + P + "b.mytable.mycol", physical); + } + + @Test + public void metalakePassthrough() { + String name = "mymetal"; + String result = + RelationalSchemaNamingBridge.convertMetadataObjectDottedFullName( + name, MetadataObject.Type.METALAKE, true); + assertEquals(name, result); + } + + @Test + public void catalogPassthrough() { + String name = "catalog"; + String result = + RelationalSchemaNamingBridge.convertMetadataObjectDottedFullName( + name, MetadataObject.Type.CATALOG, true); + assertEquals(name, result); + } + + // ------------------------------------------------------------------------- + // Part-count mismatch → input returned unchanged + // ------------------------------------------------------------------------- + + @Test + public void schemaPartCountMismatch() { + // SCHEMA expects 2 parts; 3 parts → no conversion + String name = "catalog.schema.extra"; + String result = + RelationalSchemaNamingBridge.convertMetadataObjectDottedFullName( + name, MetadataObject.Type.SCHEMA, true); + assertEquals(name, result); + } + + @Test + public void tablePartCountMismatch() { + // TABLE expects 3 parts; 2 parts → no conversion + String name = "catalog.schema"; + String result = + RelationalSchemaNamingBridge.convertMetadataObjectDottedFullName( + name, MetadataObject.Type.TABLE, true); + assertEquals(name, result); + } + + // ------------------------------------------------------------------------- + // Round-trip: logical → physical → logical + // ------------------------------------------------------------------------- + + @Test + public void schemaRoundTrip() { + String logical = "catalog.a:b:c"; + String physical = + RelationalSchemaNamingBridge.convertMetadataObjectDottedFullName( + logical, MetadataObject.Type.SCHEMA, true); + String backToLogical = + RelationalSchemaNamingBridge.convertMetadataObjectDottedFullName( + physical, MetadataObject.Type.SCHEMA, false); + assertEquals(logical, backToLogical); + } + + @Test + public void tableRoundTrip() { + String logical = "catalog.a:b.mytable"; + String physical = + RelationalSchemaNamingBridge.convertMetadataObjectDottedFullName( + logical, MetadataObject.Type.TABLE, true); + String backToLogical = + RelationalSchemaNamingBridge.convertMetadataObjectDottedFullName( + physical, MetadataObject.Type.TABLE, false); + assertEquals(logical, backToLogical); + } + + // ------------------------------------------------------------------------- + // Non-default separator + // ------------------------------------------------------------------------- + + @Test + public void customSeparatorSchema() throws Exception { + mockSeparator("/"); + String logical = "catalog.a/b/c"; + String physical = + RelationalSchemaNamingBridge.convertMetadataObjectDottedFullName( + logical, MetadataObject.Type.SCHEMA, true); + assertEquals("catalog.a" + P + "b" + P + "c", physical); + + String backToLogical = + RelationalSchemaNamingBridge.convertMetadataObjectDottedFullName( + physical, MetadataObject.Type.SCHEMA, false); + assertEquals(logical, backToLogical); + } + + // ------------------------------------------------------------------------- + // embeddedNamespaceForStorage / embeddedNamespaceForApi + // ------------------------------------------------------------------------- + + @Test + public void embeddedNsToStorage() { + Namespace ns = Namespace.of("ml", "cat", "a:b:c"); + Namespace stored = RelationalSchemaNamingBridge.embeddedNamespaceForStorage(ns); + assertEquals(Namespace.of("ml", "cat", "a" + P + "b" + P + "c"), stored); + } + + @Test + public void embeddedNsToApi() { + Namespace ns = Namespace.of("ml", "cat", "a" + P + "b" + P + "c"); + Namespace api = RelationalSchemaNamingBridge.embeddedNamespaceForApi(ns); + assertEquals(Namespace.of("ml", "cat", "a:b:c"), api); + } + + @Test + public void embeddedNsShortUnchanged() { + // Namespaces with fewer than 3 levels are returned as-is + Namespace ns = Namespace.of("ml", "cat"); + Namespace stored = RelationalSchemaNamingBridge.embeddedNamespaceForStorage(ns); + assertSame(ns, stored); + } + + // ------------------------------------------------------------------------- + // nameIdentifierForStorage / nameIdentifierForApi – SCHEMA + // ------------------------------------------------------------------------- + + @Test + public void schemaIdentToStorage() { + NameIdentifier ident = NameIdentifier.of("ml", "cat", "a:b"); + NameIdentifier stored = + RelationalSchemaNamingBridge.nameIdentifierForStorage(ident, Entity.EntityType.SCHEMA); + assertEquals("a" + P + "b", stored.name()); + } + + @Test + public void schemaIdentToApi() { + NameIdentifier ident = NameIdentifier.of("ml", "cat", "a" + P + "b"); + NameIdentifier api = + RelationalSchemaNamingBridge.nameIdentifierForApi(ident, Entity.EntityType.SCHEMA); + assertEquals("a:b", api.name()); + } + + @Test + public void flatSchemaIdent() { + NameIdentifier ident = NameIdentifier.of("ml", "cat", "flat"); + assertEquals( + "flat", + RelationalSchemaNamingBridge.nameIdentifierForStorage(ident, Entity.EntityType.SCHEMA) + .name()); + assertEquals( + "flat", + RelationalSchemaNamingBridge.nameIdentifierForApi(ident, Entity.EntityType.SCHEMA).name()); + } + + // ------------------------------------------------------------------------- + // wrapperUpdater – SCHEMA guard + // ------------------------------------------------------------------------- + + @Test + public void wrapperUpdaterRejectsSchema() { + assertThrows( + IllegalArgumentException.class, + () -> RelationalSchemaNamingBridge.wrapperUpdater(Entity.EntityType.SCHEMA, e -> e)); + } + + // ------------------------------------------------------------------------- + // genericEntityMetadataFullNameForApi + // ------------------------------------------------------------------------- + + @Test + public void genericEntityApiTable() { + // TABLE has 3 dot-parts; schema segment at index 1 gets physical→logical conversion + String physicalName = "catalog.a" + P + "b.mytable"; + GenericEntity entity = + GenericEntity.builder() + .withId(1L) + .withEntityType(Entity.EntityType.TABLE) + .withName(physicalName) + .withNamespace(Namespace.of("ml")) + .build(); + GenericEntity result = RelationalSchemaNamingBridge.genericEntityMetadataFullNameForApi(entity); + assertEquals("catalog.a:b.mytable", result.name()); + } + + @Test + public void genericEntityApiSchema() { + // SCHEMA has 2 dot-parts + String physicalName = "catalog.a" + P + "b" + P + "c"; + GenericEntity entity = + GenericEntity.builder() + .withId(2L) + .withEntityType(Entity.EntityType.SCHEMA) + .withName(physicalName) + .withNamespace(Namespace.of("ml")) + .build(); + GenericEntity result = RelationalSchemaNamingBridge.genericEntityMetadataFullNameForApi(entity); + assertEquals("catalog.a:b:c", result.name()); + } + + @Test + public void genericEntityApiFlatSchema() { + // Schema name with no physical separator → no conversion; same instance returned + String flatName = "catalog.flat"; + GenericEntity entity = + GenericEntity.builder() + .withId(3L) + .withEntityType(Entity.EntityType.SCHEMA) + .withName(flatName) + .withNamespace(Namespace.of("ml")) + .build(); + GenericEntity result = RelationalSchemaNamingBridge.genericEntityMetadataFullNameForApi(entity); + assertSame(entity, result); + } + + @Test + public void genericEntityApiNullName() { + GenericEntity entity = + GenericEntity.builder() + .withId(4L) + .withEntityType(Entity.EntityType.TABLE) + .withName(null) + .withNamespace(Namespace.of("ml")) + .build(); + GenericEntity result = RelationalSchemaNamingBridge.genericEntityMetadataFullNameForApi(entity); + assertSame(entity, result); + } + + @Test + public void genericEntityApiUnmappedType() { + // Entity types that don't map to MetadataObject.Type (e.g. TABLE_STATISTIC) are returned as-is + GenericEntity entity = + GenericEntity.builder() + .withId(5L) + .withEntityType(Entity.EntityType.TABLE_STATISTIC) + .withName("some.name") + .withNamespace(Namespace.of("ml")) + .build(); + GenericEntity result = RelationalSchemaNamingBridge.genericEntityMetadataFullNameForApi(entity); + assertSame(entity, result); + } + + // ------------------------------------------------------------------------- + // Helpers + // ------------------------------------------------------------------------- + + private static void mockSeparator(String sep) throws Exception { + Config config = Mockito.mock(Config.class); + Mockito.when(config.get(Configs.SCHEMA_SEPARATOR)).thenReturn(sep); + FieldUtils.writeField(GravitinoEnv.getInstance(), "config", config, true); + } +} From 61dcc028b82c2abdefa26b353a4c7f28a6b57c96 Mon Sep 17 00:00:00 2001 From: roryqi Date: Wed, 13 May 2026 08:37:12 +0000 Subject: [PATCH 02/44] fix(core): address code review issues in nested namespace JDBC backend - Replace Preconditions.checkNotNull with Objects.requireNonNull in JDBCBackend.batchInsertRelations for consistency with the interface - Add @SuppressWarnings explanatory comments in unchecked-cast methods - Document FILESET/TOPIC/MODEL/MODEL_VERSION passthrough in default switch branches of nameIdentifierForStorage/nameIdentifierForApi - Expand Javadoc on embeddedNamespaceForStorage/Api to warn about the index-2 layout assumption - Guard statisticEntityForApi/Storage with Preconditions.checkArgument to reject non-TableStatisticEntity subtypes at the call site - Add tests: roleEntityForStorage/Api round-trips, batchInsertRelations cache invalidation in RelationalEntityStore, and non-OWNER_REL rejection in JDBCBackend.batchInsertRelations Co-Authored-By: Claude Sonnet 4.6 --- .../storage/relational/JDBCBackend.java | 9 +- .../RelationalSchemaNamingBridge.java | 41 ++++++++- .../relational/TestJDBCBackendBatchGet.java | 18 ++++ .../relational/TestRelationalEntityStore.java | 56 ++++++++++++ .../TestRelationalSchemaNamingBridge.java | 89 +++++++++++++++++++ 5 files changed, 205 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java b/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java index c1fc087ad09..ac72e56b04e 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java @@ -29,6 +29,7 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.function.Function; import java.util.stream.Collectors; import org.apache.commons.lang3.tuple.Pair; @@ -917,10 +918,10 @@ public void batchInsertRelations( if (srcIdentifiers == null || srcIdentifiers.isEmpty()) { return; } - Preconditions.checkNotNull(relType, "relType must not be null"); - Preconditions.checkNotNull(srcType, "srcType must not be null"); - Preconditions.checkNotNull(dstIdentifier, "dstIdentifier must not be null"); - Preconditions.checkNotNull(dstType, "dstType must not be null"); + Objects.requireNonNull(relType, "relType must not be null"); + Objects.requireNonNull(srcType, "srcType must not be null"); + Objects.requireNonNull(dstIdentifier, "dstIdentifier must not be null"); + Objects.requireNonNull(dstType, "dstType must not be null"); switch (relType) { case OWNER_REL: List srcStorage = diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalSchemaNamingBridge.java b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalSchemaNamingBridge.java index bfd394b8ceb..cf9e7854e41 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalSchemaNamingBridge.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalSchemaNamingBridge.java @@ -212,7 +212,12 @@ private static String convertSchemaSegmentAt( return DOT_JOINER.join(copy); } - /** Converts the schema segment embedded at namespace index 2 when length ≥ 3. */ + /** + * Converts the schema segment at namespace index 2 (the slot occupied by the schema name in the + * standard {@code [metalake, catalog, schema, ...]} layout) from logical to physical form. Callers + * must only pass namespaces that follow this layout; if the schema sits at a different index the + * conversion will silently apply to the wrong segment. + */ public static Namespace embeddedNamespaceForStorage(Namespace ns) { if (ns.length() < 3) { return ns; @@ -223,6 +228,10 @@ public static Namespace embeddedNamespaceForStorage(Namespace ns) { return Namespace.of(copy); } + /** + * Converts the schema segment at namespace index 2 from physical to logical form. See {@link + * #embeddedNamespaceForStorage(Namespace)} for the layout constraint. + */ public static Namespace embeddedNamespaceForApi(Namespace ns) { if (ns.length() < 3) { return ns; @@ -259,10 +268,16 @@ public static NameIdentifier nameIdentifierForStorage( case TABLE_STATISTIC: return NameIdentifier.of(embeddedNamespaceForStorage(ident.namespace()), ident.name()); default: + // FILESET, TOPIC, MODEL, MODEL_VERSION and other types do not embed the schema + // in their identifier path and are forwarded as-is to the meta services. return ident; } } + /** + * Converts identifiers from physical storage naming back to the logical form used by API callers. + * Passthrough rules match {@link #nameIdentifierForStorage(NameIdentifier, Entity.EntityType)}. + */ public static NameIdentifier nameIdentifierForApi( NameIdentifier ident, Entity.EntityType entityType) { switch (entityType) { @@ -275,6 +290,7 @@ public static NameIdentifier nameIdentifierForApi( case TABLE_STATISTIC: return NameIdentifier.of(embeddedNamespaceForApi(ident.namespace()), ident.name()); default: + // FILESET, TOPIC, MODEL, MODEL_VERSION and other types are returned unchanged. return ident; } } @@ -406,7 +422,16 @@ public static FunctionEntity functionEntityForStorage(FunctionEntity e) { .build(); } + /** + * Converts a {@link StatisticEntity}'s namespace from physical storage form to logical API form. + * Only {@link TableStatisticEntity} is currently stored with an embedded schema namespace; other + * concrete subtypes are not expected and will cause an {@link IllegalArgumentException}. + */ public static StatisticEntity statisticEntityForApi(StatisticEntity e) { + Preconditions.checkArgument( + e instanceof TableStatisticEntity, + "Only TableStatisticEntity is supported by statisticEntityForApi, got: %s", + e.getClass().getSimpleName()); Namespace apiNs = embeddedNamespaceForApi(e.namespace()); if (apiNs.equals(e.namespace())) { return e; @@ -420,7 +445,15 @@ public static StatisticEntity statisticEntityForApi(StatisticEntity e) { .build(); } + /** + * Converts a {@link StatisticEntity}'s namespace to the physical storage form. See {@link + * #statisticEntityForApi(StatisticEntity)} for subtype restrictions. + */ public static StatisticEntity statisticEntityForStorage(StatisticEntity e) { + Preconditions.checkArgument( + e instanceof TableStatisticEntity, + "Only TableStatisticEntity is supported by statisticEntityForStorage, got: %s", + e.getClass().getSimpleName()); Namespace storageNs = embeddedNamespaceForStorage(e.namespace()); if (storageNs.equals(e.namespace())) { return e; @@ -442,7 +475,7 @@ public static StatisticEntity statisticEntityForStorage(StatisticEntity e) { * @param entityType bridged entity kind (e.g. {@link Entity.EntityType#TABLE}); passed to {@link * #entityForApi(Entity, Entity.EntityType)} */ - @SuppressWarnings("unchecked") + @SuppressWarnings("unchecked") // entityForApi/entityForStorage dispatch on known concrete types public static Function wrapperUpdater( Entity.EntityType entityType, Function updater) { Preconditions.checkArgument( @@ -455,7 +488,7 @@ public static Function wrapperUpdater( }; } - @SuppressWarnings("unchecked") + @SuppressWarnings("unchecked") // each case casts to the statically known subtype for that EntityType public static E entityForApi( E entity, Entity.EntityType type) { switch (type) { @@ -474,7 +507,7 @@ public static E entityForApi( } } - @SuppressWarnings("unchecked") + @SuppressWarnings("unchecked") // instanceof guards ensure each cast is safe public static E entityForStorage(E entity) { if (entity instanceof SchemaEntity) { return entity; diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/TestJDBCBackendBatchGet.java b/core/src/test/java/org/apache/gravitino/storage/relational/TestJDBCBackendBatchGet.java index a7410990b37..4b0db612de0 100644 --- a/core/src/test/java/org/apache/gravitino/storage/relational/TestJDBCBackendBatchGet.java +++ b/core/src/test/java/org/apache/gravitino/storage/relational/TestJDBCBackendBatchGet.java @@ -29,6 +29,7 @@ import org.apache.gravitino.Entity; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; +import org.apache.gravitino.SupportsRelationOperations; import org.apache.gravitino.job.JobHandle; import org.apache.gravitino.job.JobTemplate; import org.apache.gravitino.meta.BaseMetalake; @@ -1062,4 +1063,21 @@ public void testBatchGetViewsPartialResults() throws IOException { Assertions.assertEquals(1, result.size()); Assertions.assertEquals("view1", result.get(0).name()); } + + @TestTemplate + public void testBatchInsertRelationsRejectsUnsupportedRelationType() { + NameIdentifier src = NameIdentifier.of("metalake", "schema1"); + NameIdentifier dst = NameIdentifier.of("metalake", "role1"); + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + backend.batchInsertRelations( + SupportsRelationOperations.Type.TAG_METADATA_OBJECT_REL, + Lists.newArrayList(src), + Entity.EntityType.SCHEMA, + dst, + Entity.EntityType.ROLE, + false)); + } } diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalEntityStore.java b/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalEntityStore.java index 042c29681b8..5c84c9e2206 100644 --- a/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalEntityStore.java +++ b/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalEntityStore.java @@ -22,6 +22,7 @@ import static org.mockito.ArgumentMatchers.eq; import java.io.IOException; +import java.util.Arrays; import java.util.List; import java.util.function.Function; import org.apache.commons.lang3.reflect.FieldUtils; @@ -157,6 +158,61 @@ void testInsertRelationInvalidatesCacheAfterBackendInsert() dst, Entity.EntityType.TAG, SupportsRelationOperations.Type.TAG_METADATA_OBJECT_REL); } + @Test + void testBatchInsertRelationsInvalidatesCacheForAllSrcsAndDst() + throws IOException, IllegalAccessException { + NameIdentifier src1 = NameIdentifier.of("metalake", "schema1"); + NameIdentifier src2 = NameIdentifier.of("metalake", "schema2"); + NameIdentifier dst = NameIdentifier.of("metalake", "role1"); + List srcs = Arrays.asList(src1, src2); + NoOpsCache cache = (NoOpsCache) FieldUtils.readField(store, "cache", true); + + store.batchInsertRelations( + SupportsRelationOperations.Type.OWNER_REL, + srcs, + Entity.EntityType.SCHEMA, + dst, + Entity.EntityType.ROLE, + false); + + InOrder inOrder = Mockito.inOrder(backend, cache); + inOrder + .verify(backend) + .batchInsertRelations( + SupportsRelationOperations.Type.OWNER_REL, + srcs, + Entity.EntityType.SCHEMA, + dst, + Entity.EntityType.ROLE, + false); + inOrder + .verify(cache) + .invalidate(src1, Entity.EntityType.SCHEMA, SupportsRelationOperations.Type.OWNER_REL); + inOrder + .verify(cache) + .invalidate(src2, Entity.EntityType.SCHEMA, SupportsRelationOperations.Type.OWNER_REL); + inOrder + .verify(cache) + .invalidate(dst, Entity.EntityType.ROLE, SupportsRelationOperations.Type.OWNER_REL); + } + + @Test + void testBatchInsertRelationsWithEmptyListIsNoOp() throws IOException, IllegalAccessException { + NoOpsCache cache = (NoOpsCache) FieldUtils.readField(store, "cache", true); + + store.batchInsertRelations( + SupportsRelationOperations.Type.OWNER_REL, + List.of(), + Entity.EntityType.SCHEMA, + NameIdentifier.of("metalake", "role1"), + Entity.EntityType.ROLE, + false); + + Mockito.verify(backend, Mockito.never()) + .batchInsertRelations(any(), any(), any(), any(), any(), any(Boolean.class)); + Mockito.verify(cache, Mockito.never()).invalidate(any(), any(), any()); + } + @Test void testUpdateEntityRelationsInvalidatesCacheAfterBackendUpdate() throws IOException, NoSuchEntityException, EntityAlreadyExistsException, diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalSchemaNamingBridge.java b/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalSchemaNamingBridge.java index 48a436f77fb..bf6693cb9f9 100644 --- a/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalSchemaNamingBridge.java +++ b/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalSchemaNamingBridge.java @@ -22,6 +22,9 @@ import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; +import com.google.common.collect.Lists; +import java.time.Instant; +import java.util.List; import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.gravitino.Config; import org.apache.gravitino.Configs; @@ -30,7 +33,13 @@ import org.apache.gravitino.MetadataObject; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; +import org.apache.gravitino.authorization.Privilege; +import org.apache.gravitino.authorization.Privileges; +import org.apache.gravitino.authorization.SecurableObject; +import org.apache.gravitino.authorization.SecurableObjects; +import org.apache.gravitino.meta.AuditInfo; import org.apache.gravitino.meta.GenericEntity; +import org.apache.gravitino.meta.RoleEntity; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -356,10 +365,90 @@ public void genericEntityApiUnmappedType() { assertSame(entity, result); } + // ------------------------------------------------------------------------- + // roleEntityForStorage / roleEntityForApi + // ------------------------------------------------------------------------- + + @Test + public void roleWithNestedSchemaSecurableObjectToStorage() { + // SecurableObject of type SCHEMA with logical name "catalog.a:b" + List privs = Lists.newArrayList(Privileges.UseSchema.allow()); + SecurableObject catalogObj = SecurableObjects.ofCatalog("catalog", Lists.newArrayList()); + SecurableObject schemaObj = SecurableObjects.ofSchema(catalogObj, "a:b", privs); + + RoleEntity role = buildRole(Lists.newArrayList(schemaObj)); + + RoleEntity stored = RelationalSchemaNamingBridge.roleEntityForStorage(role); + SecurableObject storedSchema = stored.securableObjects().get(0); + // schema segment "a:b" → "ab" + assertEquals("catalog.a" + P + "b", storedSchema.fullName()); + } + + @Test + public void roleWithNestedSchemaSecurableObjectToApi() { + // SecurableObject of type SCHEMA with physical name "catalog.ab" + List privs = Lists.newArrayList(Privileges.UseSchema.allow()); + SecurableObject schemaObj = + SecurableObjects.parse("catalog.a" + P + "b", MetadataObject.Type.SCHEMA, privs); + + RoleEntity role = buildRole(Lists.newArrayList(schemaObj)); + + RoleEntity api = RelationalSchemaNamingBridge.roleEntityForApi(role); + SecurableObject apiSchema = api.securableObjects().get(0); + assertEquals("catalog.a:b", apiSchema.fullName()); + } + + @Test + public void roleWithFlatSchemaIsUnchanged() { + // A schema without the separator → same instance returned + List privs = Lists.newArrayList(Privileges.UseSchema.allow()); + SecurableObject catalogObj = SecurableObjects.ofCatalog("catalog", Lists.newArrayList()); + SecurableObject schemaObj = SecurableObjects.ofSchema(catalogObj, "flat", privs); + + RoleEntity role = buildRole(Lists.newArrayList(schemaObj)); + assertSame(role, RelationalSchemaNamingBridge.roleEntityForStorage(role)); + assertSame(role, RelationalSchemaNamingBridge.roleEntityForApi(role)); + } + + @Test + public void roleWithNoSecurableObjectsIsUnchanged() { + RoleEntity role = buildRole(null); + assertSame(role, RelationalSchemaNamingBridge.roleEntityForStorage(role)); + assertSame(role, RelationalSchemaNamingBridge.roleEntityForApi(role)); + } + + @Test + public void roleRoundTrip() { + List privs = Lists.newArrayList(Privileges.UseSchema.allow()); + SecurableObject catalogObj = SecurableObjects.ofCatalog("catalog", Lists.newArrayList()); + SecurableObject schemaObj = SecurableObjects.ofSchema(catalogObj, "a:b:c", privs); + + RoleEntity original = buildRole(Lists.newArrayList(schemaObj)); + RoleEntity stored = RelationalSchemaNamingBridge.roleEntityForStorage(original); + RoleEntity backToApi = RelationalSchemaNamingBridge.roleEntityForApi(stored); + + assertEquals( + original.securableObjects().get(0).fullName(), + backToApi.securableObjects().get(0).fullName()); + } + // ------------------------------------------------------------------------- // Helpers // ------------------------------------------------------------------------- + private static RoleEntity buildRole(List securableObjects) { + AuditInfo audit = + AuditInfo.builder().withCreator("test").withCreateTime(Instant.now()).build(); + return RoleEntity.builder() + .withId(1L) + .withName("role1") + .withNamespace(Namespace.of("ml")) + .withProperties(null) + .withAuditInfo(audit) + .withSecurableObjects(securableObjects) + .build(); + } + private static void mockSeparator(String sep) throws Exception { Config config = Mockito.mock(Config.class); Mockito.when(config.get(Configs.SCHEMA_SEPARATOR)).thenReturn(sep); From 5e175bb6bf8698cead9c8bdf4a5076e83e4c5570 Mon Sep 17 00:00:00 2001 From: roryqi Date: Wed, 13 May 2026 08:51:59 +0000 Subject: [PATCH 03/44] refactor(core): simplify RelationalSchemaNamingBridge and its tests - Extract mapRoleSecurableObjects helper to eliminate the duplicate builder block shared by roleEntityForStorage and roleEntityForApi - Extract statisticEntityWithNamespace helper to consolidate the type guard and builder shared by statisticEntityForApi and statisticEntityForStorage - Replace repeated Lists.newArrayList(Privileges.UseSchema.allow()) and SecurableObjects.ofCatalog("catalog", ...) constructions in tests with private static final constants USE_SCHEMA_PRIVS and CATALOG_OBJ - Remove inline WHAT-comments that restate what the assertions already express Co-Authored-By: Claude Sonnet 4.6 --- .../RelationalSchemaNamingBridge.java | 56 +++++-------------- .../TestRelationalSchemaNamingBridge.java | 56 ++++++------------- 2 files changed, 33 insertions(+), 79 deletions(-) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalSchemaNamingBridge.java b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalSchemaNamingBridge.java index cf9e7854e41..559e255fc1c 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalSchemaNamingBridge.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalSchemaNamingBridge.java @@ -92,36 +92,21 @@ public static SecurableObject securableObjectForApi(SecurableObject object) { } public static RoleEntity roleEntityForStorage(RoleEntity entity) { - List objects = entity.securableObjects(); - if (objects == null || objects.isEmpty()) { - return entity; - } - List mapped = - objects.stream() - .map(RelationalSchemaNamingBridge::securableObjectForStorage) - .collect(Collectors.toList()); - if (mapped.equals(objects)) { - return entity; - } - return RoleEntity.builder() - .withId(entity.id()) - .withName(entity.name()) - .withNamespace(entity.namespace()) - .withProperties(entity.properties()) - .withAuditInfo(entity.auditInfo()) - .withSecurableObjects(mapped) - .build(); + return mapRoleSecurableObjects(entity, RelationalSchemaNamingBridge::securableObjectForStorage); } public static RoleEntity roleEntityForApi(RoleEntity entity) { + return mapRoleSecurableObjects(entity, RelationalSchemaNamingBridge::securableObjectForApi); + } + + private static RoleEntity mapRoleSecurableObjects( + RoleEntity entity, Function mapper) { List objects = entity.securableObjects(); if (objects == null || objects.isEmpty()) { return entity; } List mapped = - objects.stream() - .map(RelationalSchemaNamingBridge::securableObjectForApi) - .collect(Collectors.toList()); + objects.stream().map(mapper).collect(Collectors.toList()); if (mapped.equals(objects)) { return entity; } @@ -428,21 +413,7 @@ public static FunctionEntity functionEntityForStorage(FunctionEntity e) { * concrete subtypes are not expected and will cause an {@link IllegalArgumentException}. */ public static StatisticEntity statisticEntityForApi(StatisticEntity e) { - Preconditions.checkArgument( - e instanceof TableStatisticEntity, - "Only TableStatisticEntity is supported by statisticEntityForApi, got: %s", - e.getClass().getSimpleName()); - Namespace apiNs = embeddedNamespaceForApi(e.namespace()); - if (apiNs.equals(e.namespace())) { - return e; - } - return TableStatisticEntity.builder() - .withId(e.id()) - .withName(e.name()) - .withValue(e.value()) - .withAuditInfo((AuditInfo) e.auditInfo()) - .withNamespace(apiNs) - .build(); + return statisticEntityWithNamespace(e, embeddedNamespaceForApi(e.namespace())); } /** @@ -450,12 +421,15 @@ public static StatisticEntity statisticEntityForApi(StatisticEntity e) { * #statisticEntityForApi(StatisticEntity)} for subtype restrictions. */ public static StatisticEntity statisticEntityForStorage(StatisticEntity e) { + return statisticEntityWithNamespace(e, embeddedNamespaceForStorage(e.namespace())); + } + + private static StatisticEntity statisticEntityWithNamespace(StatisticEntity e, Namespace ns) { Preconditions.checkArgument( e instanceof TableStatisticEntity, - "Only TableStatisticEntity is supported by statisticEntityForStorage, got: %s", + "Only TableStatisticEntity is supported, got: %s", e.getClass().getSimpleName()); - Namespace storageNs = embeddedNamespaceForStorage(e.namespace()); - if (storageNs.equals(e.namespace())) { + if (ns.equals(e.namespace())) { return e; } return TableStatisticEntity.builder() @@ -463,7 +437,7 @@ public static StatisticEntity statisticEntityForStorage(StatisticEntity e) { .withName(e.name()) .withValue(e.value()) .withAuditInfo((AuditInfo) e.auditInfo()) - .withNamespace(storageNs) + .withNamespace(ns) .build(); } diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalSchemaNamingBridge.java b/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalSchemaNamingBridge.java index bf6693cb9f9..1bb1ba68426 100644 --- a/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalSchemaNamingBridge.java +++ b/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalSchemaNamingBridge.java @@ -51,6 +51,11 @@ public class TestRelationalSchemaNamingBridge { /** ASCII-1 internal separator used in physical storage. */ private static final String P = ""; + private static final List USE_SCHEMA_PRIVS = + Lists.newArrayList(Privileges.UseSchema.allow()); + private static final SecurableObject CATALOG_OBJ = + SecurableObjects.ofCatalog("catalog", Lists.newArrayList()); + @BeforeEach public void setUp() throws Exception { mockSeparator(":"); @@ -67,7 +72,6 @@ public void tearDown() throws Exception { @Test public void schemaToStorage() { - // SCHEMA expects 2 dot-parts; schema segment "a:b:c" → "abc" String logical = "catalog.a:b:c"; String physical = RelationalSchemaNamingBridge.convertMetadataObjectDottedFullName( @@ -86,7 +90,6 @@ public void schemaToApi() { @Test public void tableToStorage() { - // TABLE expects 3 dot-parts; schema segment at index 1 String logical = "catalog.a:b.mytable"; String physical = RelationalSchemaNamingBridge.convertMetadataObjectDottedFullName( @@ -123,7 +126,6 @@ public void functionToStorage() { @Test public void columnToStorage() { - // COLUMN expects 4 dot-parts String logical = "catalog.a:b.mytable.mycol"; String physical = RelationalSchemaNamingBridge.convertMetadataObjectDottedFullName( @@ -155,7 +157,6 @@ public void catalogPassthrough() { @Test public void schemaPartCountMismatch() { - // SCHEMA expects 2 parts; 3 parts → no conversion String name = "catalog.schema.extra"; String result = RelationalSchemaNamingBridge.convertMetadataObjectDottedFullName( @@ -295,7 +296,6 @@ public void wrapperUpdaterRejectsSchema() { @Test public void genericEntityApiTable() { - // TABLE has 3 dot-parts; schema segment at index 1 gets physical→logical conversion String physicalName = "catalog.a" + P + "b.mytable"; GenericEntity entity = GenericEntity.builder() @@ -310,7 +310,6 @@ public void genericEntityApiTable() { @Test public void genericEntityApiSchema() { - // SCHEMA has 2 dot-parts String physicalName = "catalog.a" + P + "b" + P + "c"; GenericEntity entity = GenericEntity.builder() @@ -371,40 +370,24 @@ public void genericEntityApiUnmappedType() { @Test public void roleWithNestedSchemaSecurableObjectToStorage() { - // SecurableObject of type SCHEMA with logical name "catalog.a:b" - List privs = Lists.newArrayList(Privileges.UseSchema.allow()); - SecurableObject catalogObj = SecurableObjects.ofCatalog("catalog", Lists.newArrayList()); - SecurableObject schemaObj = SecurableObjects.ofSchema(catalogObj, "a:b", privs); - - RoleEntity role = buildRole(Lists.newArrayList(schemaObj)); - - RoleEntity stored = RelationalSchemaNamingBridge.roleEntityForStorage(role); - SecurableObject storedSchema = stored.securableObjects().get(0); - // schema segment "a:b" → "ab" - assertEquals("catalog.a" + P + "b", storedSchema.fullName()); + SecurableObject schemaObj = SecurableObjects.ofSchema(CATALOG_OBJ, "a:b", USE_SCHEMA_PRIVS); + RoleEntity stored = + RelationalSchemaNamingBridge.roleEntityForStorage(buildRole(Lists.newArrayList(schemaObj))); + assertEquals("catalog.a" + P + "b", stored.securableObjects().get(0).fullName()); } @Test public void roleWithNestedSchemaSecurableObjectToApi() { - // SecurableObject of type SCHEMA with physical name "catalog.ab" - List privs = Lists.newArrayList(Privileges.UseSchema.allow()); SecurableObject schemaObj = - SecurableObjects.parse("catalog.a" + P + "b", MetadataObject.Type.SCHEMA, privs); - - RoleEntity role = buildRole(Lists.newArrayList(schemaObj)); - - RoleEntity api = RelationalSchemaNamingBridge.roleEntityForApi(role); - SecurableObject apiSchema = api.securableObjects().get(0); - assertEquals("catalog.a:b", apiSchema.fullName()); + SecurableObjects.parse("catalog.a" + P + "b", MetadataObject.Type.SCHEMA, USE_SCHEMA_PRIVS); + RoleEntity api = + RelationalSchemaNamingBridge.roleEntityForApi(buildRole(Lists.newArrayList(schemaObj))); + assertEquals("catalog.a:b", api.securableObjects().get(0).fullName()); } @Test public void roleWithFlatSchemaIsUnchanged() { - // A schema without the separator → same instance returned - List privs = Lists.newArrayList(Privileges.UseSchema.allow()); - SecurableObject catalogObj = SecurableObjects.ofCatalog("catalog", Lists.newArrayList()); - SecurableObject schemaObj = SecurableObjects.ofSchema(catalogObj, "flat", privs); - + SecurableObject schemaObj = SecurableObjects.ofSchema(CATALOG_OBJ, "flat", USE_SCHEMA_PRIVS); RoleEntity role = buildRole(Lists.newArrayList(schemaObj)); assertSame(role, RelationalSchemaNamingBridge.roleEntityForStorage(role)); assertSame(role, RelationalSchemaNamingBridge.roleEntityForApi(role)); @@ -419,14 +402,11 @@ public void roleWithNoSecurableObjectsIsUnchanged() { @Test public void roleRoundTrip() { - List privs = Lists.newArrayList(Privileges.UseSchema.allow()); - SecurableObject catalogObj = SecurableObjects.ofCatalog("catalog", Lists.newArrayList()); - SecurableObject schemaObj = SecurableObjects.ofSchema(catalogObj, "a:b:c", privs); - + SecurableObject schemaObj = SecurableObjects.ofSchema(CATALOG_OBJ, "a:b:c", USE_SCHEMA_PRIVS); RoleEntity original = buildRole(Lists.newArrayList(schemaObj)); - RoleEntity stored = RelationalSchemaNamingBridge.roleEntityForStorage(original); - RoleEntity backToApi = RelationalSchemaNamingBridge.roleEntityForApi(stored); - + RoleEntity backToApi = + RelationalSchemaNamingBridge.roleEntityForApi( + RelationalSchemaNamingBridge.roleEntityForStorage(original)); assertEquals( original.securableObjects().get(0).fullName(), backToApi.securableObjects().get(0).fullName()); From cb217322d85b6d4aca70982075112026c4100620 Mon Sep 17 00:00:00 2001 From: roryqi Date: Wed, 13 May 2026 08:56:19 +0000 Subject: [PATCH 04/44] fix(core): complete cache write-through convention in RelationalEntityStore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit batchDelete and batchPut were the only write paths that bypassed the cache — callers that relied on them would silently observe stale reads. Apply the same pattern as their single-item counterparts: - batchDelete: invalidate each (ident, entityType) pair after backend - batchPut: put each entity into cache after backend Add unit tests covering the backend-then-cache ordering for both methods. Co-Authored-By: Claude Sonnet 4.6 --- .../relational/RelationalEntityStore.java | 9 +++- .../relational/TestRelationalEntityStore.java | 41 +++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStore.java b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStore.java index d531d5cc011..1119f1c8a8c 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStore.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStore.java @@ -378,13 +378,20 @@ public List updateEntityRelations( public int batchDelete( List> entitiesToDelete, boolean cascade) throws IOException { - return backend.batchDelete(entitiesToDelete, cascade); + int deleted = backend.batchDelete(entitiesToDelete, cascade); + for (Pair entry : entitiesToDelete) { + cache.invalidate(entry.getLeft(), entry.getRight()); + } + return deleted; } @Override public void batchPut(List entities, boolean overwritten) throws IOException, EntityAlreadyExistsException { backend.batchPut(entities, overwritten); + for (E entity : entities) { + cache.put(entity); + } } private Optional>> getCachedRelations( diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalEntityStore.java b/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalEntityStore.java index 5c84c9e2206..268d24bce47 100644 --- a/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalEntityStore.java +++ b/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalEntityStore.java @@ -26,11 +26,13 @@ import java.util.List; import java.util.function.Function; import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.commons.lang3.tuple.Pair; import org.apache.gravitino.Config; import org.apache.gravitino.Configs; import org.apache.gravitino.Entity; import org.apache.gravitino.EntityAlreadyExistsException; import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.meta.BaseMetalake; import org.apache.gravitino.SupportsRelationOperations; import org.apache.gravitino.cache.NoOpsCache; import org.apache.gravitino.exceptions.NoSuchEntityException; @@ -284,4 +286,43 @@ void testUpdateEntityRelationsInvalidatesCacheAfterBackendUpdate() Entity.EntityType.TABLE, SupportsRelationOperations.Type.TAG_METADATA_OBJECT_REL); } + + @Test + void testBatchDeleteInvalidatesCacheForEachEntity() + throws IOException, IllegalAccessException { + NameIdentifier ident1 = NameIdentifier.of("metalake", "catalog1"); + NameIdentifier ident2 = NameIdentifier.of("metalake", "catalog2"); + List> toDelete = + Arrays.asList( + Pair.of(ident1, Entity.EntityType.CATALOG), + Pair.of(ident2, Entity.EntityType.CATALOG)); + NoOpsCache cache = (NoOpsCache) FieldUtils.readField(store, "cache", true); + + Mockito.when(backend.batchDelete(toDelete, false)).thenReturn(2); + + int result = store.batchDelete(toDelete, false); + + Assertions.assertEquals(2, result); + InOrder inOrder = Mockito.inOrder(backend, cache); + inOrder.verify(backend).batchDelete(toDelete, false); + inOrder.verify(cache).invalidate(ident1, Entity.EntityType.CATALOG); + inOrder.verify(cache).invalidate(ident2, Entity.EntityType.CATALOG); + } + + @Test + void testBatchPutPopulatesCacheForEachEntity() + throws IOException, EntityAlreadyExistsException, IllegalAccessException { + NoOpsCache cache = (NoOpsCache) FieldUtils.readField(store, "cache", true); + + BaseMetalake e1 = Mockito.mock(BaseMetalake.class); + BaseMetalake e2 = Mockito.mock(BaseMetalake.class); + List entities = Arrays.asList(e1, e2); + + store.batchPut(entities, false); + + InOrder inOrder = Mockito.inOrder(backend, cache); + inOrder.verify(backend).batchPut(entities, false); + inOrder.verify(cache).put(e1); + inOrder.verify(cache).put(e2); + } } From caf3b96e5685e95baf4a1781fbf07e6f4629e815 Mon Sep 17 00:00:00 2001 From: roryqi Date: Wed, 13 May 2026 08:59:44 +0000 Subject: [PATCH 05/44] Revert "fix(core): complete cache write-through convention in RelationalEntityStore" This reverts commit cb217322d85b6d4aca70982075112026c4100620. --- .../relational/RelationalEntityStore.java | 9 +--- .../relational/TestRelationalEntityStore.java | 41 ------------------- 2 files changed, 1 insertion(+), 49 deletions(-) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStore.java b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStore.java index 1119f1c8a8c..d531d5cc011 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStore.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStore.java @@ -378,20 +378,13 @@ public List updateEntityRelations( public int batchDelete( List> entitiesToDelete, boolean cascade) throws IOException { - int deleted = backend.batchDelete(entitiesToDelete, cascade); - for (Pair entry : entitiesToDelete) { - cache.invalidate(entry.getLeft(), entry.getRight()); - } - return deleted; + return backend.batchDelete(entitiesToDelete, cascade); } @Override public void batchPut(List entities, boolean overwritten) throws IOException, EntityAlreadyExistsException { backend.batchPut(entities, overwritten); - for (E entity : entities) { - cache.put(entity); - } } private Optional>> getCachedRelations( diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalEntityStore.java b/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalEntityStore.java index 268d24bce47..5c84c9e2206 100644 --- a/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalEntityStore.java +++ b/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalEntityStore.java @@ -26,13 +26,11 @@ import java.util.List; import java.util.function.Function; import org.apache.commons.lang3.reflect.FieldUtils; -import org.apache.commons.lang3.tuple.Pair; import org.apache.gravitino.Config; import org.apache.gravitino.Configs; import org.apache.gravitino.Entity; import org.apache.gravitino.EntityAlreadyExistsException; import org.apache.gravitino.NameIdentifier; -import org.apache.gravitino.meta.BaseMetalake; import org.apache.gravitino.SupportsRelationOperations; import org.apache.gravitino.cache.NoOpsCache; import org.apache.gravitino.exceptions.NoSuchEntityException; @@ -286,43 +284,4 @@ void testUpdateEntityRelationsInvalidatesCacheAfterBackendUpdate() Entity.EntityType.TABLE, SupportsRelationOperations.Type.TAG_METADATA_OBJECT_REL); } - - @Test - void testBatchDeleteInvalidatesCacheForEachEntity() - throws IOException, IllegalAccessException { - NameIdentifier ident1 = NameIdentifier.of("metalake", "catalog1"); - NameIdentifier ident2 = NameIdentifier.of("metalake", "catalog2"); - List> toDelete = - Arrays.asList( - Pair.of(ident1, Entity.EntityType.CATALOG), - Pair.of(ident2, Entity.EntityType.CATALOG)); - NoOpsCache cache = (NoOpsCache) FieldUtils.readField(store, "cache", true); - - Mockito.when(backend.batchDelete(toDelete, false)).thenReturn(2); - - int result = store.batchDelete(toDelete, false); - - Assertions.assertEquals(2, result); - InOrder inOrder = Mockito.inOrder(backend, cache); - inOrder.verify(backend).batchDelete(toDelete, false); - inOrder.verify(cache).invalidate(ident1, Entity.EntityType.CATALOG); - inOrder.verify(cache).invalidate(ident2, Entity.EntityType.CATALOG); - } - - @Test - void testBatchPutPopulatesCacheForEachEntity() - throws IOException, EntityAlreadyExistsException, IllegalAccessException { - NoOpsCache cache = (NoOpsCache) FieldUtils.readField(store, "cache", true); - - BaseMetalake e1 = Mockito.mock(BaseMetalake.class); - BaseMetalake e2 = Mockito.mock(BaseMetalake.class); - List entities = Arrays.asList(e1, e2); - - store.batchPut(entities, false); - - InOrder inOrder = Mockito.inOrder(backend, cache); - inOrder.verify(backend).batchPut(entities, false); - inOrder.verify(cache).put(e1); - inOrder.verify(cache).put(e2); - } } From d87ba6a61cccc9b92e2eea39fbce9052fefa9772 Mon Sep 17 00:00:00 2001 From: roryqi Date: Wed, 13 May 2026 09:11:19 +0000 Subject: [PATCH 06/44] refactor(core): replace convertMetadataObjectDottedFullName with NameIdentifier-based approach Use the existing nameIdentifierForStorage/Api infrastructure via a sentinel metalake prefix instead of bespoke string-splitting logic. Removes the package-private convertMetadataObjectDottedFullName and convertSchemaSegmentAt helpers; updates securableObjectForStorage/Api and genericEntityMetadataFullNameForApi accordingly. Tests are replaced with equivalent coverage through the public securableObject API. Co-Authored-By: Claude Sonnet 4.6 --- .../RelationalSchemaNamingBridge.java | 102 ++++------- .../TestRelationalSchemaNamingBridge.java | 167 ++++-------------- 2 files changed, 70 insertions(+), 199 deletions(-) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalSchemaNamingBridge.java b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalSchemaNamingBridge.java index 559e255fc1c..12f6eaaf50f 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalSchemaNamingBridge.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalSchemaNamingBridge.java @@ -26,7 +26,6 @@ import java.util.stream.Collectors; import org.apache.gravitino.Entity; import org.apache.gravitino.HasIdentifier; -import org.apache.gravitino.MetadataObject; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; import org.apache.gravitino.authorization.SecurableObject; @@ -41,6 +40,7 @@ import org.apache.gravitino.meta.TableStatisticEntity; import org.apache.gravitino.meta.ViewEntity; import org.apache.gravitino.utils.HierarchicalSchemaUtil; +import org.apache.gravitino.utils.MetadataObjectUtil; /** * Translates hierarchical schema naming between the logical representation used above {@link @@ -54,9 +54,25 @@ public final class RelationalSchemaNamingBridge { private static final Joiner DOT_JOINER = Joiner.on('.'); + private static final String SENTINEL_METALAKE = "_"; private RelationalSchemaNamingBridge() {} + /** + * Reconstructs a dotted metadata-object full name from a {@link NameIdentifier} that was created + * by prepending {@link #SENTINEL_METALAKE} to the original full name. Skips the sentinel at + * {@code namespace.levels()[0]} and joins the remaining levels with the identifier's name. + */ + private static String identToMetadataObjectFullName(NameIdentifier ident) { + String[] levels = ident.namespace().levels(); + List parts = new ArrayList<>(levels.length); + for (int i = 1; i < levels.length; i++) { + parts.add(levels[i]); + } + parts.add(ident.name()); + return DOT_JOINER.join(parts); + } + private static String separator() { return HierarchicalSchemaUtil.schemaSeparator(); } @@ -68,12 +84,14 @@ private static String separator() { * Entity.EntityType#MODEL}, {@link Entity.EntityType#MODEL_VERSION}) are unchanged. */ public static SecurableObject securableObjectForStorage(SecurableObject object) { - String converted = - convertMetadataObjectDottedFullName(object.fullName(), object.type(), /* toStorage */ true); - if (converted.equals(object.fullName())) { + Entity.EntityType entityType = MetadataObjectUtil.toEntityType(object.type()); + NameIdentifier ident = NameIdentifier.parse(SENTINEL_METALAKE + "." + object.fullName()); + NameIdentifier converted = nameIdentifierForStorage(ident, entityType); + if (converted.equals(ident)) { return object; } - return SecurableObjects.parse(converted, object.type(), object.privileges()); + return SecurableObjects.parse( + identToMetadataObjectFullName(converted), object.type(), object.privileges()); } /** @@ -82,13 +100,14 @@ public static SecurableObject securableObjectForStorage(SecurableObject object) * #securableObjectForStorage(SecurableObject)}. */ public static SecurableObject securableObjectForApi(SecurableObject object) { - String converted = - convertMetadataObjectDottedFullName( - object.fullName(), object.type(), /* toStorage */ false); - if (converted.equals(object.fullName())) { + Entity.EntityType entityType = MetadataObjectUtil.toEntityType(object.type()); + NameIdentifier ident = NameIdentifier.parse(SENTINEL_METALAKE + "." + object.fullName()); + NameIdentifier converted = nameIdentifierForApi(ident, entityType); + if (converted.equals(ident)) { return object; } - return SecurableObjects.parse(converted, object.type(), object.privileges()); + return SecurableObjects.parse( + identToMetadataObjectFullName(converted), object.type(), object.privileges()); } public static RoleEntity roleEntityForStorage(RoleEntity entity) { @@ -123,80 +142,25 @@ private static RoleEntity mapRoleSecurableObjects( /** * Normalizes {@link GenericEntity#name()} when it holds a dotted metadata-object path whose * schema segment uses physical hierarchical encoding, for API callers above {@link JDBCBackend}. - * - *

Uses {@link MetadataObject.Type} derived from {@link Entity.EntityType} (same as relational - * policy/tag stubs that only set dotted {@link GenericEntity#name()}, not a full table/view - * namespace). */ public static GenericEntity genericEntityMetadataFullNameForApi(GenericEntity entity) { String name = entity.name(); if (name == null || name.isEmpty()) { return entity; } - final MetadataObject.Type moType; - try { - moType = MetadataObject.Type.valueOf(entity.type().name()); - } catch (IllegalArgumentException e) { - return entity; - } - String convertedName = convertMetadataObjectDottedFullName(name, moType, false); - - if (convertedName.equals(name)) { + NameIdentifier ident = NameIdentifier.parse(SENTINEL_METALAKE + "." + name); + NameIdentifier converted = nameIdentifierForApi(ident, entity.type()); + if (converted.equals(ident)) { return entity; } return GenericEntity.builder() .withId(entity.id()) .withEntityType(entity.type()) - .withName(convertedName) + .withName(identToMetadataObjectFullName(converted)) .withNamespace(entity.namespace()) .build(); } - /** - * Rewrites the schema segment inside a dotted metadata full name (catalog.schema[.rest]) between - * logical and physical hierarchical forms. Non-matching part counts are returned unchanged. - */ - static String convertMetadataObjectDottedFullName( - String fullName, MetadataObject.Type type, boolean toStorage) { - if (fullName == null || fullName.isEmpty()) { - return fullName; - } - switch (type) { - case SCHEMA: - return convertSchemaSegmentAt(fullName, /* expectedParts */ 2, toStorage); - case TABLE: - case VIEW: - case FUNCTION: - return convertSchemaSegmentAt(fullName, /* expectedParts */ 3, toStorage); - case COLUMN: - return convertSchemaSegmentAt(fullName, /* expectedParts */ 4, toStorage); - default: - return fullName; - } - } - - private static String convertSchemaSegmentAt( - String fullName, int expectedParts, boolean toStorage) { - List parts = SecurableObjects.DOT_SPLITTER.splitToList(fullName); - if (parts.size() != expectedParts) { - return fullName; - } - String schemaSegment = parts.get(1); - if (schemaSegment == null || schemaSegment.isEmpty()) { - return fullName; - } - String mapped = - toStorage - ? HierarchicalSchemaUtil.logicalToPhysical(schemaSegment, separator()) - : HierarchicalSchemaUtil.physicalToLogical(schemaSegment, separator()); - if (mapped.equals(schemaSegment)) { - return fullName; - } - List copy = new ArrayList<>(parts); - copy.set(1, mapped); - return DOT_JOINER.join(copy); - } - /** * Converts the schema segment at namespace index 2 (the slot occupied by the schema name in the * standard {@code [metalake, catalog, schema, ...]} layout) from logical to physical form. Callers diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalSchemaNamingBridge.java b/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalSchemaNamingBridge.java index 1bb1ba68426..10e3b38c30d 100644 --- a/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalSchemaNamingBridge.java +++ b/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalSchemaNamingBridge.java @@ -67,158 +67,65 @@ public void tearDown() throws Exception { } // ------------------------------------------------------------------------- - // convertMetadataObjectDottedFullName – each MetadataObject.Type + // securableObjectForStorage / securableObjectForApi // ------------------------------------------------------------------------- @Test - public void schemaToStorage() { - String logical = "catalog.a:b:c"; - String physical = - RelationalSchemaNamingBridge.convertMetadataObjectDottedFullName( - logical, MetadataObject.Type.SCHEMA, true); - assertEquals("catalog.a" + P + "b" + P + "c", physical); + public void securableObjectSchemaToStorage() { + SecurableObject obj = SecurableObjects.ofSchema(CATALOG_OBJ, "a:b:c", USE_SCHEMA_PRIVS); + SecurableObject result = RelationalSchemaNamingBridge.securableObjectForStorage(obj); + assertEquals("catalog.a" + P + "b" + P + "c", result.fullName()); } @Test - public void schemaToApi() { - String physical = "catalog.a" + P + "b" + P + "c"; - String logical = - RelationalSchemaNamingBridge.convertMetadataObjectDottedFullName( - physical, MetadataObject.Type.SCHEMA, false); - assertEquals("catalog.a:b:c", logical); + public void securableObjectSchemaToApi() { + SecurableObject obj = + SecurableObjects.parse("catalog.a" + P + "b" + P + "c", MetadataObject.Type.SCHEMA, USE_SCHEMA_PRIVS); + SecurableObject result = RelationalSchemaNamingBridge.securableObjectForApi(obj); + assertEquals("catalog.a:b:c", result.fullName()); } @Test - public void tableToStorage() { - String logical = "catalog.a:b.mytable"; - String physical = - RelationalSchemaNamingBridge.convertMetadataObjectDottedFullName( - logical, MetadataObject.Type.TABLE, true); - assertEquals("catalog.a" + P + "b.mytable", physical); + public void securableObjectTableToStorage() { + SecurableObject obj = + SecurableObjects.parse("catalog.a:b.mytable", MetadataObject.Type.TABLE, Lists.newArrayList()); + SecurableObject result = RelationalSchemaNamingBridge.securableObjectForStorage(obj); + assertEquals("catalog.a" + P + "b.mytable", result.fullName()); } @Test - public void tableToApi() { - String physical = "catalog.a" + P + "b.mytable"; - String logical = - RelationalSchemaNamingBridge.convertMetadataObjectDottedFullName( - physical, MetadataObject.Type.TABLE, false); - assertEquals("catalog.a:b.mytable", logical); + public void securableObjectTableToApi() { + SecurableObject obj = + SecurableObjects.parse("catalog.a" + P + "b.mytable", MetadataObject.Type.TABLE, Lists.newArrayList()); + SecurableObject result = RelationalSchemaNamingBridge.securableObjectForApi(obj); + assertEquals("catalog.a:b.mytable", result.fullName()); } @Test - public void viewToStorage() { - String logical = "catalog.a:b.myview"; - String physical = - RelationalSchemaNamingBridge.convertMetadataObjectDottedFullName( - logical, MetadataObject.Type.VIEW, true); - assertEquals("catalog.a" + P + "b.myview", physical); + public void securableObjectCatalogPassthrough() { + SecurableObject catalog = SecurableObjects.ofCatalog("catalog", Lists.newArrayList()); + assertSame(catalog, RelationalSchemaNamingBridge.securableObjectForStorage(catalog)); + assertSame(catalog, RelationalSchemaNamingBridge.securableObjectForApi(catalog)); } @Test - public void functionToStorage() { - String logical = "catalog.a:b.myfunc"; - String physical = - RelationalSchemaNamingBridge.convertMetadataObjectDottedFullName( - logical, MetadataObject.Type.FUNCTION, true); - assertEquals("catalog.a" + P + "b.myfunc", physical); + public void securableObjectTableRoundTrip() { + SecurableObject obj = + SecurableObjects.parse("catalog.a:b:c.mytable", MetadataObject.Type.TABLE, Lists.newArrayList()); + SecurableObject stored = RelationalSchemaNamingBridge.securableObjectForStorage(obj); + SecurableObject backToApi = RelationalSchemaNamingBridge.securableObjectForApi(stored); + assertEquals(obj.fullName(), backToApi.fullName()); } @Test - public void columnToStorage() { - String logical = "catalog.a:b.mytable.mycol"; - String physical = - RelationalSchemaNamingBridge.convertMetadataObjectDottedFullName( - logical, MetadataObject.Type.COLUMN, true); - assertEquals("catalog.a" + P + "b.mytable.mycol", physical); - } - - @Test - public void metalakePassthrough() { - String name = "mymetal"; - String result = - RelationalSchemaNamingBridge.convertMetadataObjectDottedFullName( - name, MetadataObject.Type.METALAKE, true); - assertEquals(name, result); - } - - @Test - public void catalogPassthrough() { - String name = "catalog"; - String result = - RelationalSchemaNamingBridge.convertMetadataObjectDottedFullName( - name, MetadataObject.Type.CATALOG, true); - assertEquals(name, result); - } - - // ------------------------------------------------------------------------- - // Part-count mismatch → input returned unchanged - // ------------------------------------------------------------------------- - - @Test - public void schemaPartCountMismatch() { - String name = "catalog.schema.extra"; - String result = - RelationalSchemaNamingBridge.convertMetadataObjectDottedFullName( - name, MetadataObject.Type.SCHEMA, true); - assertEquals(name, result); - } - - @Test - public void tablePartCountMismatch() { - // TABLE expects 3 parts; 2 parts → no conversion - String name = "catalog.schema"; - String result = - RelationalSchemaNamingBridge.convertMetadataObjectDottedFullName( - name, MetadataObject.Type.TABLE, true); - assertEquals(name, result); - } - - // ------------------------------------------------------------------------- - // Round-trip: logical → physical → logical - // ------------------------------------------------------------------------- - - @Test - public void schemaRoundTrip() { - String logical = "catalog.a:b:c"; - String physical = - RelationalSchemaNamingBridge.convertMetadataObjectDottedFullName( - logical, MetadataObject.Type.SCHEMA, true); - String backToLogical = - RelationalSchemaNamingBridge.convertMetadataObjectDottedFullName( - physical, MetadataObject.Type.SCHEMA, false); - assertEquals(logical, backToLogical); - } - - @Test - public void tableRoundTrip() { - String logical = "catalog.a:b.mytable"; - String physical = - RelationalSchemaNamingBridge.convertMetadataObjectDottedFullName( - logical, MetadataObject.Type.TABLE, true); - String backToLogical = - RelationalSchemaNamingBridge.convertMetadataObjectDottedFullName( - physical, MetadataObject.Type.TABLE, false); - assertEquals(logical, backToLogical); - } - - // ------------------------------------------------------------------------- - // Non-default separator - // ------------------------------------------------------------------------- - - @Test - public void customSeparatorSchema() throws Exception { + public void customSeparatorSecurableObject() throws Exception { mockSeparator("/"); - String logical = "catalog.a/b/c"; - String physical = - RelationalSchemaNamingBridge.convertMetadataObjectDottedFullName( - logical, MetadataObject.Type.SCHEMA, true); - assertEquals("catalog.a" + P + "b" + P + "c", physical); - - String backToLogical = - RelationalSchemaNamingBridge.convertMetadataObjectDottedFullName( - physical, MetadataObject.Type.SCHEMA, false); - assertEquals(logical, backToLogical); + SecurableObject obj = + SecurableObjects.parse("catalog.a/b.mytable", MetadataObject.Type.TABLE, Lists.newArrayList()); + SecurableObject stored = RelationalSchemaNamingBridge.securableObjectForStorage(obj); + assertEquals("catalog.a" + P + "b.mytable", stored.fullName()); + SecurableObject backToApi = RelationalSchemaNamingBridge.securableObjectForApi(stored); + assertEquals("catalog.a/b.mytable", backToApi.fullName()); } // ------------------------------------------------------------------------- From 4ff41632d0fb8e66c4af726f165f2fb814d4190e Mon Sep 17 00:00:00 2001 From: roryqi Date: Wed, 13 May 2026 09:16:40 +0000 Subject: [PATCH 07/44] fix(core): guard genericEntityMetadataFullNameForApi against non-MetadataObject entity types Entity types without a MetadataObject.Type equivalent (e.g. TABLE_STATISTIC, MODEL_VERSION) must not be schema-converted. Re-add the MetadataObject.Type.valueOf guard that was dropped in the previous refactor. Co-Authored-By: Claude Sonnet 4.6 --- .../relational/RelationalSchemaNamingBridge.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalSchemaNamingBridge.java b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalSchemaNamingBridge.java index 12f6eaaf50f..bb68431e7d8 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalSchemaNamingBridge.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalSchemaNamingBridge.java @@ -26,6 +26,7 @@ import java.util.stream.Collectors; import org.apache.gravitino.Entity; import org.apache.gravitino.HasIdentifier; +import org.apache.gravitino.MetadataObject; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; import org.apache.gravitino.authorization.SecurableObject; @@ -142,14 +143,24 @@ private static RoleEntity mapRoleSecurableObjects( /** * Normalizes {@link GenericEntity#name()} when it holds a dotted metadata-object path whose * schema segment uses physical hierarchical encoding, for API callers above {@link JDBCBackend}. + * + *

Only processes entity types that correspond to a {@link MetadataObject.Type} (e.g. TABLE, + * SCHEMA). Types without a {@link MetadataObject.Type} equivalent (e.g. TABLE_STATISTIC, + * MODEL_VERSION) are returned unchanged. */ public static GenericEntity genericEntityMetadataFullNameForApi(GenericEntity entity) { String name = entity.name(); if (name == null || name.isEmpty()) { return entity; } + final MetadataObject.Type moType; + try { + moType = MetadataObject.Type.valueOf(entity.type().name()); + } catch (IllegalArgumentException e) { + return entity; + } NameIdentifier ident = NameIdentifier.parse(SENTINEL_METALAKE + "." + name); - NameIdentifier converted = nameIdentifierForApi(ident, entity.type()); + NameIdentifier converted = nameIdentifierForApi(ident, MetadataObjectUtil.toEntityType(moType)); if (converted.equals(ident)) { return entity; } From 3c4f7db1b989d63eb5a95e196949ac3f1bcf04bf Mon Sep 17 00:00:00 2001 From: Rory Date: Wed, 13 May 2026 19:45:24 +0800 Subject: [PATCH 08/44] refactor --- .../gravitino/SupportsRelationOperations.java | 6 - .../HierarchicalSchemaRelationalBackend.java | 265 ++++++++++++++++ .../storage/relational/JDBCBackend.java | 295 ++++-------------- .../LogicalSchemaNamingRelationalBackend.java | 1 + .../relational/RelationalEntityStore.java | 7 +- .../RelationalEntityStoreIdResolver.java | 5 +- .../RelationalSchemaNamingBridge.java | 91 ++++-- .../relational/BackendTestExtension.java | 4 +- .../TestRelationalSchemaNamingBridge.java | 66 +++- 9 files changed, 447 insertions(+), 293 deletions(-) create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/HierarchicalSchemaRelationalBackend.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/LogicalSchemaNamingRelationalBackend.java diff --git a/core/src/main/java/org/apache/gravitino/SupportsRelationOperations.java b/core/src/main/java/org/apache/gravitino/SupportsRelationOperations.java index cdf6c28ce7e..25c4fc60b1d 100644 --- a/core/src/main/java/org/apache/gravitino/SupportsRelationOperations.java +++ b/core/src/main/java/org/apache/gravitino/SupportsRelationOperations.java @@ -20,7 +20,6 @@ import java.io.IOException; import java.util.List; -import java.util.Objects; import org.apache.gravitino.exceptions.NoSuchEntityException; /** @@ -189,11 +188,6 @@ default void batchInsertRelations( Entity.EntityType dstType, boolean override) throws IOException { - Objects.requireNonNull(relType, "relType must not be null"); - Objects.requireNonNull(srcIdentifiers, "srcIdentifiers must not be null"); - Objects.requireNonNull(srcType, "srcType must not be null"); - Objects.requireNonNull(dstIdentifier, "dstIdentifier must not be null"); - Objects.requireNonNull(dstType, "dstType must not be null"); for (NameIdentifier srcIdent : srcIdentifiers) { insertRelation(relType, srcIdent, srcType, dstIdentifier, dstType, override); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/HierarchicalSchemaRelationalBackend.java b/core/src/main/java/org/apache/gravitino/storage/relational/HierarchicalSchemaRelationalBackend.java new file mode 100644 index 00000000000..79af337a132 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/HierarchicalSchemaRelationalBackend.java @@ -0,0 +1,265 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gravitino.storage.relational; + +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.gravitino.Entity; +import org.apache.gravitino.EntityAlreadyExistsException; +import org.apache.gravitino.HasIdentifier; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.Namespace; +import org.apache.gravitino.RelationalEntity; +import org.apache.gravitino.exceptions.NoSuchEntityException; + +/** + * JDBC-backed {@link RelationalBackend} that translates hierarchical schema logical naming (API) to + * the physical encoding expected by relational meta services, delegating storage operations to + * {@link JDBCBackend}. + * + *

{@link JDBCBackend} is treated as storage-oriented: identifiers and entity payloads passed to + * it use physical schema segments and embedded namespaces where applicable. Naming rules live in + * {@link RelationalSchemaNamingBridge}, especially {@link + * RelationalSchemaNamingBridge#entityForApi} (covers relation list elements) and {@link + * RelationalSchemaNamingBridge#nameIdentifierForStorage}, so this class does not branch on {@link + * Type} or {@link Entity.EntityType} for relation calls. + */ +public class HierarchicalSchemaRelationalBackend extends JDBCBackend { + + @Override + public List list( + Namespace namespace, Entity.EntityType entityType, boolean allFields) throws IOException { + Namespace storageParentNs = RelationalSchemaNamingBridge.embeddedNamespaceForStorage(namespace); + List raw = super.list(storageParentNs, entityType, allFields); + return raw.stream() + .map(e -> (E) RelationalSchemaNamingBridge.entityForApi(e, entityType)) + .collect(Collectors.toList()); + } + + @Override + public void insert(E e, boolean overwritten) + throws EntityAlreadyExistsException, IOException { + super.insert(RelationalSchemaNamingBridge.entityForStorage(e), overwritten); + } + + @Override + public E update( + NameIdentifier ident, Entity.EntityType entityType, Function updater) + throws IOException, NoSuchEntityException, EntityAlreadyExistsException { + return (E) + RelationalSchemaNamingBridge.entityForApi( + super.update( + RelationalSchemaNamingBridge.nameIdentifierForStorage(ident, entityType), + entityType, + RelationalSchemaNamingBridge.wrapperUpdater(entityType, updater)), + entityType); + } + + @Override + public E get( + NameIdentifier ident, Entity.EntityType entityType) + throws NoSuchEntityException, IOException { + return (E) + RelationalSchemaNamingBridge.entityForApi( + super.get( + RelationalSchemaNamingBridge.nameIdentifierForStorage(ident, entityType), + entityType), + entityType); + } + + @Override + public List batchGet( + List identifiers, Entity.EntityType entityType) { + List storageIdents = + identifiers.stream() + .map(id -> RelationalSchemaNamingBridge.nameIdentifierForStorage(id, entityType)) + .collect(Collectors.toList()); + List got = super.batchGet(storageIdents, entityType); + return got.stream() + .map(e -> (E) RelationalSchemaNamingBridge.entityForApi(e, entityType)) + .collect(Collectors.toList()); + } + + @Override + public boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolean cascade) + throws IOException { + return super.delete( + RelationalSchemaNamingBridge.nameIdentifierForStorage(ident, entityType), + entityType, + cascade); + } + + @Override + public int batchDelete( + List> entitiesToDelete, boolean cascade) + throws IOException { + List> storagePairs = + entitiesToDelete.stream() + .map( + p -> + Pair.of( + RelationalSchemaNamingBridge.nameIdentifierForStorage( + p.getLeft(), p.getRight()), + p.getRight())) + .collect(Collectors.toList()); + return super.batchDelete(storagePairs, cascade); + } + + @Override + public void batchPut(List entities, boolean overwritten) + throws IOException, EntityAlreadyExistsException { + List storage = + entities.stream() + .map(RelationalSchemaNamingBridge::entityForStorage) + .collect(Collectors.toList()); + super.batchPut(storage, overwritten); + } + + @Override + public List listEntitiesByRelation( + Type relType, NameIdentifier nameIdentifier, Entity.EntityType identType, boolean allFields) + throws IOException { + NameIdentifier source = + RelationalSchemaNamingBridge.nameIdentifierForStorage(nameIdentifier, identType); + List out = super.listEntitiesByRelation(relType, source, identType, allFields); + return out.stream() + .map(e -> (E) RelationalSchemaNamingBridge.entityForApi(e, e.type())) + .collect(Collectors.toList()); + } + + @Override + public List> batchListEntitiesByRelation( + Type relType, List nameIdentifiers, Entity.EntityType identType) + throws IOException { + List storageIdents = + nameIdentifiers.stream() + .map(n -> RelationalSchemaNamingBridge.nameIdentifierForStorage(n, identType)) + .collect(Collectors.toList()); + Map storageToLogical = new HashMap<>(); + for (int i = 0; i < nameIdentifiers.size(); i++) { + storageToLogical.put(storageIdents.get(i), nameIdentifiers.get(i)); + } + List> raw = + super.batchListEntitiesByRelation(relType, storageIdents, identType); + return raw.stream() + .map(re -> remapRelationEntity(re, storageToLogical)) + .collect(Collectors.toList()); + } + + @SuppressWarnings("unchecked") + private static RelationalEntity remapRelationEntity( + RelationalEntity re, Map storageToLogical) { + Entity tgt = re.targetEntity(); + return new RelationalEntity<>( + re.type(), + storageToLogical.get(re.source()), + re.sourceType(), + RelationalSchemaNamingBridge.entityForApi( + (Entity & HasIdentifier) tgt, tgt.type())); + } + + @Override + public void insertRelation( + Type relType, + NameIdentifier srcIdentifier, + Entity.EntityType srcType, + NameIdentifier dstIdentifier, + Entity.EntityType dstType, + boolean override) { + super.insertRelation( + relType, + RelationalSchemaNamingBridge.nameIdentifierForStorage(srcIdentifier, srcType), + srcType, + RelationalSchemaNamingBridge.nameIdentifierForStorage(dstIdentifier, dstType), + dstType, + override); + } + + @Override + public void batchInsertRelations( + Type relType, + List srcIdentifiers, + Entity.EntityType srcType, + NameIdentifier dstIdentifier, + Entity.EntityType dstType, + boolean override) + throws IOException { + List srcStorage = + srcIdentifiers.stream() + .map(i -> RelationalSchemaNamingBridge.nameIdentifierForStorage(i, srcType)) + .collect(Collectors.toList()); + super.batchInsertRelations( + relType, + srcStorage, + srcType, + RelationalSchemaNamingBridge.nameIdentifierForStorage(dstIdentifier, dstType), + dstType, + override); + } + + @Override + public List updateEntityRelations( + Type relType, + NameIdentifier srcEntityIdent, + Entity.EntityType srcEntityType, + NameIdentifier[] destEntitiesToAdd, + NameIdentifier[] destEntitiesToRemove) + throws IOException, NoSuchEntityException, EntityAlreadyExistsException { + List out = + super.updateEntityRelations( + relType, + RelationalSchemaNamingBridge.nameIdentifierForStorage(srcEntityIdent, srcEntityType), + srcEntityType, + Arrays.stream(destEntitiesToAdd) + .map( + id -> + RelationalSchemaNamingBridge.relationDestIdentifiersForStorage(relType, id)) + .toArray(NameIdentifier[]::new), + Arrays.stream(destEntitiesToRemove) + .map( + id -> + RelationalSchemaNamingBridge.relationDestIdentifiersForStorage(relType, id)) + .toArray(NameIdentifier[]::new)); + return out.stream() + .map(e -> (E) RelationalSchemaNamingBridge.entityForApi(e, e.type())) + .collect(Collectors.toList()); + } + + @Override + public E getEntityByRelation( + Type relType, + NameIdentifier srcIdentifier, + Entity.EntityType srcType, + NameIdentifier destEntityIdent) + throws IOException, NoSuchEntityException { + E raw = + super.getEntityByRelation( + relType, + RelationalSchemaNamingBridge.nameIdentifierForStorage(srcIdentifier, srcType), + srcType, + RelationalSchemaNamingBridge.relationDestIdentifiersForStorage( + relType, destEntityIdent)); + return (E) RelationalSchemaNamingBridge.entityForApi(raw, raw.type()); + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java b/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java index ac72e56b04e..65f641ea0d4 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java @@ -26,7 +26,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import java.io.IOException; -import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Objects; @@ -95,13 +94,13 @@ * syntax, please implement the SQL statements and methods in MyBatis Mapper separately and switch * according to the {@link Configs#ENTITY_RELATIONAL_JDBC_BACKEND_URL_KEY} parameter. * - *

Hierarchical schema names use the configured logical separator in {@link NameIdentifier}s and - * {@link Namespace}s seen by callers; this layer translates schema segments to the internal - * physical form ({@link org.apache.gravitino.catalog.HierarchicalSchemaUtil#physicalSeparator()}) - * before delegating to meta services for applicable entity types, and translates back on results - * (except {@code insertSchema}, which still accepts logical {@link SchemaEntity} names for - * nested-schema materialization). Fileset, Topic, Model, and Model version entities are not - * transformed here. + *

This class is storage-oriented: {@link NameIdentifier}s, {@link Namespace}s, and entity + * payloads passed into meta services must already use the physical hierarchical encoding expected + * by the relational layer (ASCII-1 schema segments where nested schemas are stored). API-level + * logical naming is translated at the {@link RelationalEntityStore} boundary by {@link + * HierarchicalSchemaRelationalBackend} (see {@link RelationalSchemaNamingBridge}). Types such as + * Fileset, Topic, Model, and Model version are not subject to embedded-schema identifier + * conversion. */ public class JDBCBackend implements RelationalBackend { @@ -130,26 +129,11 @@ public List list( case CATALOG: return (List) CatalogMetaService.getInstance().listCatalogsByNamespace(namespace); case SCHEMA: - return (List) - SchemaMetaService.getInstance().listSchemasByNamespace(namespace).stream() - .map(RelationalSchemaNamingBridge::schemaEntityForApi) - .collect(Collectors.toList()); + return (List) SchemaMetaService.getInstance().listSchemasByNamespace(namespace); case TABLE: - return (List) - TableMetaService.getInstance() - .listTablesByNamespace( - RelationalSchemaNamingBridge.embeddedNamespaceForStorage(namespace)) - .stream() - .map(RelationalSchemaNamingBridge::tableEntityForApi) - .collect(Collectors.toList()); + return (List) TableMetaService.getInstance().listTablesByNamespace(namespace); case VIEW: - return (List) - ViewMetaService.getInstance() - .listViewsByNamespace( - RelationalSchemaNamingBridge.embeddedNamespaceForStorage(namespace)) - .stream() - .map(RelationalSchemaNamingBridge::viewEntityForApi) - .collect(Collectors.toList()); + return (List) ViewMetaService.getInstance().listViewsByNamespace(namespace); case FILESET: return (List) FilesetMetaService.getInstance().listFilesetsByNamespace(namespace); case TOPIC: @@ -159,10 +143,7 @@ public List list( case USER: return (List) UserMetaService.getInstance().listUsersByNamespace(namespace, allFields); case ROLE: - return (List) - RoleMetaService.getInstance().listRolesByNamespace(namespace).stream() - .map(RelationalSchemaNamingBridge::roleEntityForApi) - .collect(Collectors.toList()); + return (List) RoleMetaService.getInstance().listRolesByNamespace(namespace); case GROUP: return (List) GroupMetaService.getInstance().listGroupsByNamespace(namespace, allFields); case MODEL: @@ -171,13 +152,7 @@ public List list( return (List) ModelVersionMetaService.getInstance().listModelVersionsByNamespace(namespace); case FUNCTION: - return (List) - FunctionMetaService.getInstance() - .listFunctionsByNamespace( - RelationalSchemaNamingBridge.embeddedNamespaceForStorage(namespace)) - .stream() - .map(RelationalSchemaNamingBridge::functionEntityForApi) - .collect(Collectors.toList()); + return (List) FunctionMetaService.getInstance().listFunctionsByNamespace(namespace); case POLICY: return (List) PolicyMetaService.getInstance().listPoliciesByNamespace(namespace); case JOB_TEMPLATE: @@ -186,14 +161,11 @@ public List list( case JOB: return (List) JobMetaService.getInstance().listJobsByNamespace(namespace); case TABLE_STATISTIC: - Namespace statisticNs = RelationalSchemaNamingBridge.embeddedNamespaceForStorage(namespace); + Namespace statisticNs = namespace; return (List) StatisticMetaService.getInstance() .listStatisticsByEntity( - NameIdentifier.parse(statisticNs.toString()), Entity.EntityType.TABLE) - .stream() - .map(RelationalSchemaNamingBridge::statisticEntityForApi) - .collect(Collectors.toList()); + NameIdentifier.parse(statisticNs.toString()), Entity.EntityType.TABLE); default: throw new UnsupportedEntityTypeException( "Unsupported entity type: %s for list operation", entityType); @@ -220,9 +192,7 @@ public void insert(E e, boolean overwritten) } else if (e instanceof SchemaEntity) { SchemaMetaService.getInstance().insertSchema((SchemaEntity) e, overwritten); } else if (e instanceof TableEntity) { - TableMetaService.getInstance() - .insertTable( - RelationalSchemaNamingBridge.tableEntityForStorage((TableEntity) e), overwritten); + TableMetaService.getInstance().insertTable((TableEntity) e, overwritten); } else if (e instanceof FilesetEntity) { FilesetMetaService.getInstance().insertFileset((FilesetEntity) e, overwritten); } else if (e instanceof TopicEntity) { @@ -230,9 +200,7 @@ public void insert(E e, boolean overwritten) } else if (e instanceof UserEntity) { UserMetaService.getInstance().insertUser((UserEntity) e, overwritten); } else if (e instanceof RoleEntity) { - RoleMetaService.getInstance() - .insertRole( - RelationalSchemaNamingBridge.roleEntityForStorage((RoleEntity) e), overwritten); + RoleMetaService.getInstance().insertRole((RoleEntity) e, overwritten); } else if (e instanceof GroupEntity) { GroupMetaService.getInstance().insertGroup((GroupEntity) e, overwritten); } else if (e instanceof TagEntity) { @@ -247,10 +215,7 @@ public void insert(E e, boolean overwritten) } ModelVersionMetaService.getInstance().insertModelVersion((ModelVersionEntity) e); } else if (e instanceof FunctionEntity) { - FunctionMetaService.getInstance() - .insertFunction( - RelationalSchemaNamingBridge.functionEntityForStorage((FunctionEntity) e), - overwritten); + FunctionMetaService.getInstance().insertFunction((FunctionEntity) e, overwritten); } else if (e instanceof PolicyEntity) { PolicyMetaService.getInstance().insertPolicy((PolicyEntity) e, overwritten); } else if (e instanceof JobTemplateEntity) { @@ -258,9 +223,7 @@ public void insert(E e, boolean overwritten) } else if (e instanceof JobEntity) { JobMetaService.getInstance().insertJob((JobEntity) e, overwritten); } else if (e instanceof ViewEntity) { - ViewMetaService.getInstance() - .insertView( - RelationalSchemaNamingBridge.viewEntityForStorage((ViewEntity) e), overwritten); + ViewMetaService.getInstance().insertView((ViewEntity) e, overwritten); } else if (e instanceof GenericEntity) { GenericEntity genericEntity = (GenericEntity) e; throw new UnsupportedEntityTypeException( @@ -281,19 +244,9 @@ public E update( case CATALOG: return (E) CatalogMetaService.getInstance().updateCatalog(ident, updater); case SCHEMA: - return (E) - SchemaMetaService.getInstance() - .updateSchema( - RelationalSchemaNamingBridge.schemaIdentifierForStorage(ident), updater); + return (E) SchemaMetaService.getInstance().updateSchema(ident, updater); case TABLE: - return (E) - RelationalSchemaNamingBridge.tableEntityForApi( - TableMetaService.getInstance() - .updateTable( - RelationalSchemaNamingBridge.nameIdentifierForStorage( - ident, Entity.EntityType.TABLE), - RelationalSchemaNamingBridge.wrapperUpdater( - Entity.EntityType.TABLE, updater))); + return (E) TableMetaService.getInstance().updateTable(ident, updater); case FILESET: return (E) FilesetMetaService.getInstance().updateFileset(ident, updater); case TOPIC: @@ -311,27 +264,13 @@ public E update( case MODEL_VERSION: return (E) ModelVersionMetaService.getInstance().updateModelVersion(ident, updater); case FUNCTION: - return (E) - RelationalSchemaNamingBridge.functionEntityForApi( - FunctionMetaService.getInstance() - .updateFunction( - RelationalSchemaNamingBridge.nameIdentifierForStorage( - ident, Entity.EntityType.FUNCTION), - RelationalSchemaNamingBridge.wrapperUpdater( - Entity.EntityType.FUNCTION, updater))); + return (E) FunctionMetaService.getInstance().updateFunction(ident, updater); case POLICY: return (E) PolicyMetaService.getInstance().updatePolicy(ident, updater); case JOB_TEMPLATE: return (E) JobTemplateMetaService.getInstance().updateJobTemplate(ident, updater); case VIEW: - return (E) - RelationalSchemaNamingBridge.viewEntityForApi( - ViewMetaService.getInstance() - .updateView( - RelationalSchemaNamingBridge.nameIdentifierForStorage( - ident, Entity.EntityType.VIEW), - RelationalSchemaNamingBridge.wrapperUpdater( - Entity.EntityType.VIEW, updater))); + return (E) ViewMetaService.getInstance().updateView(ident, updater); default: throw new UnsupportedEntityTypeException( "Unsupported entity type: %s for update operation", entityType); @@ -348,18 +287,9 @@ public E get( case CATALOG: return (E) CatalogMetaService.getInstance().getCatalogByIdentifier(ident); case SCHEMA: - return (E) - RelationalSchemaNamingBridge.schemaEntityForApi( - SchemaMetaService.getInstance() - .getSchemaByIdentifier( - RelationalSchemaNamingBridge.schemaIdentifierForStorage(ident))); + return (E) SchemaMetaService.getInstance().getSchemaByIdentifier(ident); case TABLE: - return (E) - RelationalSchemaNamingBridge.tableEntityForApi( - TableMetaService.getInstance() - .getTableByIdentifier( - RelationalSchemaNamingBridge.nameIdentifierForStorage( - ident, Entity.EntityType.TABLE))); + return (E) TableMetaService.getInstance().getTableByIdentifier(ident); case FILESET: return (E) FilesetMetaService.getInstance().getFilesetByIdentifier(ident); case TOPIC: @@ -369,9 +299,7 @@ public E get( case GROUP: return (E) GroupMetaService.getInstance().getGroupByIdentifier(ident); case ROLE: - return (E) - RelationalSchemaNamingBridge.roleEntityForApi( - RoleMetaService.getInstance().getRoleByIdentifier(ident)); + return (E) RoleMetaService.getInstance().getRoleByIdentifier(ident); case TAG: return (E) TagMetaService.getInstance().getTagByIdentifier(ident); case MODEL: @@ -379,12 +307,7 @@ public E get( case MODEL_VERSION: return (E) ModelVersionMetaService.getInstance().getModelVersionByIdentifier(ident); case FUNCTION: - return (E) - RelationalSchemaNamingBridge.functionEntityForApi( - FunctionMetaService.getInstance() - .getFunctionByIdentifier( - RelationalSchemaNamingBridge.nameIdentifierForStorage( - ident, Entity.EntityType.FUNCTION))); + return (E) FunctionMetaService.getInstance().getFunctionByIdentifier(ident); case POLICY: return (E) PolicyMetaService.getInstance().getPolicyByIdentifier(ident); case JOB_TEMPLATE: @@ -392,12 +315,7 @@ public E get( case JOB: return (E) JobMetaService.getInstance().getJobByIdentifier(ident); case VIEW: - return (E) - RelationalSchemaNamingBridge.viewEntityForApi( - ViewMetaService.getInstance() - .getViewByIdentifier( - RelationalSchemaNamingBridge.nameIdentifierForStorage( - ident, Entity.EntityType.VIEW))); + return (E) ViewMetaService.getInstance().getViewByIdentifier(ident); default: throw new UnsupportedEntityTypeException( "Unsupported entity type: %s for get operation", entityType); @@ -424,25 +342,14 @@ public List batchGet( case CATALOG: return (List) CatalogMetaService.getInstance().batchGetCatalogByIdentifier(identifiers); case SCHEMA: - List schemaStorageIdents = - identifiers.stream() - .map(RelationalSchemaNamingBridge::schemaIdentifierForStorage) - .collect(Collectors.toList()); - return (List) - SchemaMetaService.getInstance().batchGetSchemaByIdentifier(schemaStorageIdents).stream() - .map(RelationalSchemaNamingBridge::schemaEntityForApi) - .collect(Collectors.toList()); + return (List) SchemaMetaService.getInstance().batchGetSchemaByIdentifier(identifiers); case TABLE: - Namespace tableStorageNs = - RelationalSchemaNamingBridge.embeddedNamespaceForStorage(firstNamespace); List tableStorageIdents = identifiers.stream() - .map(id -> NameIdentifier.of(tableStorageNs, id.name())) + .map(id -> NameIdentifier.of(firstNamespace, id.name())) .collect(Collectors.toList()); return (List) - TableMetaService.getInstance().batchGetTableByIdentifier(tableStorageIdents).stream() - .map(RelationalSchemaNamingBridge::tableEntityForApi) - .collect(Collectors.toList()); + TableMetaService.getInstance().batchGetTableByIdentifier(tableStorageIdents); case FILESET: return (List) FilesetMetaService.getInstance().batchGetFilesetByIdentifier(identifiers); case TOPIC: @@ -476,10 +383,7 @@ public List batchGet( List roles = Lists.newArrayList(); for (NameIdentifier identifier : identifiers) { try { - roles.add( - (E) - RelationalSchemaNamingBridge.roleEntityForApi( - RoleMetaService.getInstance().getRoleByIdentifier(identifier))); + roles.add((E) RoleMetaService.getInstance().getRoleByIdentifier(identifier)); } catch (NoSuchEntityException e) { LOG.debug("Skipping missing role during batch get: {}", identifier.name()); } @@ -489,13 +393,7 @@ public List batchGet( List views = Lists.newArrayList(); for (NameIdentifier identifier : identifiers) { try { - views.add( - (E) - RelationalSchemaNamingBridge.viewEntityForApi( - ViewMetaService.getInstance() - .getViewByIdentifier( - RelationalSchemaNamingBridge.nameIdentifierForStorage( - identifier, Entity.EntityType.VIEW)))); + views.add((E) ViewMetaService.getInstance().getViewByIdentifier(identifier)); } catch (NoSuchEntityException e) { LOG.debug("Skipping missing view during batch get: {}", identifier.name()); } @@ -516,13 +414,9 @@ public boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolea case CATALOG: return CatalogMetaService.getInstance().deleteCatalog(ident, cascade); case SCHEMA: - return SchemaMetaService.getInstance() - .deleteSchema(RelationalSchemaNamingBridge.schemaIdentifierForStorage(ident), cascade); + return SchemaMetaService.getInstance().deleteSchema(ident, cascade); case TABLE: - return TableMetaService.getInstance() - .deleteTable( - RelationalSchemaNamingBridge.nameIdentifierForStorage( - ident, Entity.EntityType.TABLE)); + return TableMetaService.getInstance().deleteTable(ident); case FILESET: return FilesetMetaService.getInstance().deleteFileset(ident); case TOPIC: @@ -540,10 +434,7 @@ public boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolea case MODEL_VERSION: return ModelVersionMetaService.getInstance().deleteModelVersion(ident); case FUNCTION: - return FunctionMetaService.getInstance() - .deleteFunction( - RelationalSchemaNamingBridge.nameIdentifierForStorage( - ident, Entity.EntityType.FUNCTION)); + return FunctionMetaService.getInstance().deleteFunction(ident); case POLICY: return PolicyMetaService.getInstance().deletePolicy(ident); case JOB_TEMPLATE: @@ -551,10 +442,7 @@ public boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolea case JOB: return JobMetaService.getInstance().deleteJob(ident); case VIEW: - return ViewMetaService.getInstance() - .deleteView( - RelationalSchemaNamingBridge.nameIdentifierForStorage( - ident, Entity.EntityType.VIEW)); + return ViewMetaService.getInstance().deleteView(ident); default: throw new UnsupportedEntityTypeException( "Unsupported entity type: %s for delete operation", entityType); @@ -729,9 +617,7 @@ public int batchDelete( 1 == namespaceSize, "All entities must be in the same namespace for batch delete operation."); - Namespace namespace = - RelationalSchemaNamingBridge.embeddedNamespaceForStorage( - deleteIdents.get(0).namespace()); + Namespace namespace = deleteIdents.get(0).namespace(); return StatisticMetaService.getInstance() .batchDeleteStatisticPOs( NameIdentifier.parse(namespace.toString()), @@ -764,13 +650,8 @@ public void batchPut(List entities, boolea 1 == entities.stream().collect(Collectors.groupingBy(HasIdentifier::namespace)).size(), "All entities must be in the same namespace for batchPut operation."); - List storageStatisticEntities = - statisticEntities.stream() - .map(RelationalSchemaNamingBridge::statisticEntityForStorage) - .collect(Collectors.toList()); - Namespace statisticNamespace = - RelationalSchemaNamingBridge.embeddedNamespaceForStorage( - statisticEntities.get(0).namespace()); + List storageStatisticEntities = statisticEntities; + Namespace statisticNamespace = statisticEntities.get(0).namespace(); StatisticMetaService.getInstance() .batchInsertStatisticPOsOnDuplicateKeyUpdate( storageStatisticEntities, @@ -791,22 +672,13 @@ public List listEntitiesByRelation( case OWNER_REL: List list = Lists.newArrayList(); OwnerMetaService.getInstance() - .getOwner( - RelationalSchemaNamingBridge.nameIdentifierForStorage(nameIdentifier, identType), - identType) + .getOwner(nameIdentifier, identType) .ifPresent(e -> list.add((E) e)); return list; case METADATA_OBJECT_ROLE_REL: return (List) RoleMetaService.getInstance() - .listRolesByMetadataObject( - RelationalSchemaNamingBridge.nameIdentifierForStorage( - nameIdentifier, identType), - identType, - allFields) - .stream() - .map(RelationalSchemaNamingBridge::roleEntityForApi) - .collect(Collectors.toList()); + .listRolesByMetadataObject(nameIdentifier, identType, allFields); case ROLE_GROUP_REL: if (identType == Entity.EntityType.ROLE) { return (List) GroupMetaService.getInstance().listGroupsByRoleIdent(nameIdentifier); @@ -818,10 +690,7 @@ public List listEntitiesByRelation( if (identType == Entity.EntityType.ROLE) { return (List) UserMetaService.getInstance().listUsersByRoleIdent(nameIdentifier); } else if (identType == Entity.EntityType.USER) { - return (List) - RoleMetaService.getInstance().listRolesByUserIdent(nameIdentifier).stream() - .map(RelationalSchemaNamingBridge::roleEntityForApi) - .collect(Collectors.toList()); + return (List) RoleMetaService.getInstance().listRolesByUserIdent(nameIdentifier); } else { throw new IllegalArgumentException( String.format("ROLE_USER_REL doesn't support type %s", identType.name())); @@ -830,35 +699,20 @@ public List listEntitiesByRelation( case POLICY_METADATA_OBJECT_REL: if (identType == Entity.EntityType.POLICY) { return (List) - PolicyMetaService.getInstance() - .listAssociatedEntitiesForPolicy(nameIdentifier) - .stream() - .map(RelationalSchemaNamingBridge::genericEntityMetadataFullNameForApi) - .collect(Collectors.toList()); + PolicyMetaService.getInstance().listAssociatedEntitiesForPolicy(nameIdentifier); } else { return (List) PolicyMetaService.getInstance() - .listPoliciesForMetadataObject( - RelationalSchemaNamingBridge.nameIdentifierForStorage( - nameIdentifier, identType), - identType); + .listPoliciesForMetadataObject(nameIdentifier, identType); } case TAG_METADATA_OBJECT_REL: if (identType == Entity.EntityType.TAG) { return (List) - TagMetaService.getInstance() - .listAssociatedMetadataObjectsForTag(nameIdentifier) - .stream() - .map(RelationalSchemaNamingBridge::genericEntityMetadataFullNameForApi) - .collect(Collectors.toList()); + TagMetaService.getInstance().listAssociatedMetadataObjectsForTag(nameIdentifier); } else { return (List) - TagMetaService.getInstance() - .listTagsForMetadataObject( - RelationalSchemaNamingBridge.nameIdentifierForStorage( - nameIdentifier, identType), - identType); + TagMetaService.getInstance().listTagsForMetadataObject(nameIdentifier, identType); } default: throw new IllegalArgumentException( @@ -872,11 +726,7 @@ public List> batchListEntitiesByRelation( throws IOException { switch (relType) { case OWNER_REL: - List ownerRelStorageIdents = - nameIdentifiers.stream() - .map(id -> RelationalSchemaNamingBridge.nameIdentifierForStorage(id, identType)) - .collect(Collectors.toList()); - return OwnerMetaService.getInstance().batchGetOwner(ownerRelStorageIdents, identType); + return OwnerMetaService.getInstance().batchGetOwner(nameIdentifiers, identType); default: throw new IllegalArgumentException( String.format("Doesn't support the relation type %s", relType)); @@ -893,12 +743,7 @@ public void insertRelation( boolean override) { switch (relType) { case OWNER_REL: - OwnerMetaService.getInstance() - .setOwner( - RelationalSchemaNamingBridge.nameIdentifierForStorage(srcIdentifier, srcType), - srcType, - RelationalSchemaNamingBridge.nameIdentifierForStorage(dstIdentifier, dstType), - dstType); + OwnerMetaService.getInstance().setOwner(srcIdentifier, srcType, dstIdentifier, dstType); break; default: throw new IllegalArgumentException( @@ -924,16 +769,8 @@ public void batchInsertRelations( Objects.requireNonNull(dstType, "dstType must not be null"); switch (relType) { case OWNER_REL: - List srcStorage = - srcIdentifiers.stream() - .map(id -> RelationalSchemaNamingBridge.nameIdentifierForStorage(id, srcType)) - .collect(Collectors.toList()); OwnerMetaService.getInstance() - .batchSetOwners( - srcStorage, - srcType, - RelationalSchemaNamingBridge.nameIdentifierForStorage(dstIdentifier, dstType), - dstType); + .batchSetOwners(srcIdentifiers, srcType, dstIdentifier, dstType); break; default: throw new IllegalArgumentException( @@ -954,20 +791,12 @@ public List updateEntityRelations( return (List) PolicyMetaService.getInstance() .associatePoliciesWithMetadataObject( - RelationalSchemaNamingBridge.nameIdentifierForStorage( - srcEntityIdent, srcEntityType), - srcEntityType, - storageIdents(destEntitiesToAdd, Entity.EntityType.POLICY), - storageIdents(destEntitiesToRemove, Entity.EntityType.POLICY)); + srcEntityIdent, srcEntityType, destEntitiesToAdd, destEntitiesToRemove); case TAG_METADATA_OBJECT_REL: return (List) TagMetaService.getInstance() .associateTagsWithMetadataObject( - RelationalSchemaNamingBridge.nameIdentifierForStorage( - srcEntityIdent, srcEntityType), - srcEntityType, - storageIdents(destEntitiesToAdd, Entity.EntityType.TAG), - storageIdents(destEntitiesToRemove, Entity.EntityType.TAG)); + srcEntityIdent, srcEntityType, destEntitiesToAdd, destEntitiesToRemove); default: throw new IllegalArgumentException( String.format("Doesn't support the relation type %s", relType)); @@ -985,35 +814,17 @@ public E getEntityByRelation( case POLICY_METADATA_OBJECT_REL: return (E) PolicyMetaService.getInstance() - .getPolicyForMetadataObject( - RelationalSchemaNamingBridge.nameIdentifierForStorage(srcIdentifier, srcType), - srcType, - RelationalSchemaNamingBridge.nameIdentifierForStorage( - destEntityIdent, Entity.EntityType.POLICY)); + .getPolicyForMetadataObject(srcIdentifier, srcType, destEntityIdent); case TAG_METADATA_OBJECT_REL: return (E) TagMetaService.getInstance() - .getTagForMetadataObject( - RelationalSchemaNamingBridge.nameIdentifierForStorage(srcIdentifier, srcType), - srcType, - RelationalSchemaNamingBridge.nameIdentifierForStorage( - destEntityIdent, Entity.EntityType.TAG)); + .getTagForMetadataObject(srcIdentifier, srcType, destEntityIdent); default: throw new IllegalArgumentException( String.format("Doesn't support the relation type %s", relType)); } } - private static NameIdentifier[] storageIdents( - NameIdentifier[] idents, Entity.EntityType entityType) { - if (idents == null) { - return null; - } - return Arrays.stream(idents) - .map(id -> RelationalSchemaNamingBridge.nameIdentifierForStorage(id, entityType)) - .toArray(NameIdentifier[]::new); - } - public enum JDBCBackendType { H2(true), MYSQL(false), diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/LogicalSchemaNamingRelationalBackend.java b/core/src/main/java/org/apache/gravitino/storage/relational/LogicalSchemaNamingRelationalBackend.java new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/LogicalSchemaNamingRelationalBackend.java @@ -0,0 +1 @@ + diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStore.java b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStore.java index d531d5cc011..367822b9b32 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStore.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStore.java @@ -55,13 +55,16 @@ /** * Relation store to store entities. This means we can store entities in a relational store. I.e., * MySQL, PostgreSQL, etc. If you want to use a different backend, you can implement the {@link - * RelationalBackend} interface + * RelationalBackend} interface. The default JDBC backend is created as {@link + * HierarchicalSchemaRelationalBackend} so hierarchical schema naming can evolve at the store + * boundary without rewriting {@link RelationalEntityStore} callers. */ public class RelationalEntityStore implements EntityStore, SupportsRelationOperations { private static final Logger LOGGER = LoggerFactory.getLogger(RelationalEntityStore.class); public static final ImmutableMap RELATIONAL_BACKENDS = ImmutableMap.of( - Configs.DEFAULT_ENTITY_RELATIONAL_STORE, JDBCBackend.class.getCanonicalName()); + Configs.DEFAULT_ENTITY_RELATIONAL_STORE, + HierarchicalSchemaRelationalBackend.class.getCanonicalName()); private RelationalBackend backend; private RelationalGarbageCollector garbageCollector; private EntityCache cache; diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStoreIdResolver.java b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStoreIdResolver.java index 895572719a0..75ba9b7a172 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStoreIdResolver.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStoreIdResolver.java @@ -169,12 +169,15 @@ private NamespacedEntityId getEntityIdsRequiringMetalakeId( private NamespacedEntityId getEntityIdsRequiringSchemaIds( NameIdentifier nameIdentifier, Entity.EntityType type) { + NameIdentifier schemaIdent = NameIdentifierUtil.getSchemaIdentifier(nameIdentifier); + String storageSchemaName = + RelationalSchemaNamingBridge.schemaIdentifierForStorage(schemaIdent).name(); SchemaIds schemaIds = SchemaMetaService.getInstance() .getSchemaIdByMetalakeNameAndCatalogNameAndSchemaName( NameIdentifierUtil.getMetalake(nameIdentifier), NameIdentifierUtil.getCatalogIdentifier(nameIdentifier).name(), - NameIdentifierUtil.getSchemaIdentifier(nameIdentifier).name()); + storageSchemaName); switch (type) { case SCHEMA: diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalSchemaNamingBridge.java b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalSchemaNamingBridge.java index bb68431e7d8..91e749a5be8 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalSchemaNamingBridge.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalSchemaNamingBridge.java @@ -29,6 +29,7 @@ import org.apache.gravitino.MetadataObject; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; +import org.apache.gravitino.SupportsRelationOperations.Type; import org.apache.gravitino.authorization.SecurableObject; import org.apache.gravitino.authorization.SecurableObjects; import org.apache.gravitino.meta.AuditInfo; @@ -125,8 +126,7 @@ private static RoleEntity mapRoleSecurableObjects( if (objects == null || objects.isEmpty()) { return entity; } - List mapped = - objects.stream().map(mapper).collect(Collectors.toList()); + List mapped = objects.stream().map(mapper).collect(Collectors.toList()); if (mapped.equals(objects)) { return entity; } @@ -149,34 +149,20 @@ private static RoleEntity mapRoleSecurableObjects( * MODEL_VERSION) are returned unchanged. */ public static GenericEntity genericEntityMetadataFullNameForApi(GenericEntity entity) { - String name = entity.name(); - if (name == null || name.isEmpty()) { - return entity; - } - final MetadataObject.Type moType; - try { - moType = MetadataObject.Type.valueOf(entity.type().name()); - } catch (IllegalArgumentException e) { - return entity; - } - NameIdentifier ident = NameIdentifier.parse(SENTINEL_METALAKE + "." + name); - NameIdentifier converted = nameIdentifierForApi(ident, MetadataObjectUtil.toEntityType(moType)); - if (converted.equals(ident)) { - return entity; - } + NameIdentifier converted = nameIdentifierForStorage(entity.nameIdentifier(), entity.type()); return GenericEntity.builder() .withId(entity.id()) .withEntityType(entity.type()) - .withName(identToMetadataObjectFullName(converted)) - .withNamespace(entity.namespace()) + .withName(converted.name()) + .withNamespace(converted.namespace()) .build(); } /** * Converts the schema segment at namespace index 2 (the slot occupied by the schema name in the - * standard {@code [metalake, catalog, schema, ...]} layout) from logical to physical form. Callers - * must only pass namespaces that follow this layout; if the schema sits at a different index the - * conversion will silently apply to the wrong segment. + * standard {@code [metalake, catalog, schema, ...]} layout) from logical to physical form. + * Callers must only pass namespaces that follow this layout; if the schema sits at a different + * index the conversion will silently apply to the wrong segment. */ public static Namespace embeddedNamespaceForStorage(Namespace ns) { if (ns.length() < 3) { @@ -270,6 +256,25 @@ public static SchemaEntity schemaEntityForApi(SchemaEntity entity) { .build(); } + /** + * Converts a {@link SchemaEntity}'s {@link SchemaEntity#name()} from logical (API) form to the + * physical encoding used by relational meta services. Inverse of {@link #schemaEntityForApi}. + */ + public static SchemaEntity schemaEntityForStorage(SchemaEntity entity) { + String physicalName = HierarchicalSchemaUtil.logicalToPhysical(entity.name(), separator()); + if (physicalName.equals(entity.name())) { + return entity; + } + return SchemaEntity.builder() + .withId(entity.id()) + .withName(physicalName) + .withNamespace(entity.namespace()) + .withComment(entity.comment()) + .withProperties(entity.properties()) + .withAuditInfo(entity.auditInfo()) + .build(); + } + public static TableEntity tableEntityForApi(TableEntity e) { Namespace apiNs = embeddedNamespaceForApi(e.namespace()); if (apiNs.equals(e.namespace())) { @@ -427,9 +432,6 @@ private static StatisticEntity statisticEntityWithNamespace(StatisticEntity e, N @SuppressWarnings("unchecked") // entityForApi/entityForStorage dispatch on known concrete types public static Function wrapperUpdater( Entity.EntityType entityType, Function updater) { - Preconditions.checkArgument( - entityType != Entity.EntityType.SCHEMA, - "Schema entity updates must go through SchemaMetaService directly, not wrapperUpdater"); return e -> { E logicalOld = entityForApi(e, entityType); E logicalNew = updater.apply(logicalOld); @@ -437,9 +439,23 @@ public static Function wrapperUpdater( }; } - @SuppressWarnings("unchecked") // each case casts to the statically known subtype for that EntityType + /** + * Converts storage-shaped entities to API naming. {@link RoleEntity} and {@link GenericEntity} + * (policy/tag relation list payloads) are normalized by concrete type before the {@code + * type}-based branch so callers may pass each element's {@link Entity.EntityType} (e.g. roles + * listed for a {@link Entity.EntityType#TABLE} metadata object still use {@code TABLE} while + * elements are roles). + */ + @SuppressWarnings( + "unchecked") // each case casts to the statically known subtype for that EntityType public static E entityForApi( E entity, Entity.EntityType type) { + if (entity instanceof RoleEntity) { + return (E) roleEntityForApi((RoleEntity) entity); + } + if (entity instanceof GenericEntity) { + return (E) genericEntityMetadataFullNameForApi((GenericEntity) entity); + } switch (type) { case SCHEMA: return (E) schemaEntityForApi((SchemaEntity) entity); @@ -459,7 +475,7 @@ public static E entityForApi( @SuppressWarnings("unchecked") // instanceof guards ensure each cast is safe public static E entityForStorage(E entity) { if (entity instanceof SchemaEntity) { - return entity; + return (E) schemaEntityForStorage((SchemaEntity) entity); } if (entity instanceof TableEntity) { return (E) tableEntityForStorage((TableEntity) entity); @@ -473,6 +489,27 @@ public static E entityForStorage(E entity) { if (entity instanceof StatisticEntity) { return (E) statisticEntityForStorage((StatisticEntity) entity); } + if (entity instanceof RoleEntity) { + return (E) roleEntityForStorage((RoleEntity) entity); + } return entity; } + + /** + * JDBC policy/tag relation APIs: a destination {@link NameIdentifier} is normalized using the + * element type implied by {@code relType} when applicable (e.g. policy or tag name for + * association rows). + */ + public static NameIdentifier relationDestIdentifiersForStorage( + Type relType, NameIdentifier ident) { + Entity.EntityType elementType; + if (relType == Type.POLICY_METADATA_OBJECT_REL) { + elementType = Entity.EntityType.POLICY; + } else if (relType == Type.TAG_METADATA_OBJECT_REL) { + elementType = Entity.EntityType.TAG; + } else { + return ident; + } + return nameIdentifierForStorage(ident, elementType); + } } diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/BackendTestExtension.java b/core/src/test/java/org/apache/gravitino/storage/relational/BackendTestExtension.java index 0920052ff70..2111826bfac 100644 --- a/core/src/test/java/org/apache/gravitino/storage/relational/BackendTestExtension.java +++ b/core/src/test/java/org/apache/gravitino/storage/relational/BackendTestExtension.java @@ -183,7 +183,7 @@ private RelationalBackend startBackend(String type) throws Exception { FieldUtils.writeField( GravitinoEnv.getInstance(), "idGenerator", RandomIdGenerator.INSTANCE, true); - RelationalBackend backend = new JDBCBackend(); + RelationalBackend backend = new HierarchicalSchemaRelationalBackend(); if ("mysql".equals(type)) { String url = baseIT.startAndInitMySQLBackend(); Mockito.when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_URL)).thenReturn(url); @@ -225,7 +225,7 @@ private RelationalBackend startBackend(String type) throws Exception { } // A simple Wrapper solves the H2 file cleanup issue. - public static class H2BackendWrapper extends JDBCBackend { + public static class H2BackendWrapper extends HierarchicalSchemaRelationalBackend { private final String path; public H2BackendWrapper(Config config, String path) { diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalSchemaNamingBridge.java b/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalSchemaNamingBridge.java index 10e3b38c30d..f801f96a26c 100644 --- a/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalSchemaNamingBridge.java +++ b/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalSchemaNamingBridge.java @@ -20,7 +20,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertSame; -import static org.junit.jupiter.api.Assertions.assertThrows; import com.google.common.collect.Lists; import java.time.Instant; @@ -40,6 +39,7 @@ import org.apache.gravitino.meta.AuditInfo; import org.apache.gravitino.meta.GenericEntity; import org.apache.gravitino.meta.RoleEntity; +import org.apache.gravitino.meta.SchemaEntity; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -80,7 +80,8 @@ public void securableObjectSchemaToStorage() { @Test public void securableObjectSchemaToApi() { SecurableObject obj = - SecurableObjects.parse("catalog.a" + P + "b" + P + "c", MetadataObject.Type.SCHEMA, USE_SCHEMA_PRIVS); + SecurableObjects.parse( + "catalog.a" + P + "b" + P + "c", MetadataObject.Type.SCHEMA, USE_SCHEMA_PRIVS); SecurableObject result = RelationalSchemaNamingBridge.securableObjectForApi(obj); assertEquals("catalog.a:b:c", result.fullName()); } @@ -88,7 +89,8 @@ public void securableObjectSchemaToApi() { @Test public void securableObjectTableToStorage() { SecurableObject obj = - SecurableObjects.parse("catalog.a:b.mytable", MetadataObject.Type.TABLE, Lists.newArrayList()); + SecurableObjects.parse( + "catalog.a:b.mytable", MetadataObject.Type.TABLE, Lists.newArrayList()); SecurableObject result = RelationalSchemaNamingBridge.securableObjectForStorage(obj); assertEquals("catalog.a" + P + "b.mytable", result.fullName()); } @@ -96,7 +98,8 @@ public void securableObjectTableToStorage() { @Test public void securableObjectTableToApi() { SecurableObject obj = - SecurableObjects.parse("catalog.a" + P + "b.mytable", MetadataObject.Type.TABLE, Lists.newArrayList()); + SecurableObjects.parse( + "catalog.a" + P + "b.mytable", MetadataObject.Type.TABLE, Lists.newArrayList()); SecurableObject result = RelationalSchemaNamingBridge.securableObjectForApi(obj); assertEquals("catalog.a:b.mytable", result.fullName()); } @@ -111,7 +114,8 @@ public void securableObjectCatalogPassthrough() { @Test public void securableObjectTableRoundTrip() { SecurableObject obj = - SecurableObjects.parse("catalog.a:b:c.mytable", MetadataObject.Type.TABLE, Lists.newArrayList()); + SecurableObjects.parse( + "catalog.a:b:c.mytable", MetadataObject.Type.TABLE, Lists.newArrayList()); SecurableObject stored = RelationalSchemaNamingBridge.securableObjectForStorage(obj); SecurableObject backToApi = RelationalSchemaNamingBridge.securableObjectForApi(stored); assertEquals(obj.fullName(), backToApi.fullName()); @@ -121,7 +125,8 @@ public void securableObjectTableRoundTrip() { public void customSeparatorSecurableObject() throws Exception { mockSeparator("/"); SecurableObject obj = - SecurableObjects.parse("catalog.a/b.mytable", MetadataObject.Type.TABLE, Lists.newArrayList()); + SecurableObjects.parse( + "catalog.a/b.mytable", MetadataObject.Type.TABLE, Lists.newArrayList()); SecurableObject stored = RelationalSchemaNamingBridge.securableObjectForStorage(obj); assertEquals("catalog.a" + P + "b.mytable", stored.fullName()); SecurableObject backToApi = RelationalSchemaNamingBridge.securableObjectForApi(stored); @@ -187,14 +192,50 @@ public void flatSchemaIdent() { } // ------------------------------------------------------------------------- - // wrapperUpdater – SCHEMA guard + // wrapperUpdater – SCHEMA // ------------------------------------------------------------------------- @Test - public void wrapperUpdaterRejectsSchema() { - assertThrows( - IllegalArgumentException.class, - () -> RelationalSchemaNamingBridge.wrapperUpdater(Entity.EntityType.SCHEMA, e -> e)); + public void wrapperUpdaterSchemaRoundTrip() throws Exception { + mockSeparator(":"); + AuditInfo audit = AuditInfo.builder().withCreator("test").withCreateTime(Instant.now()).build(); + SchemaEntity storageEntity = + SchemaEntity.builder() + .withId(1L) + .withName("a" + P + "b") + .withNamespace(Namespace.of("ml", "cat")) + .withComment("c0") + .withProperties(null) + .withAuditInfo(audit) + .build(); + + var wrapped = + RelationalSchemaNamingBridge.wrapperUpdater( + Entity.EntityType.SCHEMA, + e -> + SchemaEntity.builder() + .withId(e.id()) + .withName(e.name()) + .withNamespace(e.namespace()) + .withComment("c1") + .withProperties(e.properties()) + .withAuditInfo(e.auditInfo()) + .build()); + + SchemaEntity out = wrapped.apply(storageEntity); + assertEquals("a" + P + "b", out.name()); + assertEquals("c1", out.comment()); + + final String[] seenLogicalName = new String[1]; + var wrappedPassThrough = + RelationalSchemaNamingBridge.wrapperUpdater( + Entity.EntityType.SCHEMA, + e -> { + seenLogicalName[0] = e.name(); + return e; + }); + wrappedPassThrough.apply(storageEntity); + assertEquals("a:b", seenLogicalName[0]); } // ------------------------------------------------------------------------- @@ -324,8 +365,7 @@ public void roleRoundTrip() { // ------------------------------------------------------------------------- private static RoleEntity buildRole(List securableObjects) { - AuditInfo audit = - AuditInfo.builder().withCreator("test").withCreateTime(Instant.now()).build(); + AuditInfo audit = AuditInfo.builder().withCreator("test").withCreateTime(Instant.now()).build(); return RoleEntity.builder() .withId(1L) .withName("role1") From 6d231381d9b93b033037ecd8c54a4a8aad93c2c3 Mon Sep 17 00:00:00 2001 From: Rory Date: Wed, 13 May 2026 19:58:42 +0800 Subject: [PATCH 09/44] revert unused changes --- .../HierarchicalSchemaRelationalBackend.java | 3 +-- .../storage/relational/JDBCBackend.java | 27 ++++--------------- 2 files changed, 6 insertions(+), 24 deletions(-) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/HierarchicalSchemaRelationalBackend.java b/core/src/main/java/org/apache/gravitino/storage/relational/HierarchicalSchemaRelationalBackend.java index 79af337a132..337645788e9 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/HierarchicalSchemaRelationalBackend.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/HierarchicalSchemaRelationalBackend.java @@ -175,8 +175,7 @@ private static RelationalEntity remapRelationEntity( re.type(), storageToLogical.get(re.source()), re.sourceType(), - RelationalSchemaNamingBridge.entityForApi( - (Entity & HasIdentifier) tgt, tgt.type())); + RelationalSchemaNamingBridge.entityForApi((Entity & HasIdentifier) tgt, tgt.type())); } @Override diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java b/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java index 65f641ea0d4..e076cced64e 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java @@ -93,14 +93,6 @@ * a database that supports the JDBC protocol as storage. If the specified database has special SQL * syntax, please implement the SQL statements and methods in MyBatis Mapper separately and switch * according to the {@link Configs#ENTITY_RELATIONAL_JDBC_BACKEND_URL_KEY} parameter. - * - *

This class is storage-oriented: {@link NameIdentifier}s, {@link Namespace}s, and entity - * payloads passed into meta services must already use the physical hierarchical encoding expected - * by the relational layer (ASCII-1 schema segments where nested schemas are stored). API-level - * logical naming is translated at the {@link RelationalEntityStore} boundary by {@link - * HierarchicalSchemaRelationalBackend} (see {@link RelationalSchemaNamingBridge}). Types such as - * Fileset, Topic, Model, and Model version are not subject to embedded-schema identifier - * conversion. */ public class JDBCBackend implements RelationalBackend { @@ -161,11 +153,10 @@ public List list( case JOB: return (List) JobMetaService.getInstance().listJobsByNamespace(namespace); case TABLE_STATISTIC: - Namespace statisticNs = namespace; return (List) StatisticMetaService.getInstance() .listStatisticsByEntity( - NameIdentifier.parse(statisticNs.toString()), Entity.EntityType.TABLE); + NameIdentifier.parse(namespace.toString()), Entity.EntityType.TABLE); default: throw new UnsupportedEntityTypeException( "Unsupported entity type: %s for list operation", entityType); @@ -344,12 +335,7 @@ public List batchGet( case SCHEMA: return (List) SchemaMetaService.getInstance().batchGetSchemaByIdentifier(identifiers); case TABLE: - List tableStorageIdents = - identifiers.stream() - .map(id -> NameIdentifier.of(firstNamespace, id.name())) - .collect(Collectors.toList()); - return (List) - TableMetaService.getInstance().batchGetTableByIdentifier(tableStorageIdents); + return (List) TableMetaService.getInstance().batchGetTableByIdentifier(identifiers); case FILESET: return (List) FilesetMetaService.getInstance().batchGetFilesetByIdentifier(identifiers); case TOPIC: @@ -366,8 +352,7 @@ public List batchGet( return (List) JobTemplateMetaService.getInstance().batchGetJobTemplateByIdentifier(identifiers); case USER: - // No batch SQL yet; missing entities are silently omitted per batchGet contract. - // TODO: replace loops with true batch SQL operations for USER/GROUP/ROLE/VIEW. + // TODO: Add true batch SQL operations for users, roles, and views List users = Lists.newArrayList(); for (NameIdentifier identifier : identifiers) { try { @@ -650,12 +635,10 @@ public void batchPut(List entities, boolea 1 == entities.stream().collect(Collectors.groupingBy(HasIdentifier::namespace)).size(), "All entities must be in the same namespace for batchPut operation."); - List storageStatisticEntities = statisticEntities; - Namespace statisticNamespace = statisticEntities.get(0).namespace(); StatisticMetaService.getInstance() .batchInsertStatisticPOsOnDuplicateKeyUpdate( - storageStatisticEntities, - NameIdentifier.parse(statisticNamespace.toString()), + statisticEntities, + NameIdentifier.parse(statisticEntities.get(0).namespace().toString()), Entity.EntityType.TABLE); break; default: From cbbdb8ac96147b2ccf0ce80bb94ce43a81827256 Mon Sep 17 00:00:00 2001 From: Rory Date: Wed, 13 May 2026 20:09:13 +0800 Subject: [PATCH 10/44] address id resolver --- .../gravitino/cache/CachedEntityIdResolver.java | 13 +++++++++---- .../gravitino/storage/relational/JDBCBackend.java | 5 ----- .../storage/relational/RelationalBackend.java | 5 ----- .../relational/RelationalEntityStoreIdResolver.java | 4 +--- 4 files changed, 10 insertions(+), 17 deletions(-) diff --git a/core/src/main/java/org/apache/gravitino/cache/CachedEntityIdResolver.java b/core/src/main/java/org/apache/gravitino/cache/CachedEntityIdResolver.java index d77c685d407..e91a0f21dc8 100644 --- a/core/src/main/java/org/apache/gravitino/cache/CachedEntityIdResolver.java +++ b/core/src/main/java/org/apache/gravitino/cache/CachedEntityIdResolver.java @@ -23,6 +23,7 @@ import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.meta.EntityIdResolver; import org.apache.gravitino.meta.NamespacedEntityId; +import org.apache.gravitino.storage.relational.RelationalSchemaNamingBridge; import org.apache.gravitino.utils.NameIdentifierUtil; public class CachedEntityIdResolver implements EntityIdResolver { @@ -37,14 +38,16 @@ public CachedEntityIdResolver(EntityCache entityCache, EntityIdResolver underlyi @Override public NamespacedEntityId getEntityIds(NameIdentifier nameIdentifier, Entity.EntityType type) { + NameIdentifier cacheKey = + RelationalSchemaNamingBridge.nameIdentifierForApi(nameIdentifier, type); return entityCache - .getIfPresent(nameIdentifier, type) + .getIfPresent(cacheKey, type) .map( entity -> { - if (nameIdentifier.hasNamespace()) { + if (cacheKey.hasNamespace()) { NamespacedEntityId namespaceIds = getEntityIds( - NameIdentifierUtil.parentNameIdentifier(nameIdentifier, type), + NameIdentifierUtil.parentNameIdentifier(cacheKey, type), NameIdentifierUtil.parentEntityType(type)); return new NamespacedEntityId(entity.id(), namespaceIds.fullIds()); } else { @@ -56,8 +59,10 @@ public NamespacedEntityId getEntityIds(NameIdentifier nameIdentifier, Entity.Ent @Override public Long getEntityId(NameIdentifier nameIdentifier, Entity.EntityType type) { + NameIdentifier cacheKey = + RelationalSchemaNamingBridge.nameIdentifierForApi(nameIdentifier, type); return entityCache - .getIfPresent(nameIdentifier, type) + .getIfPresent(cacheKey, type) .map(HasIdentifier::id) .orElseGet(() -> underlyingResolver.getEntityId(nameIdentifier, type)); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java b/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java index e076cced64e..f401cd50be4 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java @@ -28,7 +28,6 @@ import java.io.IOException; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.function.Function; import java.util.stream.Collectors; import org.apache.commons.lang3.tuple.Pair; @@ -746,10 +745,6 @@ public void batchInsertRelations( if (srcIdentifiers == null || srcIdentifiers.isEmpty()) { return; } - Objects.requireNonNull(relType, "relType must not be null"); - Objects.requireNonNull(srcType, "srcType must not be null"); - Objects.requireNonNull(dstIdentifier, "dstIdentifier must not be null"); - Objects.requireNonNull(dstType, "dstType must not be null"); switch (relType) { case OWNER_REL: OwnerMetaService.getInstance() diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalBackend.java b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalBackend.java index 3e4318bb02f..ce4bc8f61a7 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalBackend.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalBackend.java @@ -116,11 +116,6 @@ E get(NameIdentifier ident, Entity.EntityType /** * Batch retrieves the entities associated with the identifiers and the entity type. * - *

Partial-result contract: for entity types that do not have a native batch-SQL - * implementation (currently {@code USER}, {@code GROUP}, {@code ROLE}, {@code VIEW}), missing - * entities are silently omitted from the result rather than causing an exception. Callers must - * therefore not assume that the returned list has the same size as {@code identifiers}. - * * @param The type of the entity returned. * @param identifiers The identifiers of the entities. * @param entityType The type of the entity. diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStoreIdResolver.java b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStoreIdResolver.java index 75ba9b7a172..f522cb7085a 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStoreIdResolver.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStoreIdResolver.java @@ -170,14 +170,12 @@ private NamespacedEntityId getEntityIdsRequiringMetalakeId( private NamespacedEntityId getEntityIdsRequiringSchemaIds( NameIdentifier nameIdentifier, Entity.EntityType type) { NameIdentifier schemaIdent = NameIdentifierUtil.getSchemaIdentifier(nameIdentifier); - String storageSchemaName = - RelationalSchemaNamingBridge.schemaIdentifierForStorage(schemaIdent).name(); SchemaIds schemaIds = SchemaMetaService.getInstance() .getSchemaIdByMetalakeNameAndCatalogNameAndSchemaName( NameIdentifierUtil.getMetalake(nameIdentifier), NameIdentifierUtil.getCatalogIdentifier(nameIdentifier).name(), - storageSchemaName); + schemaIdent.name()); switch (type) { case SCHEMA: From 6fa7da4f95650074336f5fe5044010b272ef9290 Mon Sep 17 00:00:00 2001 From: Rory Date: Wed, 13 May 2026 20:11:00 +0800 Subject: [PATCH 11/44] revert --- .../apache/gravitino/storage/relational/RelationalBackend.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalBackend.java b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalBackend.java index ce4bc8f61a7..61eb9c435ba 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalBackend.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalBackend.java @@ -119,7 +119,8 @@ E get(NameIdentifier ident, Entity.EntityType * @param The type of the entity returned. * @param identifiers The identifiers of the entities. * @param entityType The type of the entity. - * @return The entities found; may be smaller than {@code identifiers} if some are missing. + * @return The entities associated with the identifiers and the entity type, or null if the key + * does not exist. */ List batchGet( List identifiers, Entity.EntityType entityType); From 42d4809cb154a5f63bcda349f0dec4d4dcb892ec Mon Sep 17 00:00:00 2001 From: Rory Date: Wed, 13 May 2026 20:27:08 +0800 Subject: [PATCH 12/44] refactor --- .../storage/relational/RelationalSchemaNamingBridge.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalSchemaNamingBridge.java b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalSchemaNamingBridge.java index 91e749a5be8..359f653094d 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalSchemaNamingBridge.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalSchemaNamingBridge.java @@ -450,12 +450,10 @@ public static Function wrapperUpdater( "unchecked") // each case casts to the statically known subtype for that EntityType public static E entityForApi( E entity, Entity.EntityType type) { - if (entity instanceof RoleEntity) { - return (E) roleEntityForApi((RoleEntity) entity); - } if (entity instanceof GenericEntity) { return (E) genericEntityMetadataFullNameForApi((GenericEntity) entity); } + switch (type) { case SCHEMA: return (E) schemaEntityForApi((SchemaEntity) entity); @@ -467,6 +465,8 @@ public static E entityForApi( return (E) functionEntityForApi((FunctionEntity) entity); case TABLE_STATISTIC: return (E) statisticEntityForApi((StatisticEntity) entity); + case ROLE: + return (E) roleEntityForApi((RoleEntity) entity); default: return entity; } From 3d3f4dd27648ec3c33fbd7564b845881e7dcd276 Mon Sep 17 00:00:00 2001 From: Rory Date: Wed, 13 May 2026 20:49:55 +0800 Subject: [PATCH 13/44] remove convention --- .../cache/CachedEntityIdResolver.java | 13 +- .../HierarchicalSchemaRelationalBackend.java | 246 +-------- .../relational/RelationalEntityStore.java | 5 +- .../RelationalSchemaNamingBridge.java | 515 ------------------ .../TestRelationalSchemaNamingBridge.java | 384 ------------- 5 files changed, 9 insertions(+), 1154 deletions(-) delete mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/RelationalSchemaNamingBridge.java delete mode 100644 core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalSchemaNamingBridge.java diff --git a/core/src/main/java/org/apache/gravitino/cache/CachedEntityIdResolver.java b/core/src/main/java/org/apache/gravitino/cache/CachedEntityIdResolver.java index e91a0f21dc8..d77c685d407 100644 --- a/core/src/main/java/org/apache/gravitino/cache/CachedEntityIdResolver.java +++ b/core/src/main/java/org/apache/gravitino/cache/CachedEntityIdResolver.java @@ -23,7 +23,6 @@ import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.meta.EntityIdResolver; import org.apache.gravitino.meta.NamespacedEntityId; -import org.apache.gravitino.storage.relational.RelationalSchemaNamingBridge; import org.apache.gravitino.utils.NameIdentifierUtil; public class CachedEntityIdResolver implements EntityIdResolver { @@ -38,16 +37,14 @@ public CachedEntityIdResolver(EntityCache entityCache, EntityIdResolver underlyi @Override public NamespacedEntityId getEntityIds(NameIdentifier nameIdentifier, Entity.EntityType type) { - NameIdentifier cacheKey = - RelationalSchemaNamingBridge.nameIdentifierForApi(nameIdentifier, type); return entityCache - .getIfPresent(cacheKey, type) + .getIfPresent(nameIdentifier, type) .map( entity -> { - if (cacheKey.hasNamespace()) { + if (nameIdentifier.hasNamespace()) { NamespacedEntityId namespaceIds = getEntityIds( - NameIdentifierUtil.parentNameIdentifier(cacheKey, type), + NameIdentifierUtil.parentNameIdentifier(nameIdentifier, type), NameIdentifierUtil.parentEntityType(type)); return new NamespacedEntityId(entity.id(), namespaceIds.fullIds()); } else { @@ -59,10 +56,8 @@ public NamespacedEntityId getEntityIds(NameIdentifier nameIdentifier, Entity.Ent @Override public Long getEntityId(NameIdentifier nameIdentifier, Entity.EntityType type) { - NameIdentifier cacheKey = - RelationalSchemaNamingBridge.nameIdentifierForApi(nameIdentifier, type); return entityCache - .getIfPresent(cacheKey, type) + .getIfPresent(nameIdentifier, type) .map(HasIdentifier::id) .orElseGet(() -> underlyingResolver.getEntityId(nameIdentifier, type)); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/HierarchicalSchemaRelationalBackend.java b/core/src/main/java/org/apache/gravitino/storage/relational/HierarchicalSchemaRelationalBackend.java index 337645788e9..40ef0a053f0 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/HierarchicalSchemaRelationalBackend.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/HierarchicalSchemaRelationalBackend.java @@ -17,248 +17,8 @@ */ package org.apache.gravitino.storage.relational; -import java.io.IOException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.function.Function; -import java.util.stream.Collectors; -import org.apache.commons.lang3.tuple.Pair; -import org.apache.gravitino.Entity; -import org.apache.gravitino.EntityAlreadyExistsException; -import org.apache.gravitino.HasIdentifier; -import org.apache.gravitino.NameIdentifier; -import org.apache.gravitino.Namespace; -import org.apache.gravitino.RelationalEntity; -import org.apache.gravitino.exceptions.NoSuchEntityException; - /** - * JDBC-backed {@link RelationalBackend} that translates hierarchical schema logical naming (API) to - * the physical encoding expected by relational meta services, delegating storage operations to - * {@link JDBCBackend}. - * - *

{@link JDBCBackend} is treated as storage-oriented: identifiers and entity payloads passed to - * it use physical schema segments and embedded namespaces where applicable. Naming rules live in - * {@link RelationalSchemaNamingBridge}, especially {@link - * RelationalSchemaNamingBridge#entityForApi} (covers relation list elements) and {@link - * RelationalSchemaNamingBridge#nameIdentifierForStorage}, so this class does not branch on {@link - * Type} or {@link Entity.EntityType} for relation calls. + * Default JDBC-backed {@link RelationalBackend}. Extends {@link JDBCBackend} without altering + * identifiers or entity payloads so the relational meta services see the same names as callers. */ -public class HierarchicalSchemaRelationalBackend extends JDBCBackend { - - @Override - public List list( - Namespace namespace, Entity.EntityType entityType, boolean allFields) throws IOException { - Namespace storageParentNs = RelationalSchemaNamingBridge.embeddedNamespaceForStorage(namespace); - List raw = super.list(storageParentNs, entityType, allFields); - return raw.stream() - .map(e -> (E) RelationalSchemaNamingBridge.entityForApi(e, entityType)) - .collect(Collectors.toList()); - } - - @Override - public void insert(E e, boolean overwritten) - throws EntityAlreadyExistsException, IOException { - super.insert(RelationalSchemaNamingBridge.entityForStorage(e), overwritten); - } - - @Override - public E update( - NameIdentifier ident, Entity.EntityType entityType, Function updater) - throws IOException, NoSuchEntityException, EntityAlreadyExistsException { - return (E) - RelationalSchemaNamingBridge.entityForApi( - super.update( - RelationalSchemaNamingBridge.nameIdentifierForStorage(ident, entityType), - entityType, - RelationalSchemaNamingBridge.wrapperUpdater(entityType, updater)), - entityType); - } - - @Override - public E get( - NameIdentifier ident, Entity.EntityType entityType) - throws NoSuchEntityException, IOException { - return (E) - RelationalSchemaNamingBridge.entityForApi( - super.get( - RelationalSchemaNamingBridge.nameIdentifierForStorage(ident, entityType), - entityType), - entityType); - } - - @Override - public List batchGet( - List identifiers, Entity.EntityType entityType) { - List storageIdents = - identifiers.stream() - .map(id -> RelationalSchemaNamingBridge.nameIdentifierForStorage(id, entityType)) - .collect(Collectors.toList()); - List got = super.batchGet(storageIdents, entityType); - return got.stream() - .map(e -> (E) RelationalSchemaNamingBridge.entityForApi(e, entityType)) - .collect(Collectors.toList()); - } - - @Override - public boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolean cascade) - throws IOException { - return super.delete( - RelationalSchemaNamingBridge.nameIdentifierForStorage(ident, entityType), - entityType, - cascade); - } - - @Override - public int batchDelete( - List> entitiesToDelete, boolean cascade) - throws IOException { - List> storagePairs = - entitiesToDelete.stream() - .map( - p -> - Pair.of( - RelationalSchemaNamingBridge.nameIdentifierForStorage( - p.getLeft(), p.getRight()), - p.getRight())) - .collect(Collectors.toList()); - return super.batchDelete(storagePairs, cascade); - } - - @Override - public void batchPut(List entities, boolean overwritten) - throws IOException, EntityAlreadyExistsException { - List storage = - entities.stream() - .map(RelationalSchemaNamingBridge::entityForStorage) - .collect(Collectors.toList()); - super.batchPut(storage, overwritten); - } - - @Override - public List listEntitiesByRelation( - Type relType, NameIdentifier nameIdentifier, Entity.EntityType identType, boolean allFields) - throws IOException { - NameIdentifier source = - RelationalSchemaNamingBridge.nameIdentifierForStorage(nameIdentifier, identType); - List out = super.listEntitiesByRelation(relType, source, identType, allFields); - return out.stream() - .map(e -> (E) RelationalSchemaNamingBridge.entityForApi(e, e.type())) - .collect(Collectors.toList()); - } - - @Override - public List> batchListEntitiesByRelation( - Type relType, List nameIdentifiers, Entity.EntityType identType) - throws IOException { - List storageIdents = - nameIdentifiers.stream() - .map(n -> RelationalSchemaNamingBridge.nameIdentifierForStorage(n, identType)) - .collect(Collectors.toList()); - Map storageToLogical = new HashMap<>(); - for (int i = 0; i < nameIdentifiers.size(); i++) { - storageToLogical.put(storageIdents.get(i), nameIdentifiers.get(i)); - } - List> raw = - super.batchListEntitiesByRelation(relType, storageIdents, identType); - return raw.stream() - .map(re -> remapRelationEntity(re, storageToLogical)) - .collect(Collectors.toList()); - } - - @SuppressWarnings("unchecked") - private static RelationalEntity remapRelationEntity( - RelationalEntity re, Map storageToLogical) { - Entity tgt = re.targetEntity(); - return new RelationalEntity<>( - re.type(), - storageToLogical.get(re.source()), - re.sourceType(), - RelationalSchemaNamingBridge.entityForApi((Entity & HasIdentifier) tgt, tgt.type())); - } - - @Override - public void insertRelation( - Type relType, - NameIdentifier srcIdentifier, - Entity.EntityType srcType, - NameIdentifier dstIdentifier, - Entity.EntityType dstType, - boolean override) { - super.insertRelation( - relType, - RelationalSchemaNamingBridge.nameIdentifierForStorage(srcIdentifier, srcType), - srcType, - RelationalSchemaNamingBridge.nameIdentifierForStorage(dstIdentifier, dstType), - dstType, - override); - } - - @Override - public void batchInsertRelations( - Type relType, - List srcIdentifiers, - Entity.EntityType srcType, - NameIdentifier dstIdentifier, - Entity.EntityType dstType, - boolean override) - throws IOException { - List srcStorage = - srcIdentifiers.stream() - .map(i -> RelationalSchemaNamingBridge.nameIdentifierForStorage(i, srcType)) - .collect(Collectors.toList()); - super.batchInsertRelations( - relType, - srcStorage, - srcType, - RelationalSchemaNamingBridge.nameIdentifierForStorage(dstIdentifier, dstType), - dstType, - override); - } - - @Override - public List updateEntityRelations( - Type relType, - NameIdentifier srcEntityIdent, - Entity.EntityType srcEntityType, - NameIdentifier[] destEntitiesToAdd, - NameIdentifier[] destEntitiesToRemove) - throws IOException, NoSuchEntityException, EntityAlreadyExistsException { - List out = - super.updateEntityRelations( - relType, - RelationalSchemaNamingBridge.nameIdentifierForStorage(srcEntityIdent, srcEntityType), - srcEntityType, - Arrays.stream(destEntitiesToAdd) - .map( - id -> - RelationalSchemaNamingBridge.relationDestIdentifiersForStorage(relType, id)) - .toArray(NameIdentifier[]::new), - Arrays.stream(destEntitiesToRemove) - .map( - id -> - RelationalSchemaNamingBridge.relationDestIdentifiersForStorage(relType, id)) - .toArray(NameIdentifier[]::new)); - return out.stream() - .map(e -> (E) RelationalSchemaNamingBridge.entityForApi(e, e.type())) - .collect(Collectors.toList()); - } - - @Override - public E getEntityByRelation( - Type relType, - NameIdentifier srcIdentifier, - Entity.EntityType srcType, - NameIdentifier destEntityIdent) - throws IOException, NoSuchEntityException { - E raw = - super.getEntityByRelation( - relType, - RelationalSchemaNamingBridge.nameIdentifierForStorage(srcIdentifier, srcType), - srcType, - RelationalSchemaNamingBridge.relationDestIdentifiersForStorage( - relType, destEntityIdent)); - return (E) RelationalSchemaNamingBridge.entityForApi(raw, raw.type()); - } -} +public class HierarchicalSchemaRelationalBackend extends JDBCBackend {} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStore.java b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStore.java index 367822b9b32..e50fd475074 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStore.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStore.java @@ -55,9 +55,8 @@ /** * Relation store to store entities. This means we can store entities in a relational store. I.e., * MySQL, PostgreSQL, etc. If you want to use a different backend, you can implement the {@link - * RelationalBackend} interface. The default JDBC backend is created as {@link - * HierarchicalSchemaRelationalBackend} so hierarchical schema naming can evolve at the store - * boundary without rewriting {@link RelationalEntityStore} callers. + * RelationalBackend} interface. The default JDBC backend is {@link + * HierarchicalSchemaRelationalBackend}. */ public class RelationalEntityStore implements EntityStore, SupportsRelationOperations { private static final Logger LOGGER = LoggerFactory.getLogger(RelationalEntityStore.class); diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalSchemaNamingBridge.java b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalSchemaNamingBridge.java deleted file mode 100644 index 359f653094d..00000000000 --- a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalSchemaNamingBridge.java +++ /dev/null @@ -1,515 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file to - * you under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gravitino.storage.relational; - -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.function.Function; -import java.util.stream.Collectors; -import org.apache.gravitino.Entity; -import org.apache.gravitino.HasIdentifier; -import org.apache.gravitino.MetadataObject; -import org.apache.gravitino.NameIdentifier; -import org.apache.gravitino.Namespace; -import org.apache.gravitino.SupportsRelationOperations.Type; -import org.apache.gravitino.authorization.SecurableObject; -import org.apache.gravitino.authorization.SecurableObjects; -import org.apache.gravitino.meta.AuditInfo; -import org.apache.gravitino.meta.FunctionEntity; -import org.apache.gravitino.meta.GenericEntity; -import org.apache.gravitino.meta.RoleEntity; -import org.apache.gravitino.meta.SchemaEntity; -import org.apache.gravitino.meta.StatisticEntity; -import org.apache.gravitino.meta.TableEntity; -import org.apache.gravitino.meta.TableStatisticEntity; -import org.apache.gravitino.meta.ViewEntity; -import org.apache.gravitino.utils.HierarchicalSchemaUtil; -import org.apache.gravitino.utils.MetadataObjectUtil; - -/** - * Translates hierarchical schema naming between the logical representation used above {@link - * JDBCBackend} (configured separator in identifiers/namespaces) and the physical representation - * expected by relational meta services (ASCII-1 hierarchy separator in schema segments). - * - *

{@link Entity.EntityType#FILESET}, {@link Entity.EntityType#TOPIC}, {@link - * Entity.EntityType#MODEL}, and {@link Entity.EntityType#MODEL_VERSION} identifiers/namespaces are - * left unchanged by this bridge (delegated as-is to meta services). - */ -public final class RelationalSchemaNamingBridge { - - private static final Joiner DOT_JOINER = Joiner.on('.'); - private static final String SENTINEL_METALAKE = "_"; - - private RelationalSchemaNamingBridge() {} - - /** - * Reconstructs a dotted metadata-object full name from a {@link NameIdentifier} that was created - * by prepending {@link #SENTINEL_METALAKE} to the original full name. Skips the sentinel at - * {@code namespace.levels()[0]} and joins the remaining levels with the identifier's name. - */ - private static String identToMetadataObjectFullName(NameIdentifier ident) { - String[] levels = ident.namespace().levels(); - List parts = new ArrayList<>(levels.length); - for (int i = 1; i < levels.length; i++) { - parts.add(levels[i]); - } - parts.add(ident.name()); - return DOT_JOINER.join(parts); - } - - private static String separator() { - return HierarchicalSchemaUtil.schemaSeparator(); - } - - /** - * Converts the dotted metadata {@code fullName} carried by {@link SecurableObject} so the schema - * segment matches relational storage (physical hierarchical encoding). Types excluded from schema - * embedding ({@link Entity.EntityType#FILESET}, {@link Entity.EntityType#TOPIC}, {@link - * Entity.EntityType#MODEL}, {@link Entity.EntityType#MODEL_VERSION}) are unchanged. - */ - public static SecurableObject securableObjectForStorage(SecurableObject object) { - Entity.EntityType entityType = MetadataObjectUtil.toEntityType(object.type()); - NameIdentifier ident = NameIdentifier.parse(SENTINEL_METALAKE + "." + object.fullName()); - NameIdentifier converted = nameIdentifierForStorage(ident, entityType); - if (converted.equals(ident)) { - return object; - } - return SecurableObjects.parse( - identToMetadataObjectFullName(converted), object.type(), object.privileges()); - } - - /** - * Converts the dotted metadata {@code fullName} on a {@link SecurableObject} to logical schema - * naming for API callers. Passthrough rules match {@link - * #securableObjectForStorage(SecurableObject)}. - */ - public static SecurableObject securableObjectForApi(SecurableObject object) { - Entity.EntityType entityType = MetadataObjectUtil.toEntityType(object.type()); - NameIdentifier ident = NameIdentifier.parse(SENTINEL_METALAKE + "." + object.fullName()); - NameIdentifier converted = nameIdentifierForApi(ident, entityType); - if (converted.equals(ident)) { - return object; - } - return SecurableObjects.parse( - identToMetadataObjectFullName(converted), object.type(), object.privileges()); - } - - public static RoleEntity roleEntityForStorage(RoleEntity entity) { - return mapRoleSecurableObjects(entity, RelationalSchemaNamingBridge::securableObjectForStorage); - } - - public static RoleEntity roleEntityForApi(RoleEntity entity) { - return mapRoleSecurableObjects(entity, RelationalSchemaNamingBridge::securableObjectForApi); - } - - private static RoleEntity mapRoleSecurableObjects( - RoleEntity entity, Function mapper) { - List objects = entity.securableObjects(); - if (objects == null || objects.isEmpty()) { - return entity; - } - List mapped = objects.stream().map(mapper).collect(Collectors.toList()); - if (mapped.equals(objects)) { - return entity; - } - return RoleEntity.builder() - .withId(entity.id()) - .withName(entity.name()) - .withNamespace(entity.namespace()) - .withProperties(entity.properties()) - .withAuditInfo(entity.auditInfo()) - .withSecurableObjects(mapped) - .build(); - } - - /** - * Normalizes {@link GenericEntity#name()} when it holds a dotted metadata-object path whose - * schema segment uses physical hierarchical encoding, for API callers above {@link JDBCBackend}. - * - *

Only processes entity types that correspond to a {@link MetadataObject.Type} (e.g. TABLE, - * SCHEMA). Types without a {@link MetadataObject.Type} equivalent (e.g. TABLE_STATISTIC, - * MODEL_VERSION) are returned unchanged. - */ - public static GenericEntity genericEntityMetadataFullNameForApi(GenericEntity entity) { - NameIdentifier converted = nameIdentifierForStorage(entity.nameIdentifier(), entity.type()); - return GenericEntity.builder() - .withId(entity.id()) - .withEntityType(entity.type()) - .withName(converted.name()) - .withNamespace(converted.namespace()) - .build(); - } - - /** - * Converts the schema segment at namespace index 2 (the slot occupied by the schema name in the - * standard {@code [metalake, catalog, schema, ...]} layout) from logical to physical form. - * Callers must only pass namespaces that follow this layout; if the schema sits at a different - * index the conversion will silently apply to the wrong segment. - */ - public static Namespace embeddedNamespaceForStorage(Namespace ns) { - if (ns.length() < 3) { - return ns; - } - String[] lv = ns.levels(); - String[] copy = Arrays.copyOf(lv, lv.length); - copy[2] = HierarchicalSchemaUtil.logicalToPhysical(copy[2], separator()); - return Namespace.of(copy); - } - - /** - * Converts the schema segment at namespace index 2 from physical to logical form. See {@link - * #embeddedNamespaceForStorage(Namespace)} for the layout constraint. - */ - public static Namespace embeddedNamespaceForApi(Namespace ns) { - if (ns.length() < 3) { - return ns; - } - String[] lv = ns.levels(); - String[] copy = Arrays.copyOf(lv, lv.length); - copy[2] = HierarchicalSchemaUtil.physicalToLogical(copy[2], separator()); - return Namespace.of(copy); - } - - public static NameIdentifier schemaIdentifierForStorage(NameIdentifier ident) { - return NameIdentifier.of( - ident.namespace(), HierarchicalSchemaUtil.logicalToPhysical(ident.name(), separator())); - } - - public static NameIdentifier schemaIdentifierForApi(NameIdentifier ident) { - return NameIdentifier.of( - ident.namespace(), HierarchicalSchemaUtil.physicalToLogical(ident.name(), separator())); - } - - /** - * Converts identifiers carrying optional hierarchical schema segments before delegating to JDBC - * meta services (physical naming). - */ - public static NameIdentifier nameIdentifierForStorage( - NameIdentifier ident, Entity.EntityType entityType) { - switch (entityType) { - case SCHEMA: - return schemaIdentifierForStorage(ident); - case TABLE: - case VIEW: - case FUNCTION: - case COLUMN: - case TABLE_STATISTIC: - return NameIdentifier.of(embeddedNamespaceForStorage(ident.namespace()), ident.name()); - default: - // FILESET, TOPIC, MODEL, MODEL_VERSION and other types do not embed the schema - // in their identifier path and are forwarded as-is to the meta services. - return ident; - } - } - - /** - * Converts identifiers from physical storage naming back to the logical form used by API callers. - * Passthrough rules match {@link #nameIdentifierForStorage(NameIdentifier, Entity.EntityType)}. - */ - public static NameIdentifier nameIdentifierForApi( - NameIdentifier ident, Entity.EntityType entityType) { - switch (entityType) { - case SCHEMA: - return schemaIdentifierForApi(ident); - case TABLE: - case VIEW: - case FUNCTION: - case COLUMN: - case TABLE_STATISTIC: - return NameIdentifier.of(embeddedNamespaceForApi(ident.namespace()), ident.name()); - default: - // FILESET, TOPIC, MODEL, MODEL_VERSION and other types are returned unchanged. - return ident; - } - } - - public static SchemaEntity schemaEntityForApi(SchemaEntity entity) { - String logicalName = HierarchicalSchemaUtil.physicalToLogical(entity.name(), separator()); - if (logicalName.equals(entity.name())) { - return entity; - } - return SchemaEntity.builder() - .withId(entity.id()) - .withName(logicalName) - .withNamespace(entity.namespace()) - .withComment(entity.comment()) - .withProperties(entity.properties()) - .withAuditInfo(entity.auditInfo()) - .build(); - } - - /** - * Converts a {@link SchemaEntity}'s {@link SchemaEntity#name()} from logical (API) form to the - * physical encoding used by relational meta services. Inverse of {@link #schemaEntityForApi}. - */ - public static SchemaEntity schemaEntityForStorage(SchemaEntity entity) { - String physicalName = HierarchicalSchemaUtil.logicalToPhysical(entity.name(), separator()); - if (physicalName.equals(entity.name())) { - return entity; - } - return SchemaEntity.builder() - .withId(entity.id()) - .withName(physicalName) - .withNamespace(entity.namespace()) - .withComment(entity.comment()) - .withProperties(entity.properties()) - .withAuditInfo(entity.auditInfo()) - .build(); - } - - public static TableEntity tableEntityForApi(TableEntity e) { - Namespace apiNs = embeddedNamespaceForApi(e.namespace()); - if (apiNs.equals(e.namespace())) { - return e; - } - return TableEntity.builder() - .withId(e.id()) - .withName(e.name()) - .withNamespace(apiNs) - .withColumns(e.columns()) - .withAuditInfo(e.auditInfo()) - .withProperties(e.properties()) - .withPartitioning(e.partitioning()) - .withSortOrders(e.sortOrders()) - .withDistribution(e.distribution()) - .withIndexes(e.indexes()) - .withComment(e.comment()) - .build(); - } - - public static TableEntity tableEntityForStorage(TableEntity e) { - Namespace storageNs = embeddedNamespaceForStorage(e.namespace()); - if (storageNs.equals(e.namespace())) { - return e; - } - return TableEntity.builder() - .withId(e.id()) - .withName(e.name()) - .withNamespace(storageNs) - .withColumns(e.columns()) - .withAuditInfo(e.auditInfo()) - .withProperties(e.properties()) - .withPartitioning(e.partitioning()) - .withSortOrders(e.sortOrders()) - .withDistribution(e.distribution()) - .withIndexes(e.indexes()) - .withComment(e.comment()) - .build(); - } - - public static ViewEntity viewEntityForApi(ViewEntity e) { - Namespace apiNs = embeddedNamespaceForApi(e.namespace()); - if (apiNs.equals(e.namespace())) { - return e; - } - return ViewEntity.builder() - .withId(e.id()) - .withName(e.name()) - .withNamespace(apiNs) - .withColumns(e.columns()) - .withRepresentations(e.representations()) - .withDefaultCatalog(e.defaultCatalog()) - .withDefaultSchema(e.defaultSchema()) - .withComment(e.comment()) - .withProperties(e.properties()) - .withAuditInfo(e.auditInfo()) - .build(); - } - - public static ViewEntity viewEntityForStorage(ViewEntity e) { - Namespace storageNs = embeddedNamespaceForStorage(e.namespace()); - if (storageNs.equals(e.namespace())) { - return e; - } - return ViewEntity.builder() - .withId(e.id()) - .withName(e.name()) - .withNamespace(storageNs) - .withColumns(e.columns()) - .withRepresentations(e.representations()) - .withDefaultCatalog(e.defaultCatalog()) - .withDefaultSchema(e.defaultSchema()) - .withComment(e.comment()) - .withProperties(e.properties()) - .withAuditInfo(e.auditInfo()) - .build(); - } - - public static FunctionEntity functionEntityForApi(FunctionEntity e) { - Namespace apiNs = embeddedNamespaceForApi(e.namespace()); - if (apiNs.equals(e.namespace())) { - return e; - } - return FunctionEntity.builder() - .withId(e.id()) - .withName(e.name()) - .withNamespace(apiNs) - .withComment(e.comment()) - .withFunctionType(e.functionType()) - .withDeterministic(e.deterministic()) - .withDefinitions(e.definitions()) - .withAuditInfo(e.auditInfo()) - .build(); - } - - public static FunctionEntity functionEntityForStorage(FunctionEntity e) { - Namespace storageNs = embeddedNamespaceForStorage(e.namespace()); - if (storageNs.equals(e.namespace())) { - return e; - } - return FunctionEntity.builder() - .withId(e.id()) - .withName(e.name()) - .withNamespace(storageNs) - .withComment(e.comment()) - .withFunctionType(e.functionType()) - .withDeterministic(e.deterministic()) - .withDefinitions(e.definitions()) - .withAuditInfo(e.auditInfo()) - .build(); - } - - /** - * Converts a {@link StatisticEntity}'s namespace from physical storage form to logical API form. - * Only {@link TableStatisticEntity} is currently stored with an embedded schema namespace; other - * concrete subtypes are not expected and will cause an {@link IllegalArgumentException}. - */ - public static StatisticEntity statisticEntityForApi(StatisticEntity e) { - return statisticEntityWithNamespace(e, embeddedNamespaceForApi(e.namespace())); - } - - /** - * Converts a {@link StatisticEntity}'s namespace to the physical storage form. See {@link - * #statisticEntityForApi(StatisticEntity)} for subtype restrictions. - */ - public static StatisticEntity statisticEntityForStorage(StatisticEntity e) { - return statisticEntityWithNamespace(e, embeddedNamespaceForStorage(e.namespace())); - } - - private static StatisticEntity statisticEntityWithNamespace(StatisticEntity e, Namespace ns) { - Preconditions.checkArgument( - e instanceof TableStatisticEntity, - "Only TableStatisticEntity is supported, got: %s", - e.getClass().getSimpleName()); - if (ns.equals(e.namespace())) { - return e; - } - return TableStatisticEntity.builder() - .withId(e.id()) - .withName(e.name()) - .withValue(e.value()) - .withAuditInfo((AuditInfo) e.auditInfo()) - .withNamespace(ns) - .build(); - } - - /** - * Wraps a meta-service {@code updater} used from {@link JDBCBackend}: converts entities from - * storage naming to API naming, applies {@code updater}, then converts back to storage for - * relational persistence. - * - * @param entityType bridged entity kind (e.g. {@link Entity.EntityType#TABLE}); passed to {@link - * #entityForApi(Entity, Entity.EntityType)} - */ - @SuppressWarnings("unchecked") // entityForApi/entityForStorage dispatch on known concrete types - public static Function wrapperUpdater( - Entity.EntityType entityType, Function updater) { - return e -> { - E logicalOld = entityForApi(e, entityType); - E logicalNew = updater.apply(logicalOld); - return entityForStorage(logicalNew); - }; - } - - /** - * Converts storage-shaped entities to API naming. {@link RoleEntity} and {@link GenericEntity} - * (policy/tag relation list payloads) are normalized by concrete type before the {@code - * type}-based branch so callers may pass each element's {@link Entity.EntityType} (e.g. roles - * listed for a {@link Entity.EntityType#TABLE} metadata object still use {@code TABLE} while - * elements are roles). - */ - @SuppressWarnings( - "unchecked") // each case casts to the statically known subtype for that EntityType - public static E entityForApi( - E entity, Entity.EntityType type) { - if (entity instanceof GenericEntity) { - return (E) genericEntityMetadataFullNameForApi((GenericEntity) entity); - } - - switch (type) { - case SCHEMA: - return (E) schemaEntityForApi((SchemaEntity) entity); - case TABLE: - return (E) tableEntityForApi((TableEntity) entity); - case VIEW: - return (E) viewEntityForApi((ViewEntity) entity); - case FUNCTION: - return (E) functionEntityForApi((FunctionEntity) entity); - case TABLE_STATISTIC: - return (E) statisticEntityForApi((StatisticEntity) entity); - case ROLE: - return (E) roleEntityForApi((RoleEntity) entity); - default: - return entity; - } - } - - @SuppressWarnings("unchecked") // instanceof guards ensure each cast is safe - public static E entityForStorage(E entity) { - if (entity instanceof SchemaEntity) { - return (E) schemaEntityForStorage((SchemaEntity) entity); - } - if (entity instanceof TableEntity) { - return (E) tableEntityForStorage((TableEntity) entity); - } - if (entity instanceof ViewEntity) { - return (E) viewEntityForStorage((ViewEntity) entity); - } - if (entity instanceof FunctionEntity) { - return (E) functionEntityForStorage((FunctionEntity) entity); - } - if (entity instanceof StatisticEntity) { - return (E) statisticEntityForStorage((StatisticEntity) entity); - } - if (entity instanceof RoleEntity) { - return (E) roleEntityForStorage((RoleEntity) entity); - } - return entity; - } - - /** - * JDBC policy/tag relation APIs: a destination {@link NameIdentifier} is normalized using the - * element type implied by {@code relType} when applicable (e.g. policy or tag name for - * association rows). - */ - public static NameIdentifier relationDestIdentifiersForStorage( - Type relType, NameIdentifier ident) { - Entity.EntityType elementType; - if (relType == Type.POLICY_METADATA_OBJECT_REL) { - elementType = Entity.EntityType.POLICY; - } else if (relType == Type.TAG_METADATA_OBJECT_REL) { - elementType = Entity.EntityType.TAG; - } else { - return ident; - } - return nameIdentifierForStorage(ident, elementType); - } -} diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalSchemaNamingBridge.java b/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalSchemaNamingBridge.java deleted file mode 100644 index f801f96a26c..00000000000 --- a/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalSchemaNamingBridge.java +++ /dev/null @@ -1,384 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.gravitino.storage.relational; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertSame; - -import com.google.common.collect.Lists; -import java.time.Instant; -import java.util.List; -import org.apache.commons.lang3.reflect.FieldUtils; -import org.apache.gravitino.Config; -import org.apache.gravitino.Configs; -import org.apache.gravitino.Entity; -import org.apache.gravitino.GravitinoEnv; -import org.apache.gravitino.MetadataObject; -import org.apache.gravitino.NameIdentifier; -import org.apache.gravitino.Namespace; -import org.apache.gravitino.authorization.Privilege; -import org.apache.gravitino.authorization.Privileges; -import org.apache.gravitino.authorization.SecurableObject; -import org.apache.gravitino.authorization.SecurableObjects; -import org.apache.gravitino.meta.AuditInfo; -import org.apache.gravitino.meta.GenericEntity; -import org.apache.gravitino.meta.RoleEntity; -import org.apache.gravitino.meta.SchemaEntity; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -/** Unit tests for {@link RelationalSchemaNamingBridge}. */ -public class TestRelationalSchemaNamingBridge { - - /** ASCII-1 internal separator used in physical storage. */ - private static final String P = ""; - - private static final List USE_SCHEMA_PRIVS = - Lists.newArrayList(Privileges.UseSchema.allow()); - private static final SecurableObject CATALOG_OBJ = - SecurableObjects.ofCatalog("catalog", Lists.newArrayList()); - - @BeforeEach - public void setUp() throws Exception { - mockSeparator(":"); - } - - @AfterEach - public void tearDown() throws Exception { - FieldUtils.writeField(GravitinoEnv.getInstance(), "config", null, true); - } - - // ------------------------------------------------------------------------- - // securableObjectForStorage / securableObjectForApi - // ------------------------------------------------------------------------- - - @Test - public void securableObjectSchemaToStorage() { - SecurableObject obj = SecurableObjects.ofSchema(CATALOG_OBJ, "a:b:c", USE_SCHEMA_PRIVS); - SecurableObject result = RelationalSchemaNamingBridge.securableObjectForStorage(obj); - assertEquals("catalog.a" + P + "b" + P + "c", result.fullName()); - } - - @Test - public void securableObjectSchemaToApi() { - SecurableObject obj = - SecurableObjects.parse( - "catalog.a" + P + "b" + P + "c", MetadataObject.Type.SCHEMA, USE_SCHEMA_PRIVS); - SecurableObject result = RelationalSchemaNamingBridge.securableObjectForApi(obj); - assertEquals("catalog.a:b:c", result.fullName()); - } - - @Test - public void securableObjectTableToStorage() { - SecurableObject obj = - SecurableObjects.parse( - "catalog.a:b.mytable", MetadataObject.Type.TABLE, Lists.newArrayList()); - SecurableObject result = RelationalSchemaNamingBridge.securableObjectForStorage(obj); - assertEquals("catalog.a" + P + "b.mytable", result.fullName()); - } - - @Test - public void securableObjectTableToApi() { - SecurableObject obj = - SecurableObjects.parse( - "catalog.a" + P + "b.mytable", MetadataObject.Type.TABLE, Lists.newArrayList()); - SecurableObject result = RelationalSchemaNamingBridge.securableObjectForApi(obj); - assertEquals("catalog.a:b.mytable", result.fullName()); - } - - @Test - public void securableObjectCatalogPassthrough() { - SecurableObject catalog = SecurableObjects.ofCatalog("catalog", Lists.newArrayList()); - assertSame(catalog, RelationalSchemaNamingBridge.securableObjectForStorage(catalog)); - assertSame(catalog, RelationalSchemaNamingBridge.securableObjectForApi(catalog)); - } - - @Test - public void securableObjectTableRoundTrip() { - SecurableObject obj = - SecurableObjects.parse( - "catalog.a:b:c.mytable", MetadataObject.Type.TABLE, Lists.newArrayList()); - SecurableObject stored = RelationalSchemaNamingBridge.securableObjectForStorage(obj); - SecurableObject backToApi = RelationalSchemaNamingBridge.securableObjectForApi(stored); - assertEquals(obj.fullName(), backToApi.fullName()); - } - - @Test - public void customSeparatorSecurableObject() throws Exception { - mockSeparator("/"); - SecurableObject obj = - SecurableObjects.parse( - "catalog.a/b.mytable", MetadataObject.Type.TABLE, Lists.newArrayList()); - SecurableObject stored = RelationalSchemaNamingBridge.securableObjectForStorage(obj); - assertEquals("catalog.a" + P + "b.mytable", stored.fullName()); - SecurableObject backToApi = RelationalSchemaNamingBridge.securableObjectForApi(stored); - assertEquals("catalog.a/b.mytable", backToApi.fullName()); - } - - // ------------------------------------------------------------------------- - // embeddedNamespaceForStorage / embeddedNamespaceForApi - // ------------------------------------------------------------------------- - - @Test - public void embeddedNsToStorage() { - Namespace ns = Namespace.of("ml", "cat", "a:b:c"); - Namespace stored = RelationalSchemaNamingBridge.embeddedNamespaceForStorage(ns); - assertEquals(Namespace.of("ml", "cat", "a" + P + "b" + P + "c"), stored); - } - - @Test - public void embeddedNsToApi() { - Namespace ns = Namespace.of("ml", "cat", "a" + P + "b" + P + "c"); - Namespace api = RelationalSchemaNamingBridge.embeddedNamespaceForApi(ns); - assertEquals(Namespace.of("ml", "cat", "a:b:c"), api); - } - - @Test - public void embeddedNsShortUnchanged() { - // Namespaces with fewer than 3 levels are returned as-is - Namespace ns = Namespace.of("ml", "cat"); - Namespace stored = RelationalSchemaNamingBridge.embeddedNamespaceForStorage(ns); - assertSame(ns, stored); - } - - // ------------------------------------------------------------------------- - // nameIdentifierForStorage / nameIdentifierForApi – SCHEMA - // ------------------------------------------------------------------------- - - @Test - public void schemaIdentToStorage() { - NameIdentifier ident = NameIdentifier.of("ml", "cat", "a:b"); - NameIdentifier stored = - RelationalSchemaNamingBridge.nameIdentifierForStorage(ident, Entity.EntityType.SCHEMA); - assertEquals("a" + P + "b", stored.name()); - } - - @Test - public void schemaIdentToApi() { - NameIdentifier ident = NameIdentifier.of("ml", "cat", "a" + P + "b"); - NameIdentifier api = - RelationalSchemaNamingBridge.nameIdentifierForApi(ident, Entity.EntityType.SCHEMA); - assertEquals("a:b", api.name()); - } - - @Test - public void flatSchemaIdent() { - NameIdentifier ident = NameIdentifier.of("ml", "cat", "flat"); - assertEquals( - "flat", - RelationalSchemaNamingBridge.nameIdentifierForStorage(ident, Entity.EntityType.SCHEMA) - .name()); - assertEquals( - "flat", - RelationalSchemaNamingBridge.nameIdentifierForApi(ident, Entity.EntityType.SCHEMA).name()); - } - - // ------------------------------------------------------------------------- - // wrapperUpdater – SCHEMA - // ------------------------------------------------------------------------- - - @Test - public void wrapperUpdaterSchemaRoundTrip() throws Exception { - mockSeparator(":"); - AuditInfo audit = AuditInfo.builder().withCreator("test").withCreateTime(Instant.now()).build(); - SchemaEntity storageEntity = - SchemaEntity.builder() - .withId(1L) - .withName("a" + P + "b") - .withNamespace(Namespace.of("ml", "cat")) - .withComment("c0") - .withProperties(null) - .withAuditInfo(audit) - .build(); - - var wrapped = - RelationalSchemaNamingBridge.wrapperUpdater( - Entity.EntityType.SCHEMA, - e -> - SchemaEntity.builder() - .withId(e.id()) - .withName(e.name()) - .withNamespace(e.namespace()) - .withComment("c1") - .withProperties(e.properties()) - .withAuditInfo(e.auditInfo()) - .build()); - - SchemaEntity out = wrapped.apply(storageEntity); - assertEquals("a" + P + "b", out.name()); - assertEquals("c1", out.comment()); - - final String[] seenLogicalName = new String[1]; - var wrappedPassThrough = - RelationalSchemaNamingBridge.wrapperUpdater( - Entity.EntityType.SCHEMA, - e -> { - seenLogicalName[0] = e.name(); - return e; - }); - wrappedPassThrough.apply(storageEntity); - assertEquals("a:b", seenLogicalName[0]); - } - - // ------------------------------------------------------------------------- - // genericEntityMetadataFullNameForApi - // ------------------------------------------------------------------------- - - @Test - public void genericEntityApiTable() { - String physicalName = "catalog.a" + P + "b.mytable"; - GenericEntity entity = - GenericEntity.builder() - .withId(1L) - .withEntityType(Entity.EntityType.TABLE) - .withName(physicalName) - .withNamespace(Namespace.of("ml")) - .build(); - GenericEntity result = RelationalSchemaNamingBridge.genericEntityMetadataFullNameForApi(entity); - assertEquals("catalog.a:b.mytable", result.name()); - } - - @Test - public void genericEntityApiSchema() { - String physicalName = "catalog.a" + P + "b" + P + "c"; - GenericEntity entity = - GenericEntity.builder() - .withId(2L) - .withEntityType(Entity.EntityType.SCHEMA) - .withName(physicalName) - .withNamespace(Namespace.of("ml")) - .build(); - GenericEntity result = RelationalSchemaNamingBridge.genericEntityMetadataFullNameForApi(entity); - assertEquals("catalog.a:b:c", result.name()); - } - - @Test - public void genericEntityApiFlatSchema() { - // Schema name with no physical separator → no conversion; same instance returned - String flatName = "catalog.flat"; - GenericEntity entity = - GenericEntity.builder() - .withId(3L) - .withEntityType(Entity.EntityType.SCHEMA) - .withName(flatName) - .withNamespace(Namespace.of("ml")) - .build(); - GenericEntity result = RelationalSchemaNamingBridge.genericEntityMetadataFullNameForApi(entity); - assertSame(entity, result); - } - - @Test - public void genericEntityApiNullName() { - GenericEntity entity = - GenericEntity.builder() - .withId(4L) - .withEntityType(Entity.EntityType.TABLE) - .withName(null) - .withNamespace(Namespace.of("ml")) - .build(); - GenericEntity result = RelationalSchemaNamingBridge.genericEntityMetadataFullNameForApi(entity); - assertSame(entity, result); - } - - @Test - public void genericEntityApiUnmappedType() { - // Entity types that don't map to MetadataObject.Type (e.g. TABLE_STATISTIC) are returned as-is - GenericEntity entity = - GenericEntity.builder() - .withId(5L) - .withEntityType(Entity.EntityType.TABLE_STATISTIC) - .withName("some.name") - .withNamespace(Namespace.of("ml")) - .build(); - GenericEntity result = RelationalSchemaNamingBridge.genericEntityMetadataFullNameForApi(entity); - assertSame(entity, result); - } - - // ------------------------------------------------------------------------- - // roleEntityForStorage / roleEntityForApi - // ------------------------------------------------------------------------- - - @Test - public void roleWithNestedSchemaSecurableObjectToStorage() { - SecurableObject schemaObj = SecurableObjects.ofSchema(CATALOG_OBJ, "a:b", USE_SCHEMA_PRIVS); - RoleEntity stored = - RelationalSchemaNamingBridge.roleEntityForStorage(buildRole(Lists.newArrayList(schemaObj))); - assertEquals("catalog.a" + P + "b", stored.securableObjects().get(0).fullName()); - } - - @Test - public void roleWithNestedSchemaSecurableObjectToApi() { - SecurableObject schemaObj = - SecurableObjects.parse("catalog.a" + P + "b", MetadataObject.Type.SCHEMA, USE_SCHEMA_PRIVS); - RoleEntity api = - RelationalSchemaNamingBridge.roleEntityForApi(buildRole(Lists.newArrayList(schemaObj))); - assertEquals("catalog.a:b", api.securableObjects().get(0).fullName()); - } - - @Test - public void roleWithFlatSchemaIsUnchanged() { - SecurableObject schemaObj = SecurableObjects.ofSchema(CATALOG_OBJ, "flat", USE_SCHEMA_PRIVS); - RoleEntity role = buildRole(Lists.newArrayList(schemaObj)); - assertSame(role, RelationalSchemaNamingBridge.roleEntityForStorage(role)); - assertSame(role, RelationalSchemaNamingBridge.roleEntityForApi(role)); - } - - @Test - public void roleWithNoSecurableObjectsIsUnchanged() { - RoleEntity role = buildRole(null); - assertSame(role, RelationalSchemaNamingBridge.roleEntityForStorage(role)); - assertSame(role, RelationalSchemaNamingBridge.roleEntityForApi(role)); - } - - @Test - public void roleRoundTrip() { - SecurableObject schemaObj = SecurableObjects.ofSchema(CATALOG_OBJ, "a:b:c", USE_SCHEMA_PRIVS); - RoleEntity original = buildRole(Lists.newArrayList(schemaObj)); - RoleEntity backToApi = - RelationalSchemaNamingBridge.roleEntityForApi( - RelationalSchemaNamingBridge.roleEntityForStorage(original)); - assertEquals( - original.securableObjects().get(0).fullName(), - backToApi.securableObjects().get(0).fullName()); - } - - // ------------------------------------------------------------------------- - // Helpers - // ------------------------------------------------------------------------- - - private static RoleEntity buildRole(List securableObjects) { - AuditInfo audit = AuditInfo.builder().withCreator("test").withCreateTime(Instant.now()).build(); - return RoleEntity.builder() - .withId(1L) - .withName("role1") - .withNamespace(Namespace.of("ml")) - .withProperties(null) - .withAuditInfo(audit) - .withSecurableObjects(securableObjects) - .build(); - } - - private static void mockSeparator(String sep) throws Exception { - Config config = Mockito.mock(Config.class); - Mockito.when(config.get(Configs.SCHEMA_SEPARATOR)).thenReturn(sep); - FieldUtils.writeField(GravitinoEnv.getInstance(), "config", config, true); - } -} From 993e482fe918e3b31281df176a220645993b2163 Mon Sep 17 00:00:00 2001 From: Rory Date: Wed, 13 May 2026 23:24:02 +0800 Subject: [PATCH 14/44] revert change --- .../HierarchicalSchemaRelationalBackend.java | 24 ------------------- .../LogicalSchemaNamingRelationalBackend.java | 1 - .../relational/RelationalEntityStore.java | 6 ++--- .../relational/BackendTestExtension.java | 4 ++-- 4 files changed, 4 insertions(+), 31 deletions(-) delete mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/HierarchicalSchemaRelationalBackend.java delete mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/LogicalSchemaNamingRelationalBackend.java diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/HierarchicalSchemaRelationalBackend.java b/core/src/main/java/org/apache/gravitino/storage/relational/HierarchicalSchemaRelationalBackend.java deleted file mode 100644 index 40ef0a053f0..00000000000 --- a/core/src/main/java/org/apache/gravitino/storage/relational/HierarchicalSchemaRelationalBackend.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gravitino.storage.relational; - -/** - * Default JDBC-backed {@link RelationalBackend}. Extends {@link JDBCBackend} without altering - * identifiers or entity payloads so the relational meta services see the same names as callers. - */ -public class HierarchicalSchemaRelationalBackend extends JDBCBackend {} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/LogicalSchemaNamingRelationalBackend.java b/core/src/main/java/org/apache/gravitino/storage/relational/LogicalSchemaNamingRelationalBackend.java deleted file mode 100644 index 8b137891791..00000000000 --- a/core/src/main/java/org/apache/gravitino/storage/relational/LogicalSchemaNamingRelationalBackend.java +++ /dev/null @@ -1 +0,0 @@ - diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStore.java b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStore.java index e50fd475074..27381659d67 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStore.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStore.java @@ -55,15 +55,13 @@ /** * Relation store to store entities. This means we can store entities in a relational store. I.e., * MySQL, PostgreSQL, etc. If you want to use a different backend, you can implement the {@link - * RelationalBackend} interface. The default JDBC backend is {@link - * HierarchicalSchemaRelationalBackend}. + * RelationalBackend} interface. The default JDBC backend is {@link JDBCBackend}. */ public class RelationalEntityStore implements EntityStore, SupportsRelationOperations { private static final Logger LOGGER = LoggerFactory.getLogger(RelationalEntityStore.class); public static final ImmutableMap RELATIONAL_BACKENDS = ImmutableMap.of( - Configs.DEFAULT_ENTITY_RELATIONAL_STORE, - HierarchicalSchemaRelationalBackend.class.getCanonicalName()); + Configs.DEFAULT_ENTITY_RELATIONAL_STORE, JDBCBackend.class.getCanonicalName()); private RelationalBackend backend; private RelationalGarbageCollector garbageCollector; private EntityCache cache; diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/BackendTestExtension.java b/core/src/test/java/org/apache/gravitino/storage/relational/BackendTestExtension.java index 2111826bfac..0920052ff70 100644 --- a/core/src/test/java/org/apache/gravitino/storage/relational/BackendTestExtension.java +++ b/core/src/test/java/org/apache/gravitino/storage/relational/BackendTestExtension.java @@ -183,7 +183,7 @@ private RelationalBackend startBackend(String type) throws Exception { FieldUtils.writeField( GravitinoEnv.getInstance(), "idGenerator", RandomIdGenerator.INSTANCE, true); - RelationalBackend backend = new HierarchicalSchemaRelationalBackend(); + RelationalBackend backend = new JDBCBackend(); if ("mysql".equals(type)) { String url = baseIT.startAndInitMySQLBackend(); Mockito.when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_URL)).thenReturn(url); @@ -225,7 +225,7 @@ private RelationalBackend startBackend(String type) throws Exception { } // A simple Wrapper solves the H2 file cleanup issue. - public static class H2BackendWrapper extends HierarchicalSchemaRelationalBackend { + public static class H2BackendWrapper extends JDBCBackend { private final String path; public H2BackendWrapper(Config config, String path) { From ae98cf386253e13cbdd92ffa0379c0845770f4ab Mon Sep 17 00:00:00 2001 From: Rory Date: Wed, 13 May 2026 23:31:03 +0800 Subject: [PATCH 15/44] revert --- .../RelationalEntityStoreIdResolver.java | 3 +-- .../relational/BackendTestExtension.java | 1 - .../relational/TestJDBCBackendBatchGet.java | 18 ------------------ 3 files changed, 1 insertion(+), 21 deletions(-) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStoreIdResolver.java b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStoreIdResolver.java index f522cb7085a..895572719a0 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStoreIdResolver.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStoreIdResolver.java @@ -169,13 +169,12 @@ private NamespacedEntityId getEntityIdsRequiringMetalakeId( private NamespacedEntityId getEntityIdsRequiringSchemaIds( NameIdentifier nameIdentifier, Entity.EntityType type) { - NameIdentifier schemaIdent = NameIdentifierUtil.getSchemaIdentifier(nameIdentifier); SchemaIds schemaIds = SchemaMetaService.getInstance() .getSchemaIdByMetalakeNameAndCatalogNameAndSchemaName( NameIdentifierUtil.getMetalake(nameIdentifier), NameIdentifierUtil.getCatalogIdentifier(nameIdentifier).name(), - schemaIdent.name()); + NameIdentifierUtil.getSchemaIdentifier(nameIdentifier).name()); switch (type) { case SCHEMA: diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/BackendTestExtension.java b/core/src/test/java/org/apache/gravitino/storage/relational/BackendTestExtension.java index 0920052ff70..a9377c8b599 100644 --- a/core/src/test/java/org/apache/gravitino/storage/relational/BackendTestExtension.java +++ b/core/src/test/java/org/apache/gravitino/storage/relational/BackendTestExtension.java @@ -177,7 +177,6 @@ private RelationalBackend startBackend(String type) throws Exception { .thenReturn(DEFAULT_RELATIONAL_JDBC_BACKEND_MAX_WAIT_MILLISECONDS); Mockito.when(config.get(CACHE_ENABLED)).thenReturn(true); - Mockito.when(config.get(Configs.SCHEMA_SEPARATOR)).thenReturn(":"); FieldUtils.writeField(GravitinoEnv.getInstance(), "config", config, true); FieldUtils.writeField( diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/TestJDBCBackendBatchGet.java b/core/src/test/java/org/apache/gravitino/storage/relational/TestJDBCBackendBatchGet.java index 4b0db612de0..a7410990b37 100644 --- a/core/src/test/java/org/apache/gravitino/storage/relational/TestJDBCBackendBatchGet.java +++ b/core/src/test/java/org/apache/gravitino/storage/relational/TestJDBCBackendBatchGet.java @@ -29,7 +29,6 @@ import org.apache.gravitino.Entity; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; -import org.apache.gravitino.SupportsRelationOperations; import org.apache.gravitino.job.JobHandle; import org.apache.gravitino.job.JobTemplate; import org.apache.gravitino.meta.BaseMetalake; @@ -1063,21 +1062,4 @@ public void testBatchGetViewsPartialResults() throws IOException { Assertions.assertEquals(1, result.size()); Assertions.assertEquals("view1", result.get(0).name()); } - - @TestTemplate - public void testBatchInsertRelationsRejectsUnsupportedRelationType() { - NameIdentifier src = NameIdentifier.of("metalake", "schema1"); - NameIdentifier dst = NameIdentifier.of("metalake", "role1"); - - Assertions.assertThrows( - IllegalArgumentException.class, - () -> - backend.batchInsertRelations( - SupportsRelationOperations.Type.TAG_METADATA_OBJECT_REL, - Lists.newArrayList(src), - Entity.EntityType.SCHEMA, - dst, - Entity.EntityType.ROLE, - false)); - } } From 7cfe982a1746237dfcbbc30cde7ab9dae376c737 Mon Sep 17 00:00:00 2001 From: Rory Date: Thu, 14 May 2026 02:02:57 +0800 Subject: [PATCH 16/44] fix --- .../RequireSchemaConventionService.java | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/service/RequireSchemaConventionService.java diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/RequireSchemaConventionService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/RequireSchemaConventionService.java new file mode 100644 index 00000000000..e7ca994ca76 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/RequireSchemaConventionService.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.service; + +import java.util.List; +import org.apache.commons.lang3.StringUtils; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.Namespace; +import org.apache.gravitino.utils.HierarchicalSchemaUtil; + +/** + * Base for relational metadata services whose JDBC keys use hierarchical schema storage names. + * + *

API identifiers use the configured logical schema separator; persisted rows use the internal + * physical separator. Subclasses implement {@link #fetchPOByStorageIdentifier} and {@link + * #fetchPOsByStorageNamespace} after {@link #getPOForApiIdentifier} / {@link #listPOsForApiNamespace} + * rewrite identifiers or namespaces as needed. + * + * @param persistent object type loaded for the entity + */ +public abstract class RequireSchemaConventionService { + + protected RequireSchemaConventionService() {} + + /** Loads a PO using an identifier whose naming is already in storage form. */ + protected abstract E fetchPOByStorageIdentifier(NameIdentifier storageIdentifier); + + /** Lists POs under a namespace whose naming is already in storage form where applicable. */ + protected abstract List fetchPOsByStorageNamespace(Namespace storageNamespace); + + /** Converts hierarchical schema naming then loads the PO. */ + protected final E getPOForApiIdentifier(NameIdentifier apiIdentifier) { + return fetchPOByStorageIdentifier(apiIdentifierToStorage(apiIdentifier)); + } + + /** Converts hierarchical schema naming then lists POs under the namespace. */ + protected final List listPOsForApiNamespace(Namespace apiNamespace) { + return fetchPOsByStorageNamespace(apiNamespaceToStorage(apiNamespace)); + } + + private static NameIdentifier apiIdentifierToStorage(NameIdentifier apiIdentifier) { + String[] levels = apiIdentifier.namespace().levels(); + if (levels.length == 2) { + String rawName = apiIdentifier.name(); + String storageName = + StringUtils.isNotBlank(rawName) + ? HierarchicalSchemaUtil.logicalToPhysical( + rawName, HierarchicalSchemaUtil.schemaSeparator()) + : rawName; + if (storageName.equals(apiIdentifier.name())) { + return apiIdentifier; + } + return NameIdentifier.of(apiIdentifier.namespace(), storageName); + } + if (levels.length == 3) { + String rawSeg = levels[2]; + String storageSchema = + StringUtils.isNotBlank(rawSeg) + ? HierarchicalSchemaUtil.logicalToPhysical( + rawSeg, HierarchicalSchemaUtil.schemaSeparator()) + : rawSeg; + if (storageSchema.equals(levels[2])) { + return apiIdentifier; + } + return NameIdentifier.of( + Namespace.of(levels[0], levels[1], storageSchema), apiIdentifier.name()); + } + return apiIdentifier; + } + + private static Namespace apiNamespaceToStorage(Namespace apiNamespace) { + String[] levels = apiNamespace.levels(); + if (levels.length != 3) { + return apiNamespace; + } + String rawSeg = levels[2]; + String storageSchema = + StringUtils.isNotBlank(rawSeg) + ? HierarchicalSchemaUtil.logicalToPhysical( + rawSeg, HierarchicalSchemaUtil.schemaSeparator()) + : rawSeg; + if (storageSchema.equals(levels[2])) { + return apiNamespace; + } + return Namespace.of(levels[0], levels[1], storageSchema); + } +} From 26319b726200644c3de2ef2fc5d732cfeecc6611 Mon Sep 17 00:00:00 2001 From: Rory Date: Thu, 14 May 2026 02:18:53 +0800 Subject: [PATCH 17/44] cache --- .../relational/service/TableMetaService.java | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java index 02c094a5092..5f0a68cb5d0 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java @@ -58,7 +58,7 @@ import org.apache.gravitino.utils.NamespaceUtil; /** The service class for table metadata. It provides the basic database operations for table. */ -public class TableMetaService { +public class TableMetaService extends RequireSchemaConventionService { private static final TableMetaService INSTANCE = new TableMetaService(); public static TableMetaService getInstance() { @@ -466,12 +466,22 @@ private TablePO getTablePOByFullQualifiedName(NameIdentifier identifier) { private Function> tableListFetcher() { return GravitinoEnv.getInstance().cacheEnabled() ? this::listTablePOsBySchemaId - : this::listTablePOsByFullQualifiedName; + : this::listPOsForApiNamespace; } private Function tablePOFetcher() { return GravitinoEnv.getInstance().cacheEnabled() ? this::getTablePOBySchemaId - : this::getTablePOByFullQualifiedName; + : this::getPOForApiIdentifier; } + + @Override + protected TablePO fetchPOByStorageIdentifier(NameIdentifier storageIdentifier) { + return getTablePOByFullQualifiedName(storageIdentifier); + } + + @Override + protected List fetchPOsByStorageNamespace(Namespace storageNamespace) { + return listTablePOsByFullQualifiedName(storageNamespace); + } } From dbcf0969ef038df3719c2945df9e1b3330ba6e1f Mon Sep 17 00:00:00 2001 From: Rory Date: Thu, 14 May 2026 02:23:40 +0800 Subject: [PATCH 18/44] cache --- .../service/FunctionMetaService.java | 36 ++++++++++++------- .../relational/service/ViewMetaService.java | 35 +++++++++++------- 2 files changed, 46 insertions(+), 25 deletions(-) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionMetaService.java index 0377afed002..78f184544ef 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionMetaService.java @@ -55,7 +55,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class FunctionMetaService { +public class FunctionMetaService extends RequireSchemaConventionService { public static FunctionMetaService getInstance() { return INSTANCE; } @@ -270,12 +270,6 @@ public FunctionEntity updateFunction( } } - private Function functionPOFetcher() { - return GravitinoEnv.getInstance().cacheEnabled() - ? this::getFunctionPOBySchemaId - : this::getFunctionPOByFullQualifiedName; - } - private FunctionPO getFunctionPOBySchemaId(NameIdentifier ident) { Long schemaId = EntityIdService.getEntityId( @@ -332,12 +326,6 @@ private List listFunctionPOsBySchemaId(Namespace namespace) { FunctionMetaMapper.class, mapper -> mapper.listFunctionPOsBySchemaId(schemaId)); } - private Function> functionListFetcher() { - return GravitinoEnv.getInstance().cacheEnabled() - ? this::listFunctionPOsBySchemaId - : this::listFunctionPOsByFullQualifiedName; - } - private List listFunctionPOsByFullQualifiedName(Namespace namespace) { String[] namespaceLevels = namespace.levels(); List functionPOs = @@ -355,6 +343,28 @@ private List listFunctionPOsByFullQualifiedName(Namespace namespace) return functionPOs.stream().filter(po -> po.functionId() != null).collect(Collectors.toList()); } + private Function> functionListFetcher() { + return GravitinoEnv.getInstance().cacheEnabled() + ? this::listFunctionPOsBySchemaId + : this::listPOsForApiNamespace; + } + + private Function functionPOFetcher() { + return GravitinoEnv.getInstance().cacheEnabled() + ? this::getFunctionPOBySchemaId + : this::getPOForApiIdentifier; + } + + @Override + protected FunctionPO fetchPOByStorageIdentifier(NameIdentifier storageIdentifier) { + return getFunctionPOByFullQualifiedName(storageIdentifier); + } + + @Override + protected List fetchPOsByStorageNamespace(Namespace storageNamespace) { + return listFunctionPOsByFullQualifiedName(storageNamespace); + } + private void fillFunctionPOBuilderParentEntityId( FunctionPO.FunctionPOBuilder builder, Namespace ns) { NamespaceUtil.checkFunction(ns); diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewMetaService.java index 177118c8118..6fe6ea0abf2 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewMetaService.java @@ -54,7 +54,7 @@ import org.apache.gravitino.utils.NamespaceUtil; /** The service class for view metadata. It provides the basic database operations for view. */ -public class ViewMetaService { +public class ViewMetaService extends RequireSchemaConventionService { private static final ViewMetaService INSTANCE = new ViewMetaService(); @@ -87,7 +87,7 @@ public Long getViewIdBySchemaIdAndName(Long schemaId, String viewName) { baseMetricName = "listViewsByNamespace") public List listViewsByNamespace(Namespace namespace) { NamespaceUtil.checkView(namespace); - List viewPOs = listViewPOsByNamespace(namespace); + List viewPOs = listViewPOs(namespace); return viewPOs.stream().map(po -> fromViewPO(po, namespace)).collect(Collectors.toList()); } @@ -95,8 +95,7 @@ public List listViewsByNamespace(Namespace namespace) { metricsSource = GRAVITINO_RELATIONAL_STORE_METRIC_NAME, baseMetricName = "getViewByIdentifier") public ViewEntity getViewByIdentifier(NameIdentifier identifier) { - NameIdentifierUtil.checkView(identifier); - ViewPO viewPO = viewPOFetcher().apply(identifier); + ViewPO viewPO = getViewPOByIdentifier(identifier); return fromViewPO(viewPO, identifier.namespace()); } @@ -141,9 +140,7 @@ public void insertView(ViewEntity viewEntity, boolean overwrite) throws IOExcept baseMetricName = "updateViewByIdentifier") public ViewEntity updateView( NameIdentifier ident, Function updater) throws IOException { - NameIdentifierUtil.checkView(ident); - - ViewPO oldViewPO = viewPOFetcher().apply(ident); + ViewPO oldViewPO = getViewPOByIdentifier(ident); ViewEntity oldViewEntity = fromViewPO(oldViewPO, ident.namespace()); ViewEntity newEntity = (ViewEntity) updater.apply((E) oldViewEntity); Preconditions.checkArgument( @@ -203,8 +200,7 @@ public ViewEntity updateView( metricsSource = GRAVITINO_RELATIONAL_STORE_METRIC_NAME, baseMetricName = "deleteViewByIdentifier") public boolean deleteView(NameIdentifier ident) { - NameIdentifierUtil.checkView(ident); - ViewPO viewPO = viewPOFetcher().apply(ident); + ViewPO viewPO = getViewPOByIdentifier(ident); String metalakeName = ident.namespace().level(0); String catalogName = ident.namespace().level(1); String schemaName = ident.namespace().level(2); @@ -287,20 +283,25 @@ private ViewPO updateViewPO(ViewPO oldViewPO, ViewEntity newEntity) { return buildViewPO(newEntity, builder, newVersion.intValue()); } - private List listViewPOsByNamespace(Namespace namespace) { + private ViewPO getViewPOByIdentifier(NameIdentifier identifier) { + NameIdentifierUtil.checkView(identifier); + return viewPOFetcher().apply(identifier); + } + + private List listViewPOs(Namespace namespace) { return viewListFetcher().apply(namespace); } private Function> viewListFetcher() { return GravitinoEnv.getInstance().cacheEnabled() ? this::listViewPOsBySchemaId - : this::listViewPOsByFullQualifiedName; + : this::listPOsForApiNamespace; } private Function viewPOFetcher() { return GravitinoEnv.getInstance().cacheEnabled() ? this::getViewPOBySchemaId - : this::getViewPOByFullQualifiedName; + : this::getPOForApiIdentifier; } private List listViewPOsBySchemaId(Namespace namespace) { @@ -371,4 +372,14 @@ private ViewPO getViewPOByFullQualifiedName(NameIdentifier identifier) { return viewPO; } + + @Override + protected ViewPO fetchPOByStorageIdentifier(NameIdentifier storageIdentifier) { + return getViewPOByFullQualifiedName(storageIdentifier); + } + + @Override + protected List fetchPOsByStorageNamespace(Namespace storageNamespace) { + return listViewPOsByFullQualifiedName(storageNamespace); + } } From 29cb2ace3a4fed03d6b98ad01fe4bdd0b9f2fd70 Mon Sep 17 00:00:00 2001 From: Rory Date: Thu, 14 May 2026 16:52:26 +0800 Subject: [PATCH 19/44] refactor --- .../relational/service/BasePOStorageOps.java | 112 +++++++++++++ .../RequireSchemaConventionService.java | 4 +- .../relational/service/SchemaMetaService.java | 154 +++--------------- .../service/SchemaPOStorageOps.java | 99 +++++++++++ .../relational/service/TableMetaService.java | 16 +- 5 files changed, 243 insertions(+), 142 deletions(-) create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/service/BasePOStorageOps.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaPOStorageOps.java diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/BasePOStorageOps.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/BasePOStorageOps.java new file mode 100644 index 00000000000..2e351521a7f --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/BasePOStorageOps.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.service; + +import java.util.List; +import org.apache.gravitino.Entity; +import org.apache.gravitino.GravitinoEnv; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.Namespace; + +public abstract class BasePOStorageOps { + public void insertPO(Mapper mapper, PO po, boolean overwrite) { + throw new UnsupportedOperationException(); + } + + public void batchInsertPOs(Mapper mapper, List pos, boolean overwrite) { + throw new UnsupportedOperationException(); + } + + public Integer updatePO(Mapper mapper, PO newPO, PO oldPO) { + throw new UnsupportedOperationException(); + } + + public final PO getPO(Mapper mapper, NameIdentifier identifier) { + if (!capabilities().contains(Capability.GET_BY_NS_UID) + && !capabilities().contains(Capability.GET_BY_NAME)) { + throw new UnsupportedOperationException(); + } + + if (GravitinoEnv.getInstance().cacheEnabled() + && capabilities().contains(Capability.GET_BY_NS_UID)) { + Long parentId = + EntityIdService.getEntityId( + NameIdentifier.of(identifier.namespace().toString()), entityType()); + return getPO(mapper, parentId, identifier.name()); + } + + return getPOByFullName(mapper, identifier); + } + + public final List listPOs(Mapper mapper, Namespace namespace) { + if (!capabilities().contains(Capability.LIST_BY_NS_UID) + && !capabilities().contains(Capability.LIST_BY_NS_NAME)) { + throw new UnsupportedOperationException(); + } + + if (GravitinoEnv.getInstance().cacheEnabled() + && capabilities().contains(Capability.LIST_BY_NS_UID)) { + Long parentId = + EntityIdService.getEntityId(NameIdentifier.of(namespace.toString()), entityType()); + return listPOs(mapper, parentId); + } + + return listPOsByNSFullName(mapper, namespace); + } + + public PO getPO(Mapper mapper, Long parentId, String name) { + throw new UnsupportedOperationException(); + } + + public List listPOs(Mapper mapper, Long parentId) { + throw new UnsupportedOperationException(); + } + + public List listPOs(Mapper mapper, Namespace namespace, List names) { + throw new UnsupportedOperationException(); + } + + public List listPOs(Mapper mapper, List uuids) { + throw new UnsupportedOperationException(); + } + + protected PO getPOByFullName(Mapper mapper, NameIdentifier identifier) { + throw new UnsupportedOperationException(); + } + + protected List listPOsByNSFullName(Mapper mapper, Namespace namespace) { + throw new UnsupportedOperationException(); + } + + public abstract List capabilities(); + + protected abstract Entity.EntityType entityType(); + + public enum Capability { + INSERT, + BATCH_INSERT, + UPDATE, + GET_BY_NAME, + GET_BY_NS_UID, + LIST_BY_NS_UID, + LIST_BY_NS_NAME, + LIST_BY_NAME_FILTER, + LIST_BY_UID_FILTER + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/RequireSchemaConventionService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/RequireSchemaConventionService.java index e7ca994ca76..c5c76cf2596 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/RequireSchemaConventionService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/RequireSchemaConventionService.java @@ -29,8 +29,8 @@ * *

API identifiers use the configured logical schema separator; persisted rows use the internal * physical separator. Subclasses implement {@link #fetchPOByStorageIdentifier} and {@link - * #fetchPOsByStorageNamespace} after {@link #getPOForApiIdentifier} / {@link #listPOsForApiNamespace} - * rewrite identifiers or namespaces as needed. + * #fetchPOsByStorageNamespace} after {@link #getPOForApiIdentifier} / {@link + * #listPOsForApiNamespace} rewrite identifiers or namespaces as needed. * * @param persistent object type loaded for the entity */ diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java index c7edb6d26f5..7406d8b9b42 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java @@ -33,7 +33,6 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; import org.apache.gravitino.Entity; -import org.apache.gravitino.Entity.EntityType; import org.apache.gravitino.GravitinoEnv; import org.apache.gravitino.HasIdentifier; import org.apache.gravitino.MetadataObject; @@ -80,29 +79,14 @@ /** The service class for schema metadata. It provides the basic database operations for schema. */ public class SchemaMetaService { private static final SchemaMetaService INSTANCE = new SchemaMetaService(); + private SchemaPOStorageOps ops; public static SchemaMetaService getInstance() { return INSTANCE; } - private SchemaMetaService() {} - - @Monitored( - metricsSource = GRAVITINO_RELATIONAL_STORE_METRIC_NAME, - baseMetricName = "getSchemaPOByCatalogIdAndName") - public SchemaPO getSchemaPOByCatalogIdAndName(Long catalogId, String schemaName) { - SchemaPO schemaPO = - SessionUtils.getWithoutCommit( - SchemaMetaMapper.class, - mapper -> mapper.selectSchemaMetaByCatalogIdAndName(catalogId, schemaName)); - - if (schemaPO == null) { - throw new NoSuchEntityException( - NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, - Entity.EntityType.SCHEMA.name().toLowerCase(), - schemaName); - } - return schemaPO; + private SchemaMetaService() { + this.ops = new SchemaPOStorageOps(); } @Monitored( @@ -110,39 +94,19 @@ public SchemaPO getSchemaPOByCatalogIdAndName(Long catalogId, String schemaName) baseMetricName = "getSchemaIdByMetalakeNameAndCatalogNameAndSchemaName") public SchemaIds getSchemaIdByMetalakeNameAndCatalogNameAndSchemaName( String metalakeName, String catalogName, String schemaName) { - SchemaIds schemaIds = + NameIdentifier identifier = NameIdentifier.of(metalakeName, catalogName, schemaName); + SchemaPO schemaPO = SessionUtils.getWithoutCommit( - SchemaMetaMapper.class, - mapper -> - mapper.selectSchemaIdByMetalakeNameAndCatalogNameAndSchemaName( - metalakeName, catalogName, schemaName)); + SchemaMetaMapper.class, mapper -> ops.getPOByFullName(mapper, identifier)); - if (schemaIds == null) { + if (schemaPO == null) { throw new NoSuchEntityException( NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, Entity.EntityType.SCHEMA.name().toLowerCase(), schemaName); } - return schemaIds; - } - - @Monitored( - metricsSource = GRAVITINO_RELATIONAL_STORE_METRIC_NAME, - baseMetricName = "getSchemaIdByCatalogIdAndName") - public Long getSchemaIdByCatalogIdAndName(Long catalogId, String schemaName) { - Long schemaId = - SessionUtils.getWithoutCommit( - SchemaMetaMapper.class, - mapper -> mapper.selectSchemaIdByCatalogIdAndName(catalogId, schemaName)); - - if (schemaId == null) { - throw new NoSuchEntityException( - NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, - Entity.EntityType.SCHEMA.name().toLowerCase(), - schemaName); - } - return schemaId; + return new SchemaIds(schemaPO.getMetalakeId(), schemaPO.getCatalogId(), schemaPO.getSchemaId()); } @Monitored( @@ -230,11 +194,7 @@ public void insertSchema(SchemaEntity schemaEntity, boolean overwrite) throws IO SchemaPO leafPO = POConverters.initializeSchemaPOWithVersion(leafRow, leafBuilder); List schemaPosToInsert = new ArrayList<>(missingAncestorPOs); schemaPosToInsert.add(leafPO); - if (overwrite) { - mapper.batchInsertSchemaMetaOnDuplicateKeyUpdate(schemaPosToInsert); - } else { - mapper.batchInsertSchemaMeta(schemaPosToInsert); - } + ops.batchInsertPOs(mapper, schemaPosToInsert, overwrite); }); } catch (RuntimeException re) { ExceptionUtils.checkSQLException( @@ -271,7 +231,8 @@ public SchemaEntity updateSchema( SessionUtils.getWithoutCommit( SchemaMetaMapper.class, mapper -> - mapper.updateSchemaMeta( + ops.updatePO( + mapper, POConverters.updateSchemaPOWithVersion(oldSchemaPO, newEntity), oldSchemaPO))), () -> { @@ -497,93 +458,20 @@ private SchemaPO getSchemaPOByIdentifier(NameIdentifier identifier) { return schemaPOFetcher().apply(identifier); } - private SchemaPO getSchemaByFullQualifiedName( - String metalakeName, String catalogName, String schemaName) { - SchemaPO schemaPO = - SessionUtils.getWithoutCommit( - SchemaMetaMapper.class, - mapper -> - mapper.selectSchemaByFullQualifiedName(metalakeName, catalogName, schemaName)); - if (schemaPO == null) { - throw new NoSuchEntityException( - NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, - EntityType.CATALOG.name().toLowerCase(), - schemaName); - } - - return schemaPO; - } - private List listSchemaPOs(Namespace namespace) { return schemaListFetcher().apply(namespace); } - private List listSchemaPOsByCatalogId(Namespace namespace) { - Long catalogId = - EntityIdService.getEntityId( - NameIdentifier.of(namespace.levels()), Entity.EntityType.CATALOG); - - return SessionUtils.getWithoutCommit( - SchemaMetaMapper.class, mapper -> mapper.listSchemaPOsByCatalogId(catalogId)); - } - - private List listSchemaPOsByFullQualifiedName(Namespace namespace) { - String[] namespaceLevels = namespace.levels(); - List schemaPOs = - SessionUtils.getWithoutCommit( - SchemaMetaMapper.class, - mapper -> - mapper.listSchemaPOsByFullQualifiedName(namespaceLevels[0], namespaceLevels[1])); - if (schemaPOs.isEmpty() || schemaPOs.get(0).getCatalogId() == null) { - throw new NoSuchEntityException( - NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, - Entity.EntityType.CATALOG.name().toLowerCase(), - namespaceLevels[1]); - } - return schemaPOs.stream().filter(po -> po.getSchemaId() != null).collect(Collectors.toList()); - } - - private SchemaPO getSchemaPOByCatalogId(NameIdentifier identifier) { - Long catalogId = - EntityIdService.getEntityId( - NameIdentifier.of(identifier.namespace().levels()), Entity.EntityType.CATALOG); - return getSchemaPOByCatalogIdAndName(catalogId, identifier.name()); - } - - private SchemaPO getSchemaPOByFullQualifiedName(NameIdentifier identifier) { - String[] namespaceLevels = identifier.namespace().levels(); - SchemaPO schemaPO = - getSchemaByFullQualifiedName(namespaceLevels[0], namespaceLevels[1], identifier.name()); - - if (schemaPO.getCatalogId() == null) { - throw new NoSuchEntityException( - NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, - Entity.EntityType.CATALOG.name().toLowerCase(), - namespaceLevels[1]); - } - - if (schemaPO.getSchemaId() == null) { - throw new NoSuchEntityException( - NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, - Entity.EntityType.SCHEMA.name().toLowerCase(), - identifier.name()); - } - - return schemaPO; - } - private Function> schemaListFetcher() { - // If cache is enabled, we can use catalog id to fetch schemas faster or else use full qualified - // name to join several tables to get the schema list. - return GravitinoEnv.getInstance().cacheEnabled() - ? this::listSchemaPOsByCatalogId - : this::listSchemaPOsByFullQualifiedName; + return namespace -> + SessionUtils.getWithoutCommit( + SchemaMetaMapper.class, mapper -> ops.listPOs(mapper, namespace)); } private Function schemaPOFetcher() { - return GravitinoEnv.getInstance().cacheEnabled() - ? this::getSchemaPOByCatalogId - : this::getSchemaPOByFullQualifiedName; + return identifier -> + SessionUtils.getWithoutCommit( + SchemaMetaMapper.class, mapper -> ops.getPO(mapper, identifier)); } private void fillSchemaPOBuilderParentEntityId(SchemaPO.Builder builder, Namespace namespace) { @@ -605,12 +493,14 @@ public List batchGetSchemaByIdentifier(List identi List schemaNames = identifiers.stream().map(NameIdentifier::name).collect(Collectors.toList()); - return SessionUtils.doWithCommitAndFetchResult( + return SessionUtils.getWithoutCommit( SchemaMetaMapper.class, mapper -> { List schemaPOs = - mapper.batchSelectSchemaByIdentifier( - catalogIdent.namespace().level(0), catalogIdent.name(), schemaNames); + ops.listPOs( + mapper, + Namespace.of(catalogIdent.namespace().levels()[0], catalogIdent.name()), + schemaNames); return POConverters.fromSchemaPOs(schemaPOs, firstIdent.namespace()); }); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaPOStorageOps.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaPOStorageOps.java new file mode 100644 index 00000000000..4f2175173ce --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaPOStorageOps.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.service; + +import java.util.List; +import org.apache.gravitino.Entity; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.Namespace; +import org.apache.gravitino.storage.relational.mapper.SchemaMetaMapper; +import org.apache.gravitino.storage.relational.po.SchemaPO; + +public class SchemaPOStorageOps extends BasePOStorageOps { + + public SchemaPOStorageOps() {} + + @Override + public void batchInsertPOs(SchemaMetaMapper mapper, List schemaPOs, boolean overwrite) { + if (overwrite) { + mapper.batchInsertSchemaMetaOnDuplicateKeyUpdate(schemaPOs); + } else { + mapper.batchInsertSchemaMeta(schemaPOs); + } + } + + @Override + public Integer updatePO(SchemaMetaMapper mapper, SchemaPO oldPO, SchemaPO newPO) { + return mapper.updateSchemaMeta(oldPO, newPO); + } + + @Override + public SchemaPO getPO(SchemaMetaMapper mapper, Long parentId, String name) { + return mapper.selectSchemaMetaByCatalogIdAndName(parentId, name); + } + + @Override + protected SchemaPO getPOByFullName(SchemaMetaMapper mapper, NameIdentifier identifier) { + Namespace namespace = identifier.namespace(); + return mapper.selectSchemaByFullQualifiedName( + namespace.level(0), namespace.level(1), identifier.name()); + } + + @Override + public List listPOs(SchemaMetaMapper schemaMetaMapper, Long parentId) { + return schemaMetaMapper.listSchemaPOsByCatalogId(parentId); + } + + @Override + public List listPOs( + SchemaMetaMapper schemaMetaMapper, Namespace namespace, List names) { + return schemaMetaMapper.batchSelectSchemaByIdentifier( + namespace.level(0), namespace.level(1), names); + } + + @Override + public List listPOs(SchemaMetaMapper schemaMetaMapper, List uuids) { + return schemaMetaMapper.listSchemaPOsBySchemaIds(uuids); + } + + @Override + protected List listPOsByNSFullName( + SchemaMetaMapper schemaMetaMapper, Namespace namespace) { + return schemaMetaMapper.listSchemaPOsByFullQualifiedName( + namespace.level(0), namespace.level(1)); + } + + @Override + public List capabilities() { + return List.of( + Capability.BATCH_INSERT, + Capability.UPDATE, + Capability.GET_BY_NAME, + Capability.GET_BY_NS_UID, + Capability.LIST_BY_NS_NAME, + Capability.LIST_BY_NS_UID, + Capability.LIST_BY_NAME_FILTER, + Capability.LIST_BY_UID_FILTER); + } + + @Override + protected Entity.EntityType entityType() { + return Entity.EntityType.SCHEMA; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java index 5f0a68cb5d0..9b0706f1bf2 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java @@ -475,13 +475,13 @@ private Function tablePOFetcher() { : this::getPOForApiIdentifier; } - @Override - protected TablePO fetchPOByStorageIdentifier(NameIdentifier storageIdentifier) { - return getTablePOByFullQualifiedName(storageIdentifier); - } + @Override + protected TablePO fetchPOByStorageIdentifier(NameIdentifier storageIdentifier) { + return getTablePOByFullQualifiedName(storageIdentifier); + } - @Override - protected List fetchPOsByStorageNamespace(Namespace storageNamespace) { - return listTablePOsByFullQualifiedName(storageNamespace); - } + @Override + protected List fetchPOsByStorageNamespace(Namespace storageNamespace) { + return listTablePOsByFullQualifiedName(storageNamespace); + } } From 7b0c466c6b519075a11f705e77df567bd39f9e72 Mon Sep 17 00:00:00 2001 From: Rory Date: Thu, 14 May 2026 17:21:17 +0800 Subject: [PATCH 20/44] fix --- .../relational/service/SchemaMetaService.java | 4 +- .../relational/service/TableMetaService.java | 50 ++++++++----------- 2 files changed, 21 insertions(+), 33 deletions(-) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java index 7406d8b9b42..1f4e552f16a 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java @@ -95,9 +95,7 @@ private SchemaMetaService() { public SchemaIds getSchemaIdByMetalakeNameAndCatalogNameAndSchemaName( String metalakeName, String catalogName, String schemaName) { NameIdentifier identifier = NameIdentifier.of(metalakeName, catalogName, schemaName); - SchemaPO schemaPO = - SessionUtils.getWithoutCommit( - SchemaMetaMapper.class, mapper -> ops.getPOByFullName(mapper, identifier)); + SchemaPO schemaPO = schemaPOFetcher().apply(identifier); if (schemaPO == null) { throw new NoSuchEntityException( diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java index 9b0706f1bf2..98d82fb7f7d 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java @@ -58,9 +58,11 @@ import org.apache.gravitino.utils.NamespaceUtil; /** The service class for table metadata. It provides the basic database operations for table. */ -public class TableMetaService extends RequireSchemaConventionService { +public class TableMetaService { private static final TableMetaService INSTANCE = new TableMetaService(); + private final TablePOStorageOps ops = new TablePOStorageOps(); + public static TableMetaService getInstance() { return INSTANCE; } @@ -74,7 +76,7 @@ public Long getTableIdBySchemaIdAndName(Long schemaId, String tableName) { Long tableId = SessionUtils.getWithoutCommit( TableMetaMapper.class, - mapper -> mapper.selectTableIdBySchemaIdAndName(schemaId, tableName)); + mapper -> ops.getPO(mapper, schemaId, tableName)).getTableId(); if (tableId == null) { throw new NoSuchEntityException( @@ -124,11 +126,7 @@ public void insertTable(TableEntity tableEntity, boolean overwrite) throws IOExc TableMetaMapper.class, mapper -> { tablePORef.set(po); - if (overwrite) { - mapper.insertTableMetaOnDuplicateKeyUpdate(po); - } else { - mapper.insertTableMeta(po); - } + ops.insertPO(mapper, po, overwrite); }), () -> SessionUtils.doWithoutCommit( @@ -202,9 +200,10 @@ public TableEntity updateTable( SessionUtils.doMultipleWithCommit( () -> updateResult.set( - SessionUtils.getWithoutCommit( - TableMetaMapper.class, - mapper -> mapper.updateTableMeta(newTablePO, oldTablePO, newSchemaId))), + SessionUtils.getWithoutCommit( + TableMetaMapper.class, + mapper -> + ops.updatePO(mapper, newTablePO, oldTablePO))), () -> SessionUtils.doWithoutCommit( TableVersionMapper.class, @@ -347,11 +346,11 @@ public List batchGetTableByIdentifier(List identifi Objects.equals(schemaIdent, NameIdentifierUtil.getSchemaIdentifier(ident))); tableNames.add(ident.name()); } - Long schemaId = EntityIdService.getEntityId(schemaIdent, Entity.EntityType.SCHEMA); return SessionUtils.doWithCommitAndFetchResult( TableMetaMapper.class, mapper -> { - List tableList = mapper.batchSelectTableByIdentifier(schemaId, tableNames); + List tableList = + ops.listPOs(mapper, firstIdent.namespace(), tableNames); return POConverters.fromTablePOs(tableList, firstIdent.namespace()); }); } @@ -376,7 +375,7 @@ private TablePO getTablePOBySchemaIdAndName(Long schemaId, String tableName) { TablePO tablePO = SessionUtils.getWithoutCommit( TableMetaMapper.class, - mapper -> mapper.selectTableMetaBySchemaIdAndName(schemaId, tableName)); + mapper -> ops.getPO(mapper, schemaId, tableName)); if (tablePO == null) { throw new NoSuchEntityException( NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, @@ -392,8 +391,10 @@ private TablePO getTableByFullQualifiedName( SessionUtils.getWithoutCommit( TableMetaMapper.class, mapper -> - mapper.selectTableByFullQualifiedName( - metalakeName, catalogName, schemaName, tableName)); + ops.getPOByFullName( + mapper, + NameIdentifier.of( + Namespace.of(metalakeName, catalogName, schemaName), tableName))); if (tablePO == null) { throw new NoSuchEntityException( NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, @@ -413,7 +414,7 @@ private List listTablePOsBySchemaId(Namespace namespace) { EntityIdService.getEntityId( NameIdentifier.of(namespace.levels()), Entity.EntityType.SCHEMA); return SessionUtils.getWithoutCommit( - TableMetaMapper.class, mapper -> mapper.listTablePOsBySchemaId(schemaId)); + TableMetaMapper.class, mapper -> ops.listPOs(mapper, schemaId)); } private List listTablePOsByFullQualifiedName(Namespace namespace) { @@ -422,8 +423,9 @@ private List listTablePOsByFullQualifiedName(Namespace namespace) { SessionUtils.getWithoutCommit( TableMetaMapper.class, mapper -> - mapper.listTablePOsByFullQualifiedName( - namespaceLevels[0], namespaceLevels[1], namespaceLevels[2])); + ops.listPOs( + mapper, + Namespace.of(namespaceLevels[0], namespaceLevels[1], namespaceLevels[2]))); if (tablePOs.isEmpty() || tablePOs.get(0).getSchemaId() == null) { throw new NoSuchEntityException( NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, @@ -470,18 +472,6 @@ private Function> tableListFetcher() { } private Function tablePOFetcher() { - return GravitinoEnv.getInstance().cacheEnabled() - ? this::getTablePOBySchemaId - : this::getPOForApiIdentifier; - } - - @Override - protected TablePO fetchPOByStorageIdentifier(NameIdentifier storageIdentifier) { - return getTablePOByFullQualifiedName(storageIdentifier); - } - @Override - protected List fetchPOsByStorageNamespace(Namespace storageNamespace) { - return listTablePOsByFullQualifiedName(storageNamespace); } } From 74a0a7d56d68e9cec729cda0e48ca78e69dda508 Mon Sep 17 00:00:00 2001 From: roryqi Date: Thu, 14 May 2026 09:46:44 +0000 Subject: [PATCH 21/44] refactor Co-Authored-By: Claude Opus 4.7 --- .../relational/service/TableMetaService.java | 150 +++--------------- .../relational/service/TablePOStorageOps.java | 99 ++++++++++++ 2 files changed, 122 insertions(+), 127 deletions(-) create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/service/TablePOStorageOps.java diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java index 98d82fb7f7d..ab1aae1a306 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java @@ -28,10 +28,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; -import java.util.stream.Collectors; import org.apache.gravitino.Entity; -import org.apache.gravitino.Entity.EntityType; -import org.apache.gravitino.GravitinoEnv; import org.apache.gravitino.HasIdentifier; import org.apache.gravitino.MetadataObject; import org.apache.gravitino.NameIdentifier; @@ -60,31 +57,31 @@ /** The service class for table metadata. It provides the basic database operations for table. */ public class TableMetaService { private static final TableMetaService INSTANCE = new TableMetaService(); - - private final TablePOStorageOps ops = new TablePOStorageOps(); + private TablePOStorageOps ops; public static TableMetaService getInstance() { return INSTANCE; } - private TableMetaService() {} + private TableMetaService() { + this.ops = new TablePOStorageOps(); + } @Monitored( metricsSource = GRAVITINO_RELATIONAL_STORE_METRIC_NAME, baseMetricName = "getTableIdBySchemaIdAndName") public Long getTableIdBySchemaIdAndName(Long schemaId, String tableName) { - Long tableId = + TablePO tablePO = SessionUtils.getWithoutCommit( - TableMetaMapper.class, - mapper -> ops.getPO(mapper, schemaId, tableName)).getTableId(); + TableMetaMapper.class, mapper -> ops.getPO(mapper, schemaId, tableName)); - if (tableId == null) { + if (tablePO == null) { throw new NoSuchEntityException( NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, Entity.EntityType.TABLE.name().toLowerCase(), tableName); } - return tableId; + return tablePO.getTableId(); } @Monitored( @@ -200,10 +197,9 @@ public TableEntity updateTable( SessionUtils.doMultipleWithCommit( () -> updateResult.set( - SessionUtils.getWithoutCommit( - TableMetaMapper.class, - mapper -> - ops.updatePO(mapper, newTablePO, oldTablePO))), + SessionUtils.getWithoutCommit( + TableMetaMapper.class, + mapper -> ops.updatePO(mapper, newTablePO, oldTablePO))), () -> SessionUtils.doWithoutCommit( TableVersionMapper.class, @@ -349,129 +345,29 @@ public List batchGetTableByIdentifier(List identifi return SessionUtils.doWithCommitAndFetchResult( TableMetaMapper.class, mapper -> { - List tableList = - ops.listPOs(mapper, firstIdent.namespace(), tableNames); + List tableList = ops.listPOs(mapper, firstIdent.namespace(), tableNames); return POConverters.fromTablePOs(tableList, firstIdent.namespace()); }); } - private void fillTablePOBuilderParentEntityId(TablePO.Builder builder, Namespace namespace) { - NamespaceUtil.checkTable(namespace); - NamespacedEntityId namespacedEntityId = - EntityIdService.getEntityIds( - NameIdentifier.of(namespace.levels()), Entity.EntityType.SCHEMA); - builder.withMetalakeId(namespacedEntityId.namespaceIds()[0]); - builder.withCatalogId(namespacedEntityId.namespaceIds()[1]); - builder.withSchemaId(namespacedEntityId.entityId()); - } - private TablePO getTablePOByIdentifier(NameIdentifier identifier) { NameIdentifierUtil.checkTable(identifier); - - return tablePOFetcher().apply(identifier); - } - - private TablePO getTablePOBySchemaIdAndName(Long schemaId, String tableName) { - TablePO tablePO = - SessionUtils.getWithoutCommit( - TableMetaMapper.class, - mapper -> ops.getPO(mapper, schemaId, tableName)); - if (tablePO == null) { - throw new NoSuchEntityException( - NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, - Entity.EntityType.TABLE.name().toLowerCase(), - tableName); - } - return tablePO; - } - - private TablePO getTableByFullQualifiedName( - String metalakeName, String catalogName, String schemaName, String tableName) { - TablePO tablePO = - SessionUtils.getWithoutCommit( - TableMetaMapper.class, - mapper -> - ops.getPOByFullName( - mapper, - NameIdentifier.of( - Namespace.of(metalakeName, catalogName, schemaName), tableName))); - if (tablePO == null) { - throw new NoSuchEntityException( - NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, - Entity.EntityType.TABLE.name().toLowerCase(), - tableName); - } - - return tablePO; + return SessionUtils.getWithoutCommit( + TableMetaMapper.class, mapper -> ops.getPO(mapper, identifier)); } private List listTablePOs(Namespace namespace) { - return tableListFetcher().apply(namespace); - } - - private List listTablePOsBySchemaId(Namespace namespace) { - Long schemaId = - EntityIdService.getEntityId( - NameIdentifier.of(namespace.levels()), Entity.EntityType.SCHEMA); return SessionUtils.getWithoutCommit( - TableMetaMapper.class, mapper -> ops.listPOs(mapper, schemaId)); - } - - private List listTablePOsByFullQualifiedName(Namespace namespace) { - String[] namespaceLevels = namespace.levels(); - List tablePOs = - SessionUtils.getWithoutCommit( - TableMetaMapper.class, - mapper -> - ops.listPOs( - mapper, - Namespace.of(namespaceLevels[0], namespaceLevels[1], namespaceLevels[2]))); - if (tablePOs.isEmpty() || tablePOs.get(0).getSchemaId() == null) { - throw new NoSuchEntityException( - NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, - EntityType.SCHEMA.name().toLowerCase(), - namespaceLevels[2]); - } - return tablePOs.stream().filter(po -> po.getTableId() != null).collect(Collectors.toList()); - } - - private TablePO getTablePOBySchemaId(NameIdentifier identifier) { - Long schemaId = - EntityIdService.getEntityId( - NameIdentifier.of(identifier.namespace().levels()), Entity.EntityType.SCHEMA); - return getTablePOBySchemaIdAndName(schemaId, identifier.name()); - } - - private TablePO getTablePOByFullQualifiedName(NameIdentifier identifier) { - String[] namespaceLevels = identifier.namespace().levels(); - TablePO tablePO = - getTableByFullQualifiedName( - namespaceLevels[0], namespaceLevels[1], namespaceLevels[2], identifier.name()); - - if (tablePO.getSchemaId() == null) { - throw new NoSuchEntityException( - NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, - EntityType.SCHEMA.name().toLowerCase(), - namespaceLevels[2]); - } - - if (tablePO.getTableId() == null) { - throw new NoSuchEntityException( - NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, - EntityType.TABLE.name().toLowerCase(), - identifier.name()); - } - - return tablePO; - } - - private Function> tableListFetcher() { - return GravitinoEnv.getInstance().cacheEnabled() - ? this::listTablePOsBySchemaId - : this::listPOsForApiNamespace; + TableMetaMapper.class, mapper -> ops.listPOs(mapper, namespace)); } - private Function tablePOFetcher() { - + private void fillTablePOBuilderParentEntityId(TablePO.Builder builder, Namespace namespace) { + NamespaceUtil.checkTable(namespace); + NamespacedEntityId namespacedEntityId = + EntityIdService.getEntityIds( + NameIdentifier.of(namespace.levels()), Entity.EntityType.SCHEMA); + builder.withMetalakeId(namespacedEntityId.namespaceIds()[0]); + builder.withCatalogId(namespacedEntityId.namespaceIds()[1]); + builder.withSchemaId(namespacedEntityId.entityId()); } } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/TablePOStorageOps.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/TablePOStorageOps.java new file mode 100644 index 00000000000..b9e7e204a1d --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/TablePOStorageOps.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.service; + +import java.util.List; +import org.apache.gravitino.Entity; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.Namespace; +import org.apache.gravitino.storage.relational.mapper.TableMetaMapper; +import org.apache.gravitino.storage.relational.po.TablePO; + +public class TablePOStorageOps extends BasePOStorageOps { + + public TablePOStorageOps() {} + + @Override + public void insertPO(TableMetaMapper mapper, TablePO tablePO, boolean overwrite) { + if (overwrite) { + mapper.insertTableMetaOnDuplicateKeyUpdate(tablePO); + } else { + mapper.insertTableMeta(tablePO); + } + } + + @Override + public Integer updatePO(TableMetaMapper mapper, TablePO newPO, TablePO oldPO) { + return mapper.updateTableMeta(newPO, oldPO, newPO.getSchemaId()); + } + + @Override + public TablePO getPO(TableMetaMapper mapper, Long parentId, String name) { + return mapper.selectTableMetaBySchemaIdAndName(parentId, name); + } + + @Override + protected TablePO getPOByFullName(TableMetaMapper mapper, NameIdentifier identifier) { + Namespace namespace = identifier.namespace(); + return mapper.selectTableByFullQualifiedName( + namespace.level(0), namespace.level(1), namespace.level(2), identifier.name()); + } + + @Override + public List listPOs(TableMetaMapper mapper, Long parentId) { + return mapper.listTablePOsBySchemaId(parentId); + } + + @Override + public List listPOs(TableMetaMapper mapper, Namespace namespace, List names) { + Long schemaId = + EntityIdService.getEntityId( + NameIdentifier.of(namespace.levels()), Entity.EntityType.SCHEMA); + return mapper.batchSelectTableByIdentifier(schemaId, names); + } + + @Override + public List listPOs(TableMetaMapper mapper, List uuids) { + return mapper.listTablePOsByTableIds(uuids); + } + + @Override + protected List listPOsByNSFullName(TableMetaMapper mapper, Namespace namespace) { + return mapper.listTablePOsByFullQualifiedName( + namespace.level(0), namespace.level(1), namespace.level(2)); + } + + @Override + public List capabilities() { + return List.of( + Capability.INSERT, + Capability.UPDATE, + Capability.GET_BY_NAME, + Capability.GET_BY_NS_UID, + Capability.LIST_BY_NS_UID, + Capability.LIST_BY_NS_NAME, + Capability.LIST_BY_NAME_FILTER, + Capability.LIST_BY_UID_FILTER); + } + + @Override + protected Entity.EntityType entityType() { + return Entity.EntityType.TABLE; + } +} From e9134fd7de5d25299c19ba2f0a8d70bd80f20509 Mon Sep 17 00:00:00 2001 From: roryqi Date: Thu, 14 May 2026 09:55:16 +0000 Subject: [PATCH 22/44] refactor Co-Authored-By: Claude Opus 4.7 --- .../service/FunctionMetaService.java | 16 +-- .../RequireSchemaConventionService.java | 103 ------------------ .../relational/service/ViewMetaService.java | 16 +-- 3 files changed, 6 insertions(+), 129 deletions(-) delete mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/service/RequireSchemaConventionService.java diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionMetaService.java index 78f184544ef..f868b7c75c9 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionMetaService.java @@ -55,7 +55,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class FunctionMetaService extends RequireSchemaConventionService { +public class FunctionMetaService { public static FunctionMetaService getInstance() { return INSTANCE; } @@ -346,23 +346,13 @@ private List listFunctionPOsByFullQualifiedName(Namespace namespace) private Function> functionListFetcher() { return GravitinoEnv.getInstance().cacheEnabled() ? this::listFunctionPOsBySchemaId - : this::listPOsForApiNamespace; + : this::listFunctionPOsByFullQualifiedName; } private Function functionPOFetcher() { return GravitinoEnv.getInstance().cacheEnabled() ? this::getFunctionPOBySchemaId - : this::getPOForApiIdentifier; - } - - @Override - protected FunctionPO fetchPOByStorageIdentifier(NameIdentifier storageIdentifier) { - return getFunctionPOByFullQualifiedName(storageIdentifier); - } - - @Override - protected List fetchPOsByStorageNamespace(Namespace storageNamespace) { - return listFunctionPOsByFullQualifiedName(storageNamespace); + : this::getFunctionPOByFullQualifiedName; } private void fillFunctionPOBuilderParentEntityId( diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/RequireSchemaConventionService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/RequireSchemaConventionService.java deleted file mode 100644 index c5c76cf2596..00000000000 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/RequireSchemaConventionService.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.gravitino.storage.relational.service; - -import java.util.List; -import org.apache.commons.lang3.StringUtils; -import org.apache.gravitino.NameIdentifier; -import org.apache.gravitino.Namespace; -import org.apache.gravitino.utils.HierarchicalSchemaUtil; - -/** - * Base for relational metadata services whose JDBC keys use hierarchical schema storage names. - * - *

API identifiers use the configured logical schema separator; persisted rows use the internal - * physical separator. Subclasses implement {@link #fetchPOByStorageIdentifier} and {@link - * #fetchPOsByStorageNamespace} after {@link #getPOForApiIdentifier} / {@link - * #listPOsForApiNamespace} rewrite identifiers or namespaces as needed. - * - * @param persistent object type loaded for the entity - */ -public abstract class RequireSchemaConventionService { - - protected RequireSchemaConventionService() {} - - /** Loads a PO using an identifier whose naming is already in storage form. */ - protected abstract E fetchPOByStorageIdentifier(NameIdentifier storageIdentifier); - - /** Lists POs under a namespace whose naming is already in storage form where applicable. */ - protected abstract List fetchPOsByStorageNamespace(Namespace storageNamespace); - - /** Converts hierarchical schema naming then loads the PO. */ - protected final E getPOForApiIdentifier(NameIdentifier apiIdentifier) { - return fetchPOByStorageIdentifier(apiIdentifierToStorage(apiIdentifier)); - } - - /** Converts hierarchical schema naming then lists POs under the namespace. */ - protected final List listPOsForApiNamespace(Namespace apiNamespace) { - return fetchPOsByStorageNamespace(apiNamespaceToStorage(apiNamespace)); - } - - private static NameIdentifier apiIdentifierToStorage(NameIdentifier apiIdentifier) { - String[] levels = apiIdentifier.namespace().levels(); - if (levels.length == 2) { - String rawName = apiIdentifier.name(); - String storageName = - StringUtils.isNotBlank(rawName) - ? HierarchicalSchemaUtil.logicalToPhysical( - rawName, HierarchicalSchemaUtil.schemaSeparator()) - : rawName; - if (storageName.equals(apiIdentifier.name())) { - return apiIdentifier; - } - return NameIdentifier.of(apiIdentifier.namespace(), storageName); - } - if (levels.length == 3) { - String rawSeg = levels[2]; - String storageSchema = - StringUtils.isNotBlank(rawSeg) - ? HierarchicalSchemaUtil.logicalToPhysical( - rawSeg, HierarchicalSchemaUtil.schemaSeparator()) - : rawSeg; - if (storageSchema.equals(levels[2])) { - return apiIdentifier; - } - return NameIdentifier.of( - Namespace.of(levels[0], levels[1], storageSchema), apiIdentifier.name()); - } - return apiIdentifier; - } - - private static Namespace apiNamespaceToStorage(Namespace apiNamespace) { - String[] levels = apiNamespace.levels(); - if (levels.length != 3) { - return apiNamespace; - } - String rawSeg = levels[2]; - String storageSchema = - StringUtils.isNotBlank(rawSeg) - ? HierarchicalSchemaUtil.logicalToPhysical( - rawSeg, HierarchicalSchemaUtil.schemaSeparator()) - : rawSeg; - if (storageSchema.equals(levels[2])) { - return apiNamespace; - } - return Namespace.of(levels[0], levels[1], storageSchema); - } -} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewMetaService.java index 6fe6ea0abf2..53999dc008b 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewMetaService.java @@ -54,7 +54,7 @@ import org.apache.gravitino.utils.NamespaceUtil; /** The service class for view metadata. It provides the basic database operations for view. */ -public class ViewMetaService extends RequireSchemaConventionService { +public class ViewMetaService { private static final ViewMetaService INSTANCE = new ViewMetaService(); @@ -295,13 +295,13 @@ private List listViewPOs(Namespace namespace) { private Function> viewListFetcher() { return GravitinoEnv.getInstance().cacheEnabled() ? this::listViewPOsBySchemaId - : this::listPOsForApiNamespace; + : this::listViewPOsByFullQualifiedName; } private Function viewPOFetcher() { return GravitinoEnv.getInstance().cacheEnabled() ? this::getViewPOBySchemaId - : this::getPOForApiIdentifier; + : this::getViewPOByFullQualifiedName; } private List listViewPOsBySchemaId(Namespace namespace) { @@ -372,14 +372,4 @@ private ViewPO getViewPOByFullQualifiedName(NameIdentifier identifier) { return viewPO; } - - @Override - protected ViewPO fetchPOByStorageIdentifier(NameIdentifier storageIdentifier) { - return getViewPOByFullQualifiedName(storageIdentifier); - } - - @Override - protected List fetchPOsByStorageNamespace(Namespace storageNamespace) { - return listViewPOsByFullQualifiedName(storageNamespace); - } } From 8cf90f7131c53488e996fb80c7ea6f90fdc02b7a Mon Sep 17 00:00:00 2001 From: roryqi Date: Thu, 14 May 2026 10:09:18 +0000 Subject: [PATCH 23/44] refactor Co-Authored-By: Claude Opus 4.7 --- .../service/FunctionMetaService.java | 109 ++---------------- .../service/FunctionPOStorageOps.java | 90 +++++++++++++++ .../relational/service/ViewMetaService.java | 103 ++--------------- .../relational/service/ViewPOStorageOps.java | 90 +++++++++++++++ 4 files changed, 201 insertions(+), 191 deletions(-) create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionPOStorageOps.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/service/ViewPOStorageOps.java diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionMetaService.java index f868b7c75c9..dd272ff9f99 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionMetaService.java @@ -33,7 +33,6 @@ import java.util.function.Function; import java.util.stream.Collectors; import org.apache.gravitino.Entity; -import org.apache.gravitino.GravitinoEnv; import org.apache.gravitino.HasIdentifier; import org.apache.gravitino.MetadataObject; import org.apache.gravitino.NameIdentifier; @@ -56,14 +55,17 @@ import org.slf4j.LoggerFactory; public class FunctionMetaService { + private static final Logger LOG = LoggerFactory.getLogger(FunctionMetaService.class); + private static final FunctionMetaService INSTANCE = new FunctionMetaService(); + private FunctionPOStorageOps ops; + public static FunctionMetaService getInstance() { return INSTANCE; } - private static final Logger LOG = LoggerFactory.getLogger(FunctionMetaService.class); - private static final FunctionMetaService INSTANCE = new FunctionMetaService(); - - private FunctionMetaService() {} + private FunctionMetaService() { + this.ops = new FunctionPOStorageOps(); + } @Monitored( metricsSource = GRAVITINO_RELATIONAL_STORE_METRIC_NAME, @@ -115,14 +117,7 @@ public void insertFunction(FunctionEntity functionEntity, boolean overwrite) thr SessionUtils.doMultipleWithCommit( () -> SessionUtils.doWithoutCommit( - FunctionMetaMapper.class, - mapper -> { - if (overwrite) { - mapper.insertFunctionMetaOnDuplicateKeyUpdate(po); - } else { - mapper.insertFunctionMeta(po); - } - }), + FunctionMetaMapper.class, mapper -> ops.insertPO(mapper, po, overwrite)), () -> SessionUtils.doWithoutCommit( FunctionVersionMetaMapper.class, @@ -231,8 +226,8 @@ public int deleteFunctionVersionsByRetentionCount(Long versionRetentionCount, in baseMetricName = "getFunctionPOByIdentifier") FunctionPO getFunctionPOByIdentifier(NameIdentifier ident) { NameIdentifierUtil.checkFunction(ident); - - return functionPOFetcher().apply(ident); + return SessionUtils.getWithoutCommit( + FunctionMetaMapper.class, mapper -> ops.getPO(mapper, ident)); } @Monitored( @@ -260,7 +255,7 @@ public FunctionEntity updateFunction( () -> SessionUtils.doWithoutCommit( FunctionMetaMapper.class, - mapper -> mapper.updateFunctionMeta(newFunctionPO, oldFunctionPO))); + mapper -> ops.updatePO(mapper, newFunctionPO, oldFunctionPO))); return newEntity; } catch (RuntimeException re) { @@ -270,89 +265,9 @@ public FunctionEntity updateFunction( } } - private FunctionPO getFunctionPOBySchemaId(NameIdentifier ident) { - Long schemaId = - EntityIdService.getEntityId( - NameIdentifier.of(ident.namespace().levels()), Entity.EntityType.SCHEMA); - - FunctionPO functionPO = - SessionUtils.getWithoutCommit( - FunctionMetaMapper.class, - mapper -> mapper.selectFunctionMetaBySchemaIdAndName(schemaId, ident.name())); - - if (functionPO == null) { - throw new NoSuchEntityException( - NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, - Entity.EntityType.FUNCTION.name().toLowerCase(Locale.ROOT), - ident.toString()); - } - return functionPO; - } - - private FunctionPO getFunctionPOByFullQualifiedName(NameIdentifier ident) { - String[] namespaceLevels = ident.namespace().levels(); - FunctionPO functionPO = - SessionUtils.getWithoutCommit( - FunctionMetaMapper.class, - mapper -> - mapper.selectFunctionMetaByFullQualifiedName( - namespaceLevels[0], namespaceLevels[1], namespaceLevels[2], ident.name())); - - if (functionPO == null || functionPO.functionId() == null) { - throw new NoSuchEntityException( - NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, - Entity.EntityType.FUNCTION.name().toLowerCase(Locale.ROOT), - ident.name()); - } - - if (functionPO.schemaId() == null) { - throw new NoSuchEntityException( - NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, - Entity.EntityType.SCHEMA.name().toLowerCase(), - namespaceLevels[2]); - } - return functionPO; - } - private List listFunctionPOs(Namespace namespace) { - return functionListFetcher().apply(namespace); - } - - private List listFunctionPOsBySchemaId(Namespace namespace) { - Long schemaId = - EntityIdService.getEntityId( - NameIdentifier.of(namespace.levels()), Entity.EntityType.SCHEMA); return SessionUtils.getWithoutCommit( - FunctionMetaMapper.class, mapper -> mapper.listFunctionPOsBySchemaId(schemaId)); - } - - private List listFunctionPOsByFullQualifiedName(Namespace namespace) { - String[] namespaceLevels = namespace.levels(); - List functionPOs = - SessionUtils.getWithoutCommit( - FunctionMetaMapper.class, - mapper -> - mapper.listFunctionPOsByFullQualifiedName( - namespaceLevels[0], namespaceLevels[1], namespaceLevels[2])); - if (functionPOs.isEmpty() || functionPOs.get(0).schemaId() == null) { - throw new NoSuchEntityException( - NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, - Entity.EntityType.SCHEMA.name().toLowerCase(), - namespaceLevels[2]); - } - return functionPOs.stream().filter(po -> po.functionId() != null).collect(Collectors.toList()); - } - - private Function> functionListFetcher() { - return GravitinoEnv.getInstance().cacheEnabled() - ? this::listFunctionPOsBySchemaId - : this::listFunctionPOsByFullQualifiedName; - } - - private Function functionPOFetcher() { - return GravitinoEnv.getInstance().cacheEnabled() - ? this::getFunctionPOBySchemaId - : this::getFunctionPOByFullQualifiedName; + FunctionMetaMapper.class, mapper -> ops.listPOs(mapper, namespace)); } private void fillFunctionPOBuilderParentEntityId( diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionPOStorageOps.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionPOStorageOps.java new file mode 100644 index 00000000000..5ac67b34b4f --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionPOStorageOps.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.service; + +import java.util.List; +import org.apache.gravitino.Entity; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.Namespace; +import org.apache.gravitino.storage.relational.mapper.FunctionMetaMapper; +import org.apache.gravitino.storage.relational.po.FunctionPO; + +public class FunctionPOStorageOps extends BasePOStorageOps { + + public FunctionPOStorageOps() {} + + @Override + public void insertPO(FunctionMetaMapper mapper, FunctionPO functionPO, boolean overwrite) { + if (overwrite) { + mapper.insertFunctionMetaOnDuplicateKeyUpdate(functionPO); + } else { + mapper.insertFunctionMeta(functionPO); + } + } + + @Override + public Integer updatePO(FunctionMetaMapper mapper, FunctionPO newPO, FunctionPO oldPO) { + return mapper.updateFunctionMeta(newPO, oldPO); + } + + @Override + public FunctionPO getPO(FunctionMetaMapper mapper, Long parentId, String name) { + return mapper.selectFunctionMetaBySchemaIdAndName(parentId, name); + } + + @Override + protected FunctionPO getPOByFullName(FunctionMetaMapper mapper, NameIdentifier identifier) { + Namespace namespace = identifier.namespace(); + return mapper.selectFunctionMetaByFullQualifiedName( + namespace.level(0), namespace.level(1), namespace.level(2), identifier.name()); + } + + @Override + public List listPOs(FunctionMetaMapper mapper, Long parentId) { + return mapper.listFunctionPOsBySchemaId(parentId); + } + + @Override + public List listPOs(FunctionMetaMapper mapper, List uuids) { + return mapper.listFunctionPOsByFunctionIds(uuids); + } + + @Override + protected List listPOsByNSFullName(FunctionMetaMapper mapper, Namespace namespace) { + return mapper.listFunctionPOsByFullQualifiedName( + namespace.level(0), namespace.level(1), namespace.level(2)); + } + + @Override + public List capabilities() { + return List.of( + Capability.INSERT, + Capability.UPDATE, + Capability.GET_BY_NAME, + Capability.GET_BY_NS_UID, + Capability.LIST_BY_NS_UID, + Capability.LIST_BY_NS_NAME, + Capability.LIST_BY_UID_FILTER); + } + + @Override + protected Entity.EntityType entityType() { + return Entity.EntityType.FUNCTION; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewMetaService.java index 53999dc008b..8fdc740c5dc 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewMetaService.java @@ -31,8 +31,6 @@ import java.util.function.Function; import java.util.stream.Collectors; import org.apache.gravitino.Entity; -import org.apache.gravitino.Entity.EntityType; -import org.apache.gravitino.GravitinoEnv; import org.apache.gravitino.HasIdentifier; import org.apache.gravitino.MetadataObject; import org.apache.gravitino.NameIdentifier; @@ -57,12 +55,15 @@ public class ViewMetaService { private static final ViewMetaService INSTANCE = new ViewMetaService(); + private ViewPOStorageOps ops; public static ViewMetaService getInstance() { return INSTANCE; } - private ViewMetaService() {} + private ViewMetaService() { + this.ops = new ViewPOStorageOps(); + } @Monitored( metricsSource = GRAVITINO_RELATIONAL_STORE_METRIC_NAME, @@ -110,14 +111,7 @@ public void insertView(ViewEntity viewEntity, boolean overwrite) throws IOExcept SessionUtils.doMultipleWithCommit( () -> SessionUtils.doWithoutCommit( - ViewMetaMapper.class, - mapper -> { - if (overwrite) { - mapper.insertViewMetaOnDuplicateKeyUpdate(po); - } else { - mapper.insertViewMeta(po); - } - }), + ViewMetaMapper.class, mapper -> ops.insertPO(mapper, po, overwrite)), () -> SessionUtils.doWithoutCommit( ViewVersionInfoMapper.class, @@ -168,7 +162,7 @@ public ViewEntity updateView( () -> { updateResult.set( SessionUtils.getWithoutCommit( - ViewMetaMapper.class, mapper -> mapper.updateViewMeta(newViewPO, oldViewPO))); + ViewMetaMapper.class, mapper -> ops.updatePO(mapper, newViewPO, oldViewPO))); if (updateResult.get() == 0) { throw new RuntimeException("Failed to update the entity: " + ident); } @@ -285,91 +279,12 @@ private ViewPO updateViewPO(ViewPO oldViewPO, ViewEntity newEntity) { private ViewPO getViewPOByIdentifier(NameIdentifier identifier) { NameIdentifierUtil.checkView(identifier); - return viewPOFetcher().apply(identifier); + return SessionUtils.getWithoutCommit( + ViewMetaMapper.class, mapper -> ops.getPO(mapper, identifier)); } private List listViewPOs(Namespace namespace) { - return viewListFetcher().apply(namespace); - } - - private Function> viewListFetcher() { - return GravitinoEnv.getInstance().cacheEnabled() - ? this::listViewPOsBySchemaId - : this::listViewPOsByFullQualifiedName; - } - - private Function viewPOFetcher() { - return GravitinoEnv.getInstance().cacheEnabled() - ? this::getViewPOBySchemaId - : this::getViewPOByFullQualifiedName; - } - - private List listViewPOsBySchemaId(Namespace namespace) { - Long schemaId = - EntityIdService.getEntityId( - NameIdentifier.of(namespace.levels()), Entity.EntityType.SCHEMA); return SessionUtils.getWithoutCommit( - ViewMetaMapper.class, mapper -> mapper.listViewPOsBySchemaId(schemaId)); - } - - private List listViewPOsByFullQualifiedName(Namespace namespace) { - String[] namespaceLevels = namespace.levels(); - List viewPOs = - SessionUtils.getWithoutCommit( - ViewMetaMapper.class, - mapper -> - mapper.listViewPOsByFullQualifiedName( - namespaceLevels[0], namespaceLevels[1], namespaceLevels[2])); - if (viewPOs.isEmpty() || viewPOs.get(0).getSchemaId() == null) { - throw new NoSuchEntityException( - NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, - EntityType.SCHEMA.name().toLowerCase(), - namespaceLevels[2]); - } - return viewPOs.stream().filter(po -> po.getViewId() != null).collect(Collectors.toList()); - } - - private ViewPO getViewPOBySchemaId(NameIdentifier identifier) { - Long schemaId = - EntityIdService.getEntityId( - NameIdentifier.of(identifier.namespace().levels()), Entity.EntityType.SCHEMA); - ViewPO viewPO = - SessionUtils.getWithoutCommit( - ViewMetaMapper.class, - mapper -> mapper.selectViewMetaBySchemaIdAndName(schemaId, identifier.name())); - - if (viewPO == null) { - throw new NoSuchEntityException( - NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, - Entity.EntityType.VIEW.name().toLowerCase(), - identifier.name()); - } - return viewPO; - } - - private ViewPO getViewPOByFullQualifiedName(NameIdentifier identifier) { - String[] namespaceLevels = identifier.namespace().levels(); - ViewPO viewPO = - SessionUtils.getWithoutCommit( - ViewMetaMapper.class, - mapper -> - mapper.selectViewByFullQualifiedName( - namespaceLevels[0], namespaceLevels[1], namespaceLevels[2], identifier.name())); - - if (viewPO == null || viewPO.getSchemaId() == null) { - throw new NoSuchEntityException( - NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, - EntityType.SCHEMA.name().toLowerCase(), - namespaceLevels[2]); - } - - if (viewPO.getViewId() == null) { - throw new NoSuchEntityException( - NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, - EntityType.VIEW.name().toLowerCase(), - identifier.name()); - } - - return viewPO; + ViewMetaMapper.class, mapper -> ops.listPOs(mapper, namespace)); } } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewPOStorageOps.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewPOStorageOps.java new file mode 100644 index 00000000000..ffa125f1696 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewPOStorageOps.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.service; + +import java.util.List; +import org.apache.gravitino.Entity; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.Namespace; +import org.apache.gravitino.storage.relational.mapper.ViewMetaMapper; +import org.apache.gravitino.storage.relational.po.ViewPO; + +public class ViewPOStorageOps extends BasePOStorageOps { + + public ViewPOStorageOps() {} + + @Override + public void insertPO(ViewMetaMapper mapper, ViewPO viewPO, boolean overwrite) { + if (overwrite) { + mapper.insertViewMetaOnDuplicateKeyUpdate(viewPO); + } else { + mapper.insertViewMeta(viewPO); + } + } + + @Override + public Integer updatePO(ViewMetaMapper mapper, ViewPO newPO, ViewPO oldPO) { + return mapper.updateViewMeta(newPO, oldPO); + } + + @Override + public ViewPO getPO(ViewMetaMapper mapper, Long parentId, String name) { + return mapper.selectViewMetaBySchemaIdAndName(parentId, name); + } + + @Override + protected ViewPO getPOByFullName(ViewMetaMapper mapper, NameIdentifier identifier) { + Namespace namespace = identifier.namespace(); + return mapper.selectViewByFullQualifiedName( + namespace.level(0), namespace.level(1), namespace.level(2), identifier.name()); + } + + @Override + public List listPOs(ViewMetaMapper mapper, Long parentId) { + return mapper.listViewPOsBySchemaId(parentId); + } + + @Override + public List listPOs(ViewMetaMapper mapper, List uuids) { + return mapper.listViewPOsByViewIds(uuids); + } + + @Override + protected List listPOsByNSFullName(ViewMetaMapper mapper, Namespace namespace) { + return mapper.listViewPOsByFullQualifiedName( + namespace.level(0), namespace.level(1), namespace.level(2)); + } + + @Override + public List capabilities() { + return List.of( + Capability.INSERT, + Capability.UPDATE, + Capability.GET_BY_NAME, + Capability.GET_BY_NS_UID, + Capability.LIST_BY_NS_UID, + Capability.LIST_BY_NS_NAME, + Capability.LIST_BY_UID_FILTER); + } + + @Override + protected Entity.EntityType entityType() { + return Entity.EntityType.VIEW; + } +} From bac3a3c154eacfe3afb318ece46a3bf4815f5f22 Mon Sep 17 00:00:00 2001 From: roryqi Date: Thu, 14 May 2026 10:18:07 +0000 Subject: [PATCH 24/44] refactor Co-Authored-By: Claude Opus 4.7 --- .../storage/relational/service/FunctionMetaService.java | 9 ++++----- .../storage/relational/service/ViewMetaService.java | 9 ++++----- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionMetaService.java index dd272ff9f99..32bdb3a702b 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionMetaService.java @@ -89,18 +89,17 @@ public FunctionEntity getFunctionByIdentifier(NameIdentifier ident) { metricsSource = GRAVITINO_RELATIONAL_STORE_METRIC_NAME, baseMetricName = "getFunctionIdBySchemaIdAndFunctionName") public Long getFunctionIdBySchemaIdAndFunctionName(Long schemaId, String functionName) { - Long functionId = + FunctionPO functionPO = SessionUtils.getWithoutCommit( - FunctionMetaMapper.class, - mapper -> mapper.selectFunctionIdBySchemaIdAndFunctionName(schemaId, functionName)); + FunctionMetaMapper.class, mapper -> ops.getPO(mapper, schemaId, functionName)); - if (functionId == null) { + if (functionPO == null) { throw new NoSuchEntityException( NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, Entity.EntityType.FUNCTION.name().toLowerCase(Locale.ROOT), functionName); } - return functionId; + return functionPO.functionId(); } @Monitored( diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewMetaService.java index 8fdc740c5dc..c7bedb55afe 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewMetaService.java @@ -69,18 +69,17 @@ private ViewMetaService() { metricsSource = GRAVITINO_RELATIONAL_STORE_METRIC_NAME, baseMetricName = "getViewIdBySchemaIdAndName") public Long getViewIdBySchemaIdAndName(Long schemaId, String viewName) { - Long viewId = + ViewPO viewPO = SessionUtils.getWithoutCommit( - ViewMetaMapper.class, - mapper -> mapper.selectViewIdBySchemaIdAndName(schemaId, viewName)); + ViewMetaMapper.class, mapper -> ops.getPO(mapper, schemaId, viewName)); - if (viewId == null) { + if (viewPO == null) { throw new NoSuchEntityException( NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, Entity.EntityType.VIEW.name().toLowerCase(), viewName); } - return viewId; + return viewPO.getViewId(); } @Monitored( From 060815a2e893e7b35ec57cc18c643560eed87d82 Mon Sep 17 00:00:00 2001 From: roryqi Date: Thu, 14 May 2026 10:47:59 +0000 Subject: [PATCH 25/44] refactor Co-Authored-By: Claude Opus 4.7 --- .../service/MetadataObjectService.java | 33 ++++++++++++++++--- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java index ee8f7ff8ad7..315d7118914 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java @@ -96,6 +96,14 @@ public class MetadataObjectService { MetadataObjectService::getJobTemplateObjectsFullName) .build(); + static final Map> TYPE_TO_STORAGE_OPS_MAP = + ImmutableMap.>builder() + .put(MetadataObject.Type.SCHEMA, new SchemaPOStorageOps()) + .put(MetadataObject.Type.TABLE, new TablePOStorageOps()) + .put(MetadataObject.Type.VIEW, new ViewPOStorageOps()) + .put(MetadataObject.Type.FUNCTION, new FunctionPOStorageOps()) + .build(); + private static Map getPolicyObjectsFullName(List policyIds) { if (policyIds == null || policyIds.isEmpty()) { return Maps.newHashMap(); @@ -345,9 +353,13 @@ public static Map getFunctionObjectsFullName(List functionId return Maps.newHashMap(); } + @SuppressWarnings("unchecked") + BasePOStorageOps ops = + (BasePOStorageOps) + TYPE_TO_STORAGE_OPS_MAP.get(MetadataObject.Type.FUNCTION); List functionPOs = SessionUtils.getWithoutCommit( - FunctionMetaMapper.class, mapper -> mapper.listFunctionPOsByFunctionIds(functionIds)); + FunctionMetaMapper.class, mapper -> ops.listPOs(mapper, functionIds)); if (functionPOs == null || functionPOs.isEmpty()) { return new HashMap<>(); @@ -395,9 +407,13 @@ public static Map getTableObjectsFullName(List tableIds) { return Maps.newHashMap(); } + @SuppressWarnings("unchecked") + BasePOStorageOps ops = + (BasePOStorageOps) + TYPE_TO_STORAGE_OPS_MAP.get(MetadataObject.Type.TABLE); List tablePOs = SessionUtils.getWithoutCommit( - TableMetaMapper.class, mapper -> mapper.listTablePOsByTableIds(tableIds)); + TableMetaMapper.class, mapper -> ops.listPOs(mapper, tableIds)); if (tablePOs == null || tablePOs.isEmpty()) { return Maps.newHashMap(); @@ -534,9 +550,12 @@ public static Map getTopicObjectsFullName(List topicIds) { metricsSource = GRAVITINO_RELATIONAL_STORE_METRIC_NAME, baseMetricName = "getViewObjectsFullName") public static Map getViewObjectsFullName(List viewIds) { + @SuppressWarnings("unchecked") + BasePOStorageOps ops = + (BasePOStorageOps) + TYPE_TO_STORAGE_OPS_MAP.get(MetadataObject.Type.VIEW); List viewPOs = - SessionUtils.getWithoutCommit( - ViewMetaMapper.class, mapper -> mapper.listViewPOsByViewIds(viewIds)); + SessionUtils.getWithoutCommit(ViewMetaMapper.class, mapper -> ops.listPOs(mapper, viewIds)); if (viewPOs == null || viewPOs.isEmpty()) { return new HashMap<>(); } @@ -601,9 +620,13 @@ public static Map getCatalogObjectsFullName(List catalogIds) metricsSource = GRAVITINO_RELATIONAL_STORE_METRIC_NAME, baseMetricName = "getSchemaObjectsFullName") public static Map getSchemaObjectsFullName(List schemaIds) { + @SuppressWarnings("unchecked") + BasePOStorageOps ops = + (BasePOStorageOps) + TYPE_TO_STORAGE_OPS_MAP.get(MetadataObject.Type.SCHEMA); List schemaPOs = SessionUtils.getWithoutCommit( - SchemaMetaMapper.class, mapper -> mapper.listSchemaPOsBySchemaIds(schemaIds)); + SchemaMetaMapper.class, mapper -> ops.listPOs(mapper, schemaIds)); if (schemaPOs == null || schemaPOs.isEmpty()) { return new HashMap<>(); From b2c78e5ca5a605eda9ac1db3d8c61aa9ab7c3c76 Mon Sep 17 00:00:00 2001 From: roryqi Date: Thu, 14 May 2026 10:59:50 +0000 Subject: [PATCH 26/44] refactor Co-Authored-By: Claude Opus 4.7 --- .../service/FunctionMetaService.java | 4 +- .../HierarchicalSchemaPOStorageOps.java | 145 ++++++++++++++++++ .../relational/service/TableMetaService.java | 4 +- .../relational/service/ViewMetaService.java | 4 +- 4 files changed, 151 insertions(+), 6 deletions(-) create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/service/HierarchicalSchemaPOStorageOps.java diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionMetaService.java index 32bdb3a702b..6f2bda66b3f 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionMetaService.java @@ -57,14 +57,14 @@ public class FunctionMetaService { private static final Logger LOG = LoggerFactory.getLogger(FunctionMetaService.class); private static final FunctionMetaService INSTANCE = new FunctionMetaService(); - private FunctionPOStorageOps ops; + private BasePOStorageOps ops; public static FunctionMetaService getInstance() { return INSTANCE; } private FunctionMetaService() { - this.ops = new FunctionPOStorageOps(); + this.ops = new HierarchicalSchemaPOStorageOps<>(new FunctionPOStorageOps()); } @Monitored( diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/HierarchicalSchemaPOStorageOps.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/HierarchicalSchemaPOStorageOps.java new file mode 100644 index 00000000000..3a6f1109108 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/HierarchicalSchemaPOStorageOps.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.service; + +import java.util.List; +import org.apache.commons.lang3.StringUtils; +import org.apache.gravitino.Entity; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.Namespace; +import org.apache.gravitino.utils.HierarchicalSchemaUtil; + +/** + * Wraps a {@link BasePOStorageOps} so full-name lookups translate the schema segment from the + * external logical separator used by the API into the internal physical separator used in storage. + * Cache-keyed lookups already use API-form names, so they pass through unchanged. + * + * @param persistent object type + * @param MyBatis mapper type + */ +public class HierarchicalSchemaPOStorageOps extends BasePOStorageOps { + + private final BasePOStorageOps delegate; + + public HierarchicalSchemaPOStorageOps(BasePOStorageOps delegate) { + this.delegate = delegate; + } + + @Override + public void insertPO(Mapper mapper, PO po, boolean overwrite) { + delegate.insertPO(mapper, po, overwrite); + } + + @Override + public void batchInsertPOs(Mapper mapper, List pos, boolean overwrite) { + delegate.batchInsertPOs(mapper, pos, overwrite); + } + + @Override + public Integer updatePO(Mapper mapper, PO newPO, PO oldPO) { + return delegate.updatePO(mapper, newPO, oldPO); + } + + @Override + public PO getPO(Mapper mapper, Long parentId, String name) { + return delegate.getPO(mapper, parentId, name); + } + + @Override + public List listPOs(Mapper mapper, Long parentId) { + return delegate.listPOs(mapper, parentId); + } + + @Override + public List listPOs(Mapper mapper, Namespace namespace, List names) { + return delegate.listPOs(mapper, namespace, names); + } + + @Override + public List listPOs(Mapper mapper, List uuids) { + return delegate.listPOs(mapper, uuids); + } + + @Override + protected PO getPOByFullName(Mapper mapper, NameIdentifier identifier) { + return delegate.getPOByFullName(mapper, apiIdentifierToStorage(identifier)); + } + + @Override + protected List listPOsByNSFullName(Mapper mapper, Namespace namespace) { + return delegate.listPOsByNSFullName(mapper, apiNamespaceToStorage(namespace)); + } + + @Override + public List capabilities() { + return delegate.capabilities(); + } + + @Override + protected Entity.EntityType entityType() { + return delegate.entityType(); + } + + private static NameIdentifier apiIdentifierToStorage(NameIdentifier apiIdentifier) { + String[] levels = apiIdentifier.namespace().levels(); + if (levels.length == 2) { + String rawName = apiIdentifier.name(); + String storageName = + StringUtils.isNotBlank(rawName) + ? HierarchicalSchemaUtil.logicalToPhysical( + rawName, HierarchicalSchemaUtil.schemaSeparator()) + : rawName; + if (storageName.equals(apiIdentifier.name())) { + return apiIdentifier; + } + return NameIdentifier.of(apiIdentifier.namespace(), storageName); + } + if (levels.length == 3) { + String rawSeg = levels[2]; + String storageSchema = + StringUtils.isNotBlank(rawSeg) + ? HierarchicalSchemaUtil.logicalToPhysical( + rawSeg, HierarchicalSchemaUtil.schemaSeparator()) + : rawSeg; + if (storageSchema.equals(levels[2])) { + return apiIdentifier; + } + return NameIdentifier.of( + Namespace.of(levels[0], levels[1], storageSchema), apiIdentifier.name()); + } + return apiIdentifier; + } + + private static Namespace apiNamespaceToStorage(Namespace apiNamespace) { + String[] levels = apiNamespace.levels(); + if (levels.length != 3) { + return apiNamespace; + } + String rawSeg = levels[2]; + String storageSchema = + StringUtils.isNotBlank(rawSeg) + ? HierarchicalSchemaUtil.logicalToPhysical( + rawSeg, HierarchicalSchemaUtil.schemaSeparator()) + : rawSeg; + if (storageSchema.equals(levels[2])) { + return apiNamespace; + } + return Namespace.of(levels[0], levels[1], storageSchema); + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java index ab1aae1a306..16c4c46684b 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java @@ -57,14 +57,14 @@ /** The service class for table metadata. It provides the basic database operations for table. */ public class TableMetaService { private static final TableMetaService INSTANCE = new TableMetaService(); - private TablePOStorageOps ops; + private BasePOStorageOps ops; public static TableMetaService getInstance() { return INSTANCE; } private TableMetaService() { - this.ops = new TablePOStorageOps(); + this.ops = new HierarchicalSchemaPOStorageOps<>(new TablePOStorageOps()); } @Monitored( diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewMetaService.java index c7bedb55afe..d86055e84df 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewMetaService.java @@ -55,14 +55,14 @@ public class ViewMetaService { private static final ViewMetaService INSTANCE = new ViewMetaService(); - private ViewPOStorageOps ops; + private BasePOStorageOps ops; public static ViewMetaService getInstance() { return INSTANCE; } private ViewMetaService() { - this.ops = new ViewPOStorageOps(); + this.ops = new HierarchicalSchemaPOStorageOps<>(new ViewPOStorageOps()); } @Monitored( From f8690dca67e67c3222502a7ddf6e2fb0f5eaeba7 Mon Sep 17 00:00:00 2001 From: roryqi Date: Thu, 14 May 2026 11:08:36 +0000 Subject: [PATCH 27/44] refactor Co-Authored-By: Claude Opus 4.7 --- .../service/FunctionMetaService.java | 2 +- ...=> HierarchicalConventionPOStorageOp.java} | 4 +-- .../relational/service/SchemaMetaService.java | 26 +++++++------------ .../relational/service/TableMetaService.java | 2 +- .../relational/service/ViewMetaService.java | 2 +- 5 files changed, 14 insertions(+), 22 deletions(-) rename core/src/main/java/org/apache/gravitino/storage/relational/service/{HierarchicalSchemaPOStorageOps.java => HierarchicalConventionPOStorageOp.java} (96%) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionMetaService.java index 6f2bda66b3f..0b3f9c1fc42 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionMetaService.java @@ -64,7 +64,7 @@ public static FunctionMetaService getInstance() { } private FunctionMetaService() { - this.ops = new HierarchicalSchemaPOStorageOps<>(new FunctionPOStorageOps()); + this.ops = new HierarchicalConventionPOStorageOp<>(new FunctionPOStorageOps()); } @Monitored( diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/HierarchicalSchemaPOStorageOps.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/HierarchicalConventionPOStorageOp.java similarity index 96% rename from core/src/main/java/org/apache/gravitino/storage/relational/service/HierarchicalSchemaPOStorageOps.java rename to core/src/main/java/org/apache/gravitino/storage/relational/service/HierarchicalConventionPOStorageOp.java index 3a6f1109108..c5adb67b11c 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/HierarchicalSchemaPOStorageOps.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/HierarchicalConventionPOStorageOp.java @@ -33,11 +33,11 @@ * @param persistent object type * @param MyBatis mapper type */ -public class HierarchicalSchemaPOStorageOps extends BasePOStorageOps { +public class HierarchicalConventionPOStorageOp extends BasePOStorageOps { private final BasePOStorageOps delegate; - public HierarchicalSchemaPOStorageOps(BasePOStorageOps delegate) { + public HierarchicalConventionPOStorageOp(BasePOStorageOps delegate) { this.delegate = delegate; } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java index 1f4e552f16a..18187a2f9d5 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java @@ -79,14 +79,14 @@ /** The service class for schema metadata. It provides the basic database operations for schema. */ public class SchemaMetaService { private static final SchemaMetaService INSTANCE = new SchemaMetaService(); - private SchemaPOStorageOps ops; + private BasePOStorageOps ops; public static SchemaMetaService getInstance() { return INSTANCE; } private SchemaMetaService() { - this.ops = new SchemaPOStorageOps(); + this.ops = new HierarchicalConventionPOStorageOp<>(new SchemaPOStorageOps()); } @Monitored( @@ -95,7 +95,9 @@ private SchemaMetaService() { public SchemaIds getSchemaIdByMetalakeNameAndCatalogNameAndSchemaName( String metalakeName, String catalogName, String schemaName) { NameIdentifier identifier = NameIdentifier.of(metalakeName, catalogName, schemaName); - SchemaPO schemaPO = schemaPOFetcher().apply(identifier); + SchemaPO schemaPO = + SessionUtils.getWithoutCommit( + SchemaMetaMapper.class, mapper -> ops.getPO(mapper, identifier)); if (schemaPO == null) { throw new NoSuchEntityException( @@ -453,23 +455,13 @@ public int deleteSchemaMetasByLegacyTimeline(Long legacyTimeline, int limit) { private SchemaPO getSchemaPOByIdentifier(NameIdentifier identifier) { NameIdentifierUtil.checkSchema(identifier); - return schemaPOFetcher().apply(identifier); + return SessionUtils.getWithoutCommit( + SchemaMetaMapper.class, mapper -> ops.getPO(mapper, identifier)); } private List listSchemaPOs(Namespace namespace) { - return schemaListFetcher().apply(namespace); - } - - private Function> schemaListFetcher() { - return namespace -> - SessionUtils.getWithoutCommit( - SchemaMetaMapper.class, mapper -> ops.listPOs(mapper, namespace)); - } - - private Function schemaPOFetcher() { - return identifier -> - SessionUtils.getWithoutCommit( - SchemaMetaMapper.class, mapper -> ops.getPO(mapper, identifier)); + return SessionUtils.getWithoutCommit( + SchemaMetaMapper.class, mapper -> ops.listPOs(mapper, namespace)); } private void fillSchemaPOBuilderParentEntityId(SchemaPO.Builder builder, Namespace namespace) { diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java index 16c4c46684b..43a38f1ab84 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java @@ -64,7 +64,7 @@ public static TableMetaService getInstance() { } private TableMetaService() { - this.ops = new HierarchicalSchemaPOStorageOps<>(new TablePOStorageOps()); + this.ops = new HierarchicalConventionPOStorageOp<>(new TablePOStorageOps()); } @Monitored( diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewMetaService.java index d86055e84df..62871edd16f 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewMetaService.java @@ -62,7 +62,7 @@ public static ViewMetaService getInstance() { } private ViewMetaService() { - this.ops = new HierarchicalSchemaPOStorageOps<>(new ViewPOStorageOps()); + this.ops = new HierarchicalConventionPOStorageOp<>(new ViewPOStorageOps()); } @Monitored( From 16d9a56e18a40c8c25e95001c8ca4cb131fd9e90 Mon Sep 17 00:00:00 2001 From: roryqi Date: Thu, 14 May 2026 11:30:43 +0000 Subject: [PATCH 28/44] refactor Co-Authored-By: Claude Opus 4.7 --- .../service/FunctionMetaService.java | 2 +- ...> HierarchicalConventionPOStorageOps.java} | 59 +++++++++++++++---- .../relational/service/SchemaMetaService.java | 2 +- .../relational/service/TableMetaService.java | 2 +- .../relational/service/ViewMetaService.java | 2 +- 5 files changed, 52 insertions(+), 15 deletions(-) rename core/src/main/java/org/apache/gravitino/storage/relational/service/{HierarchicalConventionPOStorageOp.java => HierarchicalConventionPOStorageOps.java} (65%) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionMetaService.java index 0b3f9c1fc42..e5017f6d556 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionMetaService.java @@ -64,7 +64,7 @@ public static FunctionMetaService getInstance() { } private FunctionMetaService() { - this.ops = new HierarchicalConventionPOStorageOp<>(new FunctionPOStorageOps()); + this.ops = new HierarchicalConventionPOStorageOps<>(new FunctionPOStorageOps()); } @Monitored( diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/HierarchicalConventionPOStorageOp.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/HierarchicalConventionPOStorageOps.java similarity index 65% rename from core/src/main/java/org/apache/gravitino/storage/relational/service/HierarchicalConventionPOStorageOp.java rename to core/src/main/java/org/apache/gravitino/storage/relational/service/HierarchicalConventionPOStorageOps.java index c5adb67b11c..e76c7f840cf 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/HierarchicalConventionPOStorageOp.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/HierarchicalConventionPOStorageOps.java @@ -19,6 +19,8 @@ package org.apache.gravitino.storage.relational.service; import java.util.List; +import java.util.function.UnaryOperator; +import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.Entity; import org.apache.gravitino.NameIdentifier; @@ -26,19 +28,27 @@ import org.apache.gravitino.utils.HierarchicalSchemaUtil; /** - * Wraps a {@link BasePOStorageOps} so full-name lookups translate the schema segment from the - * external logical separator used by the API into the internal physical separator used in storage. - * Cache-keyed lookups already use API-form names, so they pass through unchanged. + * Wraps a {@link BasePOStorageOps} to bridge the hierarchical schema naming convention. Names that + * appear in API form (logical separator) are translated to storage form (physical separator) before + * delegating, and an optional rewriter can post-process POs returned from reads (typically used to + * translate a PO field from physical back to logical for callers). * * @param persistent object type * @param MyBatis mapper type */ -public class HierarchicalConventionPOStorageOp extends BasePOStorageOps { +public class HierarchicalConventionPOStorageOps extends BasePOStorageOps { private final BasePOStorageOps delegate; + private final UnaryOperator readRewriter; - public HierarchicalConventionPOStorageOp(BasePOStorageOps delegate) { + public HierarchicalConventionPOStorageOps(BasePOStorageOps delegate) { + this(delegate, UnaryOperator.identity()); + } + + public HierarchicalConventionPOStorageOps( + BasePOStorageOps delegate, UnaryOperator readRewriter) { this.delegate = delegate; + this.readRewriter = readRewriter; } @Override @@ -58,32 +68,37 @@ public Integer updatePO(Mapper mapper, PO newPO, PO oldPO) { @Override public PO getPO(Mapper mapper, Long parentId, String name) { - return delegate.getPO(mapper, parentId, name); + return applyRead(delegate.getPO(mapper, parentId, toPhysicalIfHierarchical(name))); } @Override public List listPOs(Mapper mapper, Long parentId) { - return delegate.listPOs(mapper, parentId); + return applyRead(delegate.listPOs(mapper, parentId)); } @Override public List listPOs(Mapper mapper, Namespace namespace, List names) { - return delegate.listPOs(mapper, namespace, names); + Namespace storageNs = apiNamespaceToStorage(namespace); + List storageNames = + names.stream() + .map(HierarchicalConventionPOStorageOps::toPhysicalIfHierarchical) + .collect(Collectors.toList()); + return applyRead(delegate.listPOs(mapper, storageNs, storageNames)); } @Override public List listPOs(Mapper mapper, List uuids) { - return delegate.listPOs(mapper, uuids); + return applyRead(delegate.listPOs(mapper, uuids)); } @Override protected PO getPOByFullName(Mapper mapper, NameIdentifier identifier) { - return delegate.getPOByFullName(mapper, apiIdentifierToStorage(identifier)); + return applyRead(delegate.getPOByFullName(mapper, apiIdentifierToStorage(identifier))); } @Override protected List listPOsByNSFullName(Mapper mapper, Namespace namespace) { - return delegate.listPOsByNSFullName(mapper, apiNamespaceToStorage(namespace)); + return applyRead(delegate.listPOsByNSFullName(mapper, apiNamespaceToStorage(namespace))); } @Override @@ -96,6 +111,28 @@ protected Entity.EntityType entityType() { return delegate.entityType(); } + private PO applyRead(PO po) { + return po == null ? null : readRewriter.apply(po); + } + + private List applyRead(List pos) { + if (pos == null || pos.isEmpty()) { + return pos; + } + return pos.stream().map(this::applyRead).collect(Collectors.toList()); + } + + private static String toPhysicalIfHierarchical(String name) { + if (StringUtils.isBlank(name)) { + return name; + } + String sep = HierarchicalSchemaUtil.schemaSeparator(); + if (!name.contains(sep)) { + return name; + } + return HierarchicalSchemaUtil.logicalToPhysical(name, sep); + } + private static NameIdentifier apiIdentifierToStorage(NameIdentifier apiIdentifier) { String[] levels = apiIdentifier.namespace().levels(); if (levels.length == 2) { diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java index 18187a2f9d5..910599eb618 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java @@ -86,7 +86,7 @@ public static SchemaMetaService getInstance() { } private SchemaMetaService() { - this.ops = new HierarchicalConventionPOStorageOp<>(new SchemaPOStorageOps()); + this.ops = new HierarchicalConventionPOStorageOps<>(new SchemaPOStorageOps()); } @Monitored( diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java index 43a38f1ab84..4f13765dee6 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java @@ -64,7 +64,7 @@ public static TableMetaService getInstance() { } private TableMetaService() { - this.ops = new HierarchicalConventionPOStorageOp<>(new TablePOStorageOps()); + this.ops = new HierarchicalConventionPOStorageOps<>(new TablePOStorageOps()); } @Monitored( diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewMetaService.java index 62871edd16f..1f2d4f769de 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewMetaService.java @@ -62,7 +62,7 @@ public static ViewMetaService getInstance() { } private ViewMetaService() { - this.ops = new HierarchicalConventionPOStorageOp<>(new ViewPOStorageOps()); + this.ops = new HierarchicalConventionPOStorageOps<>(new ViewPOStorageOps()); } @Monitored( From df927bb58a36cc7483907c8eb1ba82ca4e19644c Mon Sep 17 00:00:00 2001 From: roryqi Date: Thu, 14 May 2026 11:49:53 +0000 Subject: [PATCH 29/44] refactor Co-Authored-By: Claude Opus 4.7 --- .../HierarchicalConventionPOStorageOps.java | 27 ++++++++---- .../relational/service/SchemaMetaService.java | 41 ++++++++++++++++++- .../service/TestSchemaMetaService.java | 2 +- 3 files changed, 61 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/HierarchicalConventionPOStorageOps.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/HierarchicalConventionPOStorageOps.java index e76c7f840cf..c7a6c756861 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/HierarchicalConventionPOStorageOps.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/HierarchicalConventionPOStorageOps.java @@ -30,8 +30,10 @@ /** * Wraps a {@link BasePOStorageOps} to bridge the hierarchical schema naming convention. Names that * appear in API form (logical separator) are translated to storage form (physical separator) before - * delegating, and an optional rewriter can post-process POs returned from reads (typically used to - * translate a PO field from physical back to logical for callers). + * delegating. Two optional PO rewriters allow callers to translate a PO field across the boundary: + * the read rewriter is applied to POs returned from read methods (typically physical→logical), and + * the write rewriter is applied to POs passed into write methods (typically logical→physical) so + * the SQL still receives storage-form values. * * @param persistent object type * @param MyBatis mapper type @@ -40,30 +42,34 @@ public class HierarchicalConventionPOStorageOps extends BasePOStorag private final BasePOStorageOps delegate; private final UnaryOperator readRewriter; + private final UnaryOperator writeRewriter; public HierarchicalConventionPOStorageOps(BasePOStorageOps delegate) { - this(delegate, UnaryOperator.identity()); + this(delegate, UnaryOperator.identity(), UnaryOperator.identity()); } public HierarchicalConventionPOStorageOps( - BasePOStorageOps delegate, UnaryOperator readRewriter) { + BasePOStorageOps delegate, + UnaryOperator readRewriter, + UnaryOperator writeRewriter) { this.delegate = delegate; this.readRewriter = readRewriter; + this.writeRewriter = writeRewriter; } @Override public void insertPO(Mapper mapper, PO po, boolean overwrite) { - delegate.insertPO(mapper, po, overwrite); + delegate.insertPO(mapper, writeRewriter.apply(po), overwrite); } @Override public void batchInsertPOs(Mapper mapper, List pos, boolean overwrite) { - delegate.batchInsertPOs(mapper, pos, overwrite); + delegate.batchInsertPOs(mapper, applyWrite(pos), overwrite); } @Override public Integer updatePO(Mapper mapper, PO newPO, PO oldPO) { - return delegate.updatePO(mapper, newPO, oldPO); + return delegate.updatePO(mapper, writeRewriter.apply(newPO), writeRewriter.apply(oldPO)); } @Override @@ -122,6 +128,13 @@ private List applyRead(List pos) { return pos.stream().map(this::applyRead).collect(Collectors.toList()); } + private List applyWrite(List pos) { + if (pos == null || pos.isEmpty()) { + return pos; + } + return pos.stream().map(writeRewriter).collect(Collectors.toList()); + } + private static String toPhysicalIfHierarchical(String name) { if (StringUtils.isBlank(name)) { return name; diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java index 910599eb618..fc6a37163da 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java @@ -86,7 +86,46 @@ public static SchemaMetaService getInstance() { } private SchemaMetaService() { - this.ops = new HierarchicalConventionPOStorageOps<>(new SchemaPOStorageOps()); + this.ops = + new HierarchicalConventionPOStorageOps<>( + new SchemaPOStorageOps(), + SchemaMetaService::physicalToLogicalSchemaPO, + SchemaMetaService::logicalToPhysicalSchemaPO); + } + + private static SchemaPO physicalToLogicalSchemaPO(SchemaPO po) { + String name = po.getSchemaName(); + if (name == null || !name.contains(HierarchicalSchemaUtil.physicalSeparator())) { + return po; + } + return copySchemaPOWithName( + po, + HierarchicalSchemaUtil.physicalToLogical(name, HierarchicalSchemaUtil.schemaSeparator())); + } + + private static SchemaPO logicalToPhysicalSchemaPO(SchemaPO po) { + String name = po.getSchemaName(); + if (name == null || !name.contains(HierarchicalSchemaUtil.schemaSeparator())) { + return po; + } + return copySchemaPOWithName( + po, + HierarchicalSchemaUtil.logicalToPhysical(name, HierarchicalSchemaUtil.schemaSeparator())); + } + + private static SchemaPO copySchemaPOWithName(SchemaPO po, String name) { + return SchemaPO.builder() + .withSchemaId(po.getSchemaId()) + .withSchemaName(name) + .withMetalakeId(po.getMetalakeId()) + .withCatalogId(po.getCatalogId()) + .withSchemaComment(po.getSchemaComment()) + .withProperties(po.getProperties()) + .withAuditInfo(po.getAuditInfo()) + .withCurrentVersion(po.getCurrentVersion()) + .withLastVersion(po.getLastVersion()) + .withDeletedAt(po.getDeletedAt()) + .build(); } @Monitored( diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestSchemaMetaService.java b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestSchemaMetaService.java index 2c19502caa6..20e94280195 100644 --- a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestSchemaMetaService.java +++ b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestSchemaMetaService.java @@ -254,7 +254,7 @@ public void testInsertHierarchicalSchemaCreatesAncestorsAndLeaf() throws IOExcep SchemaEntity loaded = schemaMetaService.getSchemaByIdentifier( NameIdentifier.of(metalakeName, catalogName, physicalLeaf)); - Assertions.assertEquals(physicalLeaf, loaded.name()); + Assertions.assertEquals(logicalLeaf, loaded.name()); Assertions.assertEquals("nested", loaded.comment()); } From a960062502b55e7c85ad41dc3bd53f0bc6f3766f Mon Sep 17 00:00:00 2001 From: roryqi Date: Thu, 14 May 2026 12:01:36 +0000 Subject: [PATCH 30/44] test: add unit tests for HierarchicalConventionPOStorageOps Co-Authored-By: Claude Opus 4.7 --- ...estHierarchicalConventionPOStorageOps.java | 335 ++++++++++++++++++ 1 file changed, 335 insertions(+) create mode 100644 core/src/test/java/org/apache/gravitino/storage/relational/service/TestHierarchicalConventionPOStorageOps.java diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestHierarchicalConventionPOStorageOps.java b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestHierarchicalConventionPOStorageOps.java new file mode 100644 index 00000000000..f4f67dfb068 --- /dev/null +++ b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestHierarchicalConventionPOStorageOps.java @@ -0,0 +1,335 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.service; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.function.UnaryOperator; +import org.apache.gravitino.Entity; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.Namespace; +import org.apache.gravitino.utils.HierarchicalSchemaUtil; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +public class TestHierarchicalConventionPOStorageOps { + + private static final String SEP = HierarchicalSchemaUtil.schemaSeparator(); + private static final String PHYS = HierarchicalSchemaUtil.physicalSeparator(); + + private BasePOStorageOps delegate; + private Object mapper; + + @BeforeEach + @SuppressWarnings("unchecked") + public void setUp() { + delegate = mock(BasePOStorageOps.class); + mapper = new Object(); + } + + // ---------- Input name conversion ---------- + + @Test + public void getPOByParentIdConvertsHierarchicalName() { + when(delegate.getPO(eq(mapper), eq(7L), eq("ns_a" + PHYS + "ns_b"))).thenReturn("found"); + HierarchicalConventionPOStorageOps wrapper = + new HierarchicalConventionPOStorageOps<>(delegate); + + String result = wrapper.getPO(mapper, 7L, "ns_a" + SEP + "ns_b"); + + assertEquals("found", result); + verify(delegate).getPO(mapper, 7L, "ns_a" + PHYS + "ns_b"); + } + + @Test + public void getPOByParentIdLeavesSimpleNameUnchanged() { + when(delegate.getPO(eq(mapper), eq(7L), eq("plain"))).thenReturn("po"); + HierarchicalConventionPOStorageOps wrapper = + new HierarchicalConventionPOStorageOps<>(delegate); + + String result = wrapper.getPO(mapper, 7L, "plain"); + + assertEquals("po", result); + verify(delegate).getPO(mapper, 7L, "plain"); + } + + @Test + public void getPOByParentIdReturnsNullWhenDelegateMisses() { + when(delegate.getPO(any(), any(Long.class), any(String.class))).thenReturn(null); + HierarchicalConventionPOStorageOps wrapper = + new HierarchicalConventionPOStorageOps<>( + delegate, s -> s + "-rewritten", UnaryOperator.identity()); + + assertNull(wrapper.getPO(mapper, 1L, "missing")); + } + + @Test + public void listPOsByNamespaceAndNamesConvertsEachHierarchicalName() { + List names = Arrays.asList("plain", "ns_a" + SEP + "ns_b", "other"); + Namespace ns = Namespace.of("ml", "cat"); + when(delegate.listPOs(eq(mapper), eq(ns), any(List.class))) + .thenReturn(Collections.singletonList("po")); + + HierarchicalConventionPOStorageOps wrapper = + new HierarchicalConventionPOStorageOps<>(delegate); + + wrapper.listPOs(mapper, ns, names); + + ArgumentCaptor> namesCaptor = ArgumentCaptor.forClass(List.class); + verify(delegate).listPOs(eq(mapper), eq(ns), namesCaptor.capture()); + assertEquals(Arrays.asList("plain", "ns_a" + PHYS + "ns_b", "other"), namesCaptor.getValue()); + } + + // ---------- Identifier / namespace API → storage translation ---------- + + @Test + public void getPOByFullNameConvertsSchemaIdentifierName() { + NameIdentifier ident = NameIdentifier.of(Namespace.of("ml", "cat"), "ns_a" + SEP + "ns_b"); + when(delegate.getPOByFullName(any(), any(NameIdentifier.class))).thenReturn("po"); + + HierarchicalConventionPOStorageOps wrapper = + new HierarchicalConventionPOStorageOps<>(delegate); + + wrapper.getPOByFullName(mapper, ident); + + ArgumentCaptor captor = ArgumentCaptor.forClass(NameIdentifier.class); + verify(delegate).getPOByFullName(eq(mapper), captor.capture()); + NameIdentifier converted = captor.getValue(); + assertEquals("ns_a" + PHYS + "ns_b", converted.name()); + assertEquals(Namespace.of("ml", "cat"), converted.namespace()); + } + + @Test + public void getPOByFullNameConvertsSchemaSegmentForChildIdentifier() { + NameIdentifier ident = + NameIdentifier.of(Namespace.of("ml", "cat", "ns_a" + SEP + "ns_b"), "tbl"); + when(delegate.getPOByFullName(any(), any(NameIdentifier.class))).thenReturn("po"); + + HierarchicalConventionPOStorageOps wrapper = + new HierarchicalConventionPOStorageOps<>(delegate); + + wrapper.getPOByFullName(mapper, ident); + + ArgumentCaptor captor = ArgumentCaptor.forClass(NameIdentifier.class); + verify(delegate).getPOByFullName(eq(mapper), captor.capture()); + NameIdentifier converted = captor.getValue(); + assertEquals("tbl", converted.name()); + assertEquals(Namespace.of("ml", "cat", "ns_a" + PHYS + "ns_b"), converted.namespace()); + } + + @Test + public void getPOByFullNamePassesThroughWhenNoSeparatorPresent() { + NameIdentifier ident = NameIdentifier.of(Namespace.of("ml", "cat", "schema"), "tbl"); + when(delegate.getPOByFullName(any(), any(NameIdentifier.class))).thenReturn("po"); + + HierarchicalConventionPOStorageOps wrapper = + new HierarchicalConventionPOStorageOps<>(delegate); + + wrapper.getPOByFullName(mapper, ident); + + verify(delegate).getPOByFullName(mapper, ident); + } + + @Test + public void listPOsByNSFullNameConvertsSchemaSegment() { + Namespace ns = Namespace.of("ml", "cat", "ns_a" + SEP + "ns_b"); + when(delegate.listPOsByNSFullName(any(), any(Namespace.class))) + .thenReturn(Collections.singletonList("po")); + + HierarchicalConventionPOStorageOps wrapper = + new HierarchicalConventionPOStorageOps<>(delegate); + + wrapper.listPOsByNSFullName(mapper, ns); + + ArgumentCaptor nsCaptor = ArgumentCaptor.forClass(Namespace.class); + verify(delegate).listPOsByNSFullName(eq(mapper), nsCaptor.capture()); + assertEquals(Namespace.of("ml", "cat", "ns_a" + PHYS + "ns_b"), nsCaptor.getValue()); + } + + @Test + public void listPOsByNSFullNameLeavesShortNamespaceUnchanged() { + Namespace ns = Namespace.of("ml", "cat"); + when(delegate.listPOsByNSFullName(any(), any(Namespace.class))) + .thenReturn(Collections.emptyList()); + + HierarchicalConventionPOStorageOps wrapper = + new HierarchicalConventionPOStorageOps<>(delegate); + + wrapper.listPOsByNSFullName(mapper, ns); + + verify(delegate).listPOsByNSFullName(mapper, ns); + } + + // ---------- Read rewriter ---------- + + @Test + public void readRewriterAppliedToGetPOByParentId() { + when(delegate.getPO(any(), any(Long.class), any(String.class))).thenReturn("raw"); + HierarchicalConventionPOStorageOps wrapper = + new HierarchicalConventionPOStorageOps<>( + delegate, s -> s + "-rewritten", UnaryOperator.identity()); + + assertEquals("raw-rewritten", wrapper.getPO(mapper, 1L, "plain")); + } + + @Test + public void readRewriterAppliedToListPOsByParentId() { + when(delegate.listPOs(any(), any(Long.class))).thenReturn(Arrays.asList("a", "b")); + HierarchicalConventionPOStorageOps wrapper = + new HierarchicalConventionPOStorageOps<>( + delegate, s -> s.toUpperCase(), UnaryOperator.identity()); + + assertEquals(Arrays.asList("A", "B"), wrapper.listPOs(mapper, 1L)); + } + + @Test + public void readRewriterAppliedToListPOsByIds() { + when(delegate.listPOs(any(), any(List.class))).thenReturn(Arrays.asList("x", "y")); + HierarchicalConventionPOStorageOps wrapper = + new HierarchicalConventionPOStorageOps<>(delegate, s -> s + "!", UnaryOperator.identity()); + + List ids = Arrays.asList(1L, 2L); + assertEquals(Arrays.asList("x!", "y!"), wrapper.listPOs(mapper, ids)); + } + + @Test + public void readRewriterIsNotInvokedOnNullResult() { + when(delegate.getPO(any(), any(Long.class), any(String.class))).thenReturn(null); + HierarchicalConventionPOStorageOps wrapper = + new HierarchicalConventionPOStorageOps<>( + delegate, + s -> { + throw new AssertionError("rewriter must not be invoked on null"); + }, + UnaryOperator.identity()); + + assertNull(wrapper.getPO(mapper, 1L, "x")); + } + + @Test + public void readRewriterIsNotInvokedOnEmptyList() { + when(delegate.listPOs(any(), any(Long.class))).thenReturn(Collections.emptyList()); + HierarchicalConventionPOStorageOps wrapper = + new HierarchicalConventionPOStorageOps<>( + delegate, + s -> { + throw new AssertionError("rewriter must not be invoked on empty list"); + }, + UnaryOperator.identity()); + + assertEquals(Collections.emptyList(), wrapper.listPOs(mapper, 1L)); + } + + // ---------- Write rewriter ---------- + + @Test + public void writeRewriterAppliedToInsertPO() { + HierarchicalConventionPOStorageOps wrapper = + new HierarchicalConventionPOStorageOps<>( + delegate, UnaryOperator.identity(), s -> s + "-written"); + + wrapper.insertPO(mapper, "logical", true); + + verify(delegate).insertPO(mapper, "logical-written", true); + } + + @Test + public void writeRewriterAppliedToBatchInsertPOs() { + HierarchicalConventionPOStorageOps wrapper = + new HierarchicalConventionPOStorageOps<>(delegate, UnaryOperator.identity(), s -> s + "-W"); + + wrapper.batchInsertPOs(mapper, Arrays.asList("a", "b"), false); + + ArgumentCaptor> captor = ArgumentCaptor.forClass(List.class); + verify(delegate).batchInsertPOs(eq(mapper), captor.capture(), eq(false)); + assertEquals(Arrays.asList("a-W", "b-W"), captor.getValue()); + } + + @Test + public void writeRewriterAppliedToBothPOsInUpdate() { + when(delegate.updatePO(any(), eq("new-W"), eq("old-W"))).thenReturn(1); + HierarchicalConventionPOStorageOps wrapper = + new HierarchicalConventionPOStorageOps<>(delegate, UnaryOperator.identity(), s -> s + "-W"); + + assertEquals(1, wrapper.updatePO(mapper, "new", "old")); + verify(delegate).updatePO(mapper, "new-W", "old-W"); + } + + @Test + public void singleArgConstructorUsesIdentityRewritersForReadsAndWrites() { + when(delegate.getPO(any(), any(Long.class), any(String.class))).thenReturn("po"); + HierarchicalConventionPOStorageOps wrapper = + new HierarchicalConventionPOStorageOps<>(delegate); + + assertEquals("po", wrapper.getPO(mapper, 1L, "plain")); + wrapper.insertPO(mapper, "raw", false); + verify(delegate).insertPO(mapper, "raw", false); + + wrapper.batchInsertPOs(mapper, Arrays.asList("a", "b"), true); + ArgumentCaptor> captor = ArgumentCaptor.forClass(List.class); + verify(delegate).batchInsertPOs(eq(mapper), captor.capture(), eq(true)); + assertEquals(Arrays.asList("a", "b"), captor.getValue()); + } + + // ---------- Delegation ---------- + + @Test + public void capabilitiesAndEntityTypeDelegated() { + when(delegate.capabilities()) + .thenReturn( + Arrays.asList( + BasePOStorageOps.Capability.GET_BY_NAME, BasePOStorageOps.Capability.UPDATE)); + when(delegate.entityType()).thenReturn(Entity.EntityType.SCHEMA); + + HierarchicalConventionPOStorageOps wrapper = + new HierarchicalConventionPOStorageOps<>(delegate); + + assertEquals( + Arrays.asList(BasePOStorageOps.Capability.GET_BY_NAME, BasePOStorageOps.Capability.UPDATE), + wrapper.capabilities()); + assertSame(Entity.EntityType.SCHEMA, wrapper.entityType()); + } + + @Test + public void batchInsertEmptyListIsForwarded() { + HierarchicalConventionPOStorageOps wrapper = + new HierarchicalConventionPOStorageOps<>( + delegate, + UnaryOperator.identity(), + s -> { + throw new AssertionError("rewriter must not be invoked on empty list"); + }); + + wrapper.batchInsertPOs(mapper, new ArrayList<>(), false); + + verify(delegate).batchInsertPOs(eq(mapper), eq(Collections.emptyList()), eq(false)); + } +} From 3d652c87b1b1b02cf66ca144996e7b0b260150b6 Mon Sep 17 00:00:00 2001 From: Rory Date: Thu, 14 May 2026 21:16:46 +0800 Subject: [PATCH 31/44] fix --- .../storage/relational/service/BasePOStorageOps.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/BasePOStorageOps.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/BasePOStorageOps.java index 2e351521a7f..c8e7fd609b5 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/BasePOStorageOps.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/BasePOStorageOps.java @@ -23,6 +23,7 @@ import org.apache.gravitino.GravitinoEnv; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; +import org.apache.gravitino.utils.NameIdentifierUtil; public abstract class BasePOStorageOps { public void insertPO(Mapper mapper, PO po, boolean overwrite) { @@ -47,7 +48,8 @@ public final PO getPO(Mapper mapper, NameIdentifier identifier) { && capabilities().contains(Capability.GET_BY_NS_UID)) { Long parentId = EntityIdService.getEntityId( - NameIdentifier.of(identifier.namespace().toString()), entityType()); + NameIdentifier.parse(identifier.namespace().toString()), + NameIdentifierUtil.parentEntityType(entityType())); return getPO(mapper, parentId, identifier.name()); } @@ -63,7 +65,9 @@ public final List listPOs(Mapper mapper, Namespace namespace) { if (GravitinoEnv.getInstance().cacheEnabled() && capabilities().contains(Capability.LIST_BY_NS_UID)) { Long parentId = - EntityIdService.getEntityId(NameIdentifier.of(namespace.toString()), entityType()); + EntityIdService.getEntityId( + NameIdentifier.parse(namespace.toString()), + NameIdentifierUtil.parentEntityType(entityType())); return listPOs(mapper, parentId); } From e72918b13d92b9bbc434f025207220e231798a55 Mon Sep 17 00:00:00 2001 From: Rory Date: Thu, 14 May 2026 22:19:27 +0800 Subject: [PATCH 32/44] fix --- .../relational/service/BasePOStorageOps.java | 39 +++- .../TestJDBCBackendBatchInsert.java | 188 ++++++++++++++++++ .../relational/TestRelationalEntityStore.java | 89 --------- 3 files changed, 216 insertions(+), 100 deletions(-) create mode 100644 core/src/test/java/org/apache/gravitino/storage/relational/TestJDBCBackendBatchInsert.java diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/BasePOStorageOps.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/BasePOStorageOps.java index c8e7fd609b5..53dc82e537b 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/BasePOStorageOps.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/BasePOStorageOps.java @@ -27,21 +27,28 @@ public abstract class BasePOStorageOps { public void insertPO(Mapper mapper, PO po, boolean overwrite) { - throw new UnsupportedOperationException(); + throw new UnsupportedOperationException( + "insertPO is not supported by " + getClass().getSimpleName()); } public void batchInsertPOs(Mapper mapper, List pos, boolean overwrite) { - throw new UnsupportedOperationException(); + throw new UnsupportedOperationException( + "batchInsertPOs is not supported by " + getClass().getSimpleName()); } public Integer updatePO(Mapper mapper, PO newPO, PO oldPO) { - throw new UnsupportedOperationException(); + throw new UnsupportedOperationException( + "updatePO is not supported by " + getClass().getSimpleName()); } public final PO getPO(Mapper mapper, NameIdentifier identifier) { if (!capabilities().contains(Capability.GET_BY_NS_UID) && !capabilities().contains(Capability.GET_BY_NAME)) { - throw new UnsupportedOperationException(); + throw new UnsupportedOperationException( + "getPO requires GET_BY_NS_UID or GET_BY_NAME for " + + entityType() + + ", but capabilities are " + + capabilities()); } if (GravitinoEnv.getInstance().cacheEnabled() @@ -59,7 +66,11 @@ && capabilities().contains(Capability.GET_BY_NS_UID)) { public final List listPOs(Mapper mapper, Namespace namespace) { if (!capabilities().contains(Capability.LIST_BY_NS_UID) && !capabilities().contains(Capability.LIST_BY_NS_NAME)) { - throw new UnsupportedOperationException(); + throw new UnsupportedOperationException( + "listPOs requires LIST_BY_NS_UID or LIST_BY_NS_NAME for " + + entityType() + + ", but capabilities are " + + capabilities()); } if (GravitinoEnv.getInstance().cacheEnabled() @@ -75,27 +86,33 @@ && capabilities().contains(Capability.LIST_BY_NS_UID)) { } public PO getPO(Mapper mapper, Long parentId, String name) { - throw new UnsupportedOperationException(); + throw new UnsupportedOperationException( + "getPO by parent id is not supported by " + getClass().getSimpleName()); } public List listPOs(Mapper mapper, Long parentId) { - throw new UnsupportedOperationException(); + throw new UnsupportedOperationException( + "listPOs by parent id is not supported by " + getClass().getSimpleName()); } public List listPOs(Mapper mapper, Namespace namespace, List names) { - throw new UnsupportedOperationException(); + throw new UnsupportedOperationException( + "listPOs by namespace and names is not supported by " + getClass().getSimpleName()); } public List listPOs(Mapper mapper, List uuids) { - throw new UnsupportedOperationException(); + throw new UnsupportedOperationException( + "listPOs by uuids is not supported by " + getClass().getSimpleName()); } protected PO getPOByFullName(Mapper mapper, NameIdentifier identifier) { - throw new UnsupportedOperationException(); + throw new UnsupportedOperationException( + "getPOByFullName is not supported by " + getClass().getSimpleName()); } protected List listPOsByNSFullName(Mapper mapper, Namespace namespace) { - throw new UnsupportedOperationException(); + throw new UnsupportedOperationException( + "listPOsByNSFullName is not supported by " + getClass().getSimpleName()); } public abstract List capabilities(); diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/TestJDBCBackendBatchInsert.java b/core/src/test/java/org/apache/gravitino/storage/relational/TestJDBCBackendBatchInsert.java new file mode 100644 index 00000000000..c1d17aca292 --- /dev/null +++ b/core/src/test/java/org/apache/gravitino/storage/relational/TestJDBCBackendBatchInsert.java @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import org.apache.gravitino.Entity; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.meta.SchemaEntity; +import org.apache.gravitino.storage.RandomIdGenerator; +import org.apache.gravitino.utils.HierarchicalSchemaUtil; +import org.apache.gravitino.utils.NamespaceUtil; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.TestTemplate; + +/** + * Tests schema insert paths that use relational {@code batchInsertSchemaMeta} (single or batch). + */ +public class TestJDBCBackendBatchInsert extends TestJDBCBackend { + + @TestTemplate + public void testBatchInsertSingleSchemaViaBackend() throws IOException { + String metalakeName = "metalake_batch_insert_single"; + String catalogName = "catalog_batch_insert_single"; + createAndInsertMakeLake(metalakeName); + createAndInsertCatalog(metalakeName, catalogName); + + SchemaEntity schema = + createSchemaEntity( + RandomIdGenerator.INSTANCE.nextId(), + NamespaceUtil.ofSchema(metalakeName, catalogName), + "flat_schema_batch", + AUDIT_INFO); + backend.insert(schema, false); + + SchemaEntity loaded = + (SchemaEntity) + backend.get( + NameIdentifier.of(metalakeName, catalogName, "flat_schema_batch"), + Entity.EntityType.SCHEMA); + Assertions.assertEquals(schema.id(), loaded.id()); + Assertions.assertEquals("flat_schema_batch", loaded.name()); + Assertions.assertEquals(schema.namespace(), loaded.namespace()); + + List listed = + backend.list( + NamespaceUtil.ofSchema(metalakeName, catalogName), Entity.EntityType.SCHEMA, true); + Assertions.assertEquals(1, listed.size()); + Assertions.assertEquals("flat_schema_batch", listed.get(0).name()); + } + + @TestTemplate + public void testBatchInsertHierarchicalSchemaCreatesAncestorsAndLeafViaBackend() + throws IOException { + String metalakeName = "metalake_batch_insert_nested"; + String catalogName = "catalog_batch_insert_nested"; + createAndInsertMakeLake(metalakeName); + createAndInsertCatalog(metalakeName, catalogName); + + String logicalLeaf = "ns_a:ns_b:leaf"; + String sep = HierarchicalSchemaUtil.schemaSeparator(); + String physicalLeaf = HierarchicalSchemaUtil.logicalToPhysical(logicalLeaf, sep); + SchemaEntity hierarchical = + SchemaEntity.builder() + .withId(RandomIdGenerator.INSTANCE.nextId()) + .withName(physicalLeaf) + .withNamespace(NamespaceUtil.ofSchema(metalakeName, catalogName)) + .withComment("nested") + .withProperties(Collections.emptyMap()) + .withAuditInfo(AUDIT_INFO) + .build(); + backend.insert(hierarchical, false); + + List schemas = + backend.list( + NamespaceUtil.ofSchema(metalakeName, catalogName), Entity.EntityType.SCHEMA, true); + Set logicalNames = + schemas.stream() + .map(SchemaEntity::name) + .map( + n -> { + if (n != null && n.contains(HierarchicalSchemaUtil.physicalSeparator())) { + return HierarchicalSchemaUtil.physicalToLogical(n, sep); + } + return n; + }) + .collect(Collectors.toSet()); + + Assertions.assertTrue(logicalNames.contains("ns_a")); + Assertions.assertTrue(logicalNames.contains("ns_a:ns_b")); + Assertions.assertTrue(logicalNames.contains(logicalLeaf)); + + SchemaEntity loaded = + (SchemaEntity) + backend.get( + NameIdentifier.of(metalakeName, catalogName, physicalLeaf), + Entity.EntityType.SCHEMA); + Assertions.assertEquals(logicalLeaf, loaded.name()); + Assertions.assertEquals("nested", loaded.comment()); + } + + @TestTemplate + public void testBatchInsertHierarchicalSecondLeafReusesAncestorsViaBackend() throws IOException { + String metalakeName = "metalake_batch_insert_reuse"; + String catalogName = "catalog_batch_insert_reuse"; + createAndInsertMakeLake(metalakeName); + createAndInsertCatalog(metalakeName, catalogName); + + String sep = HierarchicalSchemaUtil.schemaSeparator(); + String physSep = HierarchicalSchemaUtil.physicalSeparator(); + String physicalLeaf1 = HierarchicalSchemaUtil.logicalToPhysical("ns_a:ns_b:leaf1", sep); + String physicalLeaf2 = HierarchicalSchemaUtil.logicalToPhysical("ns_a:ns_b:leaf2", sep); + String[] parts = physicalLeaf1.split(Pattern.quote(physSep), -1); + String ancestorA = parts[0]; + String ancestorAB = String.join(physSep, Arrays.copyOfRange(parts, 0, 2)); + + SchemaEntity first = + SchemaEntity.builder() + .withId(RandomIdGenerator.INSTANCE.nextId()) + .withName(physicalLeaf1) + .withNamespace(NamespaceUtil.ofSchema(metalakeName, catalogName)) + .withComment("first") + .withProperties(Collections.emptyMap()) + .withAuditInfo(AUDIT_INFO) + .build(); + backend.insert(first, false); + + long idA = + ((SchemaEntity) + backend.get( + NameIdentifier.of(metalakeName, catalogName, ancestorA), + Entity.EntityType.SCHEMA)) + .id(); + long idAB = + ((SchemaEntity) + backend.get( + NameIdentifier.of(metalakeName, catalogName, ancestorAB), + Entity.EntityType.SCHEMA)) + .id(); + + SchemaEntity second = + SchemaEntity.builder() + .withId(RandomIdGenerator.INSTANCE.nextId()) + .withName(physicalLeaf2) + .withNamespace(NamespaceUtil.ofSchema(metalakeName, catalogName)) + .withComment("second") + .withProperties(Collections.emptyMap()) + .withAuditInfo(AUDIT_INFO) + .build(); + backend.insert(second, false); + + Assertions.assertEquals( + idA, + ((SchemaEntity) + backend.get( + NameIdentifier.of(metalakeName, catalogName, ancestorA), + Entity.EntityType.SCHEMA)) + .id()); + Assertions.assertEquals( + idAB, + ((SchemaEntity) + backend.get( + NameIdentifier.of(metalakeName, catalogName, ancestorAB), + Entity.EntityType.SCHEMA)) + .id()); + } +} diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalEntityStore.java b/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalEntityStore.java index 5c84c9e2206..4878b46de0b 100644 --- a/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalEntityStore.java +++ b/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalEntityStore.java @@ -195,93 +195,4 @@ void testBatchInsertRelationsInvalidatesCacheForAllSrcsAndDst() .verify(cache) .invalidate(dst, Entity.EntityType.ROLE, SupportsRelationOperations.Type.OWNER_REL); } - - @Test - void testBatchInsertRelationsWithEmptyListIsNoOp() throws IOException, IllegalAccessException { - NoOpsCache cache = (NoOpsCache) FieldUtils.readField(store, "cache", true); - - store.batchInsertRelations( - SupportsRelationOperations.Type.OWNER_REL, - List.of(), - Entity.EntityType.SCHEMA, - NameIdentifier.of("metalake", "role1"), - Entity.EntityType.ROLE, - false); - - Mockito.verify(backend, Mockito.never()) - .batchInsertRelations(any(), any(), any(), any(), any(), any(Boolean.class)); - Mockito.verify(cache, Mockito.never()).invalidate(any(), any(), any()); - } - - @Test - void testUpdateEntityRelationsInvalidatesCacheAfterBackendUpdate() - throws IOException, NoSuchEntityException, EntityAlreadyExistsException, - IllegalAccessException { - NameIdentifier src = NameIdentifier.of("metalake", "catalog", "schema", "table1"); - NameIdentifier destToAdd = NameIdentifier.of("metalake", "tag1"); - NameIdentifier destToRemove = NameIdentifier.of("metalake", "tag2"); - NameIdentifier[] destEntitiesToAdd = new NameIdentifier[] {destToAdd}; - NameIdentifier[] destEntitiesToRemove = new NameIdentifier[] {destToRemove}; - NoOpsCache cache = (NoOpsCache) FieldUtils.readField(store, "cache", true); - - Mockito.doAnswer( - invocation -> { - Mockito.verify(cache, Mockito.never()) - .invalidate( - src, - Entity.EntityType.TABLE, - SupportsRelationOperations.Type.TAG_METADATA_OBJECT_REL); - Mockito.verify(cache, Mockito.never()) - .invalidate( - destToAdd, - Entity.EntityType.TABLE, - SupportsRelationOperations.Type.TAG_METADATA_OBJECT_REL); - Mockito.verify(cache, Mockito.never()) - .invalidate( - destToRemove, - Entity.EntityType.TABLE, - SupportsRelationOperations.Type.TAG_METADATA_OBJECT_REL); - return List.of(); - }) - .when(backend) - .updateEntityRelations( - SupportsRelationOperations.Type.TAG_METADATA_OBJECT_REL, - src, - Entity.EntityType.TABLE, - destEntitiesToAdd, - destEntitiesToRemove); - - store.updateEntityRelations( - SupportsRelationOperations.Type.TAG_METADATA_OBJECT_REL, - src, - Entity.EntityType.TABLE, - destEntitiesToAdd, - destEntitiesToRemove); - - InOrder inOrder = Mockito.inOrder(backend, cache); - inOrder - .verify(backend) - .updateEntityRelations( - SupportsRelationOperations.Type.TAG_METADATA_OBJECT_REL, - src, - Entity.EntityType.TABLE, - destEntitiesToAdd, - destEntitiesToRemove); - inOrder - .verify(cache) - .invalidate( - src, Entity.EntityType.TABLE, SupportsRelationOperations.Type.TAG_METADATA_OBJECT_REL); - inOrder - .verify(cache) - .invalidate( - destToAdd, - Entity.EntityType.TABLE, - SupportsRelationOperations.Type.TAG_METADATA_OBJECT_REL); - inOrder - .verify(cache) - .invalidate( - destToRemove, - Entity.EntityType.TABLE, - SupportsRelationOperations.Type.TAG_METADATA_OBJECT_REL); - } } From 61484651a356ecdbc43c7548b74adc83c6c4f653 Mon Sep 17 00:00:00 2001 From: Rory Date: Fri, 15 May 2026 00:41:23 +0800 Subject: [PATCH 33/44] fix ut --- .../relational/service/FunctionMetaService.java | 12 ++++++++++-- .../relational/service/TableMetaService.java | 13 +++++++++++-- .../storage/relational/service/ViewMetaService.java | 13 +++++++++++-- 3 files changed, 32 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionMetaService.java index e5017f6d556..df36c8d9450 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionMetaService.java @@ -225,8 +225,16 @@ public int deleteFunctionVersionsByRetentionCount(Long versionRetentionCount, in baseMetricName = "getFunctionPOByIdentifier") FunctionPO getFunctionPOByIdentifier(NameIdentifier ident) { NameIdentifierUtil.checkFunction(ident); - return SessionUtils.getWithoutCommit( - FunctionMetaMapper.class, mapper -> ops.getPO(mapper, ident)); + FunctionPO functionPO = + SessionUtils.getWithoutCommit(FunctionMetaMapper.class, mapper -> ops.getPO(mapper, ident)); + + if (functionPO == null) { + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, + Entity.EntityType.FUNCTION.name().toLowerCase(Locale.ROOT), + ident.name()); + } + return functionPO; } @Monitored( diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java index 4f13765dee6..5acae72486e 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java @@ -352,8 +352,17 @@ public List batchGetTableByIdentifier(List identifi private TablePO getTablePOByIdentifier(NameIdentifier identifier) { NameIdentifierUtil.checkTable(identifier); - return SessionUtils.getWithoutCommit( - TableMetaMapper.class, mapper -> ops.getPO(mapper, identifier)); + TablePO tablePO = + SessionUtils.getWithoutCommit( + TableMetaMapper.class, mapper -> ops.getPO(mapper, identifier)); + if (tablePO == null) { + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, + Entity.EntityType.TABLE.name().toLowerCase(), + identifier.name()); + } + + return tablePO; } private List listTablePOs(Namespace namespace) { diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewMetaService.java index 1f2d4f769de..fd18f8ec75a 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewMetaService.java @@ -278,8 +278,17 @@ private ViewPO updateViewPO(ViewPO oldViewPO, ViewEntity newEntity) { private ViewPO getViewPOByIdentifier(NameIdentifier identifier) { NameIdentifierUtil.checkView(identifier); - return SessionUtils.getWithoutCommit( - ViewMetaMapper.class, mapper -> ops.getPO(mapper, identifier)); + ViewPO viewPO = + SessionUtils.getWithoutCommit( + ViewMetaMapper.class, mapper -> ops.getPO(mapper, identifier)); + if (viewPO == null) { + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, + Entity.EntityType.VIEW.name().toLowerCase(), + identifier.name()); + } + + return viewPO; } private List listViewPOs(Namespace namespace) { From f31fe4adf39a156794ee8b1fc00f7db186da62e3 Mon Sep 17 00:00:00 2001 From: Rory Date: Fri, 15 May 2026 00:44:59 +0800 Subject: [PATCH 34/44] fix ut --- .../relational/service/SchemaMetaService.java | 82 ++++++++++--------- 1 file changed, 45 insertions(+), 37 deletions(-) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java index fc6a37163da..9d7f5041095 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java @@ -93,41 +93,6 @@ private SchemaMetaService() { SchemaMetaService::logicalToPhysicalSchemaPO); } - private static SchemaPO physicalToLogicalSchemaPO(SchemaPO po) { - String name = po.getSchemaName(); - if (name == null || !name.contains(HierarchicalSchemaUtil.physicalSeparator())) { - return po; - } - return copySchemaPOWithName( - po, - HierarchicalSchemaUtil.physicalToLogical(name, HierarchicalSchemaUtil.schemaSeparator())); - } - - private static SchemaPO logicalToPhysicalSchemaPO(SchemaPO po) { - String name = po.getSchemaName(); - if (name == null || !name.contains(HierarchicalSchemaUtil.schemaSeparator())) { - return po; - } - return copySchemaPOWithName( - po, - HierarchicalSchemaUtil.logicalToPhysical(name, HierarchicalSchemaUtil.schemaSeparator())); - } - - private static SchemaPO copySchemaPOWithName(SchemaPO po, String name) { - return SchemaPO.builder() - .withSchemaId(po.getSchemaId()) - .withSchemaName(name) - .withMetalakeId(po.getMetalakeId()) - .withCatalogId(po.getCatalogId()) - .withSchemaComment(po.getSchemaComment()) - .withProperties(po.getProperties()) - .withAuditInfo(po.getAuditInfo()) - .withCurrentVersion(po.getCurrentVersion()) - .withLastVersion(po.getLastVersion()) - .withDeletedAt(po.getDeletedAt()) - .build(); - } - @Monitored( metricsSource = GRAVITINO_RELATIONAL_STORE_METRIC_NAME, baseMetricName = "getSchemaIdByMetalakeNameAndCatalogNameAndSchemaName") @@ -494,8 +459,16 @@ public int deleteSchemaMetasByLegacyTimeline(Long legacyTimeline, int limit) { private SchemaPO getSchemaPOByIdentifier(NameIdentifier identifier) { NameIdentifierUtil.checkSchema(identifier); - return SessionUtils.getWithoutCommit( - SchemaMetaMapper.class, mapper -> ops.getPO(mapper, identifier)); + SchemaPO schemaPO = + SessionUtils.getWithoutCommit( + SchemaMetaMapper.class, mapper -> ops.getPO(mapper, identifier)); + if (schemaPO == null) { + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, + Entity.EntityType.SCHEMA.name().toLowerCase(), + identifier.name()); + } + return schemaPO; } private List listSchemaPOs(Namespace namespace) { @@ -542,4 +515,39 @@ private static long nextIdForNestedAncestor() { } return generator.nextId(); } + + private static SchemaPO physicalToLogicalSchemaPO(SchemaPO po) { + String name = po.getSchemaName(); + if (name == null || !name.contains(HierarchicalSchemaUtil.physicalSeparator())) { + return po; + } + return copySchemaPOWithName( + po, + HierarchicalSchemaUtil.physicalToLogical(name, HierarchicalSchemaUtil.schemaSeparator())); + } + + private static SchemaPO logicalToPhysicalSchemaPO(SchemaPO po) { + String name = po.getSchemaName(); + if (name == null || !name.contains(HierarchicalSchemaUtil.schemaSeparator())) { + return po; + } + return copySchemaPOWithName( + po, + HierarchicalSchemaUtil.logicalToPhysical(name, HierarchicalSchemaUtil.schemaSeparator())); + } + + private static SchemaPO copySchemaPOWithName(SchemaPO po, String name) { + return SchemaPO.builder() + .withSchemaId(po.getSchemaId()) + .withSchemaName(name) + .withMetalakeId(po.getMetalakeId()) + .withCatalogId(po.getCatalogId()) + .withSchemaComment(po.getSchemaComment()) + .withProperties(po.getProperties()) + .withAuditInfo(po.getAuditInfo()) + .withCurrentVersion(po.getCurrentVersion()) + .withLastVersion(po.getLastVersion()) + .withDeletedAt(po.getDeletedAt()) + .build(); + } } From 60a12ecbd1c66e47c9981bf455c251b0b0770500 Mon Sep 17 00:00:00 2001 From: Rory Date: Fri, 15 May 2026 00:53:16 +0800 Subject: [PATCH 35/44] revert --- .../relational/TestRelationalEntityStore.java | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalEntityStore.java b/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalEntityStore.java index 4878b46de0b..5c84c9e2206 100644 --- a/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalEntityStore.java +++ b/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalEntityStore.java @@ -195,4 +195,93 @@ void testBatchInsertRelationsInvalidatesCacheForAllSrcsAndDst() .verify(cache) .invalidate(dst, Entity.EntityType.ROLE, SupportsRelationOperations.Type.OWNER_REL); } + + @Test + void testBatchInsertRelationsWithEmptyListIsNoOp() throws IOException, IllegalAccessException { + NoOpsCache cache = (NoOpsCache) FieldUtils.readField(store, "cache", true); + + store.batchInsertRelations( + SupportsRelationOperations.Type.OWNER_REL, + List.of(), + Entity.EntityType.SCHEMA, + NameIdentifier.of("metalake", "role1"), + Entity.EntityType.ROLE, + false); + + Mockito.verify(backend, Mockito.never()) + .batchInsertRelations(any(), any(), any(), any(), any(), any(Boolean.class)); + Mockito.verify(cache, Mockito.never()).invalidate(any(), any(), any()); + } + + @Test + void testUpdateEntityRelationsInvalidatesCacheAfterBackendUpdate() + throws IOException, NoSuchEntityException, EntityAlreadyExistsException, + IllegalAccessException { + NameIdentifier src = NameIdentifier.of("metalake", "catalog", "schema", "table1"); + NameIdentifier destToAdd = NameIdentifier.of("metalake", "tag1"); + NameIdentifier destToRemove = NameIdentifier.of("metalake", "tag2"); + NameIdentifier[] destEntitiesToAdd = new NameIdentifier[] {destToAdd}; + NameIdentifier[] destEntitiesToRemove = new NameIdentifier[] {destToRemove}; + NoOpsCache cache = (NoOpsCache) FieldUtils.readField(store, "cache", true); + + Mockito.doAnswer( + invocation -> { + Mockito.verify(cache, Mockito.never()) + .invalidate( + src, + Entity.EntityType.TABLE, + SupportsRelationOperations.Type.TAG_METADATA_OBJECT_REL); + Mockito.verify(cache, Mockito.never()) + .invalidate( + destToAdd, + Entity.EntityType.TABLE, + SupportsRelationOperations.Type.TAG_METADATA_OBJECT_REL); + Mockito.verify(cache, Mockito.never()) + .invalidate( + destToRemove, + Entity.EntityType.TABLE, + SupportsRelationOperations.Type.TAG_METADATA_OBJECT_REL); + return List.of(); + }) + .when(backend) + .updateEntityRelations( + SupportsRelationOperations.Type.TAG_METADATA_OBJECT_REL, + src, + Entity.EntityType.TABLE, + destEntitiesToAdd, + destEntitiesToRemove); + + store.updateEntityRelations( + SupportsRelationOperations.Type.TAG_METADATA_OBJECT_REL, + src, + Entity.EntityType.TABLE, + destEntitiesToAdd, + destEntitiesToRemove); + + InOrder inOrder = Mockito.inOrder(backend, cache); + inOrder + .verify(backend) + .updateEntityRelations( + SupportsRelationOperations.Type.TAG_METADATA_OBJECT_REL, + src, + Entity.EntityType.TABLE, + destEntitiesToAdd, + destEntitiesToRemove); + inOrder + .verify(cache) + .invalidate( + src, Entity.EntityType.TABLE, SupportsRelationOperations.Type.TAG_METADATA_OBJECT_REL); + inOrder + .verify(cache) + .invalidate( + destToAdd, + Entity.EntityType.TABLE, + SupportsRelationOperations.Type.TAG_METADATA_OBJECT_REL); + inOrder + .verify(cache) + .invalidate( + destToRemove, + Entity.EntityType.TABLE, + SupportsRelationOperations.Type.TAG_METADATA_OBJECT_REL); + } } From 617ba19c04d570fdde88333824d461ff0e68c4d0 Mon Sep 17 00:00:00 2001 From: Rory Date: Fri, 15 May 2026 00:55:59 +0800 Subject: [PATCH 36/44] revert --- .../relational/TestRelationalEntityStore.java | 56 ------------------- 1 file changed, 56 deletions(-) diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalEntityStore.java b/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalEntityStore.java index 5c84c9e2206..042c29681b8 100644 --- a/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalEntityStore.java +++ b/core/src/test/java/org/apache/gravitino/storage/relational/TestRelationalEntityStore.java @@ -22,7 +22,6 @@ import static org.mockito.ArgumentMatchers.eq; import java.io.IOException; -import java.util.Arrays; import java.util.List; import java.util.function.Function; import org.apache.commons.lang3.reflect.FieldUtils; @@ -158,61 +157,6 @@ void testInsertRelationInvalidatesCacheAfterBackendInsert() dst, Entity.EntityType.TAG, SupportsRelationOperations.Type.TAG_METADATA_OBJECT_REL); } - @Test - void testBatchInsertRelationsInvalidatesCacheForAllSrcsAndDst() - throws IOException, IllegalAccessException { - NameIdentifier src1 = NameIdentifier.of("metalake", "schema1"); - NameIdentifier src2 = NameIdentifier.of("metalake", "schema2"); - NameIdentifier dst = NameIdentifier.of("metalake", "role1"); - List srcs = Arrays.asList(src1, src2); - NoOpsCache cache = (NoOpsCache) FieldUtils.readField(store, "cache", true); - - store.batchInsertRelations( - SupportsRelationOperations.Type.OWNER_REL, - srcs, - Entity.EntityType.SCHEMA, - dst, - Entity.EntityType.ROLE, - false); - - InOrder inOrder = Mockito.inOrder(backend, cache); - inOrder - .verify(backend) - .batchInsertRelations( - SupportsRelationOperations.Type.OWNER_REL, - srcs, - Entity.EntityType.SCHEMA, - dst, - Entity.EntityType.ROLE, - false); - inOrder - .verify(cache) - .invalidate(src1, Entity.EntityType.SCHEMA, SupportsRelationOperations.Type.OWNER_REL); - inOrder - .verify(cache) - .invalidate(src2, Entity.EntityType.SCHEMA, SupportsRelationOperations.Type.OWNER_REL); - inOrder - .verify(cache) - .invalidate(dst, Entity.EntityType.ROLE, SupportsRelationOperations.Type.OWNER_REL); - } - - @Test - void testBatchInsertRelationsWithEmptyListIsNoOp() throws IOException, IllegalAccessException { - NoOpsCache cache = (NoOpsCache) FieldUtils.readField(store, "cache", true); - - store.batchInsertRelations( - SupportsRelationOperations.Type.OWNER_REL, - List.of(), - Entity.EntityType.SCHEMA, - NameIdentifier.of("metalake", "role1"), - Entity.EntityType.ROLE, - false); - - Mockito.verify(backend, Mockito.never()) - .batchInsertRelations(any(), any(), any(), any(), any(), any(Boolean.class)); - Mockito.verify(cache, Mockito.never()).invalidate(any(), any(), any()); - } - @Test void testUpdateEntityRelationsInvalidatesCacheAfterBackendUpdate() throws IOException, NoSuchEntityException, EntityAlreadyExistsException, From 525f2c6f941127aa94a834c8d934844e9ef374c1 Mon Sep 17 00:00:00 2001 From: roryqi Date: Thu, 14 May 2026 18:07:05 +0000 Subject: [PATCH 37/44] fix ut --- .../gravitino/catalog/fileset/TestFilesetCatalogOperations.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/TestFilesetCatalogOperations.java b/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/TestFilesetCatalogOperations.java index a0dbce64a20..492cdcc38bc 100644 --- a/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/TestFilesetCatalogOperations.java +++ b/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/TestFilesetCatalogOperations.java @@ -325,7 +325,7 @@ public static void setUp() throws IllegalAccessException { MockedStatic catalogMetaServiceMockedStatic = Mockito.mockStatic(CatalogMetaService.class); MockedStatic schemaMetaServiceMockedStatic = - Mockito.mockStatic(SchemaMetaService.class); + Mockito.mockStatic(SchemaMetaService.class, Mockito.CALLS_REAL_METHODS); metalakeMetaServiceMockedStatic .when(MetalakeMetaService::getInstance) From 722859c3e1d477b62eef25ffff0574b9df547be6 Mon Sep 17 00:00:00 2001 From: roryqi Date: Fri, 15 May 2026 01:48:51 +0000 Subject: [PATCH 38/44] distinguish missing catalog/schema in join-based lookups Throw NoSuchEntityException with the correct entity type by checking catalogId/schemaId on the joined row, so callers see "catalog not found" or "schema not found" instead of a misleading parent error. Co-Authored-By: Claude Opus 4.7 --- .../service/SchemaPOStorageOps.java | 34 +++++++++++-- .../relational/service/TablePOStorageOps.java | 50 +++++++++++++++++-- 2 files changed, 76 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaPOStorageOps.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaPOStorageOps.java index 4f2175173ce..930f9d831a5 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaPOStorageOps.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaPOStorageOps.java @@ -19,9 +19,11 @@ package org.apache.gravitino.storage.relational.service; import java.util.List; +import java.util.stream.Collectors; import org.apache.gravitino.Entity; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; +import org.apache.gravitino.exceptions.NoSuchEntityException; import org.apache.gravitino.storage.relational.mapper.SchemaMetaMapper; import org.apache.gravitino.storage.relational.po.SchemaPO; @@ -51,8 +53,22 @@ public SchemaPO getPO(SchemaMetaMapper mapper, Long parentId, String name) { @Override protected SchemaPO getPOByFullName(SchemaMetaMapper mapper, NameIdentifier identifier) { Namespace namespace = identifier.namespace(); - return mapper.selectSchemaByFullQualifiedName( - namespace.level(0), namespace.level(1), identifier.name()); + SchemaPO po = + mapper.selectSchemaByFullQualifiedName( + namespace.level(0), namespace.level(1), identifier.name()); + // INNER JOIN on metalake/catalog: a null PO means the metalake or catalog does not exist. + if (po == null) { + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, + Entity.EntityType.CATALOG.name().toLowerCase(), + namespace.level(1)); + } + // LEFT JOIN on schema_meta: a row with non-null catalogId but null schemaId means + // the catalog exists but the schema does not. + if (po.getSchemaId() == null) { + return null; + } + return po; } @Override @@ -75,8 +91,18 @@ public List listPOs(SchemaMetaMapper schemaMetaMapper, List uuid @Override protected List listPOsByNSFullName( SchemaMetaMapper schemaMetaMapper, Namespace namespace) { - return schemaMetaMapper.listSchemaPOsByFullQualifiedName( - namespace.level(0), namespace.level(1)); + List pos = + schemaMetaMapper.listSchemaPOsByFullQualifiedName(namespace.level(0), namespace.level(1)); + // An empty result means the parent metalake or catalog does not exist. + if (pos.isEmpty()) { + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, + Entity.EntityType.CATALOG.name().toLowerCase(), + namespace.level(1)); + } + // Same LEFT JOIN behavior as getPOByFullName: filter out the placeholder row that + // represents an existing catalog without any matching schema. + return pos.stream().filter(po -> po.getSchemaId() != null).collect(Collectors.toList()); } @Override diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/TablePOStorageOps.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/TablePOStorageOps.java index b9e7e204a1d..727db88819f 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/TablePOStorageOps.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/TablePOStorageOps.java @@ -19,9 +19,11 @@ package org.apache.gravitino.storage.relational.service; import java.util.List; +import java.util.stream.Collectors; import org.apache.gravitino.Entity; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; +import org.apache.gravitino.exceptions.NoSuchEntityException; import org.apache.gravitino.storage.relational.mapper.TableMetaMapper; import org.apache.gravitino.storage.relational.po.TablePO; @@ -51,8 +53,30 @@ public TablePO getPO(TableMetaMapper mapper, Long parentId, String name) { @Override protected TablePO getPOByFullName(TableMetaMapper mapper, NameIdentifier identifier) { Namespace namespace = identifier.namespace(); - return mapper.selectTableByFullQualifiedName( - namespace.level(0), namespace.level(1), namespace.level(2), identifier.name()); + TablePO po = + mapper.selectTableByFullQualifiedName( + namespace.level(0), namespace.level(1), namespace.level(2), identifier.name()); + // INNER JOIN on metalake/catalog: a null PO means the metalake or catalog does not exist. + if (po == null) { + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, + Entity.EntityType.CATALOG.name().toLowerCase(), + namespace.level(1)); + } + // LEFT JOIN on schema_meta: a row with non-null catalogId but null schemaId means + // the catalog exists but the schema does not. + if (po.getSchemaId() == null) { + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, + Entity.EntityType.SCHEMA.name().toLowerCase(), + namespace.level(2)); + } + // LEFT JOIN on table_meta: a row with non-null schemaId but null tableId means + // the schema exists but the table does not. + if (po.getTableId() == null) { + return null; + } + return po; } @Override @@ -75,8 +99,26 @@ public List listPOs(TableMetaMapper mapper, List uuids) { @Override protected List listPOsByNSFullName(TableMetaMapper mapper, Namespace namespace) { - return mapper.listTablePOsByFullQualifiedName( - namespace.level(0), namespace.level(1), namespace.level(2)); + List pos = + mapper.listTablePOsByFullQualifiedName( + namespace.level(0), namespace.level(1), namespace.level(2)); + // INNER JOIN on metalake/catalog: an empty result means the metalake or catalog does not exist. + if (pos.isEmpty()) { + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, + Entity.EntityType.CATALOG.name().toLowerCase(), + namespace.level(1)); + } + // LEFT JOIN on schema_meta: rows with non-null catalogId but null schemaId mean the + // catalog exists but the schema does not. + if (pos.get(0).getSchemaId() == null) { + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, + Entity.EntityType.SCHEMA.name().toLowerCase(), + namespace.level(2)); + } + // LEFT JOIN on table_meta: filter out the placeholder row for a schema without tables. + return pos.stream().filter(po -> po.getTableId() != null).collect(Collectors.toList()); } @Override From 9809cb6dac21b373cf972e6669a212f7f1b836bd Mon Sep 17 00:00:00 2001 From: roryqi Date: Fri, 15 May 2026 01:57:24 +0000 Subject: [PATCH 39/44] apply join-aware missing-entity checks to view and function Use the same pattern as Table: when the join returns no row or a placeholder row, throw NoSuchEntityException with the correct entity type by inspecting catalogId/schemaId on the result. Also switch the function-by-full-name select from INNER JOIN to LEFT JOIN on the version table, matching the list query and the Table/View shape. Co-Authored-By: Claude Opus 4.7 --- .../base/FunctionMetaBaseSQLProvider.java | 2 +- .../service/FunctionPOStorageOps.java | 50 +++++++++++++++++-- .../relational/service/ViewPOStorageOps.java | 50 +++++++++++++++++-- 3 files changed, 93 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/FunctionMetaBaseSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/FunctionMetaBaseSQLProvider.java index 88d229bcf76..d15cf8bb3af 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/FunctionMetaBaseSQLProvider.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/FunctionMetaBaseSQLProvider.java @@ -168,7 +168,7 @@ public String selectFunctionMetaByFullQualifiedName( %s fm ON sm.schema_id = fm.schema_id AND fm.function_name = #{functionName} AND fm.deleted_at = 0 - INNER JOIN + LEFT JOIN %s vi ON fm.function_id = vi.function_id AND fm.function_current_version = vi.version AND vi.deleted_at = 0 diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionPOStorageOps.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionPOStorageOps.java index 5ac67b34b4f..0c60a3b4aba 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionPOStorageOps.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionPOStorageOps.java @@ -19,9 +19,11 @@ package org.apache.gravitino.storage.relational.service; import java.util.List; +import java.util.stream.Collectors; import org.apache.gravitino.Entity; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; +import org.apache.gravitino.exceptions.NoSuchEntityException; import org.apache.gravitino.storage.relational.mapper.FunctionMetaMapper; import org.apache.gravitino.storage.relational.po.FunctionPO; @@ -51,8 +53,30 @@ public FunctionPO getPO(FunctionMetaMapper mapper, Long parentId, String name) { @Override protected FunctionPO getPOByFullName(FunctionMetaMapper mapper, NameIdentifier identifier) { Namespace namespace = identifier.namespace(); - return mapper.selectFunctionMetaByFullQualifiedName( - namespace.level(0), namespace.level(1), namespace.level(2), identifier.name()); + FunctionPO po = + mapper.selectFunctionMetaByFullQualifiedName( + namespace.level(0), namespace.level(1), namespace.level(2), identifier.name()); + // INNER JOIN on metalake/catalog: a null PO means the metalake or catalog does not exist. + if (po == null) { + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, + Entity.EntityType.CATALOG.name().toLowerCase(), + namespace.level(1)); + } + // LEFT JOIN on schema_meta: a row with non-null catalogId but null schemaId means + // the catalog exists but the schema does not. + if (po.schemaId() == null) { + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, + Entity.EntityType.SCHEMA.name().toLowerCase(), + namespace.level(2)); + } + // LEFT JOIN on function_meta: a row with non-null schemaId but null functionId means + // the schema exists but the function does not. + if (po.functionId() == null) { + return null; + } + return po; } @Override @@ -67,8 +91,26 @@ public List listPOs(FunctionMetaMapper mapper, List uuids) { @Override protected List listPOsByNSFullName(FunctionMetaMapper mapper, Namespace namespace) { - return mapper.listFunctionPOsByFullQualifiedName( - namespace.level(0), namespace.level(1), namespace.level(2)); + List pos = + mapper.listFunctionPOsByFullQualifiedName( + namespace.level(0), namespace.level(1), namespace.level(2)); + // INNER JOIN on metalake/catalog: an empty result means the metalake or catalog does not exist. + if (pos.isEmpty()) { + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, + Entity.EntityType.CATALOG.name().toLowerCase(), + namespace.level(1)); + } + // LEFT JOIN on schema_meta: rows with non-null catalogId but null schemaId mean the + // catalog exists but the schema does not. + if (pos.get(0).schemaId() == null) { + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, + Entity.EntityType.SCHEMA.name().toLowerCase(), + namespace.level(2)); + } + // LEFT JOIN on function_meta: filter out the placeholder row for a schema without functions. + return pos.stream().filter(po -> po.functionId() != null).collect(Collectors.toList()); } @Override diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewPOStorageOps.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewPOStorageOps.java index ffa125f1696..a40a9a62da6 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewPOStorageOps.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewPOStorageOps.java @@ -19,9 +19,11 @@ package org.apache.gravitino.storage.relational.service; import java.util.List; +import java.util.stream.Collectors; import org.apache.gravitino.Entity; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; +import org.apache.gravitino.exceptions.NoSuchEntityException; import org.apache.gravitino.storage.relational.mapper.ViewMetaMapper; import org.apache.gravitino.storage.relational.po.ViewPO; @@ -51,8 +53,30 @@ public ViewPO getPO(ViewMetaMapper mapper, Long parentId, String name) { @Override protected ViewPO getPOByFullName(ViewMetaMapper mapper, NameIdentifier identifier) { Namespace namespace = identifier.namespace(); - return mapper.selectViewByFullQualifiedName( - namespace.level(0), namespace.level(1), namespace.level(2), identifier.name()); + ViewPO po = + mapper.selectViewByFullQualifiedName( + namespace.level(0), namespace.level(1), namespace.level(2), identifier.name()); + // INNER JOIN on metalake/catalog: a null PO means the metalake or catalog does not exist. + if (po == null) { + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, + Entity.EntityType.CATALOG.name().toLowerCase(), + namespace.level(1)); + } + // LEFT JOIN on schema_meta: a row with non-null catalogId but null schemaId means + // the catalog exists but the schema does not. + if (po.getSchemaId() == null) { + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, + Entity.EntityType.SCHEMA.name().toLowerCase(), + namespace.level(2)); + } + // LEFT JOIN on view_meta: a row with non-null schemaId but null viewId means + // the schema exists but the view does not. + if (po.getViewId() == null) { + return null; + } + return po; } @Override @@ -67,8 +91,26 @@ public List listPOs(ViewMetaMapper mapper, List uuids) { @Override protected List listPOsByNSFullName(ViewMetaMapper mapper, Namespace namespace) { - return mapper.listViewPOsByFullQualifiedName( - namespace.level(0), namespace.level(1), namespace.level(2)); + List pos = + mapper.listViewPOsByFullQualifiedName( + namespace.level(0), namespace.level(1), namespace.level(2)); + // INNER JOIN on metalake/catalog: an empty result means the metalake or catalog does not exist. + if (pos.isEmpty()) { + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, + Entity.EntityType.CATALOG.name().toLowerCase(), + namespace.level(1)); + } + // LEFT JOIN on schema_meta: rows with non-null catalogId but null schemaId mean the + // catalog exists but the schema does not. + if (pos.get(0).getSchemaId() == null) { + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, + Entity.EntityType.SCHEMA.name().toLowerCase(), + namespace.level(2)); + } + // LEFT JOIN on view_meta: filter out the placeholder row for a schema without views. + return pos.stream().filter(po -> po.getViewId() != null).collect(Collectors.toList()); } @Override From 09be3a61d36e1a8774a87432788f93784e6a18a9 Mon Sep 17 00:00:00 2001 From: roryqi Date: Fri, 15 May 2026 02:00:52 +0000 Subject: [PATCH 40/44] revert function get-by-full-name sql and storage-ops changes Keep the INNER JOIN on the version table for the function-by-full-name select. Revert the Java getPOByFullName checks accordingly so the caller continues to handle the null PO case. Co-Authored-By: Claude Opus 4.7 --- .../base/FunctionMetaBaseSQLProvider.java | 2 +- .../service/FunctionPOStorageOps.java | 26 ++----------------- 2 files changed, 3 insertions(+), 25 deletions(-) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/FunctionMetaBaseSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/FunctionMetaBaseSQLProvider.java index d15cf8bb3af..88d229bcf76 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/FunctionMetaBaseSQLProvider.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/FunctionMetaBaseSQLProvider.java @@ -168,7 +168,7 @@ public String selectFunctionMetaByFullQualifiedName( %s fm ON sm.schema_id = fm.schema_id AND fm.function_name = #{functionName} AND fm.deleted_at = 0 - LEFT JOIN + INNER JOIN %s vi ON fm.function_id = vi.function_id AND fm.function_current_version = vi.version AND vi.deleted_at = 0 diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionPOStorageOps.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionPOStorageOps.java index 0c60a3b4aba..db81f09757a 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionPOStorageOps.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionPOStorageOps.java @@ -53,30 +53,8 @@ public FunctionPO getPO(FunctionMetaMapper mapper, Long parentId, String name) { @Override protected FunctionPO getPOByFullName(FunctionMetaMapper mapper, NameIdentifier identifier) { Namespace namespace = identifier.namespace(); - FunctionPO po = - mapper.selectFunctionMetaByFullQualifiedName( - namespace.level(0), namespace.level(1), namespace.level(2), identifier.name()); - // INNER JOIN on metalake/catalog: a null PO means the metalake or catalog does not exist. - if (po == null) { - throw new NoSuchEntityException( - NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, - Entity.EntityType.CATALOG.name().toLowerCase(), - namespace.level(1)); - } - // LEFT JOIN on schema_meta: a row with non-null catalogId but null schemaId means - // the catalog exists but the schema does not. - if (po.schemaId() == null) { - throw new NoSuchEntityException( - NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, - Entity.EntityType.SCHEMA.name().toLowerCase(), - namespace.level(2)); - } - // LEFT JOIN on function_meta: a row with non-null schemaId but null functionId means - // the schema exists but the function does not. - if (po.functionId() == null) { - return null; - } - return po; + return mapper.selectFunctionMetaByFullQualifiedName( + namespace.level(0), namespace.level(1), namespace.level(2), identifier.name()); } @Override From 4e40b4849f9d7f615aeac7fcd5a2e6dad75f2d81 Mon Sep 17 00:00:00 2001 From: Rory Date: Fri, 15 May 2026 16:45:10 +0800 Subject: [PATCH 41/44] Address the comments --- .../relational/service/BasePOStorageOps.java | 68 +------- .../service/FunctionMetaService.java | 7 +- .../service/FunctionPOStorageOps.java | 15 +- .../HierarchicalConventionPOStorageOps.java | 95 ++++++----- .../service/POStorageReadRouting.java | 118 +++++++++++++ .../relational/service/SchemaMetaService.java | 11 +- .../service/SchemaPOStorageOps.java | 16 +- .../relational/service/TableMetaService.java | 6 +- .../relational/service/TablePOStorageOps.java | 16 +- .../relational/service/ViewMetaService.java | 6 +- .../relational/service/ViewPOStorageOps.java | 15 +- ...estHierarchicalConventionPOStorageOps.java | 36 ++-- .../service/TestPOStorageReadRouting.java | 157 ++++++++++++++++++ 13 files changed, 385 insertions(+), 181 deletions(-) create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/service/POStorageReadRouting.java create mode 100644 core/src/test/java/org/apache/gravitino/storage/relational/service/TestPOStorageReadRouting.java diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/BasePOStorageOps.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/BasePOStorageOps.java index 53dc82e537b..be66afc109c 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/BasePOStorageOps.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/BasePOStorageOps.java @@ -20,10 +20,8 @@ import java.util.List; import org.apache.gravitino.Entity; -import org.apache.gravitino.GravitinoEnv; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; -import org.apache.gravitino.utils.NameIdentifierUtil; public abstract class BasePOStorageOps { public void insertPO(Mapper mapper, PO po, boolean overwrite) { @@ -41,48 +39,12 @@ public Integer updatePO(Mapper mapper, PO newPO, PO oldPO) { "updatePO is not supported by " + getClass().getSimpleName()); } - public final PO getPO(Mapper mapper, NameIdentifier identifier) { - if (!capabilities().contains(Capability.GET_BY_NS_UID) - && !capabilities().contains(Capability.GET_BY_NAME)) { - throw new UnsupportedOperationException( - "getPO requires GET_BY_NS_UID or GET_BY_NAME for " - + entityType() - + ", but capabilities are " - + capabilities()); - } - - if (GravitinoEnv.getInstance().cacheEnabled() - && capabilities().contains(Capability.GET_BY_NS_UID)) { - Long parentId = - EntityIdService.getEntityId( - NameIdentifier.parse(identifier.namespace().toString()), - NameIdentifierUtil.parentEntityType(entityType())); - return getPO(mapper, parentId, identifier.name()); - } - - return getPOByFullName(mapper, identifier); - } - - public final List listPOs(Mapper mapper, Namespace namespace) { - if (!capabilities().contains(Capability.LIST_BY_NS_UID) - && !capabilities().contains(Capability.LIST_BY_NS_NAME)) { - throw new UnsupportedOperationException( - "listPOs requires LIST_BY_NS_UID or LIST_BY_NS_NAME for " - + entityType() - + ", but capabilities are " - + capabilities()); - } - - if (GravitinoEnv.getInstance().cacheEnabled() - && capabilities().contains(Capability.LIST_BY_NS_UID)) { - Long parentId = - EntityIdService.getEntityId( - NameIdentifier.parse(namespace.toString()), - NameIdentifierUtil.parentEntityType(entityType())); - return listPOs(mapper, parentId); - } - - return listPOsByNSFullName(mapper, namespace); + /** + * When {@code true} and the entity-id cache is enabled, callers may resolve rows by parent entity + * id plus short name via {@link #getPO} and {@link #listPOs}; see {@link POStorageReadRouting}. + */ + public boolean supportsParentIdRelationalRead() { + return false; } public PO getPO(Mapper mapper, Long parentId, String name) { @@ -105,29 +67,15 @@ public List listPOs(Mapper mapper, List uuids) { "listPOs by uuids is not supported by " + getClass().getSimpleName()); } - protected PO getPOByFullName(Mapper mapper, NameIdentifier identifier) { + public PO getPOByFullName(Mapper mapper, NameIdentifier identifier) { throw new UnsupportedOperationException( "getPOByFullName is not supported by " + getClass().getSimpleName()); } - protected List listPOsByNSFullName(Mapper mapper, Namespace namespace) { + public List listPOsByNSFullName(Mapper mapper, Namespace namespace) { throw new UnsupportedOperationException( "listPOsByNSFullName is not supported by " + getClass().getSimpleName()); } - public abstract List capabilities(); - protected abstract Entity.EntityType entityType(); - - public enum Capability { - INSERT, - BATCH_INSERT, - UPDATE, - GET_BY_NAME, - GET_BY_NS_UID, - LIST_BY_NS_UID, - LIST_BY_NS_NAME, - LIST_BY_NAME_FILTER, - LIST_BY_UID_FILTER - } } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionMetaService.java index df36c8d9450..b8967e70f42 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionMetaService.java @@ -226,7 +226,9 @@ public int deleteFunctionVersionsByRetentionCount(Long versionRetentionCount, in FunctionPO getFunctionPOByIdentifier(NameIdentifier ident) { NameIdentifierUtil.checkFunction(ident); FunctionPO functionPO = - SessionUtils.getWithoutCommit(FunctionMetaMapper.class, mapper -> ops.getPO(mapper, ident)); + SessionUtils.getWithoutCommit( + FunctionMetaMapper.class, + mapper -> POStorageReadRouting.getPO(mapper, ident, ops, Entity.EntityType.FUNCTION)); if (functionPO == null) { throw new NoSuchEntityException( @@ -274,7 +276,8 @@ public FunctionEntity updateFunction( private List listFunctionPOs(Namespace namespace) { return SessionUtils.getWithoutCommit( - FunctionMetaMapper.class, mapper -> ops.listPOs(mapper, namespace)); + FunctionMetaMapper.class, + mapper -> POStorageReadRouting.listPOs(mapper, namespace, ops, Entity.EntityType.FUNCTION)); } private void fillFunctionPOBuilderParentEntityId( diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionPOStorageOps.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionPOStorageOps.java index db81f09757a..21eabb0edad 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionPOStorageOps.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionPOStorageOps.java @@ -51,7 +51,7 @@ public FunctionPO getPO(FunctionMetaMapper mapper, Long parentId, String name) { } @Override - protected FunctionPO getPOByFullName(FunctionMetaMapper mapper, NameIdentifier identifier) { + public FunctionPO getPOByFullName(FunctionMetaMapper mapper, NameIdentifier identifier) { Namespace namespace = identifier.namespace(); return mapper.selectFunctionMetaByFullQualifiedName( namespace.level(0), namespace.level(1), namespace.level(2), identifier.name()); @@ -68,7 +68,7 @@ public List listPOs(FunctionMetaMapper mapper, List uuids) { } @Override - protected List listPOsByNSFullName(FunctionMetaMapper mapper, Namespace namespace) { + public List listPOsByNSFullName(FunctionMetaMapper mapper, Namespace namespace) { List pos = mapper.listFunctionPOsByFullQualifiedName( namespace.level(0), namespace.level(1), namespace.level(2)); @@ -92,15 +92,8 @@ protected List listPOsByNSFullName(FunctionMetaMapper mapper, Namesp } @Override - public List capabilities() { - return List.of( - Capability.INSERT, - Capability.UPDATE, - Capability.GET_BY_NAME, - Capability.GET_BY_NS_UID, - Capability.LIST_BY_NS_UID, - Capability.LIST_BY_NS_NAME, - Capability.LIST_BY_UID_FILTER); + public boolean supportsParentIdRelationalRead() { + return true; } @Override diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/HierarchicalConventionPOStorageOps.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/HierarchicalConventionPOStorageOps.java index c7a6c756861..607fd24cc47 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/HierarchicalConventionPOStorageOps.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/HierarchicalConventionPOStorageOps.java @@ -28,12 +28,13 @@ import org.apache.gravitino.utils.HierarchicalSchemaUtil; /** - * Wraps a {@link BasePOStorageOps} to bridge the hierarchical schema naming convention. Names that - * appear in API form (logical separator) are translated to storage form (physical separator) before - * delegating. Two optional PO rewriters allow callers to translate a PO field across the boundary: - * the read rewriter is applied to POs returned from read methods (typically physical→logical), and - * the write rewriter is applied to POs passed into write methods (typically logical→physical) so - * the SQL still receives storage-form values. + * Wraps a {@link BasePOStorageOps} to bridge the hierarchical schema naming convention. Identifiers + * and namespace segments in logical form (logical separator) are translated to physical form + * (physical separator) before delegating. Two optional PO rewriters allow callers to translate a PO + * field across the boundary: {@code physicalToLogicalRewriter} is applied to POs returned from read + * methods (typically physical→logical), and {@code logicalToPhysicalRewriter} is applied to POs + * passed into write methods (typically logical→physical) so the SQL still receives storage-form + * values. * * @param persistent object type * @param MyBatis mapper type @@ -41,8 +42,8 @@ public class HierarchicalConventionPOStorageOps extends BasePOStorageOps { private final BasePOStorageOps delegate; - private final UnaryOperator readRewriter; - private final UnaryOperator writeRewriter; + private final UnaryOperator physicalToLogicalRewriter; + private final UnaryOperator logicalToPhysicalRewriter; public HierarchicalConventionPOStorageOps(BasePOStorageOps delegate) { this(delegate, UnaryOperator.identity(), UnaryOperator.identity()); @@ -50,16 +51,16 @@ public HierarchicalConventionPOStorageOps(BasePOStorageOps delegate) public HierarchicalConventionPOStorageOps( BasePOStorageOps delegate, - UnaryOperator readRewriter, - UnaryOperator writeRewriter) { + UnaryOperator physicalToLogicalRewriter, + UnaryOperator logicalToPhysicalRewriter) { this.delegate = delegate; - this.readRewriter = readRewriter; - this.writeRewriter = writeRewriter; + this.physicalToLogicalRewriter = physicalToLogicalRewriter; + this.logicalToPhysicalRewriter = logicalToPhysicalRewriter; } @Override public void insertPO(Mapper mapper, PO po, boolean overwrite) { - delegate.insertPO(mapper, writeRewriter.apply(po), overwrite); + delegate.insertPO(mapper, logicalToPhysicalRewriter.apply(po), overwrite); } @Override @@ -69,7 +70,8 @@ public void batchInsertPOs(Mapper mapper, List pos, boolean overwrite) { @Override public Integer updatePO(Mapper mapper, PO newPO, PO oldPO) { - return delegate.updatePO(mapper, writeRewriter.apply(newPO), writeRewriter.apply(oldPO)); + return delegate.updatePO( + mapper, logicalToPhysicalRewriter.apply(newPO), logicalToPhysicalRewriter.apply(oldPO)); } @Override @@ -83,13 +85,13 @@ public List listPOs(Mapper mapper, Long parentId) { } @Override - public List listPOs(Mapper mapper, Namespace namespace, List names) { - Namespace storageNs = apiNamespaceToStorage(namespace); - List storageNames = + public List listPOs(Mapper mapper, Namespace logicalNamespace, List names) { + Namespace physicalNamespace = logicalToPhysicalNamespace(logicalNamespace); + List physicalNames = names.stream() .map(HierarchicalConventionPOStorageOps::toPhysicalIfHierarchical) .collect(Collectors.toList()); - return applyRead(delegate.listPOs(mapper, storageNs, storageNames)); + return applyRead(delegate.listPOs(mapper, physicalNamespace, physicalNames)); } @Override @@ -98,18 +100,20 @@ public List listPOs(Mapper mapper, List uuids) { } @Override - protected PO getPOByFullName(Mapper mapper, NameIdentifier identifier) { - return applyRead(delegate.getPOByFullName(mapper, apiIdentifierToStorage(identifier))); + public PO getPOByFullName(Mapper mapper, NameIdentifier logical) { + NameIdentifier physical = logicalToPhysicalIdentifier(logical); + return applyRead(delegate.getPOByFullName(mapper, physical)); } @Override - protected List listPOsByNSFullName(Mapper mapper, Namespace namespace) { - return applyRead(delegate.listPOsByNSFullName(mapper, apiNamespaceToStorage(namespace))); + public List listPOsByNSFullName(Mapper mapper, Namespace logical) { + Namespace physical = logicalToPhysicalNamespace(logical); + return applyRead(delegate.listPOsByNSFullName(mapper, physical)); } @Override - public List capabilities() { - return delegate.capabilities(); + public boolean supportsParentIdRelationalRead() { + return delegate.supportsParentIdRelationalRead(); } @Override @@ -118,7 +122,7 @@ protected Entity.EntityType entityType() { } private PO applyRead(PO po) { - return po == null ? null : readRewriter.apply(po); + return po == null ? null : physicalToLogicalRewriter.apply(po); } private List applyRead(List pos) { @@ -132,7 +136,7 @@ private List applyWrite(List pos) { if (pos == null || pos.isEmpty()) { return pos; } - return pos.stream().map(writeRewriter).collect(Collectors.toList()); + return pos.stream().map(logicalToPhysicalRewriter).collect(Collectors.toList()); } private static String toPhysicalIfHierarchical(String name) { @@ -146,50 +150,49 @@ private static String toPhysicalIfHierarchical(String name) { return HierarchicalSchemaUtil.logicalToPhysical(name, sep); } - private static NameIdentifier apiIdentifierToStorage(NameIdentifier apiIdentifier) { - String[] levels = apiIdentifier.namespace().levels(); + private static NameIdentifier logicalToPhysicalIdentifier(NameIdentifier logical) { + String[] levels = logical.namespace().levels(); if (levels.length == 2) { - String rawName = apiIdentifier.name(); - String storageName = + String rawName = logical.name(); + String physicalName = StringUtils.isNotBlank(rawName) ? HierarchicalSchemaUtil.logicalToPhysical( rawName, HierarchicalSchemaUtil.schemaSeparator()) : rawName; - if (storageName.equals(apiIdentifier.name())) { - return apiIdentifier; + if (physicalName.equals(logical.name())) { + return logical; } - return NameIdentifier.of(apiIdentifier.namespace(), storageName); + return NameIdentifier.of(logical.namespace(), physicalName); } if (levels.length == 3) { String rawSeg = levels[2]; - String storageSchema = + String physicalSchema = StringUtils.isNotBlank(rawSeg) ? HierarchicalSchemaUtil.logicalToPhysical( rawSeg, HierarchicalSchemaUtil.schemaSeparator()) : rawSeg; - if (storageSchema.equals(levels[2])) { - return apiIdentifier; + if (physicalSchema.equals(levels[2])) { + return logical; } - return NameIdentifier.of( - Namespace.of(levels[0], levels[1], storageSchema), apiIdentifier.name()); + return NameIdentifier.of(Namespace.of(levels[0], levels[1], physicalSchema), logical.name()); } - return apiIdentifier; + return logical; } - private static Namespace apiNamespaceToStorage(Namespace apiNamespace) { - String[] levels = apiNamespace.levels(); + private static Namespace logicalToPhysicalNamespace(Namespace logical) { + String[] levels = logical.levels(); if (levels.length != 3) { - return apiNamespace; + return logical; } String rawSeg = levels[2]; - String storageSchema = + String physicalSchema = StringUtils.isNotBlank(rawSeg) ? HierarchicalSchemaUtil.logicalToPhysical( rawSeg, HierarchicalSchemaUtil.schemaSeparator()) : rawSeg; - if (storageSchema.equals(levels[2])) { - return apiNamespace; + if (physicalSchema.equals(levels[2])) { + return logical; } - return Namespace.of(levels[0], levels[1], storageSchema); + return Namespace.of(levels[0], levels[1], physicalSchema); } } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/POStorageReadRouting.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/POStorageReadRouting.java new file mode 100644 index 00000000000..3b41ecb59c3 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/POStorageReadRouting.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.service; + +import java.util.List; +import org.apache.gravitino.Entity; +import org.apache.gravitino.GravitinoEnv; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.Namespace; +import org.apache.gravitino.utils.NameIdentifierUtil; + +/** + * Routes relational PO reads between parent-id based SQL (when the entity-id cache is enabled) and + * full qualified name SQL. Callers keep cache policy in one place instead of embedding it in {@link + * BasePOStorageOps}. + */ +public final class POStorageReadRouting { + + private POStorageReadRouting() {} + + /** + * Loads a PO using parent-id based SQL when the entity-id cache is enabled and {@code ops} + * supports that path; otherwise uses full qualified name SQL. + * + * @param mapper MyBatis mapper session + * @param identifier entity name identifier (logical naming when wrapped by hierarchical ops) + * @param ops storage operations delegate + * @param entityType entity type for {@code identifier} (used to resolve the parent id) + * @param persistent object type + * @param mapper type + * @return loaded PO or null when the delegate returns null + */ + public static PO getPO( + Mapper mapper, + NameIdentifier identifier, + BasePOStorageOps ops, + Entity.EntityType entityType) { + return getPO(mapper, identifier, ops, entityType, GravitinoEnv.getInstance().cacheEnabled()); + } + + /** + * Same as {@link #getPO} but uses an explicit cache flag (typically for tests). + * + * @param cacheEnabled when true, prefer parent-id based reads when supported + */ + public static PO getPO( + Mapper mapper, + NameIdentifier identifier, + BasePOStorageOps ops, + Entity.EntityType entityType, + boolean cacheEnabled) { + if (cacheEnabled && ops.supportsParentIdRelationalRead()) { + Long parentId = + EntityIdService.getEntityId( + NameIdentifier.parse(identifier.namespace().toString()), + NameIdentifierUtil.parentEntityType(entityType)); + return ops.getPO(mapper, parentId, identifier.name()); + } + return ops.getPOByFullName(mapper, identifier); + } + + /** + * Lists POs under a namespace using parent-id based SQL when the entity-id cache is enabled and + * {@code ops} supports that path; otherwise uses full qualified namespace SQL. + * + * @param mapper MyBatis mapper session + * @param namespace parent namespace for the listed entities + * @param ops storage operations delegate + * @param entityType entity type stored under {@code namespace} (used to resolve the parent id) + * @param persistent object type + * @param mapper type + * @return list from the delegate (may be empty) + */ + public static List listPOs( + Mapper mapper, + Namespace namespace, + BasePOStorageOps ops, + Entity.EntityType entityType) { + return listPOs(mapper, namespace, ops, entityType, GravitinoEnv.getInstance().cacheEnabled()); + } + + /** + * Same as {@link #listPOs} but uses an explicit cache flag (typically for tests). + * + * @param cacheEnabled when true, prefer parent-id based reads when supported + */ + public static List listPOs( + Mapper mapper, + Namespace namespace, + BasePOStorageOps ops, + Entity.EntityType entityType, + boolean cacheEnabled) { + if (cacheEnabled && ops.supportsParentIdRelationalRead()) { + Long parentId = + EntityIdService.getEntityId( + NameIdentifier.parse(namespace.toString()), + NameIdentifierUtil.parentEntityType(entityType)); + return ops.listPOs(mapper, parentId); + } + return ops.listPOsByNSFullName(mapper, namespace); + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java index 9d7f5041095..da755e07e6d 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java @@ -101,7 +101,9 @@ public SchemaIds getSchemaIdByMetalakeNameAndCatalogNameAndSchemaName( NameIdentifier identifier = NameIdentifier.of(metalakeName, catalogName, schemaName); SchemaPO schemaPO = SessionUtils.getWithoutCommit( - SchemaMetaMapper.class, mapper -> ops.getPO(mapper, identifier)); + SchemaMetaMapper.class, + mapper -> + POStorageReadRouting.getPO(mapper, identifier, ops, Entity.EntityType.SCHEMA)); if (schemaPO == null) { throw new NoSuchEntityException( @@ -461,7 +463,9 @@ private SchemaPO getSchemaPOByIdentifier(NameIdentifier identifier) { NameIdentifierUtil.checkSchema(identifier); SchemaPO schemaPO = SessionUtils.getWithoutCommit( - SchemaMetaMapper.class, mapper -> ops.getPO(mapper, identifier)); + SchemaMetaMapper.class, + mapper -> + POStorageReadRouting.getPO(mapper, identifier, ops, Entity.EntityType.SCHEMA)); if (schemaPO == null) { throw new NoSuchEntityException( NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, @@ -473,7 +477,8 @@ private SchemaPO getSchemaPOByIdentifier(NameIdentifier identifier) { private List listSchemaPOs(Namespace namespace) { return SessionUtils.getWithoutCommit( - SchemaMetaMapper.class, mapper -> ops.listPOs(mapper, namespace)); + SchemaMetaMapper.class, + mapper -> POStorageReadRouting.listPOs(mapper, namespace, ops, Entity.EntityType.SCHEMA)); } private void fillSchemaPOBuilderParentEntityId(SchemaPO.Builder builder, Namespace namespace) { diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaPOStorageOps.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaPOStorageOps.java index 930f9d831a5..3ddba833890 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaPOStorageOps.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaPOStorageOps.java @@ -51,7 +51,7 @@ public SchemaPO getPO(SchemaMetaMapper mapper, Long parentId, String name) { } @Override - protected SchemaPO getPOByFullName(SchemaMetaMapper mapper, NameIdentifier identifier) { + public SchemaPO getPOByFullName(SchemaMetaMapper mapper, NameIdentifier identifier) { Namespace namespace = identifier.namespace(); SchemaPO po = mapper.selectSchemaByFullQualifiedName( @@ -89,7 +89,7 @@ public List listPOs(SchemaMetaMapper schemaMetaMapper, List uuid } @Override - protected List listPOsByNSFullName( + public List listPOsByNSFullName( SchemaMetaMapper schemaMetaMapper, Namespace namespace) { List pos = schemaMetaMapper.listSchemaPOsByFullQualifiedName(namespace.level(0), namespace.level(1)); @@ -106,16 +106,8 @@ protected List listPOsByNSFullName( } @Override - public List capabilities() { - return List.of( - Capability.BATCH_INSERT, - Capability.UPDATE, - Capability.GET_BY_NAME, - Capability.GET_BY_NS_UID, - Capability.LIST_BY_NS_NAME, - Capability.LIST_BY_NS_UID, - Capability.LIST_BY_NAME_FILTER, - Capability.LIST_BY_UID_FILTER); + public boolean supportsParentIdRelationalRead() { + return true; } @Override diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java index 5acae72486e..d4486c88a1d 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java @@ -354,7 +354,8 @@ private TablePO getTablePOByIdentifier(NameIdentifier identifier) { NameIdentifierUtil.checkTable(identifier); TablePO tablePO = SessionUtils.getWithoutCommit( - TableMetaMapper.class, mapper -> ops.getPO(mapper, identifier)); + TableMetaMapper.class, + mapper -> POStorageReadRouting.getPO(mapper, identifier, ops, Entity.EntityType.TABLE)); if (tablePO == null) { throw new NoSuchEntityException( NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, @@ -367,7 +368,8 @@ private TablePO getTablePOByIdentifier(NameIdentifier identifier) { private List listTablePOs(Namespace namespace) { return SessionUtils.getWithoutCommit( - TableMetaMapper.class, mapper -> ops.listPOs(mapper, namespace)); + TableMetaMapper.class, + mapper -> POStorageReadRouting.listPOs(mapper, namespace, ops, Entity.EntityType.TABLE)); } private void fillTablePOBuilderParentEntityId(TablePO.Builder builder, Namespace namespace) { diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/TablePOStorageOps.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/TablePOStorageOps.java index 727db88819f..d687c93ec88 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/TablePOStorageOps.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/TablePOStorageOps.java @@ -51,7 +51,7 @@ public TablePO getPO(TableMetaMapper mapper, Long parentId, String name) { } @Override - protected TablePO getPOByFullName(TableMetaMapper mapper, NameIdentifier identifier) { + public TablePO getPOByFullName(TableMetaMapper mapper, NameIdentifier identifier) { Namespace namespace = identifier.namespace(); TablePO po = mapper.selectTableByFullQualifiedName( @@ -98,7 +98,7 @@ public List listPOs(TableMetaMapper mapper, List uuids) { } @Override - protected List listPOsByNSFullName(TableMetaMapper mapper, Namespace namespace) { + public List listPOsByNSFullName(TableMetaMapper mapper, Namespace namespace) { List pos = mapper.listTablePOsByFullQualifiedName( namespace.level(0), namespace.level(1), namespace.level(2)); @@ -122,16 +122,8 @@ protected List listPOsByNSFullName(TableMetaMapper mapper, Namespace na } @Override - public List capabilities() { - return List.of( - Capability.INSERT, - Capability.UPDATE, - Capability.GET_BY_NAME, - Capability.GET_BY_NS_UID, - Capability.LIST_BY_NS_UID, - Capability.LIST_BY_NS_NAME, - Capability.LIST_BY_NAME_FILTER, - Capability.LIST_BY_UID_FILTER); + public boolean supportsParentIdRelationalRead() { + return true; } @Override diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewMetaService.java index fd18f8ec75a..4384e2b27b6 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewMetaService.java @@ -280,7 +280,8 @@ private ViewPO getViewPOByIdentifier(NameIdentifier identifier) { NameIdentifierUtil.checkView(identifier); ViewPO viewPO = SessionUtils.getWithoutCommit( - ViewMetaMapper.class, mapper -> ops.getPO(mapper, identifier)); + ViewMetaMapper.class, + mapper -> POStorageReadRouting.getPO(mapper, identifier, ops, Entity.EntityType.VIEW)); if (viewPO == null) { throw new NoSuchEntityException( NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, @@ -293,6 +294,7 @@ private ViewPO getViewPOByIdentifier(NameIdentifier identifier) { private List listViewPOs(Namespace namespace) { return SessionUtils.getWithoutCommit( - ViewMetaMapper.class, mapper -> ops.listPOs(mapper, namespace)); + ViewMetaMapper.class, + mapper -> POStorageReadRouting.listPOs(mapper, namespace, ops, Entity.EntityType.VIEW)); } } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewPOStorageOps.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewPOStorageOps.java index a40a9a62da6..231b624b0b5 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewPOStorageOps.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewPOStorageOps.java @@ -51,7 +51,7 @@ public ViewPO getPO(ViewMetaMapper mapper, Long parentId, String name) { } @Override - protected ViewPO getPOByFullName(ViewMetaMapper mapper, NameIdentifier identifier) { + public ViewPO getPOByFullName(ViewMetaMapper mapper, NameIdentifier identifier) { Namespace namespace = identifier.namespace(); ViewPO po = mapper.selectViewByFullQualifiedName( @@ -90,7 +90,7 @@ public List listPOs(ViewMetaMapper mapper, List uuids) { } @Override - protected List listPOsByNSFullName(ViewMetaMapper mapper, Namespace namespace) { + public List listPOsByNSFullName(ViewMetaMapper mapper, Namespace namespace) { List pos = mapper.listViewPOsByFullQualifiedName( namespace.level(0), namespace.level(1), namespace.level(2)); @@ -114,15 +114,8 @@ protected List listPOsByNSFullName(ViewMetaMapper mapper, Namespace name } @Override - public List capabilities() { - return List.of( - Capability.INSERT, - Capability.UPDATE, - Capability.GET_BY_NAME, - Capability.GET_BY_NS_UID, - Capability.LIST_BY_NS_UID, - Capability.LIST_BY_NS_NAME, - Capability.LIST_BY_UID_FILTER); + public boolean supportsParentIdRelationalRead() { + return true; } @Override diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestHierarchicalConventionPOStorageOps.java b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestHierarchicalConventionPOStorageOps.java index f4f67dfb068..a2ab01d112b 100644 --- a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestHierarchicalConventionPOStorageOps.java +++ b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestHierarchicalConventionPOStorageOps.java @@ -21,6 +21,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; @@ -108,7 +109,7 @@ public void listPOsByNamespaceAndNamesConvertsEachHierarchicalName() { assertEquals(Arrays.asList("plain", "ns_a" + PHYS + "ns_b", "other"), namesCaptor.getValue()); } - // ---------- Identifier / namespace API → storage translation ---------- + // ---------- Identifier / namespace logical → physical translation ---------- @Test public void getPOByFullNameConvertsSchemaIdentifierName() { @@ -188,10 +189,10 @@ public void listPOsByNSFullNameLeavesShortNamespaceUnchanged() { verify(delegate).listPOsByNSFullName(mapper, ns); } - // ---------- Read rewriter ---------- + // ---------- physicalToLogicalRewriter (read path) ---------- @Test - public void readRewriterAppliedToGetPOByParentId() { + public void physicalToLogicalRewriterAppliedToGetPOByParentId() { when(delegate.getPO(any(), any(Long.class), any(String.class))).thenReturn("raw"); HierarchicalConventionPOStorageOps wrapper = new HierarchicalConventionPOStorageOps<>( @@ -201,7 +202,7 @@ public void readRewriterAppliedToGetPOByParentId() { } @Test - public void readRewriterAppliedToListPOsByParentId() { + public void physicalToLogicalRewriterAppliedToListPOsByParentId() { when(delegate.listPOs(any(), any(Long.class))).thenReturn(Arrays.asList("a", "b")); HierarchicalConventionPOStorageOps wrapper = new HierarchicalConventionPOStorageOps<>( @@ -211,7 +212,7 @@ public void readRewriterAppliedToListPOsByParentId() { } @Test - public void readRewriterAppliedToListPOsByIds() { + public void physicalToLogicalRewriterAppliedToListPOsByIds() { when(delegate.listPOs(any(), any(List.class))).thenReturn(Arrays.asList("x", "y")); HierarchicalConventionPOStorageOps wrapper = new HierarchicalConventionPOStorageOps<>(delegate, s -> s + "!", UnaryOperator.identity()); @@ -221,7 +222,7 @@ public void readRewriterAppliedToListPOsByIds() { } @Test - public void readRewriterIsNotInvokedOnNullResult() { + public void physicalToLogicalRewriterIsNotInvokedOnNullResult() { when(delegate.getPO(any(), any(Long.class), any(String.class))).thenReturn(null); HierarchicalConventionPOStorageOps wrapper = new HierarchicalConventionPOStorageOps<>( @@ -235,7 +236,7 @@ public void readRewriterIsNotInvokedOnNullResult() { } @Test - public void readRewriterIsNotInvokedOnEmptyList() { + public void physicalToLogicalRewriterIsNotInvokedOnEmptyList() { when(delegate.listPOs(any(), any(Long.class))).thenReturn(Collections.emptyList()); HierarchicalConventionPOStorageOps wrapper = new HierarchicalConventionPOStorageOps<>( @@ -248,10 +249,10 @@ public void readRewriterIsNotInvokedOnEmptyList() { assertEquals(Collections.emptyList(), wrapper.listPOs(mapper, 1L)); } - // ---------- Write rewriter ---------- + // ---------- logicalToPhysicalRewriter (write path) ---------- @Test - public void writeRewriterAppliedToInsertPO() { + public void logicalToPhysicalRewriterAppliedToInsertPO() { HierarchicalConventionPOStorageOps wrapper = new HierarchicalConventionPOStorageOps<>( delegate, UnaryOperator.identity(), s -> s + "-written"); @@ -262,7 +263,7 @@ public void writeRewriterAppliedToInsertPO() { } @Test - public void writeRewriterAppliedToBatchInsertPOs() { + public void logicalToPhysicalRewriterAppliedToBatchInsertPOs() { HierarchicalConventionPOStorageOps wrapper = new HierarchicalConventionPOStorageOps<>(delegate, UnaryOperator.identity(), s -> s + "-W"); @@ -274,7 +275,7 @@ public void writeRewriterAppliedToBatchInsertPOs() { } @Test - public void writeRewriterAppliedToBothPOsInUpdate() { + public void logicalToPhysicalRewriterAppliedToBothPOsInUpdate() { when(delegate.updatePO(any(), eq("new-W"), eq("old-W"))).thenReturn(1); HierarchicalConventionPOStorageOps wrapper = new HierarchicalConventionPOStorageOps<>(delegate, UnaryOperator.identity(), s -> s + "-W"); @@ -284,7 +285,7 @@ public void writeRewriterAppliedToBothPOsInUpdate() { } @Test - public void singleArgConstructorUsesIdentityRewritersForReadsAndWrites() { + public void singleArgConstructorUsesIdentityRewritersForBothDirections() { when(delegate.getPO(any(), any(Long.class), any(String.class))).thenReturn("po"); HierarchicalConventionPOStorageOps wrapper = new HierarchicalConventionPOStorageOps<>(delegate); @@ -302,19 +303,14 @@ public void singleArgConstructorUsesIdentityRewritersForReadsAndWrites() { // ---------- Delegation ---------- @Test - public void capabilitiesAndEntityTypeDelegated() { - when(delegate.capabilities()) - .thenReturn( - Arrays.asList( - BasePOStorageOps.Capability.GET_BY_NAME, BasePOStorageOps.Capability.UPDATE)); + public void supportsParentIdRelationalReadAndEntityTypeDelegated() { + when(delegate.supportsParentIdRelationalRead()).thenReturn(true); when(delegate.entityType()).thenReturn(Entity.EntityType.SCHEMA); HierarchicalConventionPOStorageOps wrapper = new HierarchicalConventionPOStorageOps<>(delegate); - assertEquals( - Arrays.asList(BasePOStorageOps.Capability.GET_BY_NAME, BasePOStorageOps.Capability.UPDATE), - wrapper.capabilities()); + assertTrue(wrapper.supportsParentIdRelationalRead()); assertSame(Entity.EntityType.SCHEMA, wrapper.entityType()); } diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestPOStorageReadRouting.java b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestPOStorageReadRouting.java new file mode 100644 index 00000000000..3b57ded40a0 --- /dev/null +++ b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestPOStorageReadRouting.java @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.service; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mockStatic; + +import java.util.Collections; +import java.util.List; +import org.apache.gravitino.Entity; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.Namespace; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; + +public class TestPOStorageReadRouting { + + private static final Object MAPPER = new Object(); + + @Test + public void getPOUsesFullNameWhenCacheDisabled() { + RecordingOps ops = new RecordingOps(true); + NameIdentifier id = NameIdentifier.of("ml", "cat", "schema"); + String result = POStorageReadRouting.getPO(MAPPER, id, ops, Entity.EntityType.SCHEMA, false); + assertEquals("by-full", result); + assertFalse(ops.getPOByParentCalled); + assertTrue(ops.getPOByFullNameCalled); + } + + @Test + public void getPOUsesFullNameWhenCacheEnabledButParentPathUnsupported() { + RecordingOps ops = new RecordingOps(false); + String result = + POStorageReadRouting.getPO( + MAPPER, NameIdentifier.of("ml", "cat", "schema"), ops, Entity.EntityType.SCHEMA, true); + assertEquals("by-full", result); + assertFalse(ops.getPOByParentCalled); + assertTrue(ops.getPOByFullNameCalled); + } + + @Test + public void getPOUsesParentIdWhenCacheEnabledAndSupported() { + try (MockedStatic entityIds = mockStatic(EntityIdService.class)) { + entityIds + .when(() -> EntityIdService.getEntityId(any(), eq(Entity.EntityType.CATALOG))) + .thenReturn(42L); + RecordingOps ops = new RecordingOps(true); + NameIdentifier id = NameIdentifier.of("ml", "cat", "schema"); + String result = POStorageReadRouting.getPO(MAPPER, id, ops, Entity.EntityType.SCHEMA, true); + assertEquals("by-parent", result); + assertTrue(ops.getPOByParentCalled); + assertFalse(ops.getPOByFullNameCalled); + assertEquals(Long.valueOf(42), ops.seenParentId); + assertEquals("schema", ops.seenShortName); + } + } + + @Test + public void listPOsUsesFullNamespaceWhenCacheDisabled() { + RecordingOps ops = new RecordingOps(true); + Namespace ns = Namespace.of("ml", "cat", "sch"); + List result = + POStorageReadRouting.listPOs(MAPPER, ns, ops, Entity.EntityType.TABLE, false); + assertTrue(result.isEmpty()); + assertFalse(ops.listByParentCalled); + assertTrue(ops.listByFullNsCalled); + } + + @Test + public void listPOsUsesParentIdWhenCacheEnabledAndSupported() { + try (MockedStatic entityIds = mockStatic(EntityIdService.class)) { + entityIds + .when(() -> EntityIdService.getEntityId(any(), eq(Entity.EntityType.SCHEMA))) + .thenReturn(7L); + RecordingOps ops = new RecordingOps(true); + Namespace ns = Namespace.of("ml", "cat", "sch"); + List out = + POStorageReadRouting.listPOs(MAPPER, ns, ops, Entity.EntityType.TABLE, true); + assertEquals(Collections.singletonList("row"), out); + assertTrue(ops.listByParentCalled); + assertFalse(ops.listByFullNsCalled); + assertEquals(Long.valueOf(7), ops.seenListParentId); + } + } + + private static final class RecordingOps extends BasePOStorageOps { + private final boolean supportsParent; + boolean getPOByParentCalled; + boolean getPOByFullNameCalled; + boolean listByParentCalled; + boolean listByFullNsCalled; + Long seenParentId; + String seenShortName; + Long seenListParentId; + + RecordingOps(boolean supportsParent) { + this.supportsParent = supportsParent; + } + + @Override + public boolean supportsParentIdRelationalRead() { + return supportsParent; + } + + @Override + public String getPO(Object mapper, Long parentId, String name) { + getPOByParentCalled = true; + seenParentId = parentId; + seenShortName = name; + return "by-parent"; + } + + @Override + public String getPOByFullName(Object mapper, NameIdentifier identifier) { + getPOByFullNameCalled = true; + return "by-full"; + } + + @Override + public List listPOs(Object mapper, Long parentId) { + listByParentCalled = true; + seenListParentId = parentId; + return Collections.singletonList("row"); + } + + @Override + public List listPOsByNSFullName(Object mapper, Namespace namespace) { + listByFullNsCalled = true; + return Collections.emptyList(); + } + + @Override + protected Entity.EntityType entityType() { + return Entity.EntityType.SCHEMA; + } + } +} From 9be45fce927ce12898b3925174f80fdf0dfb90ed Mon Sep 17 00:00:00 2001 From: Rory Date: Fri, 15 May 2026 21:17:41 +0800 Subject: [PATCH 42/44] rename --- .../storage/relational/service/BasePOStorageOps.java | 4 ++-- .../storage/relational/service/FunctionPOStorageOps.java | 4 ++-- .../service/HierarchicalConventionPOStorageOps.java | 4 ++-- .../storage/relational/service/SchemaPOStorageOps.java | 4 ++-- .../storage/relational/service/TablePOStorageOps.java | 4 ++-- .../storage/relational/service/ViewPOStorageOps.java | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/BasePOStorageOps.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/BasePOStorageOps.java index be66afc109c..e53fb701a56 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/BasePOStorageOps.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/BasePOStorageOps.java @@ -62,9 +62,9 @@ public List listPOs(Mapper mapper, Namespace namespace, List names) "listPOs by namespace and names is not supported by " + getClass().getSimpleName()); } - public List listPOs(Mapper mapper, List uuids) { + public List listPOs(Mapper mapper, List entityIds) { throw new UnsupportedOperationException( - "listPOs by uuids is not supported by " + getClass().getSimpleName()); + "listPOs by entityIds is not supported by " + getClass().getSimpleName()); } public PO getPOByFullName(Mapper mapper, NameIdentifier identifier) { diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionPOStorageOps.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionPOStorageOps.java index 21eabb0edad..eb453869d26 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionPOStorageOps.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/FunctionPOStorageOps.java @@ -63,8 +63,8 @@ public List listPOs(FunctionMetaMapper mapper, Long parentId) { } @Override - public List listPOs(FunctionMetaMapper mapper, List uuids) { - return mapper.listFunctionPOsByFunctionIds(uuids); + public List listPOs(FunctionMetaMapper mapper, List entityIds) { + return mapper.listFunctionPOsByFunctionIds(entityIds); } @Override diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/HierarchicalConventionPOStorageOps.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/HierarchicalConventionPOStorageOps.java index 607fd24cc47..4f2b570ddb7 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/HierarchicalConventionPOStorageOps.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/HierarchicalConventionPOStorageOps.java @@ -95,8 +95,8 @@ public List listPOs(Mapper mapper, Namespace logicalNamespace, List } @Override - public List listPOs(Mapper mapper, List uuids) { - return applyRead(delegate.listPOs(mapper, uuids)); + public List listPOs(Mapper mapper, List entityIds) { + return applyRead(delegate.listPOs(mapper, entityIds)); } @Override diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaPOStorageOps.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaPOStorageOps.java index 3ddba833890..568437b4c51 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaPOStorageOps.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaPOStorageOps.java @@ -84,8 +84,8 @@ public List listPOs( } @Override - public List listPOs(SchemaMetaMapper schemaMetaMapper, List uuids) { - return schemaMetaMapper.listSchemaPOsBySchemaIds(uuids); + public List listPOs(SchemaMetaMapper schemaMetaMapper, List entityIds) { + return schemaMetaMapper.listSchemaPOsBySchemaIds(entityIds); } @Override diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/TablePOStorageOps.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/TablePOStorageOps.java index d687c93ec88..7c9654a885c 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/TablePOStorageOps.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/TablePOStorageOps.java @@ -93,8 +93,8 @@ public List listPOs(TableMetaMapper mapper, Namespace namespace, List listPOs(TableMetaMapper mapper, List uuids) { - return mapper.listTablePOsByTableIds(uuids); + public List listPOs(TableMetaMapper mapper, List entityIds) { + return mapper.listTablePOsByTableIds(entityIds); } @Override diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewPOStorageOps.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewPOStorageOps.java index 231b624b0b5..23aaeb3d075 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewPOStorageOps.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/ViewPOStorageOps.java @@ -85,8 +85,8 @@ public List listPOs(ViewMetaMapper mapper, Long parentId) { } @Override - public List listPOs(ViewMetaMapper mapper, List uuids) { - return mapper.listViewPOsByViewIds(uuids); + public List listPOs(ViewMetaMapper mapper, List entityIds) { + return mapper.listViewPOsByViewIds(entityIds); } @Override From f79d0dd68142752a2235eee8ef06d120167670ad Mon Sep 17 00:00:00 2001 From: roryqi Date: Sun, 17 May 2026 16:26:51 +0000 Subject: [PATCH 43/44] fix(core): split insertSchema name on logical separator SchemaMetaService.insertSchema previously split the entity name on the physical separator (HierarchicalSchemaUtil.physicalSeparator()), so the ancestor-row creation path was only reachable when the caller already handed a storage-form name. Production callers (SchemaOperationDispatcher .importSchema and ManagedSchemaOperations.createSchema) pass the external API form, so the leaf was inserted with no ancestor rows. Split on the logical separator and let HierarchicalConventionPOStorageOps .batchInsertPOs apply its write rewriter to translate each PO to storage form before SQL execution. For the direct batchSelectSchemaByIdentifier lookup that bypasses the rewriter, convert logical -> physical at the call site. Update TestSchemaMetaService accordingly so both hierarchical tests build SchemaEntities with logical-form names. Co-Authored-By: Claude Opus 4.7 --- .../relational/service/SchemaMetaService.java | 28 +++++++++------ .../service/TestSchemaMetaService.java | 36 +++++-------------- 2 files changed, 26 insertions(+), 38 deletions(-) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java index da755e07e6d..204a016cd31 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java @@ -139,23 +139,24 @@ public List listSchemasByNamespace(Namespace namespace) { public void insertSchema(SchemaEntity schemaEntity, boolean overwrite) throws IOException { try { NameIdentifierUtil.checkSchema(schemaEntity.nameIdentifier()); - // Callers above this service (e.g. JDBCBackend + naming bridge) pass storage-form schema - // names: nested paths use the internal physical separator, not the external logical one. - String physicalSep = HierarchicalSchemaUtil.physicalSeparator(); + // SchemaEntity arrives in API/logical form (separator = HierarchicalSchemaUtil + // .schemaSeparator()). We split here on the logical separator and build ancestor rows in + // logical form. HierarchicalConventionPOStorageOps.batchInsertPOs applies its write + // rewriter to translate each PO's name to storage form before SQL execution. + String logicalSep = HierarchicalSchemaUtil.schemaSeparator(); String schemaName = schemaEntity.name(); List rowsToInsert = new ArrayList<>(); - if (schemaName == null || !schemaName.contains(physicalSep)) { + if (schemaName == null || !schemaName.contains(logicalSep)) { rowsToInsert.add(schemaEntity); } else { - // Segments of the storage-form name; e.g. [A, B, C] -> ancestor rows "A", "A"+sep+"B", then - // leaf. - String[] parts = schemaName.split(Pattern.quote(physicalSep), -1); + // Segments of the logical name; e.g. "A:B:C" -> ancestor rows "A", "A:B", then leaf. + String[] parts = schemaName.split(Pattern.quote(logicalSep), -1); for (int nSeg = 1; nSeg < parts.length; nSeg++) { - String ancestorPhysical = String.join(physicalSep, Arrays.copyOf(parts, nSeg)); + String ancestorLogical = String.join(logicalSep, Arrays.copyOf(parts, nSeg)); SchemaEntity ancestor = SchemaEntity.builder() .withId(nextIdForNestedAncestor()) - .withName(ancestorPhysical) + .withName(ancestorLogical) .withNamespace(schemaEntity.namespace()) .withComment(null) .withProperties(Collections.emptyMap()) @@ -174,11 +175,14 @@ public void insertSchema(SchemaEntity schemaEntity, boolean overwrite) throws IO if (n > 1) { SchemaEntity firstAncestor = rowsToInsert.get(0); Namespace ancestorNs = firstAncestor.namespace(); + // The mapper queries storage (physical) form names directly, so convert before + // the lookup and compare in physical form. List ancestorPhysicalNames = rowsToInsert.subList(0, n - 1).stream() .map(SchemaEntity::name) + .map(name -> HierarchicalSchemaUtil.logicalToPhysical(name, logicalSep)) .collect(Collectors.toList()); - Set existingAncestorNames = + Set existingPhysicalNames = mapper .batchSelectSchemaByIdentifier( ancestorNs.level(0), ancestorNs.level(1), ancestorPhysicalNames) @@ -186,7 +190,9 @@ public void insertSchema(SchemaEntity schemaEntity, boolean overwrite) throws IO .map(SchemaPO::getSchemaName) .collect(Collectors.toSet()); for (SchemaEntity row : rowsToInsert.subList(0, n - 1)) { - if (existingAncestorNames.contains(row.name())) { + String physicalName = + HierarchicalSchemaUtil.logicalToPhysical(row.name(), logicalSep); + if (existingPhysicalNames.contains(physicalName)) { continue; } SchemaPO.Builder builder = SchemaPO.builder(); diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestSchemaMetaService.java b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestSchemaMetaService.java index 20e94280195..599c32886e3 100644 --- a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestSchemaMetaService.java +++ b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestSchemaMetaService.java @@ -24,11 +24,9 @@ import java.io.IOException; import java.time.Instant; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Set; -import java.util.regex.Pattern; import java.util.stream.Collectors; import org.apache.gravitino.Entity; import org.apache.gravitino.EntityAlreadyExistsException; @@ -38,7 +36,6 @@ import org.apache.gravitino.meta.TopicEntity; import org.apache.gravitino.storage.RandomIdGenerator; import org.apache.gravitino.storage.relational.TestJDBCBackend; -import org.apache.gravitino.utils.HierarchicalSchemaUtil; import org.apache.gravitino.utils.NameIdentifierUtil; import org.apache.gravitino.utils.NamespaceUtil; import org.junit.jupiter.api.Assertions; @@ -220,12 +217,10 @@ public void testInsertHierarchicalSchemaCreatesAncestorsAndLeaf() throws IOExcep SchemaMetaService schemaMetaService = SchemaMetaService.getInstance(); String logicalLeaf = "ns_a:ns_b:leaf"; - String sep = HierarchicalSchemaUtil.schemaSeparator(); - String physicalLeaf = HierarchicalSchemaUtil.logicalToPhysical(logicalLeaf, sep); SchemaEntity hierarchical = SchemaEntity.builder() .withId(RandomIdGenerator.INSTANCE.nextId()) - .withName(physicalLeaf) + .withName(logicalLeaf) .withNamespace(NamespaceUtil.ofSchema(metalakeName, catalogName)) .withComment("nested") .withProperties(Collections.emptyMap()) @@ -235,17 +230,7 @@ public void testInsertHierarchicalSchemaCreatesAncestorsAndLeaf() throws IOExcep List schemas = schemaMetaService.listSchemasByNamespace(NamespaceUtil.ofSchema(metalakeName, catalogName)); - Set logicalNames = - schemas.stream() - .map(SchemaEntity::name) - .map( - n -> { - if (n != null && n.contains(HierarchicalSchemaUtil.physicalSeparator())) { - return HierarchicalSchemaUtil.physicalToLogical(n, sep); - } - return n; - }) - .collect(Collectors.toSet()); + Set logicalNames = schemas.stream().map(SchemaEntity::name).collect(Collectors.toSet()); Assertions.assertTrue(logicalNames.contains("ns_a")); Assertions.assertTrue(logicalNames.contains("ns_a:ns_b")); @@ -253,7 +238,7 @@ public void testInsertHierarchicalSchemaCreatesAncestorsAndLeaf() throws IOExcep SchemaEntity loaded = schemaMetaService.getSchemaByIdentifier( - NameIdentifier.of(metalakeName, catalogName, physicalLeaf)); + NameIdentifier.of(metalakeName, catalogName, logicalLeaf)); Assertions.assertEquals(logicalLeaf, loaded.name()); Assertions.assertEquals("nested", loaded.comment()); } @@ -264,18 +249,15 @@ public void testInsertHierarchicalSecondLeafReusesAncestorsWithoutUpsert() throw createAndInsertCatalog(metalakeName, catalogName); SchemaMetaService schemaMetaService = SchemaMetaService.getInstance(); - String sep = HierarchicalSchemaUtil.schemaSeparator(); - String physSep = HierarchicalSchemaUtil.physicalSeparator(); - String physicalLeaf1 = HierarchicalSchemaUtil.logicalToPhysical("ns_a:ns_b:leaf1", sep); - String physicalLeaf2 = HierarchicalSchemaUtil.logicalToPhysical("ns_a:ns_b:leaf2", sep); - String[] parts = physicalLeaf1.split(Pattern.quote(physSep), -1); - String ancestorA = parts[0]; - String ancestorAB = String.join(physSep, Arrays.copyOfRange(parts, 0, 2)); + String leaf1 = "ns_a:ns_b:leaf1"; + String leaf2 = "ns_a:ns_b:leaf2"; + String ancestorA = "ns_a"; + String ancestorAB = "ns_a:ns_b"; SchemaEntity first = SchemaEntity.builder() .withId(RandomIdGenerator.INSTANCE.nextId()) - .withName(physicalLeaf1) + .withName(leaf1) .withNamespace(NamespaceUtil.ofSchema(metalakeName, catalogName)) .withComment("first") .withProperties(Collections.emptyMap()) @@ -295,7 +277,7 @@ public void testInsertHierarchicalSecondLeafReusesAncestorsWithoutUpsert() throw SchemaEntity second = SchemaEntity.builder() .withId(RandomIdGenerator.INSTANCE.nextId()) - .withName(physicalLeaf2) + .withName(leaf2) .withNamespace(NamespaceUtil.ofSchema(metalakeName, catalogName)) .withComment("second") .withProperties(Collections.emptyMap()) From cb3f4736fe0ccac82bdbea5f89250f2cfe5be2cb Mon Sep 17 00:00:00 2001 From: Rory Date: Mon, 18 May 2026 01:05:46 +0800 Subject: [PATCH 44/44] Fix schema --- .../relational/service/SchemaMetaService.java | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java index 204a016cd31..5cc65c178d5 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java @@ -175,24 +175,16 @@ public void insertSchema(SchemaEntity schemaEntity, boolean overwrite) throws IO if (n > 1) { SchemaEntity firstAncestor = rowsToInsert.get(0); Namespace ancestorNs = firstAncestor.namespace(); - // The mapper queries storage (physical) form names directly, so convert before - // the lookup and compare in physical form. - List ancestorPhysicalNames = + List ancestorNames = rowsToInsert.subList(0, n - 1).stream() .map(SchemaEntity::name) - .map(name -> HierarchicalSchemaUtil.logicalToPhysical(name, logicalSep)) .collect(Collectors.toList()); - Set existingPhysicalNames = - mapper - .batchSelectSchemaByIdentifier( - ancestorNs.level(0), ancestorNs.level(1), ancestorPhysicalNames) - .stream() + Set existingLogicalNames = + ops.listPOs(mapper, ancestorNs, ancestorNames).stream() .map(SchemaPO::getSchemaName) .collect(Collectors.toSet()); for (SchemaEntity row : rowsToInsert.subList(0, n - 1)) { - String physicalName = - HierarchicalSchemaUtil.logicalToPhysical(row.name(), logicalSep); - if (existingPhysicalNames.contains(physicalName)) { + if (existingLogicalNames.contains(row.name())) { continue; } SchemaPO.Builder builder = SchemaPO.builder();