From 4c2987d1a6be7f9c2d7ac073d2d6940fb249286b Mon Sep 17 00:00:00 2001 From: Rekha Seethamraju Date: Tue, 31 Mar 2026 16:55:17 -0700 Subject: [PATCH] Create /tableConfigs/tune endpoint that validates and tunes TableConfigs and returns the tuned config. --- .../TableConfigsRestletResource.java | 56 ++++++++++--- .../api/TableConfigsRestletResourceTest.java | 78 +++++++++++++++++++ .../builder/ControllerRequestURLBuilder.java | 4 + 3 files changed, 129 insertions(+), 9 deletions(-) diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableConfigsRestletResource.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableConfigsRestletResource.java index 9acef500a649..921909b1b078 100644 --- a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableConfigsRestletResource.java +++ b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableConfigsRestletResource.java @@ -239,13 +239,13 @@ public ConfigSuccessResponse addConfig( } if (offlineTableConfig != null) { - tuneConfig(offlineTableConfig, schema); + applyTuning(offlineTableConfig, schema); if (!ignoreActiveTasks) { PinotTableRestletResource.tableTasksValidation(offlineTableConfig, _pinotHelixTaskResourceManager); } } if (realtimeTableConfig != null) { - tuneConfig(realtimeTableConfig, schema); + applyTuning(realtimeTableConfig, schema); if (!ignoreActiveTasks) { PinotTableRestletResource.tableTasksValidation(realtimeTableConfig, _pinotHelixTaskResourceManager); } @@ -407,7 +407,7 @@ public ConfigSuccessResponse updateConfig( LOGGER.info("Updated schema: {}", tableName); if (offlineTableConfig != null) { - tuneConfig(offlineTableConfig, schema); + applyTuning(offlineTableConfig, schema); if (_pinotHelixResourceManager.hasOfflineTable(tableName)) { _pinotHelixResourceManager.updateTableConfig(offlineTableConfig, forceTableSchemaUpdate); LOGGER.info("Updated offline table config: {}", tableName); @@ -417,7 +417,7 @@ public ConfigSuccessResponse updateConfig( } } if (realtimeTableConfig != null) { - tuneConfig(realtimeTableConfig, schema); + applyTuning(realtimeTableConfig, schema); if (_pinotHelixResourceManager.hasRealtimeTable(tableName)) { _pinotHelixResourceManager.updateTableConfig(realtimeTableConfig, forceTableSchemaUpdate); LOGGER.info("Updated realtime table config: {}", tableName); @@ -459,6 +459,47 @@ public String validateConfig(String tableConfigsStr, + "(ALL|TASK|UPSERT|TENANT|MINION_INSTANCES|ACTIVE_TASKS)") @QueryParam("validationTypesToSkip") @Nullable String typesToSkip, @Context HttpHeaders httpHeaders, @Context Request request) { + Pair> tableConfigsAndUnrecognizedProps = + parseAndValidateTableConfigs(tableConfigsStr, typesToSkip, httpHeaders, request); + TableConfigs tableConfigs = tableConfigsAndUnrecognizedProps.getLeft(); + ObjectNode response = JsonUtils.objectToJsonNode(tableConfigs).deepCopy(); + response.set("unrecognizedProperties", JsonUtils.objectToJsonNode(tableConfigsAndUnrecognizedProps.getRight())); + return response.toString(); + } + + /** + * Validates and tunes the {@link TableConfigs} as provided in the tableConfigsStr json, by applying tuner configs, + * ensuring min replicas and storage quota constraints, and returns the tuned TableConfigs. + */ + @POST + @Path("/tableConfigs/tune") + @Produces(MediaType.APPLICATION_JSON) + @ApiOperation(value = "Tune the TableConfigs", + notes = "Validates and applies tuning (tuner configs, min replicas, storage quota) to the TableConfigs, " + + "returning the result that would be stored on create/update") + @ManualAuthorization // performed after parsing TableConfigs + public String tuneConfig(String tableConfigsStr, + @ApiParam(value = "comma separated list of validation type(s) to skip. supported types: " + + "(ALL|TASK|UPSERT|TENANT|MINION_INSTANCES|ACTIVE_TASKS)") + @QueryParam("validationTypesToSkip") @Nullable String typesToSkip, @Context HttpHeaders httpHeaders, + @Context Request request) { + Pair> tableConfigsAndUnrecognizedProps = + parseAndValidateTableConfigs(tableConfigsStr, typesToSkip, httpHeaders, request); + TableConfigs tableConfigs = tableConfigsAndUnrecognizedProps.getLeft(); + Schema schema = tableConfigs.getSchema(); + if (tableConfigs.getOffline() != null) { + applyTuning(tableConfigs.getOffline(), schema); + } + if (tableConfigs.getRealtime() != null) { + applyTuning(tableConfigs.getRealtime(), schema); + } + ObjectNode response = JsonUtils.objectToJsonNode(tableConfigs).deepCopy(); + response.set("unrecognizedProperties", JsonUtils.objectToJsonNode(tableConfigsAndUnrecognizedProps.getRight())); + return response.toString(); + } + + private Pair> parseAndValidateTableConfigs(String tableConfigsStr, + @Nullable String typesToSkip, HttpHeaders httpHeaders, Request request) { Pair> tableConfigsAndUnrecognizedProps; try { tableConfigsAndUnrecognizedProps = @@ -481,13 +522,10 @@ public String validateConfig(String tableConfigsStr, if (!accessControl.hasAccess(httpHeaders, TargetType.TABLE, rawTableName, Actions.Table.VALIDATE_TABLE_CONFIGS)) { throw new ControllerApplicationException(LOGGER, "Permission denied", Response.Status.FORBIDDEN); } - - ObjectNode response = JsonUtils.objectToJsonNode(tableConfigs).deepCopy(); - response.set("unrecognizedProperties", JsonUtils.objectToJsonNode(tableConfigsAndUnrecognizedProps.getRight())); - return response.toString(); + return tableConfigsAndUnrecognizedProps; } - private void tuneConfig(TableConfig tableConfig, Schema schema) { + private void applyTuning(TableConfig tableConfig, Schema schema) { TableConfigTunerUtils.applyTunerConfigs(_pinotHelixResourceManager, tableConfig, schema, Collections.emptyMap()); TableConfigUtils.ensureMinReplicas(tableConfig, _controllerConf.getDefaultTableMinReplicas()); TableConfigUtils.ensureStorageQuotaConstraints(tableConfig, _controllerConf.getDimTableMaxSize()); diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/api/TableConfigsRestletResourceTest.java b/pinot-controller/src/test/java/org/apache/pinot/controller/api/TableConfigsRestletResourceTest.java index ea80b910a02a..0712579b8565 100644 --- a/pinot-controller/src/test/java/org/apache/pinot/controller/api/TableConfigsRestletResourceTest.java +++ b/pinot-controller/src/test/java/org/apache/pinot/controller/api/TableConfigsRestletResourceTest.java @@ -730,6 +730,84 @@ public void testGetConfigCompatibility() sendDeleteRequest(DEFAULT_INSTANCE.getControllerRequestURLBuilder().forSchemaDelete(tableName)); } + @Test + public void testTuneConfig() + throws IOException { + String tuneConfigUrl = DEFAULT_INSTANCE.getControllerRequestURLBuilder().forTableConfigsTune(); + + String tableName = "testTune"; + TableConfigs tableConfigs; + + // invalid json + try { + tableConfigs = new TableConfigs(tableName, createDummySchema(tableName), createOfflineTableConfig(tableName), + createRealtimeTableConfig(tableName)); + sendPostRequest(tuneConfigUrl, tableConfigs.toPrettyJsonString().replace("\"offline\"", "offline\"")); + fail("Tune of a TableConfigs with invalid json string should have failed"); + } catch (Exception e) { + // expected + } + + // null table configs + try { + tableConfigs = new TableConfigs(tableName, createDummySchema(tableName), null, null); + sendPostRequest(tuneConfigUrl, tableConfigs.toPrettyJsonString()); + fail("Tune of a TableConfigs with null offline and realtime tableConfig should have failed"); + } catch (Exception e) { + // expected + } + + // replicas are bumped up to min replicas + String tableName1 = "testTuneReplicas"; + TableConfig replicaTestOfflineTableConfig = createOfflineTableConfig(tableName1); + TableConfig replicaTestRealtimeTableConfig = createRealtimeTableConfig(tableName1); + replicaTestOfflineTableConfig.getValidationConfig().setReplication("1"); + replicaTestRealtimeTableConfig.getValidationConfig().setReplication("1"); + tableConfigs = new TableConfigs(tableName1, createDummySchema(tableName1), replicaTestOfflineTableConfig, + replicaTestRealtimeTableConfig); + String response = sendPostRequest(tuneConfigUrl, tableConfigs.toPrettyJsonString()); + TableConfigs tuned = JsonUtils.stringToObject(response, TableConfigs.class); + Assert.assertEquals(tuned.getOffline().getReplication(), DEFAULT_MIN_NUM_REPLICAS); + Assert.assertEquals(tuned.getRealtime().getReplication(), DEFAULT_MIN_NUM_REPLICAS); + + // dim table storage quota is capped + String tableName2 = "testTuneQuota"; + TableConfig offlineDimTableConfig = createOfflineDimTableConfig(tableName2); + tableConfigs = new TableConfigs(tableName2, createDummySchemaWithPrimaryKey(tableName2), offlineDimTableConfig, + null); + response = sendPostRequest(tuneConfigUrl, tableConfigs.toPrettyJsonString()); + tuned = JsonUtils.stringToObject(response, TableConfigs.class); + Assert.assertEquals(tuned.getOffline().getQuotaConfig().getStorage(), + DEFAULT_INSTANCE.getControllerConfig().getDimTableMaxSize()); + + // tuner configs are applied + String tableName3 = "testTuneTunerConfig"; + Schema schema3 = createDummySchema(tableName3); + tableConfigs = new TableConfigs(tableName3, schema3, createOfflineTunerTableConfig(tableName3), + createRealtimeTunerTableConfig(tableName3)); + response = sendPostRequest(tuneConfigUrl, tableConfigs.toPrettyJsonString()); + tuned = JsonUtils.stringToObject(response, TableConfigs.class); + Assert.assertTrue(tuned.getOffline().getIndexingConfig().getInvertedIndexColumns() + .containsAll(schema3.getDimensionNames())); + Assert.assertTrue(tuned.getOffline().getIndexingConfig().getNoDictionaryColumns() + .containsAll(schema3.getMetricNames())); + Assert.assertTrue(tuned.getRealtime().getIndexingConfig().getInvertedIndexColumns() + .containsAll(schema3.getDimensionNames())); + Assert.assertTrue(tuned.getRealtime().getIndexingConfig().getNoDictionaryColumns() + .containsAll(schema3.getMetricNames())); + + // response includes unrecognizedProperties + String tableName4 = "testTuneUnrecognized"; + TableConfig offlineTableConfig = createOfflineTableConfig(tableName4); + tableConfigs = new TableConfigs(tableName4, createDummySchema(tableName4), offlineTableConfig, null); + ObjectNode tableConfigsJson = JsonUtils.objectToJsonNode(tableConfigs).deepCopy(); + tableConfigsJson.put("illegalKey1", 1); + response = sendPostRequest(tuneConfigUrl, tableConfigsJson.toPrettyString()); + JsonNode responseJson = JsonUtils.stringToJsonNode(response); + Assert.assertTrue(responseJson.has("unrecognizedProperties")); + Assert.assertTrue(responseJson.get("unrecognizedProperties").has("/illegalKey1")); + } + @Test public void testValidateConfigWithClusterValidationSkipTypes() throws IOException { diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/builder/ControllerRequestURLBuilder.java b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/builder/ControllerRequestURLBuilder.java index a7fa09019824..4346cc394b9a 100644 --- a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/builder/ControllerRequestURLBuilder.java +++ b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/builder/ControllerRequestURLBuilder.java @@ -409,6 +409,10 @@ public String forTableConfigsValidate() { return StringUtil.join("/", _baseUrl, "tableConfigs", "validate"); } + public String forTableConfigsTune() { + return StringUtil.join("/", _baseUrl, "tableConfigs", "tune"); + } + public String forSegmentReload(String tableName, String segmentName, boolean forceDownload) { return StringUtil.join("/", _baseUrl, "segments", tableName, encode(segmentName), "reload?forceDownload=" + forceDownload);