Skip to content

Add POST /tableConfigs/tune endpoint to apply tuning and return tuned config #18056

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
rseetham wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Add POST /tableConfigs/tune endpoint to apply tuning and return tuned config #18056

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 47 additions & 9 deletions .../src/main/java/org/apache/pinot/controller/api/resources/TableConfigsRestletResource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -239,13 +239,13 @@ public ConfigSuccessResponse addConfig(
}

if (offlineTableConfig != null) {
tuneConfig(offlineTableConfig, schema);
applyTuning(offlineTableConfig, schema);
if (!ignoreActiveTasks) {
PinotTableRestletResource.tableTasksValidation(offlineTableConfig, _pinotHelixTaskResourceManager);
}
}
if (realtimeTableConfig != null) {
tuneConfig(realtimeTableConfig, schema);
applyTuning(realtimeTableConfig, schema);
if (!ignoreActiveTasks) {
PinotTableRestletResource.tableTasksValidation(realtimeTableConfig, _pinotHelixTaskResourceManager);
}
Expand Down Expand Up @@ -407,7 +407,7 @@ public ConfigSuccessResponse updateConfig(
LOGGER.info("Updated schema: {}", tableName);

if (offlineTableConfig != null) {
tuneConfig(offlineTableConfig, schema);
applyTuning(offlineTableConfig, schema);
if (_pinotHelixResourceManager.hasOfflineTable(tableName)) {
_pinotHelixResourceManager.updateTableConfig(offlineTableConfig, forceTableSchemaUpdate);
LOGGER.info("Updated offline table config: {}", tableName);
Expand All @@ -417,7 +417,7 @@ public ConfigSuccessResponse updateConfig(
}
}
if (realtimeTableConfig != null) {
tuneConfig(realtimeTableConfig, schema);
applyTuning(realtimeTableConfig, schema);
if (_pinotHelixResourceManager.hasRealtimeTable(tableName)) {
_pinotHelixResourceManager.updateTableConfig(realtimeTableConfig, forceTableSchemaUpdate);
LOGGER.info("Updated realtime table config: {}", tableName);
Expand Down Expand Up @@ -459,6 +459,47 @@ public String validateConfig(String tableConfigsStr,
+ "(ALL|TASK|UPSERT|TENANT|MINION_INSTANCES|ACTIVE_TASKS)")
@QueryParam("validationTypesToSkip") @Nullable String typesToSkip, @Context HttpHeaders httpHeaders,
@Context Request request) {
Pair<TableConfigs, Map<String, Object>> tableConfigsAndUnrecognizedProps =
parseAndValidateTableConfigs(tableConfigsStr, typesToSkip, httpHeaders, request);
TableConfigs tableConfigs = tableConfigsAndUnrecognizedProps.getLeft();
ObjectNode response = JsonUtils.objectToJsonNode(tableConfigs).deepCopy();
response.set("unrecognizedProperties", JsonUtils.objectToJsonNode(tableConfigsAndUnrecognizedProps.getRight()));
return response.toString();
}

/**
* Validates and tunes the {@link TableConfigs} as provided in the tableConfigsStr json, by applying tuner configs,
* ensuring min replicas and storage quota constraints, and returns the tuned TableConfigs.
*/
@POST
@Path("/tableConfigs/tune")
Copy link
Copy Markdown
Collaborator

@shauryachats shauryachats Apr 7, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Instead of a new endpoint, could we consider adding a applyTuneConfigs param to POST /tableConfigs/validate since this endpoint validates as well as applies tuner configs to the input table config - hence it is an extension of the validate endpoint.

Copy link
Copy Markdown
Contributor Author

@rseetham rseetham Apr 9, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Validate also tuning configs is not obvious. The tune configs could be extended in the future for more tuning. The validate is there to make sure the tuning does not fail devise the input is invalid.

@Produces(MediaType.APPLICATION_JSON)
@ApiOperation(value = "Tune the TableConfigs",
notes = "Validates and applies tuning (tuner configs, min replicas, storage quota) to the TableConfigs, "
+ "returning the result that would be stored on create/update")
@ManualAuthorization // performed after parsing TableConfigs
public String tuneConfig(String tableConfigsStr,
@ApiParam(value = "comma separated list of validation type(s) to skip. supported types: "
+ "(ALL|TASK|UPSERT|TENANT|MINION_INSTANCES|ACTIVE_TASKS)")
@QueryParam("validationTypesToSkip") @Nullable String typesToSkip, @Context HttpHeaders httpHeaders,
@Context Request request) {
Pair<TableConfigs, Map<String, Object>> tableConfigsAndUnrecognizedProps =
parseAndValidateTableConfigs(tableConfigsStr, typesToSkip, httpHeaders, request);
TableConfigs tableConfigs = tableConfigsAndUnrecognizedProps.getLeft();
Schema schema = tableConfigs.getSchema();
if (tableConfigs.getOffline() != null) {
applyTuning(tableConfigs.getOffline(), schema);
}
if (tableConfigs.getRealtime() != null) {
applyTuning(tableConfigs.getRealtime(), schema);
}
ObjectNode response = JsonUtils.objectToJsonNode(tableConfigs).deepCopy();
response.set("unrecognizedProperties", JsonUtils.objectToJsonNode(tableConfigsAndUnrecognizedProps.getRight()));
return response.toString();
}

private Pair<TableConfigs, Map<String, Object>> parseAndValidateTableConfigs(String tableConfigsStr,
@Nullable String typesToSkip, HttpHeaders httpHeaders, Request request) {
Pair<TableConfigs, Map<String, Object>> tableConfigsAndUnrecognizedProps;
try {
tableConfigsAndUnrecognizedProps =
Expand All @@ -481,13 +522,10 @@ public String validateConfig(String tableConfigsStr,
if (!accessControl.hasAccess(httpHeaders, TargetType.TABLE, rawTableName, Actions.Table.VALIDATE_TABLE_CONFIGS)) {
throw new ControllerApplicationException(LOGGER, "Permission denied", Response.Status.FORBIDDEN);
}

ObjectNode response = JsonUtils.objectToJsonNode(tableConfigs).deepCopy();
response.set("unrecognizedProperties", JsonUtils.objectToJsonNode(tableConfigsAndUnrecognizedProps.getRight()));
return response.toString();
return tableConfigsAndUnrecognizedProps;
}

private void tuneConfig(TableConfig tableConfig, Schema schema) {
private void applyTuning(TableConfig tableConfig, Schema schema) {
TableConfigTunerUtils.applyTunerConfigs(_pinotHelixResourceManager, tableConfig, schema, Collections.emptyMap());
TableConfigUtils.ensureMinReplicas(tableConfig, _controllerConf.getDefaultTableMinReplicas());
TableConfigUtils.ensureStorageQuotaConstraints(tableConfig, _controllerConf.getDimTableMaxSize());
Expand Down
78 changes: 78 additions & 0 deletions ...roller/src/test/java/org/apache/pinot/controller/api/TableConfigsRestletResourceTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -730,6 +730,84 @@ public void testGetConfigCompatibility()
sendDeleteRequest(DEFAULT_INSTANCE.getControllerRequestURLBuilder().forSchemaDelete(tableName));
}

@Test
public void testTuneConfig()
throws IOException {
String tuneConfigUrl = DEFAULT_INSTANCE.getControllerRequestURLBuilder().forTableConfigsTune();

String tableName = "testTune";
TableConfigs tableConfigs;

// invalid json
try {
tableConfigs = new TableConfigs(tableName, createDummySchema(tableName), createOfflineTableConfig(tableName),
createRealtimeTableConfig(tableName));
sendPostRequest(tuneConfigUrl, tableConfigs.toPrettyJsonString().replace("\"offline\"", "offline\""));
fail("Tune of a TableConfigs with invalid json string should have failed");
} catch (Exception e) {
// expected
}

// null table configs
try {
tableConfigs = new TableConfigs(tableName, createDummySchema(tableName), null, null);
sendPostRequest(tuneConfigUrl, tableConfigs.toPrettyJsonString());
fail("Tune of a TableConfigs with null offline and realtime tableConfig should have failed");
} catch (Exception e) {
// expected
}

// replicas are bumped up to min replicas
String tableName1 = "testTuneReplicas";
TableConfig replicaTestOfflineTableConfig = createOfflineTableConfig(tableName1);
TableConfig replicaTestRealtimeTableConfig = createRealtimeTableConfig(tableName1);
replicaTestOfflineTableConfig.getValidationConfig().setReplication("1");
replicaTestRealtimeTableConfig.getValidationConfig().setReplication("1");
tableConfigs = new TableConfigs(tableName1, createDummySchema(tableName1), replicaTestOfflineTableConfig,
replicaTestRealtimeTableConfig);
String response = sendPostRequest(tuneConfigUrl, tableConfigs.toPrettyJsonString());
TableConfigs tuned = JsonUtils.stringToObject(response, TableConfigs.class);
Assert.assertEquals(tuned.getOffline().getReplication(), DEFAULT_MIN_NUM_REPLICAS);
Assert.assertEquals(tuned.getRealtime().getReplication(), DEFAULT_MIN_NUM_REPLICAS);

// dim table storage quota is capped
String tableName2 = "testTuneQuota";
TableConfig offlineDimTableConfig = createOfflineDimTableConfig(tableName2);
tableConfigs = new TableConfigs(tableName2, createDummySchemaWithPrimaryKey(tableName2), offlineDimTableConfig,
null);
response = sendPostRequest(tuneConfigUrl, tableConfigs.toPrettyJsonString());
tuned = JsonUtils.stringToObject(response, TableConfigs.class);
Assert.assertEquals(tuned.getOffline().getQuotaConfig().getStorage(),
DEFAULT_INSTANCE.getControllerConfig().getDimTableMaxSize());

// tuner configs are applied
String tableName3 = "testTuneTunerConfig";
Schema schema3 = createDummySchema(tableName3);
tableConfigs = new TableConfigs(tableName3, schema3, createOfflineTunerTableConfig(tableName3),
createRealtimeTunerTableConfig(tableName3));
response = sendPostRequest(tuneConfigUrl, tableConfigs.toPrettyJsonString());
tuned = JsonUtils.stringToObject(response, TableConfigs.class);
Assert.assertTrue(tuned.getOffline().getIndexingConfig().getInvertedIndexColumns()
.containsAll(schema3.getDimensionNames()));
Assert.assertTrue(tuned.getOffline().getIndexingConfig().getNoDictionaryColumns()
.containsAll(schema3.getMetricNames()));
Assert.assertTrue(tuned.getRealtime().getIndexingConfig().getInvertedIndexColumns()
.containsAll(schema3.getDimensionNames()));
Assert.assertTrue(tuned.getRealtime().getIndexingConfig().getNoDictionaryColumns()
.containsAll(schema3.getMetricNames()));

// response includes unrecognizedProperties
String tableName4 = "testTuneUnrecognized";
TableConfig offlineTableConfig = createOfflineTableConfig(tableName4);
tableConfigs = new TableConfigs(tableName4, createDummySchema(tableName4), offlineTableConfig, null);
ObjectNode tableConfigsJson = JsonUtils.objectToJsonNode(tableConfigs).deepCopy();
tableConfigsJson.put("illegalKey1", 1);
response = sendPostRequest(tuneConfigUrl, tableConfigsJson.toPrettyString());
JsonNode responseJson = JsonUtils.stringToJsonNode(response);
Assert.assertTrue(responseJson.has("unrecognizedProperties"));
Assert.assertTrue(responseJson.get("unrecognizedProperties").has("/illegalKey1"));
}

@Test
public void testValidateConfigWithClusterValidationSkipTypes()
throws IOException {
Expand Down
4 changes: 4 additions & 0 deletions pinot-spi/src/main/java/org/apache/pinot/spi/utils/builder/ControllerRequestURLBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -409,6 +409,10 @@ public String forTableConfigsValidate() {
return StringUtil.join("/", _baseUrl, "tableConfigs", "validate");
}

public String forTableConfigsTune() {
return StringUtil.join("/", _baseUrl, "tableConfigs", "tune");
}

public String forSegmentReload(String tableName, String segmentName, boolean forceDownload) {
return StringUtil.join("/", _baseUrl, "segments", tableName, encode(segmentName),
"reload?forceDownload=" + forceDownload);
Expand Down
Loading