From 9add23a612069d2c43f9127ba059b180b2d05f74 Mon Sep 17 00:00:00 2001 From: Lucas Capistrant Date: Thu, 2 Apr 2026 16:50:25 -0500 Subject: [PATCH 1/3] builder for reindexCascade in the console --- .../compaction-config-dialog.spec.tsx.snap | 786 +++++++++++++++++ .../rule-provider-editor.spec.tsx.snap | 823 ++++++++++++++++++ .../virtual-columns-editor.spec.tsx.snap | 184 ++++ .../compaction-config-dialog.spec.tsx | 43 + .../compaction-config-dialog.tsx | 190 +++- .../rule-provider-editor.scss | 95 ++ .../rule-provider-editor.spec.tsx | 58 ++ .../rule-provider-editor.tsx | 275 ++++++ .../virtual-columns-editor.scss | 67 ++ .../virtual-columns-editor.spec.tsx | 48 + .../virtual-columns-editor.tsx | 144 +++ .../compaction-config/compaction-config.tsx | 1 + .../reindex-cascade-config.spec.ts | 156 ++++ .../reindex-cascade-config.tsx | 730 ++++++++++++++++ web-console/src/druid-models/index.ts | 1 + .../datasources-view/datasources-view.tsx | 17 +- 16 files changed, 3577 insertions(+), 41 deletions(-) create mode 100644 web-console/src/dialogs/compaction-config-dialog/__snapshots__/rule-provider-editor.spec.tsx.snap create mode 100644 web-console/src/dialogs/compaction-config-dialog/__snapshots__/virtual-columns-editor.spec.tsx.snap create mode 100644 web-console/src/dialogs/compaction-config-dialog/rule-provider-editor.scss create mode 100644 web-console/src/dialogs/compaction-config-dialog/rule-provider-editor.spec.tsx create mode 100644 web-console/src/dialogs/compaction-config-dialog/rule-provider-editor.tsx create mode 100644 web-console/src/dialogs/compaction-config-dialog/virtual-columns-editor.scss create mode 100644 web-console/src/dialogs/compaction-config-dialog/virtual-columns-editor.spec.tsx create mode 100644 web-console/src/dialogs/compaction-config-dialog/virtual-columns-editor.tsx create mode 100644 web-console/src/druid-models/compaction-config/reindex-cascade-config.spec.ts create mode 100644 web-console/src/druid-models/compaction-config/reindex-cascade-config.tsx diff --git a/web-console/src/dialogs/compaction-config-dialog/__snapshots__/compaction-config-dialog.spec.tsx.snap b/web-console/src/dialogs/compaction-config-dialog/__snapshots__/compaction-config-dialog.spec.tsx.snap index b9b48cf6e58b..396a85030636 100644 --- a/web-console/src/dialogs/compaction-config-dialog/__snapshots__/compaction-config-dialog.spec.tsx.snap +++ b/web-console/src/dialogs/compaction-config-dialog/__snapshots__/compaction-config-dialog.spec.tsx.snap @@ -1230,6 +1230,792 @@ exports[`CompactionConfigDialog matches snapshot with compactionConfig (range pa `; +exports[`CompactionConfigDialog matches snapshot with reindexCascade compactionConfig 1`] = ` + + +
+ + + + + + + + + Segment granularity used for intervals where no partitioning rule matches. This is the default time bucketing for segments. +

, + "label": "Default segment granularity", + "name": "defaultSegmentGranularity", + "required": true, + "suggestions": [ + "MINUTE", + "FIFTEEN_MINUTE", + "HOUR", + "DAY", + "MONTH", + "QUARTER", + "YEAR", + ], + "type": "string", + }, + { + "info":

+ Partitioning strategy used for intervals where no partitioning rule matches. Use + + + dynamic + + for best-effort rollup or + + range + + for range-based partitioning. +

, + "label": "Default partitioning type", + "name": "defaultPartitionsSpec.type", + "suggestions": [ + "dynamic", + "range", + ], + "type": "string", + }, + { + "defaultValue": 5000000, + "defined": [Function], + "info": + Determines how many rows are in each segment. + , + "label": "Default max rows per segment", + "name": "defaultPartitionsSpec.maxRowsPerSegment", + "type": "number", + }, + { + "defaultValue": 20000000, + "defined": [Function], + "info": + Total number of rows in segments waiting for being pushed. + , + "label": "Default max total rows", + "name": "defaultPartitionsSpec.maxTotalRows", + "type": "number", + }, + { + "defined": [Function], + "info":

+ The dimensions to partition on. +

, + "label": "Default partition dimensions", + "name": "defaultPartitionsSpec.partitionDimensions", + "required": true, + "type": "string-array", + }, + { + "defined": [Function], + "info":

+ Target number of rows per segment. Either + + targetRowsPerSegment + + or + + + maxRowsPerSegment + + must be set. +

, + "label": "Default target rows per segment", + "name": "defaultPartitionsSpec.targetRowsPerSegment", + "required": [Function], + "type": "number", + "zeroMeansUndefined": true, + }, + { + "customDialog": [Function], + "customSummary": [Function], + "defined": [Function], + "info":

+ Optional virtual columns used if your default partitions spec range partitioning definition references virtual columns. +

, + "label": "Default partitioning virtual columns", + "name": "defaultPartitioningVirtualColumns", + "type": "custom", + }, + { + "customDialog": [Function], + "customSummary": [Function], + "info":

+ Configure the reindexing rules that control how data is compacted as it ages. Rules define partitioning, deletion, index spec, and data schema changes. +

, + "label": "Rule provider", + "name": "ruleProvider", + "required": true, + "type": "custom", + }, + { + "adjustment": [Function], + "defaultValue": [Function], + "info":

+ Choose whether to skip recent data and how the offset is calculated. + + disabled + + + means no skip offset. + + fromLatest + + skips relative to the end of the latest segment. + + fromNow + + skips relative to the current time. +

, + "label": "Skip offset", + "name": "skipOffsetType", + "suggestions": [ + "disabled", + "fromLatest", + "fromNow", + ], + "type": "string", + }, + { + "defined": [Function], + "info":

+ ISO 8601 period. Skips data newer than this offset from the end of the latest segment. +

, + "label": "Skip offset value", + "name": "skipOffsetFromLatest", + "suggestions": [ + "PT0H", + "PT1H", + "P1D", + "P3D", + ], + "type": "string", + }, + { + "defined": [Function], + "info":

+ ISO 8601 period. Skips data newer than this offset from the current time. +

, + "label": "Skip offset value", + "name": "skipOffsetFromNow", + "suggestions": [ + "PT0H", + "PT1H", + "P1D", + "P3D", + ], + "type": "string", + }, + { + "defaultValue": 25, + "hideInMore": true, + "info":

+ Priority of compaction tasks. +

, + "min": 0, + "name": "taskPriority", + "type": "number", + }, + { + "hideInMore": true, + "info":

+ Maximum total input segment size in bytes per compaction task. +

, + "name": "inputSegmentSizeBytes", + "type": "size-bytes", + }, + { + "defaultValue": false, + "info":

+ Enable concurrent append and replace for the datasource. Recommended if you are appending data to a datasource while compaction is running. +

, + "label": "Task context: concurrent locks", + "name": "taskContext.useConcurrentLocks", + "type": "boolean", + }, + { + "info":

+ Maximum number of tasks (including the controller) for MSQ compaction. Must be at least 2 (one controller, one worker). +

, + "label": "Task context: max num tasks", + "min": 2, + "name": "taskContext.maxNumTasks", + "placeholder": "(cluster default)", + "type": "number", + "zeroMeansUndefined": true, + }, + { + "hideInMore": true, + "info":

+ Maximum number of rows to hold in memory before persisting. Lower values reduce memory usage but may increase disk I/O. +

, + "label": "Task context: max rows in memory", + "name": "taskContext.maxRowsInMemory", + "placeholder": "(default)", + "type": "number", + "zeroMeansUndefined": true, + }, + { + "hideInMore": true, + "info":

+ Maximum frame size in bytes for MSQ tasks. Increase if tasks fail due to frame size limits. +

, + "label": "Task context: max frame size", + "name": "taskContext.maxFrameSize", + "placeholder": "(default)", + "type": "number", + "zeroMeansUndefined": true, + }, + { + "hideInMore": true, + "info":

+ Full task context map. Common settings are available as dedicated fields above. Use this to set additional MSQ context parameters. +

, + "label": "Task context: additional settings", + "name": "taskContext", + "type": "json", + }, + { + "hideInMore": true, + "info":

+ Tuning config for compaction tasks. Note: you cannot set + + partitionsSpec + + inside + + + tuningConfig + + for cascading reindexing — partitioning is controlled by rules and defaults. +

, + "name": "tuningConfig", + "type": "json", + }, + ] + } + model={ + { + "dataSource": "test1", + "defaultPartitionsSpec": { + "maxRowsPerSegment": 5000000, + "type": "dynamic", + }, + "defaultSegmentGranularity": "DAY", + "ruleProvider": { + "deletionRules": [ + { + "deleteWhere": { + "column": "isRobot", + "matchValue": "true", + "type": "equals", + }, + "id": "remove-bots", + "olderThan": "P90D", + }, + ], + "type": "inline", + }, + "type": "reindexCascade", + } + } + onChange={[Function]} + /> + +
+
+
+ + + + +
+
+ +`; + +exports[`CompactionConfigDialog matches snapshot with useSupervisors (inline template) 1`] = ` + + +
+ + + + + + + + + The offset for searching segments to be compacted. Strongly recommended to set for realtime dataSources. +

, + "name": "skipOffsetFromLatest", + "suggestions": [ + "PT0H", + "PT1H", + "P1D", + "P3D", + ], + "type": "string", + }, + { + "info":

+ For perfect rollup, you should use either + + hashed + + (partitioning based on the hash of dimensions in each row) or + + range + + (based on several dimensions). For best-effort rollup, you should use + + dynamic + + . +

, + "label": "Partitioning type", + "name": "tuningConfig.partitionsSpec.type", + "suggestions": [ + "dynamic", + "hashed", + "range", + ], + "type": "string", + }, + { + "defaultValue": 5000000, + "defined": [Function], + "info": + Determines how many rows are in each segment. + , + "name": "tuningConfig.partitionsSpec.maxRowsPerSegment", + "type": "number", + }, + { + "defaultValue": 20000000, + "defined": [Function], + "info": + Total number of rows in segments waiting for being pushed. + , + "name": "tuningConfig.partitionsSpec.maxTotalRows", + "type": "number", + }, + { + "defined": [Function], + "info": +

+ If the segments generated are a sub-optimal size for the requested partition dimensions, consider setting this field. +

+

+ A target row count for each partition. Each partition will have a row count close to the target assuming evenly distributed keys. Defaults to 5 million if numShards is null. +

+

+ If + + numShards + + is left unspecified, the Parallel task will determine + + + numShards + + automatically by + + targetRowsPerSegment + + . +

+

+ Note that either + + targetRowsPerSegment + + or + + numShards + + will be used to evenly distribute rows and find the best partitioning. Leave blank to show all properties. +

+ , + "name": "tuningConfig.partitionsSpec.targetRowsPerSegment", + "placeholder": "(defaults to 500000)", + "type": "number", + "zeroMeansUndefined": true, + }, + { + "defined": [Function], + "info": +

+ Target number of rows to include in a partition, should be a number that targets segments of 500MB~1GB. +

+

+ + maxRowsPerSegment + + renamed to + + targetRowsPerSegment + +

+

+ If + + numShards + + is left unspecified, the Parallel task will determine + + + numShards + + automatically by + + targetRowsPerSegment + + . +

+

+ Note that either + + targetRowsPerSegment + + or + + numShards + + will be used to evenly distribute rows and find the best partitioning. Leave blank to show all properties. +

+ , + "name": "tuningConfig.partitionsSpec.maxRowsPerSegment", + "type": "number", + "zeroMeansUndefined": true, + }, + { + "defined": [Function], + "info": +

+ If you know the optimal number of shards and want to speed up the time it takes for compaction to run, set this field. +

+

+ Directly specify the number of shards to create. If this is specified and 'intervals' is specified in the granularitySpec, the index task can skip the determine intervals/partitions pass through the data. +

+

+ Note that either + + targetRowsPerSegment + + or + + numShards + + will be used to evenly distribute rows across partitions and find the best partitioning. Leave blank to show all properties. +

+ , + "name": "tuningConfig.partitionsSpec.numShards", + "type": "number", + "zeroMeansUndefined": true, + }, + { + "defined": [Function], + "info":

+ The dimensions to partition on. Leave blank to select all dimensions. +

, + "name": "tuningConfig.partitionsSpec.partitionDimensions", + "placeholder": "(all dimensions)", + "type": "string-array", + }, + { + "defined": [Function], + "info":

+ The dimension to partition on. +

, + "name": "tuningConfig.partitionsSpec.partitionDimension", + "required": true, + "type": "string", + }, + { + "defined": [Function], + "info":

+ The dimensions to partition on. +

, + "name": "tuningConfig.partitionsSpec.partitionDimensions", + "required": true, + "type": "string-array", + }, + { + "defined": [Function], + "info": +

+ Target number of rows to include in a partition, should be a number that targets segments of 500MB~1GB. +

+

+ Note that either + + targetRowsPerSegment + + or + + maxRowsPerSegment + + will be used to find the best partitioning. Leave blank to show all properties. +

+ , + "name": "tuningConfig.partitionsSpec.targetRowsPerSegment", + "required": [Function], + "type": "number", + "zeroMeansUndefined": true, + }, + { + "defined": [Function], + "info": +

+ Maximum number of rows to include in a partition. +

+

+ Note that either + + targetRowsPerSegment + + or + + maxRowsPerSegment + + will be used to find the best partitioning. Leave blank to show all properties. +

+ , + "name": "tuningConfig.partitionsSpec.maxRowsPerSegment", + "required": [Function], + "type": "number", + "zeroMeansUndefined": true, + }, + { + "defaultValue": false, + "defined": [Function], + "info":

+ Assume that input data has already been grouped on time and dimensions. Ingestion will run faster, but may choose sub-optimal partitions if this assumption is violated. +

, + "name": "tuningConfig.partitionsSpec.assumeGrouped", + "type": "boolean", + }, + { + "defaultValue": 1, + "info": + Maximum number of tasks which can be run at the same time. The supervisor task would spawn worker tasks up to maxNumConcurrentSubTasks regardless of the available task slots. If this value is set to 1, the supervisor task processes data ingestion on its own instead of spawning worker tasks. If this value is set to too large, too many worker tasks can be created which might block other ingestion. + , + "min": 1, + "name": "tuningConfig.maxNumConcurrentSubTasks", + "type": "number", + }, + { + "defaultValue": -1, + "info": +

+ Limit of the number of segments to merge in a single phase when merging segments for publishing. This limit affects the total number of columns present in a set of segments to merge. If the limit is exceeded, segment merging occurs in multiple phases. Druid merges at least 2 segments per phase, regardless of this setting. +

+

+ Default: -1 (unlimited) +

+ , + "min": -1, + "name": "tuningConfig.maxColumnsToMerge", + "type": "number", + }, + { + "defaultValue": 10, + "defined": [Function], + "info": + Maximum number of merge tasks which can be run at the same time. + , + "min": 1, + "name": "tuningConfig.totalNumMergeTasks", + "type": "number", + }, + { + "adjustment": [Function], + "defaultValue": 1073741824, + "hideInMore": true, + "info": + Maximum number of bytes of input segments to process in a single task. If a single segment is larger than this number, it will be processed by itself in a single task (input segments are never split across tasks). + , + "min": 1000000, + "name": "tuningConfig.splitHintSpec.maxSplitSize", + "type": "number", + }, + { + "adjustment": [Function], + "defaultValue": 1000, + "hideInMore": true, + "info": + Maximum number of input segments to process in a single subtask. This limit is to avoid task failures when the ingestion spec is too long. There are two known limits on the max size of serialized ingestion spec, i.e., the max ZNode size in ZooKeeper ( + + jute.maxbuffer + + ) and the max packet size in MySQL ( + + max_allowed_packet + + ). These can make ingestion tasks fail if the serialized ingestion spec size hits one of them. + , + "label": "Max num files (segments)", + "min": 1, + "name": "tuningConfig.splitHintSpec.maxNumFiles", + "type": "number", + }, + ] + } + model={ + { + "dataSource": "test1", + "tuningConfig": { + "partitionsSpec": { + "type": "dynamic", + }, + }, + } + } + onChange={[Function]} + /> + +

+ If you want to append data to a datasource while compaction is running, you need to enable concurrent append and replace for the datasource by updating the compaction settings. +

+

+ For more information refer to the + + + documentation + + . +

+ + } + inlineInfo={true} + > + + + +
+
+
+ + + + +
+
+ +`; + exports[`CompactionConfigDialog matches snapshot without compactionConfig 1`] = ` + +
+
+ + Unique identifier for this rule. +

, + "name": "id", + "required": true, + "type": "string", + }, + { + "info":

+ ISO 8601 period defining the age threshold. The rule applies to data older than the current time minus this period. +

, + "name": "olderThan", + "required": true, + "suggestions": [ + "P1D", + "P7D", + "P30D", + "P90D", + "P180D", + "P365D", + ], + "type": "string", + }, + { + "info":

+ Human-readable description of this rule. +

, + "name": "description", + "type": "string", + }, + { + "info":

+ Time granularity for segment buckets. +

, + "name": "segmentGranularity", + "required": true, + "suggestions": [ + "MINUTE", + "FIFTEEN_MINUTE", + "HOUR", + "DAY", + "MONTH", + "QUARTER", + "YEAR", + ], + "type": "string", + }, + { + "info":

+ Use + + dynamic + + for best-effort rollup or + + range + + for range-based partitioning. +

, + "label": "Partitioning type", + "name": "partitionsSpec.type", + "suggestions": [ + "dynamic", + "range", + ], + "type": "string", + }, + { + "defaultValue": 5000000, + "defined": [Function], + "info": + Determines how many rows are in each segment. + , + "label": "Max rows per segment", + "name": "partitionsSpec.maxRowsPerSegment", + "type": "number", + }, + { + "defined": [Function], + "info":

+ The dimensions to partition on. +

, + "label": "Partition dimensions", + "name": "partitionsSpec.partitionDimensions", + "required": true, + "type": "string-array", + }, + { + "defined": [Function], + "info":

+ Target number of rows per segment for range partitioning. +

, + "label": "Target rows per segment", + "name": "partitionsSpec.targetRowsPerSegment", + "type": "number", + "zeroMeansUndefined": true, + }, + { + "customDialog": [Function], + "customSummary": [Function], + "defined": [Function], + "info":

+ Virtual columns for partitioning by nested or derived fields. +

, + "name": "virtualColumns", + "type": "custom", + }, + ] + } + onChange={[Function]} + onNewRule={[Function]} + rules={[]} + title="Partitioning rules" + /> + + Unique identifier for this rule. +

, + "name": "id", + "required": true, + "type": "string", + }, + { + "info":

+ ISO 8601 period defining the age threshold. The rule applies to data older than the current time minus this period. +

, + "name": "olderThan", + "required": true, + "suggestions": [ + "P1D", + "P7D", + "P30D", + "P90D", + "P180D", + "P365D", + ], + "type": "string", + }, + { + "info":

+ Human-readable description of this rule. +

, + "name": "description", + "type": "string", + }, + { + "info":

+ A Druid filter matching rows to + + delete + + . The compacted data retains rows that do not match this filter. Multiple deletion rules combine as + + + NOT(A OR B OR C) + + . +

, + "name": "deleteWhere", + "required": true, + "type": "json", + }, + { + "hideInMore": true, + "info":

+ Virtual columns for filtering on nested or derived fields. +

, + "name": "virtualColumns", + "type": "json", + }, + ] + } + onChange={[Function]} + onNewRule={[Function]} + rules={[]} + title="Deletion rules" + /> + + Unique identifier for this rule. +

, + "name": "id", + "required": true, + "type": "string", + }, + { + "info":

+ ISO 8601 period defining the age threshold. The rule applies to data older than the current time minus this period. +

, + "name": "olderThan", + "required": true, + "suggestions": [ + "P1D", + "P7D", + "P30D", + "P90D", + "P180D", + "P365D", + ], + "type": "string", + }, + { + "info":

+ Human-readable description of this rule. +

, + "name": "description", + "type": "string", + }, + { + "info":

+ An IndexSpec object defining bitmap type, metric compression, and other encoding settings for compacted segments. +

, + "name": "indexSpec", + "required": true, + "type": "json", + }, + ] + } + onChange={[Function]} + onNewRule={[Function]} + rules={[]} + title="Index spec rules" + /> + + Unique identifier for this rule. +

, + "name": "id", + "required": true, + "type": "string", + }, + { + "info":

+ ISO 8601 period defining the age threshold. The rule applies to data older than the current time minus this period. +

, + "name": "olderThan", + "required": true, + "suggestions": [ + "P1D", + "P7D", + "P30D", + "P90D", + "P180D", + "P365D", + ], + "type": "string", + }, + { + "info":

+ Human-readable description of this rule. +

, + "name": "description", + "type": "string", + }, + { + "info":

+ Query granularity for the compacted segments. Leave unset to preserve existing granularity. +

, + "name": "queryGranularity", + "placeholder": "(unset)", + "suggestions": [ + "MINUTE", + "FIFTEEN_MINUTE", + "HOUR", + "DAY", + "MONTH", + "QUARTER", + "YEAR", + ], + "type": "string", + }, + { + "info":

+ Whether to enable rollup. Set to + + true + + only when + + metricsSpec + + is defined. +

, + "name": "rollup", + "type": "boolean", + }, + { + "info": +

+ Array of aggregator factories for rollup metrics. Example: +

+ 
+                  [
+  {
+    "type": "longSum",
+    "name": "added",
+    "fieldName": "added"
+  },
+  {
+    "type": "longSum",
+    "name": "deleted",
+    "fieldName": "deleted"
+  }
+]
+
+ , + "name": "metricsSpec", + "type": "json", + }, + { + "info": +

+ Dimensions config for the compacted segments. Example: +

+ 
+                  {
+  "dimensions": [
+    "page",
+    { "type": "string", "name": "channel" }
+  ]
+}
+
+ , + "name": "dimensionsSpec", + "type": "json", + }, + { + "hideInMore": true, + "info":

+ List of aggregate projections. +

, + "name": "projections", + "type": "json", + }, + ] + } + onChange={[Function]} + onNewRule={[Function]} + rules={[]} + title="Data schema rules" + /> +
+
+
+
+ + +
+
+ +`; + +exports[`RuleProviderEditor matches snapshot with rules 1`] = ` + + +
+
+ + Unique identifier for this rule. +

, + "name": "id", + "required": true, + "type": "string", + }, + { + "info":

+ ISO 8601 period defining the age threshold. The rule applies to data older than the current time minus this period. +

, + "name": "olderThan", + "required": true, + "suggestions": [ + "P1D", + "P7D", + "P30D", + "P90D", + "P180D", + "P365D", + ], + "type": "string", + }, + { + "info":

+ Human-readable description of this rule. +

, + "name": "description", + "type": "string", + }, + { + "info":

+ Time granularity for segment buckets. +

, + "name": "segmentGranularity", + "required": true, + "suggestions": [ + "MINUTE", + "FIFTEEN_MINUTE", + "HOUR", + "DAY", + "MONTH", + "QUARTER", + "YEAR", + ], + "type": "string", + }, + { + "info":

+ Use + + dynamic + + for best-effort rollup or + + range + + for range-based partitioning. +

, + "label": "Partitioning type", + "name": "partitionsSpec.type", + "suggestions": [ + "dynamic", + "range", + ], + "type": "string", + }, + { + "defaultValue": 5000000, + "defined": [Function], + "info": + Determines how many rows are in each segment. + , + "label": "Max rows per segment", + "name": "partitionsSpec.maxRowsPerSegment", + "type": "number", + }, + { + "defined": [Function], + "info":

+ The dimensions to partition on. +

, + "label": "Partition dimensions", + "name": "partitionsSpec.partitionDimensions", + "required": true, + "type": "string-array", + }, + { + "defined": [Function], + "info":

+ Target number of rows per segment for range partitioning. +

, + "label": "Target rows per segment", + "name": "partitionsSpec.targetRowsPerSegment", + "type": "number", + "zeroMeansUndefined": true, + }, + { + "customDialog": [Function], + "customSummary": [Function], + "defined": [Function], + "info":

+ Virtual columns for partitioning by nested or derived fields. +

, + "name": "virtualColumns", + "type": "custom", + }, + ] + } + onChange={[Function]} + onNewRule={[Function]} + rules={ + [ + { + "id": "daily-30d", + "olderThan": "P30D", + "partitionsSpec": { + "maxRowsPerSegment": 5000000, + "type": "dynamic", + }, + "segmentGranularity": "DAY", + }, + ] + } + title="Partitioning rules" + /> + + Unique identifier for this rule. +

, + "name": "id", + "required": true, + "type": "string", + }, + { + "info":

+ ISO 8601 period defining the age threshold. The rule applies to data older than the current time minus this period. +

, + "name": "olderThan", + "required": true, + "suggestions": [ + "P1D", + "P7D", + "P30D", + "P90D", + "P180D", + "P365D", + ], + "type": "string", + }, + { + "info":

+ Human-readable description of this rule. +

, + "name": "description", + "type": "string", + }, + { + "info":

+ A Druid filter matching rows to + + delete + + . The compacted data retains rows that do not match this filter. Multiple deletion rules combine as + + + NOT(A OR B OR C) + + . +

, + "name": "deleteWhere", + "required": true, + "type": "json", + }, + { + "hideInMore": true, + "info":

+ Virtual columns for filtering on nested or derived fields. +

, + "name": "virtualColumns", + "type": "json", + }, + ] + } + onChange={[Function]} + onNewRule={[Function]} + rules={ + [ + { + "deleteWhere": { + "column": "isRobot", + "matchValue": "true", + "type": "equals", + }, + "id": "remove-bots", + "olderThan": "P90D", + }, + ] + } + title="Deletion rules" + /> + + Unique identifier for this rule. +

, + "name": "id", + "required": true, + "type": "string", + }, + { + "info":

+ ISO 8601 period defining the age threshold. The rule applies to data older than the current time minus this period. +

, + "name": "olderThan", + "required": true, + "suggestions": [ + "P1D", + "P7D", + "P30D", + "P90D", + "P180D", + "P365D", + ], + "type": "string", + }, + { + "info":

+ Human-readable description of this rule. +

, + "name": "description", + "type": "string", + }, + { + "info":

+ An IndexSpec object defining bitmap type, metric compression, and other encoding settings for compacted segments. +

, + "name": "indexSpec", + "required": true, + "type": "json", + }, + ] + } + onChange={[Function]} + onNewRule={[Function]} + rules={[]} + title="Index spec rules" + /> + + Unique identifier for this rule. +

, + "name": "id", + "required": true, + "type": "string", + }, + { + "info":

+ ISO 8601 period defining the age threshold. The rule applies to data older than the current time minus this period. +

, + "name": "olderThan", + "required": true, + "suggestions": [ + "P1D", + "P7D", + "P30D", + "P90D", + "P180D", + "P365D", + ], + "type": "string", + }, + { + "info":

+ Human-readable description of this rule. +

, + "name": "description", + "type": "string", + }, + { + "info":

+ Query granularity for the compacted segments. Leave unset to preserve existing granularity. +

, + "name": "queryGranularity", + "placeholder": "(unset)", + "suggestions": [ + "MINUTE", + "FIFTEEN_MINUTE", + "HOUR", + "DAY", + "MONTH", + "QUARTER", + "YEAR", + ], + "type": "string", + }, + { + "info":

+ Whether to enable rollup. Set to + + true + + only when + + metricsSpec + + is defined. +

, + "name": "rollup", + "type": "boolean", + }, + { + "info": +

+ Array of aggregator factories for rollup metrics. Example: +

+ 
+                  [
+  {
+    "type": "longSum",
+    "name": "added",
+    "fieldName": "added"
+  },
+  {
+    "type": "longSum",
+    "name": "deleted",
+    "fieldName": "deleted"
+  }
+]
+
+ , + "name": "metricsSpec", + "type": "json", + }, + { + "info": +

+ Dimensions config for the compacted segments. Example: +

+ 
+                  {
+  "dimensions": [
+    "page",
+    { "type": "string", "name": "channel" }
+  ]
+}
+
+ , + "name": "dimensionsSpec", + "type": "json", + }, + { + "hideInMore": true, + "info":

+ List of aggregate projections. +

, + "name": "projections", + "type": "json", + }, + ] + } + onChange={[Function]} + onNewRule={[Function]} + rules={[]} + title="Data schema rules" + /> +
+
+
+
+ + +
+
+ +`; diff --git a/web-console/src/dialogs/compaction-config-dialog/__snapshots__/virtual-columns-editor.spec.tsx.snap b/web-console/src/dialogs/compaction-config-dialog/__snapshots__/virtual-columns-editor.spec.tsx.snap new file mode 100644 index 000000000000..542e81980fc2 --- /dev/null +++ b/web-console/src/dialogs/compaction-config-dialog/__snapshots__/virtual-columns-editor.spec.tsx.snap @@ -0,0 +1,184 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`VirtualColumnsEditor matches snapshot with no virtual columns 1`] = ` + + +
+
+ + Add virtual column + +
+
+
+
+ + +
+
+ +`; + +exports[`VirtualColumnsEditor matches snapshot with virtual columns 1`] = ` + + +
+
+ +
+ + + city_and_country + +
+ +
+ +
+ + Output name of the virtual column. +

, + "name": "name", + "required": true, + "type": "string", + }, + { + "info":

+ Druid expression that computes the virtual column value. +

, + "name": "expression", + "required": true, + "type": "string", + }, + { + "info":

+ Output type of the expression result. +

, + "name": "outputType", + "suggestions": [ + "STRING", + "LONG", + "FLOAT", + "DOUBLE", + "COMPLEX", + ], + "type": "string", + }, + ] + } + model={ + { + "expression": "concat(cityName, ':', countryName)", + "name": "city_and_country", + "outputType": "STRING", + "type": "expression", + } + } + onChange={[Function]} + /> +
+ + + + Add virtual column + +
+
+
+
+ + +
+
+ +`; diff --git a/web-console/src/dialogs/compaction-config-dialog/compaction-config-dialog.spec.tsx b/web-console/src/dialogs/compaction-config-dialog/compaction-config-dialog.spec.tsx index cd620c9fe121..cd805f941172 100644 --- a/web-console/src/dialogs/compaction-config-dialog/compaction-config-dialog.spec.tsx +++ b/web-console/src/dialogs/compaction-config-dialog/compaction-config-dialog.spec.tsx @@ -81,4 +81,47 @@ describe('CompactionConfigDialog', () => { ); expect(compactionDialog).toMatchSnapshot(); }); + + it('matches snapshot with useSupervisors (inline template)', () => { + const compactionDialog = shallow( + {}} + onSave={() => {}} + onDelete={() => {}} + datasource="test1" + compactionConfig={undefined} + useSupervisors + />, + ); + expect(compactionDialog).toMatchSnapshot(); + }); + + it('matches snapshot with reindexCascade compactionConfig', () => { + const compactionDialog = shallow( + {}} + onSave={() => {}} + onDelete={() => {}} + datasource="test1" + compactionConfig={{ + type: 'reindexCascade', + dataSource: 'test1', + defaultSegmentGranularity: 'DAY', + defaultPartitionsSpec: { type: 'dynamic', maxRowsPerSegment: 5000000 }, + ruleProvider: { + type: 'inline', + deletionRules: [ + { + id: 'remove-bots', + olderThan: 'P90D', + deleteWhere: { type: 'equals', column: 'isRobot', matchValue: 'true' }, + }, + ], + }, + }} + useSupervisors + />, + ); + expect(compactionDialog).toMatchSnapshot(); + }); }); diff --git a/web-console/src/dialogs/compaction-config-dialog/compaction-config-dialog.tsx b/web-console/src/dialogs/compaction-config-dialog/compaction-config-dialog.tsx index 35132d79fe79..4a52f703aa75 100644 --- a/web-console/src/dialogs/compaction-config-dialog/compaction-config-dialog.tsx +++ b/web-console/src/dialogs/compaction-config-dialog/compaction-config-dialog.tsx @@ -16,10 +16,20 @@ * limitations under the License. */ -import { Button, Callout, Classes, Code, Dialog, Intent, Switch } from '@blueprintjs/core'; +import { + Button, + Callout, + Classes, + Code, + Dialog, + FormGroup, + HTMLSelect, + Intent, + Switch, +} from '@blueprintjs/core'; import React, { useState } from 'react'; -import type { FormJsonTabs } from '../../components'; +import type { Field, FormJsonTabs } from '../../components'; import { AutoForm, ExternalLink, @@ -28,16 +38,24 @@ import { JsonInput, PopoverText, } from '../../components'; -import type { CompactionConfig } from '../../druid-models'; +import type { + CompactionConfig, + ExpressionVirtualColumn, + InlineRuleProvider, +} from '../../druid-models'; import { COMPACTION_CONFIG_FIELDS, compactionConfigHasLegacyInputSegmentSizeBytesSet, + newReindexCascadeConfig, + REINDEX_CASCADE_CONFIG_FIELDS, } from '../../druid-models'; import { getLink } from '../../links'; import { deepDelete, deepGet, deepSet, formatBytesCompact } from '../../utils'; import { CompactionHistoryDialog } from '../compaction-history-dialog/compaction-history-dialog'; import { COMPACTION_CONFIG_COMPLETIONS } from './compaction-config-completions'; +import { RuleProviderEditor } from './rule-provider-editor'; +import { VirtualColumnsEditor } from './virtual-columns-editor'; import './compaction-config-dialog.scss'; @@ -47,12 +65,19 @@ export interface CompactionConfigDialogProps { onDelete: () => void; datasource: string; compactionConfig: CompactionConfig | undefined; + useSupervisors?: boolean; +} + +type TemplateType = 'inline' | 'reindexCascade'; + +function getTemplateType(config: CompactionConfig): TemplateType { + return config.type === 'reindexCascade' ? 'reindexCascade' : 'inline'; } export const CompactionConfigDialog = React.memo(function CompactionConfigDialog( props: CompactionConfigDialogProps, ) { - const { datasource, compactionConfig, onSave, onClose, onDelete } = props; + const { datasource, compactionConfig, useSupervisors, onSave, onClose, onDelete } = props; const [showHistory, setShowHistory] = useState(false); const [currentTab, setCurrentTab] = useState('form'); @@ -64,9 +89,81 @@ export const CompactionConfigDialog = React.memo(function CompactionConfigDialog ); const [jsonError, setJsonError] = useState(); - const issueWithCurrentConfig = AutoForm.issueWithModel(currentConfig, COMPACTION_CONFIG_FIELDS); + const templateType = getTemplateType(currentConfig); + const isReindexCascade = templateType === 'reindexCascade'; + + const fields: Field[] = isReindexCascade + ? (REINDEX_CASCADE_CONFIG_FIELDS as Field[]) + : COMPACTION_CONFIG_FIELDS; + const issueWithCurrentConfig = AutoForm.issueWithModel(currentConfig, fields); const disableSubmit = Boolean(jsonError || issueWithCurrentConfig); + function handleTemplateTypeChange(newType: TemplateType) { + if (newType === templateType) return; + if (newType === 'reindexCascade') { + setCurrentConfig(newReindexCascadeConfig(datasource)); + } else { + setCurrentConfig({ + dataSource: datasource, + tuningConfig: { partitionsSpec: { type: 'dynamic' } }, + }); + } + } + + function makeVirtualColumnsDialog({ + value, + onValueChange, + onClose: dialogClose, + }: { + value: any; + onValueChange: (v: any) => void; + onClose: () => void; + }) { + return ( + { + onValueChange(vcs.length ? vcs : undefined); + }} + onClose={dialogClose} + /> + ); + } + + // Build the reindexCascade field list with customDialog wired up + function getFieldsWithDialogs(): Field[] { + if (!isReindexCascade) return fields; + return fields.map(field => { + if (field.name === 'ruleProvider') { + return { + ...field, + customDialog: ({ + value, + onValueChange, + onClose: dialogClose, + }: { + value: any; + onValueChange: (v: any) => void; + onClose: () => void; + }) => ( + + ), + }; + } + if (field.name === 'defaultPartitioningVirtualColumns') { + return { + ...field, + customDialog: makeVirtualColumnsDialog, + }; + } + return field; + }); + } + if (showHistory) { return ( setShowHistory(false)} /> @@ -81,7 +178,7 @@ export const CompactionConfigDialog = React.memo(function CompactionConfigDialog canOutsideClickClose={false} title={`Compaction config: ${datasource}`} > - {compactionConfigHasLegacyInputSegmentSizeBytesSet(currentConfig) && ( + {!isReindexCascade && compactionConfigHasLegacyInputSegmentSizeBytesSet(currentConfig) && (

Your current config sets the legacy inputSegmentSizeBytes to{' '} @@ -107,51 +204,64 @@ export const CompactionConfigDialog = React.memo(function CompactionConfigDialog

{currentTab === 'form' ? ( <> + {useSupervisors && ( + + handleTemplateTypeChange(e.target.value as TemplateType)} + > + + + + + )} setCurrentConfig(m as CompactionConfig)} /> - -

- If you want to append data to a datasource while compaction is running, you need - to enable concurrent append and replace for the datasource by updating the - compaction settings. -

-

- For more information refer to the{' '} - - documentation - - . -

- - } - > - { - setCurrentConfig( - (deepGet(currentConfig, 'taskContext.useConcurrentLocks') - ? deepDelete(currentConfig, 'taskContext.useConcurrentLocks') - : deepSet(currentConfig, 'taskContext.useConcurrentLocks', true)) as any, - ); - }} - /> - + {!isReindexCascade && ( + +

+ If you want to append data to a datasource while compaction is running, you + need to enable concurrent append and replace for the datasource by updating + the compaction settings. +

+

+ For more information refer to the{' '} + + documentation + + . +

+ + } + > + { + setCurrentConfig( + (deepGet(currentConfig, 'taskContext.useConcurrentLocks') + ? deepDelete(currentConfig, 'taskContext.useConcurrentLocks') + : deepSet(currentConfig, 'taskContext.useConcurrentLocks', true)) as any, + ); + }} + /> + + )} ) : ( AutoForm.issueWithModel(value, COMPACTION_CONFIG_FIELDS)} + issueWithValue={value => AutoForm.issueWithModel(value, fields)} height="100%" - jsonCompletions={COMPACTION_CONFIG_COMPLETIONS} + jsonCompletions={!isReindexCascade ? COMPACTION_CONFIG_COMPLETIONS : undefined} /> )}
diff --git a/web-console/src/dialogs/compaction-config-dialog/rule-provider-editor.scss b/web-console/src/dialogs/compaction-config-dialog/rule-provider-editor.scss new file mode 100644 index 000000000000..40fb911fdcea --- /dev/null +++ b/web-console/src/dialogs/compaction-config-dialog/rule-provider-editor.scss @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@import '../../variables'; + +.rule-provider-editor { + &.#{$bp-ns}-dialog { + height: 80vh; + width: 700px; + } + + .form-json-selector { + margin: 15px; + } + + .content { + margin: 0 15px 10px 0; + padding: 0 5px 0 15px; + flex: 1; + overflow: auto; + } + + .rule-sections { + display: flex; + flex-direction: column; + gap: 10px; + } + + .rule-section { + .rule-section-header { + display: flex; + align-items: center; + gap: 5px; + + .rule-section-title { + margin: 0; + } + + .rule-section-count { + opacity: 0.6; + } + } + + .rule-section-description { + margin: 0 0 10px 30px; + opacity: 0.7; + font-size: 12px; + } + } + + .rule-card { + margin: 0 0 8px 0; + padding: 0; + + .rule-card-header { + display: flex; + align-items: center; + padding: 5px; + gap: 5px; + + .rule-card-summary { + font-weight: 500; + } + + .rule-card-age { + opacity: 0.6; + font-size: 12px; + } + + .spacer { + flex: 1; + } + } + + .rule-card-form { + padding: 10px 15px; + border-top: 1px solid rgba(255, 255, 255, 0.1); + } + } +} diff --git a/web-console/src/dialogs/compaction-config-dialog/rule-provider-editor.spec.tsx b/web-console/src/dialogs/compaction-config-dialog/rule-provider-editor.spec.tsx new file mode 100644 index 000000000000..e1b131533dc1 --- /dev/null +++ b/web-console/src/dialogs/compaction-config-dialog/rule-provider-editor.spec.tsx @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { shallow } from '../../utils/shallow-renderer'; + +import { RuleProviderEditor } from './rule-provider-editor'; + +describe('RuleProviderEditor', () => { + it('matches snapshot with empty provider', () => { + const editor = shallow( + {}} onSave={() => {}} ruleProvider={undefined} />, + ); + expect(editor).toMatchSnapshot(); + }); + + it('matches snapshot with rules', () => { + const editor = shallow( + {}} + onSave={() => {}} + ruleProvider={{ + type: 'inline', + partitioningRules: [ + { + id: 'daily-30d', + olderThan: 'P30D', + segmentGranularity: 'DAY', + partitionsSpec: { type: 'dynamic', maxRowsPerSegment: 5000000 }, + }, + ], + deletionRules: [ + { + id: 'remove-bots', + olderThan: 'P90D', + deleteWhere: { type: 'equals', column: 'isRobot', matchValue: 'true' }, + }, + ], + }} + />, + ); + expect(editor).toMatchSnapshot(); + }); +}); diff --git a/web-console/src/dialogs/compaction-config-dialog/rule-provider-editor.tsx b/web-console/src/dialogs/compaction-config-dialog/rule-provider-editor.tsx new file mode 100644 index 000000000000..8bc8dac90e1f --- /dev/null +++ b/web-console/src/dialogs/compaction-config-dialog/rule-provider-editor.tsx @@ -0,0 +1,275 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Button, Card, Classes, Collapse, Dialog, H5, Intent } from '@blueprintjs/core'; +import { IconNames } from '@blueprintjs/icons'; +import React, { useState } from 'react'; + +import type { FormJsonTabs } from '../../components'; +import { AutoForm, FormJsonSelector, JsonInput } from '../../components'; +import type { + DataSchemaRule, + DeletionRule, + ExpressionVirtualColumn, + IndexSpecRule, + InlineRuleProvider, + PartitioningRule, + ReindexingRule, +} from '../../druid-models'; +import { + DATA_SCHEMA_RULE_FIELDS, + DELETION_RULE_FIELDS, + INDEX_SPEC_RULE_FIELDS, + newDataSchemaRule, + newDeletionRule, + newIndexSpecRule, + newPartitioningRule, + PARTITIONING_RULE_FIELDS, + summarizeRule, +} from '../../druid-models'; +import { swapElements } from '../../utils'; + +import { VirtualColumnsEditor } from './virtual-columns-editor'; + +import './rule-provider-editor.scss'; + +export interface RuleProviderEditorProps { + onClose: () => void; + onSave: (ruleProvider: InlineRuleProvider) => void; + ruleProvider: InlineRuleProvider | undefined; +} + +function makeVirtualColumnsDialog({ + value, + onValueChange, + onClose: dialogClose, +}: { + value: any; + onValueChange: (v: any) => void; + onClose: () => void; +}) { + return ( + { + onValueChange(vcs.length ? vcs : undefined); + }} + onClose={dialogClose} + /> + ); +} + +// Augment partitioning rule fields with customDialog for virtualColumns +const PARTITIONING_RULE_FIELDS_WITH_DIALOGS = PARTITIONING_RULE_FIELDS.map(field => { + if (field.name === 'virtualColumns') { + return { ...field, customDialog: makeVirtualColumnsDialog }; + } + return field; +}); + +export const RuleProviderEditor = React.memo(function RuleProviderEditor( + props: RuleProviderEditorProps, +) { + const { onClose, onSave, ruleProvider } = props; + + const [currentTab, setCurrentTab] = useState('form'); + const [currentProvider, setCurrentProvider] = useState( + ruleProvider || { type: 'inline' }, + ); + const [jsonError, setJsonError] = useState(); + + return ( + + { + setJsonError(undefined); + setCurrentTab(t); + }} + /> +
+ {currentTab === 'form' ? ( +
+ + title="Partitioning rules" + description="Control segment granularity and partitioning as data ages. Non-additive: only one applies per interval." + rules={currentProvider.partitioningRules || []} + fields={PARTITIONING_RULE_FIELDS_WITH_DIALOGS} + onNewRule={newPartitioningRule} + onChange={rules => + setCurrentProvider({ ...currentProvider, partitioningRules: rules }) + } + /> + + title="Deletion rules" + description="Specify rows to remove during compaction. Additive: multiple rules can combine." + rules={currentProvider.deletionRules || []} + fields={DELETION_RULE_FIELDS} + onNewRule={newDeletionRule} + onChange={rules => setCurrentProvider({ ...currentProvider, deletionRules: rules })} + /> + + title="Index spec rules" + description="Control compression and encoding settings. Non-additive: only one applies per interval." + rules={currentProvider.indexSpecRules || []} + fields={INDEX_SPEC_RULE_FIELDS} + onNewRule={newIndexSpecRule} + onChange={rules => setCurrentProvider({ ...currentProvider, indexSpecRules: rules })} + /> + + title="Data schema rules" + description="Control dimensions, metrics, query granularity, rollup, and projections. Non-additive: only one applies per interval." + rules={currentProvider.dataSchemaRules || []} + fields={DATA_SCHEMA_RULE_FIELDS} + onNewRule={newDataSchemaRule} + onChange={rules => setCurrentProvider({ ...currentProvider, dataSchemaRules: rules })} + /> +
+ ) : ( + setCurrentProvider(v)} + setError={setJsonError} + height="100%" + /> + )} +
+
+
+
+
+ + ); +}); + +// --- Generic rule section component --- + +interface RuleSectionProps { + title: string; + description: string; + rules: R[]; + fields: any[]; + onNewRule: () => R; + onChange: (rules: R[]) => void; +} + +function RuleSection(props: RuleSectionProps) { + const { title, description, rules, fields, onNewRule, onChange } = props; + const [sectionOpen, setSectionOpen] = useState(true); + const [openRules, setOpenRules] = useState>({}); + + function toggleRule(index: number) { + setOpenRules({ ...openRules, [index]: !openRules[index] }); + } + + function addRule() { + onChange([...rules, onNewRule()]); + setOpenRules({ ...openRules, [rules.length]: true }); + } + + function deleteRule(index: number) { + onChange(rules.filter((_r, i) => i !== index)); + } + + function changeRule(index: number, newRule: R) { + onChange(rules.map((r, i) => (i === index ? newRule : r))); + } + + function moveRule(index: number, direction: number) { + onChange(swapElements(rules, index, index + direction)); + } + + return ( +
+
+
+ +

{description}

+ {rules.map((rule, index) => ( + +
+
+ +
+ changeRule(index, m as R)} /> +
+ + + ))} + + +
+ ); +} diff --git a/web-console/src/dialogs/compaction-config-dialog/virtual-columns-editor.scss b/web-console/src/dialogs/compaction-config-dialog/virtual-columns-editor.scss new file mode 100644 index 000000000000..10cf8ecc0eca --- /dev/null +++ b/web-console/src/dialogs/compaction-config-dialog/virtual-columns-editor.scss @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@import '../../variables'; + +.virtual-columns-editor { + &.#{$bp-ns}-dialog { + height: 60vh; + width: 550px; + } + + .form-json-selector { + margin: 15px; + } + + .content { + margin: 0 15px 10px 0; + padding: 0 5px 0 15px; + flex: 1; + overflow: auto; + } + + .vc-list { + display: flex; + flex-direction: column; + gap: 8px; + } + + .vc-card { + padding: 0; + + .vc-card-header { + display: flex; + align-items: center; + padding: 5px; + gap: 5px; + + .vc-card-name { + font-weight: 500; + } + + .spacer { + flex: 1; + } + } + + .vc-card-form { + padding: 10px 15px; + border-top: 1px solid rgba(255, 255, 255, 0.1); + } + } +} diff --git a/web-console/src/dialogs/compaction-config-dialog/virtual-columns-editor.spec.tsx b/web-console/src/dialogs/compaction-config-dialog/virtual-columns-editor.spec.tsx new file mode 100644 index 000000000000..332fb4fed19c --- /dev/null +++ b/web-console/src/dialogs/compaction-config-dialog/virtual-columns-editor.spec.tsx @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { shallow } from '../../utils/shallow-renderer'; + +import { VirtualColumnsEditor } from './virtual-columns-editor'; + +describe('VirtualColumnsEditor', () => { + it('matches snapshot with no virtual columns', () => { + const editor = shallow( + {}} onSave={() => {}} virtualColumns={undefined} />, + ); + expect(editor).toMatchSnapshot(); + }); + + it('matches snapshot with virtual columns', () => { + const editor = shallow( + {}} + onSave={() => {}} + virtualColumns={[ + { + type: 'expression', + name: 'city_and_country', + expression: "concat(cityName, ':', countryName)", + outputType: 'STRING', + }, + ]} + />, + ); + expect(editor).toMatchSnapshot(); + }); +}); diff --git a/web-console/src/dialogs/compaction-config-dialog/virtual-columns-editor.tsx b/web-console/src/dialogs/compaction-config-dialog/virtual-columns-editor.tsx new file mode 100644 index 000000000000..3727b9df4397 --- /dev/null +++ b/web-console/src/dialogs/compaction-config-dialog/virtual-columns-editor.tsx @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Button, Card, Classes, Collapse, Dialog, Intent } from '@blueprintjs/core'; +import { IconNames } from '@blueprintjs/icons'; +import React, { useState } from 'react'; + +import type { FormJsonTabs } from '../../components'; +import { AutoForm, FormJsonSelector, JsonInput } from '../../components'; +import type { ExpressionVirtualColumn } from '../../druid-models'; +import { EXPRESSION_VIRTUAL_COLUMN_FIELDS, newExpressionVirtualColumn } from '../../druid-models'; + +import './virtual-columns-editor.scss'; + +export interface VirtualColumnsEditorProps { + onClose: () => void; + onSave: (virtualColumns: ExpressionVirtualColumn[]) => void; + virtualColumns: ExpressionVirtualColumn[] | undefined; +} + +export const VirtualColumnsEditor = React.memo(function VirtualColumnsEditor( + props: VirtualColumnsEditorProps, +) { + const { onClose, onSave, virtualColumns } = props; + + const [currentTab, setCurrentTab] = useState('form'); + const [currentColumns, setCurrentColumns] = useState( + virtualColumns || [], + ); + const [jsonError, setJsonError] = useState(); + const [openCards, setOpenCards] = useState>( + Object.fromEntries((virtualColumns || []).map((_, i) => [i, true])), + ); + + function addColumn() { + setCurrentColumns([...currentColumns, newExpressionVirtualColumn()]); + setOpenCards({ ...openCards, [currentColumns.length]: true }); + } + + function deleteColumn(index: number) { + setCurrentColumns(currentColumns.filter((_c, i) => i !== index)); + } + + function changeColumn(index: number, col: ExpressionVirtualColumn) { + setCurrentColumns(currentColumns.map((c, i) => (i === index ? col : c))); + } + + function toggleCard(index: number) { + setOpenCards({ ...openCards, [index]: !openCards[index] }); + } + + return ( + + { + setJsonError(undefined); + setCurrentTab(t); + }} + /> +
+ {currentTab === 'form' ? ( +
+ {currentColumns.map((col, index) => ( + +
+
+ +
+ changeColumn(index, m as ExpressionVirtualColumn)} + /> +
+ + + ))} + +
+ ) : ( + setCurrentColumns(v)} + setError={setJsonError} + height="100%" + /> + )} +
+
+
+
+
+ + ); +}); diff --git a/web-console/src/druid-models/compaction-config/compaction-config.tsx b/web-console/src/druid-models/compaction-config/compaction-config.tsx index de4a2f47a843..fcaca5d6dd21 100644 --- a/web-console/src/druid-models/compaction-config/compaction-config.tsx +++ b/web-console/src/druid-models/compaction-config/compaction-config.tsx @@ -22,6 +22,7 @@ import type { Field } from '../../components'; import { deepGet, deepSet, oneOfKnown } from '../../utils'; export interface CompactionConfig { + type?: string; dataSource: string; skipOffsetFromLatest?: string; tuningConfig?: any; diff --git a/web-console/src/druid-models/compaction-config/reindex-cascade-config.spec.ts b/web-console/src/druid-models/compaction-config/reindex-cascade-config.spec.ts new file mode 100644 index 000000000000..6b2c981587c9 --- /dev/null +++ b/web-console/src/druid-models/compaction-config/reindex-cascade-config.spec.ts @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + newDataSchemaRule, + newDeletionRule, + newExpressionVirtualColumn, + newIndexSpecRule, + newPartitioningRule, + newReindexCascadeConfig, + summarizeRule, + summarizeRuleProvider, + summarizeVirtualColumns, +} from './reindex-cascade-config'; + +describe('reindex-cascade-config', () => { + describe('summarizeRuleProvider', () => { + it('returns (none) for undefined', () => { + expect(summarizeRuleProvider(undefined)).toBe('(none)'); + }); + + it('returns (no rules) for empty provider', () => { + expect(summarizeRuleProvider({ type: 'inline' })).toBe('(no rules)'); + }); + + it('summarizes partitioning rules', () => { + expect( + summarizeRuleProvider({ + type: 'inline', + partitioningRules: [newPartitioningRule()], + }), + ).toBe('1 partitioning rule'); + }); + + it('summarizes multiple rule types', () => { + expect( + summarizeRuleProvider({ + type: 'inline', + partitioningRules: [newPartitioningRule(), newPartitioningRule()], + deletionRules: [newDeletionRule()], + indexSpecRules: [newIndexSpecRule()], + dataSchemaRules: [newDataSchemaRule()], + }), + ).toBe('2 partitioning rules, 1 deletion rule, 1 index spec rule, 1 data schema rule'); + }); + }); + + describe('summarizeVirtualColumns', () => { + it('returns (none) for undefined', () => { + expect(summarizeVirtualColumns(undefined)).toBe('(none)'); + }); + + it('returns (none) for empty array', () => { + expect(summarizeVirtualColumns([])).toBe('(none)'); + }); + + it('summarizes named columns', () => { + expect( + summarizeVirtualColumns([ + { type: 'expression', name: 'v0', expression: 'x + 1' }, + { type: 'expression', name: 'v1', expression: 'y + 2' }, + ]), + ).toBe('v0, v1'); + }); + + it('shows (unnamed) for columns without names', () => { + expect(summarizeVirtualColumns([{ type: 'expression', name: '', expression: '' }])).toBe( + '(unnamed)', + ); + }); + }); + + describe('summarizeRule', () => { + it('uses description if present', () => { + expect(summarizeRule({ id: 'rule-1', olderThan: 'P30D', description: 'My rule' })).toBe( + 'My rule', + ); + }); + + it('falls back to id', () => { + expect(summarizeRule({ id: 'rule-1', olderThan: 'P30D' })).toBe('rule-1'); + }); + }); + + describe('newReindexCascadeConfig', () => { + it('returns correct defaults', () => { + const config = newReindexCascadeConfig('wikipedia'); + expect(config.type).toBe('reindexCascade'); + expect(config.dataSource).toBe('wikipedia'); + expect(config.defaultSegmentGranularity).toBe('DAY'); + expect(config.defaultPartitionsSpec).toEqual({ type: 'dynamic', maxRowsPerSegment: 5000000 }); + expect(config.ruleProvider).toEqual({ type: 'inline' }); + }); + }); + + describe('factory functions', () => { + it('newPartitioningRule returns correct defaults', () => { + const rule = newPartitioningRule(); + expect(rule.id).toBeTruthy(); + expect(rule.olderThan).toBe('P30D'); + expect(rule.segmentGranularity).toBe('DAY'); + expect(rule.partitionsSpec.type).toBe('dynamic'); + }); + + it('newDeletionRule returns correct defaults', () => { + const rule = newDeletionRule(); + expect(rule.id).toBeTruthy(); + expect(rule.olderThan).toBe('P90D'); + expect(rule.deleteWhere).toBeDefined(); + }); + + it('newIndexSpecRule returns correct defaults', () => { + const rule = newIndexSpecRule(); + expect(rule.id).toBeTruthy(); + expect(rule.olderThan).toBe('P90D'); + expect(rule.indexSpec).toEqual({}); + }); + + it('newDataSchemaRule returns correct defaults', () => { + const rule = newDataSchemaRule(); + expect(rule.id).toBeTruthy(); + expect(rule.olderThan).toBe('P30D'); + expect(rule.queryGranularity).toBeUndefined(); + }); + + it('generates unique IDs', () => { + const rule1 = newPartitioningRule(); + const rule2 = newPartitioningRule(); + expect(rule1.id).not.toBe(rule2.id); + }); + }); + + describe('newExpressionVirtualColumn', () => { + it('returns correct defaults', () => { + const vc = newExpressionVirtualColumn(); + expect(vc.type).toBe('expression'); + expect(vc.name).toBe(''); + expect(vc.expression).toBe(''); + }); + }); +}); diff --git a/web-console/src/druid-models/compaction-config/reindex-cascade-config.tsx b/web-console/src/druid-models/compaction-config/reindex-cascade-config.tsx new file mode 100644 index 000000000000..f2b3c60e7fa5 --- /dev/null +++ b/web-console/src/druid-models/compaction-config/reindex-cascade-config.tsx @@ -0,0 +1,730 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Code } from '@blueprintjs/core'; +import { v4 as uuidv4 } from 'uuid'; + +import type { Field } from '../../components'; +import { deepGet, oneOfKnown, pluralIfNeeded } from '../../utils'; + +// --- Virtual column types --- + +export interface ExpressionVirtualColumn { + type: 'expression'; + name: string; + expression: string; + outputType?: string; +} + +export const EXPRESSION_VIRTUAL_COLUMN_FIELDS: Field[] = [ + { + name: 'name', + type: 'string', + required: true, + info:

Output name of the virtual column.

, + }, + { + name: 'expression', + type: 'string', + required: true, + info:

Druid expression that computes the virtual column value.

, + }, + { + name: 'outputType', + type: 'string', + suggestions: ['STRING', 'LONG', 'FLOAT', 'DOUBLE', 'COMPLEX'], + info:

Output type of the expression result.

, + }, +]; + +export function newExpressionVirtualColumn(): ExpressionVirtualColumn { + return { + type: 'expression', + name: '', + expression: '', + }; +} + +export function summarizeVirtualColumns( + virtualColumns: ExpressionVirtualColumn[] | undefined, +): string { + if (!virtualColumns?.length) return '(none)'; + return virtualColumns.map(vc => vc.name || '(unnamed)').join(', '); +} + +// --- Reindexing rule types --- + +export interface ReindexingRule { + id: string; + olderThan: string; + description?: string; +} + +export interface PartitioningRule extends ReindexingRule { + segmentGranularity: string; + partitionsSpec: any; + virtualColumns?: any; +} + +export interface DeletionRule extends ReindexingRule { + deleteWhere: any; + virtualColumns?: any; +} + +export interface IndexSpecRule extends ReindexingRule { + indexSpec: any; +} + +export interface DataSchemaRule extends ReindexingRule { + dimensionsSpec?: any; + metricsSpec?: any[]; + queryGranularity?: string; + rollup?: boolean; + projections?: any[]; +} + +// --- Rule provider types --- + +export interface InlineRuleProvider { + type: 'inline'; + partitioningRules?: PartitioningRule[]; + deletionRules?: DeletionRule[]; + indexSpecRules?: IndexSpecRule[]; + dataSchemaRules?: DataSchemaRule[]; +} + +export type RuleProvider = InlineRuleProvider; + +// --- Top-level reindex cascade config --- + +export interface ReindexCascadeConfig { + type: 'reindexCascade'; + dataSource: string; + defaultSegmentGranularity: string; + defaultPartitionsSpec: any; + defaultPartitioningVirtualColumns?: any; + ruleProvider: RuleProvider; + taskPriority?: number; + inputSegmentSizeBytes?: number; + taskContext?: Record; + skipOffsetFromLatest?: string; + skipOffsetFromNow?: string; + tuningConfig?: any; +} + +// --- Factory functions for new rules --- + +export function newPartitioningRule(): PartitioningRule { + return { + id: uuidv4(), + olderThan: 'P30D', + segmentGranularity: 'DAY', + partitionsSpec: { type: 'dynamic', maxRowsPerSegment: 5000000 }, + }; +} + +export function newDeletionRule(): DeletionRule { + return { + id: uuidv4(), + olderThan: 'P90D', + deleteWhere: { type: 'equals', column: '', matchValueType: 'STRING', matchValue: '' }, + }; +} + +export function newIndexSpecRule(): IndexSpecRule { + return { + id: uuidv4(), + olderThan: 'P90D', + indexSpec: {}, + }; +} + +export function newDataSchemaRule(): DataSchemaRule { + return { + id: uuidv4(), + olderThan: 'P30D', + }; +} + +// --- Summary helpers --- + +export function summarizeRuleProvider(ruleProvider: RuleProvider | undefined): string { + if (!ruleProvider) return '(none)'; + const parts: string[] = []; + if (ruleProvider.partitioningRules?.length) { + parts.push(pluralIfNeeded(ruleProvider.partitioningRules.length, 'partitioning rule')); + } + if (ruleProvider.deletionRules?.length) { + parts.push(pluralIfNeeded(ruleProvider.deletionRules.length, 'deletion rule')); + } + if (ruleProvider.indexSpecRules?.length) { + parts.push(pluralIfNeeded(ruleProvider.indexSpecRules.length, 'index spec rule')); + } + if (ruleProvider.dataSchemaRules?.length) { + parts.push(pluralIfNeeded(ruleProvider.dataSchemaRules.length, 'data schema rule')); + } + return parts.length ? parts.join(', ') : '(no rules)'; +} + +export function summarizeRule(rule: ReindexingRule): string { + return rule.description || rule.id; +} + +// --- Shared field constants --- + +const SEGMENT_GRANULARITY_SUGGESTIONS = [ + 'MINUTE', + 'FIFTEEN_MINUTE', + 'HOUR', + 'DAY', + 'MONTH', + 'QUARTER', + 'YEAR', +]; + +const KNOWN_DEFAULT_PARTITION_TYPES = ['dynamic', 'range']; + +const PERIOD_SUGGESTIONS = ['P1D', 'P7D', 'P30D', 'P90D', 'P180D', 'P365D']; + +// --- Top-level fields for ReindexCascadeConfig --- + +export const REINDEX_CASCADE_CONFIG_FIELDS: Field[] = [ + { + name: 'defaultSegmentGranularity', + label: 'Default segment granularity', + type: 'string', + required: true, + suggestions: SEGMENT_GRANULARITY_SUGGESTIONS, + info: ( +

+ Segment granularity used for intervals where no partitioning rule matches. This is the + default time bucketing for segments. +

+ ), + }, + { + name: 'defaultPartitionsSpec.type', + label: 'Default partitioning type', + type: 'string', + suggestions: ['dynamic', 'range'], + info: ( +

+ Partitioning strategy used for intervals where no partitioning rule matches. Use{' '} + dynamic for best-effort rollup or range for range-based + partitioning. +

+ ), + }, + // defaultPartitionsSpec: dynamic + { + name: 'defaultPartitionsSpec.maxRowsPerSegment', + label: 'Default max rows per segment', + type: 'number', + defaultValue: 5000000, + defined: c => + oneOfKnown( + deepGet(c, 'defaultPartitionsSpec.type'), + KNOWN_DEFAULT_PARTITION_TYPES, + 'dynamic', + ), + info: <>Determines how many rows are in each segment., + }, + { + name: 'defaultPartitionsSpec.maxTotalRows', + label: 'Default max total rows', + type: 'number', + defaultValue: 20000000, + defined: c => + oneOfKnown( + deepGet(c, 'defaultPartitionsSpec.type'), + KNOWN_DEFAULT_PARTITION_TYPES, + 'dynamic', + ), + info: <>Total number of rows in segments waiting for being pushed., + }, + // defaultPartitionsSpec: range + { + name: 'defaultPartitionsSpec.partitionDimensions', + label: 'Default partition dimensions', + type: 'string-array', + defined: c => + oneOfKnown(deepGet(c, 'defaultPartitionsSpec.type'), KNOWN_DEFAULT_PARTITION_TYPES, 'range'), + required: true, + info:

The dimensions to partition on.

, + }, + { + name: 'defaultPartitionsSpec.targetRowsPerSegment', + label: 'Default target rows per segment', + type: 'number', + zeroMeansUndefined: true, + defined: c => + oneOfKnown(deepGet(c, 'defaultPartitionsSpec.type'), KNOWN_DEFAULT_PARTITION_TYPES, 'range'), + required: c => + !deepGet(c, 'defaultPartitionsSpec.targetRowsPerSegment') && + !deepGet(c, 'defaultPartitionsSpec.maxRowsPerSegment'), + info: ( +

+ Target number of rows per segment. Either targetRowsPerSegment or{' '} + maxRowsPerSegment must be set. +

+ ), + }, + { + name: 'defaultPartitioningVirtualColumns', + label: 'Default partitioning virtual columns', + type: 'custom', + defined: c => + oneOfKnown(deepGet(c, 'defaultPartitionsSpec.type'), KNOWN_DEFAULT_PARTITION_TYPES, 'range'), + customSummary: summarizeVirtualColumns, + info: ( +

+ Optional virtual columns used if your default partitions spec range partitioning definition + references virtual columns. +

+ ), + // customDialog is set by the CompactionConfigDialog component + }, + // ruleProvider (custom dialog) + { + name: 'ruleProvider', + label: 'Rule provider', + type: 'custom', + required: true, + customSummary: summarizeRuleProvider, + info: ( +

+ Configure the reindexing rules that control how data is compacted as it ages. Rules define + partitioning, deletion, index spec, and data schema changes. +

+ ), + // customDialog is set by the CompactionConfigDialog component + }, + // Skip offset - virtual selector with 3 options: disabled, fromLatest, fromNow + { + name: 'skipOffsetType', + label: 'Skip offset', + type: 'string', + suggestions: ['disabled', 'fromLatest', 'fromNow'], + defaultValue: (c: ReindexCascadeConfig) => { + if (c.skipOffsetFromNow) return 'fromNow'; + if (c.skipOffsetFromLatest) return 'fromLatest'; + return 'disabled'; + }, + adjustment: c => { + const skipType = (c as any).skipOffsetType; + const adjusted = { ...c }; + if (skipType === 'fromNow') { + if (!adjusted.skipOffsetFromNow) { + adjusted.skipOffsetFromNow = adjusted.skipOffsetFromLatest || 'P1D'; + } + delete adjusted.skipOffsetFromLatest; + } else if (skipType === 'fromLatest') { + if (!adjusted.skipOffsetFromLatest) { + adjusted.skipOffsetFromLatest = adjusted.skipOffsetFromNow || 'P1D'; + } + delete adjusted.skipOffsetFromNow; + } else { + // disabled + delete adjusted.skipOffsetFromLatest; + delete adjusted.skipOffsetFromNow; + } + delete (adjusted as any).skipOffsetType; + return adjusted; + }, + info: ( +

+ Choose whether to skip recent data and how the offset is calculated. disabled{' '} + means no skip offset. fromLatest skips relative to the end of the latest + segment. fromNow skips relative to the current time. +

+ ), + }, + { + name: 'skipOffsetFromLatest', + label: 'Skip offset value', + type: 'string', + suggestions: ['PT0H', 'PT1H', 'P1D', 'P3D'], + defined: c => Boolean(c.skipOffsetFromLatest), + info: ( +

ISO 8601 period. Skips data newer than this offset from the end of the latest segment.

+ ), + }, + { + name: 'skipOffsetFromNow', + label: 'Skip offset value', + type: 'string', + suggestions: ['PT0H', 'PT1H', 'P1D', 'P3D'], + defined: c => Boolean(c.skipOffsetFromNow), + info:

ISO 8601 period. Skips data newer than this offset from the current time.

, + }, + { + name: 'taskPriority', + type: 'number', + defaultValue: 25, + min: 0, + hideInMore: true, + info:

Priority of compaction tasks.

, + }, + { + name: 'inputSegmentSizeBytes', + type: 'size-bytes', + hideInMore: true, + info:

Maximum total input segment size in bytes per compaction task.

, + }, + // Promoted task context fields + { + name: 'taskContext.useConcurrentLocks', + label: 'Task context: concurrent locks', + type: 'boolean', + defaultValue: false, + info: ( +

+ Enable concurrent append and replace for the datasource. Recommended if you are appending + data to a datasource while compaction is running. +

+ ), + }, + { + name: 'taskContext.maxNumTasks', + label: 'Task context: max num tasks', + type: 'number', + min: 2, + placeholder: '(cluster default)', + zeroMeansUndefined: true, + info: ( +

+ Maximum number of tasks (including the controller) for MSQ compaction. Must be at least 2 + (one controller, one worker). +

+ ), + }, + { + name: 'taskContext.maxRowsInMemory', + label: 'Task context: max rows in memory', + type: 'number', + zeroMeansUndefined: true, + placeholder: '(default)', + hideInMore: true, + info: ( +

+ Maximum number of rows to hold in memory before persisting. Lower values reduce memory usage + but may increase disk I/O. +

+ ), + }, + { + name: 'taskContext.maxFrameSize', + label: 'Task context: max frame size', + type: 'number', + zeroMeansUndefined: true, + placeholder: '(default)', + hideInMore: true, + info: ( +

+ Maximum frame size in bytes for MSQ tasks. Increase if tasks fail due to frame size limits. +

+ ), + }, + { + name: 'taskContext', + label: 'Task context: additional settings', + type: 'json', + hideInMore: true, + info: ( +

+ Full task context map. Common settings are available as dedicated fields above. Use this to + set additional MSQ context parameters. +

+ ), + }, + { + name: 'tuningConfig', + type: 'json', + hideInMore: true, + info: ( +

+ Tuning config for compaction tasks. Note: you cannot set partitionsSpec inside{' '} + tuningConfig for cascading reindexing — partitioning is controlled by rules and + defaults. +

+ ), + }, +]; + +// --- Field arrays for individual rule types --- + +export const PARTITIONING_RULE_FIELDS: Field[] = [ + { + name: 'id', + type: 'string', + required: true, + info:

Unique identifier for this rule.

, + }, + { + name: 'olderThan', + type: 'string', + required: true, + suggestions: PERIOD_SUGGESTIONS, + info: ( +

+ ISO 8601 period defining the age threshold. The rule applies to data older than the current + time minus this period. +

+ ), + }, + { + name: 'description', + type: 'string', + info:

Human-readable description of this rule.

, + }, + { + name: 'segmentGranularity', + type: 'string', + required: true, + suggestions: SEGMENT_GRANULARITY_SUGGESTIONS, + info:

Time granularity for segment buckets.

, + }, + { + name: 'partitionsSpec.type', + label: 'Partitioning type', + type: 'string', + suggestions: ['dynamic', 'range'], + info: ( +

+ Use dynamic for best-effort rollup or range for range-based + partitioning. +

+ ), + }, + { + name: 'partitionsSpec.maxRowsPerSegment', + label: 'Max rows per segment', + type: 'number', + defaultValue: 5000000, + defined: r => oneOfKnown(deepGet(r, 'partitionsSpec.type'), ['dynamic', 'range'], 'dynamic'), + info: <>Determines how many rows are in each segment., + }, + { + name: 'partitionsSpec.partitionDimensions', + label: 'Partition dimensions', + type: 'string-array', + defined: r => deepGet(r, 'partitionsSpec.type') === 'range', + required: true, + info:

The dimensions to partition on.

, + }, + { + name: 'partitionsSpec.targetRowsPerSegment', + label: 'Target rows per segment', + type: 'number', + zeroMeansUndefined: true, + defined: r => deepGet(r, 'partitionsSpec.type') === 'range', + info:

Target number of rows per segment for range partitioning.

, + }, + { + name: 'virtualColumns', + type: 'custom', + defined: r => deepGet(r, 'partitionsSpec.type') === 'range', + customSummary: summarizeVirtualColumns, + info:

Virtual columns for partitioning by nested or derived fields.

, + // customDialog is set by the RuleProviderEditor component + }, +]; + +export const DELETION_RULE_FIELDS: Field[] = [ + { + name: 'id', + type: 'string', + required: true, + info:

Unique identifier for this rule.

, + }, + { + name: 'olderThan', + type: 'string', + required: true, + suggestions: PERIOD_SUGGESTIONS, + info: ( +

+ ISO 8601 period defining the age threshold. The rule applies to data older than the current + time minus this period. +

+ ), + }, + { + name: 'description', + type: 'string', + info:

Human-readable description of this rule.

, + }, + { + name: 'deleteWhere', + type: 'json', + required: true, + info: ( +

+ A Druid filter matching rows to delete. The compacted data retains rows + that do not match this filter. Multiple deletion rules combine as{' '} + NOT(A OR B OR C). +

+ ), + }, + { + name: 'virtualColumns', + type: 'json', + hideInMore: true, + info:

Virtual columns for filtering on nested or derived fields.

, + }, +]; + +export const INDEX_SPEC_RULE_FIELDS: Field[] = [ + { + name: 'id', + type: 'string', + required: true, + info:

Unique identifier for this rule.

, + }, + { + name: 'olderThan', + type: 'string', + required: true, + suggestions: PERIOD_SUGGESTIONS, + info: ( +

+ ISO 8601 period defining the age threshold. The rule applies to data older than the current + time minus this period. +

+ ), + }, + { + name: 'description', + type: 'string', + info:

Human-readable description of this rule.

, + }, + { + name: 'indexSpec', + type: 'json', + required: true, + info: ( +

+ An IndexSpec object defining bitmap type, metric compression, and other encoding settings + for compacted segments. +

+ ), + }, +]; + +export const DATA_SCHEMA_RULE_FIELDS: Field[] = [ + { + name: 'id', + type: 'string', + required: true, + info:

Unique identifier for this rule.

, + }, + { + name: 'olderThan', + type: 'string', + required: true, + suggestions: PERIOD_SUGGESTIONS, + info: ( +

+ ISO 8601 period defining the age threshold. The rule applies to data older than the current + time minus this period. +

+ ), + }, + { + name: 'description', + type: 'string', + info:

Human-readable description of this rule.

, + }, + { + name: 'queryGranularity', + type: 'string', + placeholder: '(unset)', + suggestions: SEGMENT_GRANULARITY_SUGGESTIONS, + info: ( +

+ Query granularity for the compacted segments. Leave unset to preserve existing granularity. +

+ ), + }, + { + name: 'rollup', + type: 'boolean', + info: ( +

+ Whether to enable rollup. Set to true only when metricsSpec is + defined. +

+ ), + }, + { + name: 'metricsSpec', + type: 'json', + info: ( + <> +

Array of aggregator factories for rollup metrics. Example:

+ 
+          {`[
+  {
+    "type": "longSum",
+    "name": "added",
+    "fieldName": "added"
+  },
+  {
+    "type": "longSum",
+    "name": "deleted",
+    "fieldName": "deleted"
+  }
+]`}
+
+ + ), + }, + { + name: 'dimensionsSpec', + type: 'json', + info: ( + <> +

Dimensions config for the compacted segments. Example:

+ 
{`{
+  "dimensions": [
+    "page",
+    { "type": "string", "name": "channel" }
+  ]
+}`}
+ + ), + }, + { + name: 'projections', + type: 'json', + hideInMore: true, + info:

List of aggregate projections.

, + }, +]; + +// --- Default config --- + +export function newReindexCascadeConfig(dataSource: string): ReindexCascadeConfig { + return { + type: 'reindexCascade', + dataSource, + defaultSegmentGranularity: 'DAY', + defaultPartitionsSpec: { type: 'dynamic', maxRowsPerSegment: 5000000 }, + ruleProvider: { type: 'inline' }, + }; +} diff --git a/web-console/src/druid-models/index.ts b/web-console/src/druid-models/index.ts index 7a3bb2218a97..3e51b9ec781a 100644 --- a/web-console/src/druid-models/index.ts +++ b/web-console/src/druid-models/index.ts @@ -20,6 +20,7 @@ export * from './array-ingest-mode/array-ingest-mode'; export * from './async-query/async-query'; export * from './broker-dynamic-config/broker-dynamic-config'; export * from './compaction-config/compaction-config'; +export * from './compaction-config/reindex-cascade-config'; export * from './compaction-dynamic-config/compaction-dynamic-config'; export * from './compaction-status/compaction-status'; export * from './console/console'; diff --git a/web-console/src/views/datasources-view/datasources-view.tsx b/web-console/src/views/datasources-view/datasources-view.tsx index 49e0f0e29966..767bbe0cc48f 100644 --- a/web-console/src/views/datasources-view/datasources-view.tsx +++ b/web-console/src/views/datasources-view/datasources-view.tsx @@ -316,6 +316,7 @@ export interface DatasourcesViewState { datasourcesAndDefaultRulesState: QueryState; showUnused: boolean; + useSupervisors: boolean; retentionDialogOpenOn?: RetentionDialogOpenOn; compactionDialogOpenOn?: CompactionConfigDialogOpenOn; datasourceToMarkAsUnusedAllSegmentsIn?: string; @@ -420,6 +421,7 @@ GROUP BY 1, 2`; datasourcesAndDefaultRulesState: QueryState.INIT, showUnused: false, + useSupervisors: false, useUnuseAction: 'unuse', useUnuseInterval: '', showForceCompact: false, @@ -628,6 +630,18 @@ GROUP BY 1, 2`; c => c.dataSource, ); + // Check if supervisor-based compaction is enabled + try { + const supervisorEnabledResp = await Api.instance.get( + '/druid/indexer/v1/compaction/isSupervisorEnabled', + { signal }, + ); + this.setState({ useSupervisors: Boolean(supervisorEnabledResp.data) }); + } catch { + // If the endpoint is not available, default to false + this.setState({ useSupervisors: false }); + } + return { ...datasourcesAndDefaultRules, datasources: datasourcesAndDefaultRules.datasources.map(ds => ({ @@ -1146,7 +1160,7 @@ GROUP BY 1, 2`; } private renderCompactionConfigDialog() { - const { datasourcesAndDefaultRulesState, compactionDialogOpenOn } = this.state; + const { datasourcesAndDefaultRulesState, compactionDialogOpenOn, useSupervisors } = this.state; if (!compactionDialogOpenOn || !datasourcesAndDefaultRulesState.data) return; return ( @@ -1156,6 +1170,7 @@ GROUP BY 1, 2`; onClose={() => this.setState({ compactionDialogOpenOn: undefined })} onSave={this.saveCompaction} onDelete={this.deleteCompaction} + useSupervisors={useSupervisors} /> ); } From 7e140521f28b5fd25dc4428d4172028a10fb6786 Mon Sep 17 00:00:00 2001 From: Lucas Capistrant Date: Tue, 14 Apr 2026 12:17:18 -0500 Subject: [PATCH 2/3] Update docs --- docs/data-management/cascading-reindexing.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/data-management/cascading-reindexing.md b/docs/data-management/cascading-reindexing.md index 1cc11764ad05..df21a5eb0e48 100644 --- a/docs/data-management/cascading-reindexing.md +++ b/docs/data-management/cascading-reindexing.md @@ -103,9 +103,17 @@ The template requires `defaultSegmentGranularity` and `defaultPartitionsSpec`. T 1. **No partitioning rules defined at all.** If you only define deletion, index spec, or data schema rules, all intervals use the default granularity and partitions spec. 2. **Non-partitioning rules have a more recent threshold than the newest partitioning rule.** For example, if your only partitioning rule is `olderThan: P90D` but you have a deletion rule with `olderThan: P30D`, intervals between 30 and 90 days old will use the defaults. -## Supervisor spec reference +## Submit a cascading reindexing supervisor -To submit a cascading reindexing supervisor, wrap the template spec inside a compaction supervisor spec: +You can submit a cascading reindexing supervisor through the web console or the API. + +### Web console + +In the web console, go to the **Datasources** view and click **Edit compaction config** for a datasource. If your cluster has compaction supervisors enabled, the dialog includes a **Template type** dropdown. Select **Cascading reindexing** to configure the template, including default partitioning, skip offsets, and reindexing rules. + +### API + +To submit a cascading reindexing supervisor via the API, wrap the template spec inside a compaction supervisor spec: ```json { From 4b87a659491bbeb7ca372e8d25e586dc19623ed9 Mon Sep 17 00:00:00 2001 From: Lucas Capistrant Date: Tue, 14 Apr 2026 12:21:35 -0500 Subject: [PATCH 3/3] I guess dropdown isn't in the dictionary --- docs/data-management/cascading-reindexing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/data-management/cascading-reindexing.md b/docs/data-management/cascading-reindexing.md index df21a5eb0e48..95f13add01ea 100644 --- a/docs/data-management/cascading-reindexing.md +++ b/docs/data-management/cascading-reindexing.md @@ -109,7 +109,7 @@ You can submit a cascading reindexing supervisor through the web console or the ### Web console -In the web console, go to the **Datasources** view and click **Edit compaction config** for a datasource. If your cluster has compaction supervisors enabled, the dialog includes a **Template type** dropdown. Select **Cascading reindexing** to configure the template, including default partitioning, skip offsets, and reindexing rules. +In the web console, go to the **Datasources** view and click **Edit compaction config** for a datasource. If your cluster has compaction supervisors enabled, the dialog includes a **Template type** drop-down. Select **Cascading reindexing** to configure the template, including default partitioning, skip offsets, and reindexing rules. ### API