Skip to content

Commit 490400e

Browse files
vdegansvdegans
committed
Add cache settings
1 parent a99b73b commit 490400e

6 files changed

Lines changed: 91 additions & 23 deletions

File tree

‎exec/java-exec/src/main/java/org/apache/drill/exec/cache/CustomCacheManager.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -30,14 +30,14 @@ public class CustomCacheManager {
3030
private static void loadConfig() {
3131
DrillConfig config = DrillConfig.create();
3232

33-
queryMaxEntries = getConfigInt(config, "custom.cache.query.max_entries", 100);
34-
queryTtlMinutes = getConfigInt(config, "custom.cache.query.ttl_minutes", 300);
35-
transformMaxEntries = getConfigInt(config, "custom.cache.transform.max_entries", 100);
36-
transformTtlMinutes = getConfigInt(config, "custom.cache.transform.ttl_minutes", 300);
33+
queryMaxEntries = getConfigInt(config, "planner.query.cache.max_entries_amount", 100);
34+
queryTtlMinutes = getConfigInt(config, "planner.query.cache.plan_cache_ttl_minutes", 300);
35+
transformMaxEntries = getConfigInt(config, "planner.transform.cache.max_entries_amount", 100);
36+
transformTtlMinutes = getConfigInt(config, "planner.transform.plan_cache_ttl_minutes", 300);
3737

3838
queryCache = Caffeine.newBuilder()
3939
.maximumSize(queryMaxEntries)
40-
.expireAfterWrite(queryTtlMinutes, TimeUnit.MINUTES)
40+
.expireAfterWrite(queryTtlMinutes, TimeUnit.MILLISECONDS)
4141
.recordStats()
4242
.build();
4343

@@ -75,4 +75,4 @@ public static void logCacheStats() {
7575
logger.info("Transform Cache Stats: " + transformCache.stats());
7676
logger.info("Transform Cache Size: " + transformCache.estimatedSize());
7777
}
78-
}
78+
}

‎exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,20 @@ public class PlannerSettings implements Context{
132132
public static final String UNIONALL_DISTRIBUTE_KEY = "planner.enable_unionall_distribute";
133133
public static final BooleanValidator UNIONALL_DISTRIBUTE = new BooleanValidator(UNIONALL_DISTRIBUTE_KEY, null);
134134

135+
public static final BooleanValidator PLAN_CACHE = new BooleanValidator("planner.cache.enable",
136+
new OptionDescription("Enables caching of generated query plans in memory, so repeated queries can bypass the planning phase and execute faster.")
137+
);
138+
139+
// Only settable in config, due to pub-sub requirements for recreating the cache on value change
140+
// public static final RangeLongValidator PLAN_CACHE_TTL = new RangeLongValidator("planner.cache.ttl_minutes",
141+
// 0, Long.MAX_VALUE,
142+
// new OptionDescription("Time-to-live for cached query plans in minutes. Plans older than this are evicted. Default is 0 (disabled)")
143+
// );
144+
// public static final RangeLongValidator MAX_CACHE_ENTRIES = new RangeLongValidator("planner.cache.max_entries",
145+
// 1, Long.MAX_VALUE,
146+
// new OptionDescription("Maximum total number of entries for cached query plans. When exceeded, least recently used plans are evicted.")
147+
// );
148+
135149
// ------------------------------------------- Index planning related options BEGIN --------------------------------------------------------------
136150
public static final String USE_SIMPLE_OPTIMIZER_KEY = "planner.use_simple_optimizer";
137151
public static final BooleanValidator USE_SIMPLE_OPTIMIZER = new BooleanValidator(USE_SIMPLE_OPTIMIZER_KEY,
@@ -416,6 +430,10 @@ public boolean isUnionAllDistributeEnabled() {
416430
return options.getOption(UNIONALL_DISTRIBUTE);
417431
}
418432

433+
public boolean isPlanCacheEnabled() {
434+
return options.getOption(PLAN_CACHE);
435+
}
436+
419437
public boolean isParquetRowGroupFilterPushdownPlanningEnabled() {
420438
return options.getOption(PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING);
421439
}

‎exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java‎

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -410,14 +410,20 @@ protected RelNode transform(PlannerType plannerType, PlannerPhase phase, RelNode
410410
final Stopwatch watch = Stopwatch.createStarted();
411411
final RuleSet rules = config.getRules(phase, input);
412412
final RelTraitSet toTraits = targetTraits.simplify();
413+
final OptionManager options = context.getOptions();
414+
final boolean planCacheEnabled = options.getOption(PlannerSettings.PLAN_CACHE);
413415

414-
// Create a cache key based on the input parameters
415-
CacheKey key = new CacheKey(plannerType, phase, input, targetTraits);
416+
CacheKey key = null;
416417

417-
RelNode cachedResult = CustomCacheManager.getTransformedPlan(key);
418-
if (cachedResult != null) {
418+
if (planCacheEnabled) {
419+
// Create a cache key based on the input parameters
420+
key = new CacheKey(plannerType, phase, input, targetTraits);
421+
422+
RelNode cachedResult = CustomCacheManager.getTransformedPlan(key);
423+
if (cachedResult != null) {
419424
CustomCacheManager.logCacheStats();
420425
return cachedResult;
426+
}
421427
}
422428

423429
final RelNode output;
@@ -465,9 +471,14 @@ protected RelNode transform(PlannerType plannerType, PlannerPhase phase, RelNode
465471
}
466472
}
467473

468-
// Store the result in the cache before returning
469-
CustomCacheManager.putTransformedPlan(key, output);
470-
CustomCacheManager.logCacheStats();
474+
if (planCacheEnabled) {
475+
logger.info("planCache enabled, storing transformedplan");
476+
// Store the result in the cache before returning
477+
CustomCacheManager.putTransformedPlan(key, output);
478+
CustomCacheManager.logCacheStats();
479+
} else {
480+
logger.info("planCache disabled, not storing transformedplan");
481+
}
471482

472483
if (log) {
473484
log(plannerType, phase, output, logger, watch);

‎exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,10 @@ public static CaseInsensitiveMap<OptionDefinition> createDefaultOptionDefinition
8282
// here.
8383
@SuppressWarnings("deprecation")
8484
final OptionDefinition[] definitions = new OptionDefinition[]{
85+
new OptionDefinition(PlannerSettings.PLAN_CACHE),
86+
// new OptionDefinition(PlannerSettings.PLAN_CACHE_TTL),
87+
// new OptionDefinition(PlannerSettings.MAX_CACHE_ENTRIES),
88+
8589
new OptionDefinition(PlannerSettings.CONSTANT_FOLDING),
8690
new OptionDefinition(PlannerSettings.EXCHANGE),
8791
new OptionDefinition(PlannerSettings.HASHAGG),

‎exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java‎

Lines changed: 39 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
import org.apache.drill.exec.physical.base.PhysicalOperator;
4040
import org.apache.drill.exec.planner.fragment.Fragment;
4141
import org.apache.drill.exec.planner.fragment.MakeFragmentsVisitor;
42+
import org.apache.drill.exec.planner.physical.PlannerSettings;
4243
import org.apache.drill.exec.planner.sql.DirectPlan;
4344
import org.apache.drill.exec.planner.sql.DrillSqlWorker;
4445
import org.apache.drill.exec.proto.BitControl.PlanFragment;
@@ -56,6 +57,7 @@
5657
import org.apache.drill.exec.rpc.UserClientConnection;
5758
import org.apache.drill.exec.server.DrillbitContext;
5859
import org.apache.drill.exec.server.FailureUtils;
60+
import org.apache.drill.exec.server.options.OptionManager;
5961
import org.apache.drill.exec.server.options.OptionSet;
6062
import org.apache.drill.exec.testing.ControlsInjector;
6163
import org.apache.drill.exec.testing.ControlsInjectorFactory;
@@ -595,23 +597,50 @@ private void logWorkUnit(QueryWorkUnit queryWorkUnit) {
595597
queryId, queryWorkUnit.stringifyFragments()));
596598
}
597599

598-
private void runSQL(final String sql) throws ExecutionSetupException {
599-
final Pointer<String> textPlan = new Pointer<>();
600+
private PhysicalPlan getOrBuildPlan(String sql, Pointer<String> textPlan, boolean planCacheEnabled)
601+
throws ExecutionSetupException {
600602

601-
PhysicalPlan plan = CustomCacheManager.getQueryPlan(sql);
603+
PhysicalPlan plan = null;
604+
605+
if (planCacheEnabled) {
606+
logger.info("Cache enabled, checking entries");
607+
plan = CustomCacheManager.getQueryPlan(sql);
608+
609+
if (plan == null) {
610+
logger.info("Cache miss, generating new plan");
611+
plan = DrillSqlWorker.getPlan(queryContext, sql, textPlan);
612+
613+
if (isCacheableQuery(sql)) {
614+
CustomCacheManager.putQueryPlan(sql, plan);
615+
CustomCacheManager.logCacheStats();
616+
}
617+
} else {
618+
logger.info("Using cached plan");
619+
}
602620

603-
if (plan == null) {
604-
logger.info("Cache miss, generating new plan");
605-
plan = DrillSqlWorker.getPlan(queryContext, sql, textPlan);
606621
} else {
607-
logger.info("Using cached plan");
622+
plan = DrillSqlWorker.getPlan(queryContext, sql, textPlan);
608623
}
609624

610-
if(sql.trim().startsWith("SELECT")) {
611-
CustomCacheManager.putQueryPlan(sql, plan);
612-
CustomCacheManager.logCacheStats();
625+
return plan;
626+
}
627+
628+
private boolean isCacheableQuery(String sql) {
629+
return sql.trim().toUpperCase().startsWith("SELECT");
630+
}
631+
632+
private void runSQL(final String sql) throws ExecutionSetupException {
633+
final Pointer<String> textPlan = new Pointer<>();
634+
final OptionManager options = queryContext.getOptions();
635+
final boolean planCacheEnabled = options.getOption(PlannerSettings.PLAN_CACHE);
636+
if (planCacheEnabled) {
637+
logger.info("PlanCache is enabled");
638+
} else {
639+
logger.info("PlanCache is disabled");
613640
}
614641

642+
PhysicalPlan plan = getOrBuildPlan(sql, textPlan, planCacheEnabled);
643+
615644
runPhysicalPlan(plan, textPlan);
616645
}
617646

‎exec/java-exec/src/main/resources/drill-module.conf‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -697,6 +697,12 @@ drill.exec.options: {
697697
planner.width.max_per_node: 0,
698698
planner.width.max_per_query: 1000,
699699

700+
planner.cache.enable: true,
701+
planner.query.cache.ttl_minutes: 0,
702+
planner.transform.cache.ttl_minutes: 0,
703+
planner.query.cache.max_entries_amount: 100,
704+
planner.transform.cache.max_entries_amount: 100,
705+
700706
prepare.statement.create_timeout_ms: 30000,
701707
security.admin.user_groups: "%drill_process_user_groups%",
702708
security.admin.users: "%drill_process_user%",

0 commit comments

Comments
 (0)