Add effective query to search explain results (#14552)

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: danotorrey

changelog/unreleased/issue-14552.toml

@@ -16,6 +16,7 @@
  */
 package org.graylog.plugins.views.search;
 
+import com.fasterxml.jackson.annotation.JsonInclude;
 import org.graylog.plugins.views.search.errors.SearchError;
 import org.graylog2.indexer.indexset.MongoIndexSet;
 import org.graylog2.indexer.ranges.IndexRange;
@@ -32,7 +33,15 @@ public record SearchResult(Map<String, QueryExplainResult> queries) {
     public record QueryExplainResult(Map<String, ExplainResult> searchTypes) {
     }
 
-    public record ExplainResult(String queryString, Set<IndexRangeResult> searchedIndexRanges) {
+    public record ExplainResult(String queryString, Set<IndexRangeResult> searchedIndexRanges,
+                                @JsonInclude(JsonInclude.Include.NON_NULL) String effectiveQuery) {
+        public ExplainResult(String queryString, Set<IndexRangeResult> searchedIndexRanges) {
+            this(queryString, searchedIndexRanges, null);
+        }
+
+        public ExplainResult withEffectiveQuery(final String effectiveQuery) {
+            return new ExplainResult(queryString, searchedIndexRanges, effectiveQuery);
+        }
     }
 
     public record IndexRangeResult(String indexName, long begin, long end, boolean isWarmTiered,

graylog2-server/src/main/java/org/graylog/plugins/views/search/ExplainResults.java

@@ -27,6 +27,9 @@
 import org.graylog.plugins.views.search.QueryResult;
 import org.graylog.plugins.views.search.Search;
 import org.graylog.plugins.views.search.SearchJob;
+import org.graylog.plugins.views.search.SearchType;
+import org.graylog.plugins.views.search.searchfilters.EffectiveQueryComposer;
+import org.graylog.plugins.views.search.searchfilters.model.UsedSearchFilter;
 import org.graylog.plugins.views.search.elasticsearch.ElasticsearchQueryString;
 import org.graylog.plugins.views.search.errors.QueryError;
 import org.graylog.plugins.views.search.errors.SearchError;
@@ -38,6 +41,7 @@
 import org.slf4j.LoggerFactory;
 
 import java.util.Collection;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
@@ -62,17 +66,20 @@ public class QueryEngine {
     private final Executor dataLakeJobsQueryPool;
     private final ElasticsearchBackendProvider elasticsearchBackendProvider;
     private final Map<String, QueryBackend<? extends GeneratedQueryContext>> unversionedBackends;
+    private final EffectiveQueryComposer effectiveQueryComposer;
 
     @Inject
     public QueryEngine(Configuration configuration,
                        ElasticsearchBackendProvider elasticsearchBackendProvider,
                        Map<String, QueryBackend<? extends GeneratedQueryContext>> unversionedBackends,
                        Set<QueryMetadataDecorator> queryMetadataDecorators,
-                       QueryParser queryParser) {
+                       QueryParser queryParser,
+                       EffectiveQueryComposer effectiveQueryComposer) {
         this.elasticsearchBackendProvider = elasticsearchBackendProvider;
         this.unversionedBackends = unversionedBackends;
         this.queryMetadataDecorators = queryMetadataDecorators;
         this.queryParser = queryParser;
+        this.effectiveQueryComposer = effectiveQueryComposer;
 
         this.indexerJobsQueryPool = createThreadPool(
                 configuration.searchQueryEngineIndexerJobsPoolSize(),
@@ -111,12 +118,31 @@ public ExplainResults explain(SearchJob searchJob, Set<SearchError> validationEr
                     var backend = getBackendForQuery(q);
                     final GeneratedQueryContext generatedQueryContext = backend.generate(q, Set.of(), timezone);
 
-                    return backend.explain(searchJob, q, generatedQueryContext);
+                    return withEffectiveQueries(q, backend.explain(searchJob, q, generatedQueryContext));
                 }));
 
         return new ExplainResults(searchJob.getSearchId(), new ExplainResults.SearchResult(queries), validationErrors);
     }
 
+    private ExplainResults.QueryExplainResult withEffectiveQueries(final Query query, final ExplainResults.QueryExplainResult queryExplainResult) {
+        final Map<String, SearchType> searchTypesById = query.searchTypes().stream()
+                .collect(Collectors.toMap(SearchType::id, s -> s, (a, b) -> a));
+        final String baseQuery = query.query().queryString();
+
+        final Map<String, ExplainResults.ExplainResult> enriched = queryExplainResult.searchTypes().entrySet().stream()
+                .collect(Collectors.toMap(Map.Entry::getKey, entry -> {
+                    final List<UsedSearchFilter> filters = new java.util.ArrayList<>(
+                            query.filters() != null ? query.filters() : List.of());
+                    final SearchType searchType = searchTypesById.get(entry.getKey());
+                    if (searchType != null && searchType.filters() != null) {
+                        filters.addAll(searchType.filters());
+                    }
+                    return entry.getValue().withEffectiveQuery(effectiveQueryComposer.compose(baseQuery, filters));
+                }));
+
+        return new ExplainResults.QueryExplainResult(enriched);
+    }
+
     @WithSpan
     public SearchJob execute(SearchJob searchJob, Set<SearchError> validationErrors, DateTimeZone timezone) {
         final Set<Query> validQueries = searchJob.getSearch().queries()

graylog2-server/src/main/java/org/graylog/plugins/views/search/engine/QueryEngine.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog.plugins.views.search.rest;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.graylog.plugins.views.search.searchfilters.model.UsedSearchFilter;
+
+import java.util.List;
+
+public record EffectiveQueryRequest(@JsonProperty("query_string") String queryString,
+                                    @JsonProperty("filters") List<UsedSearchFilter> filters) {
+    @JsonCreator
+    public EffectiveQueryRequest {
+        queryString = queryString == null ? "" : queryString;
+        filters = filters == null ? List.of() : filters;
+    }
+}

graylog2-server/src/main/java/org/graylog/plugins/views/search/rest/EffectiveQueryRequest.java

@@ -16,6 +16,7 @@
  */
 package org.graylog.plugins.views.search.rest;
 
+import com.google.common.collect.ImmutableSet;
 import com.google.common.eventbus.EventBus;
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.Parameter;
@@ -41,20 +42,23 @@
 import jakarta.ws.rs.core.MediaType;
 import jakarta.ws.rs.core.Response;
 import org.apache.shiro.authz.annotation.RequiresAuthentication;
-import org.graylog.events.processor.systemnotification.TemplateRenderRequest;
 import org.graylog.plugins.views.audit.ViewsAuditEventTypes;
 import org.graylog.plugins.views.search.ExplainResults;
+import org.graylog.plugins.views.search.Query;
 import org.graylog.plugins.views.search.Search;
 import org.graylog.plugins.views.search.SearchDomain;
 import org.graylog.plugins.views.search.SearchJob;
 import org.graylog.plugins.views.search.db.SearchJobService;
+import org.graylog.plugins.views.search.elasticsearch.ElasticsearchQueryString;
 import org.graylog.plugins.views.search.engine.SearchExecutor;
 import org.graylog.plugins.views.search.events.SearchJobExecutionEvent;
 import org.graylog.plugins.views.search.permissions.SearchUser;
+import org.graylog.plugins.views.search.searchtypes.MessageList;
 import org.graylog2.audit.jersey.AuditEvent;
 import org.graylog2.audit.jersey.NoAuditEvent;
 import org.graylog2.indexer.searches.SearchesClusterConfig;
 import org.graylog2.plugin.cluster.ClusterConfigService;
+import org.graylog2.plugin.indexer.searches.timeranges.RelativeRange;
 import org.graylog2.plugin.rest.PluginRestResource;
 import org.graylog2.shared.rest.PublicCloudAPI;
 import org.graylog2.shared.rest.resources.RestResource;
@@ -65,6 +69,7 @@
 
 import java.net.URI;
 import java.util.List;
+import java.util.Set;
 import java.util.stream.Collectors;
 
 @PublicCloudAPI
@@ -203,6 +208,32 @@ public ExplainResults explainQuery(@Parameter(name = "id") @PathParam("id") Stri
         return searchExecutor.explain(id, searchUser, executionState);
     }
 
+    @POST
+    @Operation(summary = "Explain an ad-hoc query plus search filters",
+               description = "Synthesizes a search from the given query and search filters and explains it, so callers "
+                       + "without a saved search (e.g. the event definition editor) can read the effective query from "
+                       + "the returned explain results.")
+    @ApiResponses({
+            @ApiResponse(responseCode = "200", description = "Explain results (read effective_query per search type)",
+                    content = @Content(schema = @Schema(implementation = ExplainResults.class)))
+    })
+    @Path("effective_query")
+    @NoAuditEvent("Does not return any actual data")
+    public ExplainResults explainEffectiveQuery(@RequestBody(content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = EffectiveQueryRequest.class)))
+                                                EffectiveQueryRequest request,
+                                                @Context SearchUser searchUser) {
+        final Query query = Query.builder()
+                .id("effective-query")
+                .searchTypes(Set.of(MessageList.builder().id("effective-query-message-list").build()))
+                .query(ElasticsearchQueryString.of(request.queryString()))
+                .filters(request.filters())
+                .timerange(RelativeRange.allTime())
+                .build();
+        final Search search = Search.builder().queries(ImmutableSet.of(query)).build();
+
+        return searchExecutor.explain(search, searchUser, ExecutionState.empty());
+    }
+
     @POST
     @Operation(summary = "Execute a new synchronous search", description = "Executes a new search and waits for its result")
     @ApiResponses({

graylog2-server/src/main/java/org/graylog/plugins/views/search/rest/SearchResource.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog.plugins.views.search.searchfilters;
+
+import org.graylog.plugins.views.search.searchfilters.model.UsedSearchFilter;
+
+import java.util.List;
+
+public class DefaultEffectiveQueryComposer implements EffectiveQueryComposer {
+
+    @Override
+    public String compose(final String baseQuery, final List<UsedSearchFilter> filters) {
+        return baseQuery == null ? "" : baseQuery;
+    }
+}

graylog2-server/src/main/java/org/graylog/plugins/views/search/searchfilters/DefaultEffectiveQueryComposer.java

@@ -0,0 +1,26 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog.plugins.views.search.searchfilters;
+
+import org.graylog.plugins.views.search.searchfilters.model.UsedSearchFilter;
+
+import java.util.List;
+
+public interface EffectiveQueryComposer {
+
+    String compose(String baseQuery, List<UsedSearchFilter> filters);
+}

graylog2-server/src/main/java/org/graylog/plugins/views/search/searchfilters/EffectiveQueryComposer.java

@@ -17,6 +17,8 @@
 package org.graylog.plugins.views.search.searchfilters.module;
 
 import com.google.inject.multibindings.OptionalBinder;
+import org.graylog.plugins.views.search.searchfilters.DefaultEffectiveQueryComposer;
+import org.graylog.plugins.views.search.searchfilters.EffectiveQueryComposer;
 import org.graylog.plugins.views.search.searchfilters.db.IgnoreSearchFilters;
 import org.graylog.plugins.views.search.searchfilters.db.SearchFiltersReFetcher;
 import org.graylog.plugins.views.search.searchfilters.db.UsedSearchFiltersToQueryStringsMapper;
@@ -33,5 +35,7 @@ protected void configure() {
         OptionalBinder.newOptionalBinder(binder(), SearchFiltersReFetcher.class)
                 .setDefault().to(IgnoreSearchFilters.class);
 
+        OptionalBinder.newOptionalBinder(binder(), EffectiveQueryComposer.class)
+                .setDefault().to(DefaultEffectiveQueryComposer.class);
     }
 }