Support effective query in search explain results

graylog2-server/src/main/java/org/graylog/plugins/views/search/ExplainResults.java

@@ -16,6 +16,7 @@
  */
 package org.graylog.plugins.views.search;
 
+import com.fasterxml.jackson.annotation.JsonInclude;
 import org.graylog.plugins.views.search.errors.SearchError;
 import org.graylog2.indexer.indexset.MongoIndexSet;
 import org.graylog2.indexer.ranges.IndexRange;
@@ -32,7 +33,15 @@ public record SearchResult(Map<String, QueryExplainResult> queries) {
     public record QueryExplainResult(Map<String, ExplainResult> searchTypes) {
     }
 
-    public record ExplainResult(String queryString, Set<IndexRangeResult> searchedIndexRanges) {
+    public record ExplainResult(String queryString, Set<IndexRangeResult> searchedIndexRanges,
+                                @JsonInclude(JsonInclude.Include.NON_NULL) String effectiveQuery) {
+        public ExplainResult(String queryString, Set<IndexRangeResult> searchedIndexRanges) {
+            this(queryString, searchedIndexRanges, null);
+        }
+
+        public ExplainResult withEffectiveQuery(final String effectiveQuery) {
+            return new ExplainResult(queryString, searchedIndexRanges, effectiveQuery);
+        }
     }
 
     public record IndexRangeResult(String indexName, long begin, long end, boolean isWarmTiered,

graylog2-server/src/main/java/org/graylog/plugins/views/search/engine/QueryEngine.java

@@ -27,6 +27,9 @@
 import org.graylog.plugins.views.search.QueryResult;
 import org.graylog.plugins.views.search.Search;
 import org.graylog.plugins.views.search.SearchJob;
+import org.graylog.plugins.views.search.SearchType;
+import org.graylog.plugins.views.search.searchfilters.EffectiveQueryComposer;
+import org.graylog.plugins.views.search.searchfilters.model.UsedSearchFilter;
 import org.graylog.plugins.views.search.elasticsearch.ElasticsearchQueryString;
 import org.graylog.plugins.views.search.errors.QueryError;
 import org.graylog.plugins.views.search.errors.SearchError;
@@ -38,6 +41,7 @@
 import org.slf4j.LoggerFactory;
 
 import java.util.Collection;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
@@ -62,17 +66,20 @@ public class QueryEngine {
     private final Executor dataLakeJobsQueryPool;
     private final ElasticsearchBackendProvider elasticsearchBackendProvider;
     private final Map<String, QueryBackend<? extends GeneratedQueryContext>> unversionedBackends;
+    private final EffectiveQueryComposer effectiveQueryComposer;
 
     @Inject
     public QueryEngine(Configuration configuration,
                        ElasticsearchBackendProvider elasticsearchBackendProvider,
                        Map<String, QueryBackend<? extends GeneratedQueryContext>> unversionedBackends,
                        Set<QueryMetadataDecorator> queryMetadataDecorators,
-                       QueryParser queryParser) {
+                       QueryParser queryParser,
+                       EffectiveQueryComposer effectiveQueryComposer) {
         this.elasticsearchBackendProvider = elasticsearchBackendProvider;
         this.unversionedBackends = unversionedBackends;
         this.queryMetadataDecorators = queryMetadataDecorators;
         this.queryParser = queryParser;
+        this.effectiveQueryComposer = effectiveQueryComposer;
 
         this.indexerJobsQueryPool = createThreadPool(
                 configuration.searchQueryEngineIndexerJobsPoolSize(),
@@ -111,12 +118,31 @@ public ExplainResults explain(SearchJob searchJob, Set<SearchError> validationEr
                     var backend = getBackendForQuery(q);
                     final GeneratedQueryContext generatedQueryContext = backend.generate(q, Set.of(), timezone);
 
-                    return backend.explain(searchJob, q, generatedQueryContext);
+                    return withEffectiveQueries(q, backend.explain(searchJob, q, generatedQueryContext));
                 }));
 
         return new ExplainResults(searchJob.getSearchId(), new ExplainResults.SearchResult(queries), validationErrors);
     }
 
+    private ExplainResults.QueryExplainResult withEffectiveQueries(final Query query, final ExplainResults.QueryExplainResult queryExplainResult) {
+        final Map<String, SearchType> searchTypesById = query.searchTypes().stream()
+                .collect(Collectors.toMap(SearchType::id, s -> s, (a, b) -> a));
+        final String baseQuery = query.query().queryString();
+
+        final Map<String, ExplainResults.ExplainResult> enriched = queryExplainResult.searchTypes().entrySet().stream()
+                .collect(Collectors.toMap(Map.Entry::getKey, entry -> {
+                    final List<UsedSearchFilter> filters = new java.util.ArrayList<>(
+                            query.filters() != null ? query.filters() : List.of());
+                    final SearchType searchType = searchTypesById.get(entry.getKey());
+                    if (searchType != null && searchType.filters() != null) {
+                        filters.addAll(searchType.filters());
+                    }
+                    return entry.getValue().withEffectiveQuery(effectiveQueryComposer.compose(baseQuery, filters));
+                }));
+
+        return new ExplainResults.QueryExplainResult(enriched);
+    }
+
     @WithSpan
     public SearchJob execute(SearchJob searchJob, Set<SearchError> validationErrors, DateTimeZone timezone) {
         final Set<Query> validQueries = searchJob.getSearch().queries()

graylog2-server/src/main/java/org/graylog/plugins/views/search/rest/EffectiveQueryRequest.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog.plugins.views.search.rest;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.graylog.plugins.views.search.searchfilters.model.UsedSearchFilter;
+
+import java.util.List;
+
+public record EffectiveQueryRequest(@JsonProperty("query_string") String queryString,
+                                    @JsonProperty("filters") List<UsedSearchFilter> filters) {
+    @JsonCreator
+    public EffectiveQueryRequest {
+        queryString = queryString == null ? "" : queryString;
+        filters = filters == null ? List.of() : filters;
+    }
+}

graylog2-server/src/main/java/org/graylog/plugins/views/search/rest/SearchResource.java

@@ -16,6 +16,7 @@
  */
 package org.graylog.plugins.views.search.rest;
 
+import com.google.common.collect.ImmutableSet;
 import com.google.common.eventbus.EventBus;
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.Parameter;
@@ -41,20 +42,23 @@
 import jakarta.ws.rs.core.MediaType;
 import jakarta.ws.rs.core.Response;
 import org.apache.shiro.authz.annotation.RequiresAuthentication;
-import org.graylog.events.processor.systemnotification.TemplateRenderRequest;
 import org.graylog.plugins.views.audit.ViewsAuditEventTypes;
 import org.graylog.plugins.views.search.ExplainResults;
+import org.graylog.plugins.views.search.Query;
 import org.graylog.plugins.views.search.Search;
 import org.graylog.plugins.views.search.SearchDomain;
 import org.graylog.plugins.views.search.SearchJob;
 import org.graylog.plugins.views.search.db.SearchJobService;
+import org.graylog.plugins.views.search.elasticsearch.ElasticsearchQueryString;
 import org.graylog.plugins.views.search.engine.SearchExecutor;
 import org.graylog.plugins.views.search.events.SearchJobExecutionEvent;
 import org.graylog.plugins.views.search.permissions.SearchUser;
+import org.graylog.plugins.views.search.searchtypes.MessageList;
 import org.graylog2.audit.jersey.AuditEvent;
 import org.graylog2.audit.jersey.NoAuditEvent;
 import org.graylog2.indexer.searches.SearchesClusterConfig;
 import org.graylog2.plugin.cluster.ClusterConfigService;
+import org.graylog2.plugin.indexer.searches.timeranges.RelativeRange;
 import org.graylog2.plugin.rest.PluginRestResource;
 import org.graylog2.shared.rest.PublicCloudAPI;
 import org.graylog2.shared.rest.resources.RestResource;
@@ -65,6 +69,7 @@
 
 import java.net.URI;
 import java.util.List;
+import java.util.Set;
 import java.util.stream.Collectors;
 
 @PublicCloudAPI
@@ -203,6 +208,32 @@ public ExplainResults explainQuery(@Parameter(name = "id") @PathParam("id") Stri
         return searchExecutor.explain(id, searchUser, executionState);
     }
 
+    @POST
+    @Operation(summary = "Explain an ad-hoc query plus search filters",
+               description = "Synthesizes a search from the given query and search filters and explains it, so callers "
+                       + "without a saved search (e.g. the event definition editor) can read the effective query from "
+                       + "the returned explain results.")
+    @ApiResponses({
+            @ApiResponse(responseCode = "200", description = "Explain results (read effective_query per search type)",
+                    content = @Content(schema = @Schema(implementation = ExplainResults.class)))
+    })
+    @Path("effective_query")
+    @NoAuditEvent("Does not return any actual data")
+    public ExplainResults explainEffectiveQuery(@RequestBody(content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = EffectiveQueryRequest.class)))
+                                                EffectiveQueryRequest request,
+                                                @Context SearchUser searchUser) {
+        final Query query = Query.builder()
+                .id("effective-query")
+                .searchTypes(Set.of(MessageList.builder().id("effective-query-message-list").build()))
+                .query(ElasticsearchQueryString.of(request.queryString()))
+                .filters(request.filters())
+                .timerange(RelativeRange.allTime())
+                .build();
+        final Search search = Search.builder().queries(ImmutableSet.of(query)).build();
+
+        return searchExecutor.explain(search, searchUser, ExecutionState.empty());
+    }
+
     @POST
     @Operation(summary = "Execute a new synchronous search", description = "Executes a new search and waits for its result")
     @ApiResponses({

graylog2-server/src/main/java/org/graylog/plugins/views/search/searchfilters/DefaultEffectiveQueryComposer.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog.plugins.views.search.searchfilters;
+
+import org.graylog.plugins.views.search.searchfilters.model.UsedSearchFilter;
+
+import java.util.List;
+
+public class DefaultEffectiveQueryComposer implements EffectiveQueryComposer {
+
+    @Override
+    public String compose(final String baseQuery, final List<UsedSearchFilter> filters) {
+        return baseQuery == null ? "" : baseQuery;
+    }
+}

graylog2-server/src/main/java/org/graylog/plugins/views/search/searchfilters/EffectiveQueryComposer.java

@@ -0,0 +1,26 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog.plugins.views.search.searchfilters;
+
+import org.graylog.plugins.views.search.searchfilters.model.UsedSearchFilter;
+
+import java.util.List;
+
+public interface EffectiveQueryComposer {
+
+    String compose(String baseQuery, List<UsedSearchFilter> filters);
+}

graylog2-server/src/main/java/org/graylog/plugins/views/search/searchfilters/module/SearchFiltersModule.java

@@ -17,6 +17,8 @@
 package org.graylog.plugins.views.search.searchfilters.module;
 
 import com.google.inject.multibindings.OptionalBinder;
+import org.graylog.plugins.views.search.searchfilters.DefaultEffectiveQueryComposer;
+import org.graylog.plugins.views.search.searchfilters.EffectiveQueryComposer;
 import org.graylog.plugins.views.search.searchfilters.db.IgnoreSearchFilters;
 import org.graylog.plugins.views.search.searchfilters.db.SearchFiltersReFetcher;
 import org.graylog.plugins.views.search.searchfilters.db.UsedSearchFiltersToQueryStringsMapper;
@@ -33,5 +35,7 @@ protected void configure() {
         OptionalBinder.newOptionalBinder(binder(), SearchFiltersReFetcher.class)
                 .setDefault().to(IgnoreSearchFilters.class);
 
+        OptionalBinder.newOptionalBinder(binder(), EffectiveQueryComposer.class)
+                .setDefault().to(DefaultEffectiveQueryComposer.class);
     }
 }

graylog2-web-interface/src/components/common/SearchFiltersFormControls.tsx

@@ -15,7 +15,7 @@
  * <http://www.mongodb.com/licensing/server-side-public-license>.
  */
 import React, { useEffect, useMemo } from 'react';
-import { Formik } from 'formik';
+import { Formik, useFormikContext } from 'formik';
 import { OrderedMap } from 'immutable';
 import { v4 as uuidv4 } from 'uuid';
 
@@ -27,9 +27,22 @@ type Props = {
   filters: SearchFilter[];
   onChange: (filters: OrderedMap<string, SearchFilter>) => void;
   hideFiltersPreview?: (val: boolean) => void;
+  queryString?: string;
 };
 
-function SearchFiltersFormControls({ filters, onChange, hideFiltersPreview = () => {} }: Props) {
+// Keeps the isolated Formik's `queryString` field in sync with the manually-entered query
+// living outside this form, without resetting `searchFilters` on every keystroke.
+const SyncQueryString = ({ queryString }: { queryString: string }) => {
+  const { setFieldValue } = useFormikContext();
+
+  useEffect(() => {
+    setFieldValue('queryString', queryString);
+  }, [queryString, setFieldValue]);
+
+  return null;
+};
+
+function SearchFiltersFormControls({ filters, onChange, hideFiltersPreview = () => {}, queryString = '' }: Props) {
   const searchFiltersPlugin = usePluginEntities('eventDefinitions.components.searchForm') ?? [];
   const pluggableControls = searchFiltersPlugin.map((controlFn) => controlFn()).filter((control) => !!control);
 
@@ -41,8 +54,8 @@ function SearchFiltersFormControls({ filters, onChange, hideFiltersPreview = ()
       filters.map((filter) => [filter.id || uuidv4(), { frontendId: filter.id || uuidv4(), ...filter }]),
     );
 
-    return { searchFilters };
-  }, [filters]);
+    return { searchFilters, queryString };
+  }, [filters, queryString]);
 
   if (!pluggableControls.length)
     return <SearchFilterBanner onHide={() => hideFiltersPreview(true)} pluggableControls={pluggableControls} />;
@@ -58,7 +71,10 @@ function SearchFiltersFormControls({ filters, onChange, hideFiltersPreview = ()
 
   return (
     <Formik onSubmit={handleSearchFiltersChange} initialValues={initialFilters}>
-      <SearchFiltersComponent />
+      <>
+        <SyncQueryString queryString={queryString} />
+        <SearchFiltersComponent />
+      </>
     </Formik>
   );
 }

graylog2-web-interface/src/components/event-definitions/event-definition-types/FilterForm.tsx

@@ -39,6 +39,7 @@ import type { RelativeTimeRangeWithEnd } from 'views/logic/queries/Query';
 import Search from 'views/logic/search/Search';
 import { extractDurationAndUnit } from 'components/common/TimeUnitInput';
 import { Alert, ButtonToolbar, ControlLabel, FormGroup, HelpBlock, Input } from 'components/bootstrap';
+import usePluginEntities from 'hooks/usePluginEntities';
 import RelativeTime from 'components/common/RelativeTime';
 import Parameter from 'views/logic/parameters/Parameter';
 import type { ParameterJson } from 'views/logic/parameters/Parameter';
@@ -183,6 +184,7 @@ const QueryParameters = ({ eventDefinition, onChange, validation, userCanViewLoo
 const FilterForm = ({ currentUser, eventDefinition, onChange, streams, validation }: Props) => {
   const { execute_every_ms: executeEveryMs, search_within_ms: searchWithinMs } = eventDefinition.config;
   const [currentConfig, setCurrentConfig] = useState(eventDefinition.config);
+  const searchQueryPreviews = usePluginEntities('eventDefinitions.components.searchQueryPreview') ?? [];
   const searchWithin = extractDurationAndUnit(searchWithinMs, TIME_UNITS);
   const executeEvery = extractDurationAndUnit(executeEveryMs, TIME_UNITS);
   const { userTimezone } = useUserDateTime();
@@ -557,6 +559,14 @@ const FilterForm = ({ currentUser, eventDefinition, onChange, streams, validatio
         </FormGroup>
       )}
 
+      {/* Extension point (filled by the enterprise search-filters plugin) for a read-only preview of the
+          effective query beneath the Search Query. Scoped (Illuminate) definitions hide the base query, so
+          the preview is not rendered for them. */}
+      {onlyFilters ||
+        searchQueryPreviews.map(({ component: SearchQueryPreview, key }) => (
+          <SearchQueryPreview key={key} queryString={currentConfig.query ?? ''} filters={currentConfig.filters ?? []} />
+        ))}
+
       {onlyFilters || (
         <QueryParameters
           eventDefinition={eventDefinition}
@@ -576,6 +586,7 @@ const FilterForm = ({ currentUser, eventDefinition, onChange, streams, validatio
               filters={eventDefinition.config.filters}
               onChange={handleSearchFiltersChange}
               hideFiltersPreview={hideFiltersPreview}
+              queryString={currentConfig.query}
             />
           </div>
         </FormGroup>

graylog2-web-interface/src/components/event-definitions/types.ts

@@ -19,7 +19,7 @@ import type React from 'react';
 
 import type { SearchBarControl } from 'views/types';
 import type User from 'logic/users/User';
-import type { EventDefinition } from 'components/event-definitions/event-definitions-types';
+import type { EventDefinition, SearchFilter } from 'components/event-definitions/event-definitions-types';
 import type { Attribute } from 'stores/PaginationTypes';
 
 export type AlertType = 'alert' | 'event' | 'event_definition';
@@ -105,5 +105,9 @@ declare module 'graylog-web-plugin/plugin' {
     'eventDefinitions.components.sigmaGitImport'?: Array<{ component: React.FC; key: string }>;
     'eventDefinitions.components.sigmaFileUpload'?: Array<{ component: React.FC; key: string }>;
     'eventDefinitions.components.sigmaOptions'?: Array<{ component: React.FC; key: string }>;
+    'eventDefinitions.components.searchQueryPreview'?: Array<{
+      component: React.ComponentType<{ queryString: string; filters: SearchFilter[] }>;
+      key: string;
+    }>;
   }
 }