fix: Streams Data Lake column not shown on cloud

graylog2-web-interface/src/@types/graylog-web-plugin/index.d.ts

@@ -19,11 +19,12 @@ import type Immutable from 'immutable';
 
 import type FetchError from 'logic/errors/FetchError';
 import type { DataTieringConfig } from 'components/indices/data-tiering';
+import type { Attribute } from 'stores/PaginationTypes';
 import type { QualifiedUrl } from 'routing/Routes';
 import type User from 'logic/users/User';
 import type { EventDefinition } from 'components/event-definitions/event-definitions-types';
 import type { Stream } from 'logic/streams/types';
-import type { ColumnRenderer } from 'components/common/EntityDataTable/types';
+import type { ColumnRenderersByAttribute } from 'components/common/EntityDataTable/types';
 import type { StepType } from 'components/common/Wizard';
 import type { InputSetupWizardStep } from 'components/inputs/InputSetupWizard';
 import type { TelemetryEventType } from 'logic/telemetry/TelemetryContext';
@@ -198,6 +199,12 @@ type IndexRetentionConfig = {
   summaryComponent: React.ComponentType<IndexRetentionSummaryComponentProps>;
 };
 
+type StreamsOverviewTableElement = {
+  attributeName: string;
+  attributes: Array<Attribute>;
+  columnRenderers: ColumnRenderersByAttribute<Stream>;
+};
+
 declare module 'graylog-web-plugin/plugin' {
   type Id = string;
   type Wildcard = '*';
@@ -327,11 +334,6 @@ declare module 'graylog-web-plugin/plugin' {
       timestamp_to: string;
       restore_history: Array<{ id: string }>;
     }>;
-    getStreamDataLakeTableElements: (permission: Immutable.List<string>) => {
-      attributeName: string;
-      attributes: Array<{ id: string; title: string }>;
-      columnRenderer: { data_lake: ColumnRenderer<Stream> };
-    };
     DataLakeStreamDeleteWarning: React.ComponentType;
   }
 
@@ -391,6 +393,9 @@ declare module 'graylog-web-plugin/plugin' {
      */
     pageNavigation?: Array<PageNavigation>;
     dataLake?: Array<PluginDataLake>;
+    // Use this for stream-overview-only columns. Use `components.shared.entityTableElements`
+    // when the extension should participate in the generic entity-table mechanism.
+    'components.streams.overview.tableElements'?: Array<StreamsOverviewTableElement>;
     dataTiering?: Array<DataTiering>;
     defaultNavigation?: Array<PluginNavigation>;
     navigationItems?: Array<PluginNavigationItems>;

graylog2-web-interface/src/components/streams/StreamsOverview/ColumnRenderers.tsx

@@ -15,8 +15,6 @@
  * <http://www.mongodb.com/licensing/server-side-public-license>.
  */
 import * as React from 'react';
-import { PluginStore } from 'graylog-web-plugin/plugin';
-import type Immutable from 'immutable';
 
 import type { ColumnRenderersByAttribute } from 'components/common/EntityDataTable/types';
 import type { Output } from 'hooks/useOutputs';
@@ -34,7 +32,6 @@ import OutputsCell from './cells/OutputsCell';
 import ArchivingsCell from './cells/ArchivingsCell';
 import DestinationFilterRulesCell from './cells/DestinationFilterRulesCell';
 
-const getStreamDataLakeTableElements = PluginStore.exports('dataLake')?.[0]?.getStreamDataLakeTableElements;
 const pipelineRenderer = {
   pipelines: {
     renderCell: (_pipeline: any[], stream) => <PipelinesCell stream={stream} />,
@@ -44,8 +41,7 @@ const pipelineRenderer = {
 const customColumnRenderers = (
   indexSets: Array<IndexSet>,
   isPipelineColumnPermitted: boolean,
-  permissions: Immutable.List<string>,
-  pluggableColumnRenderers?: ColumnRenderersByAttribute<Stream>,
+  extensionColumnRenderers?: ColumnRenderersByAttribute<Stream>,
 ): ColumnRenderers<Stream> => ({
   attributes: {
     title: {
@@ -81,8 +77,7 @@ const customColumnRenderers = (
       renderCell: (_archiving: boolean, stream) => <ArchivingsCell stream={stream} indexSets={indexSets} />,
       staticWidth: 'matchHeader' as const,
     },
-    ...(getStreamDataLakeTableElements?.(permissions)?.columnRenderer || {}),
-    ...(pluggableColumnRenderers || {}),
+    ...(extensionColumnRenderers || {}),
   },
 });
 

graylog2-web-interface/src/components/streams/StreamsOverview/Constants.ts

@@ -14,25 +14,15 @@
  * along with this program. If not, see
  * <http://www.mongodb.com/licensing/server-side-public-license>.
  */
-
-import { PluginStore } from 'graylog-web-plugin/plugin';
-import type Immutable from 'immutable';
-import type { Permission } from 'graylog-web-plugin/plugin';
-
 import type { Attribute, Sort } from 'stores/PaginationTypes';
 
-const getStreamDataLakeTableElements = PluginStore.exports('dataLake')?.[0]?.getStreamDataLakeTableElements;
-
 const getStreamTableElements = (
-  permissions: Immutable.List<Permission>,
   isPipelineColumnPermitted: boolean,
-  pluggableAttributes?: {
+  extensionAttributes?: {
     attributeNames?: Array<string>;
     attributes?: Array<Attribute>;
   },
 ) => {
-  const streamDataLakeTableElements = getStreamDataLakeTableElements?.(permissions);
-
   const defaultLayout = {
     entityTableId: 'streams',
     defaultPageSize: 20,
@@ -44,9 +34,8 @@ const getStreamTableElements = (
       ...(isPipelineColumnPermitted ? ['pipelines'] : []),
       'outputs',
       'archiving',
-      ...(streamDataLakeTableElements?.attributeName ? [streamDataLakeTableElements.attributeName] : []),
+      ...(extensionAttributes?.attributeNames || []),
       'destination_filters',
-      ...(pluggableAttributes?.attributeNames || []),
       'disabled',
       'throughput',
     ],
@@ -57,9 +46,8 @@ const getStreamTableElements = (
       ...(isPipelineColumnPermitted ? ['pipelines'] : []),
       'outputs',
       'archiving',
-      ...(streamDataLakeTableElements?.attributeName ? [streamDataLakeTableElements.attributeName] : []),
+      ...(extensionAttributes?.attributeNames || []),
       'destination_filters',
-      ...(pluggableAttributes?.attributeNames || []),
       'disabled',
       'throughput',
       'created_at',
@@ -72,10 +60,9 @@ const getStreamTableElements = (
     { id: 'rules', title: 'Stream Rules' },
     ...(isPipelineColumnPermitted ? [{ id: 'pipelines', title: 'Pipelines' }] : []),
     { id: 'outputs', title: 'Outputs' },
-    { id: 'destination_filters', title: 'Filter Rules' },
     { id: 'archiving', title: 'Archiving' },
-    ...(streamDataLakeTableElements?.attributes || []),
-    ...(pluggableAttributes?.attributes || []),
+    ...(extensionAttributes?.attributes || []),
+    { id: 'destination_filters', title: 'Filter Rules' },
   ];
 
   return {

graylog2-web-interface/src/components/streams/StreamsOverview/StreamsOverview.test.tsx

@@ -18,6 +18,7 @@ import React from 'react';
 import { render, screen, within } from 'wrappedTestingLibrary';
 import userEvent from '@testing-library/user-event';
 import * as Immutable from 'immutable';
+import { PluginManifest, PluginStore } from 'graylog-web-plugin/plugin';
 
 import { indexSets } from 'fixtures/indexSets';
 import { asMock, MockStore } from 'helpers/mocking';
@@ -245,4 +246,41 @@ describe('StreamsOverview', () => {
 
     expect(screen.queryByText('Only prod logs')).not.toBeInTheDocument();
   });
+
+  it('should render stream overview table elements from plugins', async () => {
+    const plugin = new PluginManifest({}, {
+      'components.streams.overview.tableElements': [
+        {
+          attributeName: 'data_lake',
+          attributes: [{ id: 'data_lake', title: 'Data Lake' }],
+          columnRenderers: {
+            data_lake: {
+              renderCell: () => 'Preview logs',
+              staticWidth: 'matchHeader',
+            },
+          },
+        },
+      ],
+    });
+
+    PluginStore.register(plugin);
+    asMock(useFetchEntities).mockReturnValue(paginatedStreams());
+    asMock(useUserLayoutPreferences).mockReturnValue({
+      data: {
+        ...layoutPreferences,
+        attributes: undefined,
+      },
+      isInitialLoading: false,
+      refetch: () => {},
+    });
+
+    try {
+      renderSut();
+
+      await screen.findByText('Data Lake');
+      await screen.findByText('Preview logs');
+    } finally {
+      PluginStore.unregister(plugin);
+    }
+  });
 });

graylog2-web-interface/src/components/streams/StreamsOverview/StreamsOverview.tsx

@@ -17,7 +17,6 @@
 import React, { useEffect, useMemo } from 'react';
 import { useQueryClient } from '@tanstack/react-query';
 
-import useCurrentUser from 'hooks/useCurrentUser';
 import QueryHelper from 'components/common/QueryHelper';
 import type { Stream } from 'stores/streams/StreamsStore';
 import StreamsStore from 'stores/streams/StreamsStore';
@@ -27,13 +26,13 @@ import getStreamTableElements from 'components/streams/StreamsOverview/Constants
 import FilterValueRenderers from 'components/streams/StreamsOverview/FilterValueRenderers';
 import useTableElements from 'components/streams/StreamsOverview/hooks/useTableComponents';
 import PaginatedEntityTable from 'components/common/PaginatedEntityTable';
-import usePluggableEntityTableElements from 'hooks/usePluggableEntityTableElements';
 import { CurrentUserStore } from 'stores/users/CurrentUserStore';
 import type { SearchParams } from 'stores/PaginationTypes';
 import type { PaginatedResponse } from 'components/common/PaginatedEntityTable/useFetchEntities';
 
 import CustomColumnRenderers from './ColumnRenderers';
 import usePipelineColumn from './hooks/usePipelineColumn';
+import useStreamsOverviewExtensions from './hooks/useStreamsOverviewExtensions';
 
 const useRefetchStreamsOnStoreChange = (refetchStreams: () => void) => {
   useEffect(() => {
@@ -48,26 +47,23 @@ const useRefetchStreamsOnStoreChange = (refetchStreams: () => void) => {
 type Props = {
   indexSets: Array<IndexSet>;
 };
-const entityName = 'stream';
 
 const StreamsOverview = ({ indexSets }: Props) => {
   const queryClient = useQueryClient();
   const { isPipelineColumnPermitted } = usePipelineColumn();
-  const currentUser = useCurrentUser();
-  const { pluggableColumnRenderers, pluggableAttributes, pluggableExpandedSections } =
-    usePluggableEntityTableElements<Stream>(null, entityName);
+  const { columnRenderers: extensionColumnRenderers, attributes: extensionAttributes, expandedSections: pluggableExpandedSections } =
+    useStreamsOverviewExtensions();
 
   const { entityActions, expandedSections, bulkActions } = useTableElements({ indexSets, pluggableExpandedSections });
   useRefetchStreamsOnStoreChange(() => queryClient.invalidateQueries({ queryKey: KEY_PREFIX }));
 
   const columnRenderers = useMemo(
-    () =>
-      CustomColumnRenderers(indexSets, isPipelineColumnPermitted, currentUser.permissions, pluggableColumnRenderers),
-    [indexSets, isPipelineColumnPermitted, currentUser.permissions, pluggableColumnRenderers],
+    () => CustomColumnRenderers(indexSets, isPipelineColumnPermitted, extensionColumnRenderers),
+    [extensionColumnRenderers, indexSets, isPipelineColumnPermitted],
   );
   const { additionalAttributes, defaultLayout } = useMemo(
-    () => getStreamTableElements(currentUser.permissions, isPipelineColumnPermitted, pluggableAttributes),
-    [currentUser.permissions, isPipelineColumnPermitted, pluggableAttributes],
+    () => getStreamTableElements(isPipelineColumnPermitted, extensionAttributes),
+    [extensionAttributes, isPipelineColumnPermitted],
   );
 
   const fetchEntities = (options: SearchParams): Promise<PaginatedResponse<Stream>> => {

graylog2-web-interface/src/components/streams/StreamsOverview/hooks/useStreamsOverviewExtensions.ts

@@ -0,0 +1,67 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+import { useMemo } from 'react';
+
+import usePluginEntities from 'hooks/usePluginEntities';
+import usePluggableEntityTableElements from 'hooks/usePluggableEntityTableElements';
+import type { Stream } from 'stores/streams/StreamsStore';
+import type { Attribute } from 'stores/PaginationTypes';
+import type { ColumnRenderersByAttribute, ExpandedSectionRenderer } from 'components/common/EntityDataTable/types';
+
+const entityName = 'stream';
+const streamOverviewTableElementsExport = 'components.streams.overview.tableElements';
+
+const useStreamsOverviewExtensions = (): {
+  columnRenderers: ColumnRenderersByAttribute<Stream>;
+  attributes: {
+    attributeNames: Array<string>;
+    attributes: Array<Attribute>;
+  };
+  expandedSections: { [sectionName: string]: ExpandedSectionRenderer<Stream> };
+} => {
+  const {
+    pluggableColumnRenderers,
+    pluggableAttributes,
+    pluggableExpandedSections,
+  } = usePluggableEntityTableElements<Stream>(null, entityName);
+  const pluginTableElements = usePluginEntities(streamOverviewTableElementsExport);
+
+  return useMemo(
+    () => ({
+      // Stream overview extensions are stream-specific and should be applied
+      // before generic entity table extensions so generic plugins can still override them.
+      columnRenderers: {
+        ...pluginTableElements.reduce((acc, curr) => ({ ...acc, ...curr.columnRenderers }), {}),
+        ...pluggableColumnRenderers,
+      },
+      attributes: {
+        attributeNames: [
+          ...pluginTableElements.map(({ attributeName }) => attributeName),
+          ...pluggableAttributes.attributeNames,
+        ],
+        attributes: [
+          ...pluginTableElements.flatMap(({ attributes }) => attributes),
+          ...pluggableAttributes.attributes,
+        ],
+      },
+      expandedSections: pluggableExpandedSections,
+    }),
+    [pluginTableElements, pluggableAttributes, pluggableColumnRenderers, pluggableExpandedSections],
+  );
+};
+
+export default useStreamsOverviewExtensions;