Add setting for backfilling events on setup (#103)

* Add setting for backfill depth

* Fix a couple of random bugs

* Run formatter

* Add chaining to exception

Author: xvaier

packages/flare/bin/constants.py

@@ -15,6 +15,7 @@ class PasswordKeys(Enum):
     INGEST_FULL_EVENT_DATA = "ingest_full_event_data"
     SEVERITIES_FILTER = "severities_filter"
     SOURCE_TYPES_FILTER = "source_types_filter"
+    NUMBER_OF_DAYS_TO_BACKFILL = "number_of_days_to_backfill"
 
 
 class DataStoreKeys(Enum):

packages/flare/bin/cron_job_ingest_events.py

@@ -61,7 +61,12 @@ def main(
         # for identifiers 30 days prior to the day a tenant was first configured.
         start_date = data_store.get_earliest_ingested_by_tenant(tenant_id)
         if not start_date:
-            start_date = datetime.now(timezone.utc) - timedelta(days=30)
+            number_of_days_to_backfill = get_number_of_days_to_backfill(
+                storage_passwords=storage_passwords
+            )
+            start_date = datetime.now(timezone.utc) - timedelta(
+                days=number_of_days_to_backfill
+            )
             data_store.set_earliest_ingested_by_tenant(tenant_id, start_date)
 
         for event, next_token in fetch_feed(
@@ -142,6 +147,18 @@ def get_api_key(storage_passwords: client.StoragePasswords) -> str:
     return api_key
 
 
+def get_number_of_days_to_backfill(storage_passwords: client.StoragePasswords) -> int:
+    number_of_days_to_backfill = get_storage_password_value(
+        storage_passwords=storage_passwords,
+        password_key=PasswordKeys.NUMBER_OF_DAYS_TO_BACKFILL.value,
+    )
+
+    try:
+        return int(number_of_days_to_backfill) if number_of_days_to_backfill else 30
+    except Exception as e:
+        raise Exception("Number of days to backfill not a number") from e
+
+
 def get_tenant_ids(storage_passwords: client.StoragePasswords) -> list[int]:
     stored_tenant_ids = get_storage_password_value(
         storage_passwords=storage_passwords, password_key=PasswordKeys.TENANT_IDS.value

packages/react-components/src/components/ConfigurationUserPreferencesStep.tsx

@@ -26,6 +26,7 @@ import {
     saveConfiguration,
     convertSourceTypeFilterToArray,
     fetchTenantIds,
+    fetchNumberOfDaysToBackfill,
 } from '../utils/setupConfiguration';
 import './ConfigurationGlobalStep.css';
 import './ConfigurationUserPreferencesStep.css';
@@ -36,6 +37,7 @@ import { ToastKeys, toastManager } from './ToastManager';
 import Tooltip from './Tooltip';
 import FlareLogoLoading from './FlareLogoLoading';
 import TenantSelection from './TenantSelection';
+import Input from './Input';
 
 const ConfigurationUserPreferencesStep: FC<{
     configurationStep: ConfigurationStep;
@@ -54,6 +56,8 @@ const ConfigurationUserPreferencesStep: FC<{
     const [indexNames, setIndexNames] = useState<string[]>([]);
     const [isIngestingFullEventData, setIsIngestingFullEventData] = useState(false);
     const [isLoading, setIsLoading] = useState(false);
+    const [numberOfDaysToBackfill, setNumberOfDaysToBackfill] = useState<string>();
+    const [isFirstSetup, setIsFirstSetup] = useState(false);
 
     const handleIndexNameChange = (e): void => setIndexName(e.target.value);
     const handleIsIngestingFullEventDataChange = (e): void =>
@@ -68,7 +72,8 @@ const ConfigurationUserPreferencesStep: FC<{
             indexName,
             isIngestingFullEventData,
             getSeverityFilterValue(selectedSeverities, severities),
-            getSourceTypesFilterValue(selectedSourceTypes, sourceTypeCategories)
+            getSourceTypesFilterValue(selectedSourceTypes, sourceTypeCategories),
+            numberOfDaysToBackfill
         )
             .then(() => {
                 setIsLoading(false);
@@ -101,6 +106,7 @@ const ConfigurationUserPreferencesStep: FC<{
                 fetchSeveritiesFilter(),
                 fetchSourceTypeFilters(apiKey),
                 fetchSourceTypesFilter(),
+                fetchNumberOfDaysToBackfill(),
             ])
                 .then(
                     ([
@@ -113,7 +119,14 @@ const ConfigurationUserPreferencesStep: FC<{
                         severitiesFilter,
                         allSourceTypeCategories,
                         sourceTypeFilter,
+                        numberOfDaysToBackfillSaved,
                     ]) => {
+                        // The form can't be submitted without any tenant ids
+                        // so the absence of tenant ids indicates that it is the first setup.
+                        if (!tenantIds.length) {
+                            setIsFirstSetup(true);
+                        }
+                        setNumberOfDaysToBackfill(numberOfDaysToBackfillSaved ?? '');
                         setSelectedTenantIds(new Set(tenantIds));
                         setIsIngestingFullEventData(ingestFullEventData);
                         setIndexName(index);
@@ -265,6 +278,25 @@ const ConfigurationUserPreferencesStep: FC<{
                         />
                     </span>
                 </div>
+                <div className="form-item">
+                    <div className="label-tooltip">
+                        <Label>Number of days to backfill events</Label>
+                        <Tooltip>
+                            <div>
+                                This field can only be set when setting up the app for the first
+                                time.
+                            </div>
+                        </Tooltip>
+                    </div>
+                    <Input
+                        onChange={(e): void => setNumberOfDaysToBackfill(e.target.value)}
+                        value={numberOfDaysToBackfill}
+                        min="0"
+                        type="number"
+                        placeholder="30"
+                        disabled={!isFirstSetup}
+                    />
+                </div>
                 <div className="button-group">
                     <Button onClick={(): void => onNavigateBackClick()} isSecondary>
                         Back

packages/react-components/src/components/Input.css

@@ -0,0 +1,5 @@
+input {
+    color: var(--text-color);
+    border: 0px;
+    outline: none;
+}

packages/react-components/src/components/Input.tsx

@@ -0,0 +1,9 @@
+import React, { FC } from 'react';
+
+import './Input.css';
+
+const Input: FC<React.ComponentProps<'input'>> = ({ ...props }) => {
+    return <input {...props} />;
+};
+
+export default Input;

packages/react-components/src/global.css

@@ -77,10 +77,17 @@ input {
     border: 1px solid var(--secondary-text-color);
     border-radius: 40px;
     margin-top: 0.5rem;
+
+    &:disabled {
+        cursor: not-allowed;
+        color: var(--secondary-text-color);
+    }
 }
 
 input:hover {
-    border: 1px solid var(--button-bg-color);
+    :not(:disabled) {
+        border: 1px solid var(--button-bg-color);
+    }
 }
 
 input::placeholder {

packages/react-components/src/models/constants.ts

@@ -16,4 +16,5 @@ export enum PasswordKeys {
     INGEST_FULL_EVENT_DATA = 'ingest_full_event_data',
     SEVERITIES_FILTER = 'severities_filter',
     SOURCE_TYPES_FILTER = 'source_types_filter',
+    NUMBER_OF_DAYS_TO_BACKFILL = 'number_of_days_to_backfill',
 }

packages/react-components/src/utils/setupConfiguration.ts

@@ -121,7 +121,8 @@ async function saveConfiguration(
     indexName: string,
     isIngestingFullEventData: boolean,
     severitiesFilter: string,
-    sourceTypesFilter: string
+    sourceTypesFilter: string,
+    numberOfDaysToBackfill?: string
 ): Promise<void> {
     const service = createService();
     const storagePasswords = await promisify(service.storagePasswords().fetch)();
@@ -132,6 +133,11 @@ async function saveConfiguration(
         PasswordKeys.INGEST_FULL_EVENT_DATA,
         `${isIngestingFullEventData}`
     );
+    await savePassword(
+        storagePasswords,
+        PasswordKeys.NUMBER_OF_DAYS_TO_BACKFILL,
+        numberOfDaysToBackfill ?? ''
+    );
     await savePassword(storagePasswords, PasswordKeys.SEVERITIES_FILTER, `${severitiesFilter}`);
     await savePassword(storagePasswords, PasswordKeys.SOURCE_TYPES_FILTER, `${sourceTypesFilter}`);
     await saveIndexForIngestion(service, indexName);
@@ -244,6 +250,12 @@ async function fetchTenantIds(): Promise<number[]> {
     });
 }
 
+async function fetchNumberOfDaysToBackfill(): Promise<string | undefined> {
+    return fetchPassword(PasswordKeys.NUMBER_OF_DAYS_TO_BACKFILL).then((numberOfDaysToBackfill) => {
+        return numberOfDaysToBackfill;
+    });
+}
+
 async function fetchIngestFullEventData(): Promise<boolean> {
     return fetchPassword(PasswordKeys.INGEST_FULL_EVENT_DATA).then((isIngestingFullEventData) => {
         return isIngestingFullEventData === 'true';
@@ -446,6 +458,7 @@ export {
     fetchSeveritiesFilter,
     fetchSourceTypeFilters,
     fetchSourceTypesFilter,
+    fetchNumberOfDaysToBackfill,
     fetchTenantIds,
     fetchUserTenants,
     fetchVersionName,