Skip to content

Commit f5d7375

Browse files
TyMarcTyMarc
authored
Merge pull request #73 from Flared/mahinse/change_ingest_metadata_default
Change the option from 'Ingest metadata only' to 'Ingest full event data' and make it off by default
2 parents cb9f782 + 4a5548a commit f5d7375

9 files changed

Lines changed: 45 additions & 36 deletions

File tree

‎packages/flare/bin/constants.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
class PasswordKeys(Enum):
1414
API_KEY = "api_key"
1515
TENANT_ID = "tenant_id"
16-
INGEST_METADATA_ONLY = "ingest_metadata_only"
16+
INGEST_FULL_EVENT_DATA = "ingest_full_event_data"
1717
SEVERITIES_FILTER = "severities_filter"
1818
SOURCE_TYPES_FILTER = "source_types_filter"
1919

‎packages/flare/bin/cron_job_ingest_events.py‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,9 @@ def main(
9595

9696
api_key = get_api_key(storage_passwords=storage_passwords)
9797
tenant_id = get_tenant_id(storage_passwords=storage_passwords)
98-
ingest_metadata_only = get_ingest_metadata_only(storage_passwords=storage_passwords)
98+
ingest_full_event_data = get_ingest_full_event_data(
99+
storage_passwords=storage_passwords
100+
)
99101
severities_filter = get_severities_filter(storage_passwords=storage_passwords)
100102
source_types_filter = get_source_types_filter(storage_passwords=storage_passwords)
101103

@@ -107,7 +109,7 @@ def main(
107109
kvstore=kvstore,
108110
api_key=api_key,
109111
tenant_id=tenant_id,
110-
ingest_metadata_only=ingest_metadata_only,
112+
ingest_full_event_data=ingest_full_event_data,
111113
severities=severities_filter,
112114
source_types=source_types_filter,
113115
flare_api_cls=flare_api_cls,
@@ -156,11 +158,11 @@ def get_tenant_id(storage_passwords: StoragePasswords) -> int:
156158
return tenant_id
157159

158160

159-
def get_ingest_metadata_only(storage_passwords: StoragePasswords) -> bool:
161+
def get_ingest_full_event_data(storage_passwords: StoragePasswords) -> bool:
160162
return (
161163
get_storage_password_value(
162164
storage_passwords=storage_passwords,
163-
password_key=PasswordKeys.INGEST_METADATA_ONLY.value,
165+
password_key=PasswordKeys.INGEST_FULL_EVENT_DATA.value,
164166
)
165167
== "true"
166168
)
@@ -312,7 +314,7 @@ def fetch_feed(
312314
kvstore: KVStoreCollections,
313315
api_key: str,
314316
tenant_id: int,
315-
ingest_metadata_only: bool,
317+
ingest_full_event_data: bool,
316318
severities: list[str],
317319
source_types: list[str],
318320
flare_api_cls: FlareAPI = FlareAPI,
@@ -326,7 +328,7 @@ def fetch_feed(
326328
for event_next in flare_api.fetch_feed_events(
327329
next=next,
328330
start_date=start_date,
329-
ingest_metadata_only=ingest_metadata_only,
331+
ingest_full_event_data=ingest_full_event_data,
330332
severities=severities,
331333
source_types=source_types,
332334
):

‎packages/flare/bin/flare.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ def fetch_feed_events(
4949
*,
5050
next: Optional[str] = None,
5151
start_date: Optional[date] = None,
52-
ingest_metadata_only: bool,
52+
ingest_full_event_data: bool,
5353
severities: list[str],
5454
source_types: list[str],
5555
) -> Iterator[tuple[dict, str]]:
@@ -63,7 +63,7 @@ def fetch_feed_events(
6363
self.logger.debug(event_feed)
6464
next_token = event_feed["next"]
6565
for event in event_feed["items"]:
66-
if not ingest_metadata_only:
66+
if ingest_full_event_data:
6767
event = self._fetch_full_event_from_uid(
6868
uid=event["metadata"]["uid"]
6969
)

‎packages/flare/tests/bin/conftest.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ def fetch_feed_events(
104104
self,
105105
next: Optional[str],
106106
start_date: Optional[datetime],
107-
ingest_metadata_only: bool,
107+
ingest_full_event_data: bool,
108108
severities: list[str],
109109
source_types: list[str],
110110
) -> List[tuple[dict, str]]:

‎packages/flare/tests/bin/test_flare_wrapper.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ def test_flare_full_data_without_metadata(
4343
for event, next_token in flare_api.fetch_feed_events(
4444
next=None,
4545
start_date=None,
46-
ingest_metadata_only=True,
46+
ingest_full_event_data=False,
4747
severities=[],
4848
source_types=[],
4949
):
@@ -106,7 +106,7 @@ def test_flare_full_data_with_metadata(
106106
for event, next_token in flare_api.fetch_feed_events(
107107
next=None,
108108
start_date=None,
109-
ingest_metadata_only=False,
109+
ingest_full_event_data=True,
110110
severities=[],
111111
source_types=[],
112112
):
@@ -150,7 +150,7 @@ def test_flare_full_data_with_metadata_and_exception(
150150
flare_api.fetch_feed_events(
151151
next=None,
152152
start_date=None,
153-
ingest_metadata_only=False,
153+
ingest_full_event_data=True,
154154
severities=[],
155155
source_types=[],
156156
)

‎packages/flare/tests/bin/test_ingest_events.py‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
from cron_job_ingest_events import fetch_feed
2020
from cron_job_ingest_events import get_api_key
2121
from cron_job_ingest_events import get_collection_value
22+
from cron_job_ingest_events import get_ingest_full_event_data
2223
from cron_job_ingest_events import get_last_fetched
2324
from cron_job_ingest_events import get_last_ingested_tenant_id
2425
from cron_job_ingest_events import get_start_date
@@ -188,6 +189,12 @@ def test_get_last_fetched_expect_none(kvstore: FakeKVStoreCollections) -> None:
188189
assert get_last_fetched(kvstore=kvstore) is None
189190

190191

192+
def test_get_default_ingest_full_event_data_value(
193+
storage_passwords: FakeStoragePasswords,
194+
) -> None:
195+
assert get_ingest_full_event_data(storage_passwords=storage_passwords) is False
196+
197+
191198
@pytest.mark.parametrize(
192199
"kvstore",
193200
[
@@ -212,7 +219,7 @@ def test_fetch_feed_expect_exception(
212219
kvstore=kvstore,
213220
api_key="some_key",
214221
tenant_id=11111,
215-
ingest_metadata_only=False,
222+
ingest_full_event_data=True,
216223
severities=[],
217224
source_types=[],
218225
):
@@ -238,7 +245,7 @@ def test_fetch_feed_expect_feed_response(
238245
kvstore=kvstore,
239246
api_key="some_key",
240247
tenant_id=11111,
241-
ingest_metadata_only=False,
248+
ingest_full_event_data=True,
242249
severities=[],
243250
source_types=[],
244251
flare_api_cls=FakeFlareAPI,

‎packages/react-components/src/components/ConfigurationUserPreferencesStep.tsx‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ import {
1717
fetchCurrentIndexName,
1818
fetchSeverityFilters,
1919
fetchSourceTypeFilters,
20-
fetchIngestMetadataOnly,
20+
fetchIngestFullEventData,
2121
fetchSeveritiesFilter,
2222
fetchTenantId,
2323
fetchUserTenants,
@@ -51,13 +51,13 @@ const ConfigurationUserPreferencesStep: FC<{
5151
const [selectedSourceTypes, setSelectedSourceTypes] = useState<SourceType[]>([]);
5252
const [indexName, setIndexName] = useState('');
5353
const [indexNames, setIndexNames] = useState<string[]>([]);
54-
const [isIngestingMetadataOnly, setIsIngestingMetadataOnly] = useState(false);
54+
const [isIngestingFullEventData, setIsIngestingFullEventData] = useState(false);
5555
const [isLoading, setIsLoading] = useState(false);
5656

5757
const handleTenantIdChange = (e): void => setTenantId(parseInt(e.target.value, 10));
5858
const handleIndexNameChange = (e): void => setIndexName(e.target.value);
59-
const handleIsIngestingMetadataChange = (e): void =>
60-
setIsIngestingMetadataOnly(e.target.checked);
59+
const handleIsIngestingFullEventDataChange = (e): void =>
60+
setIsIngestingFullEventData(e.target.checked);
6161

6262
const handleSubmitUserPreferences = (): void => {
6363
setIsLoading(true);
@@ -66,7 +66,7 @@ const ConfigurationUserPreferencesStep: FC<{
6666
apiKey,
6767
Number(tenantId),
6868
indexName,
69-
isIngestingMetadataOnly,
69+
isIngestingFullEventData,
7070
getSeverityFilterValue(selectedSeverities, severities),
7171
getSourceTypesFilterValue(selectedSourceTypes, sourceTypeCategories)
7272
)
@@ -93,7 +93,7 @@ const ConfigurationUserPreferencesStep: FC<{
9393
if (configurationStep === ConfigurationStep.UserPreferences) {
9494
Promise.all([
9595
fetchTenantId(),
96-
fetchIngestMetadataOnly(),
96+
fetchIngestFullEventData(),
9797
fetchCurrentIndexName(),
9898
fetchUserTenants(apiKey),
9999
fetchAvailableIndexNames(),
@@ -105,7 +105,7 @@ const ConfigurationUserPreferencesStep: FC<{
105105
.then(
106106
([
107107
id,
108-
ingestMetadataOnly,
108+
ingestFullEventData,
109109
index,
110110
userTenants,
111111
availableIndexNames,
@@ -115,7 +115,7 @@ const ConfigurationUserPreferencesStep: FC<{
115115
sourceTypeFilter,
116116
]) => {
117117
setTenantId(id);
118-
setIsIngestingMetadataOnly(ingestMetadataOnly);
118+
setIsIngestingFullEventData(ingestFullEventData);
119119
setIndexName(index);
120120
if (id === undefined && userTenants.length > 0) {
121121
setTenantId(userTenants[0].id);
@@ -250,18 +250,18 @@ const ConfigurationUserPreferencesStep: FC<{
250250
</div>
251251
<div className="form-item">
252252
<div className="label-tooltip">
253-
<Label>Basic event ingestion</Label>
253+
<Label>Full event data ingestion</Label>
254254
<Tooltip>
255255
<div>
256-
Select this option if you want to ingest only the metadata of the
257-
events instead of the full data to it.
256+
Select this option if you want to ingest the full data of the events
257+
instead of the metadata of them.
258258
</div>
259259
</Tooltip>
260260
</div>
261261
<span className="switch-container">
262262
<Switch
263-
value={isIngestingMetadataOnly}
264-
onChange={handleIsIngestingMetadataChange}
263+
value={isIngestingFullEventData}
264+
onChange={handleIsIngestingFullEventDataChange}
265265
/>
266266
</span>
267267
</div>

‎packages/react-components/src/models/constants.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ export const KV_COLLECTION_VALUE = 'value';
1515
export enum PasswordKeys {
1616
API_KEY = 'api_key',
1717
TENANT_ID = 'tenant_id',
18-
INGEST_METADATA_ONLY = 'ingest_metadata_only',
18+
INGEST_FULL_EVENT_DATA = 'ingest_full_event_data',
1919
SEVERITIES_FILTER = 'severities_filter',
2020
SOURCE_TYPES_FILTER = 'source_types_filter',
2121
}

‎packages/react-components/src/utils/setupConfiguration.ts‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,7 @@ async function saveConfiguration(
131131
apiKey: string,
132132
tenantId: number,
133133
indexName: string,
134-
isIngestingMetadataOnly: boolean,
134+
isIngestingFullEventData: boolean,
135135
severitiesFilter: string,
136136
sourceTypesFilter: string
137137
): Promise<void> {
@@ -141,8 +141,8 @@ async function saveConfiguration(
141141
await savePassword(storagePasswords, PasswordKeys.TENANT_ID, `${tenantId}`);
142142
await savePassword(
143143
storagePasswords,
144-
PasswordKeys.INGEST_METADATA_ONLY,
145-
`${isIngestingMetadataOnly}`
144+
PasswordKeys.INGEST_FULL_EVENT_DATA,
145+
`${isIngestingFullEventData}`
146146
);
147147
await savePassword(storagePasswords, PasswordKeys.SEVERITIES_FILTER, `${severitiesFilter}`);
148148
await savePassword(storagePasswords, PasswordKeys.SOURCE_TYPES_FILTER, `${sourceTypesFilter}`);
@@ -243,9 +243,9 @@ async function fetchTenantId(): Promise<number | undefined> {
243243
});
244244
}
245245

246-
async function fetchIngestMetadataOnly(): Promise<boolean> {
247-
return fetchPassword(PasswordKeys.INGEST_METADATA_ONLY).then((isIngestingMetadataOnly) => {
248-
return isIngestingMetadataOnly === 'true';
246+
async function fetchIngestFullEventData(): Promise<boolean> {
247+
return fetchPassword(PasswordKeys.INGEST_FULL_EVENT_DATA).then((isIngestingFullEventData) => {
248+
return isIngestingFullEventData === 'true';
249249
});
250250
}
251251

@@ -434,7 +434,7 @@ export {
434434
fetchSeverityFilters,
435435
fetchCollectionItems,
436436
fetchCurrentIndexName,
437-
fetchIngestMetadataOnly,
437+
fetchIngestFullEventData,
438438
fetchSeveritiesFilter,
439439
fetchSourceTypeFilters,
440440
fetchSourceTypesFilter,

0 commit comments

Comments
 (0)