Skip to content

Commit 893bec9

Browse files
mjabascal10mjabascal10
committed
feat: update filter for sophos, m365 and aws
1 parent f88057e commit 893bec9

4 files changed

Lines changed: 1709 additions & 0 deletions

File tree

‎backend/src/main/resources/config/liquibase/changelog/20250527001_update_filter_sophos_central.xml‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<databaseChangeLog
3+
xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
4+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
5+
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.5.xsd">
6+
7+
<changeSet id="20250527001" author="JocLRojas">
8+
9+
<sql dbms="postgresql" splitStatements="true" stripComments="true">
10+
<![CDATA[
11+
12+
UPDATE public.utm_logstash_filter
13+
SET filter_version='1.0.0',
14+
logstash_filter='# Sophos_Central filter, version 1.0.0 using "SF syslog file guide 20.0"
15+
16+
# See: https://docs.sophos.com/nsg/sophos-firewall/20.0/pdf/sf-syslog-guide-20.0.pdf
17+
# and https://docs.sophos.com/nsg/sophos-firewall/20.0/Help/en-us/webhelp/onlinehelp/AdministratorHelp/Logs/TroubleshootingLogs/LogFileDetails/index.html#https-ftp-waf
18+
19+
pipeline:
20+
- dataTypes:
21+
- sophos-central
22+
steps:
23+
- json:
24+
source: raw
25+
26+
- rename:
27+
from:
28+
- log.coreremedyitems
29+
to: log.coreremedyItems
30+
31+
- rename:
32+
from:
33+
- log.createdat
34+
to: log.createdAt
35+
36+
- rename:
37+
from:
38+
- log.customerid
39+
to: log.customerId
40+
41+
- rename:
42+
from:
43+
- log.endpointid
44+
to: log.endpointId
45+
46+
- rename:
47+
from:
48+
- log.endpointtype
49+
to: log.endpointType
50+
51+
- rename:
52+
from:
53+
- log.sourceinfo.ip
54+
to: log.ip
55+
56+
- rename:
57+
from:
58+
- log.userid
59+
to: log.userId
60+
61+
- rename:
62+
from:
63+
- log.sourceinfo
64+
to: log.sourceInfo'
65+
WHERE id=1527;
66+
]]>
67+
</sql>
68+
</changeSet>
69+
</databaseChangeLog>

‎backend/src/main/resources/config/liquibase/changelog/20250527002_update_filter_m365.xml‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<databaseChangeLog
3+
xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
4+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
5+
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.5.xsd">
6+
7+
<changeSet id="20250527002" author="JocLRojas">
8+
9+
<sql dbms="postgresql" splitStatements="true" stripComments="true">
10+
<![CDATA[
11+
12+
UPDATE public.utm_logstash_filter
13+
SET filter_version='1.0.0',
14+
logstash_filter='# Microsoft 365 filter, version 1.0.0
15+
# Based on Official documentation
16+
# See https://learn.microsoft.com/en-us/compliance/assurance/assurance-microsoft-365-audit-log-collection
17+
# https://learn.microsoft.com/es-es/office/office-365-management-api/aip-unified-audit-logs-best-practices
18+
# https://learn.microsoft.com/en-us/purview/audit-log-activities
19+
pipeline:
20+
- dataTypes:
21+
- o365
22+
steps:
23+
- json:
24+
source: raw
25+
26+
- rename:
27+
from:
28+
- log.AppAccessContext.AADSessionId
29+
to: log.appAccessContextAADSessionId
30+
31+
- rename:
32+
from:
33+
- log.AppAccessContext.APIId
34+
to: log.appAccessContextAPIId
35+
36+
- rename:
37+
from:
38+
- log.AppAccessContext.ClientAppId
39+
to: log.appAccessContextClientAppId
40+
41+
- rename:
42+
from:
43+
- log.AppAccessContext.CorrelationId
44+
to: log.appAccessContextCorrelationId
45+
46+
- rename:
47+
from:
48+
- log.AppAccessContext.IssuedAtTime
49+
to: log.deviceTime
50+
51+
- rename:
52+
from:
53+
- log.AppAccessContext.UniqueTokenId
54+
to: log.appAccessContextUniqueTokenId
55+
56+
- rename:
57+
from:
58+
- log.ClientIPAddress
59+
to: origin.ip
60+
61+
- rename:
62+
from:
63+
- log.Operation
64+
to: action
65+
66+
- rename:
67+
from:
68+
- log.Version
69+
to: log.version
70+
71+
- rename:
72+
from:
73+
- log.ClientIP
74+
to: log.clientIP
75+
76+
- rename:
77+
from:
78+
- log.ResultStatus
79+
to: actionResult
80+
81+
- rename:
82+
from:
83+
- log.UserId
84+
to: origin.user
85+
86+
# Drop unnecessary events
87+
- drop:
88+
where:
89+
variables:
90+
- get: action
91+
as: action
92+
oftype: string
93+
expression: action=="SupervisionRuleMatch" || action=="SupervisoryReviewTag" || action=="ComplianceManagerAutomationChange" || action=="LabelContentExplorerAccessedItem" || action=="CreateCopilotPlugin" || action=="CreateCopilotPromptBook" || action=="DeleteCopilotPlugin" || action=="DeleteCopilotPromptBook" || action=="DisableCopilotPlugin" || action=="DisableCopilotPromptBook" || action=="EnableCopilotPlugin" || action=="EnableCopilotPromptBook" || action=="CopilotInteraction" || action=="UpdateCopilotPlugin" || action=="UpdateCopilotPromptBook" || action=="UpdateCopilotSettings" || action=="ApproveDisposal" || action=="ExtendRetention" || action=="RelabelItem" || action=="SearchUpdated" || action=="CaseUpdated" || action=="SearchPermissionUpdated" || action=="HoldUpdated" || action=="PreviewItemDownloaded" || action=="PreviewItemListed" || action=="SearchCreated" || action=="CaseAdded" || action=="HoldCreated" || action=="SearchRemoved" || action=="HoldRemoved" || action=="SearchExportDownloaded" || action=="SearchPreviewed" || action=="SearchResultsPurged" || action=="RemovedSearchResultsSentToZoom" || action=="RemovedSearchExported" || action=="RemovedSearchPreviewed" || action=="RemovedSearchResultsPurged" || action=="SearchReportRemoved" || action=="SearchResultsSentToZoom" || action=="SearchStarted" || action=="SearchExported" || action=="SearchReport" || action=="SearchStopped" || action=="SearchViewed" || action=="ViewedSearchExported" || action=="ViewedSearchPreviewed" || action=="AddRemediatedData" || action=="BurnJob" || action=="CreateWorkingSet" || action=="CreateWorkingSetSearch" || action=="CreateTag" || action=="DeleteWorkingSetSearch" || action=="DeleteTag" || action=="DownloadDocument" || action=="UpdateTag" || action=="ExportJob" || action=="UpdateWorkingSetSearch" || action=="PreviewWorkingSetSearch" || action=="ErrorRemediationJob" || action=="TagFiles" || action=="TagJob" || action=="ViewDocument" || action=="Copy" || action=="Create" || action=="ApplyRecordLabel" || action=="HardDelete" || action=="Send" || action=="Update" || action=="FileAccessed" || action=="FileAccessedExtended" || action=="ComplianceSettingChanged" || action=="LockRecord" || action=="UnlockRecord" || action=="FileCheckedIn" || action=="FileCheckedOut" || action=="FileCopied" || action=="FileDeletedFirstStageRecycleBin" || action=="FileDeletedSecondStageRecycleBin" || action=="RecordDelete" || action=="DocumentSensitivityMismatchDetected" || action=="FileCheckOutDiscarded" || action=="FileDownloaded" || action=="FileModifiedExtended" || action=="FilePreviewed" || action=="SearchQueryPerformed" || action=="FileRecycled" || action=="FolderRecycled" || action=="FileVersionsAllMinorsRecycled" || action=="FileVersionsAllRecycled" || action=="FileVersionRecycled" || action=="FileRestored" || action=="FileUploaded" || action=="PageViewed" || action=="PageViewedExtended" || action=="ClientViewSignaled" || action=="PagePrefetched" || action=="FolderCopied" || action=="FolderCreated" || action=="FolderDeletedFirstStageRecycleBin" || action=="FolderDeletedSecondStageRecycleBin" || action=="FolderRestored" || action=="InformationBarriersInsightsReportCompleted" || action=="InformationBarriersInsightsReportOneDriveSectionQueried" || action=="InformationBarriersInsightsReportSchedule" || action=="InformationBarriersInsightsReportSharePointSectionQueried" || action=="updateddeviceconfiguration" || action=="UpdatedPolicyConfigPriority" || action=="BackupPolicyActivated" || action=="RestoreTaskActivated" || action=="BackupItemAdded" || action=="BackupItemRemoved" || action=="RestoreTaskCompleted" || action=="DraftRestoreTaskCreated" || action=="NewBackupPolicyCreated" || action=="DraftRestoreTaskDeleted" || action=="DraftRestoreTaskEdited" || action=="BackupPolicyPaused" || action=="GetBackupItem" || action=="ViewBackupPolicyDetails" || action=="GetRestoreTaskDetails" || action=="ListAllBackupPolicies" || action=="ListAllBackupItemsInPolicies" || action=="ListAllBackupItemsInTenant" || action=="ListAllBackupItemsInWorkload" || action=="GetAllRestoreArtifactsInTask" || action=="ListAllRestorePoints" || action=="ListAllRestoreTasks" || action=="BackupItemRestoreCompleted" || action=="BackupItemRestoreTriggered" || action=="SetAdvancedFeatures" || action=="RunAntiVirusScan" || action=="LogsCollection" || action=="TaggingConfigurationUpdated" || action=="AlertExcelDownloaded" || action=="RemediationActionAdded" || action=="RemediationActionUpdated" || action=="SensorCreated" || action=="SensorDeploymentAccessKeyReceived" || action=="SensorDeploymentAccessKeyUpdated" || action=="SensorActivationMethodConfigurationUpdated" || action=="DomainControllerCoverageExcelDownloaded" || action=="MonitoringAlertUpdated" || action=="ReportDownloaded" || action=="AlertNotificationsRecipientAdded" || action=="MonitoringAlertNotificationRecipientAdded" || action=="WorkspaceCreated" || action=="AddCommentToIncident." || action=="AssignUserToIncident" || action=="UpdateIncidentStatus" || action=="AddTagsToIncident" || action=="RemoveTagsFromIncident" || action=="CreateComment" || action=="CreateForm" || action=="MoveForm" || action=="ViewForm" || action=="PreviewForm" || action=="ExportForm" || action=="AllowShareFormForCopy" || action=="DisallowShareFormForCopy" || action=="AddFormCoauthor" || action=="RemoveFormCoauthor" || action=="ViewRuntimeForm" || action=="CreateResponse" || action=="UpdateResponse" || action=="ViewResponses" || action=="ViewResponse" || action=="GetSummaryLink" || action=="DeleteSummaryLink" || action=="ProInvitation" || action=="ListForms" || action=="SubmitResponse" || action=="ConnectToExcelWorkbook" || action=="CollectionCreated" || action=="CollectionUpdated" || action=="CollectionHardDeleted" || action=="CollectionSoftDeleted" || action=="CollectionRenamed" || action=="MovedFormIntoCollection" || action=="MovedFormOutofCollection" || action=="PlanCopied" || action=="TaskAssigned" || action=="TaskCompleted" || action=="PlanListRead" || action=="TaskListRead" || action=="ProjectCreated" || action=="RoadmapCreated" || action=="RoadmapItemCreated" || action=="TaskCreated" || action=="ProjectListAccessed" || action=="RoadmapAccessed" || action=="RoadmapItemAccessed" || action=="TaskAccessed" || action=="AuditSearchCreated" || action=="AuditSearchCompleted" || action=="AuditSearchCancelled" || action=="AuditSearchExportJobCreated" || action=="AuditSearchExportJobCompleted" || action=="AuditSearchExportResultsDownloaded" || action=="EntityCreated" || action=="ClassificationAdded" || action=="ClassificationDefinitionCreated" || action=="GlossaryTermAssigned" || action=="GlossaryTermCreated" || action=="BotAddedToTeam" || action=="ChannelAdded" || action=="ConnectorAdded" || action=="MeetingDetail" || action=="MeetingParticipantDetail" || action=="MemberAdded" || action=="TabAdded" || action=="SensitivityLabelApplied" || action=="SensitivityLabelChanged" || action=="ChatCreated" || action=="TeamCreated" || action=="MessageDeleted" || action=="MessageEditedHasLink" || action=="MessagesExported" || action=="RecordingExported" || action=="TranscriptsExported" || action=="FailedValidation" || action=="ChatRetrieved" || action=="MessageHostedContentsListed" || action=="PerformedCardAction" || action=="MessageSent" || action=="AINotesUpdate" || action=="LiveNotesUpdate" || action=="AppPublishedToCatalog" || action=="MessageRead" || action=="InviteeResponded" || action=="ChannelOwnerResponded" || action=="MessagesListed" || action=="MessageCreatedHasLink" || action=="MessageCreatedNotification" || action=="MessageDeletedNotification" || action=="MessageUpdatedNotification" || action=="InviteSent" || action=="SubscribedToMessages" || action=="AppUpdatedInCatalog" || action=="ChatUpdated" || action=="MessageUpdated" || action=="TabUpdated" || action=="AppUpgraded" || action=="MessageSent" || action=="ScheduleGroupAdded" || action=="ShiftAdded" || action=="TimeOffAdded" || action=="OpenShiftAdded" || action=="ScheduleShared" || action=="ClockedIn" || action=="ClockedOut" || action=="BreakEnded" || action=="TimeClockEntryAdded" || action=="RequestAdded" || action=="RequestRespondedTo" || action=="WorkforceIntegrationAdded" || action=="OffShiftDialogAccepted" || action=="CreateUpdateRequest" || action=="EditUpdateRequest" || action=="SubmitUpdate" || action=="ViewUpdate" || action=="AcceptedSharingLinkOnFolder" || action=="FolderSharingLinkShared" || action=="LinkedEntityCreated" || action=="SubTaskCreated" || action=="TaskCreated" || action=="TaskRead" || action=="TaskListCreated" || action=="TaskListRead" || action=="AccessedOdataLink" || action=="CanceledQuery" || action=="DeletedResult" || action=="DownloadedReport" || action=="ExecutedQuery" || action=="UploadedOrgData" || action=="ViewedExplore" || action=="QuarantineReleaseRequestDeny" || action=="QuarantinePreview" || action=="QuarantineReleaseRequest" || action=="QuarantineViewHeader" || action=="UpdateUsageReportsPrivacySetting" || action=="NewAdaptiveScope" || action=="NewComplianceTag" || action=="NewRetentionCompliancePolicy" || action=="RemoveAdaptiveScope" || action=="RemoveComplianceTag" || action=="SetRestrictiveRetentionUI" || action=="ExchangeDataProactivelyPreserved" || action=="SharePointDataProactivelyPreserved" || action=="ListCreated" || action=="ListColumnCreated" || action=="ListContentTypeCreated" || action=="ListItemCreated" || action=="SiteColumnCreated" || action=="SiteContentTypeCreated" || action=="ListContentTypeDeleted" || action=="SiteColumnDeleted" || action=="SiteContentTypeDeleted" || action=="ListItemRecycled" || action=="ListItemRestored" || action=="ListColumnUpdated" || action=="ListContentTypeUpdated" || action=="SiteColumnUpdated" || action=="SiteContentTypeUpdated" || action=="SharingInvitationCreated" || action=="AccessRequestUpdated" || action=="SharingInvitationUpdated" || action=="SharingInvitationRevoked" || action=="AllowedDataLocationAdded" || action=="SiteGeoMoveCancelled" || action=="MigrationJobCompleted" || action=="SiteGeoMoveCompleted" || action=="SiteCollectionCreated" || action=="HubSiteOrphanHubDeleted" || action=="PreviewModeEnabledSet" || action=="LegacyWorkflowEnabledSet" || action=="OfficeOnDemandSet" || action=="PeopleResultsScopeSet" || action=="NewsFeedEnabledSet" || action=="HubSiteJoined" || action=="SiteCollectionQuotaModified" || action=="HubSiteRegistered" || action=="SiteGeoMoveScheduled" || action=="GeoQuotaAllocated" || action=="SiteAdminChangeRequest" || action=="ManagedSyncClientAllowed" || action=="FileSyncDownloadedFull" || action=="FileSyncUploadedFull" || action=="DataShareCreated" || action=="DataShareDeleted" || action=="GenerateCopyOfLakeData" || action=="DownloadCopyOfLakeData" || action=="SoftDeleteSettingsUpdated" || action=="CloseConversation" || action=="OpenConversation" || action=="MessageCreation" || action=="MessageDeleted" || action=="FileDownloaded" || action=="DataExport" || action=="ThreadAccessFailure" || action=="MarkedMessageChanged" || action=="RemoveCuratedTopic" || action=="UsagePolicyAcceptance" || action=="AdminThreadMuted" || action=="AdminThreadUnmuted" || action=="FileUpdateDescription" || action=="MessageUpdated" || action=="FileVisited" || action=="ThreadViewed" || action=="PulseSubmit" || action=="PulseCreate" || action=="PulseExtendDeadline" || action=="PulseInvite" || action=="PulseCancel" || action=="PulseShareResults" || action=="PulseCreateDraft" || action=="PulseDeleteDraft"
94+
95+
# Removing unused fields
96+
- delete:
97+
fields:
98+
- log.AppAccessContext'
99+
WHERE id=1527;
100+
]]>
101+
</sql>
102+
</changeSet>
103+
</databaseChangeLog>

0 commit comments

Comments
 (0)