diff --git a/.script/tests/KqlvalidationsTests/CustomTables/Fivetran_AuditTrail_CL.json b/.script/tests/KqlvalidationsTests/CustomTables/Fivetran_AuditTrail_CL.json new file mode 100644 index 00000000000..fccb43f58de --- /dev/null +++ b/.script/tests/KqlvalidationsTests/CustomTables/Fivetran_AuditTrail_CL.json @@ -0,0 +1,17 @@ +{ + "Name": "Fivetran_AuditTrail_CL", + "Properties": [ + { "Name": "TimeGenerated", "Type": "DateTime" }, + { "Name": "id", "Type": "String" }, + { "Name": "captured_at", "Type": "DateTime" }, + { "Name": "user_id", "Type": "String" }, + { "Name": "action", "Type": "String" }, + { "Name": "interaction_method", "Type": "String" }, + { "Name": "primary_resource_type", "Type": "String" }, + { "Name": "primary_resource_id", "Type": "String" }, + { "Name": "secondary_resource_type", "Type": "String" }, + { "Name": "secondary_resource_id", "Type": "String" }, + { "Name": "old_values", "Type": "Dynamic" }, + { "Name": "new_values", "Type": "Dynamic" } + ] +} diff --git a/.script/tests/KqlvalidationsTests/CustomTables/Fivetran_CL.json b/.script/tests/KqlvalidationsTests/CustomTables/Fivetran_CL.json new file mode 100644 index 00000000000..59400ba03d8 --- /dev/null +++ b/.script/tests/KqlvalidationsTests/CustomTables/Fivetran_CL.json @@ -0,0 +1,21 @@ +{ + "Name": "Fivetran_CL", + "Properties": [ + { + "Name": "TimeGenerated", + "Type": "DateTime" + }, + { + "Name": "Level", + "Type": "String" + }, + { + "Name": "CreatedAt", + "Type": "DateTime" + }, + { + "Name": "Message", + "Type": "String" + } + ] +} diff --git a/.script/tests/KqlvalidationsTests/CustomTables/Fivetran_Platform_CL.json b/.script/tests/KqlvalidationsTests/CustomTables/Fivetran_Platform_CL.json new file mode 100644 index 00000000000..e179c2f64cb --- /dev/null +++ b/.script/tests/KqlvalidationsTests/CustomTables/Fivetran_Platform_CL.json @@ -0,0 +1,8 @@ +{ + "Name": "Fivetran_Platform_CL", + "Properties": [ + { "Name": "TimeGenerated", "Type": "DateTime" }, + { "Name": "FivetranTable", "Type": "String" }, + { "Name": "Record", "Type": "Dynamic" } + ] +} diff --git a/.script/tests/detectionTemplateSchemaValidation/ValidConnectorIds.json b/.script/tests/detectionTemplateSchemaValidation/ValidConnectorIds.json index 658cbbb7015..3518d02a486 100644 --- a/.script/tests/detectionTemplateSchemaValidation/ValidConnectorIds.json +++ b/.script/tests/detectionTemplateSchemaValidation/ValidConnectorIds.json @@ -328,5 +328,6 @@ "TailscaleCCF", "TailscalePremiumCCF", "WhisperSecurityConnector", - "VaronisSaaSAlertsPush" + "VaronisSaaSAlertsPush", + "Fivetran" ] diff --git a/Logos/Fivetran.svg b/Logos/Fivetran.svg new file mode 100644 index 00000000000..ad508afd23e --- /dev/null +++ b/Logos/Fivetran.svg @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/Solutions/Fivetran/Analytic Rules/FivetranAuthFailures.yaml b/Solutions/Fivetran/Analytic Rules/FivetranAuthFailures.yaml new file mode 100644 index 00000000000..717fc49b7ad --- /dev/null +++ b/Solutions/Fivetran/Analytic Rules/FivetranAuthFailures.yaml @@ -0,0 +1,45 @@ +id: bbfbff4e-134d-4e21-a56b-93bc1de7cc2c +name: Fivetran - Repeated authentication failures +description: 'Detects an unusually high number of Fivetran log events referencing authentication, authorization or credential failures within a short window. Repeated failures can indicate brute-force or credential-stuffing attempts against the accounts Fivetran uses, credential theft/misuse, or (more benignly) expired or revoked credentials. The threshold is tuned so routine one-off credential expiry does not alert.' +kind: Scheduled +severity: Medium +status: Available +requiredDataConnectors: + - connectorId: Fivetran + dataTypes: + - Fivetran_CL +queryFrequency: 1h +queryPeriod: 1h +triggerOperator: GreaterThan +triggerThreshold: 0 +tactics: + - CredentialAccess +relevantTechniques: + - T1110 +query: | + let threshold = 10; + Fivetran_CL + | where Level in ("SEVERE", "WARNING") + | where Message has_any ( + "authentication failed", "authentication error", "unauthorized", + "invalid credentials", "invalid token", "access denied", + "permission denied", "forbidden", "401", "403", + "token expired", "credential", "auth failure", "login failed") + | summarize FailureCount = count(), SampleMessages = make_set(Message, 5), + FirstSeen = min(TimeGenerated), LastSeen = max(TimeGenerated) + by bin(TimeGenerated, 1h) + | where FailureCount > threshold + | sort by FailureCount desc + | extend Application = "Fivetran" +eventGroupingSettings: + aggregationKind: AlertPerResult +customDetails: + FailureCount: FailureCount + FirstSeen: FirstSeen + LastSeen: LastSeen +entityMappings: + - entityType: CloudApplication + fieldMappings: + - identifier: Name + columnName: Application +version: 1.0.0 diff --git a/Solutions/Fivetran/Analytic Rules/FivetranIngestionGap.yaml b/Solutions/Fivetran/Analytic Rules/FivetranIngestionGap.yaml new file mode 100644 index 00000000000..439556f0504 --- /dev/null +++ b/Solutions/Fivetran/Analytic Rules/FivetranIngestionGap.yaml @@ -0,0 +1,39 @@ +id: 5f539c60-f2ed-4a22-b779-75a50ea1b919 +name: Fivetran - Log ingestion gap (telemetry loss) +description: 'Alerts when no Fivetran logs have been received in Fivetran_CL for more than 2 hours. A gap can indicate a broken Fivetran-to-Sentinel pipeline, disabled logging, expired app credentials, or deliberately impaired security telemetry.' +kind: Scheduled +severity: Medium +status: Available +requiredDataConnectors: + - connectorId: Fivetran + dataTypes: + - Fivetran_CL +queryFrequency: 1h +queryPeriod: 14d +triggerOperator: GreaterThan +triggerThreshold: 0 +tactics: + - DefenseEvasion +relevantTechniques: + - T1562 +query: | + let lookback = 14d; + let gapThreshold = 2h; + Fivetran_CL + | where TimeGenerated > ago(lookback) + | summarize LastLogReceived = max(TimeGenerated) + | extend GapDuration = now() - LastLogReceived + | where GapDuration > gapThreshold + | extend Reason = strcat("No Fivetran logs received for ", format_timespan(GapDuration, 'dd.hh:mm:ss')) + | extend Application = "Fivetran" +eventGroupingSettings: + aggregationKind: SingleAlert +customDetails: + LastLogReceived: LastLogReceived + Reason: Reason +entityMappings: + - entityType: CloudApplication + fieldMappings: + - identifier: Name + columnName: Application +version: 1.0.0 diff --git a/Solutions/Fivetran/Data Connectors/Fivetran_CCF/Fivetran_DCR.json b/Solutions/Fivetran/Data Connectors/Fivetran_CCF/Fivetran_DCR.json new file mode 100644 index 00000000000..b14fdaf286a --- /dev/null +++ b/Solutions/Fivetran/Data Connectors/Fivetran_CCF/Fivetran_DCR.json @@ -0,0 +1,36 @@ +[ + { + "name": "fivetranDCR1", + "apiVersion": "2022-06-01", + "type": "Microsoft.Insights/dataCollectionRules", + "location": "{{location}}", + "kind": "Direct", + "properties": { + "streamDeclarations": { + "Custom-Fivetran_CL": { + "columns": [ + { "name": "Level", "type": "string" }, + { "name": "CreatedAt", "type": "datetime" }, + { "name": "Message", "type": "string" } + ] + } + }, + "destinations": { + "logAnalytics": [ + { + "workspaceResourceId": "{{workspaceResourceId}}", + "name": "clv2ws1" + } + ] + }, + "dataFlows": [ + { + "streams": [ "Custom-Fivetran_CL" ], + "destinations": [ "clv2ws1" ], + "outputStream": "Custom-Fivetran_CL", + "transformKql": "source | extend TimeGenerated = CreatedAt" + } + ] + } + } +] diff --git a/Solutions/Fivetran/Data Connectors/Fivetran_CCF/Fivetran_DataConnector.json b/Solutions/Fivetran/Data Connectors/Fivetran_CCF/Fivetran_DataConnector.json new file mode 100644 index 00000000000..ca5ecce5d9c --- /dev/null +++ b/Solutions/Fivetran/Data Connectors/Fivetran_CCF/Fivetran_DataConnector.json @@ -0,0 +1,95 @@ +{ + "id": "Fivetran", + "title": "Fivetran Platform Logs", + "publisher": "Michael Woods", + "descriptionMarkdown": "Ingests Fivetran platform and connector logs into Microsoft Sentinel via the Azure Monitor Logs Ingestion API (DCR-based custom table). Fivetran PUSHES its own logs to a Data Collection Rule using an Entra ID app; this tile shows connection status and provides sample queries. It does not itself move data - ingestion is driven by Fivetran plus the DCR.", + "graphQueries": [ + { + "metricName": "Total Fivetran log events received", + "legend": "Fivetran_CL", + "baseQuery": "Fivetran_CL" + } + ], + "sampleQueries": [ + { + "description": "All Fivetran log events (latest first)", + "query": "Fivetran_CL\n| sort by TimeGenerated desc" + }, + { + "description": "Errors and warnings only", + "query": "Fivetran_CL\n| where Level in (\"SEVERE\", \"WARNING\")\n| sort by TimeGenerated desc" + }, + { + "description": "Event volume by level over time", + "query": "Fivetran_CL\n| summarize count() by Level, bin(TimeGenerated, 1h)" + } + ], + "dataTypes": [ + { + "name": "Fivetran_CL", + "lastDataReceivedQuery": "Fivetran_CL\n| summarize Time = max(TimeGenerated)\n| where isnotempty(Time)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "Fivetran_CL\n| summarize LastLogReceived = max(TimeGenerated)\n| project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": false + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ] + }, + "instructionSteps": [ + { + "title": "1. Register an Entra ID app for Fivetran", + "description": "In Microsoft Entra ID > App registrations, create an app and a client secret. Record the Application (Client) ID, Directory (Tenant) ID and secret value." + }, + { + "title": "2. Create the custom table, DCE and DCR", + "description": "Deploy Fivetran_CL (columns TimeGenerated, Level, CreatedAt, Message) and a Data Collection Rule with stream Custom-Fivetran_CL and transform `source | extend TimeGenerated = CreatedAt`. See the michaelwoods-fivetran skill azure-setup.md." + }, + { + "title": "3. Grant the app access to the DCR", + "description": "Assign the Entra app the Monitoring Metrics Publisher role on the DCR." + }, + { + "title": "4. Configure Fivetran", + "description": "In Fivetran's external-log (Azure Monitor) setup form, choose the Logs Ingestion API mode and enter the Client ID, Tenant ID, Client secret, Logs Ingestion URI (or DCR logIngestion endpoint), DCR immutableId and stream name Custom-Fivetran_CL. Click Save & Test." + } + ], + "metadata": { + "id": "138ff805-42c0-4c6c-b687-df18187a20fe", + "version": "1.0.0", + "kind": "dataConnector", + "source": { "kind": "community" }, + "author": { "name": "Michael Woods" }, + "support": { "name": "Michael Woods", "tier": "developer" } + } +} diff --git a/Solutions/Fivetran/Data Connectors/Fivetran_CCF/Fivetran_Table.json b/Solutions/Fivetran/Data Connectors/Fivetran_CCF/Fivetran_Table.json new file mode 100644 index 00000000000..515b00f2735 --- /dev/null +++ b/Solutions/Fivetran/Data Connectors/Fivetran_CCF/Fivetran_Table.json @@ -0,0 +1,37 @@ +[ + { + "name": "Fivetran_CL", + "type": "Microsoft.OperationalInsights/workspaces/tables", + "apiVersion": "2021-12-01-preview", + "properties": { + "totalRetentionInDays": 365, + "plan": "Analytics", + "schema": { + "name": "Fivetran_CL", + "description": "Fivetran platform and connector logs pushed via the Azure Monitor Logs Ingestion API (DCR-based custom table).", + "columns": [ + { + "name": "TimeGenerated", + "type": "DateTime", + "description": "Log Analytics ingestion timestamp, derived from CreatedAt in the DCR transform." + }, + { + "name": "Level", + "type": "String", + "description": "Log level reported by Fivetran (INFO, WARNING, SEVERE)." + }, + { + "name": "CreatedAt", + "type": "DateTime", + "description": "Timestamp of the log event as sent by Fivetran." + }, + { + "name": "Message", + "type": "String", + "description": "Log message body." + } + ] + } + } + } +] diff --git a/Solutions/Fivetran/Data/Solution_Fivetran.json b/Solutions/Fivetran/Data/Solution_Fivetran.json new file mode 100644 index 00000000000..2b4dd9b26ca --- /dev/null +++ b/Solutions/Fivetran/Data/Solution_Fivetran.json @@ -0,0 +1,31 @@ +{ + "Name": "Fivetran", + "Author": "Michael Woods - michaelwoods@microsoft.com", + "Logo": "", + "Description": "The Fivetran Solution for Microsoft Sentinel ingests Fivetran platform and connector logs into a DCR-based custom table using the Azure Monitor Logs Ingestion API. Fivetran pushes its own logs directly to a Data Collection Rule; this solution ships the custom table, DCR, a data-connector tile, a parser, two analytics rules, a hunting query and a workbook to give SecOps visibility of the Fivetran data pipeline.", + "Data Connectors": [ + "Data Connectors/Fivetran_CCF/Fivetran_DataConnector.json" + ], + "Parsers": [ + "Parsers/Fivetran.yaml", + "Parsers/ASimAuthenticationFivetran.yaml", + "Parsers/vimAuthenticationFivetran.yaml", + "Parsers/ASimAuditEventFivetran.yaml", + "Parsers/vimAuditEventFivetran.yaml" + ], + "Analytic Rules": [ + "Analytic Rules/FivetranIngestionGap.yaml", + "Analytic Rules/FivetranAuthFailures.yaml" + ], + "Hunting Queries": [ + "Hunting Queries/FivetranSevereSpike.yaml" + ], + "Workbooks": [ + "Workbooks/Fivetran.json" + ], + "BasePath": "C:\\GitHub\\Azure-Sentinel\\Solutions\\Fivetran", + "Version": "3.0.1", + "Metadata": "SolutionMetadata.json", + "TemplateSpec": true, + "Is1PConnector": false +} diff --git a/Solutions/Fivetran/Hunting Queries/FivetranSevereSpike.yaml b/Solutions/Fivetran/Hunting Queries/FivetranSevereSpike.yaml new file mode 100644 index 00000000000..d037f248bdd --- /dev/null +++ b/Solutions/Fivetran/Hunting Queries/FivetranSevereSpike.yaml @@ -0,0 +1,18 @@ +id: 2de1fd60-97f4-4072-b78a-94dc7b047a63 +name: Fivetran - Spike in SEVERE log events +description: 'Surfaces hourly windows with an unusually high number of Fivetran SEVERE-level events, which can indicate connector failures, credential problems, or data-pipeline tampering.' +requiredDataConnectors: + - connectorId: Fivetran + dataTypes: + - Fivetran_CL +tactics: + - DefenseEvasion +relevantTechniques: + - T1562 +query: | + Fivetran_CL + | where Level == "SEVERE" + | summarize SevereCount = count(), SampleMessages = make_set(Message, 5) by bin(TimeGenerated, 1h) + | where SevereCount > 10 + | sort by TimeGenerated desc +version: 1.0.0 diff --git a/Solutions/Fivetran/MITRE-COVERAGE.md b/Solutions/Fivetran/MITRE-COVERAGE.md new file mode 100644 index 00000000000..c9d374c2fb9 --- /dev/null +++ b/Solutions/Fivetran/MITRE-COVERAGE.md @@ -0,0 +1,49 @@ +# MITRE ATT&CK coverage - Fivetran solution + +Verified 2026-07-22 against +https://learn.microsoft.com/en-us/azure/sentinel/mitre-coverage (Sentinel aligns +to **ATT&CK v18**) and the live ATT&CK knowledge base. + +## Mappings + +| Artifact | Tactic | Technique | Why it is appropriate | +| --- | --- | --- | --- | +| `Analytic Rules/FivetranIngestionGap.yaml` | Defense Evasion | **T1562** (Impair Defenses) | A stop in Fivetran log ingestion = loss/impairment of security telemetry, the core behaviour T1562 covers (disrupting logging/forwarding pipelines and SIEM ingestion). | +| `Analytic Rules/FivetranAuthFailures.yaml` | Credential Access | **T1110** (Brute Force) | A spike of authentication/authorization/credential failures in the Fivetran log stream matches the brute-force / credential-stuffing pattern against the accounts Fivetran uses. Threshold-based (>10/hour) so routine one-off credential expiry does not alert. Same mapping used by comparable Content Hub rules (Bitglass MultipleFailedLogins, GitLab BruteForce). | +| `Hunting Queries/FivetranSevereSpike.yaml` | Defense Evasion | **T1562** (Impair Defenses) | A surge of SEVERE pipeline errors can indicate the log source being degraded or tampered with. Corrected from the initial `Impact`/`T1499` (Endpoint DoS), which specifically means resource-exhaustion/flooding and does not describe pipeline errors. | + +The detections cover two real, distinct risks: **protecting the integrity and +availability of the Fivetran security log source** (Defense Evasion / T1562) and +**detecting credential attacks against the Fivetran service accounts** (Credential +Access / T1110). Accuracy was prioritised over breadth - no technique is claimed +that the query does not actually evidence. + +### Honesty note on T1110 + +The only signal fields Fivetran emits are `Level`, `CreatedAt` and `Message`; +there is no structured account or source-IP field, so no entity mappings are +asserted. The T1110 claim rests on a *volume threshold* of failure-keyword +messages, which is the defensible brute-force signal. A low count could also be +benign credential expiry, which is why the rule only fires above the threshold. + +## Important version note (why we use T1562, not T1685) + +In the current MITRE ATT&CK knowledge base, **Impair Defenses was renumbered from +`T1562` to `T1685`** (the old `T1562` and `T1562.008` URLs now redirect to +`T1685` / `T1685/002`). This is verifiable directly at +https://attack.mitre.org/techniques/T1562/ which issues an HTTP redirect to +https://attack.mitre.org/techniques/T1685/ (checked 2026-07-26). + +However, **Microsoft Sentinel and the Azure-Sentinel Content Hub tooling still use +`T1562`** - every current solution rule maps to `T1562` and there are zero uses of +`T1685` in the repo. Using `T1685` today would fail solution validation and would +not light up the Sentinel MITRE coverage matrix. Revisit if/when Sentinel tooling +adopts the `T1685` numbering. + +## How to confirm coverage after deployment + +In the Defender portal: **Microsoft Sentinel > Threat management > MITRE ATT&CK**. +Search technique `T1562`; once the ingestion-gap analytics rule is active (and the +hunting query present) the tile shows active/simulated coverage for Defense Evasion +- Impair Defenses. Search `T1110`; once the auth-failures analytics rule is active +the tile shows coverage for Credential Access - Brute Force. diff --git a/Solutions/Fivetran/PACKAGING.md b/Solutions/Fivetran/PACKAGING.md new file mode 100644 index 00000000000..f306863bbe6 --- /dev/null +++ b/Solutions/Fivetran/PACKAGING.md @@ -0,0 +1,94 @@ +# Packaging the Fivetran solution (Content Hub) + +The `Package/` folder (`mainTemplate.json`, `createUiDefinition.json` and the +versioned `.zip`) is **generated by the Azure-Sentinel V3 packaging +tool** from the source artifacts in this folder. Do not hand-write mainTemplate. + +## Source files you author (already present here) + +- `SolutionMetadata.json` +- `Data/Solution_Fivetran.json` (manifest listing every artifact) +- `ReleaseNotes.md` +- `Data Connectors/`, `Parsers/`, `Analytic Rules/`, `Hunting Queries/`, `Workbooks/` + +## Status: the package IS generated + +`Package/mainTemplate.json`, `Package/createUiDefinition.json` and +`Package/3.0.0.zip` (version 3.0.0) are present in this folder. They were produced +by the official V3 tool `createSolutionV3.ps1` and validated with ARM-TTK +(**48/49 pass**; the single failure is the documented false positive +`IDs Should Be Derived From ResourceIDs` on the auto-generated `contentProductId` +/ `id` content-metadata fields, which the Azure-Sentinel pipeline itself ignores - +see `.github/actions/entrypoint.ps1` and the V3 tool README). Only regenerate if +you change the source artifacts. + +## Generate the package (canonical: full clone) + +Prerequisites: **PowerShell 7.1+**, **Node.js**, the `powershell-yaml` module, and +a clone of `github.com/Azure/Azure-Sentinel` (the tool dot-sources shared modules +from `Tools/Create-Azure-Sentinel-Solution/common` and `.script/package-automation`). + +1. Copy this `Fivetran` folder into the clone under `Solutions/`. +2. Update `BasePath` in `Data/Solution_Fivetran.json` to the repo path, + e.g. `C:\GitHub\Azure-Sentinel\Solutions\Fivetran`. +3. `Install-Module powershell-yaml` +4. From the repo root run: + +```powershell +pwsh Tools/Create-Azure-Sentinel-Solution/V3/createSolutionV3.ps1 ` + -SolutionDataFolderPath "\Solutions\Fivetran\Data" ` + -VersionMode local -VersionBump patch +``` + +`-VersionMode local` skips the Partner Center catalog API (use it for a private / +first build). The tool writes `Package/mainTemplate.json`, +`Package/createUiDefinition.json` and `Package/.zip`. + +## Generate the package (no git clone - verified standalone method) + +The tool does NOT need the whole repo, only a few folders arranged as a minimal +repo root. This is how the current `Package/` was produced on a machine without +git initially installed: + +1. Install PowerShell 7 (`winget install Microsoft.PowerShell`) and, for the + metadata step, git (`winget install Git.Git`). `Install-Module powershell-yaml`. +2. Fetch just the needed subtrees with degit (no full clone): + +```powershell +npx --yes degit Azure/Azure-Sentinel/Tools/Create-Azure-Sentinel-Solution/V3 v3 +npx --yes degit --force Azure/Azure-Sentinel/Tools/Create-Azure-Sentinel-Solution/common common +npx --yes degit --force Azure/Azure-Sentinel/.script/package-automation pkgauto +npx --yes degit --force Azure/Azure-Sentinel/Tools/Create-Azure-Sentinel-Solution/arm-ttk armttk +``` + +3. Assemble a fake repo root: + `repo/Tools/Create-Azure-Sentinel-Solution/{V3,common,arm-ttk}`, + `repo/.script/package-automation`, `repo/Solutions/Fivetran/...`, + and a `repo/Workbooks/WorkbooksMetadata.json` containing one entry whose + `templateRelativePath` matches the workbook filename (`Fivetran.json`). +4. Set `BasePath` in the copied `Data` file to the fake `repo\Solutions\Fivetran`. +5. `git init` + one commit in `repo/` (the metadata step calls `git log` for dates), + ensure git is on PATH, and run the tool from the repo root (`Set-Location repo`) + with `-VersionMode local`. + +Gotchas that will bite you: (a) run pwsh from the repo root or `git log` throws +`not a git repository`; (b) missing `Workbooks/WorkbooksMetadata.json` aborts the +workbook step; (c) missing `arm-ttk/download-arm-ttk.ps1` aborts final validation; +(d) each `local` run increments the version - to land on a clean `3.0.0`, set the +Data file `Version` to `2.0.0` and run once with `-VersionBump major`. + +## Validate before submitting + +- ARM-TTK runs automatically at the end of the tool (`Test-AzTemplate` on + `Package/`). Treat the `contentProductId`/`id` `IDs Should Be Derived From + ResourceIDs` failure as an expected false positive. +- All JSON parses and all YAML is well-formed (see the skill REVIEW step). +- GUIDs in analytics rule / hunting query / connector metadata are unique. +- Run the repo's solution validation (`.script/` checks) if publishing a PR to + Content Hub. + +## Logos + +Content Hub expects an SVG under `Logos/` referenced from `SolutionMetadata.json` +and `createUiDefinition.json`. The manifest currently references +`Logos/Fivetran.svg`; add that asset (or switch to a Michael Woods logo) before publishing. diff --git a/Solutions/Fivetran/Package/3.0.1.zip b/Solutions/Fivetran/Package/3.0.1.zip new file mode 100644 index 00000000000..9302fd5daea Binary files /dev/null and b/Solutions/Fivetran/Package/3.0.1.zip differ diff --git a/Solutions/Fivetran/Package/createUiDefinition.json b/Solutions/Fivetran/Package/createUiDefinition.json new file mode 100644 index 00000000000..949242d54ea --- /dev/null +++ b/Solutions/Fivetran/Package/createUiDefinition.json @@ -0,0 +1,228 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/0.1.2-preview/CreateUIDefinition.MultiVm.json#", + "handler": "Microsoft.Azure.CreateUIDef", + "version": "0.1.2-preview", + "parameters": { + "config": { + "isWizard": false, + "basics": { + "description": "\n\n**Note:** Please refer to the following before installing the solution: \n\n• Review the solution [Release Notes](https://github.com/Azure/Azure-Sentinel/tree/master/Solutions/Fivetran/ReleaseNotes.md)\n\n • There may be [known issues](https://aka.ms/sentinelsolutionsknownissues) pertaining to this Solution, please refer to them before installing.\n\nThe Fivetran Solution for Microsoft Sentinel ingests Fivetran platform and connector logs into a DCR-based custom table using the Azure Monitor Logs Ingestion API. Fivetran pushes its own logs directly to a Data Collection Rule; this solution ships the custom table, DCR, a data-connector tile, a parser, two analytics rules, a hunting query and a workbook to give SecOps visibility of the Fivetran data pipeline.\n\n**Data Connectors:** 1, **Parsers:** 5, **Workbooks:** 1, **Analytic Rules:** 2, **Hunting Queries:** 1\n\n[Learn more about Microsoft Sentinel](https://aka.ms/azuresentinel) | [Learn more about Solutions](https://aka.ms/azuresentinelsolutionsdoc)", + "subscription": { + "resourceProviders": [ + "Microsoft.OperationsManagement/solutions", + "Microsoft.OperationalInsights/workspaces/providers/alertRules", + "Microsoft.Insights/workbooks", + "Microsoft.Logic/workflows" + ] + }, + "location": { + "metadata": { + "hidden": "Hiding location, we get it from the log analytics workspace" + }, + "visible": false + }, + "resourceGroup": { + "allowExisting": true + } + } + }, + "basics": [ + { + "name": "getLAWorkspace", + "type": "Microsoft.Solutions.ArmApiControl", + "toolTip": "This filters by workspaces that exist in the Resource Group selected", + "condition": "[greater(length(resourceGroup().name),0)]", + "request": { + "method": "GET", + "path": "[concat(subscription().id,'/providers/Microsoft.OperationalInsights/workspaces?api-version=2020-08-01')]" + } + }, + { + "name": "workspace", + "type": "Microsoft.Common.DropDown", + "label": "Workspace", + "placeholder": "Select a workspace", + "toolTip": "This dropdown will list only workspace that exists in the Resource Group selected", + "constraints": { + "allowedValues": "[map(filter(basics('getLAWorkspace').value, (filter) => contains(toLower(filter.id), toLower(resourceGroup().name))), (item) => parse(concat('{\"label\":\"', item.name, '\",\"value\":\"', item.name, '\"}')))]", + "required": true + }, + "visible": true + } + ], + "steps": [ + { + "name": "dataconnectors", + "label": "Data Connectors", + "bladeTitle": "Data Connectors", + "elements": [ + { + "name": "dataconnectors1-text", + "type": "Microsoft.Common.TextBlock", + "options": { + "text": "This Solution installs the data connector for Fivetran. You can get Fivetran custom log data in your Microsoft Sentinel workspace. After installing the solution, configure and enable this data connector by following guidance in Manage solution view." + } + }, + { + "name": "dataconnectors-parser-text", + "type": "Microsoft.Common.TextBlock", + "options": { + "text": "The Solution installs a parser that transforms the ingested data into Microsoft Sentinel normalized format. The normalized format enables better correlation of different types of data from different data sources to drive end-to-end outcomes seamlessly in security monitoring, hunting, incident investigation and response scenarios in Microsoft Sentinel." + } + }, + { + "name": "dataconnectors-link1", + "type": "Microsoft.Common.TextBlock", + "options": { + "link": { + "label": "Learn more about connecting data sources", + "uri": "https://docs.microsoft.com/azure/sentinel/connect-data-sources" + } + } + } + ] + }, + { + "name": "workbooks", + "label": "Workbooks", + "subLabel": { + "preValidation": "Configure the workbooks", + "postValidation": "Done" + }, + "bladeTitle": "Workbooks", + "elements": [ + { + "name": "workbooks-text", + "type": "Microsoft.Common.TextBlock", + "options": { + "text": "This solution installs workbook(s) to help you gain insights into the telemetry collected in Microsoft Sentinel. After installing the solution, start using the workbook in Manage solution view." + } + }, + { + "name": "workbooks-link", + "type": "Microsoft.Common.TextBlock", + "options": { + "link": { + "label": "Learn more", + "uri": "https://docs.microsoft.com/azure/sentinel/tutorial-monitor-your-data" + } + } + }, + { + "name": "workbook1", + "type": "Microsoft.Common.Section", + "label": "Fivetran", + "elements": [ + { + "name": "workbook1-text", + "type": "Microsoft.Common.TextBlock", + "options": { + "text": "Visualizes Fivetran platform and connector log volume, severity breakdown and ingestion health." + } + } + ] + } + ] + }, + { + "name": "analytics", + "label": "Analytics", + "subLabel": { + "preValidation": "Configure the analytics", + "postValidation": "Done" + }, + "bladeTitle": "Analytics", + "elements": [ + { + "name": "analytics-text", + "type": "Microsoft.Common.TextBlock", + "options": { + "text": "This solution installs the following analytic rule templates. After installing the solution, create and enable analytic rules in Manage solution view." + } + }, + { + "name": "analytics-link", + "type": "Microsoft.Common.TextBlock", + "options": { + "link": { + "label": "Learn more", + "uri": "https://docs.microsoft.com/azure/sentinel/tutorial-detect-threats-custom?WT.mc_id=Portal-Microsoft_Azure_CreateUIDef" + } + } + }, + { + "name": "analytic1", + "type": "Microsoft.Common.Section", + "label": "Fivetran - Log ingestion gap (telemetry loss)", + "elements": [ + { + "name": "analytic1-text", + "type": "Microsoft.Common.TextBlock", + "options": { + "text": "Alerts when no Fivetran logs have been received in Fivetran_CL for more than 2 hours. A gap can indicate a broken Fivetran-to-Sentinel pipeline, disabled logging, expired app credentials, or deliberately impaired security telemetry." + } + } + ] + }, + { + "name": "analytic2", + "type": "Microsoft.Common.Section", + "label": "Fivetran - Repeated authentication failures", + "elements": [ + { + "name": "analytic2-text", + "type": "Microsoft.Common.TextBlock", + "options": { + "text": "Detects an unusually high number of Fivetran log events referencing authentication, authorization or credential failures within a short window. Repeated failures can indicate brute-force or credential-stuffing attempts against the accounts Fivetran uses, credential theft/misuse, or (more benignly) expired or revoked credentials. The threshold is tuned so routine one-off credential expiry does not alert." + } + } + ] + } + ] + }, + { + "name": "huntingqueries", + "label": "Hunting Queries", + "bladeTitle": "Hunting Queries", + "elements": [ + { + "name": "huntingqueries-text", + "type": "Microsoft.Common.TextBlock", + "options": { + "text": "This solution installs the following hunting queries. After installing the solution, run these hunting queries to hunt for threats in Manage solution view. " + } + }, + { + "name": "huntingqueries-link", + "type": "Microsoft.Common.TextBlock", + "options": { + "link": { + "label": "Learn more", + "uri": "https://docs.microsoft.com/azure/sentinel/hunting" + } + } + }, + { + "name": "huntingquery1", + "type": "Microsoft.Common.Section", + "label": "Fivetran - Spike in SEVERE log events", + "elements": [ + { + "name": "huntingquery1-text", + "type": "Microsoft.Common.TextBlock", + "options": { + "text": "Surfaces hourly windows with an unusually high number of Fivetran SEVERE-level events, which can indicate connector failures, credential problems, or data-pipeline tampering. This hunting query depends on Fivetran data connector (Fivetran_CL Parser or Table)" + } + } + ] + } + ] + } + ], + "outputs": { + "workspace-location": "[first(map(filter(basics('getLAWorkspace').value, (filter) => and(contains(toLower(filter.id), toLower(resourceGroup().name)),equals(filter.name,basics('workspace')))), (item) => item.location))]", + "location": "[location()]", + "workspace": "[basics('workspace')]" + } + } +} diff --git a/Solutions/Fivetran/Package/mainTemplate.json b/Solutions/Fivetran/Package/mainTemplate.json new file mode 100644 index 00000000000..713f15e1143 --- /dev/null +++ b/Solutions/Fivetran/Package/mainTemplate.json @@ -0,0 +1,1572 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "author": "Michael Woods - michaelwoods@microsoft.com", + "comments": "Solution template for Fivetran" + }, + "parameters": { + "location": { + "type": "string", + "minLength": 1, + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "Not used, but needed to pass arm-ttk test `Location-Should-Not-Be-Hardcoded`. We instead use the `workspace-location` which is derived from the LA workspace" + } + }, + "workspace-location": { + "type": "string", + "defaultValue": "", + "metadata": { + "description": "[concat('Region to deploy solution resources -- separate from location selection',parameters('location'))]" + } + }, + "workspace": { + "defaultValue": "", + "type": "string", + "metadata": { + "description": "Workspace name for Log Analytics where Microsoft Sentinel is setup" + } + }, + "workbook1-name": { + "type": "string", + "defaultValue": "Fivetran", + "minLength": 1, + "metadata": { + "description": "Name for the workbook" + } + } + }, + "variables": { + "email": "michaelwoods@microsoft.com", + "_email": "[variables('email')]", + "_solutionName": "Fivetran", + "_solutionVersion": "3.0.1", + "solutionId": "michaelwoods.azure-sentinel-solution-fivetran", + "_solutionId": "[variables('solutionId')]", + "uiConfigId1": "Fivetran", + "_uiConfigId1": "[variables('uiConfigId1')]", + "dataConnectorContentId1": "Fivetran", + "_dataConnectorContentId1": "[variables('dataConnectorContentId1')]", + "dataConnectorId1": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId1'))]", + "_dataConnectorId1": "[variables('dataConnectorId1')]", + "dataConnectorTemplateSpecName1": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat(parameters('workspace'),'-dc-',uniquestring(variables('_dataConnectorContentId1'))))]", + "dataConnectorVersion1": "1.0.0", + "_dataConnectorcontentProductId1": "[concat(take(variables('_solutionId'),50),'-','dc','-', uniqueString(concat(variables('_solutionId'),'-','DataConnector','-',variables('_dataConnectorContentId1'),'-', variables('dataConnectorVersion1'))))]", + "parserObject1": { + "_parserName1": "[concat(parameters('workspace'),'/','Fivetran')]", + "_parserId1": "[resourceId('Microsoft.OperationalInsights/workspaces/savedSearches', parameters('workspace'), 'Fivetran')]", + "parserTemplateSpecName1": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat(parameters('workspace'),'-pr-',uniquestring('Fivetran-Parser')))]", + "parserVersion1": "1.0.0", + "parserContentId1": "Fivetran-Parser" + }, + "parserObject2": { + "_parserName2": "[concat(parameters('workspace'),'/','ASimAuthenticationFivetran')]", + "_parserId2": "[resourceId('Microsoft.OperationalInsights/workspaces/savedSearches', parameters('workspace'), 'ASimAuthenticationFivetran')]", + "parserTemplateSpecName2": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat(parameters('workspace'),'-pr-',uniquestring('ASimAuthenticationFivetran-Parser')))]", + "parserVersion2": "0.1.0", + "parserContentId2": "ASimAuthenticationFivetran-Parser" + }, + "parserObject3": { + "_parserName3": "[concat(parameters('workspace'),'/','vimAuthenticationFivetran')]", + "_parserId3": "[resourceId('Microsoft.OperationalInsights/workspaces/savedSearches', parameters('workspace'), 'vimAuthenticationFivetran')]", + "parserTemplateSpecName3": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat(parameters('workspace'),'-pr-',uniquestring('vimAuthenticationFivetran-Parser')))]", + "parserVersion3": "0.1.0", + "parserContentId3": "vimAuthenticationFivetran-Parser" + }, + "parserObject4": { + "_parserName4": "[concat(parameters('workspace'),'/','ASimAuditEventFivetran')]", + "_parserId4": "[resourceId('Microsoft.OperationalInsights/workspaces/savedSearches', parameters('workspace'), 'ASimAuditEventFivetran')]", + "parserTemplateSpecName4": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat(parameters('workspace'),'-pr-',uniquestring('ASimAuditEventFivetran-Parser')))]", + "parserVersion4": "0.1.0", + "parserContentId4": "ASimAuditEventFivetran-Parser" + }, + "parserObject5": { + "_parserName5": "[concat(parameters('workspace'),'/','vimAuditEventFivetran')]", + "_parserId5": "[resourceId('Microsoft.OperationalInsights/workspaces/savedSearches', parameters('workspace'), 'vimAuditEventFivetran')]", + "parserTemplateSpecName5": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat(parameters('workspace'),'-pr-',uniquestring('vimAuditEventFivetran-Parser')))]", + "parserVersion5": "0.1.0", + "parserContentId5": "vimAuditEventFivetran-Parser" + }, + "analyticRuleObject1": { + "analyticRuleVersion1": "1.0.0", + "_analyticRulecontentId1": "5f539c60-f2ed-4a22-b779-75a50ea1b919", + "analyticRuleId1": "[resourceId('Microsoft.SecurityInsights/AlertRuleTemplates', '5f539c60-f2ed-4a22-b779-75a50ea1b919')]", + "analyticRuleTemplateSpecName1": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat(parameters('workspace'),'-ar-',uniquestring('5f539c60-f2ed-4a22-b779-75a50ea1b919')))]", + "_analyticRulecontentProductId1": "[concat(take(variables('_solutionId'),50),'-','ar','-', uniqueString(concat(variables('_solutionId'),'-','AnalyticsRule','-','5f539c60-f2ed-4a22-b779-75a50ea1b919','-', '1.0.0')))]" + }, + "analyticRuleObject2": { + "analyticRuleVersion2": "1.0.0", + "_analyticRulecontentId2": "bbfbff4e-134d-4e21-a56b-93bc1de7cc2c", + "analyticRuleId2": "[resourceId('Microsoft.SecurityInsights/AlertRuleTemplates', 'bbfbff4e-134d-4e21-a56b-93bc1de7cc2c')]", + "analyticRuleTemplateSpecName2": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat(parameters('workspace'),'-ar-',uniquestring('bbfbff4e-134d-4e21-a56b-93bc1de7cc2c')))]", + "_analyticRulecontentProductId2": "[concat(take(variables('_solutionId'),50),'-','ar','-', uniqueString(concat(variables('_solutionId'),'-','AnalyticsRule','-','bbfbff4e-134d-4e21-a56b-93bc1de7cc2c','-', '1.0.0')))]" + }, + "huntingQueryObject1": { + "huntingQueryVersion1": "1.0.0", + "_huntingQuerycontentId1": "2de1fd60-97f4-4072-b78a-94dc7b047a63", + "huntingQueryTemplateSpecName1": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat(parameters('workspace'),'-hq-',uniquestring('2de1fd60-97f4-4072-b78a-94dc7b047a63')))]" + }, + "workbookVersion1": "1.0.0", + "workbookContentId1": "FivetranWorkbook", + "workbookId1": "[resourceId('Microsoft.Insights/workbooks', variables('workbookContentId1'))]", + "workbookTemplateSpecName1": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat(parameters('workspace'),'-wb-',uniquestring(variables('_workbookContentId1'))))]", + "_workbookContentId1": "[variables('workbookContentId1')]", + "workspaceResourceId": "[resourceId('microsoft.OperationalInsights/Workspaces', parameters('workspace'))]", + "_workbookcontentProductId1": "[concat(take(variables('_solutionId'),50),'-','wb','-', uniqueString(concat(variables('_solutionId'),'-','Workbook','-',variables('_workbookContentId1'),'-', variables('workbookVersion1'))))]", + "_solutioncontentProductId": "[concat(take(variables('_solutionId'),50),'-','sl','-', uniqueString(concat(variables('_solutionId'),'-','Solution','-',variables('_solutionId'),'-', variables('_solutionVersion'))))]" + }, + "resources": [ + { + "type": "Microsoft.OperationalInsights/workspaces/providers/contentTemplates", + "apiVersion": "2023-04-01-preview", + "name": "[variables('dataConnectorTemplateSpecName1')]", + "location": "[parameters('workspace-location')]", + "dependsOn": [ + "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" + ], + "properties": { + "description": "Fivetran data connector with template version 3.0.1", + "mainTemplate": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "[variables('dataConnectorVersion1')]", + "parameters": {}, + "variables": {}, + "resources": [ + { + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',variables('_dataConnectorContentId1'))]", + "apiVersion": "2021-03-01-preview", + "type": "Microsoft.OperationalInsights/workspaces/providers/dataConnectors", + "location": "[parameters('workspace-location')]", + "kind": "GenericUI", + "properties": { + "connectorUiConfig": { + "id": "[variables('_uiConfigId1')]", + "title": "Fivetran Platform Logs", + "publisher": "Michael Woods", + "descriptionMarkdown": "Ingests Fivetran platform and connector logs into Microsoft Sentinel via the Azure Monitor Logs Ingestion API (DCR-based custom table). Fivetran PUSHES its own logs to a Data Collection Rule using an Entra ID app; this tile shows connection status and provides sample queries. It does not itself move data - ingestion is driven by Fivetran plus the DCR.", + "graphQueries": [ + { + "metricName": "Total Fivetran log events received", + "legend": "Fivetran_CL", + "baseQuery": "Fivetran_CL" + } + ], + "sampleQueries": [ + { + "description": "All Fivetran log events (latest first)", + "query": "Fivetran_CL\n| sort by TimeGenerated desc" + }, + { + "description": "Errors and warnings only", + "query": "Fivetran_CL\n| where Level in (\"SEVERE\", \"WARNING\")\n| sort by TimeGenerated desc" + }, + { + "description": "Event volume by level over time", + "query": "Fivetran_CL\n| summarize count() by Level, bin(TimeGenerated, 1h)" + } + ], + "dataTypes": [ + { + "name": "Fivetran_CL", + "lastDataReceivedQuery": "Fivetran_CL\n| summarize Time = max(TimeGenerated)\n| where isnotempty(Time)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "Fivetran_CL\n| summarize LastLogReceived = max(TimeGenerated)\n| project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": false + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ] + }, + "instructionSteps": [ + { + "description": "In Microsoft Entra ID > App registrations, create an app and a client secret. Record the Application (Client) ID, Directory (Tenant) ID and secret value.", + "title": "1. Register an Entra ID app for Fivetran" + }, + { + "description": "Deploy Fivetran_CL (columns TimeGenerated, Level, CreatedAt, Message) and a Data Collection Rule with stream Custom-Fivetran_CL and transform `source | extend TimeGenerated = CreatedAt`. See the michaelwoods-fivetran skill azure-setup.md.", + "title": "2. Create the custom table, DCE and DCR" + }, + { + "description": "Assign the Entra app the Monitoring Metrics Publisher role on the DCR.", + "title": "3. Grant the app access to the DCR" + }, + { + "description": "In Fivetran's external-log (Azure Monitor) setup form, choose the Logs Ingestion API mode and enter the Client ID, Tenant ID, Client secret, Logs Ingestion URI (or DCR logIngestion endpoint), DCR immutableId and stream name Custom-Fivetran_CL. Click Save & Test.", + "title": "4. Configure Fivetran" + } + ], + "metadata": { + "id": "138ff805-42c0-4c6c-b687-df18187a20fe", + "version": "1.0.0", + "kind": "dataConnector", + "source": { + "kind": "community" + }, + "author": { + "name": "Michael Woods" + }, + "support": { + "name": "Michael Woods", + "tier": "developer" + } + } + } + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2023-04-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('DataConnector-', last(split(variables('_dataConnectorId1'),'/'))))]", + "properties": { + "parentId": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId1'))]", + "contentId": "[variables('_dataConnectorContentId1')]", + "kind": "DataConnector", + "version": "[variables('dataConnectorVersion1')]", + "source": { + "kind": "Solution", + "name": "Fivetran", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "Michael Woods", + "email": "[variables('_email')]" + }, + "support": { + "name": "Michael Woods", + "email": "michaelwoods@microsoft.com", + "tier": "Community", + "link": "https://github.com/woodsonsecurity" + } + } + } + ] + }, + "packageKind": "Solution", + "packageVersion": "[variables('_solutionVersion')]", + "packageName": "[variables('_solutionName')]", + "packageId": "[variables('_solutionId')]", + "contentSchemaVersion": "3.0.0", + "contentId": "[variables('_dataConnectorContentId1')]", + "contentKind": "DataConnector", + "displayName": "Fivetran Platform Logs", + "contentProductId": "[variables('_dataConnectorcontentProductId1')]", + "id": "[variables('_dataConnectorcontentProductId1')]", + "version": "[variables('dataConnectorVersion1')]" + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2023-04-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('DataConnector-', last(split(variables('_dataConnectorId1'),'/'))))]", + "dependsOn": [ + "[variables('_dataConnectorId1')]" + ], + "location": "[parameters('workspace-location')]", + "properties": { + "parentId": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId1'))]", + "contentId": "[variables('_dataConnectorContentId1')]", + "kind": "DataConnector", + "version": "[variables('dataConnectorVersion1')]", + "source": { + "kind": "Solution", + "name": "Fivetran", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "Michael Woods", + "email": "[variables('_email')]" + }, + "support": { + "name": "Michael Woods", + "email": "michaelwoods@microsoft.com", + "tier": "Community", + "link": "https://github.com/woodsonsecurity" + } + } + }, + { + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',variables('_dataConnectorContentId1'))]", + "apiVersion": "2021-03-01-preview", + "type": "Microsoft.OperationalInsights/workspaces/providers/dataConnectors", + "location": "[parameters('workspace-location')]", + "kind": "GenericUI", + "properties": { + "connectorUiConfig": { + "title": "Fivetran Platform Logs", + "publisher": "Michael Woods", + "descriptionMarkdown": "Ingests Fivetran platform and connector logs into Microsoft Sentinel via the Azure Monitor Logs Ingestion API (DCR-based custom table). Fivetran PUSHES its own logs to a Data Collection Rule using an Entra ID app; this tile shows connection status and provides sample queries. It does not itself move data - ingestion is driven by Fivetran plus the DCR.", + "graphQueries": [ + { + "metricName": "Total Fivetran log events received", + "legend": "Fivetran_CL", + "baseQuery": "Fivetran_CL" + } + ], + "dataTypes": [ + { + "name": "Fivetran_CL", + "lastDataReceivedQuery": "Fivetran_CL\n| summarize Time = max(TimeGenerated)\n| where isnotempty(Time)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "Fivetran_CL\n| summarize LastLogReceived = max(TimeGenerated)\n| project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "sampleQueries": [ + { + "description": "All Fivetran log events (latest first)", + "query": "Fivetran_CL\n| sort by TimeGenerated desc" + }, + { + "description": "Errors and warnings only", + "query": "Fivetran_CL\n| where Level in (\"SEVERE\", \"WARNING\")\n| sort by TimeGenerated desc" + }, + { + "description": "Event volume by level over time", + "query": "Fivetran_CL\n| summarize count() by Level, bin(TimeGenerated, 1h)" + } + ], + "availability": { + "status": 1, + "isPreview": false + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ] + }, + "instructionSteps": [ + { + "description": "In Microsoft Entra ID > App registrations, create an app and a client secret. Record the Application (Client) ID, Directory (Tenant) ID and secret value.", + "title": "1. Register an Entra ID app for Fivetran" + }, + { + "description": "Deploy Fivetran_CL (columns TimeGenerated, Level, CreatedAt, Message) and a Data Collection Rule with stream Custom-Fivetran_CL and transform `source | extend TimeGenerated = CreatedAt`. See the michaelwoods-fivetran skill azure-setup.md.", + "title": "2. Create the custom table, DCE and DCR" + }, + { + "description": "Assign the Entra app the Monitoring Metrics Publisher role on the DCR.", + "title": "3. Grant the app access to the DCR" + }, + { + "description": "In Fivetran's external-log (Azure Monitor) setup form, choose the Logs Ingestion API mode and enter the Client ID, Tenant ID, Client secret, Logs Ingestion URI (or DCR logIngestion endpoint), DCR immutableId and stream name Custom-Fivetran_CL. Click Save & Test.", + "title": "4. Configure Fivetran" + } + ], + "id": "[variables('_uiConfigId1')]" + } + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/contentTemplates", + "apiVersion": "2023-04-01-preview", + "name": "[variables('parserObject1').parserTemplateSpecName1]", + "location": "[parameters('workspace-location')]", + "dependsOn": [ + "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" + ], + "properties": { + "description": "Fivetran Data Parser with template version 3.0.1", + "mainTemplate": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "[variables('parserObject1').parserVersion1]", + "parameters": {}, + "variables": {}, + "resources": [ + { + "name": "[variables('parserObject1')._parserName1]", + "apiVersion": "2025-07-01", + "type": "Microsoft.OperationalInsights/workspaces/savedSearches", + "location": "[parameters('workspace-location')]", + "properties": { + "eTag": "*", + "displayName": "Parser for Fivetran logs", + "category": "Microsoft Sentinel Parser", + "functionAlias": "Fivetran", + "query": "let Fivetran_view = Fivetran_CL\n | extend\n EventVendor = 'Fivetran',\n EventProduct = 'Fivetran Platform',\n LogLevel = column_ifexists('Level', ''),\n EventTime = column_ifexists('CreatedAt', datetime(null)),\n LogMessage = column_ifexists('Message', '')\n | project TimeGenerated, EventVendor, EventProduct, LogLevel, EventTime, LogMessage;\nFivetran_view\n", + "functionParameters": "", + "version": 2, + "tags": [ + { + "name": "description", + "value": "" + } + ] + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2022-01-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('Parser-', last(split(variables('parserObject1')._parserId1,'/'))))]", + "dependsOn": [ + "[variables('parserObject1')._parserId1]" + ], + "properties": { + "parentId": "[resourceId('Microsoft.OperationalInsights/workspaces/savedSearches', parameters('workspace'), 'Fivetran')]", + "contentId": "[variables('parserObject1').parserContentId1]", + "kind": "Parser", + "version": "[variables('parserObject1').parserVersion1]", + "source": { + "name": "Fivetran", + "kind": "Solution", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "Michael Woods", + "email": "[variables('_email')]" + }, + "support": { + "name": "Michael Woods", + "email": "michaelwoods@microsoft.com", + "tier": "Community", + "link": "https://github.com/woodsonsecurity" + } + } + } + ] + }, + "packageKind": "Solution", + "packageVersion": "[variables('_solutionVersion')]", + "packageName": "[variables('_solutionName')]", + "packageId": "[variables('_solutionId')]", + "contentSchemaVersion": "3.0.0", + "contentId": "[variables('parserObject1').parserContentId1]", + "contentKind": "Parser", + "displayName": "Parser for Fivetran logs", + "contentProductId": "[concat(take(variables('_solutionId'),50),'-','pr','-', uniqueString(concat(variables('_solutionId'),'-','Parser','-',variables('parserObject1').parserContentId1,'-', '1.0.0')))]", + "id": "[concat(take(variables('_solutionId'),50),'-','pr','-', uniqueString(concat(variables('_solutionId'),'-','Parser','-',variables('parserObject1').parserContentId1,'-', '1.0.0')))]", + "version": "[variables('parserObject1').parserVersion1]" + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/savedSearches", + "apiVersion": "2025-07-01", + "name": "[variables('parserObject1')._parserName1]", + "location": "[parameters('workspace-location')]", + "properties": { + "eTag": "*", + "displayName": "Parser for Fivetran logs", + "category": "Microsoft Sentinel Parser", + "functionAlias": "Fivetran", + "query": "let Fivetran_view = Fivetran_CL\n | extend\n EventVendor = 'Fivetran',\n EventProduct = 'Fivetran Platform',\n LogLevel = column_ifexists('Level', ''),\n EventTime = column_ifexists('CreatedAt', datetime(null)),\n LogMessage = column_ifexists('Message', '')\n | project TimeGenerated, EventVendor, EventProduct, LogLevel, EventTime, LogMessage;\nFivetran_view\n", + "functionParameters": "", + "version": 2, + "tags": [ + { + "name": "description", + "value": "" + } + ] + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2022-01-01-preview", + "location": "[parameters('workspace-location')]", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('Parser-', last(split(variables('parserObject1')._parserId1,'/'))))]", + "dependsOn": [ + "[variables('parserObject1')._parserId1]" + ], + "properties": { + "parentId": "[resourceId('Microsoft.OperationalInsights/workspaces/savedSearches', parameters('workspace'), 'Fivetran')]", + "contentId": "[variables('parserObject1').parserContentId1]", + "kind": "Parser", + "version": "[variables('parserObject1').parserVersion1]", + "source": { + "kind": "Solution", + "name": "Fivetran", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "Michael Woods", + "email": "[variables('_email')]" + }, + "support": { + "name": "Michael Woods", + "email": "michaelwoods@microsoft.com", + "tier": "Community", + "link": "https://github.com/woodsonsecurity" + } + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/contentTemplates", + "apiVersion": "2023-04-01-preview", + "name": "[variables('parserObject2').parserTemplateSpecName2]", + "location": "[parameters('workspace-location')]", + "dependsOn": [ + "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" + ], + "properties": { + "description": "ASimAuthenticationFivetran Data Parser with template version 3.0.1", + "mainTemplate": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "[variables('parserObject2').parserVersion2]", + "parameters": {}, + "variables": {}, + "resources": [ + { + "name": "[variables('parserObject2')._parserName2]", + "apiVersion": "2025-07-01", + "type": "Microsoft.OperationalInsights/workspaces/savedSearches", + "location": "[parameters('workspace-location')]", + "properties": { + "eTag": "*", + "displayName": "Authentication ASIM parser for Fivetran", + "category": "Microsoft Sentinel Parser", + "functionAlias": "ASimAuthenticationFivetran", + "query": "vimAuthenticationFivetran(disabled = disabled)\n", + "functionParameters": "disabled:bool=False", + "version": 2, + "tags": [ + { + "name": "description", + "value": "" + } + ] + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2022-01-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('Parser-', last(split(variables('parserObject2')._parserId2,'/'))))]", + "dependsOn": [ + "[variables('parserObject2')._parserId2]" + ], + "properties": { + "parentId": "[resourceId('Microsoft.OperationalInsights/workspaces/savedSearches', parameters('workspace'), 'ASimAuthenticationFivetran')]", + "contentId": "[variables('parserObject2').parserContentId2]", + "kind": "Parser", + "version": "[variables('parserObject2').parserVersion2]", + "source": { + "name": "Fivetran", + "kind": "Solution", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "Michael Woods", + "email": "[variables('_email')]" + }, + "support": { + "name": "Michael Woods", + "email": "michaelwoods@microsoft.com", + "tier": "Community", + "link": "https://github.com/woodsonsecurity" + } + } + } + ] + }, + "packageKind": "Solution", + "packageVersion": "[variables('_solutionVersion')]", + "packageName": "[variables('_solutionName')]", + "packageId": "[variables('_solutionId')]", + "contentSchemaVersion": "3.0.0", + "contentId": "[variables('parserObject2').parserContentId2]", + "contentKind": "Parser", + "displayName": "Authentication ASIM parser for Fivetran", + "contentProductId": "[concat(take(variables('_solutionId'),50),'-','pr','-', uniqueString(concat(variables('_solutionId'),'-','Parser','-',variables('parserObject2').parserContentId2,'-', '0.1.0')))]", + "id": "[concat(take(variables('_solutionId'),50),'-','pr','-', uniqueString(concat(variables('_solutionId'),'-','Parser','-',variables('parserObject2').parserContentId2,'-', '0.1.0')))]", + "version": "[variables('parserObject2').parserVersion2]" + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/savedSearches", + "apiVersion": "2025-07-01", + "name": "[variables('parserObject2')._parserName2]", + "location": "[parameters('workspace-location')]", + "properties": { + "eTag": "*", + "displayName": "Authentication ASIM parser for Fivetran", + "category": "Microsoft Sentinel Parser", + "functionAlias": "ASimAuthenticationFivetran", + "query": "vimAuthenticationFivetran(disabled = disabled)\n", + "functionParameters": "disabled:bool=False", + "version": 2, + "tags": [ + { + "name": "description", + "value": "" + } + ] + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2022-01-01-preview", + "location": "[parameters('workspace-location')]", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('Parser-', last(split(variables('parserObject2')._parserId2,'/'))))]", + "dependsOn": [ + "[variables('parserObject2')._parserId2]" + ], + "properties": { + "parentId": "[resourceId('Microsoft.OperationalInsights/workspaces/savedSearches', parameters('workspace'), 'ASimAuthenticationFivetran')]", + "contentId": "[variables('parserObject2').parserContentId2]", + "kind": "Parser", + "version": "[variables('parserObject2').parserVersion2]", + "source": { + "kind": "Solution", + "name": "Fivetran", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "Michael Woods", + "email": "[variables('_email')]" + }, + "support": { + "name": "Michael Woods", + "email": "michaelwoods@microsoft.com", + "tier": "Community", + "link": "https://github.com/woodsonsecurity" + } + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/contentTemplates", + "apiVersion": "2023-04-01-preview", + "name": "[variables('parserObject3').parserTemplateSpecName3]", + "location": "[parameters('workspace-location')]", + "dependsOn": [ + "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" + ], + "properties": { + "description": "vimAuthenticationFivetran Data Parser with template version 3.0.1", + "mainTemplate": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "[variables('parserObject3').parserVersion3]", + "parameters": {}, + "variables": {}, + "resources": [ + { + "name": "[variables('parserObject3')._parserName3]", + "apiVersion": "2025-07-01", + "type": "Microsoft.OperationalInsights/workspaces/savedSearches", + "location": "[parameters('workspace-location')]", + "properties": { + "eTag": "*", + "displayName": "Authentication ASIM filtering parser for Fivetran", + "category": "Microsoft Sentinel Parser", + "functionAlias": "vimAuthenticationFivetran", + "query": "let AuthEvents = dynamic(['connection_successful','connection_failure','diagnostic_access_granted','diagnostic_access_ended','diagnostic_access_expired']);\nlet parser = (\n starttime: datetime = datetime(null),\n endtime: datetime = datetime(null),\n srcipaddr_has_any_prefix: dynamic = dynamic([]),\n srchostname_has_any: dynamic = dynamic([]),\n username_has_any: dynamic = dynamic([]),\n targetappname_has_any: dynamic = dynamic([]),\n eventtype_in: dynamic = dynamic([]),\n eventresult: string = '*',\n eventresultdetails_in: dynamic = dynamic([]),\n disabled: bool = false)\n{\n Fivetran_CL | where not(disabled)\n | where (isnull(starttime) or TimeGenerated >= starttime)\n and (isnull(endtime) or TimeGenerated <= endtime)\n // Fivetran external-log events are standardized JSON; if the DCR lands the whole\n // event in Message, parse_json extracts it. Falls back to text if not JSON.\n | extend _evt = parse_json(Message)\n | extend\n EventOriginalType = coalesce(tostring(_evt.event), column_ifexists('event_s', ''), ''),\n _conn_name = coalesce(tostring(_evt.connection_name), column_ifexists('connection_name_s', ''), ''),\n _conn_type = coalesce(tostring(_evt.connector_type), column_ifexists('connector_type_s', ''), ''),\n _conn_id = coalesce(tostring(_evt.connection_id), column_ifexists('connection_id_s', ''), ''),\n _exc = coalesce(tostring(_evt.exception_id), column_ifexists('exception_id_s', ''), '')\n | extend EventOriginalType = iff(isempty(EventOriginalType),\n tostring(extract(@'(?i)(connection_successful|connection_failure|diagnostic_access_granted|diagnostic_access_ended|diagnostic_access_expired)', 1, tostring(Message))),\n EventOriginalType)\n | where EventOriginalType in (AuthEvents)\n | extend\n EventType = iff(EventOriginalType in ('diagnostic_access_ended', 'diagnostic_access_expired'), 'Logoff', 'Logon'),\n EventResult = case(\n EventOriginalType == 'connection_failure', 'Failure',\n Level =~ 'SEVERE', 'Failure',\n 'Success')\n | extend EventResultDetails = iff(EventResult == 'Failure', 'Other', '')\n | where (eventresult == '*' or EventResult =~ eventresult)\n | where (array_length(eventtype_in) == 0 or EventType in~ (eventtype_in))\n | where (array_length(eventresultdetails_in) == 0 or EventResultDetails in~ (eventresultdetails_in))\n | extend\n EventCount = int(1),\n EventStartTime = todatetime(coalesce(column_ifexists('CreatedAt', TimeGenerated), TimeGenerated)),\n EventEndTime = todatetime(coalesce(column_ifexists('CreatedAt', TimeGenerated), TimeGenerated)),\n EventVendor = 'Fivetran',\n EventProduct = 'Fivetran Platform',\n EventSchema = 'Authentication',\n EventSchemaVersion = '0.1.4',\n EventOriginalSeverity = tostring(Level),\n EventMessage = tostring(Message),\n EventOriginalUid = _exc,\n Dvc = 'Fivetran',\n DvcHostname = 'Fivetran',\n TargetAppName = iff(isnotempty(_conn_type), _conn_type, 'Fivetran'),\n TargetAppType = 'Service',\n TargetUsername = iff(isnotempty(_conn_name), _conn_name, 'Unknown'),\n ActingAppName = 'Fivetran',\n ActingAppType = 'Service'\n | extend\n TargetUsernameType = 'Simple',\n User = TargetUsername,\n Application = TargetAppName\n | where (array_length(username_has_any) == 0 or TargetUsername has_any (username_has_any))\n | where (array_length(targetappname_has_any) == 0 or TargetAppName has_any (targetappname_has_any))\n // Fivetran connection-auth events carry no source IP or source hostname, so any\n // caller filtering on those must exclude this source.\n | where (array_length(srcipaddr_has_any_prefix) == 0)\n | where (array_length(srchostname_has_any) == 0)\n | project-away _evt, _conn_name, _conn_type, _conn_id, _exc\n};\nparser(\n starttime = starttime,\n endtime = endtime,\n srcipaddr_has_any_prefix = srcipaddr_has_any_prefix,\n srchostname_has_any = srchostname_has_any,\n username_has_any = username_has_any,\n targetappname_has_any = targetappname_has_any,\n eventtype_in = eventtype_in,\n eventresult = eventresult,\n eventresultdetails_in = eventresultdetails_in,\n disabled = disabled)\n", + "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),srcipaddr_has_any_prefix:dynamic=dynamic([]),srchostname_has_any:dynamic=dynamic([]),username_has_any:dynamic=dynamic([]),targetappname_has_any:dynamic=dynamic([]),eventtype_in:dynamic=dynamic([]),eventresult:string='*',eventresultdetails_in:dynamic=dynamic([]),disabled:bool=False", + "version": 2, + "tags": [ + { + "name": "description", + "value": "" + } + ] + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2022-01-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('Parser-', last(split(variables('parserObject3')._parserId3,'/'))))]", + "dependsOn": [ + "[variables('parserObject3')._parserId3]" + ], + "properties": { + "parentId": "[resourceId('Microsoft.OperationalInsights/workspaces/savedSearches', parameters('workspace'), 'vimAuthenticationFivetran')]", + "contentId": "[variables('parserObject3').parserContentId3]", + "kind": "Parser", + "version": "[variables('parserObject3').parserVersion3]", + "source": { + "name": "Fivetran", + "kind": "Solution", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "Michael Woods", + "email": "[variables('_email')]" + }, + "support": { + "name": "Michael Woods", + "email": "michaelwoods@microsoft.com", + "tier": "Community", + "link": "https://github.com/woodsonsecurity" + } + } + } + ] + }, + "packageKind": "Solution", + "packageVersion": "[variables('_solutionVersion')]", + "packageName": "[variables('_solutionName')]", + "packageId": "[variables('_solutionId')]", + "contentSchemaVersion": "3.0.0", + "contentId": "[variables('parserObject3').parserContentId3]", + "contentKind": "Parser", + "displayName": "Authentication ASIM filtering parser for Fivetran", + "contentProductId": "[concat(take(variables('_solutionId'),50),'-','pr','-', uniqueString(concat(variables('_solutionId'),'-','Parser','-',variables('parserObject3').parserContentId3,'-', '0.1.0')))]", + "id": "[concat(take(variables('_solutionId'),50),'-','pr','-', uniqueString(concat(variables('_solutionId'),'-','Parser','-',variables('parserObject3').parserContentId3,'-', '0.1.0')))]", + "version": "[variables('parserObject3').parserVersion3]" + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/savedSearches", + "apiVersion": "2025-07-01", + "name": "[variables('parserObject3')._parserName3]", + "location": "[parameters('workspace-location')]", + "properties": { + "eTag": "*", + "displayName": "Authentication ASIM filtering parser for Fivetran", + "category": "Microsoft Sentinel Parser", + "functionAlias": "vimAuthenticationFivetran", + "query": "let AuthEvents = dynamic(['connection_successful','connection_failure','diagnostic_access_granted','diagnostic_access_ended','diagnostic_access_expired']);\nlet parser = (\n starttime: datetime = datetime(null),\n endtime: datetime = datetime(null),\n srcipaddr_has_any_prefix: dynamic = dynamic([]),\n srchostname_has_any: dynamic = dynamic([]),\n username_has_any: dynamic = dynamic([]),\n targetappname_has_any: dynamic = dynamic([]),\n eventtype_in: dynamic = dynamic([]),\n eventresult: string = '*',\n eventresultdetails_in: dynamic = dynamic([]),\n disabled: bool = false)\n{\n Fivetran_CL | where not(disabled)\n | where (isnull(starttime) or TimeGenerated >= starttime)\n and (isnull(endtime) or TimeGenerated <= endtime)\n // Fivetran external-log events are standardized JSON; if the DCR lands the whole\n // event in Message, parse_json extracts it. Falls back to text if not JSON.\n | extend _evt = parse_json(Message)\n | extend\n EventOriginalType = coalesce(tostring(_evt.event), column_ifexists('event_s', ''), ''),\n _conn_name = coalesce(tostring(_evt.connection_name), column_ifexists('connection_name_s', ''), ''),\n _conn_type = coalesce(tostring(_evt.connector_type), column_ifexists('connector_type_s', ''), ''),\n _conn_id = coalesce(tostring(_evt.connection_id), column_ifexists('connection_id_s', ''), ''),\n _exc = coalesce(tostring(_evt.exception_id), column_ifexists('exception_id_s', ''), '')\n | extend EventOriginalType = iff(isempty(EventOriginalType),\n tostring(extract(@'(?i)(connection_successful|connection_failure|diagnostic_access_granted|diagnostic_access_ended|diagnostic_access_expired)', 1, tostring(Message))),\n EventOriginalType)\n | where EventOriginalType in (AuthEvents)\n | extend\n EventType = iff(EventOriginalType in ('diagnostic_access_ended', 'diagnostic_access_expired'), 'Logoff', 'Logon'),\n EventResult = case(\n EventOriginalType == 'connection_failure', 'Failure',\n Level =~ 'SEVERE', 'Failure',\n 'Success')\n | extend EventResultDetails = iff(EventResult == 'Failure', 'Other', '')\n | where (eventresult == '*' or EventResult =~ eventresult)\n | where (array_length(eventtype_in) == 0 or EventType in~ (eventtype_in))\n | where (array_length(eventresultdetails_in) == 0 or EventResultDetails in~ (eventresultdetails_in))\n | extend\n EventCount = int(1),\n EventStartTime = todatetime(coalesce(column_ifexists('CreatedAt', TimeGenerated), TimeGenerated)),\n EventEndTime = todatetime(coalesce(column_ifexists('CreatedAt', TimeGenerated), TimeGenerated)),\n EventVendor = 'Fivetran',\n EventProduct = 'Fivetran Platform',\n EventSchema = 'Authentication',\n EventSchemaVersion = '0.1.4',\n EventOriginalSeverity = tostring(Level),\n EventMessage = tostring(Message),\n EventOriginalUid = _exc,\n Dvc = 'Fivetran',\n DvcHostname = 'Fivetran',\n TargetAppName = iff(isnotempty(_conn_type), _conn_type, 'Fivetran'),\n TargetAppType = 'Service',\n TargetUsername = iff(isnotempty(_conn_name), _conn_name, 'Unknown'),\n ActingAppName = 'Fivetran',\n ActingAppType = 'Service'\n | extend\n TargetUsernameType = 'Simple',\n User = TargetUsername,\n Application = TargetAppName\n | where (array_length(username_has_any) == 0 or TargetUsername has_any (username_has_any))\n | where (array_length(targetappname_has_any) == 0 or TargetAppName has_any (targetappname_has_any))\n // Fivetran connection-auth events carry no source IP or source hostname, so any\n // caller filtering on those must exclude this source.\n | where (array_length(srcipaddr_has_any_prefix) == 0)\n | where (array_length(srchostname_has_any) == 0)\n | project-away _evt, _conn_name, _conn_type, _conn_id, _exc\n};\nparser(\n starttime = starttime,\n endtime = endtime,\n srcipaddr_has_any_prefix = srcipaddr_has_any_prefix,\n srchostname_has_any = srchostname_has_any,\n username_has_any = username_has_any,\n targetappname_has_any = targetappname_has_any,\n eventtype_in = eventtype_in,\n eventresult = eventresult,\n eventresultdetails_in = eventresultdetails_in,\n disabled = disabled)\n", + "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),srcipaddr_has_any_prefix:dynamic=dynamic([]),srchostname_has_any:dynamic=dynamic([]),username_has_any:dynamic=dynamic([]),targetappname_has_any:dynamic=dynamic([]),eventtype_in:dynamic=dynamic([]),eventresult:string='*',eventresultdetails_in:dynamic=dynamic([]),disabled:bool=False", + "version": 2, + "tags": [ + { + "name": "description", + "value": "" + } + ] + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2022-01-01-preview", + "location": "[parameters('workspace-location')]", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('Parser-', last(split(variables('parserObject3')._parserId3,'/'))))]", + "dependsOn": [ + "[variables('parserObject3')._parserId3]" + ], + "properties": { + "parentId": "[resourceId('Microsoft.OperationalInsights/workspaces/savedSearches', parameters('workspace'), 'vimAuthenticationFivetran')]", + "contentId": "[variables('parserObject3').parserContentId3]", + "kind": "Parser", + "version": "[variables('parserObject3').parserVersion3]", + "source": { + "kind": "Solution", + "name": "Fivetran", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "Michael Woods", + "email": "[variables('_email')]" + }, + "support": { + "name": "Michael Woods", + "email": "michaelwoods@microsoft.com", + "tier": "Community", + "link": "https://github.com/woodsonsecurity" + } + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/contentTemplates", + "apiVersion": "2023-04-01-preview", + "name": "[variables('parserObject4').parserTemplateSpecName4]", + "location": "[parameters('workspace-location')]", + "dependsOn": [ + "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" + ], + "properties": { + "description": "ASimAuditEventFivetran Data Parser with template version 3.0.1", + "mainTemplate": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "[variables('parserObject4').parserVersion4]", + "parameters": {}, + "variables": {}, + "resources": [ + { + "name": "[variables('parserObject4')._parserName4]", + "apiVersion": "2025-07-01", + "type": "Microsoft.OperationalInsights/workspaces/savedSearches", + "location": "[parameters('workspace-location')]", + "properties": { + "eTag": "*", + "displayName": "Audit Event ASIM parser for Fivetran", + "category": "Microsoft Sentinel Parser", + "functionAlias": "ASimAuditEventFivetran", + "query": "vimAuditEventFivetran(disabled = disabled)\n", + "functionParameters": "disabled:bool=False", + "version": 2, + "tags": [ + { + "name": "description", + "value": "" + } + ] + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2022-01-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('Parser-', last(split(variables('parserObject4')._parserId4,'/'))))]", + "dependsOn": [ + "[variables('parserObject4')._parserId4]" + ], + "properties": { + "parentId": "[resourceId('Microsoft.OperationalInsights/workspaces/savedSearches', parameters('workspace'), 'ASimAuditEventFivetran')]", + "contentId": "[variables('parserObject4').parserContentId4]", + "kind": "Parser", + "version": "[variables('parserObject4').parserVersion4]", + "source": { + "name": "Fivetran", + "kind": "Solution", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "Michael Woods", + "email": "[variables('_email')]" + }, + "support": { + "name": "Michael Woods", + "email": "michaelwoods@microsoft.com", + "tier": "Community", + "link": "https://github.com/woodsonsecurity" + } + } + } + ] + }, + "packageKind": "Solution", + "packageVersion": "[variables('_solutionVersion')]", + "packageName": "[variables('_solutionName')]", + "packageId": "[variables('_solutionId')]", + "contentSchemaVersion": "3.0.0", + "contentId": "[variables('parserObject4').parserContentId4]", + "contentKind": "Parser", + "displayName": "Audit Event ASIM parser for Fivetran", + "contentProductId": "[concat(take(variables('_solutionId'),50),'-','pr','-', uniqueString(concat(variables('_solutionId'),'-','Parser','-',variables('parserObject4').parserContentId4,'-', '0.1.0')))]", + "id": "[concat(take(variables('_solutionId'),50),'-','pr','-', uniqueString(concat(variables('_solutionId'),'-','Parser','-',variables('parserObject4').parserContentId4,'-', '0.1.0')))]", + "version": "[variables('parserObject4').parserVersion4]" + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/savedSearches", + "apiVersion": "2025-07-01", + "name": "[variables('parserObject4')._parserName4]", + "location": "[parameters('workspace-location')]", + "properties": { + "eTag": "*", + "displayName": "Audit Event ASIM parser for Fivetran", + "category": "Microsoft Sentinel Parser", + "functionAlias": "ASimAuditEventFivetran", + "query": "vimAuditEventFivetran(disabled = disabled)\n", + "functionParameters": "disabled:bool=False", + "version": 2, + "tags": [ + { + "name": "description", + "value": "" + } + ] + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2022-01-01-preview", + "location": "[parameters('workspace-location')]", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('Parser-', last(split(variables('parserObject4')._parserId4,'/'))))]", + "dependsOn": [ + "[variables('parserObject4')._parserId4]" + ], + "properties": { + "parentId": "[resourceId('Microsoft.OperationalInsights/workspaces/savedSearches', parameters('workspace'), 'ASimAuditEventFivetran')]", + "contentId": "[variables('parserObject4').parserContentId4]", + "kind": "Parser", + "version": "[variables('parserObject4').parserVersion4]", + "source": { + "kind": "Solution", + "name": "Fivetran", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "Michael Woods", + "email": "[variables('_email')]" + }, + "support": { + "name": "Michael Woods", + "email": "michaelwoods@microsoft.com", + "tier": "Community", + "link": "https://github.com/woodsonsecurity" + } + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/contentTemplates", + "apiVersion": "2023-04-01-preview", + "name": "[variables('parserObject5').parserTemplateSpecName5]", + "location": "[parameters('workspace-location')]", + "dependsOn": [ + "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" + ], + "properties": { + "description": "vimAuditEventFivetran Data Parser with template version 3.0.1", + "mainTemplate": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "[variables('parserObject5').parserVersion5]", + "parameters": {}, + "variables": {}, + "resources": [ + { + "name": "[variables('parserObject5')._parserName5]", + "apiVersion": "2025-07-01", + "type": "Microsoft.OperationalInsights/workspaces/savedSearches", + "location": "[parameters('workspace-location')]", + "properties": { + "eTag": "*", + "displayName": "Audit Event ASIM filtering parser for Fivetran", + "category": "Microsoft Sentinel Parser", + "functionAlias": "vimAuditEventFivetran", + "query": "let AuditConnectorEvents = dynamic(['create_table','drop_table','alter_table','create_schema','schema_migration_start','schema_migration_end','change_schema_config','change_schema_config_via_sync','create_connector','forced_resync_connector','forced_resync_table','update_state','diagnostic_access_granted','diagnostic_access_ended','diagnostic_access_expired']);\nlet parser = (\n starttime: datetime = datetime(null),\n endtime: datetime = datetime(null),\n srcipaddr_has_any_prefix: dynamic = dynamic([]),\n eventtype_in: dynamic = dynamic([]),\n eventresult: string = '*',\n actorusername_has_any: dynamic = dynamic([]),\n operation_has_any: dynamic = dynamic([]),\n object_has_any: dynamic = dynamic([]),\n newvalue_has_any: dynamic = dynamic([]),\n disabled: bool = false)\n{\n Fivetran_CL | where not(disabled)\n | where (isnull(starttime) or TimeGenerated >= starttime)\n and (isnull(endtime) or TimeGenerated <= endtime)\n | extend _evt = parse_json(Message)\n | extend\n _event = coalesce(tostring(_evt.event), column_ifexists('event_s', ''), ''),\n _conn_name = coalesce(tostring(_evt.connection_name), column_ifexists('connection_name_s', ''), ''),\n _conn_type = coalesce(tostring(_evt.connector_type), column_ifexists('connector_type_s', ''), ''),\n _exc = coalesce(tostring(_evt.exception_id), column_ifexists('exception_id_s', ''), ''),\n _data = _evt.data\n | extend _event = iff(isempty(_event),\n tostring(extract(@'(?i)\\b([A-Z_]{4,}|create_table|drop_table|alter_table|create_schema|schema_migration_start|schema_migration_end|change_schema_config|change_schema_config_via_sync|create_connector|forced_resync_connector|forced_resync_table|update_state|diagnostic_access_granted|diagnostic_access_ended|diagnostic_access_expired)\\b', 1, tostring(Message))),\n _event)\n // Include documented connector config events, plus Audit Trail user actions\n // (Fivetran emits these as UPPERCASE_WITH_UNDERSCORES event names).\n | extend _isUpperAudit = (isnotempty(_event) and _event == toupper(_event) and _event matches regex @'^[A-Z][A-Z0-9_]{3,}$')\n | where _event in (AuditConnectorEvents) or _isUpperAudit\n | extend Operation = _event\n | extend EventType = case(\n Operation has_any ('create', 'CREATE', 'add', 'ADD'), 'Create',\n Operation has_any ('drop', 'DROP', 'delete', 'DELETE', 'remove', 'REMOVE'), 'Delete',\n Operation has_any ('pause', 'PAUSE', 'disable', 'DISABLE'), 'Disable',\n Operation has_any ('resume', 'RESUME', 'enable', 'ENABLE'), 'Enable',\n Operation has_any ('migration_start', 'INITIALIZE', 'initialize'), 'Initialize',\n Operation has_any ('edit', 'EDIT', 'alter', 'change', 'update', 'UPDATE', 'set', 'SET', 'modify', 'MODIFY'), 'Set',\n Operation has_any ('diagnostic_access', 'access', 'ACCESS', 'read', 'READ'), 'Read',\n 'Other')\n | extend EventResult = case(\n isnotempty(_exc), 'Failure',\n Level =~ 'SEVERE', 'Failure',\n 'Success')\n | where (eventresult == '*' or EventResult =~ eventresult)\n | where (array_length(eventtype_in) == 0 or EventType in~ (eventtype_in))\n | where (array_length(operation_has_any) == 0 or Operation has_any (operation_has_any))\n | extend\n Object = coalesce(tostring(_data['table']), tostring(_data['schema']), tostring(_data['name']), iff(isnotempty(_conn_name), _conn_name, ''), ''),\n NewValue = coalesce(tostring(_data['new_value']), tostring(_data['status']), ''),\n OldValue = coalesce(tostring(_data['old_value']), ''),\n ActorUsername = coalesce(tostring(_data['actor']), tostring(_data['user']), tostring(_data['email']), tostring(_data['owner']), '')\n | extend\n Object = iff(isempty(Object), 'Unknown', Object),\n ObjectType = case(\n Operation has 'table' or Operation has 'schema', 'Directory Service Object',\n Operation has_any ('CONNECTION', 'CONNECTOR', 'connector'), 'Cloud Resource',\n 'Other')\n | where (array_length(object_has_any) == 0 or Object has_any (object_has_any))\n | where (array_length(newvalue_has_any) == 0 or NewValue has_any (newvalue_has_any))\n | where (array_length(actorusername_has_any) == 0 or ActorUsername has_any (actorusername_has_any))\n | extend\n EventCount = int(1),\n EventStartTime = todatetime(coalesce(column_ifexists('CreatedAt', TimeGenerated), TimeGenerated)),\n EventEndTime = todatetime(coalesce(column_ifexists('CreatedAt', TimeGenerated), TimeGenerated)),\n EventVendor = 'Fivetran',\n EventProduct = 'Fivetran Platform',\n EventSchema = 'AuditEvent',\n EventSchemaVersion = '0.1.2',\n EventOriginalType = _event,\n EventOriginalSeverity = tostring(Level),\n EventOriginalUid = _exc,\n EventMessage = tostring(Message),\n Dvc = 'Fivetran',\n DvcHostname = 'Fivetran',\n TargetAppName = iff(isnotempty(_conn_type), _conn_type, 'Fivetran'),\n TargetAppType = 'Service'\n | extend\n ActorUsername = iff(isempty(ActorUsername), 'Unknown', ActorUsername),\n ActorUsernameType = 'Simple',\n User = ActorUsername,\n Application = TargetAppName,\n Value = NewValue\n // Fivetran audit events carry no source IP, so any caller filtering on it excludes this source.\n | where (array_length(srcipaddr_has_any_prefix) == 0)\n | project-away _evt, _event, _conn_name, _conn_type, _exc, _data, _isUpperAudit\n};\nparser(\n starttime = starttime,\n endtime = endtime,\n srcipaddr_has_any_prefix = srcipaddr_has_any_prefix,\n eventtype_in = eventtype_in,\n eventresult = eventresult,\n actorusername_has_any = actorusername_has_any,\n operation_has_any = operation_has_any,\n object_has_any = object_has_any,\n newvalue_has_any = newvalue_has_any,\n disabled = disabled)\n", + "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),srcipaddr_has_any_prefix:dynamic=dynamic([]),eventtype_in:dynamic=dynamic([]),eventresult:string='*',actorusername_has_any:dynamic=dynamic([]),operation_has_any:dynamic=dynamic([]),object_has_any:dynamic=dynamic([]),newvalue_has_any:dynamic=dynamic([]),disabled:bool=False", + "version": 2, + "tags": [ + { + "name": "description", + "value": "" + } + ] + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2022-01-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('Parser-', last(split(variables('parserObject5')._parserId5,'/'))))]", + "dependsOn": [ + "[variables('parserObject5')._parserId5]" + ], + "properties": { + "parentId": "[resourceId('Microsoft.OperationalInsights/workspaces/savedSearches', parameters('workspace'), 'vimAuditEventFivetran')]", + "contentId": "[variables('parserObject5').parserContentId5]", + "kind": "Parser", + "version": "[variables('parserObject5').parserVersion5]", + "source": { + "name": "Fivetran", + "kind": "Solution", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "Michael Woods", + "email": "[variables('_email')]" + }, + "support": { + "name": "Michael Woods", + "email": "michaelwoods@microsoft.com", + "tier": "Community", + "link": "https://github.com/woodsonsecurity" + } + } + } + ] + }, + "packageKind": "Solution", + "packageVersion": "[variables('_solutionVersion')]", + "packageName": "[variables('_solutionName')]", + "packageId": "[variables('_solutionId')]", + "contentSchemaVersion": "3.0.0", + "contentId": "[variables('parserObject5').parserContentId5]", + "contentKind": "Parser", + "displayName": "Audit Event ASIM filtering parser for Fivetran", + "contentProductId": "[concat(take(variables('_solutionId'),50),'-','pr','-', uniqueString(concat(variables('_solutionId'),'-','Parser','-',variables('parserObject5').parserContentId5,'-', '0.1.0')))]", + "id": "[concat(take(variables('_solutionId'),50),'-','pr','-', uniqueString(concat(variables('_solutionId'),'-','Parser','-',variables('parserObject5').parserContentId5,'-', '0.1.0')))]", + "version": "[variables('parserObject5').parserVersion5]" + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/savedSearches", + "apiVersion": "2025-07-01", + "name": "[variables('parserObject5')._parserName5]", + "location": "[parameters('workspace-location')]", + "properties": { + "eTag": "*", + "displayName": "Audit Event ASIM filtering parser for Fivetran", + "category": "Microsoft Sentinel Parser", + "functionAlias": "vimAuditEventFivetran", + "query": "let AuditConnectorEvents = dynamic(['create_table','drop_table','alter_table','create_schema','schema_migration_start','schema_migration_end','change_schema_config','change_schema_config_via_sync','create_connector','forced_resync_connector','forced_resync_table','update_state','diagnostic_access_granted','diagnostic_access_ended','diagnostic_access_expired']);\nlet parser = (\n starttime: datetime = datetime(null),\n endtime: datetime = datetime(null),\n srcipaddr_has_any_prefix: dynamic = dynamic([]),\n eventtype_in: dynamic = dynamic([]),\n eventresult: string = '*',\n actorusername_has_any: dynamic = dynamic([]),\n operation_has_any: dynamic = dynamic([]),\n object_has_any: dynamic = dynamic([]),\n newvalue_has_any: dynamic = dynamic([]),\n disabled: bool = false)\n{\n Fivetran_CL | where not(disabled)\n | where (isnull(starttime) or TimeGenerated >= starttime)\n and (isnull(endtime) or TimeGenerated <= endtime)\n | extend _evt = parse_json(Message)\n | extend\n _event = coalesce(tostring(_evt.event), column_ifexists('event_s', ''), ''),\n _conn_name = coalesce(tostring(_evt.connection_name), column_ifexists('connection_name_s', ''), ''),\n _conn_type = coalesce(tostring(_evt.connector_type), column_ifexists('connector_type_s', ''), ''),\n _exc = coalesce(tostring(_evt.exception_id), column_ifexists('exception_id_s', ''), ''),\n _data = _evt.data\n | extend _event = iff(isempty(_event),\n tostring(extract(@'(?i)\\b([A-Z_]{4,}|create_table|drop_table|alter_table|create_schema|schema_migration_start|schema_migration_end|change_schema_config|change_schema_config_via_sync|create_connector|forced_resync_connector|forced_resync_table|update_state|diagnostic_access_granted|diagnostic_access_ended|diagnostic_access_expired)\\b', 1, tostring(Message))),\n _event)\n // Include documented connector config events, plus Audit Trail user actions\n // (Fivetran emits these as UPPERCASE_WITH_UNDERSCORES event names).\n | extend _isUpperAudit = (isnotempty(_event) and _event == toupper(_event) and _event matches regex @'^[A-Z][A-Z0-9_]{3,}$')\n | where _event in (AuditConnectorEvents) or _isUpperAudit\n | extend Operation = _event\n | extend EventType = case(\n Operation has_any ('create', 'CREATE', 'add', 'ADD'), 'Create',\n Operation has_any ('drop', 'DROP', 'delete', 'DELETE', 'remove', 'REMOVE'), 'Delete',\n Operation has_any ('pause', 'PAUSE', 'disable', 'DISABLE'), 'Disable',\n Operation has_any ('resume', 'RESUME', 'enable', 'ENABLE'), 'Enable',\n Operation has_any ('migration_start', 'INITIALIZE', 'initialize'), 'Initialize',\n Operation has_any ('edit', 'EDIT', 'alter', 'change', 'update', 'UPDATE', 'set', 'SET', 'modify', 'MODIFY'), 'Set',\n Operation has_any ('diagnostic_access', 'access', 'ACCESS', 'read', 'READ'), 'Read',\n 'Other')\n | extend EventResult = case(\n isnotempty(_exc), 'Failure',\n Level =~ 'SEVERE', 'Failure',\n 'Success')\n | where (eventresult == '*' or EventResult =~ eventresult)\n | where (array_length(eventtype_in) == 0 or EventType in~ (eventtype_in))\n | where (array_length(operation_has_any) == 0 or Operation has_any (operation_has_any))\n | extend\n Object = coalesce(tostring(_data['table']), tostring(_data['schema']), tostring(_data['name']), iff(isnotempty(_conn_name), _conn_name, ''), ''),\n NewValue = coalesce(tostring(_data['new_value']), tostring(_data['status']), ''),\n OldValue = coalesce(tostring(_data['old_value']), ''),\n ActorUsername = coalesce(tostring(_data['actor']), tostring(_data['user']), tostring(_data['email']), tostring(_data['owner']), '')\n | extend\n Object = iff(isempty(Object), 'Unknown', Object),\n ObjectType = case(\n Operation has 'table' or Operation has 'schema', 'Directory Service Object',\n Operation has_any ('CONNECTION', 'CONNECTOR', 'connector'), 'Cloud Resource',\n 'Other')\n | where (array_length(object_has_any) == 0 or Object has_any (object_has_any))\n | where (array_length(newvalue_has_any) == 0 or NewValue has_any (newvalue_has_any))\n | where (array_length(actorusername_has_any) == 0 or ActorUsername has_any (actorusername_has_any))\n | extend\n EventCount = int(1),\n EventStartTime = todatetime(coalesce(column_ifexists('CreatedAt', TimeGenerated), TimeGenerated)),\n EventEndTime = todatetime(coalesce(column_ifexists('CreatedAt', TimeGenerated), TimeGenerated)),\n EventVendor = 'Fivetran',\n EventProduct = 'Fivetran Platform',\n EventSchema = 'AuditEvent',\n EventSchemaVersion = '0.1.2',\n EventOriginalType = _event,\n EventOriginalSeverity = tostring(Level),\n EventOriginalUid = _exc,\n EventMessage = tostring(Message),\n Dvc = 'Fivetran',\n DvcHostname = 'Fivetran',\n TargetAppName = iff(isnotempty(_conn_type), _conn_type, 'Fivetran'),\n TargetAppType = 'Service'\n | extend\n ActorUsername = iff(isempty(ActorUsername), 'Unknown', ActorUsername),\n ActorUsernameType = 'Simple',\n User = ActorUsername,\n Application = TargetAppName,\n Value = NewValue\n // Fivetran audit events carry no source IP, so any caller filtering on it excludes this source.\n | where (array_length(srcipaddr_has_any_prefix) == 0)\n | project-away _evt, _event, _conn_name, _conn_type, _exc, _data, _isUpperAudit\n};\nparser(\n starttime = starttime,\n endtime = endtime,\n srcipaddr_has_any_prefix = srcipaddr_has_any_prefix,\n eventtype_in = eventtype_in,\n eventresult = eventresult,\n actorusername_has_any = actorusername_has_any,\n operation_has_any = operation_has_any,\n object_has_any = object_has_any,\n newvalue_has_any = newvalue_has_any,\n disabled = disabled)\n", + "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),srcipaddr_has_any_prefix:dynamic=dynamic([]),eventtype_in:dynamic=dynamic([]),eventresult:string='*',actorusername_has_any:dynamic=dynamic([]),operation_has_any:dynamic=dynamic([]),object_has_any:dynamic=dynamic([]),newvalue_has_any:dynamic=dynamic([]),disabled:bool=False", + "version": 2, + "tags": [ + { + "name": "description", + "value": "" + } + ] + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2022-01-01-preview", + "location": "[parameters('workspace-location')]", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('Parser-', last(split(variables('parserObject5')._parserId5,'/'))))]", + "dependsOn": [ + "[variables('parserObject5')._parserId5]" + ], + "properties": { + "parentId": "[resourceId('Microsoft.OperationalInsights/workspaces/savedSearches', parameters('workspace'), 'vimAuditEventFivetran')]", + "contentId": "[variables('parserObject5').parserContentId5]", + "kind": "Parser", + "version": "[variables('parserObject5').parserVersion5]", + "source": { + "kind": "Solution", + "name": "Fivetran", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "Michael Woods", + "email": "[variables('_email')]" + }, + "support": { + "name": "Michael Woods", + "email": "michaelwoods@microsoft.com", + "tier": "Community", + "link": "https://github.com/woodsonsecurity" + } + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/contentTemplates", + "apiVersion": "2023-04-01-preview", + "name": "[variables('analyticRuleObject1').analyticRuleTemplateSpecName1]", + "location": "[parameters('workspace-location')]", + "dependsOn": [ + "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" + ], + "properties": { + "description": "FivetranIngestionGap_AnalyticalRules Analytics Rule with template version 3.0.1", + "mainTemplate": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "[variables('analyticRuleObject1').analyticRuleVersion1]", + "parameters": {}, + "variables": {}, + "resources": [ + { + "type": "Microsoft.SecurityInsights/AlertRuleTemplates", + "name": "[variables('analyticRuleObject1')._analyticRulecontentId1]", + "apiVersion": "2023-02-01-preview", + "kind": "Scheduled", + "location": "[parameters('workspace-location')]", + "properties": { + "description": "Alerts when no Fivetran logs have been received in Fivetran_CL for more than 2 hours. A gap can indicate a broken Fivetran-to-Sentinel pipeline, disabled logging, expired app credentials, or deliberately impaired security telemetry.", + "displayName": "Fivetran - Log ingestion gap (telemetry loss)", + "enabled": false, + "query": "let lookback = 14d;\nlet gapThreshold = 2h;\nFivetran_CL\n| where TimeGenerated > ago(lookback)\n| summarize LastLogReceived = max(TimeGenerated)\n| extend GapDuration = now() - LastLogReceived\n| where GapDuration > gapThreshold\n| extend Reason = strcat(\"No Fivetran logs received for \", format_timespan(GapDuration, 'dd.hh:mm:ss'))\n| extend Application = \"Fivetran\"\n", + "queryFrequency": "PT1H", + "queryPeriod": "P14D", + "severity": "Medium", + "suppressionDuration": "PT1H", + "suppressionEnabled": false, + "triggerOperator": "GreaterThan", + "triggerThreshold": 0, + "status": "Available", + "requiredDataConnectors": [ + { + "connectorId": "Fivetran", + "dataTypes": [ + "Fivetran_CL" + ] + } + ], + "tactics": [ + "DefenseEvasion" + ], + "techniques": [ + "T1562" + ], + "entityMappings": [ + { + "fieldMappings": [ + { + "identifier": "Name", + "columnName": "Application" + } + ], + "entityType": "CloudApplication" + } + ], + "eventGroupingSettings": { + "aggregationKind": "SingleAlert" + }, + "customDetails": { + "LastLogReceived": "LastLogReceived", + "Reason": "Reason" + } + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2022-01-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('AnalyticsRule-', last(split(variables('analyticRuleObject1').analyticRuleId1,'/'))))]", + "properties": { + "description": "Fivetran Analytics Rule 1", + "parentId": "[variables('analyticRuleObject1').analyticRuleId1]", + "contentId": "[variables('analyticRuleObject1')._analyticRulecontentId1]", + "kind": "AnalyticsRule", + "version": "[variables('analyticRuleObject1').analyticRuleVersion1]", + "source": { + "kind": "Solution", + "name": "Fivetran", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "Michael Woods", + "email": "[variables('_email')]" + }, + "support": { + "name": "Michael Woods", + "email": "michaelwoods@microsoft.com", + "tier": "Community", + "link": "https://github.com/woodsonsecurity" + } + } + } + ] + }, + "packageKind": "Solution", + "packageVersion": "[variables('_solutionVersion')]", + "packageName": "[variables('_solutionName')]", + "packageId": "[variables('_solutionId')]", + "contentSchemaVersion": "3.0.0", + "contentId": "[variables('analyticRuleObject1')._analyticRulecontentId1]", + "contentKind": "AnalyticsRule", + "displayName": "Fivetran - Log ingestion gap (telemetry loss)", + "contentProductId": "[variables('analyticRuleObject1')._analyticRulecontentProductId1]", + "id": "[variables('analyticRuleObject1')._analyticRulecontentProductId1]", + "version": "[variables('analyticRuleObject1').analyticRuleVersion1]" + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/contentTemplates", + "apiVersion": "2023-04-01-preview", + "name": "[variables('analyticRuleObject2').analyticRuleTemplateSpecName2]", + "location": "[parameters('workspace-location')]", + "dependsOn": [ + "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" + ], + "properties": { + "description": "FivetranAuthFailures_AnalyticalRules Analytics Rule with template version 3.0.1", + "mainTemplate": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "[variables('analyticRuleObject2').analyticRuleVersion2]", + "parameters": {}, + "variables": {}, + "resources": [ + { + "type": "Microsoft.SecurityInsights/AlertRuleTemplates", + "name": "[variables('analyticRuleObject2')._analyticRulecontentId2]", + "apiVersion": "2023-02-01-preview", + "kind": "Scheduled", + "location": "[parameters('workspace-location')]", + "properties": { + "description": "Detects an unusually high number of Fivetran log events referencing authentication, authorization or credential failures within a short window. Repeated failures can indicate brute-force or credential-stuffing attempts against the accounts Fivetran uses, credential theft/misuse, or (more benignly) expired or revoked credentials. The threshold is tuned so routine one-off credential expiry does not alert.", + "displayName": "Fivetran - Repeated authentication failures", + "enabled": false, + "query": "let threshold = 10;\nFivetran_CL\n| where Level in (\"SEVERE\", \"WARNING\")\n| where Message has_any (\n \"authentication failed\", \"authentication error\", \"unauthorized\",\n \"invalid credentials\", \"invalid token\", \"access denied\",\n \"permission denied\", \"forbidden\", \"401\", \"403\",\n \"token expired\", \"credential\", \"auth failure\", \"login failed\")\n| summarize FailureCount = count(), SampleMessages = make_set(Message, 5),\n FirstSeen = min(TimeGenerated), LastSeen = max(TimeGenerated)\n by bin(TimeGenerated, 1h)\n| where FailureCount > threshold\n| sort by FailureCount desc\n| extend Application = \"Fivetran\"\n", + "queryFrequency": "PT1H", + "queryPeriod": "PT1H", + "severity": "Medium", + "suppressionDuration": "PT1H", + "suppressionEnabled": false, + "triggerOperator": "GreaterThan", + "triggerThreshold": 0, + "status": "Available", + "requiredDataConnectors": [ + { + "connectorId": "Fivetran", + "dataTypes": [ + "Fivetran_CL" + ] + } + ], + "tactics": [ + "CredentialAccess" + ], + "techniques": [ + "T1110" + ], + "entityMappings": [ + { + "fieldMappings": [ + { + "identifier": "Name", + "columnName": "Application" + } + ], + "entityType": "CloudApplication" + } + ], + "eventGroupingSettings": { + "aggregationKind": "AlertPerResult" + }, + "customDetails": { + "FailureCount": "FailureCount", + "FirstSeen": "FirstSeen", + "LastSeen": "LastSeen" + } + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2022-01-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('AnalyticsRule-', last(split(variables('analyticRuleObject2').analyticRuleId2,'/'))))]", + "properties": { + "description": "Fivetran Analytics Rule 2", + "parentId": "[variables('analyticRuleObject2').analyticRuleId2]", + "contentId": "[variables('analyticRuleObject2')._analyticRulecontentId2]", + "kind": "AnalyticsRule", + "version": "[variables('analyticRuleObject2').analyticRuleVersion2]", + "source": { + "kind": "Solution", + "name": "Fivetran", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "Michael Woods", + "email": "[variables('_email')]" + }, + "support": { + "name": "Michael Woods", + "email": "michaelwoods@microsoft.com", + "tier": "Community", + "link": "https://github.com/woodsonsecurity" + } + } + } + ] + }, + "packageKind": "Solution", + "packageVersion": "[variables('_solutionVersion')]", + "packageName": "[variables('_solutionName')]", + "packageId": "[variables('_solutionId')]", + "contentSchemaVersion": "3.0.0", + "contentId": "[variables('analyticRuleObject2')._analyticRulecontentId2]", + "contentKind": "AnalyticsRule", + "displayName": "Fivetran - Repeated authentication failures", + "contentProductId": "[variables('analyticRuleObject2')._analyticRulecontentProductId2]", + "id": "[variables('analyticRuleObject2')._analyticRulecontentProductId2]", + "version": "[variables('analyticRuleObject2').analyticRuleVersion2]" + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/contentTemplates", + "apiVersion": "2023-04-01-preview", + "name": "[variables('huntingQueryObject1').huntingQueryTemplateSpecName1]", + "location": "[parameters('workspace-location')]", + "dependsOn": [ + "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" + ], + "properties": { + "description": "FivetranSevereSpike_HuntingQueries Hunting Query with template version 3.0.1", + "mainTemplate": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "[variables('huntingQueryObject1').huntingQueryVersion1]", + "parameters": {}, + "variables": {}, + "resources": [ + { + "type": "Microsoft.OperationalInsights/savedSearches", + "apiVersion": "2025-07-01", + "name": "Fivetran_Hunting_Query_1", + "location": "[parameters('workspace-location')]", + "properties": { + "eTag": "*", + "displayName": "Fivetran - Spike in SEVERE log events", + "category": "Hunting Queries", + "query": "Fivetran_CL\n| where Level == \"SEVERE\"\n| summarize SevereCount = count(), SampleMessages = make_set(Message, 5) by bin(TimeGenerated, 1h)\n| where SevereCount > 10\n| sort by TimeGenerated desc\n", + "version": 2, + "tags": [ + { + "name": "description", + "value": "Surfaces hourly windows with an unusually high number of Fivetran SEVERE-level events, which can indicate connector failures, credential problems, or data-pipeline tampering." + }, + { + "name": "tactics", + "value": "DefenseEvasion" + }, + { + "name": "techniques", + "value": "T1562" + } + ] + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2022-01-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('HuntingQuery-', last(split(resourceId('Microsoft.OperationalInsights/savedSearches', variables('huntingQueryObject1')._huntingQuerycontentId1),'/'))))]", + "properties": { + "description": "Fivetran Hunting Query 1", + "parentId": "[resourceId('Microsoft.OperationalInsights/savedSearches', variables('huntingQueryObject1')._huntingQuerycontentId1)]", + "contentId": "[variables('huntingQueryObject1')._huntingQuerycontentId1]", + "kind": "HuntingQuery", + "version": "[variables('huntingQueryObject1').huntingQueryVersion1]", + "source": { + "kind": "Solution", + "name": "Fivetran", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "Michael Woods", + "email": "[variables('_email')]" + }, + "support": { + "name": "Michael Woods", + "email": "michaelwoods@microsoft.com", + "tier": "Community", + "link": "https://github.com/woodsonsecurity" + } + } + } + ] + }, + "packageKind": "Solution", + "packageVersion": "[variables('_solutionVersion')]", + "packageName": "[variables('_solutionName')]", + "packageId": "[variables('_solutionId')]", + "contentSchemaVersion": "3.0.0", + "contentId": "[variables('huntingQueryObject1')._huntingQuerycontentId1]", + "contentKind": "HuntingQuery", + "displayName": "Fivetran - Spike in SEVERE log events", + "contentProductId": "[concat(take(variables('_solutionId'),50),'-','hq','-', uniqueString(concat(variables('_solutionId'),'-','HuntingQuery','-',variables('huntingQueryObject1')._huntingQuerycontentId1,'-', '1.0.0')))]", + "id": "[concat(take(variables('_solutionId'),50),'-','hq','-', uniqueString(concat(variables('_solutionId'),'-','HuntingQuery','-',variables('huntingQueryObject1')._huntingQuerycontentId1,'-', '1.0.0')))]", + "version": "1.0.0" + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/contentTemplates", + "apiVersion": "2023-04-01-preview", + "name": "[variables('workbookTemplateSpecName1')]", + "location": "[parameters('workspace-location')]", + "dependsOn": [ + "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" + ], + "properties": { + "description": "Fivetran Workbook with template version 3.0.1", + "mainTemplate": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "[variables('workbookVersion1')]", + "parameters": {}, + "variables": {}, + "resources": [ + { + "type": "Microsoft.Insights/workbooks", + "name": "[variables('workbookContentId1')]", + "location": "[parameters('workspace-location')]", + "kind": "shared", + "apiVersion": "2021-08-01", + "metadata": { + "description": "Visualizes Fivetran platform and connector log volume, severity breakdown and ingestion health." + }, + "properties": { + "displayName": "[parameters('workbook1-name')]", + "serializedData": "{\"version\":\"Notebook/1.0\",\"items\":[{\"type\":1,\"content\":{\"json\":\"# Fivetran - Log Overview\\nOverview of Fivetran platform and connector logs ingested into Microsoft Sentinel via the Logs Ingestion API (Fivetran_CL).\"},\"name\":\"title\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"Fivetran_CL\\n| summarize Count = count() by Level\\n| render piechart\",\"size\":1,\"title\":\"Events by level\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\"},\"name\":\"byLevel\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"Fivetran_CL\\n| summarize Count = count() by bin(TimeGenerated, 1h), Level\\n| render timechart\",\"size\":0,\"title\":\"Event volume over time\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\"},\"name\":\"overTime\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"Fivetran_CL\\n| where Level == 'SEVERE'\\n| project TimeGenerated, Level, Message\\n| sort by TimeGenerated desc\\n| take 100\",\"size\":0,\"title\":\"Recent SEVERE events\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\"},\"name\":\"recentSevere\"}],\"$schema\":\"https://github.com/Microsoft/Application-Insights-Workbooks/blob/master/schema/workbook.json\"}\r\n", + "version": "1.0", + "sourceId": "[variables('workspaceResourceId')]", + "category": "sentinel" + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2022-01-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('Workbook-', last(split(variables('workbookId1'),'/'))))]", + "properties": { + "description": "@{workbookKey=FivetranWorkbook; logoFileName=Fivetran.svg; description=Visualizes Fivetran platform and connector log volume, severity breakdown and ingestion health.; dataTypesDependencies=System.Object[]; dataConnectorsDependencies=System.Object[]; previewImagesFileNames=System.Object[]; version=1.0.0; title=Fivetran; templateRelativePath=Fivetran.json; subtitle=; provider=Fivetran}.description", + "parentId": "[variables('workbookId1')]", + "contentId": "[variables('_workbookContentId1')]", + "kind": "Workbook", + "version": "[variables('workbookVersion1')]", + "source": { + "kind": "Solution", + "name": "Fivetran", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "Michael Woods", + "email": "[variables('_email')]" + }, + "support": { + "name": "Michael Woods", + "email": "michaelwoods@microsoft.com", + "tier": "Community", + "link": "https://github.com/woodsonsecurity" + }, + "dependencies": { + "operator": "AND", + "criteria": [ + { + "contentId": "Fivetran_CL", + "kind": "DataType" + }, + { + "contentId": "Fivetran", + "kind": "DataConnector" + } + ] + } + } + } + ] + }, + "packageKind": "Solution", + "packageVersion": "[variables('_solutionVersion')]", + "packageName": "[variables('_solutionName')]", + "packageId": "[variables('_solutionId')]", + "contentSchemaVersion": "3.0.0", + "contentId": "[variables('_workbookContentId1')]", + "contentKind": "Workbook", + "displayName": "[parameters('workbook1-name')]", + "contentProductId": "[variables('_workbookcontentProductId1')]", + "id": "[variables('_workbookcontentProductId1')]", + "version": "[variables('workbookVersion1')]" + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/contentPackages", + "apiVersion": "2023-04-01-preview", + "location": "[parameters('workspace-location')]", + "properties": { + "version": "3.0.1", + "kind": "Solution", + "contentSchemaVersion": "3.0.0", + "displayName": "Fivetran", + "publisherDisplayName": "Michael Woods", + "descriptionHtml": "

Note: Please refer to the following before installing the solution:

\n

• Review the solution Release Notes

\n

• There may be known issues pertaining to this Solution, please refer to them before installing.

\n

The Fivetran Solution for Microsoft Sentinel ingests Fivetran platform and connector logs into a DCR-based custom table using the Azure Monitor Logs Ingestion API. Fivetran pushes its own logs directly to a Data Collection Rule; this solution ships the custom table, DCR, a data-connector tile, a parser, two analytics rules, a hunting query and a workbook to give SecOps visibility of the Fivetran data pipeline.

\n

Data Connectors: 1, Parsers: 5, Workbooks: 1, Analytic Rules: 2, Hunting Queries: 1

\n

Learn more about Microsoft Sentinel | Learn more about Solutions

\n", + "contentKind": "Solution", + "contentProductId": "[variables('_solutioncontentProductId')]", + "id": "[variables('_solutioncontentProductId')]", + "icon": "", + "contentId": "[variables('_solutionId')]", + "parentId": "[variables('_solutionId')]", + "source": { + "kind": "Solution", + "name": "Fivetran", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "Michael Woods", + "email": "[variables('_email')]" + }, + "support": { + "name": "Michael Woods", + "email": "michaelwoods@microsoft.com", + "tier": "Community", + "link": "https://github.com/woodsonsecurity" + }, + "dependencies": { + "operator": "AND", + "criteria": [ + { + "kind": "DataConnector", + "contentId": "[variables('_dataConnectorContentId1')]", + "version": "[variables('dataConnectorVersion1')]" + }, + { + "kind": "Parser", + "contentId": "[variables('parserObject1').parserContentId1]", + "version": "[variables('parserObject1').parserVersion1]" + }, + { + "kind": "Parser", + "contentId": "[variables('parserObject2').parserContentId2]", + "version": "[variables('parserObject2').parserVersion2]" + }, + { + "kind": "Parser", + "contentId": "[variables('parserObject3').parserContentId3]", + "version": "[variables('parserObject3').parserVersion3]" + }, + { + "kind": "Parser", + "contentId": "[variables('parserObject4').parserContentId4]", + "version": "[variables('parserObject4').parserVersion4]" + }, + { + "kind": "Parser", + "contentId": "[variables('parserObject5').parserContentId5]", + "version": "[variables('parserObject5').parserVersion5]" + }, + { + "kind": "AnalyticsRule", + "contentId": "[variables('analyticRuleObject1')._analyticRulecontentId1]", + "version": "[variables('analyticRuleObject1').analyticRuleVersion1]" + }, + { + "kind": "AnalyticsRule", + "contentId": "[variables('analyticRuleObject2')._analyticRulecontentId2]", + "version": "[variables('analyticRuleObject2').analyticRuleVersion2]" + }, + { + "kind": "HuntingQuery", + "contentId": "[variables('huntingQueryObject1')._huntingQuerycontentId1]", + "version": "[variables('huntingQueryObject1').huntingQueryVersion1]" + }, + { + "kind": "Workbook", + "contentId": "[variables('_workbookContentId1')]", + "version": "[variables('workbookVersion1')]" + } + ] + }, + "firstPublishDate": "2026-07-22", + "providers": [ + "Michael Woods" + ], + "categories": { + "domains": [ + "IT Operations", + "Security - Threat Protection" + ] + } + }, + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/', variables('_solutionId'))]" + } + ], + "outputs": {} +} diff --git a/Solutions/Fivetran/Package/testParameters.json b/Solutions/Fivetran/Package/testParameters.json new file mode 100644 index 00000000000..582a413eeb4 --- /dev/null +++ b/Solutions/Fivetran/Package/testParameters.json @@ -0,0 +1,32 @@ +{ + "location": { + "type": "string", + "minLength": 1, + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "Not used, but needed to pass arm-ttk test `Location-Should-Not-Be-Hardcoded`. We instead use the `workspace-location` which is derived from the LA workspace" + } + }, + "workspace-location": { + "type": "string", + "defaultValue": "", + "metadata": { + "description": "[concat('Region to deploy solution resources -- separate from location selection',parameters('location'))]" + } + }, + "workspace": { + "defaultValue": "", + "type": "string", + "metadata": { + "description": "Workspace name for Log Analytics where Microsoft Sentinel is setup" + } + }, + "workbook1-name": { + "type": "string", + "defaultValue": "Fivetran", + "minLength": 1, + "metadata": { + "description": "Name for the workbook" + } + } +} diff --git a/Solutions/Fivetran/Parsers/ASimAuditEventFivetran.yaml b/Solutions/Fivetran/Parsers/ASimAuditEventFivetran.yaml new file mode 100644 index 00000000000..bc424106cb8 --- /dev/null +++ b/Solutions/Fivetran/Parsers/ASimAuditEventFivetran.yaml @@ -0,0 +1,14 @@ +id: ASimAuditEventFivetran +Function: + Title: Audit Event ASIM parser for Fivetran + Version: '0.1.0' + LastUpdated: '2026-07-29' +Category: Microsoft Sentinel Parser +FunctionName: ASimAuditEventFivetran +FunctionAlias: ASimAuditEventFivetran +FunctionParams: + - Name: disabled + Type: bool + Default: false +FunctionQuery: | + vimAuditEventFivetran(disabled = disabled) diff --git a/Solutions/Fivetran/Parsers/ASimAuthenticationFivetran.yaml b/Solutions/Fivetran/Parsers/ASimAuthenticationFivetran.yaml new file mode 100644 index 00000000000..139c6996443 --- /dev/null +++ b/Solutions/Fivetran/Parsers/ASimAuthenticationFivetran.yaml @@ -0,0 +1,14 @@ +id: ASimAuthenticationFivetran +Function: + Title: Authentication ASIM parser for Fivetran + Version: '0.1.0' + LastUpdated: '2026-07-29' +Category: Microsoft Sentinel Parser +FunctionName: ASimAuthenticationFivetran +FunctionAlias: ASimAuthenticationFivetran +FunctionParams: + - Name: disabled + Type: bool + Default: false +FunctionQuery: | + vimAuthenticationFivetran(disabled = disabled) diff --git a/Solutions/Fivetran/Parsers/Fivetran.yaml b/Solutions/Fivetran/Parsers/Fivetran.yaml new file mode 100644 index 00000000000..0ac3bd60bf9 --- /dev/null +++ b/Solutions/Fivetran/Parsers/Fivetran.yaml @@ -0,0 +1,18 @@ +id: Fivetran_Parser +Function: + Title: Parser for Fivetran logs + Version: '1.0.0' + LastUpdated: '2026-07-22' +Category: Microsoft Sentinel Parser +FunctionName: Fivetran +FunctionAlias: Fivetran +FunctionQuery: | + let Fivetran_view = Fivetran_CL + | extend + EventVendor = 'Fivetran', + EventProduct = 'Fivetran Platform', + LogLevel = column_ifexists('Level', ''), + EventTime = column_ifexists('CreatedAt', datetime(null)), + LogMessage = column_ifexists('Message', '') + | project TimeGenerated, EventVendor, EventProduct, LogLevel, EventTime, LogMessage; + Fivetran_view diff --git a/Solutions/Fivetran/Parsers/vimAuditEventFivetran.yaml b/Solutions/Fivetran/Parsers/vimAuditEventFivetran.yaml new file mode 100644 index 00000000000..31536c53750 --- /dev/null +++ b/Solutions/Fivetran/Parsers/vimAuditEventFivetran.yaml @@ -0,0 +1,138 @@ +id: vimAuditEventFivetran +Function: + Title: Audit Event ASIM filtering parser for Fivetran + Version: '0.1.0' + LastUpdated: '2026-07-29' +Category: Microsoft Sentinel Parser +FunctionName: vimAuditEventFivetran +FunctionAlias: vimAuditEventFivetran +FunctionParams: + - Name: starttime + Type: datetime + Default: 'datetime(null)' + - Name: endtime + Type: datetime + Default: 'datetime(null)' + - Name: srcipaddr_has_any_prefix + Type: dynamic + Default: 'dynamic([])' + - Name: eventtype_in + Type: dynamic + Default: 'dynamic([])' + - Name: eventresult + Type: string + Default: '*' + - Name: actorusername_has_any + Type: dynamic + Default: 'dynamic([])' + - Name: operation_has_any + Type: dynamic + Default: 'dynamic([])' + - Name: object_has_any + Type: dynamic + Default: 'dynamic([])' + - Name: newvalue_has_any + Type: dynamic + Default: 'dynamic([])' + - Name: disabled + Type: bool + Default: false +FunctionQuery: | + let AuditConnectorEvents = dynamic(['create_table','drop_table','alter_table','create_schema','schema_migration_start','schema_migration_end','change_schema_config','change_schema_config_via_sync','create_connector','forced_resync_connector','forced_resync_table','update_state','diagnostic_access_granted','diagnostic_access_ended','diagnostic_access_expired']); + let parser = ( + starttime: datetime = datetime(null), + endtime: datetime = datetime(null), + srcipaddr_has_any_prefix: dynamic = dynamic([]), + eventtype_in: dynamic = dynamic([]), + eventresult: string = '*', + actorusername_has_any: dynamic = dynamic([]), + operation_has_any: dynamic = dynamic([]), + object_has_any: dynamic = dynamic([]), + newvalue_has_any: dynamic = dynamic([]), + disabled: bool = false) + { + Fivetran_CL | where not(disabled) + | where (isnull(starttime) or TimeGenerated >= starttime) + and (isnull(endtime) or TimeGenerated <= endtime) + | extend _evt = parse_json(Message) + | extend + _event = coalesce(tostring(_evt.event), column_ifexists('event_s', ''), ''), + _conn_name = coalesce(tostring(_evt.connection_name), column_ifexists('connection_name_s', ''), ''), + _conn_type = coalesce(tostring(_evt.connector_type), column_ifexists('connector_type_s', ''), ''), + _exc = coalesce(tostring(_evt.exception_id), column_ifexists('exception_id_s', ''), ''), + _data = _evt.data + | extend _event = iff(isempty(_event), + tostring(extract(@'(?i)\b([A-Z_]{4,}|create_table|drop_table|alter_table|create_schema|schema_migration_start|schema_migration_end|change_schema_config|change_schema_config_via_sync|create_connector|forced_resync_connector|forced_resync_table|update_state|diagnostic_access_granted|diagnostic_access_ended|diagnostic_access_expired)\b', 1, tostring(Message))), + _event) + // Include documented connector config events, plus Audit Trail user actions + // (Fivetran emits these as UPPERCASE_WITH_UNDERSCORES event names). + | extend _isUpperAudit = (isnotempty(_event) and _event == toupper(_event) and _event matches regex @'^[A-Z][A-Z0-9_]{3,}$') + | where _event in (AuditConnectorEvents) or _isUpperAudit + | extend Operation = _event + | extend EventType = case( + Operation has_any ('create', 'CREATE', 'add', 'ADD'), 'Create', + Operation has_any ('drop', 'DROP', 'delete', 'DELETE', 'remove', 'REMOVE'), 'Delete', + Operation has_any ('pause', 'PAUSE', 'disable', 'DISABLE'), 'Disable', + Operation has_any ('resume', 'RESUME', 'enable', 'ENABLE'), 'Enable', + Operation has_any ('migration_start', 'INITIALIZE', 'initialize'), 'Initialize', + Operation has_any ('edit', 'EDIT', 'alter', 'change', 'update', 'UPDATE', 'set', 'SET', 'modify', 'MODIFY'), 'Set', + Operation has_any ('diagnostic_access', 'access', 'ACCESS', 'read', 'READ'), 'Read', + 'Other') + | extend EventResult = case( + isnotempty(_exc), 'Failure', + Level =~ 'SEVERE', 'Failure', + 'Success') + | where (eventresult == '*' or EventResult =~ eventresult) + | where (array_length(eventtype_in) == 0 or EventType in~ (eventtype_in)) + | where (array_length(operation_has_any) == 0 or Operation has_any (operation_has_any)) + | extend + Object = coalesce(tostring(_data['table']), tostring(_data['schema']), tostring(_data['name']), iff(isnotempty(_conn_name), _conn_name, ''), ''), + NewValue = coalesce(tostring(_data['new_value']), tostring(_data['status']), ''), + OldValue = coalesce(tostring(_data['old_value']), ''), + ActorUsername = coalesce(tostring(_data['actor']), tostring(_data['user']), tostring(_data['email']), tostring(_data['owner']), '') + | extend + Object = iff(isempty(Object), 'Unknown', Object), + ObjectType = case( + Operation has 'table' or Operation has 'schema', 'Directory Service Object', + Operation has_any ('CONNECTION', 'CONNECTOR', 'connector'), 'Cloud Resource', + 'Other') + | where (array_length(object_has_any) == 0 or Object has_any (object_has_any)) + | where (array_length(newvalue_has_any) == 0 or NewValue has_any (newvalue_has_any)) + | where (array_length(actorusername_has_any) == 0 or ActorUsername has_any (actorusername_has_any)) + | extend + EventCount = int(1), + EventStartTime = todatetime(coalesce(column_ifexists('CreatedAt', TimeGenerated), TimeGenerated)), + EventEndTime = todatetime(coalesce(column_ifexists('CreatedAt', TimeGenerated), TimeGenerated)), + EventVendor = 'Fivetran', + EventProduct = 'Fivetran Platform', + EventSchema = 'AuditEvent', + EventSchemaVersion = '0.1.2', + EventOriginalType = _event, + EventOriginalSeverity = tostring(Level), + EventOriginalUid = _exc, + EventMessage = tostring(Message), + Dvc = 'Fivetran', + DvcHostname = 'Fivetran', + TargetAppName = iff(isnotempty(_conn_type), _conn_type, 'Fivetran'), + TargetAppType = 'Service' + | extend + ActorUsername = iff(isempty(ActorUsername), 'Unknown', ActorUsername), + ActorUsernameType = 'Simple', + User = ActorUsername, + Application = TargetAppName, + Value = NewValue + // Fivetran audit events carry no source IP, so any caller filtering on it excludes this source. + | where (array_length(srcipaddr_has_any_prefix) == 0) + | project-away _evt, _event, _conn_name, _conn_type, _exc, _data, _isUpperAudit + }; + parser( + starttime = starttime, + endtime = endtime, + srcipaddr_has_any_prefix = srcipaddr_has_any_prefix, + eventtype_in = eventtype_in, + eventresult = eventresult, + actorusername_has_any = actorusername_has_any, + operation_has_any = operation_has_any, + object_has_any = object_has_any, + newvalue_has_any = newvalue_has_any, + disabled = disabled) diff --git a/Solutions/Fivetran/Parsers/vimAuthenticationFivetran.yaml b/Solutions/Fivetran/Parsers/vimAuthenticationFivetran.yaml new file mode 100644 index 00000000000..1beaa9e2a71 --- /dev/null +++ b/Solutions/Fivetran/Parsers/vimAuthenticationFivetran.yaml @@ -0,0 +1,120 @@ +id: vimAuthenticationFivetran +Function: + Title: Authentication ASIM filtering parser for Fivetran + Version: '0.1.0' + LastUpdated: '2026-07-29' +Category: Microsoft Sentinel Parser +FunctionName: vimAuthenticationFivetran +FunctionAlias: vimAuthenticationFivetran +FunctionParams: + - Name: starttime + Type: datetime + Default: 'datetime(null)' + - Name: endtime + Type: datetime + Default: 'datetime(null)' + - Name: srcipaddr_has_any_prefix + Type: dynamic + Default: 'dynamic([])' + - Name: srchostname_has_any + Type: dynamic + Default: 'dynamic([])' + - Name: username_has_any + Type: dynamic + Default: 'dynamic([])' + - Name: targetappname_has_any + Type: dynamic + Default: 'dynamic([])' + - Name: eventtype_in + Type: dynamic + Default: 'dynamic([])' + - Name: eventresult + Type: string + Default: '*' + - Name: eventresultdetails_in + Type: dynamic + Default: 'dynamic([])' + - Name: disabled + Type: bool + Default: false +FunctionQuery: | + let AuthEvents = dynamic(['connection_successful','connection_failure','diagnostic_access_granted','diagnostic_access_ended','diagnostic_access_expired']); + let parser = ( + starttime: datetime = datetime(null), + endtime: datetime = datetime(null), + srcipaddr_has_any_prefix: dynamic = dynamic([]), + srchostname_has_any: dynamic = dynamic([]), + username_has_any: dynamic = dynamic([]), + targetappname_has_any: dynamic = dynamic([]), + eventtype_in: dynamic = dynamic([]), + eventresult: string = '*', + eventresultdetails_in: dynamic = dynamic([]), + disabled: bool = false) + { + Fivetran_CL | where not(disabled) + | where (isnull(starttime) or TimeGenerated >= starttime) + and (isnull(endtime) or TimeGenerated <= endtime) + // Fivetran external-log events are standardized JSON; if the DCR lands the whole + // event in Message, parse_json extracts it. Falls back to text if not JSON. + | extend _evt = parse_json(Message) + | extend + EventOriginalType = coalesce(tostring(_evt.event), column_ifexists('event_s', ''), ''), + _conn_name = coalesce(tostring(_evt.connection_name), column_ifexists('connection_name_s', ''), ''), + _conn_type = coalesce(tostring(_evt.connector_type), column_ifexists('connector_type_s', ''), ''), + _conn_id = coalesce(tostring(_evt.connection_id), column_ifexists('connection_id_s', ''), ''), + _exc = coalesce(tostring(_evt.exception_id), column_ifexists('exception_id_s', ''), '') + | extend EventOriginalType = iff(isempty(EventOriginalType), + tostring(extract(@'(?i)(connection_successful|connection_failure|diagnostic_access_granted|diagnostic_access_ended|diagnostic_access_expired)', 1, tostring(Message))), + EventOriginalType) + | where EventOriginalType in (AuthEvents) + | extend + EventType = iff(EventOriginalType in ('diagnostic_access_ended', 'diagnostic_access_expired'), 'Logoff', 'Logon'), + EventResult = case( + EventOriginalType == 'connection_failure', 'Failure', + Level =~ 'SEVERE', 'Failure', + 'Success') + | extend EventResultDetails = iff(EventResult == 'Failure', 'Other', '') + | where (eventresult == '*' or EventResult =~ eventresult) + | where (array_length(eventtype_in) == 0 or EventType in~ (eventtype_in)) + | where (array_length(eventresultdetails_in) == 0 or EventResultDetails in~ (eventresultdetails_in)) + | extend + EventCount = int(1), + EventStartTime = todatetime(coalesce(column_ifexists('CreatedAt', TimeGenerated), TimeGenerated)), + EventEndTime = todatetime(coalesce(column_ifexists('CreatedAt', TimeGenerated), TimeGenerated)), + EventVendor = 'Fivetran', + EventProduct = 'Fivetran Platform', + EventSchema = 'Authentication', + EventSchemaVersion = '0.1.4', + EventOriginalSeverity = tostring(Level), + EventMessage = tostring(Message), + EventOriginalUid = _exc, + Dvc = 'Fivetran', + DvcHostname = 'Fivetran', + TargetAppName = iff(isnotempty(_conn_type), _conn_type, 'Fivetran'), + TargetAppType = 'Service', + TargetUsername = iff(isnotempty(_conn_name), _conn_name, 'Unknown'), + ActingAppName = 'Fivetran', + ActingAppType = 'Service' + | extend + TargetUsernameType = 'Simple', + User = TargetUsername, + Application = TargetAppName + | where (array_length(username_has_any) == 0 or TargetUsername has_any (username_has_any)) + | where (array_length(targetappname_has_any) == 0 or TargetAppName has_any (targetappname_has_any)) + // Fivetran connection-auth events carry no source IP or source hostname, so any + // caller filtering on those must exclude this source. + | where (array_length(srcipaddr_has_any_prefix) == 0) + | where (array_length(srchostname_has_any) == 0) + | project-away _evt, _conn_name, _conn_type, _conn_id, _exc + }; + parser( + starttime = starttime, + endtime = endtime, + srcipaddr_has_any_prefix = srcipaddr_has_any_prefix, + srchostname_has_any = srchostname_has_any, + username_has_any = username_has_any, + targetappname_has_any = targetappname_has_any, + eventtype_in = eventtype_in, + eventresult = eventresult, + eventresultdetails_in = eventresultdetails_in, + disabled = disabled) diff --git a/Solutions/Fivetran/Platform-Connector-Ingest/DEPLOYMENT-GUIDE.md b/Solutions/Fivetran/Platform-Connector-Ingest/DEPLOYMENT-GUIDE.md new file mode 100644 index 00000000000..b7fcaf0b38f --- /dev/null +++ b/Solutions/Fivetran/Platform-Connector-Ingest/DEPLOYMENT-GUIDE.md @@ -0,0 +1,199 @@ +# Deployment guide: Fivetran Platform Connector ingest into Sentinel + +End-to-end runbook for the third connection path (Platform Connector -> ADLS -> Azure +Function -> Logs Ingestion API -> Sentinel). Every value in angle brackets is yours to +set; nothing here is environment-specific. + +## Prerequisites + +- A Log Analytics workspace with Microsoft Sentinel enabled (`` in + resource group ``). +- The Fivetran Platform Connector already landing parquet in an ADLS Gen2 account + (``, container ``, lake root `//`), in + resource group ``. +- Azure CLI (`az`) 2.60+ with the `eventgrid` extension, and Azure Functions Core Tools + v4 (`func`). +- Contributor on `` and ``, plus rights to create role + assignments on the storage account and DCR. + +Set shell variables once: + +```bash +SUB= +SENTINEL_RG= +STORAGE_RG= +FUNC_RG= +LOCATION= +WORKSPACE= +ADLS= +CONTAINER= +LAKE_PREFIX=/ # path under the container, no table segment +FUNC_APP=func-fivetran-platform +FUNC_SA= # small storage account for the function host +UAMI=id-fivetran-platform-ingest +``` + +## 1. Tables + DCR (Sentinel side, bicep) + +```bash +az bicep build -f bicep/sentinel-tables-dcr.bicep # optional local compile + +az deployment group create \ + -g "$SENTINEL_RG" \ + -f bicep/sentinel-tables-dcr.bicep \ + -p workspaceName="$WORKSPACE" +``` + +Record the outputs `logsIngestionEndpoint` and `dcrImmutableId`; you set them as function +app settings in step 4. (You will re-run this in step 3 with `functionPrincipalId` once +the identity exists, to grant it Monitoring Metrics Publisher on the DCR.) + +## 2. User-assigned managed identity (created first) + +```bash +az identity create -g "$FUNC_RG" -n "$UAMI" +UAMI_ID=$(az identity show -g "$FUNC_RG" -n "$UAMI" --query id -o tsv) +UAMI_PRINCIPAL=$(az identity show -g "$FUNC_RG" -n "$UAMI" --query principalId -o tsv) +UAMI_CLIENT=$(az identity show -g "$FUNC_RG" -n "$UAMI" --query clientId -o tsv) +``` + +## 3. Pre-grant RBAC (managed identity, no keys) + +Grant the identity read on the lake, and Monitoring Metrics Publisher on the DCR. Because +the identity already exists, both grants are in place before the function first runs. + +```bash +# Read the lake parquet +az role assignment create \ + --assignee-object-id "$UAMI_PRINCIPAL" --assignee-principal-type ServicePrincipal \ + --role "Storage Blob Data Reader" \ + --scope "/subscriptions/$SUB/resourceGroups/$STORAGE_RG/providers/Microsoft.Storage/storageAccounts/$ADLS" + +# Post to the DCR: re-run the bicep with the principal id (idempotent role assignment) +az deployment group create \ + -g "$SENTINEL_RG" \ + -f bicep/sentinel-tables-dcr.bicep \ + -p workspaceName="$WORKSPACE" functionPrincipalId="$UAMI_PRINCIPAL" +``` + +## 4. Flex Consumption function app + +```bash +az functionapp create \ + --resource-group "$FUNC_RG" \ + --name "$FUNC_APP" \ + --flexconsumption-location "$LOCATION" \ + --runtime python \ + --runtime-version 3.11 \ + --storage-account "$FUNC_SA" \ + --assign-identity "$UAMI_ID" + +# App settings: DCR endpoint + rule id from step 1 outputs, and the UAMI client id so +# DefaultAzureCredential selects the user-assigned identity. +az functionapp config appsettings set -g "$FUNC_RG" -n "$FUNC_APP" --settings \ + DCR_ENDPOINT="" \ + DCR_RULE_ID="" \ + AUDIT_STREAM="Custom-Fivetran_AuditTrail_CL" \ + PLATFORM_STREAM="Custom-Fivetran_Platform_CL" \ + AZURE_CLIENT_ID="$UAMI_CLIENT" + +# Security hardening +az functionapp update -g "$FUNC_RG" -n "$FUNC_APP" --set httpsOnly=true +az functionapp config set -g "$FUNC_RG" -n "$FUNC_APP" --min-tls-version 1.2 +az resource update --resource-group "$FUNC_RG" --name scm \ + --namespace Microsoft.Web --resource-type basicPublishingCredentialsPolicies \ + --parent "sites/$FUNC_APP" --set properties.allow=false +az resource update --resource-group "$FUNC_RG" --name ftp \ + --namespace Microsoft.Web --resource-type basicPublishingCredentialsPolicies \ + --parent "sites/$FUNC_APP" --set properties.allow=false +``` + +Publish the code from this folder: + +```bash +cd function +func azure functionapp publish "$FUNC_APP" --python +cd .. +``` + +## 5. Event Grid subscription (filtered, dead-lettered) + +Create a dead-letter container, then subscribe the function to Blob Created events for +the lake data files only. + +```bash +# Dead-letter container (can be on the function storage account) +az storage container create --account-name "$FUNC_SA" --name eventgrid-deadletter --auth-mode login + +FUNC_ID=$(az functionapp show -g "$FUNC_RG" -n "$FUNC_APP" --query id -o tsv) +DEADLETTER_ID=$(az storage account show -g "$FUNC_RG" -n "$FUNC_SA" --query id -o tsv) + +az eventgrid event-subscription create \ + --name fivetran-platform-ingest \ + --source-resource-id "/subscriptions/$SUB/resourceGroups/$STORAGE_RG/providers/Microsoft.Storage/storageAccounts/$ADLS" \ + --endpoint-type azurefunction \ + --endpoint "$FUNC_ID/functions/FivetranPlatformIngest" \ + --included-event-types Microsoft.Storage.BlobCreated \ + --subject-begins-with "/blobServices/default/containers/$CONTAINER/blobs/$LAKE_PREFIX/" \ + --subject-ends-with ".parquet" \ + --deadletter-endpoint "$DEADLETTER_ID/blobServices/default/containers/eventgrid-deadletter" \ + --max-delivery-attempts 30 \ + --event-ttl 1440 +``` + +The subject filter drops everything except `.parquet` under the lake root; the function +further restricts to `//data/*.parquet`, so `_delta_log` checkpoints and Iceberg +metadata never ingest. If you only want the audit table, narrow `--subject-begins-with` +to `.../$LAKE_PREFIX/audit_trail/`. + +### Fallback: webhook endpoint + +If the `azurefunction` endpoint type is unavailable in your environment, use a webhook to +the Event Grid extension endpoint instead (rotate the system key periodically): + +```bash +KEY=$(az functionapp keys list -g "$FUNC_RG" -n "$FUNC_APP" --query systemKeys.eventgrid_extension -o tsv) +ENDPOINT="https://$FUNC_APP.azurewebsites.net/runtime/webhooks/eventgrid?functionName=FivetranPlatformIngest&code=$KEY" +# ...same az eventgrid event-subscription create but with: +# --endpoint-type webhook --endpoint "$ENDPOINT" +``` + +## 6. Content (parsers + hunting query) + +Import the three files in `content/` into the workspace (as saved functions / a hunting +query), or package them via your content pipeline: + +- `vimAuditEventFivetranAuditTrail.yaml` +- `ASimAuditEventFivetranAuditTrail.yaml` +- `FivetranAuditTrailSensitiveChanges_ASIM_Hunting.yaml` + +## 7. Verify + +```kql +Fivetran_AuditTrail_CL | take 10 +Fivetran_Platform_CL | summarize count() by FivetranTable +ASimAuditEventFivetranAuditTrail | take 10 +``` + +Allow 10-15 minutes after the first blob event for rows to appear (table/DCR schema +warm-up). Check function health in Application Insights: + +```kql +traces | where operation_Name == "FivetranPlatformIngest" | order by timestamp desc | take 50 +``` + +## 8. ASIM validation (before proposing upstream) + +Run `ASimSchemaTester` and `ASimDataTester` against `vimAuditEventFivetranAuditTrail` in +a workspace with real rows. Register Fivetran Platform with the `imAuditEvent` unifying +parser for the cross-source blend. + +## Troubleshooting + +| Symptom | Likely cause | Fix | +| --- | --- | --- | +| Function shows "0 functions" | Dependency build (old Consumption plan) | Confirm the app is on **Flex Consumption**; Flex builds Python deps remotely. | +| 403 reading the blob | Identity missing Storage Blob Data Reader, or `AZURE_CLIENT_ID` not set | Re-check step 3 grant and the `AZURE_CLIENT_ID` app setting. | +| 403 posting to Logs Ingestion | Identity missing Monitoring Metrics Publisher on the DCR | Re-run step 3 bicep with `functionPrincipalId`. | +| No rows, no invocations | Subject filter too narrow, or Event Grid not firing | Check the Event Grid subscription metrics and the dead-letter container. | +| Duplicate audit rows | Delta/Iceberg compaction re-emit (expected) | The ASIM parser dedupes by `id`; query through the parser, not the raw table. | diff --git a/Solutions/Fivetran/Platform-Connector-Ingest/IMPROVEMENTS.md b/Solutions/Fivetran/Platform-Connector-Ingest/IMPROVEMENTS.md new file mode 100644 index 00000000000..7eade028f5f --- /dev/null +++ b/Solutions/Fivetran/Platform-Connector-Ingest/IMPROVEMENTS.md @@ -0,0 +1,81 @@ +# Improvements in this reference build + +This connector is a hardened, fully-IaC evolution of a first-generation +"Platform Connector to Sentinel" build. Each improvement below is a concrete change you +can see in the files, with the reason it matters. + +## 1. Flex Consumption hosting (removes the packaging gotcha) + +- **Before:** Linux Consumption (Y1). A Python function with a compiled dependency + (`pyarrow`) had to be published with a remote build, and a run-from-package mismatch + could leave the app showing "zero functions" with no clear error. +- **Now:** **Flex Consumption**. Flex builds Python dependencies remotely as part of the + platform deployment flow, so `pyarrow` and friends resolve without the run-from-package + workaround. Deploy with `az functionapp create --flexconsumption-location `. +- **Why it matters:** removes the single most common first-deploy failure for a + parquet-reading Python function. + +## 2. User-assigned managed identity (removes the RBAC chicken-and-egg) + +- **Before:** system-assigned identity. It does not exist until the app is created, so + the `Storage Blob Data Reader` and `Monitoring Metrics Publisher` grants could only be + made after first deploy, and were lost if the app was recreated. +- **Now:** a **user-assigned managed identity** created first. You pre-grant both roles + to a stable principal, then attach it to the app. The `sentinel-tables-dcr.bicep` + `functionPrincipalId` parameter grants Monitoring Metrics Publisher on the DCR at + deploy time, so the role is in place before the first event. Set `AZURE_CLIENT_ID` so + `DefaultAzureCredential` selects the identity. +- **Why it matters:** RBAC survives app recreation and there is no ordering trap. + +## 3. Event Grid hardening (reliability + cost + no shared secret) + +- **Before:** a webhook subscription that embedded the function system key in the + endpoint URL, with no dead-letter and coarse filtering. +- **Now:** + - **`azurefunction` endpoint type** so Event Grid manages the handshake and the + function key is not pasted into a URL you have to rotate by hand. + - **Dead-letter destination** on a storage container, so events are never silently + lost if the function is briefly unavailable. + - **Retry policy** (max delivery attempts + event TTL) made explicit. + - **Tight subject filter** (`subject-begins-with` the lake root, `subject-ends-with + .parquet`) so `_delta_log` checkpoint parquet and Iceberg metadata never invoke the + function, cutting invocation cost and noise. The function still re-checks + `/
/data/*.parquet` as defence in depth. +- **Why it matters:** at-least-once delivery is real, cheaper, and has no long-lived + secret in a URL. + +## 4. Reference table on Analytics with tuned retention (not Basic/Auxiliary) + +- **Consideration:** it is tempting to put the high-volume reference table on the Basic + or Auxiliary plan to cut cost. +- **Decision:** keep `Fivetran_Platform_CL` on the **Analytics** plan. Basic and + Auxiliary plans restrict cross-table joins, and the entire purpose of these + reference/dimension tables is to be **joined** for enrichment (for example resolving a + `user_id` to an email). Instead, control cost with a **shorter interactive retention** + (`referenceRetentionInDays`, default 90 days) because dimension data changes rarely. +- **Why it matters:** a Basic/Auxiliary "saving" would break the enrichment use case the + table exists for. + +## 5. Security hardening on the function app + +Applied in the deploy guide: +- HTTPS only, minimum TLS 1.2. +- SCM and FTP **basic auth disabled** (managed-identity / OIDC publishing only). +- No storage account keys in app settings for data access; the Logs Ingestion and blob + reads both use the managed identity. + +## 6. Portability and safety + +- Every environment-specific value is a ``; there are no customer names, + resource ids, subscription ids, or lake paths anywhere in this folder. +- The two custom tables are registered in + `.script/tests/KqlvalidationsTests/CustomTables/` so the shipped parsers and hunting + query validate in CI against a known schema. + +## Unchanged on purpose + +- The **at-least-once + dedupe-by-id** design. Delta/Iceberg compaction re-emits rows; + the ASIM parser dedupes with `arg_max(TimeGenerated, *) by id`. This is correct and is + kept as-is. +- The **two-tables-one-DCR** split (typed audit vs generic reference envelope). +- The parser logic and ASIM AuditEvent mapping. diff --git a/Solutions/Fivetran/Platform-Connector-Ingest/README.md b/Solutions/Fivetran/Platform-Connector-Ingest/README.md new file mode 100644 index 00000000000..2bc65794df4 --- /dev/null +++ b/Solutions/Fivetran/Platform-Connector-Ingest/README.md @@ -0,0 +1,135 @@ +# Fivetran Platform Connector ingest (third connection path, + ASIM) + +A push connector that ingests the **Fivetran Platform Connector** tables from the +Managed Data Lake (ADLS Gen2) into Microsoft Sentinel custom tables, then normalises +the security-relevant `audit_trail` table to **ASIM AuditEvent**. + +This is a fully worked, deployable reference. It contains no customer-specific names, +resource ids, or subscription ids: every environment value is a `` you set +at deploy time. + +## Where this fits: three ways to connect Fivetran to Sentinel + +| # | Path | What it is | Best for | +| --- | --- | --- | --- | +| 1 | **External Logs -> DCR -> `Fivetran_CL`** | Fivetran pushes its platform/connector log stream straight into a DCR-based custom table. This is the connector shipped in the main `Solutions/Fivetran/` solution. | Real-time platform/connector monitoring and alerting. | +| 2 | **Platform Connector -> ADLS -> ADX external table** | Fivetran lands the full metadata + `AUDIT_TRAIL` tables as parquet in a lake; an ADX external table queries them on demand via `adx()`. | Long-retention compliance archive with no per-GB ingestion cost. | +| 3 | **Platform Connector -> ADLS -> Function -> Logs Ingestion -> Sentinel** (this folder) | An Event Grid triggered Azure Function reads the lake parquet and ingests the rows into typed Sentinel tables, normalising `audit_trail` to ASIM. | Native Analytics Rules, UEBA, entity mapping, and ASIM cross-source correlation over the audit data. | + +Path 1 carries the platform/connector **log stream** but not the full, structured +**AUDIT_TRAIL** (several audit tables are not available through the External Logs +connector). The audit data only exists in the **Platform Connector -> ADLS parquet**. +This connector is how that structured audit data reaches Sentinel as a first-class, +detectable table. + +## Source layout + +The Fivetran Platform Connector lands 12 tables into the managed data lake, each as its +own Delta + Iceberg (UniForm) table under `//
/`: + +- `data/*.parquet` - the rows +- `_delta_log/` - Delta transaction log +- `metadata/*.metadata.json` + `*.avro` - Iceberg manifests +- `orphans/` - superseded files + +``` +audit_trail <- security events (typed, ASIM AuditEvent) +user team team_membership role role_permission role_connector_type +resource_membership connection destination account connector_type <- reference/metadata +``` + +## Architecture + +``` +Fivetran Platform Connector --(daily sync)--> ADLS Gen2 (Delta+Iceberg parquet) + | + Blob Created event on .../
/data/*.parquet + | + Event Grid (subject filter + dead-letter) + | + Azure Function (Python, Flex Consumption, managed identity) + reads parquet, routes by table + | + +------------------------------+-----------------------------+ + | audit_trail (typed) | 11 reference tables | + v v | + Custom-Fivetran_AuditTrail_CL Custom-Fivetran_Platform_CL | + \_____________ Logs Ingestion API ____________/ | + | + DCR (kind: Direct, 2 streams + transforms) + | + Fivetran_AuditTrail_CL Fivetran_Platform_CL + | + ASimAuditEventFivetranAuditTrail (dedupe by id) --> content +``` + +### Two tables, one DCR (design decision) + +- **`Fivetran_AuditTrail_CL`** (typed) - the security events. Typed columns + (`action`, `user_id`, `primary_resource_*`, `old_values`, `new_values`, ...) so + detections and the ASIM parser are exact. +- **`Fivetran_Platform_CL`** (generic envelope: `FivetranTable`, `Record` dynamic) - + the other 11 reference/metadata tables. These are dimension/lookup data (who is in + which team/role, which connection maps to which destination), used to **enrich and + join**, not to alert on. A generic envelope ingests all of them with one stream and + survives Fivetran schema drift. Any table can be promoted to a typed table later if a + detection needs typed columns. + +### At-least-once + dedupe (Delta/Iceberg reality) + +Delta/Iceberg **compaction rewrites data files**, which re-emits rows already ingested. +The connector is therefore deliberately **at-least-once**; correctness comes from +**deduping on the immutable primary key**, not from the transport: + +- `audit_trail`: the ASIM parser `vimAuditEventFivetranAuditTrail` dedupes with + `summarize arg_max(TimeGenerated, *) by id` before normalising, so every consumer gets + de-duplicated rows automatically. +- reference tables: dedupe by the row id inside `Record` at query time, or build a + materialized view keyed on it if you query them hot. + +## ASIM normalisation + +`audit_trail` is a genuine event stream, so it maps to **ASIM AuditEvent 0.1.2**: + +| ASIM field | From AUDIT_TRAIL | +| --- | --- | +| EventStartTime / EventEndTime | `captured_at` | +| Operation / EventOriginalType | `action` | +| EventType | derived from `action` (Create/Delete/Set/Enable/Disable/Read/Other) | +| EventResult | `Success` (AUDIT_TRAIL records completed actions) | +| Object / ObjectType | `primary_resource_id` / `primary_resource_type` | +| OldValue / NewValue | `old_values` / `new_values` | +| ActorUserId / ActorUsername | `user_id` | +| AdditionalFields | `interaction_method`, `secondary_resource_*` | + +The 11 reference tables are **not** ASIM events (they are dimension tables), so no ASIM +parser is shipped for them. They are the enrichment source (for example join +`ActorUserId` to the `user` table for an email address). + +Parsers + consuming content live in `content/`: +- `vimAuditEventFivetranAuditTrail.yaml` - filtering parser (full ASIM audit params). +- `ASimAuditEventFivetranAuditTrail.yaml` - parameter-less parser. +- `FivetranAuditTrailSensitiveChanges_ASIM_Hunting.yaml` - hunting query that consumes + the parser. + +## Files + +- `function/function_app.py` - Event Grid triggered Python function, routes 12 tables. +- `function/requirements.txt`, `host.json`, `local.settings.json.example`. +- `fivetran-platform-dcr.json` - DCR body (2 streams + transforms) for CLI use. +- `bicep/sentinel-tables-dcr.bicep` - tables + DCR + optional DCR role assignment + (compile with `az bicep build`). +- `content/` - the ASIM parsers + consuming hunting query. +- `test/test_audittrail_parser.py` - offline dummy-data test of the parser mapping. +- `DEPLOYMENT-GUIDE.md` - end-to-end deploy runbook (Flex Consumption + managed identity). +- `IMPROVEMENTS.md` - what this reference improves versus a first-generation build. + +## Quick start + +See `DEPLOYMENT-GUIDE.md` for the full runbook. In short: + +1. Deploy `bicep/sentinel-tables-dcr.bicep` to the Sentinel resource group (tables + DCR). +2. Create a user-assigned managed identity and pre-grant it the two roles. +3. Deploy the Flex Consumption function app with that identity; set the DCR app settings. +4. Create the Event Grid subscription (subject filtered, with dead-lettering). +5. Import the `content/` parsers and verify with the KQL in the guide. diff --git a/Solutions/Fivetran/Platform-Connector-Ingest/bicep/sentinel-tables-dcr.bicep b/Solutions/Fivetran/Platform-Connector-Ingest/bicep/sentinel-tables-dcr.bicep new file mode 100644 index 00000000000..0f9d7d1edd0 --- /dev/null +++ b/Solutions/Fivetran/Platform-Connector-Ingest/bicep/sentinel-tables-dcr.bicep @@ -0,0 +1,190 @@ +// Fivetran Platform Connector -> Sentinel ingest: Sentinel-side infrastructure as code. +// +// Deploys the Sentinel side of the ADLS -> Function -> Logs Ingestion API -> Sentinel +// pipeline for the Fivetran Platform Connector tables: +// 1. typed custom table Fivetran_AuditTrail_CL (security AUDIT_TRAIL events) +// 2. generic custom table Fivetran_Platform_CL ({FivetranTable, Record dynamic}) +// for the other 11 reference/metadata tables (user, team, role, connection, ...) +// 3. Data Collection Rule (kind: Direct) with one stream per table + transforms. +// 4. (optional) Monitoring Metrics Publisher on the DCR for the ingest function's +// managed identity, when its principalId is supplied. With a user-assigned managed +// identity you can supply this up front, so the role is in place before first run +// (no chicken-and-egg with a system-assigned identity created after the DCR). +// +// The Flex Consumption function app, its user-assigned managed identity, the Event Grid +// system topic + subscription, and the Storage Blob Data Reader grant are in +// DEPLOYMENT-GUIDE.md (CLI) because the storage account typically lives in a different +// resource group / subscription owned by the data platform team. +// +// api-versions: dataCollectionRules 2024-03-11 (exposes properties.endpoints.logsIngestion), +// workspaces/tables 2022-10-01, roleAssignments 2022-04-01. +// +// Compile: az bicep build -f sentinel-tables-dcr.bicep +// Deploy: az deployment group create -g -f sentinel-tables-dcr.bicep \ +// -p workspaceName= functionPrincipalId= + +targetScope = 'resourceGroup' + +@description('Azure region. Must match the Log Analytics workspace region.') +param location string = resourceGroup().location + +@description('Name of the EXISTING Log Analytics workspace with Sentinel enabled.') +param workspaceName string + +@description('Data Collection Rule name.') +param dcrName string = 'dcr-fivetran-platform' + +@description('Interactive (Analytics) retention in days for the audit table. 30-730.') +@minValue(30) +@maxValue(730) +param auditRetentionInDays int = 365 + +@description('Total retention in days (interactive + archive) for the audit table. 30-2556.') +@minValue(30) +@maxValue(2556) +param auditTotalRetentionInDays int = 730 + +@description('Interactive (Analytics) retention in days for the reference table. 30-730. Reference/dimension data changes rarely, so a shorter interactive window is usually enough while keeping the Analytics plan so it can be joined for enrichment.') +@minValue(30) +@maxValue(730) +param referenceRetentionInDays int = 90 + +@description('Optional: object (principal) id of the ingest function\'s managed identity. When set, grants it Monitoring Metrics Publisher on the DCR. Prefer a user-assigned managed identity so this is known before deploy.') +param functionPrincipalId string = '' + +var auditTable = 'Fivetran_AuditTrail_CL' +var platformTable = 'Fivetran_Platform_CL' +var auditStream = 'Custom-${auditTable}' +var platformStream = 'Custom-${platformTable}' +// Built-in role: Monitoring Metrics Publisher +var monitoringMetricsPublisherRoleId = '3913510d-42f4-4e42-8a64-420c390055eb' + +resource workspace 'Microsoft.OperationalInsights/workspaces@2022-10-01' existing = { + name: workspaceName +} + +resource auditTableRes 'Microsoft.OperationalInsights/workspaces/tables@2022-10-01' = { + parent: workspace + name: auditTable + properties: { + plan: 'Analytics' + retentionInDays: auditRetentionInDays + totalRetentionInDays: auditTotalRetentionInDays + schema: { + name: auditTable + description: 'Fivetran Platform Connector AUDIT_TRAIL events, ingested from the Managed Data Lake parquet via the Logs Ingestion API.' + columns: [ + { name: 'TimeGenerated', type: 'dateTime', description: 'Ingestion timestamp, derived from captured_at in the DCR transform.' } + { name: 'id', type: 'string', description: 'Immutable AUDIT_TRAIL row id (primary key; used for dedupe).' } + { name: 'captured_at', type: 'dateTime', description: 'When the audited action occurred.' } + { name: 'user_id', type: 'string', description: 'Fivetran user id that performed the action (actor).' } + { name: 'action', type: 'string', description: 'The action performed (operation).' } + { name: 'interaction_method', type: 'string', description: 'How the action was performed (UI, API, terraform, ...).' } + { name: 'primary_resource_type', type: 'string', description: 'Type of the primary object acted on.' } + { name: 'primary_resource_id', type: 'string', description: 'Id of the primary object acted on.' } + { name: 'secondary_resource_type', type: 'string', description: 'Type of the secondary object.' } + { name: 'secondary_resource_id', type: 'string', description: 'Id of the secondary object.' } + { name: 'old_values', type: 'dynamic', description: 'Prior values (JSON), parsed in the DCR transform.' } + { name: 'new_values', type: 'dynamic', description: 'New values (JSON), parsed in the DCR transform.' } + ] + } + } +} + +resource platformTableRes 'Microsoft.OperationalInsights/workspaces/tables@2022-10-01' = { + parent: workspace + name: platformTable + properties: { + // Analytics (not Basic/Auxiliary): these reference tables are used to JOIN/enrich + // audit and other events. Basic and Auxiliary plans restrict cross-table joins, so + // keep Analytics here and control cost with a shorter interactive retention instead. + plan: 'Analytics' + retentionInDays: referenceRetentionInDays + totalRetentionInDays: referenceRetentionInDays + schema: { + name: platformTable + description: 'Fivetran Platform Connector reference/metadata tables (user, team, role, connection, destination, account, ...) as a generic envelope; full row in Record.' + columns: [ + { name: 'TimeGenerated', type: 'dateTime', description: 'Ingestion timestamp (now() in the DCR transform).' } + { name: 'FivetranTable', type: 'string', description: 'Source Fivetran table name (e.g. user, role, connection).' } + { name: 'Record', type: 'dynamic', description: 'Full source row as JSON, parsed in the DCR transform.' } + ] + } + } +} + +resource dcr 'Microsoft.Insights/dataCollectionRules@2024-03-11' = { + name: dcrName + location: location + kind: 'Direct' + properties: { + streamDeclarations: { + '${auditStream}': { + columns: [ + { name: 'id', type: 'string' } + { name: 'captured_at', type: 'datetime' } + { name: 'user_id', type: 'string' } + { name: 'action', type: 'string' } + { name: 'interaction_method', type: 'string' } + { name: 'primary_resource_type', type: 'string' } + { name: 'primary_resource_id', type: 'string' } + { name: 'secondary_resource_type', type: 'string' } + { name: 'secondary_resource_id', type: 'string' } + { name: 'old_values', type: 'string' } + { name: 'new_values', type: 'string' } + ] + } + '${platformStream}': { + columns: [ + { name: 'FivetranTable', type: 'string' } + { name: 'Record', type: 'string' } + ] + } + } + destinations: { + logAnalytics: [ + { + workspaceResourceId: workspace.id + name: 'sentinelWorkspace' + } + ] + } + dataFlows: [ + { + streams: [ auditStream ] + destinations: [ 'sentinelWorkspace' ] + transformKql: 'source | extend TimeGenerated = captured_at | extend old_values = parse_json(old_values), new_values = parse_json(new_values)' + outputStream: auditStream + } + { + streams: [ platformStream ] + destinations: [ 'sentinelWorkspace' ] + transformKql: 'source | extend TimeGenerated = now() | extend Record = parse_json(Record)' + outputStream: platformStream + } + ] + } + dependsOn: [ + auditTableRes + platformTableRes + ] +} + +resource dcrRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (!empty(functionPrincipalId)) { + name: guid(dcr.id, functionPrincipalId, monitoringMetricsPublisherRoleId) + scope: dcr + properties: { + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', monitoringMetricsPublisherRoleId) + principalId: functionPrincipalId + principalType: 'ServicePrincipal' + } +} + +@description('Paste into the function app setting DCR_RULE_ID.') +output dcrImmutableId string = dcr.properties.immutableId + +@description('Paste into the function app setting DCR_ENDPOINT.') +output logsIngestionEndpoint string = dcr.properties.endpoints.logsIngestion + +output auditStream string = auditStream +output platformStream string = platformStream diff --git a/Solutions/Fivetran/Platform-Connector-Ingest/content/ASimAuditEventFivetranAuditTrail.yaml b/Solutions/Fivetran/Platform-Connector-Ingest/content/ASimAuditEventFivetranAuditTrail.yaml new file mode 100644 index 00000000000..03789939326 --- /dev/null +++ b/Solutions/Fivetran/Platform-Connector-Ingest/content/ASimAuditEventFivetranAuditTrail.yaml @@ -0,0 +1,28 @@ +Parser: + Title: Audit Event ASIM parser for Fivetran Platform AUDIT_TRAIL + Version: '0.1.0' + LastUpdated: 2026-07-29 +Product: + Name: Fivetran Platform +Normalization: + Schema: AuditEvent + Version: '0.1.2' +References: +- Title: ASIM Audit Event Schema + Link: https://learn.microsoft.com/azure/sentinel/normalization-schema-audit +- Title: ASIM + Link: https://learn.microsoft.com/azure/sentinel/normalization +Description: | + ASIM Audit Event parameter-less parser for the Fivetran Platform Connector + AUDIT_TRAIL table (Fivetran_AuditTrail_CL). Wraps the filtering parser + vimAuditEventFivetranAuditTrail with default (no) filters. Use + ASimAuditEventFivetranAuditTrail for interactive queries and + vimAuditEventFivetranAuditTrail in analytics and workbooks. +ParserName: ASimAuditEventFivetranAuditTrail +EquivalentBuiltInParser: _ASim_AuditEvent_FivetranAuditTrail +ParserParams: + - Name: disabled + Type: bool + Default: false +ParserQuery: | + vimAuditEventFivetranAuditTrail(disabled = disabled) diff --git a/Solutions/Fivetran/Platform-Connector-Ingest/content/FivetranAuditTrailSensitiveChanges_ASIM_Hunting.yaml b/Solutions/Fivetran/Platform-Connector-Ingest/content/FivetranAuditTrailSensitiveChanges_ASIM_Hunting.yaml new file mode 100644 index 00000000000..8454cb2c146 --- /dev/null +++ b/Solutions/Fivetran/Platform-Connector-Ingest/content/FivetranAuditTrailSensitiveChanges_ASIM_Hunting.yaml @@ -0,0 +1,52 @@ +id: c2f4a6e1-4b7d-4c0a-9d2e-7a1b3c5d9e02 +name: Fivetran Platform - Sensitive AUDIT_TRAIL changes (ASIM AuditEvent) +description: | + 'ASIM-normalised hunting query over the Fivetran Platform Connector AUDIT_TRAIL table + (Fivetran_AuditTrail_CL) using the ASimAuditEventFivetranAuditTrail parser. Surfaces + high-impact administrative changes: deletions of connections/destinations, role and + role-permission changes, team/membership changes, and account edits. Because the source + is the fully structured AUDIT_TRAIL (not free-text log messages), Actor (user_id), + Object (resource id/type) and old/new values are exact. Review for unexpected actors, + out-of-hours changes, or bursts of destructive operations. The parser dedupes lake + compaction re-emits by audit id. Once added to the imAuditEvent unifying parser this + logic also runs cross-source.' +requiredDataConnectors: + - connectorId: FivetranPlatform + dataTypes: + - Fivetran_AuditTrail_CL +tactics: + - Impact + - Persistence + - DefenseEvasion +relevantTechniques: + - T1531 + - T1098 + - T1562 +query: | + ASimAuditEventFivetranAuditTrail + | where EventType in ('Delete', 'Disable') + or TargetAppName has_any ('role', 'destination', 'connection', 'account', 'membership', 'team', 'user') + | project + TimeGenerated, + EventType, + Operation, + Object, + ObjectType, + ActorUserId, + ActorUsername, + TargetAppName, + OldValue, + NewValue, + EventOriginalUid, + EventMessage + | sort by TimeGenerated desc +entityMappings: + - entityType: Account + fieldMappings: + - identifier: Name + columnName: ActorUsername + - entityType: CloudResource + fieldMappings: + - identifier: Name + columnName: Object +version: 1.0.0 diff --git a/Solutions/Fivetran/Platform-Connector-Ingest/content/vimAuditEventFivetranAuditTrail.yaml b/Solutions/Fivetran/Platform-Connector-Ingest/content/vimAuditEventFivetranAuditTrail.yaml new file mode 100644 index 00000000000..17154e45b2a --- /dev/null +++ b/Solutions/Fivetran/Platform-Connector-Ingest/content/vimAuditEventFivetranAuditTrail.yaml @@ -0,0 +1,155 @@ +Parser: + Title: Audit Event ASIM filtering parser for Fivetran Platform AUDIT_TRAIL + Version: '0.1.0' + LastUpdated: 2026-07-29 +Product: + Name: Fivetran Platform +Normalization: + Schema: AuditEvent + Version: '0.1.2' +References: +- Title: ASIM Audit Event Schema + Link: https://learn.microsoft.com/azure/sentinel/normalization-schema-audit +- Title: ASIM + Link: https://learn.microsoft.com/azure/sentinel/normalization +- Title: Fivetran Platform AUDIT_TRAIL table reference + Link: https://fivetran.com/docs/logs/fivetran-platform/table-reference +Description: | + ASIM Audit Event filtering parser for the Fivetran Platform Connector AUDIT_TRAIL + table, ingested from the Managed Data Lake into Fivetran_AuditTrail_CL by the + Path B ingest connector. Unlike the External Log (Fivetran_CL) best-effort parser, + this reads the fully structured audit columns (action, user_id, primary/secondary + resource, old_values/new_values) so Actor, Object, Operation and values are exact. + + Delta/Iceberg compaction in the lake can re-emit rows already ingested, so the + parser dedupes by the immutable audit `id` (arg_max on TimeGenerated) before + normalising. AUDIT_TRAIL records completed administrative actions, so EventResult + is Success; there is no source IP in AUDIT_TRAIL. +ParserName: vimAuditEventFivetranAuditTrail +EquivalentBuiltInParser: _Im_AuditEvent_FivetranAuditTrail +ParserParams: + - Name: starttime + Type: datetime + Default: datetime(null) + - Name: endtime + Type: datetime + Default: datetime(null) + - Name: srcipaddr_has_any_prefix + Type: dynamic + Default: 'dynamic([])' + - Name: eventtype_in + Type: dynamic + Default: 'dynamic([])' + - Name: eventresult + Type: string + Default: '*' + - Name: actorusername_has_any + Type: dynamic + Default: 'dynamic([])' + - Name: operation_has_any + Type: dynamic + Default: 'dynamic([])' + - Name: object_has_any + Type: dynamic + Default: 'dynamic([])' + - Name: newvalue_has_any + Type: dynamic + Default: 'dynamic([])' + - Name: disabled + Type: bool + Default: false +ParserQuery: | + let parser = ( + starttime: datetime = datetime(null), + endtime: datetime = datetime(null), + srcipaddr_has_any_prefix: dynamic = dynamic([]), + eventtype_in: dynamic = dynamic([]), + eventresult: string = '*', + actorusername_has_any: dynamic = dynamic([]), + operation_has_any: dynamic = dynamic([]), + object_has_any: dynamic = dynamic([]), + newvalue_has_any: dynamic = dynamic([]), + disabled: bool = false) + { + Fivetran_AuditTrail_CL | where not(disabled) + | where (isnull(starttime) or TimeGenerated >= starttime) + and (isnull(endtime) or TimeGenerated <= endtime) + // Dedupe compaction re-emits: keep the latest landed copy of each audit id. + | summarize arg_max(TimeGenerated, *) by id + | extend + _action = tostring(action), + _primType = tostring(primary_resource_type), + _primId = tostring(primary_resource_id), + _secType = tostring(secondary_resource_type), + _secId = tostring(secondary_resource_id) + | extend Operation = _action + | extend EventType = case( + _action has_any ('create', 'add', 'insert'), 'Create', + _action has_any ('delete', 'remove', 'drop'), 'Delete', + _action has_any ('pause', 'disable'), 'Disable', + _action has_any ('resume', 'enable'), 'Enable', + _action has_any ('edit', 'update', 'alter', 'change', 'modify', 'set'), 'Set', + _action has_any ('read', 'view', 'access', 'get'), 'Read', + 'Other') + // AUDIT_TRAIL records completed actions; there is no failure indicator. + | extend EventResult = 'Success' + | where (eventresult == '*' or EventResult =~ eventresult) + | where (array_length(eventtype_in) == 0 or EventType in~ (eventtype_in)) + | where (array_length(operation_has_any) == 0 or Operation has_any (operation_has_any)) + | extend + Object = tostring(coalesce(_primId, _primType, 'Unknown')), + ObjectType = case( + _primType has_any ('connector', 'connection', 'destination'), 'Cloud Resource', + _primType has_any ('user', 'team', 'role', 'account', 'membership'), 'Directory Service Object', + 'Other') + | where (array_length(object_has_any) == 0 or Object has_any (object_has_any)) + | extend + OldValue = tostring(old_values), + NewValue = tostring(new_values) + | where (array_length(newvalue_has_any) == 0 or NewValue has_any (newvalue_has_any)) + | extend + ActorUserId = tostring(user_id), + ActorUsername = tostring(user_id) + | extend + ActorUsername = iff(isempty(ActorUsername), 'Unknown', ActorUsername), + ActorUsernameType = 'Other', + ActorUserIdType = 'Other' + | where (array_length(actorusername_has_any) == 0 or ActorUsername has_any (actorusername_has_any)) + | extend + EventCount = int(1), + EventStartTime = todatetime(coalesce(column_ifexists('captured_at', TimeGenerated), TimeGenerated)), + EventEndTime = todatetime(coalesce(column_ifexists('captured_at', TimeGenerated), TimeGenerated)), + EventVendor = 'Fivetran', + EventProduct = 'Fivetran Platform', + EventSchema = 'AuditEvent', + EventSchemaVersion = '0.1.2', + EventOriginalType = _action, + EventOriginalUid = tostring(id), + EventMessage = strcat(_action, ' ', _primType, ' ', _primId), + Dvc = 'Fivetran', + DvcHostname = 'Fivetran', + TargetAppName = iff(isnotempty(_primType), _primType, 'Fivetran'), + TargetAppType = 'Service' + | extend + User = ActorUsername, + Application = TargetAppName, + Value = NewValue, + AdditionalFields = bag_pack( + 'interaction_method', tostring(interaction_method), + 'secondary_resource_type', _secType, + 'secondary_resource_id', _secId) + // AUDIT_TRAIL carries no source IP, so any caller filtering on it excludes this source. + | where (array_length(srcipaddr_has_any_prefix) == 0) + | project-away _action, _primType, _primId, _secType, _secId + }; + parser( + starttime = starttime, + endtime = endtime, + srcipaddr_has_any_prefix = srcipaddr_has_any_prefix, + eventtype_in = eventtype_in, + eventresult = eventresult, + actorusername_has_any = actorusername_has_any, + operation_has_any = operation_has_any, + object_has_any = object_has_any, + newvalue_has_any = newvalue_has_any, + disabled = disabled) diff --git a/Solutions/Fivetran/Platform-Connector-Ingest/fivetran-platform-dcr.json b/Solutions/Fivetran/Platform-Connector-Ingest/fivetran-platform-dcr.json new file mode 100644 index 00000000000..c43b2635d5d --- /dev/null +++ b/Solutions/Fivetran/Platform-Connector-Ingest/fivetran-platform-dcr.json @@ -0,0 +1,51 @@ +{ + "location": "", + "kind": "Direct", + "properties": { + "streamDeclarations": { + "Custom-Fivetran_AuditTrail_CL": { + "columns": [ + { "name": "id", "type": "string" }, + { "name": "captured_at", "type": "datetime" }, + { "name": "user_id", "type": "string" }, + { "name": "action", "type": "string" }, + { "name": "interaction_method", "type": "string" }, + { "name": "primary_resource_type", "type": "string" }, + { "name": "primary_resource_id", "type": "string" }, + { "name": "secondary_resource_type", "type": "string" }, + { "name": "secondary_resource_id", "type": "string" }, + { "name": "old_values", "type": "string" }, + { "name": "new_values", "type": "string" } + ] + }, + "Custom-Fivetran_Platform_CL": { + "columns": [ + { "name": "FivetranTable", "type": "string" }, + { "name": "Record", "type": "string" } + ] + } + }, + "destinations": { + "logAnalytics": [ + { + "workspaceResourceId": "/subscriptions//resourceGroups//providers/Microsoft.OperationalInsights/workspaces/", + "name": "sentinelWorkspace" + } + ] + }, + "dataFlows": [ + { + "streams": [ "Custom-Fivetran_AuditTrail_CL" ], + "destinations": [ "sentinelWorkspace" ], + "transformKql": "source | extend TimeGenerated = captured_at | extend old_values = parse_json(old_values), new_values = parse_json(new_values)", + "outputStream": "Custom-Fivetran_AuditTrail_CL" + }, + { + "streams": [ "Custom-Fivetran_Platform_CL" ], + "destinations": [ "sentinelWorkspace" ], + "transformKql": "source | extend TimeGenerated = now() | extend Record = parse_json(Record)", + "outputStream": "Custom-Fivetran_Platform_CL" + } + ] + } +} diff --git a/Solutions/Fivetran/Platform-Connector-Ingest/function/function_app.py b/Solutions/Fivetran/Platform-Connector-Ingest/function/function_app.py new file mode 100644 index 00000000000..6d44a7f70ee --- /dev/null +++ b/Solutions/Fivetran/Platform-Connector-Ingest/function/function_app.py @@ -0,0 +1,164 @@ +""" +Fivetran Platform Connector -> Sentinel ingest function (all Platform tables). + +Event Grid (Blob Created on the ADLS Gen2 lake) -> this Azure Function -> Azure +Monitor Logs Ingestion API -> DCR (kind: Direct) -> Sentinel custom tables. + +The Fivetran Platform Connector lands 12 tables into the managed data lake, each as +its own Delta + Iceberg (UniForm) table: + + audit_trail, user, team, team_membership, role, role_permission, + role_connector_type, resource_membership, connection, destination, + account, connector_type + +Row parquet lives under .../
/data/*.parquet, alongside a Delta _delta_log/ and +Iceberg metadata/ (*.metadata.json + *.avro). We only process blobs whose path contains +'/
/data/' and end in '.parquet', which skips _delta_log checkpoint parquet, +Iceberg avro/json metadata, and orphans/. + +Routing (two Sentinel tables, one DCR): + - audit_trail -> typed Custom-Fivetran_AuditTrail_CL (security events, typed cols) + - all others -> generic Custom-Fivetran_Platform_CL ({FivetranTable, Record json}) + +Delta/Iceberg compaction can rewrite data files, re-emitting rows already ingested, so +downstream KQL MUST dedupe by the immutable primary key (audit `id`, else per-table id +inside Record). This connector is deliberately at-least-once; correctness comes from the +dedupe (the ASIM parser dedupes audit_trail by id), not the transport. + +Auth is managed identity end to end (DefaultAzureCredential): + - Storage Blob Data Reader on the storage account -> read the parquet + - Monitoring Metrics Publisher on the DCR -> post to logsIngestion + When a user-assigned managed identity is used, set the app setting AZURE_CLIENT_ID to + the identity's client id so DefaultAzureCredential selects it. + +App settings (see local.settings.json.example): + DCR_ENDPOINT DCR logs ingestion endpoint (https://-xxxx..ingest.monitor.azure.com) + DCR_RULE_ID DCR properties.immutableId + AUDIT_STREAM default Custom-Fivetran_AuditTrail_CL + PLATFORM_STREAM default Custom-Fivetran_Platform_CL + AZURE_CLIENT_ID (optional) client id of the user-assigned managed identity +""" + +import io +import json +import logging +import os + +import azure.functions as func +import pyarrow.parquet as pq +from azure.identity import DefaultAzureCredential +from azure.monitor.ingestion import LogsIngestionClient +from azure.storage.blob import BlobClient + +app = func.FunctionApp() + +_credential = DefaultAzureCredential() +_logs_client = LogsIngestionClient( + endpoint=os.environ["DCR_ENDPOINT"], + credential=_credential, + logging_enable=False, +) + +_RULE_ID = os.environ["DCR_RULE_ID"] +_AUDIT_STREAM = os.environ.get("AUDIT_STREAM", "Custom-Fivetran_AuditTrail_CL") +_PLATFORM_STREAM = os.environ.get("PLATFORM_STREAM", "Custom-Fivetran_Platform_CL") + +# The 12 Fivetran Platform Connector tables we ingest. +_TABLES = { + "audit_trail", + "user", + "team", + "team_membership", + "role", + "role_permission", + "role_connector_type", + "resource_membership", + "connection", + "destination", + "account", + "connector_type", +} + +# Typed projection for audit_trail. old_values / new_values are JSON strings in parquet +# and are parsed to dynamic by the DCR transformKql. +_AUDIT_COLUMNS = [ + "id", + "captured_at", + "user_id", + "action", + "interaction_method", + "primary_resource_type", + "primary_resource_id", + "secondary_resource_type", + "secondary_resource_id", + "old_values", + "new_values", +] + + +def _jsonable(value): + """Make parquet cell values JSON-serialisable (datetimes -> ISO 8601).""" + isoformat = getattr(value, "isoformat", None) + return isoformat() if callable(isoformat) else value + + +def _table_name(blob_url: str) -> str | None: + """Return the Fivetran table name from '.../
/data/.parquet'.""" + marker = "/data/" + if marker not in blob_url: + return None + prefix = blob_url.rsplit(marker, 1)[0] + return prefix.rsplit("/", 1)[-1] + + +def _audit_rows(records: list[dict]) -> list[dict]: + rows = [] + for rec in records: + rows.append({col: _jsonable(rec.get(col)) for col in _AUDIT_COLUMNS if col in rec}) + return rows + + +def _platform_rows(table: str, records: list[dict]) -> list[dict]: + rows = [] + for rec in records: + clean = {k: _jsonable(v) for k, v in rec.items()} + rows.append({"FivetranTable": table, "Record": json.dumps(clean, default=str)}) + return rows + + +@app.function_name(name="FivetranPlatformIngest") +@app.event_grid_trigger(arg_name="event") +def ingest(event: func.EventGridEvent): + data = event.get_json() + blob_url = data.get("url", "") + api = data.get("api", "") + + if api not in ("PutBlob", "PutBlockList", "FlushWithClose", ""): + logging.info("Skipping event api=%s for %s", api, blob_url) + return + + if not blob_url.endswith(".parquet") or "/data/" not in blob_url: + logging.info("Skipping non /data/*.parquet blob: %s", blob_url) + return + + table = _table_name(blob_url) + if table not in _TABLES: + logging.info("Skipping unknown/uningested table '%s': %s", table, blob_url) + return + + logging.info("Processing %s blob: %s", table, blob_url) + blob = BlobClient.from_blob_url(blob_url, credential=_credential) + raw = blob.download_blob().readall() + records = pq.read_table(io.BytesIO(raw)).to_pylist() + if not records: + logging.info("No rows in %s", blob_url) + return + + if table == "audit_trail": + stream, rows = _AUDIT_STREAM, _audit_rows(records) + else: + stream, rows = _PLATFORM_STREAM, _platform_rows(table, records) + + # LogsIngestionClient gzips and splits into <=1 MB requests automatically. + _logs_client.upload(rule_id=_RULE_ID, stream_name=stream, logs=rows) + logging.info("Uploaded %d rows from %s to %s", len(rows), table, stream) diff --git a/Solutions/Fivetran/Platform-Connector-Ingest/function/host.json b/Solutions/Fivetran/Platform-Connector-Ingest/function/host.json new file mode 100644 index 00000000000..06d01bdaa95 --- /dev/null +++ b/Solutions/Fivetran/Platform-Connector-Ingest/function/host.json @@ -0,0 +1,15 @@ +{ + "version": "2.0", + "logging": { + "applicationInsights": { + "samplingSettings": { + "isEnabled": true, + "excludedTypes": "Request" + } + } + }, + "extensionBundle": { + "id": "Microsoft.Azure.Functions.ExtensionBundle", + "version": "[4.*, 5.0.0)" + } +} diff --git a/Solutions/Fivetran/Platform-Connector-Ingest/function/local.settings.json.example b/Solutions/Fivetran/Platform-Connector-Ingest/function/local.settings.json.example new file mode 100644 index 00000000000..99cf2aa1983 --- /dev/null +++ b/Solutions/Fivetran/Platform-Connector-Ingest/function/local.settings.json.example @@ -0,0 +1,12 @@ +{ + "IsEncrypted": false, + "Values": { + "AzureWebJobsStorage": "", + "FUNCTIONS_WORKER_RUNTIME": "python", + "DCR_ENDPOINT": "https://-xxxx..ingest.monitor.azure.com", + "DCR_RULE_ID": "dcr-0123456789abcdef0123456789abcdef", + "AUDIT_STREAM": "Custom-Fivetran_AuditTrail_CL", + "PLATFORM_STREAM": "Custom-Fivetran_Platform_CL", + "AZURE_CLIENT_ID": "" + } +} diff --git a/Solutions/Fivetran/Platform-Connector-Ingest/function/requirements.txt b/Solutions/Fivetran/Platform-Connector-Ingest/function/requirements.txt new file mode 100644 index 00000000000..11b096557c1 --- /dev/null +++ b/Solutions/Fivetran/Platform-Connector-Ingest/function/requirements.txt @@ -0,0 +1,5 @@ +azure-functions +azure-identity +azure-monitor-ingestion +azure-storage-blob +pyarrow diff --git a/Solutions/Fivetran/Platform-Connector-Ingest/test/test_audittrail_parser.py b/Solutions/Fivetran/Platform-Connector-Ingest/test/test_audittrail_parser.py new file mode 100644 index 00000000000..2d41483eae5 --- /dev/null +++ b/Solutions/Fivetran/Platform-Connector-Ingest/test/test_audittrail_parser.py @@ -0,0 +1,134 @@ +"""Dummy-data test mirroring vimAuditEventFivetranAuditTrail KQL mapping. + +Python reimplementation of the parser's normalisation + dedupe logic, run over +sample Fivetran_AuditTrail_CL rows to assert ASIM AuditEvent conformance before a +live ASimSchemaTester/ASimDataTester run. Mirrors the pattern of the ASIM-vNext +test/test_parsers.py harness. Run: python test_audittrail_parser.py +""" + +MANDATORY = [ + "EventCount", "EventStartTime", "EventEndTime", "EventType", "EventResult", + "EventProduct", "EventVendor", "EventSchema", "EventSchemaVersion", "Dvc", + "Operation", "Object", +] +ALLOWED_EVENTTYPE = { + "Set", "Read", "Create", "Delete", "Execute", "Install", "Clear", + "Enable", "Disable", "Initialize", "Start", "Stop", "Other", +} + + +def _event_type(action: str) -> str: + a = action.lower() + if any(k in a for k in ("create", "add", "insert")): + return "Create" + if any(k in a for k in ("delete", "remove", "drop")): + return "Delete" + if any(k in a for k in ("pause", "disable")): + return "Disable" + if any(k in a for k in ("resume", "enable")): + return "Enable" + if any(k in a for k in ("edit", "update", "alter", "change", "modify", "set")): + return "Set" + if any(k in a for k in ("read", "view", "access", "get")): + return "Read" + return "Other" + + +def _object_type(prim_type: str) -> str: + p = prim_type.lower() + if any(k in p for k in ("connector", "connection", "destination")): + return "Cloud Resource" + if any(k in p for k in ("user", "team", "role", "account", "membership")): + return "Directory Service Object" + return "Other" + + +def normalise(rows): + # Dedupe by id keeping latest TimeGenerated (mirrors arg_max(TimeGenerated,*) by id). + latest = {} + for r in rows: + cur = latest.get(r["id"]) + if cur is None or r["TimeGenerated"] > cur["TimeGenerated"]: + latest[r["id"]] = r + out = [] + for r in latest.values(): + out.append({ + "EventCount": 1, + "EventStartTime": r["captured_at"], + "EventEndTime": r["captured_at"], + "Operation": r["action"], + "EventType": _event_type(r["action"]), + "EventResult": "Success", + "EventProduct": "Fivetran Platform", + "EventVendor": "Fivetran", + "EventSchema": "AuditEvent", + "EventSchemaVersion": "0.1.2", + "Dvc": "Fivetran", + "Object": r["primary_resource_id"] or r["primary_resource_type"] or "Unknown", + "ObjectType": _object_type(r["primary_resource_type"]), + "ActorUserId": r["user_id"], + "ActorUsername": r["user_id"] or "Unknown", + "OldValue": r["old_values"], + "NewValue": r["new_values"], + "EventOriginalUid": r["id"], + }) + return out + + +SAMPLE = [ + {"id": "a1", "TimeGenerated": "2026-07-20T01:00:00Z", "captured_at": "2026-07-20T01:00:00Z", + "user_id": "u_100", "action": "CREATE_CONNECTION", "interaction_method": "api", + "primary_resource_type": "connection", "primary_resource_id": "c_55", + "secondary_resource_type": "", "secondary_resource_id": "", "old_values": "{}", "new_values": '{"name":"pg"}'}, + {"id": "a2", "TimeGenerated": "2026-07-20T02:00:00Z", "captured_at": "2026-07-20T02:00:00Z", + "user_id": "u_101", "action": "DELETE_DESTINATION", "interaction_method": "ui", + "primary_resource_type": "destination", "primary_resource_id": "d_9", + "secondary_resource_type": "", "secondary_resource_id": "", "old_values": '{"x":1}', "new_values": "{}"}, + {"id": "a3", "TimeGenerated": "2026-07-20T03:00:00Z", "captured_at": "2026-07-20T03:00:00Z", + "user_id": "u_102", "action": "EDIT_ROLE_PERMISSION", "interaction_method": "api", + "primary_resource_type": "role", "primary_resource_id": "r_3", + "secondary_resource_type": "connector_type", "secondary_resource_id": "postgres", + "old_values": '{"perm":"read"}', "new_values": '{"perm":"admin"}'}, + # Duplicate of a1 from a compaction rewrite, later TimeGenerated -> must dedupe to one. + {"id": "a1", "TimeGenerated": "2026-07-21T05:00:00Z", "captured_at": "2026-07-20T01:00:00Z", + "user_id": "u_100", "action": "CREATE_CONNECTION", "interaction_method": "api", + "primary_resource_type": "connection", "primary_resource_id": "c_55", + "secondary_resource_type": "", "secondary_resource_id": "", "old_values": "{}", "new_values": '{"name":"pg"}'}, +] + + +def main(): + checks = 0 + result = normalise(SAMPLE) + + assert len(result) == 3, f"dedupe failed: expected 3 unique ids, got {len(result)}" + checks += 1 + + by_id = {r["EventOriginalUid"]: r for r in result} + assert by_id["a1"]["EventType"] == "Create" + checks += 1 + assert by_id["a2"]["EventType"] == "Delete" + checks += 1 + assert by_id["a3"]["EventType"] == "Set" + checks += 1 + assert by_id["a1"]["ObjectType"] == "Cloud Resource" + checks += 1 + assert by_id["a3"]["ObjectType"] == "Directory Service Object" + checks += 1 + assert by_id["a3"]["NewValue"] == '{"perm":"admin"}' + checks += 1 + assert by_id["a2"]["ActorUserId"] == "u_101" + checks += 1 + + for r in result: + for f in MANDATORY: + assert f in r and r[f] not in (None, ""), f"missing mandatory {f}" + assert r["EventResult"] == "Success" + assert r["EventType"] in ALLOWED_EVENTTYPE, r["EventType"] + checks += 1 + + print(f"OK - {checks} check groups passed over {len(result)} normalised rows") + + +if __name__ == "__main__": + main() diff --git a/Solutions/Fivetran/README.md b/Solutions/Fivetran/README.md new file mode 100644 index 00000000000..726696e392d --- /dev/null +++ b/Solutions/Fivetran/README.md @@ -0,0 +1,62 @@ +# Fivetran - Microsoft Sentinel Solution + +Ingests Fivetran platform and connector logs into a DCR-based custom table +(`Fivetran_CL`) in Microsoft Sentinel using the Azure Monitor +**Logs Ingestion API**. Fivetran PUSHES its logs to the Data Collection Rule +using an Entra ID app registration; nothing polls Fivetran. + +## Contents + +| Artifact | File | +| --- | --- | +| Custom table (ARM) | `Data Connectors/Fivetran_CCF/Fivetran_Table.json` | +| Data Collection Rule (ARM) | `Data Connectors/Fivetran_CCF/Fivetran_DCR.json` | +| Data connector tile | `Data Connectors/Fivetran_CCF/Fivetran_DataConnector.json` | +| Parser (KQL function `Fivetran`) | `Parsers/Fivetran.yaml` | +| ASIM Authentication parser (parameterless) | `Parsers/ASimAuthenticationFivetran.yaml` | +| ASIM Authentication parser (filtering) | `Parsers/vimAuthenticationFivetran.yaml` | +| ASIM AuditEvent parser (parameterless) | `Parsers/ASimAuditEventFivetran.yaml` | +| ASIM AuditEvent parser (filtering) | `Parsers/vimAuditEventFivetran.yaml` | +| Analytics rule - ingestion gap (Defense Evasion / T1562) | `Analytic Rules/FivetranIngestionGap.yaml` | +| Analytics rule - auth failures (Credential Access / T1110) | `Analytic Rules/FivetranAuthFailures.yaml` | +| Hunting query - SEVERE spike | `Hunting Queries/FivetranSevereSpike.yaml` | +| Overview workbook | `Workbooks/Fivetran.json` | +| Solution manifest | `Data/Solution_Fivetran.json` | +| Publisher metadata | `SolutionMetadata.json` | +| Change history | `ReleaseNotes.md` | + +## Deployment order + +1. Deploy `Fivetran_Table.json` and `Fivetran_DCR.json`. The data connector + tile's instruction steps cover the Entra ID app registration, Data Collection + Endpoint and `Monitoring Metrics Publisher` RBAC assignment on the DCR. +2. Configure Fivetran's external-log (Azure Monitor) connector in Logs Ingestion + API mode with the DCR values. +3. Install the parser, analytics rule, hunting query and workbook (bundled when + the solution is packaged - see `PACKAGING.md`). + +## Design note - push, not poll + +This is a push feed, so the solution deliberately has **no** `PollingConfig.json` +and **no** CCF `DataConnectorDefinition.json` (kind Customizable). The connector +tile is a classic status/documentation connector whose connection state is +computed by a KQL `IsConnectedQuery` against the custom table. Data movement is +performed by Fivetran plus the DCR, not by Sentinel. + +## Alternative connection paths + +The shipped solution above is the real-time push feed (External Logs -> DCR -> +`Fivetran_CL`). For the structured Fivetran **Platform Connector** audit data (the +full `AUDIT_TRAIL` and metadata tables, which are not carried by the External Logs +feed), a self-contained reference build is provided under +[`Platform-Connector-Ingest/`](Platform-Connector-Ingest/README.md): an Event Grid +triggered Azure Function reads the lake parquet and ingests it into typed Sentinel +tables (`Fivetran_AuditTrail_CL`, `Fivetran_Platform_CL`), with an ASIM AuditEvent +parser and hunting query. It is documented IaC (not part of this Content Hub package) +because it depends on a customer-owned ADLS Gen2 lake. + +## Prerequisites (customer) + +- Log Analytics workspace with Microsoft Sentinel enabled. +- Entra ID app registration (Client ID, Client Secret, Tenant ID). +- `Monitoring Metrics Publisher` role for that app on the DCR. diff --git a/Solutions/Fivetran/ReleaseNotes.md b/Solutions/Fivetran/ReleaseNotes.md new file mode 100644 index 00000000000..e7204e55bbc --- /dev/null +++ b/Solutions/Fivetran/ReleaseNotes.md @@ -0,0 +1,4 @@ +| **Version** | **Date Modified (DD-MM-YYYY)** | **Change History** | +|-------------|--------------------------------|---------------------------------------------| +| 3.0.1 | 29-07-2026 | Added four ASIM parsers over `Fivetran_CL`: Authentication (`ASimAuthenticationFivetran` / `vimAuthenticationFivetran`) and AuditEvent (`ASimAuditEventFivetran` / `vimAuditEventFivetran`), enabling normalized cross-source queries and ASIM-based analytics. | +| 3.0.0 | 22-07-2026 | Initial Solution release. DCR-based custom table (Logs Ingestion API), data-connector tile, parser, two analytics rules (ingestion-gap / Defense Evasion T1562 and auth-failures / Credential Access T1110), SEVERE-spike hunting query and overview workbook. | diff --git a/Solutions/Fivetran/SolutionMetadata.json b/Solutions/Fivetran/SolutionMetadata.json new file mode 100644 index 00000000000..4769e400aa8 --- /dev/null +++ b/Solutions/Fivetran/SolutionMetadata.json @@ -0,0 +1,15 @@ +{ + "publisherId": "michaelwoods", + "offerId": "azure-sentinel-solution-fivetran", + "firstPublishDate": "2026-07-22", + "providers": [ "Michael Woods" ], + "categories": { + "domains": [ "IT Operations", "Security - Threat Protection" ] + }, + "support": { + "name": "Michael Woods", + "email": "michaelwoods@microsoft.com", + "tier": "Community", + "link": "https://github.com/woodsonsecurity" + } +} diff --git a/Solutions/Fivetran/Workbooks/Fivetran.json b/Solutions/Fivetran/Workbooks/Fivetran.json new file mode 100644 index 00000000000..a50e8a7367c --- /dev/null +++ b/Solutions/Fivetran/Workbooks/Fivetran.json @@ -0,0 +1,50 @@ +{ + "version": "Notebook/1.0", + "items": [ + { + "type": 1, + "content": { + "json": "# Fivetran - Log Overview\nOverview of Fivetran platform and connector logs ingested into Microsoft Sentinel via the Logs Ingestion API (Fivetran_CL)." + }, + "name": "title" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "Fivetran_CL\n| summarize Count = count() by Level\n| render piechart", + "size": 1, + "title": "Events by level", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces" + }, + "name": "byLevel" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "Fivetran_CL\n| summarize Count = count() by bin(TimeGenerated, 1h), Level\n| render timechart", + "size": 0, + "title": "Event volume over time", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces" + }, + "name": "overTime" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "Fivetran_CL\n| where Level == 'SEVERE'\n| project TimeGenerated, Level, Message\n| sort by TimeGenerated desc\n| take 100", + "size": 0, + "title": "Recent SEVERE events", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces" + }, + "name": "recentSevere" + } + ], + "fallbackResourceIds": [], + "$schema": "https://github.com/Microsoft/Application-Insights-Workbooks/blob/master/schema/workbook.json" +}