|
| 1 | +function Get-CIPPGeoIPLocationBatch { |
| 2 | + <# |
| 3 | + .SYNOPSIS |
| 4 | + Resolve many IPs to geo-location in one pass, warming the knownlocationdbv2 cache. |
| 5 | + .DESCRIPTION |
| 6 | + Normalizes + de-dupes the input IPs and drops redacted / reserved / private / link-local |
| 7 | + addresses (never geolocatable). Remaining IPs are seeded from knownlocationdbv2 (fresh |
| 8 | + entries only); cache misses are resolved in bulk via the geoipdb /GetIPInfoBatch endpoint |
| 9 | + (which proxies ip-api's batch API, 100 IPs per upstream request). Successful results are |
| 10 | + written back to knownlocationdbv2 and cachegeoip so later processing is a cache hit. |
| 11 | +
|
| 12 | + Returns a hashtable keyed by normalized IP -> flattened location object |
| 13 | + @{ CountryOrRegion; City; Proxy; Hosting; ASName }. Failed/unknown lookups are NOT cached |
| 14 | + (no poisoning) and are absent from the returned hashtable. |
| 15 | +
|
| 16 | + Used both at ingestion (warm the cache up front) and as a per-batch prefetch in the audit |
| 17 | + log processor (so the per-record loop is a pure in-memory lookup). |
| 18 | + .PARAMETER IPs |
| 19 | + IP addresses to resolve. Duplicates, reserved IPs and ports are handled automatically. |
| 20 | + #> |
| 21 | + [CmdletBinding()] |
| 22 | + [OutputType([hashtable])] |
| 23 | + param( |
| 24 | + [string[]]$IPs |
| 25 | + ) |
| 26 | + |
| 27 | + # 20s timeout, up to 3 attempts for the geoip HTTP calls. The short timeout stops a single |
| 28 | + # hung IP from stalling the whole batch; the retries ride out transient blips before we give up. |
| 29 | + function Invoke-GeoRetry { |
| 30 | + param([string]$Uri, [string]$Method = 'GET', $Body, [string]$ContentType, [int]$Retries = 3, [int]$TimeoutSec = 20) |
| 31 | + $lastErr = $null |
| 32 | + for ($attempt = 1; $attempt -le $Retries; $attempt++) { |
| 33 | + try { |
| 34 | + if ($PSBoundParameters.ContainsKey('Body')) { |
| 35 | + return Invoke-CIPPRestMethod -Uri $Uri -Method $Method -Body $Body -ContentType $ContentType -TimeoutSec $TimeoutSec |
| 36 | + } else { |
| 37 | + return Invoke-CIPPRestMethod -Uri $Uri -Method $Method -TimeoutSec $TimeoutSec |
| 38 | + } |
| 39 | + } catch { |
| 40 | + $lastErr = $_ |
| 41 | + if ($attempt -lt $Retries) { Start-Sleep -Milliseconds (300 * $attempt) } |
| 42 | + } |
| 43 | + } |
| 44 | + throw $lastErr |
| 45 | + } |
| 46 | + |
| 47 | + $ClientIpRegex = [regex]'^(?<IP>(?:\d{1,3}(?:\.\d{1,3}){3}|\[[0-9a-fA-F:]+\]|[0-9a-fA-F:]+))(?::\d+)?$' |
| 48 | + $ReservedIpRegex = [regex]::new( |
| 49 | + '^(?:10\.|127\.|0\.|169\.254\.|192\.168\.|172\.(?:1[6-9]|2[0-9]|3[01])\.|100\.(?:6[4-9]|[7-9][0-9]|1[01][0-9]|12[0-7])\.|(?:22[4-9]|23[0-9]|24[0-9]|25[0-5])\.|::1?$|fe[89ab]|f[cd]|ff)', |
| 50 | + [System.Text.RegularExpressions.RegexOptions]::IgnoreCase) |
| 51 | + |
| 52 | + $Result = @{} |
| 53 | + |
| 54 | + # Normalize (strip :port / brackets), drop redacted + reserved, de-dupe |
| 55 | + $Distinct = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase) |
| 56 | + foreach ($ip in $IPs) { |
| 57 | + if ([string]::IsNullOrWhiteSpace($ip)) { continue } |
| 58 | + $clean = $ClientIpRegex.Replace(([string]$ip).Trim(), '$1') -replace '[\[\]]', '' |
| 59 | + if ([string]::IsNullOrWhiteSpace($clean) -or $clean -match '[X]+') { continue } |
| 60 | + if ($ReservedIpRegex.IsMatch($clean)) { continue } |
| 61 | + $null = $Distinct.Add($clean) |
| 62 | + } |
| 63 | + if ($Distinct.Count -eq 0) { return $Result } |
| 64 | + |
| 65 | + $LocationTable = Get-CIPPTable -TableName 'knownlocationdbv2' |
| 66 | + $ValidAfter = (Get-Date).AddDays(-90).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') |
| 67 | + |
| 68 | + # 1) Seed from knownlocationdbv2 (fresh, non-Unknown entries); collect the misses |
| 69 | + $ToResolve = [System.Collections.Generic.List[string]]::new() |
| 70 | + foreach ($ip in $Distinct) { |
| 71 | + $cached = Get-CIPPAzDataTableEntity @LocationTable -Filter "PartitionKey eq 'ip' and RowKey eq '$ip' and Timestamp ge datetime'$ValidAfter'" |
| 72 | + if ($cached -and $cached.CountryOrRegion -and $cached.CountryOrRegion -ne 'Unknown') { |
| 73 | + $Result[$ip] = [pscustomobject]@{ |
| 74 | + CountryOrRegion = $cached.CountryOrRegion |
| 75 | + City = $cached.City |
| 76 | + Proxy = $cached.Proxy |
| 77 | + Hosting = $cached.Hosting |
| 78 | + ASName = $cached.ASName |
| 79 | + } |
| 80 | + } else { |
| 81 | + $ToResolve.Add($ip) |
| 82 | + } |
| 83 | + } |
| 84 | + if ($ToResolve.Count -eq 0) { return $Result } |
| 85 | + |
| 86 | + # 2) Bulk-resolve the misses via geoipdb /GetIPInfoBatch (chunk to 100 to bound payloads) |
| 87 | + $CacheGeoIPTable = Get-CippTable -TableName 'cachegeoip' |
| 88 | + $KnownEntities = [System.Collections.Generic.List[object]]::new() |
| 89 | + $CacheGeoEntities = [System.Collections.Generic.List[object]]::new() |
| 90 | + |
| 91 | + for ($i = 0; $i -lt $ToResolve.Count; $i += 100) { |
| 92 | + $chunk = @($ToResolve[$i..([Math]::Min($i + 99, $ToResolve.Count - 1))]) |
| 93 | + $payload = '[' + (($chunk | ForEach-Object { $_ | ConvertTo-Json }) -join ',') + ']' |
| 94 | + $resp = $null |
| 95 | + try { |
| 96 | + $resp = Invoke-GeoRetry -Uri 'https://geoipdb.azurewebsites.net/api/GetIPInfoBatch' -Method POST -Body $payload -ContentType 'application/json' |
| 97 | + } catch { |
| 98 | + #Write-LogMessage -API GeoIPLocation -message "Bulk geoip lookup failed, falling back to single lookups for $($chunk.Count) IP(s): $($_.Exception.Message)" -sev Warning |
| 99 | + $fb = [System.Collections.Generic.List[object]]::new() |
| 100 | + foreach ($ip in $chunk) { |
| 101 | + try { |
| 102 | + $s = Invoke-GeoRetry -Uri "https://geoipdb.azurewebsites.net/api/GetIPInfo?IP=$ip" |
| 103 | + if ($s -and $s.status -ne 'fail') { $fb.Add([pscustomobject]@{ query = $ip; status = 'success'; countryCode = $s.countryCode; city = $s.city; proxy = $s.proxy; hosting = $s.hosting; asname = $s.asname }) } |
| 104 | + } catch { } |
| 105 | + } |
| 106 | + $resp = $fb |
| 107 | + } |
| 108 | + foreach ($r in $resp) { |
| 109 | + $ip = [string]$r.query |
| 110 | + if ([string]::IsNullOrWhiteSpace($ip) -or $r.status -ne 'success') { continue } |
| 111 | + $loc = [pscustomobject]@{ |
| 112 | + CountryOrRegion = if ($r.countryCode) { $r.countryCode } else { 'Unknown' } |
| 113 | + City = if ($r.city) { $r.city } else { 'Unknown' } |
| 114 | + Proxy = if ($null -ne $r.proxy) { $r.proxy } else { 'Unknown' } |
| 115 | + Hosting = if ($null -ne $r.hosting) { $r.hosting } else { 'Unknown' } |
| 116 | + ASName = if ($r.asname) { $r.asname } else { 'Unknown' } |
| 117 | + } |
| 118 | + $Result[$ip] = $loc |
| 119 | + # Only cache real results - never persist Unknown (no poisoning, matches single path) |
| 120 | + if ($loc.CountryOrRegion -ne 'Unknown') { |
| 121 | + $KnownEntities.Add(@{ |
| 122 | + PartitionKey = 'ip' |
| 123 | + RowKey = $ip |
| 124 | + CountryOrRegion = "$($loc.CountryOrRegion)" |
| 125 | + City = "$($loc.City)" |
| 126 | + Proxy = "$($loc.Proxy)" |
| 127 | + Hosting = "$($loc.Hosting)" |
| 128 | + ASName = "$($loc.ASName)" |
| 129 | + }) |
| 130 | + $CacheGeoEntities.Add(@{ |
| 131 | + PartitionKey = 'IP' |
| 132 | + RowKey = $ip |
| 133 | + Data = [string]($r | ConvertTo-Json -Compress) |
| 134 | + }) |
| 135 | + } |
| 136 | + } |
| 137 | + } |
| 138 | + |
| 139 | + # 3) Batch-write the caches |
| 140 | + if ($KnownEntities.Count -gt 0) { |
| 141 | + try { $null = Add-CIPPAzDataTableEntity @LocationTable -Entity @($KnownEntities) -Force } |
| 142 | + catch { Write-LogMessage -API GeoIPLocation -message "Failed to cache $($KnownEntities.Count) bulk geo results: $($_.Exception.Message)" -sev Warning } |
| 143 | + } |
| 144 | + if ($CacheGeoEntities.Count -gt 0) { |
| 145 | + try { $null = Add-AzDataTableEntity @CacheGeoIPTable -Entity @($CacheGeoEntities) -Force } catch {} |
| 146 | + } |
| 147 | + |
| 148 | + return $Result |
| 149 | +} |
![pull[bot]](https://avatars.githubusercontent.com/in/12910?v=4&size=40){kind=avatar}
0 commit comments