|
| 1 | +# Copyright (c) Microsoft Corporation. |
| 2 | +# Licensed under the MIT License. |
| 3 | + |
| 4 | +<# |
| 5 | +.SYNOPSIS |
| 6 | + Scans test data files for sensitive data patterns. |
| 7 | +.DESCRIPTION |
| 8 | + Checks staged test data files for email addresses and domain names outside |
| 9 | + allowed test domains, public IP addresses, and credential-like patterns. |
| 10 | +#> |
| 11 | +[CmdletBinding()] |
| 12 | +param( |
| 13 | + [Parameter(Mandatory)] |
| 14 | + [string[]]$Files |
| 15 | +) |
| 16 | + |
| 17 | +$failed = $false |
| 18 | + |
| 19 | +# RFC-reserved TLDs that can't be publicly registered (safe for test data) |
| 20 | +# cspell:ignore Tlds |
| 21 | +$reservedTlds = @('local', 'test', 'example', 'invalid', 'localhost', 'internal', 'lan') |
| 22 | + |
| 23 | +# Allowed domains in test data (entries are regex fragments with escaped dots) |
| 24 | +# cspell:ignore vnext apikey fabrikam microsoftonline cloudapp |
| 25 | +$allowedDomains = @( |
| 26 | + 'fabrikam\.com', |
| 27 | + 'example\.com', |
| 28 | + 'contoso\.com', |
| 29 | + 'contoso\.lab', |
| 30 | + 'contoso\.mail\.onmicrosoft\.com', |
| 31 | + # Microsoft service domains (product constants in Exchange configuration) |
| 32 | + 'microsoft\.com', |
| 33 | + 'microsoftonline-p\.com', |
| 34 | + 'microsoftonline\.com', |
| 35 | + 'outlook\.com', |
| 36 | + 'live\.com', |
| 37 | + 'live-int\.com', |
| 38 | + 'office365\.com', |
| 39 | + 'office\.com', |
| 40 | + 'msn\.com', |
| 41 | + 'passport\.com', |
| 42 | + 'cloudapp\.net' |
| 43 | +) |
| 44 | +$allowedDomainsPattern = $allowedDomains -join '|' |
| 45 | + |
| 46 | +# Non-public IP ranges: RFC 1918 + loopback + APIPA + CGNAT + TEST-NET + benchmark + multicast + broadcast |
| 47 | +# cspell:ignore APIPA CGNAT |
| 48 | +$privateIpPattern = '^(10\.|172\.(1[6-9]|2[0-9]|3[01])\.|192\.168\.|127\.|0\.0\.0\.0|255\.255\.255\.255|169\.254\.|100\.(6[4-9]|[7-9][0-9]|1[01][0-9]|12[0-7])\.|192\.0\.2\.|198\.51\.100\.|203\.0\.113\.|198\.1[89]\.|22[4-9]\.|2[3-5][0-9]\.)' |
| 49 | + |
| 50 | +# Specific email addresses that are allowed regardless of domain |
| 51 | +$allowedEmails = @( |
| 52 | + 'ExToolsFeedback@microsoft.com' |
| 53 | +) |
| 54 | + |
| 55 | +foreach ($file in $Files) { |
| 56 | + if (-not (Test-Path $file)) { continue } |
| 57 | + |
| 58 | + try { |
| 59 | + $lines = @(Get-Content -Path $file -ErrorAction Stop) |
| 60 | + } catch { |
| 61 | + Write-Host " BLOCKED: $file - Unable to read file: $($_.Exception.Message)" -ForegroundColor Red |
| 62 | + $failed = $true |
| 63 | + continue |
| 64 | + } |
| 65 | + if ($lines.Count -eq 0) { continue } |
| 66 | + |
| 67 | + $domainsInFile = @{} |
| 68 | + |
| 69 | + # Patterns defined once outside the per-line loop for performance |
| 70 | + $tldPattern = '(com|net|org|edu|gov|io|info|biz|co|us|uk|de|au|ca|jp|fr|in|eu|cloud|app|dev|lab)' |
| 71 | + $fqdnRegex = '(?i)\b([a-zA-Z0-9][a-zA-Z0-9-]*(?:\.[a-zA-Z0-9-]+)*\.' + $tldPattern + ')\b' |
| 72 | + $credentialKeywords = 'password|secret|apikey|token|credential' |
| 73 | + $safeValues = @('true', 'false', 'Enabled', 'Disabled', 'Changed', 'None', 'NotConfigured') |
| 74 | + $assignRegex = '(?i)\b(?:' + $credentialKeywords + ')\s*[:=]\s*[''"]([^''"]+)[''"]' |
| 75 | + $xmlAttrRegex = '(?i)(?:key|name)\s*=\s*[''"](?:' + $credentialKeywords + ')[''"].*?value\s*=\s*[''"]([^''"]+)[''"]' |
| 76 | + # cspell:ignore clixml |
| 77 | + $clixmlRegex = '(?i)N\s*=\s*[''"](?:' + $credentialKeywords + ')[''"]>([^<]+)<' |
| 78 | + |
| 79 | + $lineNumber = 0 |
| 80 | + foreach ($line in $lines) { |
| 81 | + $lineNumber++ |
| 82 | + |
| 83 | + # Check for email addresses outside allowed domains |
| 84 | + $emailMatches = [regex]::Matches($line, '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}') |
| 85 | + foreach ($match in $emailMatches) { |
| 86 | + if ($match.Value -in $allowedEmails) { continue } |
| 87 | + $emailTld = ($match.Value -split '\.')[-1].ToLower() |
| 88 | + if ($emailTld -in $reservedTlds) { continue } |
| 89 | + if ($match.Value -notmatch "@(?:.*\.)?($allowedDomainsPattern)$") { |
| 90 | + Write-Host " BLOCKED: $file`:$lineNumber - Unrecognized email domain: $($match.Value)" -ForegroundColor Red |
| 91 | + $failed = $true |
| 92 | + } |
| 93 | + } |
| 94 | + |
| 95 | + # Check for bare domain names outside allowed domains |
| 96 | + # Skip XML namespaces, .NET assembly/binary contexts, and known product strings |
| 97 | + # cspell:ignore fqdn msilog microsft |
| 98 | + # Note: \\Microsft\.Net\\ is a known typo in the default IIS web.config comment template |
| 99 | + # shipped with Exchange ("located in \Windows\Microsft.Net\Frameworks\v2.x\Config") |
| 100 | + if ($line -notmatch 'xmlns|assembly|codeBase|PublicKeyToken|\.dll|\.exe|\\Microsoft\.NET\\|\\Microsft\.Net\\|ASP\.NET|WMI\.NET|\.msilog') { |
| 101 | + # Match FQDNs ending in known public TLDs |
| 102 | + $fqdnMatches = [regex]::Matches($line, $fqdnRegex) |
| 103 | + foreach ($fqdnMatch in $fqdnMatches) { |
| 104 | + $fqdn = $fqdnMatch.Groups[1].Value |
| 105 | + # Skip Windows file paths and email domain portions (already checked by email scanner) |
| 106 | + $matchIndex = $fqdnMatch.Index |
| 107 | + if ($matchIndex -ge 1 -and $line.Substring($matchIndex - 1, 1) -in @('\', '@')) { continue } |
| 108 | + if ($fqdn -match '(?i)^System\.') { continue } |
| 109 | + if ($fqdn -notmatch "(?i)(?:^|\.)($allowedDomainsPattern)$") { |
| 110 | + $fqdnLower = $fqdn.ToLower() |
| 111 | + if (-not $domainsInFile.ContainsKey($fqdnLower)) { |
| 112 | + $domainsInFile[$fqdnLower] = 0 |
| 113 | + } |
| 114 | + $domainsInFile[$fqdnLower]++ |
| 115 | + } |
| 116 | + } |
| 117 | + } |
| 118 | + |
| 119 | + # Check for public IP addresses - only on lines that look like network/address context |
| 120 | + # CLIXML files contain many dotted-quad version numbers (assembly versions, OIDs, etc.) |
| 121 | + # so we only flag IPs on lines with network-related property names |
| 122 | + if ($line -match 'Address|IPAddress|IPRange|Subnet|Gateway|DNS|Binding|Proxy|SmartHost|Remote|Endpoint') { |
| 123 | + $ipMatches = [regex]::Matches($line, '\b(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\b') |
| 124 | + foreach ($match in $ipMatches) { |
| 125 | + $ip = $match.Value |
| 126 | + $octets = $ip -split '\.' |
| 127 | + # Skip if first octet is 0 or any octet > 255 (not a valid IP) |
| 128 | + if ([int]$octets[0] -eq 0 -or ($octets | Where-Object { [int]$_ -gt 255 }).Count -gt 0) { continue } |
| 129 | + # Skip subnet masks |
| 130 | + if ($ip -match '^255\.') { continue } |
| 131 | + # Skip version-like patterns |
| 132 | + if ($line -match 'Version|Build|FileVersion|ProductVersion|ExchangeBuild') { continue } |
| 133 | + if ($ip -notmatch $privateIpPattern) { |
| 134 | + Write-Host " BLOCKED: $file`:$lineNumber - Public IP address: $ip" -ForegroundColor Red |
| 135 | + $failed = $true |
| 136 | + } |
| 137 | + } |
| 138 | + } |
| 139 | + |
| 140 | + # Check for credential patterns in multiple formats |
| 141 | + $credentialFound = $false |
| 142 | + |
| 143 | + # Assignment form: password = "value", secret: 'value' |
| 144 | + $assignMatches = [regex]::Matches($line, $assignRegex) |
| 145 | + foreach ($m in $assignMatches) { |
| 146 | + if ($m.Groups[1].Value -notin $safeValues) { |
| 147 | + $credentialFound = $true |
| 148 | + break |
| 149 | + } |
| 150 | + } |
| 151 | + |
| 152 | + # XML attribute form: key="password" value="secret" |
| 153 | + if (-not $credentialFound) { |
| 154 | + $xmlAttrMatches = [regex]::Matches($line, $xmlAttrRegex) |
| 155 | + foreach ($m in $xmlAttrMatches) { |
| 156 | + if ($m.Groups[1].Value -notin $safeValues) { |
| 157 | + $credentialFound = $true |
| 158 | + break |
| 159 | + } |
| 160 | + } |
| 161 | + } |
| 162 | + |
| 163 | + # CLIXML element form: N="Password">value</ |
| 164 | + if (-not $credentialFound) { |
| 165 | + $clixmlMatches = [regex]::Matches($line, $clixmlRegex) |
| 166 | + foreach ($m in $clixmlMatches) { |
| 167 | + if ($m.Groups[1].Value.Trim() -notin $safeValues) { |
| 168 | + $credentialFound = $true |
| 169 | + break |
| 170 | + } |
| 171 | + } |
| 172 | + } |
| 173 | + |
| 174 | + if ($credentialFound) { |
| 175 | + Write-Host " BLOCKED: $file`:$lineNumber - Possible credential value detected" -ForegroundColor Red |
| 176 | + $failed = $true |
| 177 | + } |
| 178 | + } |
| 179 | + |
| 180 | + # Report unrecognized domains found in this file (one line per domain) |
| 181 | + foreach ($entry in $domainsInFile.GetEnumerator() | Sort-Object -Property Value -Descending) { |
| 182 | + Write-Host " BLOCKED: $file - Unrecognized domain: $($entry.Key) ($($entry.Value) occurrences)" -ForegroundColor Red |
| 183 | + $failed = $true |
| 184 | + } |
| 185 | +} |
| 186 | + |
| 187 | +if ($failed) { |
| 188 | + Write-Host "`nSensitive data check failed. Review flagged items above." -ForegroundColor Red |
| 189 | + return 1 |
| 190 | +} else { |
| 191 | + Write-Host " No sensitive data found." -ForegroundColor Green |
| 192 | + return 0 |
| 193 | +} |
0 commit comments