-
Notifications
You must be signed in to change notification settings - Fork 49
fix(pii): never echo original PII and generate guaranteed-invalid dummies #594
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
e80aceb
571d443
6d677d8
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| package pii | ||
|
|
||
| import ( | ||
| "strings" | ||
| "testing" | ||
| ) | ||
|
|
||
| // generatorLabels is every label GenerateReplacement routes to a dedicated | ||
| // generator, plus an unknown label to cover the generic fallback path. | ||
| var generatorLabels = []string{ | ||
| labelSurname, labelFirstName, labelBuildingNum, labelDateOfBirth, | ||
| labelEmail, labelPhoneNumber, labelCity, labelURL, labelCompanyName, | ||
| labelState, labelZip, labelStreet, labelCountry, labelSSN, | ||
| labelDriverLicenseNum, labelPassportID, labelNationalID, labelIDCardNum, | ||
| labelTaxNum, labelLicensePlateNum, labelPassword, labelIBAN, labelAge, | ||
| labelSecurityToken, labelCreditCardNumber, labelUsername, | ||
| "UNKNOWNLABEL", | ||
| } | ||
|
|
||
| // A replacement equal to the original would leak the PII it is supposed to | ||
| // mask, so GenerateReplacement guarantees inequality. Feeding a previous | ||
| // output back in as the original is the most collision-prone input possible, | ||
| // and must still always produce something different. | ||
| func TestGenerateReplacementNeverReturnsOriginal(t *testing.T) { | ||
| service := NewGeneratorService() | ||
| for _, label := range generatorLabels { | ||
| original := service.GenerateReplacement(label, "") | ||
| for i := 0; i < 100; i++ { | ||
| if replacement := service.GenerateReplacement(label, original); replacement == original { | ||
| t.Fatalf("GenerateReplacement(%q) returned the original %q", label, original) | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| func TestGenerateReplacementUnknownLabelUsesPlaceholder(t *testing.T) { | ||
| service := NewGeneratorService() | ||
| replacement := service.GenerateReplacement("SOMETHINGELSE", "raw value") | ||
| if !strings.HasPrefix(replacement, "[REDACTED_SOMETHINGELSE_") { | ||
| t.Errorf("expected generic placeholder for unknown label, got %q", replacement) | ||
| } | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -71,44 +71,78 @@ func EmailGenerator(rng *rand.Rand, original string) string { | |
| return fmt.Sprintf("%s.%s@%s", firstName, lastName, domain) | ||
| } | ||
|
|
||
| // PhoneGenerator generates dummy phone numbers | ||
| // PhoneGenerator generates dummy phone numbers. The exchange is fixed to 555 | ||
| // and the line number to 0100-0199 — the NANP block reserved for fictional | ||
| // use — so the output looks real but can never be a dialable subscriber | ||
| // number belonging to someone. | ||
| func PhoneGenerator(rng *rand.Rand, original string) string { | ||
| // Generate a random 3-digit area code (200-999) | ||
| areaCode := 200 + rng.Intn(800) | ||
|
|
||
| // Generate a random 3-digit exchange (200-999) | ||
| exchange := 200 + rng.Intn(800) | ||
|
|
||
| // Generate a random 4-digit number | ||
| number := 1000 + rng.Intn(9000) | ||
| // Line number in the reserved fictional range 0100-0199 | ||
| line := 100 + rng.Intn(100) | ||
|
|
||
| // Randomly choose format | ||
| formats := []string{"%d-%d-%d", "%d.%d.%d", "(%d) %d-%d"} | ||
| formats := []string{"%d-555-%04d", "%d.555.%04d", "(%d) 555-%04d"} | ||
| format := formats[rng.Intn(len(formats))] | ||
|
|
||
| return fmt.Sprintf(format, areaCode, exchange, number) | ||
| return fmt.Sprintf(format, areaCode, line) | ||
| } | ||
|
|
||
| // SSNGenerator generates dummy SSN numbers (SOCIALNUM) | ||
| // SSNGenerator generates dummy SSN numbers (SOCIALNUM). The area number is | ||
| // drawn from 900-999, which the SSA has never issued and has reserved against | ||
| // future issuance (every issued prefix, including everything in the SSA high | ||
| // group list, lies in 001-899), so the output looks real but can never | ||
| // collide with an actual person's SSN. The group number stays in 10-49 | ||
| // because IRS ITINs also start with 9 and use group ranges 50-65, 70-88, | ||
| // 90-92, and 94-99; staying below 50 avoids colliding with those too. | ||
| func SSNGenerator(rng *rand.Rand, original string) string { | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The solution works. If you want to make it even more realistic, avoid all prefixes shown in this list https://www.ssa.gov/employer/ssnvs/highgroup.txt These are all active US SSNs
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Good pointer, and it sent me down a useful rabbit hole. I'd argue the 900-999 range already avoids every prefix in that list, with a stronger guarantee than consuming the list would give:
Your comment did surface one real gap though: IRS ITINs also start with 9 and use group digits 50-65, 70-88, 90-92, and 94-99, so a generated 9XX dummy could have matched a real ITIN. Fixed in 6d677d8 by constraining the group digits to 10-49, with a test asserting both ranges.
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Wow, your comment is incredible insightful! Thank you! I didn't know that the SSA changed to random numbers. In that light, going with the 9XX prefix makes a lot of sense. Thank you also for considering ITINs! |
||
| // Generate random numbers, avoiding obvious patterns | ||
| first := 100 + rng.Intn(900) // 100-999 | ||
| second := 10 + rng.Intn(90) // 10-99 | ||
| third := 1000 + rng.Intn(9000) // 1000-9999 | ||
| area := 900 + rng.Intn(100) // 900-999: never issued as SSNs | ||
| group := 10 + rng.Intn(40) // 10-49: outside all ITIN group ranges | ||
| serial := 1000 + rng.Intn(9000) // 1000-9999 | ||
|
|
||
| return fmt.Sprintf("%d-%d-%d", area, group, serial) | ||
| } | ||
|
|
||
| return fmt.Sprintf("%d-%d-%d", first, second, third) | ||
| // luhnCheckDigit returns the check digit that would make digits followed by | ||
| // that digit pass the Luhn checksum. | ||
| func luhnCheckDigit(digits []int) int { | ||
| sum := 0 | ||
| // Walk right to left; with the check digit occupying the final position, | ||
| // the rightmost digit here sits in a doubled position. | ||
| for i := 0; i < len(digits); i++ { | ||
| d := digits[len(digits)-1-i] | ||
| if i%2 == 0 { | ||
| d *= 2 | ||
| if d > 9 { | ||
| d -= 9 | ||
| } | ||
| } | ||
| sum += d | ||
| } | ||
| return (10 - sum%10) % 10 | ||
| } | ||
|
|
||
| // CreditCardGenerator generates dummy credit card numbers | ||
| // CreditCardGenerator generates dummy credit card numbers that deliberately | ||
| // fail the Luhn checksum: the final digit is chosen to be anything except the | ||
| // valid check digit, so the output looks real but is never an issuable card | ||
| // number. | ||
| func CreditCardGenerator(rng *rand.Rand, original string) string { | ||
| // Generate 4 groups of 4 digits | ||
| groups := make([]int, 4) | ||
| for i := range groups { | ||
| groups[i] = 1000 + rng.Intn(9000) | ||
| digits := make([]int, 16) | ||
| // Lead with a real-looking network digit (Visa/Mastercard/Discover style) | ||
| digits[0] = 4 + rng.Intn(3) | ||
| for i := 1; i < 15; i++ { | ||
| digits[i] = rng.Intn(10) | ||
| } | ||
| digits[15] = (luhnCheckDigit(digits[:15]) + 1 + rng.Intn(9)) % 10 | ||
|
|
||
| var groups [4]string | ||
| for g := 0; g < 4; g++ { | ||
| groups[g] = fmt.Sprintf("%d%d%d%d", digits[g*4], digits[g*4+1], digits[g*4+2], digits[g*4+3]) | ||
| } | ||
|
|
||
| // Randomly choose format | ||
| formats := []string{"%d %d %d %d", "%d-%d-%d-%d"} | ||
| formats := []string{"%s %s %s %s", "%s-%s-%s-%s"} | ||
| format := formats[rng.Intn(len(formats))] | ||
|
|
||
| return fmt.Sprintf(format, groups[0], groups[1], groups[2], groups[3]) | ||
|
|
@@ -158,7 +192,7 @@ func StreetGenerator(rng *rand.Rand, original string) string { | |
| "King Street", "Manor Road", "Park Lane", "The Crescent", "Green Lane", | ||
| "Mill Lane", "New Road", "Chapel Street", "West End", "North Terrace", | ||
| } | ||
| return streetNames[rng.Intn(len(streetNames))] | ||
| return pickExcluding(rng, streetNames, original) | ||
| } | ||
|
|
||
| // ZipCodeGenerator generates dummy zip codes | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
TIL :)