Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 15 additions & 2 deletions src/backend/pii/generator_service.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,10 +50,23 @@ func NewGeneratorService() *GeneratorService {
}
}

// GenerateReplacement generates a replacement for the given PII label and original text
// generateAttempts is how many times a label's generator may collide with the
// original before we fall back to the deterministic placeholder.
const generateAttempts = 3

// GenerateReplacement generates a replacement for the given PII label and
// original text. The replacement is guaranteed to differ from the original:
// returning it unchanged would leak the very PII we are masking, so after a
// few collisions we fall back to GenericGenerator's [REDACTED_...] placeholder,
// which embeds a hash of the original and therefore never equals it.
func (s *GeneratorService) GenerateReplacement(label, originalText string) string {
generator := s.getGeneratorForLabel(label)
return generator(originalText)
for attempt := 0; attempt < generateAttempts; attempt++ {
if replacement := generator(originalText); replacement != originalText {
return replacement
}
}
return piiGenerators.GenericGenerator(label, originalText)
}

// getGeneratorForLabel returns the appropriate generator function for the given label
Expand Down
42 changes: 42 additions & 0 deletions src/backend/pii/generator_service_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
package pii

import (
"strings"
"testing"
)

// generatorLabels is every label GenerateReplacement routes to a dedicated
// generator, plus an unknown label to cover the generic fallback path.
var generatorLabels = []string{
labelSurname, labelFirstName, labelBuildingNum, labelDateOfBirth,
labelEmail, labelPhoneNumber, labelCity, labelURL, labelCompanyName,
labelState, labelZip, labelStreet, labelCountry, labelSSN,
labelDriverLicenseNum, labelPassportID, labelNationalID, labelIDCardNum,
labelTaxNum, labelLicensePlateNum, labelPassword, labelIBAN, labelAge,
labelSecurityToken, labelCreditCardNumber, labelUsername,
"UNKNOWNLABEL",
}

// A replacement equal to the original would leak the PII it is supposed to
// mask, so GenerateReplacement guarantees inequality. Feeding a previous
// output back in as the original is the most collision-prone input possible,
// and must still always produce something different.
func TestGenerateReplacementNeverReturnsOriginal(t *testing.T) {
service := NewGeneratorService()
for _, label := range generatorLabels {
original := service.GenerateReplacement(label, "")
for i := 0; i < 100; i++ {
if replacement := service.GenerateReplacement(label, original); replacement == original {
t.Fatalf("GenerateReplacement(%q) returned the original %q", label, original)
}
}
}
}

func TestGenerateReplacementUnknownLabelUsesPlaceholder(t *testing.T) {
service := NewGeneratorService()
replacement := service.GenerateReplacement("SOMETHINGELSE", "raw value")
if !strings.HasPrefix(replacement, "[REDACTED_SOMETHINGELSE_") {
t.Errorf("expected generic placeholder for unknown label, got %q", replacement)
}
}
76 changes: 55 additions & 21 deletions src/backend/pii/generators/pii_generators.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,44 +71,78 @@ func EmailGenerator(rng *rand.Rand, original string) string {
return fmt.Sprintf("%s.%s@%s", firstName, lastName, domain)
}

// PhoneGenerator generates dummy phone numbers
// PhoneGenerator generates dummy phone numbers. The exchange is fixed to 555
// and the line number to 0100-0199 — the NANP block reserved for fictional
// use — so the output looks real but can never be a dialable subscriber
// number belonging to someone.
func PhoneGenerator(rng *rand.Rand, original string) string {
// Generate a random 3-digit area code (200-999)
areaCode := 200 + rng.Intn(800)

// Generate a random 3-digit exchange (200-999)
exchange := 200 + rng.Intn(800)

// Generate a random 4-digit number
number := 1000 + rng.Intn(9000)
// Line number in the reserved fictional range 0100-0199
line := 100 + rng.Intn(100)

// Randomly choose format
formats := []string{"%d-%d-%d", "%d.%d.%d", "(%d) %d-%d"}
formats := []string{"%d-555-%04d", "%d.555.%04d", "(%d) 555-%04d"}

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

TIL :)

format := formats[rng.Intn(len(formats))]

return fmt.Sprintf(format, areaCode, exchange, number)
return fmt.Sprintf(format, areaCode, line)
}

// SSNGenerator generates dummy SSN numbers (SOCIALNUM)
// SSNGenerator generates dummy SSN numbers (SOCIALNUM). The area number is
// drawn from 900-999, which the SSA has never issued and has reserved against
// future issuance (every issued prefix, including everything in the SSA high
// group list, lies in 001-899), so the output looks real but can never
// collide with an actual person's SSN. The group number stays in 10-49
// because IRS ITINs also start with 9 and use group ranges 50-65, 70-88,
// 90-92, and 94-99; staying below 50 avoids colliding with those too.
func SSNGenerator(rng *rand.Rand, original string) string {

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The solution works. If you want to make it even more realistic, avoid all prefixes shown in this list

https://www.ssa.gov/employer/ssnvs/highgroup.txt

These are all active US SSNs

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good pointer, and it sent me down a useful rabbit hole. I'd argue the 900-999 range already avoids every prefix in that list, with a stronger guarantee than consuming the list would give:

  • The high group list was frozen on June 25, 2011, the day the SSA switched to randomized assignment. Since then the SSA assigns from essentially all of 001-899 (excluding 000 and 666), so a prefix being absent from the list no longer means it is inactive. Generating from the list's complement within 001-899 would still produce SSNs that can belong to real people.
  • Every prefix that has ever been issued (and everything in the list) lies in 001-899. 900-999 is reserved against SSN issuance, so the current generator avoids 100% of the listed prefixes by construction, and stays correct as randomization fills in the rest of 001-899.

Your comment did surface one real gap though: IRS ITINs also start with 9 and use group digits 50-65, 70-88, 90-92, and 94-99, so a generated 9XX dummy could have matched a real ITIN. Fixed in 6d677d8 by constraining the group digits to 10-49, with a test asserting both ranges.

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Wow, your comment is incredible insightful! Thank you!

I didn't know that the SSA changed to random numbers. In that light, going with the 9XX prefix makes a lot of sense. Thank you also for considering ITINs!

// Generate random numbers, avoiding obvious patterns
first := 100 + rng.Intn(900) // 100-999
second := 10 + rng.Intn(90) // 10-99
third := 1000 + rng.Intn(9000) // 1000-9999
area := 900 + rng.Intn(100) // 900-999: never issued as SSNs
group := 10 + rng.Intn(40) // 10-49: outside all ITIN group ranges
serial := 1000 + rng.Intn(9000) // 1000-9999

return fmt.Sprintf("%d-%d-%d", area, group, serial)
}

return fmt.Sprintf("%d-%d-%d", first, second, third)
// luhnCheckDigit returns the check digit that would make digits followed by
// that digit pass the Luhn checksum.
func luhnCheckDigit(digits []int) int {
sum := 0
// Walk right to left; with the check digit occupying the final position,
// the rightmost digit here sits in a doubled position.
for i := 0; i < len(digits); i++ {
d := digits[len(digits)-1-i]
if i%2 == 0 {
d *= 2
if d > 9 {
d -= 9
}
}
sum += d
}
return (10 - sum%10) % 10
}

// CreditCardGenerator generates dummy credit card numbers
// CreditCardGenerator generates dummy credit card numbers that deliberately
// fail the Luhn checksum: the final digit is chosen to be anything except the
// valid check digit, so the output looks real but is never an issuable card
// number.
func CreditCardGenerator(rng *rand.Rand, original string) string {
// Generate 4 groups of 4 digits
groups := make([]int, 4)
for i := range groups {
groups[i] = 1000 + rng.Intn(9000)
digits := make([]int, 16)
// Lead with a real-looking network digit (Visa/Mastercard/Discover style)
digits[0] = 4 + rng.Intn(3)
for i := 1; i < 15; i++ {
digits[i] = rng.Intn(10)
}
digits[15] = (luhnCheckDigit(digits[:15]) + 1 + rng.Intn(9)) % 10

var groups [4]string
for g := 0; g < 4; g++ {
groups[g] = fmt.Sprintf("%d%d%d%d", digits[g*4], digits[g*4+1], digits[g*4+2], digits[g*4+3])
}

// Randomly choose format
formats := []string{"%d %d %d %d", "%d-%d-%d-%d"}
formats := []string{"%s %s %s %s", "%s-%s-%s-%s"}
format := formats[rng.Intn(len(formats))]

return fmt.Sprintf(format, groups[0], groups[1], groups[2], groups[3])
Expand Down Expand Up @@ -158,7 +192,7 @@ func StreetGenerator(rng *rand.Rand, original string) string {
"King Street", "Manor Road", "Park Lane", "The Crescent", "Green Lane",
"Mill Lane", "New Road", "Chapel Street", "West End", "North Terrace",
}
return streetNames[rng.Intn(len(streetNames))]
return pickExcluding(rng, streetNames, original)
}

// ZipCodeGenerator generates dummy zip codes
Expand Down
136 changes: 105 additions & 31 deletions src/backend/pii/generators/pii_generators_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ func TestUsernameGenerator(t *testing.T) {

func TestDateOfBirthGenerator(t *testing.T) {
rng := getTestRand(42)

// Test with empty original
result := DateOfBirthGenerator(rng, "")
datePattern := regexp.MustCompile(`^\d{2}/\d{2}/\d{4}$`)
Expand Down Expand Up @@ -141,7 +141,7 @@ func TestStreetGenerator(t *testing.T) {

func TestZipCodeGenerator(t *testing.T) {
rng := getTestRand(42)

// Test basic 5-digit zip
result := ZipCodeGenerator(rng, "")
zipPattern := regexp.MustCompile(`^\d{5}$`)
Expand Down Expand Up @@ -255,9 +255,9 @@ func TestUrlGenerator(t *testing.T) {
}

// Check that it uses reserved domains
hasReservedDomain := strings.Contains(result, "example.") ||
strings.Contains(result, "test.") ||
strings.Contains(result, "invalid.") ||
hasReservedDomain := strings.Contains(result, "example.") ||
strings.Contains(result, "test.") ||
strings.Contains(result, "invalid.") ||
strings.Contains(result, "localhost.")
if !hasReservedDomain {
t.Errorf("UrlGenerator returned URL with non-reserved domain: %s", result)
Expand Down Expand Up @@ -435,6 +435,80 @@ func TestGenericGenerator(t *testing.T) {
}
}

// The exchange must be 555 and the line number 0100-0199 — the NANP block
// reserved for fictional use — so a generated number is never dialable.
func TestPhoneGeneratorFictionalRange(t *testing.T) {
phonePattern := regexp.MustCompile(`^(\(\d{3}\) 555-01\d{2}|\d{3}-555-01\d{2}|\d{3}\.555\.01\d{2})$`)
for seed := int64(0); seed < 50; seed++ {
result := PhoneGenerator(getTestRand(seed), "")
if !phonePattern.MatchString(result) {
t.Errorf("PhoneGenerator returned number outside the 555-01XX fictional range: %s", result)
}
}
}

// The area number must fall in 900-999, which the SSA has never issued, so a
// generated SSN can never collide with a real person's. The group number must
// stay in 10-49: IRS ITINs also start with 9 and use groups 50-65, 70-88,
// 90-92, and 94-99, so anything 50+ risks colliding with a real ITIN.
func TestSSNGeneratorNeverIssuedArea(t *testing.T) {
for seed := int64(0); seed < 50; seed++ {
result := SSNGenerator(getTestRand(seed), "")
var area, group, serial int
if _, err := fmt.Sscanf(result, "%d-%d-%d", &area, &group, &serial); err != nil {
t.Fatalf("SSNGenerator returned unparseable SSN: %s", result)
}
if area < 900 || area > 999 {
t.Errorf("SSNGenerator returned area %d, expected never-issued range 900-999: %s", area, result)
}
if group < 10 || group > 49 {
t.Errorf("SSNGenerator returned group %d, expected 10-49 to avoid ITIN ranges: %s", group, result)
}
}
}

// luhnValid reports whether a full card number (check digit included) passes
// the Luhn checksum.
func luhnValid(digits []int) bool {
sum := 0
for i := 0; i < len(digits); i++ {
d := digits[len(digits)-1-i]
if i%2 == 1 {
d *= 2
if d > 9 {
d -= 9
}
}
sum += d
}
return sum%10 == 0
}

// Generated card numbers must deliberately fail the Luhn checksum so they can
// never be an issuable card number.
func TestCreditCardGeneratorFailsLuhn(t *testing.T) {
for seed := int64(0); seed < 100; seed++ {
result := CreditCardGenerator(getTestRand(seed), "")
cleaned := strings.NewReplacer(" ", "", "-", "").Replace(result)
digits := make([]int, 0, len(cleaned))
for _, ch := range cleaned {
digits = append(digits, int(ch-'0'))
}
if luhnValid(digits) {
t.Errorf("CreditCardGenerator returned a Luhn-valid number: %s", result)
}
}
}

// Echoing the original street back would leak the PII we are masking.
func TestStreetGeneratorExcludesOriginal(t *testing.T) {
for seed := int64(0); seed < 100; seed++ {
if result := StreetGenerator(getTestRand(seed), "Main St"); result == "Main St" {
t.Errorf("StreetGenerator returned the original street name (seed %d)", seed)
}
}
}

// Test that generators produce different outputs with different seeds
func TestGeneratorsDifferentSeeds(t *testing.T) {
rng1 := getTestRand(1)
Expand Down Expand Up @@ -464,32 +538,32 @@ func TestGeneratorsSameSeed(t *testing.T) {
// Test all generators return non-empty strings
func TestAllGeneratorsNonEmpty(t *testing.T) {
generators := map[string]func(*rand.Rand, string) string{
"EmailGenerator": EmailGenerator,
"PhoneGenerator": PhoneGenerator,
"SSNGenerator": SSNGenerator,
"CreditCardGenerator": CreditCardGenerator,
"UsernameGenerator": UsernameGenerator,
"DateOfBirthGenerator": DateOfBirthGenerator,
"StreetGenerator": StreetGenerator,
"ZipCodeGenerator": ZipCodeGenerator,
"CityGenerator": CityGenerator,
"BuildingNumGenerator": BuildingNumGenerator,
"FirstNameGenerator": FirstNameGenerator,
"SurnameGenerator": SurnameGenerator,
"IDCardNumGenerator": IDCardNumGenerator,
"DriverLicenseNumGenerator": DriverLicenseNumGenerator,
"TaxNumGenerator": TaxNumGenerator,
"UrlGenerator": UrlGenerator,
"CompanyNameGenerator": CompanyNameGenerator,
"StateGenerator": StateGenerator,
"CountryGenerator": CountryGenerator,
"PassportIdGenerator": PassportIdGenerator,
"NationalIdGenerator": NationalIdGenerator,
"LicensePlateNumGenerator": LicensePlateNumGenerator,
"PasswordGenerator": PasswordGenerator,
"IbanGenerator": IbanGenerator,
"AgeGenerator": AgeGenerator,
"SecurityTokenGenerator": SecurityTokenGenerator,
"EmailGenerator": EmailGenerator,
"PhoneGenerator": PhoneGenerator,
"SSNGenerator": SSNGenerator,
"CreditCardGenerator": CreditCardGenerator,
"UsernameGenerator": UsernameGenerator,
"DateOfBirthGenerator": DateOfBirthGenerator,
"StreetGenerator": StreetGenerator,
"ZipCodeGenerator": ZipCodeGenerator,
"CityGenerator": CityGenerator,
"BuildingNumGenerator": BuildingNumGenerator,
"FirstNameGenerator": FirstNameGenerator,
"SurnameGenerator": SurnameGenerator,
"IDCardNumGenerator": IDCardNumGenerator,
"DriverLicenseNumGenerator": DriverLicenseNumGenerator,
"TaxNumGenerator": TaxNumGenerator,
"UrlGenerator": UrlGenerator,
"CompanyNameGenerator": CompanyNameGenerator,
"StateGenerator": StateGenerator,
"CountryGenerator": CountryGenerator,
"PassportIdGenerator": PassportIdGenerator,
"NationalIdGenerator": NationalIdGenerator,
"LicensePlateNumGenerator": LicensePlateNumGenerator,
"PasswordGenerator": PasswordGenerator,
"IbanGenerator": IbanGenerator,
"AgeGenerator": AgeGenerator,
"SecurityTokenGenerator": SecurityTokenGenerator,
}

for name, generator := range generators {
Expand Down
Loading