[DANON-73] Exclude system users from anonymization (#123)

ncovercash · web-flow · commit a4ea7ef32269 · 2026-05-19T13:47:05.000-04:00
diff --git a/src/main/java/org/folio/anonymization/jobs/AddressAnonymization.java b/src/main/java/org/folio/anonymization/jobs/AddressAnonymization.java
@@ -12,6 +12,7 @@
 import org.folio.anonymization.jobs.templates.BatchGenerationFromTablePart;
 import org.folio.anonymization.jobs.templates.ReplaceValueFromListPart;
 import org.folio.anonymization.util.RandomValueUtils;
+import org.folio.anonymization.util.SystemUserExclusionUtil;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
@@ -121,7 +122,8 @@ public List<JobBuilder> getBuilders(TenantExecutionContext tenant) {
                         field,
                         condition,
                         replacementValues(field, Math.max(end - start, 5))
-                      )
+                      ),
+                    SystemUserExclusionUtil.getExclusionCondition(field, tenant)
                   )
                 )
                 .toList()
diff --git a/src/main/java/org/folio/anonymization/jobs/CustomFieldAnonymization.java b/src/main/java/org/folio/anonymization/jobs/CustomFieldAnonymization.java
@@ -34,6 +34,7 @@
 import org.folio.anonymization.jobs.templates.ReplaceValuePart;
 import org.folio.anonymization.util.NumberUtils;
 import org.folio.anonymization.util.RandomValueUtils;
+import org.folio.anonymization.util.SystemUserExclusionUtil;
 import org.jooq.Condition;
 import org.jooq.Field;
 import org.jooq.JSONB;
@@ -445,7 +446,8 @@ public List<JobBuilder> getBuilders(TenantExecutionContext tenant) {
                               "Redact " + field.toString() + " with refId " + r.get("refId") + " on " + label,
                               field.withJsonPath(field.jsonPath() + "." + r.get("refId")),
                               condition
-                            )
+                            ),
+                          SystemUserExclusionUtil.getExclusionCondition(field, tenant)
                         )
                       )
                     );
diff --git a/src/main/java/org/folio/anonymization/jobs/DateOfBirthAnonymization.java b/src/main/java/org/folio/anonymization/jobs/DateOfBirthAnonymization.java
@@ -13,6 +13,7 @@
 import org.folio.anonymization.domain.job.TenantExecutionContext;
 import org.folio.anonymization.jobs.templates.BatchGenerationFromTablePart;
 import org.folio.anonymization.jobs.templates.ReplaceJSONBValuePart;
+import org.folio.anonymization.util.SystemUserExclusionUtil;
 import org.jooq.Field;
 import org.jooq.JSONB;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -60,7 +61,8 @@ public List<JobBuilder> getBuilders(TenantExecutionContext tenant) {
                         field,
                         condition,
                         RANDOM_DOB_SQL
-                      )
+                      ),
+                    SystemUserExclusionUtil.getExclusionCondition(field, tenant)
                   )
                 )
                 .toList()
diff --git a/src/main/java/org/folio/anonymization/jobs/EmailAddressAnonymization.java b/src/main/java/org/folio/anonymization/jobs/EmailAddressAnonymization.java
@@ -12,6 +12,7 @@
 import org.folio.anonymization.jobs.templates.BatchGenerationFromTablePart;
 import org.folio.anonymization.jobs.templates.ReplaceValueFromListPart;
 import org.folio.anonymization.util.RandomValueUtils;
+import org.folio.anonymization.util.SystemUserExclusionUtil;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
@@ -68,7 +69,8 @@ public List<JobBuilder> getBuilders(TenantExecutionContext tenant) {
                         field,
                         condition,
                         RandomValueUtils.emails(Math.max(end - start, 5))
-                      )
+                      ),
+                    SystemUserExclusionUtil.getExclusionCondition(field, tenant)
                   )
                 )
                 .toList()
diff --git a/src/main/java/org/folio/anonymization/jobs/FirstNameAnonymization.java b/src/main/java/org/folio/anonymization/jobs/FirstNameAnonymization.java
@@ -12,6 +12,7 @@
 import org.folio.anonymization.jobs.templates.BatchGenerationFromTablePart;
 import org.folio.anonymization.jobs.templates.ReplaceValueFromListPart;
 import org.folio.anonymization.util.RandomValueUtils;
+import org.folio.anonymization.util.SystemUserExclusionUtil;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
@@ -89,7 +90,8 @@ public List<JobBuilder> getBuilders(TenantExecutionContext tenant) {
                         field,
                         condition,
                         RandomValueUtils.firstNames(Math.max(end - start, 5))
-                      )
+                      ),
+                    SystemUserExclusionUtil.getExclusionCondition(field, tenant)
                   )
                 )
                 .toList()
diff --git a/src/main/java/org/folio/anonymization/jobs/LastNameAnonymization.java b/src/main/java/org/folio/anonymization/jobs/LastNameAnonymization.java
@@ -12,6 +12,7 @@
 import org.folio.anonymization.jobs.templates.BatchGenerationFromTablePart;
 import org.folio.anonymization.jobs.templates.ReplaceValueFromListPart;
 import org.folio.anonymization.util.RandomValueUtils;
+import org.folio.anonymization.util.SystemUserExclusionUtil;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
@@ -88,7 +89,8 @@ public List<JobBuilder> getBuilders(TenantExecutionContext tenant) {
                         field,
                         condition,
                         RandomValueUtils.lastNames(Math.max(end - start, 5))
-                      )
+                      ),
+                    SystemUserExclusionUtil.getExclusionCondition(field, tenant)
                   )
                 )
                 .toList()
diff --git a/src/main/java/org/folio/anonymization/jobs/MiddleNameAnonymization.java b/src/main/java/org/folio/anonymization/jobs/MiddleNameAnonymization.java
@@ -12,6 +12,7 @@
 import org.folio.anonymization.jobs.templates.BatchGenerationFromTablePart;
 import org.folio.anonymization.jobs.templates.ReplaceValueFromListPart;
 import org.folio.anonymization.util.RandomValueUtils;
+import org.folio.anonymization.util.SystemUserExclusionUtil;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
@@ -58,7 +59,8 @@ public List<JobBuilder> getBuilders(TenantExecutionContext tenant) {
                         field,
                         condition,
                         RandomValueUtils.middleNames(Math.max(end - start, 5))
-                      )
+                      ),
+                    SystemUserExclusionUtil.getExclusionCondition(field, tenant)
                   )
                 )
                 .toList()
diff --git a/src/main/java/org/folio/anonymization/jobs/PhoneNumberAnonymization.java b/src/main/java/org/folio/anonymization/jobs/PhoneNumberAnonymization.java
@@ -15,6 +15,7 @@
 import org.folio.anonymization.jobs.templates.ReplaceJSONBValuePart;
 import org.folio.anonymization.jobs.templates.ReplaceValuePart;
 import org.folio.anonymization.util.RandomValueUtils;
+import org.folio.anonymization.util.SystemUserExclusionUtil;
 import org.jooq.Field;
 import org.jooq.JSONB;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -104,7 +105,8 @@ public List<JobBuilder> getBuilders(TenantExecutionContext tenant) {
                           baseReplacement
                         );
                       }
-                    }
+                    },
+                    SystemUserExclusionUtil.getExclusionCondition(field, tenant)
                   )
                 )
                 .toList()
diff --git a/src/main/java/org/folio/anonymization/jobs/UserBarcodeAnonymization.java b/src/main/java/org/folio/anonymization/jobs/UserBarcodeAnonymization.java
@@ -21,12 +21,15 @@
 import org.folio.anonymization.jobs.templates.BatchGenerationFromTablePart;
 import org.folio.anonymization.jobs.templates.CreateTablePart;
 import org.folio.anonymization.jobs.templates.DropTablePart;
+import org.folio.anonymization.jobs.templates.ExcludeGeneratedValuesPart;
+import org.folio.anonymization.jobs.templates.FindSystemUsersPart;
 import org.folio.anonymization.jobs.templates.GenerateValuesPart;
 import org.folio.anonymization.jobs.templates.InsertIntoTablePart;
 import org.folio.anonymization.jobs.templates.ReplaceJSONBValuePart;
 import org.folio.anonymization.jobs.templates.ReplaceValuePart;
 import org.folio.anonymization.util.DBUtils;
 import org.folio.anonymization.util.RandomValueUtils;
+import org.folio.anonymization.util.SystemUserExclusionUtil;
 import org.jooq.Field;
 import org.jooq.JSONB;
 import org.jooq.Table;
@@ -37,6 +40,9 @@
 @Component
 public class UserBarcodeAnonymization implements JobFactory {
 
+  // used to find system user barcodes that should be excluded from anonymization
+  private static final FieldReference USERS_TABLE_FIELD = new FieldReference("users", "users", "jsonb", "$.barcode");
+
   private static final List<FieldReference> FIELDS = List.of(
     new FieldReference("circulation_storage", "actual_cost_record", "jsonb", "$.user.barcode"),
     new FieldReference("circulation_storage", "request", "jsonb", "$.requester.barcode"),
@@ -50,7 +56,7 @@ public class UserBarcodeAnonymization implements JobFactory {
     new FieldReference("requests_mediated", "mediated_request", "requester_barcode"),
     new FieldReference("requests_mediated", "mediated_request", "proxy_barcode"),
     new FieldReference("users", "user_tenant", "barcode"),
-    new FieldReference("users", "users", "jsonb", "$.barcode")
+    USERS_TABLE_FIELD
   );
 
   @Autowired
@@ -82,7 +88,9 @@ public List<JobBuilder> getBuilders(TenantExecutionContext tenant) {
           .toList(),
         ctx -> {
           Table<?> tempTableFinal = table(name("public", "_danon_" + ctx.tenant().tenant().id() + "_user_barcodes"));
-          Table<?> tempTableStaging = table(name("public", "_danon_" + ctx.tenant().tenant().id() + "_user_barcodes_staging"));
+          Table<?> tempTableStaging = table(
+            name("public", "_danon_" + ctx.tenant().tenant().id() + "_user_barcodes_staging")
+          );
 
           Field<String> originalValue = field("original_value", SQLDataType.VARCHAR.notNull());
           Field<String> newValue = field("new_value", SQLDataType.VARCHAR.null_());
@@ -95,6 +103,8 @@ public List<JobBuilder> getBuilders(TenantExecutionContext tenant) {
               "enumerate",
               "generate-new-values-prep",
               "generate-new-values",
+              "exclude-system-user-values-prep",
+              "exclude-system-user-values",
               "apply-new-values-prep",
               "apply-new-values",
               "cleanup"
@@ -169,6 +179,30 @@ public List<JobBuilder> getBuilders(TenantExecutionContext tenant) {
             );
           }
 
+          job.scheduleParts(
+            "exclude-system-user-values-prep",
+            List.of(
+              new FindSystemUsersPart(
+                "Find system user values",
+                USERS_TABLE_FIELD.table(tenant.tenant()),
+                field("{0}->>'barcode'", String.class, USERS_TABLE_FIELD.baseColumn(tenant.tenant(), JSONB.class)),
+                systemUserValues ->
+                  job.scheduleParts(
+                    "exclude-system-user-values",
+                    List.of(
+                      new ExcludeGeneratedValuesPart(
+                        "Exclude system user values from anonymization",
+                        tempTableFinal,
+                        originalValue,
+                        newValue,
+                        systemUserValues
+                      )
+                    )
+                  )
+              )
+            )
+          );
+
           job.scheduleParts(
             "apply-new-values-prep",
             JobConfigurationProperty
@@ -205,7 +239,8 @@ public List<JobBuilder> getBuilders(TenantExecutionContext tenant) {
                           )
                       );
                     }
-                  }
+                  },
+                  SystemUserExclusionUtil.getExclusionCondition(field, tenant)
                 )
               )
               .toList()
diff --git a/src/main/java/org/folio/anonymization/jobs/UserExternalSystemIdAnonymization.java b/src/main/java/org/folio/anonymization/jobs/UserExternalSystemIdAnonymization.java
@@ -21,6 +21,8 @@
 import org.folio.anonymization.jobs.templates.BatchGenerationFromTablePart;
 import org.folio.anonymization.jobs.templates.CreateTablePart;
 import org.folio.anonymization.jobs.templates.DropTablePart;
+import org.folio.anonymization.jobs.templates.ExcludeGeneratedValuesPart;
+import org.folio.anonymization.jobs.templates.FindSystemUsersPart;
 import org.folio.anonymization.jobs.templates.GenerateValuesPart;
 import org.folio.anonymization.jobs.templates.InsertIntoTablePart;
 import org.folio.anonymization.jobs.templates.ReplaceJSONBValuePart;
@@ -37,11 +39,19 @@
 @Component
 public class UserExternalSystemIdAnonymization implements JobFactory {
 
+  // special handling to capture system user ones from here that should not be anonymized
+  private static final FieldReference USERS_TABLE_FIELD = new FieldReference(
+    "users",
+    "users",
+    "jsonb",
+    "$.externalSystemId"
+  );
+
   private static final List<FieldReference> FIELDS = List.of(
     new FieldReference("oa", "party", "p_orcid_id"),
     new FieldReference("users", "staging_users", "jsonb", "$.externalSystemId"),
     new FieldReference("users", "user_tenant", "external_system_id"),
-    new FieldReference("users", "users", "jsonb", "$.externalSystemId")
+    USERS_TABLE_FIELD
   );
 
   @Autowired
@@ -90,6 +100,8 @@ public List<JobBuilder> getBuilders(TenantExecutionContext tenant) {
               "enumerate",
               "generate-new-values-prep",
               "generate-new-values",
+              "exclude-system-user-values-prep",
+              "exclude-system-user-values",
               "apply-new-values-prep",
               "apply-new-values",
               "cleanup"
@@ -164,6 +176,34 @@ public List<JobBuilder> getBuilders(TenantExecutionContext tenant) {
             );
           }
 
+          job.scheduleParts(
+            "exclude-system-user-values-prep",
+            List.of(
+              new FindSystemUsersPart(
+                "Find system user values",
+                USERS_TABLE_FIELD.table(tenant.tenant()),
+                field(
+                  "{0}->>'externalSystemId'",
+                  String.class,
+                  USERS_TABLE_FIELD.baseColumn(tenant.tenant(), JSONB.class)
+                ),
+                systemUserValues ->
+                  job.scheduleParts(
+                    "exclude-system-user-values",
+                    List.of(
+                      new ExcludeGeneratedValuesPart(
+                        "Exclude system user values from anonymization",
+                        tempTableFinal,
+                        originalValue,
+                        newValue,
+                        systemUserValues
+                      )
+                    )
+                  )
+              )
+            )
+          );
+
           job.scheduleParts(
             "apply-new-values-prep",
             JobConfigurationProperty
diff --git a/src/main/java/org/folio/anonymization/jobs/UsernameAnonymization.java b/src/main/java/org/folio/anonymization/jobs/UsernameAnonymization.java
@@ -21,12 +21,15 @@
 import org.folio.anonymization.jobs.templates.BatchGenerationFromTablePart;
 import org.folio.anonymization.jobs.templates.CreateTablePart;
 import org.folio.anonymization.jobs.templates.DropTablePart;
+import org.folio.anonymization.jobs.templates.ExcludeGeneratedValuesPart;
+import org.folio.anonymization.jobs.templates.FindSystemUsersPart;
 import org.folio.anonymization.jobs.templates.GenerateValuesPart;
 import org.folio.anonymization.jobs.templates.InsertIntoTablePart;
 import org.folio.anonymization.jobs.templates.ReplaceJSONBValuePart;
 import org.folio.anonymization.jobs.templates.ReplaceValuePart;
 import org.folio.anonymization.util.DBUtils;
 import org.folio.anonymization.util.RandomValueUtils;
+import org.folio.anonymization.util.SystemUserExclusionUtil;
 import org.jooq.Field;
 import org.jooq.JSONB;
 import org.jooq.Table;
@@ -37,6 +40,9 @@
 @Component
 public class UsernameAnonymization implements JobFactory {
 
+  // special handling to capture system user ones from here that should not be anonymized
+  private static final FieldReference USERS_TABLE_FIELD = new FieldReference("users", "users", "jsonb", "$.username");
+
   private static final List<FieldReference> FIELDS = List.of(
     new FieldReference("circulation_storage", "print_events", "jsonb", "$.requesterName"),
     new FieldReference("consortia_keycloak", "inactive_user_tenant", "username"),
@@ -162,7 +168,7 @@ public class UsernameAnonymization implements JobFactory {
     new FieldReference("users", "staging_users", "jsonb", "$.metadata.createdByUsername"),
     new FieldReference("users", "staging_users", "jsonb", "$.metadata.updatedByUsername"),
     new FieldReference("users", "user_tenant", "username"),
-    new FieldReference("users", "users", "jsonb", "$.username"),
+    USERS_TABLE_FIELD,
     new FieldReference("users", "users", "jsonb", "$.metadata.createdByUsername"),
     new FieldReference("users", "users", "jsonb", "$.metadata.updatedByUsername")
   );
@@ -214,6 +220,8 @@ public List<JobBuilder> getBuilders(TenantExecutionContext tenant) {
               "enumerate",
               "generate-new-values-prep",
               "generate-new-values",
+              "exclude-system-user-values-prep",
+              "exclude-system-user-values",
               "apply-new-values-prep",
               "apply-new-values",
               "cleanup"
@@ -288,6 +296,30 @@ public List<JobBuilder> getBuilders(TenantExecutionContext tenant) {
             );
           }
 
+          job.scheduleParts(
+            "exclude-system-user-values-prep",
+            List.of(
+              new FindSystemUsersPart(
+                "Find system user values",
+                USERS_TABLE_FIELD.table(tenant.tenant()),
+                field("{0}->>'username'", String.class, USERS_TABLE_FIELD.baseColumn(tenant.tenant(), JSONB.class)),
+                systemUserValues ->
+                  job.scheduleParts(
+                    "exclude-system-user-values",
+                    List.of(
+                      new ExcludeGeneratedValuesPart(
+                        "Exclude system user values from anonymization",
+                        tempTableFinal,
+                        originalValue,
+                        newValue,
+                        systemUserValues
+                      )
+                    )
+                  )
+              )
+            )
+          );
+
           job.scheduleParts(
             "apply-new-values-prep",
             JobConfigurationProperty
@@ -324,7 +356,8 @@ public List<JobBuilder> getBuilders(TenantExecutionContext tenant) {
                           )
                       );
                     }
-                  }
+                  },
+                  SystemUserExclusionUtil.getExclusionCondition(field, tenant)
                 )
               )
               .toList()
diff --git a/src/main/java/org/folio/anonymization/jobs/templates/BatchGenerationFromTablePart.java b/src/main/java/org/folio/anonymization/jobs/templates/BatchGenerationFromTablePart.java
diff --git a/src/main/java/org/folio/anonymization/jobs/templates/ExcludeGeneratedValuesPart.java b/src/main/java/org/folio/anonymization/jobs/templates/ExcludeGeneratedValuesPart.java
diff --git a/src/main/java/org/folio/anonymization/jobs/templates/FindSystemUsersPart.java b/src/main/java/org/folio/anonymization/jobs/templates/FindSystemUsersPart.java
diff --git a/src/main/java/org/folio/anonymization/util/SystemUserExclusionUtil.java b/src/main/java/org/folio/anonymization/util/SystemUserExclusionUtil.java

Original file line number	Diff line number	Diff line change
`@@ -15,6 +15,7 @@`
`15`	`15`	`import org.folio.anonymization.jobs.templates.ReplaceJSONBValuePart;`
`16`	`16`	`import org.folio.anonymization.jobs.templates.ReplaceValuePart;`
`17`	`17`	`import org.folio.anonymization.util.RandomValueUtils;`
	`18`	`+import org.folio.anonymization.util.SystemUserExclusionUtil;`
`18`	`19`	`import org.jooq.Field;`
`19`	`20`	`import org.jooq.JSONB;`
`20`	`21`	`import org.springframework.beans.factory.annotation.Autowired;`
`@@ -104,7 +105,8 @@ public List<JobBuilder> getBuilders(TenantExecutionContext tenant) {`
`104`	`105`	`baseReplacement`
`105`	`106`	`);`
`106`	`107`	`}`
`107`		`- }`
	`108`	`+ },`
	`109`	`+ SystemUserExclusionUtil.getExclusionCondition(field, tenant)`
`108`	`110`	`)`
`109`	`111`	`)`
`110`	`112`	`.toList()`