Merge pull request #360 from diffix/piotr/no-sql-layer-lcf

Remove bucket seed from LCF noise calculation

Author: pdobacz

pg_diffix/aggregation/noise.h

@@ -15,8 +15,8 @@ extern double generate_layered_noise(const seed_t *seeds, int seeds_count,
                                      const char *step_name, double layer_sd);
 
 /*
- * Returns the noisy LCF threshold for the given noise layers.
+ * Returns the noisy LCF threshold for the given noise layer.
  */
-extern double generate_lcf_threshold(const seed_t *seeds, int seeds_count);
+extern double generate_lcf_threshold(seed_t seed);
 
 #endif /* PG_DIFFIX_NOISE_H */

src/aggregation/count_distinct.c

@@ -80,33 +80,31 @@ static void set_value_sorting_globals(Oid element_type)
   g_compare_values_func = &g_compare_values_typentry->cmp_proc_finfo;
 }
 
-static bool aid_set_is_high_count(seed_t bucket_seed, const List *aid_values_set)
+static bool aid_set_is_high_count(const List *aid_values_set)
 {
   if (list_length(aid_values_set) < g_config.low_count_min_threshold)
     return false; /* Fewer AID values than minimum threshold, value is low-count. */
 
   seed_t aid_seed = hash_set_to_seed(aid_values_set);
-
-  seed_t seeds[] = {bucket_seed, aid_seed};
-  double threshold = generate_lcf_threshold(seeds, ARRAY_LENGTH(seeds));
+  double threshold = generate_lcf_threshold(aid_seed);
 
   return list_length(aid_values_set) >= threshold;
 }
 
-static bool aid_sets_are_high_count(seed_t bucket_seed, const List *aid_values_sets)
+static bool aid_sets_are_high_count(const List *aid_values_sets)
 {
   ListCell *cell;
   foreach (cell, aid_values_sets)
   {
     const List *aid_values_set = (const List *)lfirst(cell);
-    if (!aid_set_is_high_count(bucket_seed, aid_values_set))
+    if (!aid_set_is_high_count(aid_values_set))
       return false;
   }
   return true;
 }
 
 /* Returns a list with the tracker entries that are low count. */
-static List *filter_lc_entries(seed_t bucket_seed, DistinctTracker_hash *tracker)
+static List *filter_lc_entries(DistinctTracker_hash *tracker)
 {
   List *lc_entries = NIL;
 
@@ -115,7 +113,7 @@ static List *filter_lc_entries(seed_t bucket_seed, DistinctTracker_hash *tracker
   DistinctTrackerHashEntry *entry = NULL;
   while ((entry = DistinctTracker_iterate(tracker, &it)) != NULL)
   {
-    if (!aid_sets_are_high_count(bucket_seed, entry->aid_values_sets))
+    if (!aid_sets_are_high_count(entry->aid_values_sets))
       lc_entries = lappend(lc_entries, entry);
   }
 
@@ -352,7 +350,7 @@ static CountDistinctResult count_distinct_calculate_final(CountDistinctState *st
 
   DistinctTracker_hash *tracker = state->tracker;
 
-  List *lc_entries = filter_lc_entries(bucket_seed, tracker);
+  List *lc_entries = filter_lc_entries(tracker);
   list_sort(lc_entries, &compare_tracker_entries_by_value); /* Needed to ensure determinism. */
 
   CountDistinctResult result = {0};

src/aggregation/low_count.c

@@ -14,12 +14,10 @@ typedef struct AidResult
   bool low_count;
 } AidResult;
 
-static AidResult calculate_aid_result(seed_t bucket_seed, const AidTrackerState *tracker)
+static AidResult calculate_aid_result(const AidTrackerState *tracker)
 {
   AidResult result = {.aid_seed = tracker->aid_seed};
-
-  seed_t seeds[] = {bucket_seed, tracker->aid_seed};
-  result.threshold = generate_lcf_threshold(seeds, ARRAY_LENGTH(seeds));
+  result.threshold = generate_lcf_threshold(tracker->aid_seed);
   result.low_count = tracker->aid_set->members < result.threshold;
 
   return result;
@@ -83,11 +81,10 @@ static Datum agg_finalize(AnonAggState *base_state, Bucket *bucket, BucketDescri
   LowCountState *state = (LowCountState *)base_state;
 
   bool low_count = false;
-  seed_t bucket_seed = compute_bucket_seed(bucket, bucket_desc);
 
   for (int i = 0; i < state->trackers_count; i++)
   {
-    AidResult result = calculate_aid_result(bucket_seed, state->trackers[i]);
+    AidResult result = calculate_aid_result(state->trackers[i]);
     low_count = low_count || result.low_count;
   }
 

src/aggregation/noise.c

@@ -112,15 +112,15 @@ double generate_layered_noise(const seed_t *seeds, int seeds_count,
   return noise;
 }
 
-double generate_lcf_threshold(const seed_t *seeds, int seeds_count)
+double generate_lcf_threshold(seed_t seed)
 {
   /* 
    * `low_count_mean_gap` is the number of (total!) standard deviations between
    * `low_count_min_threshold` and desired mean.
    */
   double threshold_mean = (double)g_config.low_count_min_threshold +
                           g_config.low_count_mean_gap * g_config.low_count_layer_sd * sqrt(2.0);
-  double noise = generate_layered_noise(seeds, seeds_count, "suppress", g_config.low_count_layer_sd);
+  double noise = generate_layered_noise(&seed, 1, "suppress", g_config.low_count_layer_sd);
   double noisy_threshold = threshold_mean + noise;
   return Max(noisy_threshold, g_config.low_count_min_threshold);
 }

test/expected/noisy.out

@@ -212,6 +212,19 @@ SELECT COUNT(*), COUNT(city), COUNT(DISTINCT city) FROM london_customers;
      0 |     0 |     0
 (1 row)
 
+-- LCF doesn't depend on the bucket seed, both queries should have same noisy threshold.
+SELECT diffix.floor_by(age, 30), COUNT(*) FROM test_patients GROUP BY 1;
+ floor_by | count 
+----------+-------
+          |    23
+(1 row)
+
+SELECT diffix.floor_by(age, 30), diffix.floor_by(age, 106), COUNT(*) FROM test_patients GROUP BY 1, 2;
+ floor_by | floor_by | count 
+----------+----------+-------
+          |          |     9
+(1 row)
+
 ----------------------------------------------------------------
 -- Empty tables
 ----------------------------------------------------------------

test/sql/noisy.sql

@@ -56,6 +56,10 @@ SELECT city FROM test_customers GROUP BY 1 HAVING length(city) <> 4;
 
 SELECT COUNT(*), COUNT(city), COUNT(DISTINCT city) FROM london_customers;
 
+-- LCF doesn't depend on the bucket seed, both queries should have same noisy threshold.
+SELECT diffix.floor_by(age, 30), COUNT(*) FROM test_patients GROUP BY 1;
+SELECT diffix.floor_by(age, 30), diffix.floor_by(age, 106), COUNT(*) FROM test_patients GROUP BY 1, 2;
+
 ----------------------------------------------------------------
 -- Empty tables
 ----------------------------------------------------------------