Skip to content

Commit d40d90b

Browse files
athiramanuathiramanu
authored
Detect changes in AB Testing (ABT) experiment metadata during config fetch (#8002)
Updated to detect changes in AB Testing (ABT) experiment metadata. This ensures that if an experiment's variant or affected parameter keys change, the associated config keys are correctly identified as changed, even if the parameter values remain the same. Key changes: - Added constant to . - Implemented in to map affected config keys to their respective experiment descriptions. - Modified to compare experiment metadata for each config key. - Updated with cases for changed, deleted, and key-shifted experiments. Validation: - Adding an experiment with 100% exposure assigns user to a variant - Updating exposure to 0% returns default value to user - Updating exposure back to 100% returns the previously assigned variant - Updating the variant value returns the new value to user Testing on sample Android app: [Link](https://screencast.googleplex.com/cast/NTQ0NzUyNTU4NTY0OTY2NHwxOWRlY2VkNi1kYw)
1 parent 554fe26 commit d40d90b

4 files changed

Lines changed: 143 additions & 6 deletions

File tree

‎firebase-config/CHANGELOG.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
# Unreleased
22

3+
- [fixed] Remote Config Realtime updates now trigger when a parameter's experiment
4+
or variant assignment changes, ensuring more accurate A/B test analytics and
5+
consistent user experiences. #8002
6+
37
# 23.0.1
48

59
- [changed] Bumped internal dependencies.

‎firebase-config/src/main/java/com/google/firebase/remoteconfig/RemoteConfigConstants.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,12 +97,14 @@ public final class RemoteConfigConstants {
9797
*/
9898
@StringDef({
9999
ExperimentDescriptionFieldKey.EXPERIMENT_ID,
100-
ExperimentDescriptionFieldKey.VARIANT_ID
100+
ExperimentDescriptionFieldKey.VARIANT_ID,
101+
ExperimentDescriptionFieldKey.AFFECTED_PARAMETER_KEYS
101102
})
102103
@Retention(RetentionPolicy.SOURCE)
103104
public @interface ExperimentDescriptionFieldKey {
104105
String EXPERIMENT_ID = "experimentId";
105106
String VARIANT_ID = "variantId";
107+
String AFFECTED_PARAMETER_KEYS = "affectedParameterKeys";
106108
}
107109

108110
private RemoteConfigConstants() {}

‎firebase-config/src/main/java/com/google/firebase/remoteconfig/internal/ConfigContainer.java‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@
1414

1515
package com.google.firebase.remoteconfig.internal;
1616

17+
import static com.google.firebase.remoteconfig.RemoteConfigConstants.ExperimentDescriptionFieldKey.AFFECTED_PARAMETER_KEYS;
18+
import static com.google.firebase.remoteconfig.RemoteConfigConstants.ExperimentDescriptionFieldKey.EXPERIMENT_ID;
19+
1720
import com.google.errorprone.annotations.CanIgnoreReturnValue;
1821
import java.util.Date;
1922
import java.util.HashMap;
@@ -37,6 +40,7 @@ public class ConfigContainer {
3740
static final String ABT_EXPERIMENTS_KEY = "abt_experiments_key";
3841
static final String PERSONALIZATION_METADATA_KEY = "personalization_metadata_key";
3942
static final String TEMPLATE_VERSION_NUMBER_KEY = "template_version_number_key";
43+
static final String ROLLOUT_ID_PREFIX = "rollout";
4044
static final String ROLLOUT_METADATA_KEY = "rollout_metadata_key";
4145
public static final String ROLLOUT_METADATA_AFFECTED_KEYS = "affectedParameterKeys";
4246
public static final String ROLLOUT_METADATA_ID = "rolloutId";
@@ -220,6 +224,30 @@ private Map<String, Map<String, String>> createRolloutParameterKeyMap() throws J
220224
return rolloutMetadataMap;
221225
}
222226

227+
/** Creates a map where the key is the config key and the value is the experiment description. */
228+
private Map<String, JSONObject> createExperimentsMap() throws JSONException {
229+
Map<String, JSONObject> experimentsMap = new HashMap<>();
230+
JSONArray abtExperiments = this.getAbtExperiments();
231+
// Iterate through all experiments to check if it has the `affectedParameterKeys` field.
232+
for (int i = 0; i < abtExperiments.length(); i++) {
233+
JSONObject experiment = abtExperiments.getJSONObject(i);
234+
if (!experiment.has(AFFECTED_PARAMETER_KEYS)
235+
|| experiment.getString(EXPERIMENT_ID).startsWith(ROLLOUT_ID_PREFIX)) {
236+
continue;
237+
}
238+
239+
// Since a config key can only have one experiment associated with it, map the key to the
240+
// experiment.
241+
JSONArray affectedKeys = experiment.getJSONArray(AFFECTED_PARAMETER_KEYS);
242+
for (int j = 0; j < affectedKeys.length(); j++) {
243+
String key = affectedKeys.getString(j);
244+
experimentsMap.put(key, experiment);
245+
}
246+
}
247+
248+
return experimentsMap;
249+
}
250+
223251
/**
224252
* @param other The other {@link ConfigContainer} against which to compute the diff
225253
* @return The set of config keys that have changed between the this config and {@code other}
@@ -233,6 +261,10 @@ public Set<String> getChangedParams(ConfigContainer other) throws JSONException
233261
Map<String, Map<String, String>> rolloutMetadataMap = this.createRolloutParameterKeyMap();
234262
Map<String, Map<String, String>> otherRolloutMetadataMap = other.createRolloutParameterKeyMap();
235263

264+
// Config key to experiments map.
265+
Map<String, JSONObject> experimentsMap = this.createExperimentsMap();
266+
Map<String, JSONObject> otherExperimentsMap = other.createExperimentsMap();
267+
236268
Set<String> changed = new HashSet<>();
237269
Iterator<String> keys = this.getConfigs().keys();
238270
while (keys.hasNext()) {
@@ -284,6 +316,20 @@ public Set<String> getChangedParams(ConfigContainer other) throws JSONException
284316
continue;
285317
}
286318

319+
// If one and only one of the experiments map contains the key, add it to changed.
320+
if (experimentsMap.containsKey(key) != otherExperimentsMap.containsKey(key)) {
321+
changed.add(key);
322+
continue;
323+
}
324+
325+
// If both experiment maps contains the key, compare the experiments to see if it's different.
326+
if (otherExperimentsMap.containsKey(key)
327+
&& experimentsMap.containsKey(key)
328+
&& !otherExperimentsMap.get(key).toString().equals(experimentsMap.get(key).toString())) {
329+
changed.add(key);
330+
continue;
331+
}
332+
287333
// Since the key is the same in both configs, remove it from otherConfig
288334
otherConfig.remove(key);
289335
}

‎firebase-config/src/test/java/com/google/firebase/remoteconfig/internal/ConfigContainerTest.java‎

Lines changed: 90 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
package com.google.firebase.remoteconfig.internal;
1616

1717
import static com.google.common.truth.Truth.assertThat;
18+
import static com.google.firebase.remoteconfig.RemoteConfigConstants.ExperimentDescriptionFieldKey.AFFECTED_PARAMETER_KEYS;
1819
import static com.google.firebase.remoteconfig.RemoteConfigConstants.ExperimentDescriptionFieldKey.EXPERIMENT_ID;
1920
import static com.google.firebase.remoteconfig.RemoteConfigConstants.ExperimentDescriptionFieldKey.VARIANT_ID;
2021
import static com.google.firebase.remoteconfig.internal.Personalization.ARM_INDEX;
@@ -138,23 +139,102 @@ public void getChangedParams_sameP13nMetadata_returnsEmptySet() throws Exception
138139
}
139140

140141
@Test
141-
public void getChangedParams_changedExperimentsMetadata_returnsNoParamKeys() throws Exception {
142+
public void getChangedParams_sameExperimentsMetadata_returnsEmptySet() throws Exception {
143+
JSONArray activeExperiments = generateAbtExperiments(1);
144+
JSONArray fetchedExperiments = generateAbtExperiments(1);
145+
142146
ConfigContainer config =
143147
ConfigContainer.newBuilder()
144-
.replaceConfigsWith(ImmutableMap.of("string_param", "value_1"))
148+
.replaceConfigsWith(ImmutableMap.of("abt_test_key_1", "value_1"))
149+
.withAbtExperiments(activeExperiments)
145150
.build();
146151

147152
ConfigContainer other =
148153
ConfigContainer.newBuilder()
149-
.replaceConfigsWith(ImmutableMap.of("string_param", "value_1"))
150-
.withAbtExperiments(generateAbtExperiments(1))
154+
.replaceConfigsWith(ImmutableMap.of("abt_test_key_1", "value_1"))
155+
.withAbtExperiments(fetchedExperiments)
151156
.build();
152157

153158
Set<String> changedParams = config.getChangedParams(other);
154159

155160
assertThat(changedParams).isEmpty();
156161
}
157162

163+
@Test
164+
public void getChangedParams_changedExperimentsMetadata_returnsUpdatedKey() throws Exception {
165+
JSONArray activeExperiments = generateAbtExperiments(1);
166+
JSONArray fetchedExperiments = generateAbtExperiments(1);
167+
168+
activeExperiments.getJSONObject(0).put(VARIANT_ID, "32");
169+
170+
ConfigContainer config =
171+
ConfigContainer.newBuilder()
172+
.replaceConfigsWith(ImmutableMap.of("abt_test_key_1", "value_1"))
173+
.withAbtExperiments(activeExperiments)
174+
.build();
175+
176+
ConfigContainer other =
177+
ConfigContainer.newBuilder()
178+
.replaceConfigsWith(ImmutableMap.of("abt_test_key_1", "value_1"))
179+
.withAbtExperiments(fetchedExperiments)
180+
.build();
181+
182+
Set<String> changedParams = config.getChangedParams(other);
183+
184+
assertThat(changedParams).containsExactly("abt_test_key_1");
185+
}
186+
187+
@Test
188+
public void getChangedParams_deletedExperiment_returnsUpdatedKey() throws Exception {
189+
JSONArray activeExperiments = generateAbtExperiments(1);
190+
JSONArray fetchedExperiments = new JSONArray();
191+
192+
ConfigContainer config =
193+
ConfigContainer.newBuilder()
194+
.replaceConfigsWith(ImmutableMap.of("abt_test_key_1", "value_1"))
195+
.withAbtExperiments(activeExperiments)
196+
.build();
197+
198+
ConfigContainer other =
199+
ConfigContainer.newBuilder()
200+
.replaceConfigsWith(ImmutableMap.of("abt_test_key_1", "value_1"))
201+
.withAbtExperiments(fetchedExperiments)
202+
.build();
203+
204+
Set<String> changedParams = config.getChangedParams(other);
205+
206+
assertThat(changedParams).containsExactly("abt_test_key_1");
207+
}
208+
209+
@Test
210+
public void getChangedParams_changedExperimentsKeys_returnsUpdatedKey() throws Exception {
211+
JSONArray activeExperiments = generateAbtExperiments(1);
212+
JSONArray fetchedExperiments = generateAbtExperiments(1);
213+
214+
fetchedExperiments
215+
.getJSONObject(0)
216+
.getJSONArray(AFFECTED_PARAMETER_KEYS)
217+
.put(0, "abt_test_key_2");
218+
219+
ConfigContainer config =
220+
ConfigContainer.newBuilder()
221+
.replaceConfigsWith(
222+
ImmutableMap.of("abt_test_key_1", "value_1", "abt_test_key_2", "value_2"))
223+
.withAbtExperiments(activeExperiments)
224+
.build();
225+
226+
ConfigContainer other =
227+
ConfigContainer.newBuilder()
228+
.replaceConfigsWith(
229+
ImmutableMap.of("abt_test_key_1", "value_1", "abt_test_key_2", "value_2"))
230+
.withAbtExperiments(fetchedExperiments)
231+
.build();
232+
233+
Set<String> changedParams = config.getChangedParams(other);
234+
235+
assertThat(changedParams).containsExactly("abt_test_key_1", "abt_test_key_2");
236+
}
237+
158238
@Test
159239
public void getChangedParams_noChanges_returnsEmptySet() throws Exception {
160240
ConfigContainer config =
@@ -452,9 +532,14 @@ public void getChangedParams_unchangedRolloutMetadata_returnsNoKey() throws Exce
452532

453533
private static JSONArray generateAbtExperiments(int numExperiments) throws JSONException {
454534
JSONArray experiments = new JSONArray();
535+
JSONArray experimentKeys = new JSONArray();
536+
experimentKeys.put("abt_test_key_1");
455537
for (int experimentNum = 1; experimentNum <= numExperiments; experimentNum++) {
456538
experiments.put(
457-
new JSONObject().put(EXPERIMENT_ID, "exp" + experimentNum).put(VARIANT_ID, "var1"));
539+
new JSONObject()
540+
.put(EXPERIMENT_ID, "exp_" + experimentNum)
541+
.put(VARIANT_ID, "var1")
542+
.put(AFFECTED_PARAMETER_KEYS, experimentKeys));
458543
}
459544
return experiments;
460545
}

0 commit comments

Comments
 (0)