|
1169 | 1169 | end |
1170 | 1170 |
|
1171 | 1171 | describe 'Holdout Decision Service Tests' do |
1172 | | - let(:holdout_test_data_path) do |
1173 | | - File.join(File.dirname(__FILE__), 'test_data', 'holdout_test_data.json') |
1174 | | - end |
1175 | | - |
1176 | | - let(:holdout_test_data) do |
1177 | | - JSON.parse(File.read(holdout_test_data_path)) |
1178 | | - end |
1179 | | - |
1180 | | - let(:datafile_with_holdouts) do |
1181 | | - holdout_test_data['datafileWithHoldouts'] |
1182 | | - end |
1183 | | - |
1184 | 1172 | let(:config_with_holdouts) do |
1185 | 1173 | Optimizely::DatafileProjectConfig.new( |
1186 | | - datafile_with_holdouts, |
| 1174 | + OptimizelySpec::CONFIG_BODY_WITH_HOLDOUTS_JSON, |
1187 | 1175 | spy_logger, |
1188 | 1176 | error_handler |
1189 | 1177 | ) |
1190 | 1178 | end |
1191 | 1179 |
|
1192 | 1180 | let(:project_with_holdouts) do |
1193 | 1181 | Optimizely::Project.new( |
1194 | | - datafile: datafile_with_holdouts, |
| 1182 | + datafile: OptimizelySpec::CONFIG_BODY_WITH_HOLDOUTS_JSON, |
1195 | 1183 | logger: spy_logger, |
1196 | 1184 | error_handler: error_handler |
1197 | 1185 | ) |
|
1208 | 1196 | describe '#get_variations_for_feature_list with holdouts' do |
1209 | 1197 | describe 'when holdout is active and user is bucketed' do |
1210 | 1198 | it 'should return holdout decision with variation' do |
1211 | | - feature_flag = config_with_holdouts.feature_flag_key_map['test_flag_1'] |
| 1199 | + feature_flag = config_with_holdouts.feature_flag_key_map['boolean_feature'] |
1212 | 1200 | expect(feature_flag).not_to be_nil |
1213 | 1201 |
|
1214 | | - holdout = config_with_holdouts.get_holdout('holdout_included_1') |
| 1202 | + holdout = config_with_holdouts.holdouts.first |
1215 | 1203 | expect(holdout).not_to be_nil |
1216 | 1204 |
|
1217 | 1205 | user_context = project_with_holdouts.create_user_context('testUserId', {}) |
|
1243 | 1231 |
|
1244 | 1232 | describe 'when holdout is inactive' do |
1245 | 1233 | it 'should not bucket users and log appropriate message' do |
1246 | | - feature_flag = config_with_holdouts.feature_flag_key_map['test_flag_1'] |
| 1234 | + feature_flag = config_with_holdouts.feature_flag_key_map['boolean_feature'] |
1247 | 1235 | expect(feature_flag).not_to be_nil |
1248 | 1236 |
|
1249 | | - holdout = config_with_holdouts.get_holdout('holdout_global_1') |
| 1237 | + holdout = config_with_holdouts.holdouts.first |
1250 | 1238 | expect(holdout).not_to be_nil |
1251 | 1239 |
|
1252 | 1240 | # Mock holdout as inactive |
|
1275 | 1263 |
|
1276 | 1264 | describe 'when user is not bucketed into holdout' do |
1277 | 1265 | it 'should execute successfully with valid result structure' do |
1278 | | - feature_flag = config_with_holdouts.feature_flag_key_map['test_flag_1'] |
| 1266 | + feature_flag = config_with_holdouts.feature_flag_key_map['boolean_feature'] |
1279 | 1267 | expect(feature_flag).not_to be_nil |
1280 | 1268 |
|
1281 | | - holdout = config_with_holdouts.get_holdout('holdout_included_1') |
| 1269 | + holdout = config_with_holdouts.holdouts.first |
1282 | 1270 | expect(holdout).not_to be_nil |
1283 | 1271 |
|
1284 | 1272 | user_context = project_with_holdouts.create_user_context('testUserId', {}) |
|
1299 | 1287 |
|
1300 | 1288 | describe 'with user attributes for audience targeting' do |
1301 | 1289 | it 'should evaluate holdout with user attributes' do |
1302 | | - feature_flag = config_with_holdouts.feature_flag_key_map['test_flag_1'] |
| 1290 | + feature_flag = config_with_holdouts.feature_flag_key_map['boolean_feature'] |
1303 | 1291 | expect(feature_flag).not_to be_nil |
1304 | 1292 |
|
1305 | | - holdout = config_with_holdouts.get_holdout('holdout_included_1') |
| 1293 | + holdout = config_with_holdouts.holdouts.first |
1306 | 1294 | expect(holdout).not_to be_nil |
1307 | 1295 |
|
1308 | 1296 | user_attributes = { |
|
1329 | 1317 |
|
1330 | 1318 | describe 'with multiple holdouts' do |
1331 | 1319 | it 'should handle multiple holdouts for a single feature flag' do |
1332 | | - feature_flag = config_with_holdouts.feature_flag_key_map['test_flag_1'] |
| 1320 | + feature_flag = config_with_holdouts.feature_flag_key_map['boolean_feature'] |
1333 | 1321 | expect(feature_flag).not_to be_nil |
1334 | 1322 |
|
1335 | 1323 | user_context = project_with_holdouts.create_user_context('testUserId', {}) |
|
1351 | 1339 |
|
1352 | 1340 | describe 'with empty user ID' do |
1353 | 1341 | it 'should allow holdout bucketing with empty user ID' do |
1354 | | - feature_flag = config_with_holdouts.feature_flag_key_map['test_flag_1'] |
| 1342 | + feature_flag = config_with_holdouts.feature_flag_key_map['boolean_feature'] |
1355 | 1343 | expect(feature_flag).not_to be_nil |
1356 | 1344 |
|
1357 | 1345 | # Empty user ID should still be valid for bucketing |
|
1376 | 1364 |
|
1377 | 1365 | describe 'with decision reasons' do |
1378 | 1366 | it 'should populate decision reasons for holdouts' do |
1379 | | - feature_flag = config_with_holdouts.feature_flag_key_map['test_flag_1'] |
| 1367 | + feature_flag = config_with_holdouts.feature_flag_key_map['boolean_feature'] |
1380 | 1368 | expect(feature_flag).not_to be_nil |
1381 | 1369 |
|
1382 | | - holdout = config_with_holdouts.get_holdout('holdout_included_1') |
| 1370 | + holdout = config_with_holdouts.holdouts.first |
1383 | 1371 | expect(holdout).not_to be_nil |
1384 | 1372 |
|
1385 | 1373 | user_context = project_with_holdouts.create_user_context('testUserId', {}) |
|
1407 | 1395 | describe '#get_variation_for_feature with holdouts' do |
1408 | 1396 | describe 'when user is bucketed into holdout' do |
1409 | 1397 | it 'should return holdout decision before checking experiments or rollouts' do |
1410 | | - feature_flag = config_with_holdouts.feature_flag_key_map['test_flag_1'] |
| 1398 | + feature_flag = config_with_holdouts.feature_flag_key_map['boolean_feature'] |
1411 | 1399 | expect(feature_flag).not_to be_nil |
1412 | 1400 |
|
1413 | 1401 | user_context = project_with_holdouts.create_user_context('testUserId', {}) |
|
1432 | 1420 |
|
1433 | 1421 | describe 'when holdout returns no decision' do |
1434 | 1422 | it 'should fall through to experiment and rollout evaluation' do |
1435 | | - feature_flag = config_with_holdouts.feature_flag_key_map['test_flag_1'] |
| 1423 | + feature_flag = config_with_holdouts.feature_flag_key_map['boolean_feature'] |
1436 | 1424 | expect(feature_flag).not_to be_nil |
1437 | 1425 |
|
1438 | 1426 | # Use a user ID that won't be bucketed into holdout |
|
1453 | 1441 |
|
1454 | 1442 | describe 'with decision options' do |
1455 | 1443 | it 'should respect decision options when evaluating holdouts' do |
1456 | | - feature_flag = config_with_holdouts.feature_flag_key_map['test_flag_1'] |
| 1444 | + feature_flag = config_with_holdouts.feature_flag_key_map['boolean_feature'] |
1457 | 1445 | expect(feature_flag).not_to be_nil |
1458 | 1446 |
|
1459 | 1447 | user_context = project_with_holdouts.create_user_context('testUserId', {}) |
|
1474 | 1462 |
|
1475 | 1463 | describe 'holdout priority and evaluation order' do |
1476 | 1464 | it 'should evaluate holdouts before experiments' do |
1477 | | - feature_flag = config_with_holdouts.feature_flag_key_map['test_flag_1'] |
| 1465 | + feature_flag = config_with_holdouts.feature_flag_key_map['boolean_feature'] |
1478 | 1466 | expect(feature_flag).not_to be_nil |
1479 | 1467 |
|
1480 | 1468 | user_context = project_with_holdouts.create_user_context('testUserId', {}) |
|
1500 | 1488 | end |
1501 | 1489 |
|
1502 | 1490 | it 'should evaluate global holdouts for all flags' do |
1503 | | - feature_flag = config_with_holdouts.feature_flag_key_map['test_flag_1'] |
| 1491 | + feature_flag = config_with_holdouts.feature_flag_key_map['boolean_feature'] |
1504 | 1492 | expect(feature_flag).not_to be_nil |
1505 | 1493 |
|
1506 | 1494 | # Get global holdouts |
|
1525 | 1513 |
|
1526 | 1514 | it 'should respect included and excluded flags configuration' do |
1527 | 1515 | # Test that flags in excludedFlags are not affected by that holdout |
1528 | | - feature_flag = config_with_holdouts.feature_flag_key_map['test_flag_3'] |
| 1516 | + feature_flag = config_with_holdouts.feature_flag_key_map['boolean_feature'] |
1529 | 1517 |
|
1530 | 1518 | if feature_flag |
1531 | 1519 | # Get holdouts for this flag |
1532 | | - holdouts_for_flag = config_with_holdouts.get_holdouts_for_flag('test_flag_3') |
| 1520 | + holdouts_for_flag = config_with_holdouts.get_holdouts_for_flag('boolean_feature') |
1533 | 1521 |
|
1534 | 1522 | # Should not include holdouts that exclude this flag |
1535 | 1523 | excluded_holdout = holdouts_for_flag.find { |h| h['key'] == 'excluded_holdout' } |
|
1540 | 1528 |
|
1541 | 1529 | describe 'holdout logging and error handling' do |
1542 | 1530 | it 'should log when holdout evaluation starts' do |
1543 | | - feature_flag = config_with_holdouts.feature_flag_key_map['test_flag_1'] |
| 1531 | + feature_flag = config_with_holdouts.feature_flag_key_map['boolean_feature'] |
1544 | 1532 | expect(feature_flag).not_to be_nil |
1545 | 1533 |
|
1546 | 1534 | user_context = project_with_holdouts.create_user_context('testUserId', {}) |
|
1558 | 1546 | end |
1559 | 1547 |
|
1560 | 1548 | it 'should handle missing holdout configuration gracefully' do |
1561 | | - feature_flag = config_with_holdouts.feature_flag_key_map['test_flag_1'] |
| 1549 | + feature_flag = config_with_holdouts.feature_flag_key_map['boolean_feature'] |
1562 | 1550 | expect(feature_flag).not_to be_nil |
1563 | 1551 |
|
1564 | 1552 | # Temporarily remove holdouts |
|
1581 | 1569 | end |
1582 | 1570 |
|
1583 | 1571 | it 'should handle invalid holdout data gracefully' do |
1584 | | - feature_flag = config_with_holdouts.feature_flag_key_map['test_flag_1'] |
| 1572 | + feature_flag = config_with_holdouts.feature_flag_key_map['boolean_feature'] |
1585 | 1573 | expect(feature_flag).not_to be_nil |
1586 | 1574 |
|
1587 | 1575 | user_context = project_with_holdouts.create_user_context('testUserId', {}) |
|
1601 | 1589 |
|
1602 | 1590 | describe 'holdout bucketing behavior' do |
1603 | 1591 | it 'should use consistent bucketing for the same user' do |
1604 | | - feature_flag = config_with_holdouts.feature_flag_key_map['test_flag_1'] |
| 1592 | + feature_flag = config_with_holdouts.feature_flag_key_map['boolean_feature'] |
1605 | 1593 | expect(feature_flag).not_to be_nil |
1606 | 1594 |
|
1607 | 1595 | user_id = 'consistent_user' |
|
1640 | 1628 | end |
1641 | 1629 |
|
1642 | 1630 | it 'should use bucketing ID when provided' do |
1643 | | - feature_flag = config_with_holdouts.feature_flag_key_map['test_flag_1'] |
| 1631 | + feature_flag = config_with_holdouts.feature_flag_key_map['boolean_feature'] |
1644 | 1632 | expect(feature_flag).not_to be_nil |
1645 | 1633 |
|
1646 | 1634 | user_attributes = { |
|
1663 | 1651 | end |
1664 | 1652 |
|
1665 | 1653 | it 'should handle different traffic allocations' do |
1666 | | - feature_flag = config_with_holdouts.feature_flag_key_map['test_flag_1'] |
| 1654 | + feature_flag = config_with_holdouts.feature_flag_key_map['boolean_feature'] |
1667 | 1655 | expect(feature_flag).not_to be_nil |
1668 | 1656 |
|
1669 | 1657 | # Test with multiple users to see varying bucketing results |
|
1691 | 1679 |
|
1692 | 1680 | describe 'holdout integration with feature experiments' do |
1693 | 1681 | it 'should check holdouts before feature experiments' do |
1694 | | - feature_flag = config_with_holdouts.feature_flag_key_map['test_flag_1'] |
| 1682 | + feature_flag = config_with_holdouts.feature_flag_key_map['boolean_feature'] |
1695 | 1683 | expect(feature_flag).not_to be_nil |
1696 | 1684 |
|
1697 | 1685 | user_context = project_with_holdouts.create_user_context('testUserId', {}) |
|
1713 | 1701 | end |
1714 | 1702 |
|
1715 | 1703 | it 'should fall back to experiments if no holdout decision' do |
1716 | | - feature_flag = config_with_holdouts.feature_flag_key_map['test_flag_1'] |
| 1704 | + feature_flag = config_with_holdouts.feature_flag_key_map['boolean_feature'] |
1717 | 1705 | expect(feature_flag).not_to be_nil |
1718 | 1706 |
|
1719 | 1707 | user_context = project_with_holdouts.create_user_context('non_holdout_user_123', {}) |
|
0 commit comments