diff --git a/data/data-files/client-side-eval/basic-values.yml b/data/data-files/client-side-eval/basic-values.yml index 2f4bf3b6..b582146a 100644 --- a/data/data-files/client-side-eval/basic-values.yml +++ b/data/data-files/client-side-eval/basic-values.yml @@ -1,4 +1,6 @@ --- +# Covers FLGEA 1.1.2: client-side SDKs surface pre-evaluated outputs (value, variationIndex, reason) +# Covers FLGEA 1.6.3: post-evaluation returns correct variation value or default name: basic values - parameters: diff --git a/data/data-files/client-side-eval/errors-wrong-type.yml b/data/data-files/client-side-eval/errors-wrong-type.yml index 9c37a0ec..162ab623 100644 --- a/data/data-files/client-side-eval/errors-wrong-type.yml +++ b/data/data-files/client-side-eval/errors-wrong-type.yml @@ -1,4 +1,8 @@ --- +# Covers FLGEA 1.6.3: post-evaluation type mismatch returns WRONG_TYPE error and default value +# Covers FLGEA 1.3: evaluation reason ERROR with errorKind WRONG_TYPE +# FLGEDETAIL 1.3.1: errorKind WRONG_TYPE when flag value does not match requested type +# FLGEDETAIL 1.1.2: default value returned on type mismatch name: wrong type errors # These test are only relevant for SDKs that have variation methods for specific types diff --git a/data/data-files/client-side-eval/errors.yml b/data/data-files/client-side-eval/errors.yml index 302e828b..d7a8304e 100644 --- a/data/data-files/client-side-eval/errors.yml +++ b/data/data-files/client-side-eval/errors.yml @@ -1,4 +1,8 @@ --- +# Covers FLGEA 1.6.1: flag not found returns ERROR with errorKind FLAG_NOT_FOUND +# FLGEDETAIL 1.2.2: reason kind ERROR when flag could not be evaluated +# FLGEDETAIL 1.3.1: errorKind FLAG_NOT_FOUND when flag key unknown +# FLGEDETAIL 1.1.2: default value returned when evaluation fails name: errors - skipEvaluateAllFlags: true diff --git a/data/data-files/client-side-eval/reasons.yml b/data/data-files/client-side-eval/reasons.yml index 64025b48..c997aca8 100644 --- a/data/data-files/client-side-eval/reasons.yml +++ b/data/data-files/client-side-eval/reasons.yml @@ -1,4 +1,8 @@ --- +# Covers FLGEA 1.3: all evaluation reason kinds (OFF, FALLTHROUGH, TARGET_MATCH, RULE_MATCH, PREREQUISITE_FAILED, ERROR) +# FLGEDETAIL 1.2.2: reason kind values OFF, FALLTHROUGH, TARGET_MATCH, RULE_MATCH, PREREQUISITE_FAILED, ERROR +# FLGEDETAIL 1.2.3: ruleIndex, ruleId fields for RULE_MATCH; prerequisiteKey for PREREQUISITE_FAILED; inExperiment for experiments +# FLGEDETAIL 1.3.1: errorKind MALFORMED_FLAG and EXCEPTION name: evaluation reasons - parameters: diff --git a/data/data-files/server-side-eval/attribute-references.yml b/data/data-files/server-side-eval/attribute-references.yml index d0388ad3..147f46ed 100644 --- a/data/data-files/server-side-eval/attribute-references.yml +++ b/data/data-files/server-side-eval/attribute-references.yml @@ -1,4 +1,9 @@ --- +# Covers ATREF 1.2.1: plain references (no leading slash) treated as literal attribute name +# Covers ATREF 1.2.2: ~1 escape sequence interpreted as / +# Covers ATREF 1.2.3: ~0 escape sequence interpreted as ~ +# Also tests nested attribute traversal via pointer-style references +# Also covers FLGEA 1.2: context attribute lookup in clause evaluation name: attribute references constants: diff --git a/data/data-files/server-side-eval/builtin-attrs.yml b/data/data-files/server-side-eval/builtin-attrs.yml index 3c72d667..f9bed2f2 100644 --- a/data/data-files/server-side-eval/builtin-attrs.yml +++ b/data/data-files/server-side-eval/builtin-attrs.yml @@ -1,4 +1,5 @@ --- +# FLGEA 1.2: context attribute lookup for built-in attributes name: built-in attributes other than kind constants: diff --git a/data/data-files/server-side-eval/clause-kind-matching.yml b/data/data-files/server-side-eval/clause-kind-matching.yml index 1229ab2e..ba9f6c07 100644 --- a/data/data-files/server-side-eval/clause-kind-matching.yml +++ b/data/data-files/server-side-eval/clause-kind-matching.yml @@ -1,4 +1,5 @@ --- +# FLGEA 1.6.2.3: clause evaluation for kind attribute name: clause kind matching constants: diff --git a/data/data-files/server-side-eval/errors-bad-attribute-reference.yml b/data/data-files/server-side-eval/errors-bad-attribute-reference.yml index 2f70e438..45da815e 100644 --- a/data/data-files/server-side-eval/errors-bad-attribute-reference.yml +++ b/data/data-files/server-side-eval/errors-bad-attribute-reference.yml @@ -1,4 +1,8 @@ --- +# Covers ATREF 1.1.1: non-conforming strings are not valid attribute references +# Each parameter tests a different invalid reference: missing attribute, empty path +# component, invalid escape sequences. All must produce MALFORMED_FLAG errors. +# Also covers FLGEA 1.6.2.3: clause returns MALFORMED_FLAG for invalid attribute reference name: bad attribute reference errors - parameters: diff --git a/data/data-files/server-side-eval/errors-wrong-type.yml b/data/data-files/server-side-eval/errors-wrong-type.yml index bc6d453b..f07c78b1 100644 --- a/data/data-files/server-side-eval/errors-wrong-type.yml +++ b/data/data-files/server-side-eval/errors-wrong-type.yml @@ -1,4 +1,7 @@ --- +# FLGEA 1.6.3.3: post-evaluation WRONG_TYPE error +# FLGEDETAIL 1.3.1: errorKind WRONG_TYPE when flag value does not match requested type +# FLGEDETAIL 1.1.2: default value returned on type mismatch name: wrong type errors # These test are only relevant for SDKs that have variation methods for specific types diff --git a/data/data-files/server-side-eval/errors.yml b/data/data-files/server-side-eval/errors.yml index f2c649d3..18913d90 100644 --- a/data/data-files/server-side-eval/errors.yml +++ b/data/data-files/server-side-eval/errors.yml @@ -1,4 +1,8 @@ --- +# FLGEDETAIL 1.2.2: reason kind ERROR when flag could not be evaluated +# FLGEDETAIL 1.3.1: errorKind FLAG_NOT_FOUND and MALFORMED_FLAG +# FLGEDETAIL 1.1.2: default value returned when evaluation fails +# FLGEDETAIL 1.1.3: variationIndex absent when default value returned name: evaluation failures () constants: @@ -82,6 +86,7 @@ sdkData: variations: [ "" ] evaluations: + # FLGEA 1.6.1: FLAG_NOT_FOUND error - name: flag not found flagKey: no-such-flag context: @@ -91,6 +96,7 @@ evaluations: value: reason: { "kind": "ERROR", "errorKind": "FLAG_NOT_FOUND" } + # FLGEA 1.6.2.9: MALFORMED_FLAG for invalid off variation index - name: off variation too low flagKey: off-variation-too-low- context: @@ -105,6 +111,7 @@ evaluations: default: expect: + # FLGEA 1.6.2: off with undefined offVariation returns default - name: flag is off but has no off variation flagKey: no-off-variation- context: @@ -114,6 +121,7 @@ evaluations: value: reason: { kind: "OFF" } # this is *not* considered a "malformed flag" condition even though it returns the default + # FLGEA 1.6.2.9: MALFORMED_FLAG for invalid fallthrough variation index - name: fallthrough variation too low flagKey: fallthrough-variation-too-low- context: @@ -128,6 +136,7 @@ evaluations: default: expect: + # FLGEA 1.6.2.9: MALFORMED_FLAG for invalid target variation index - name: target variation too low flagKey: target-match-variation-too-low- context: @@ -142,6 +151,7 @@ evaluations: default: expect: + # FLGEA 1.6.2.9: MALFORMED_FLAG for invalid rule variation index - name: rule variation too low flagKey: rule-match-variation-too-low- context: diff --git a/data/data-files/server-side-eval/json-variations.yml b/data/data-files/server-side-eval/json-variations.yml index 10307706..64786cae 100644 --- a/data/data-files/server-side-eval/json-variations.yml +++ b/data/data-files/server-side-eval/json-variations.yml @@ -1,4 +1,5 @@ --- +# FLGEA 1.1.2: evaluation outputs with JSON variation values name: JSON variations constants: diff --git a/data/data-files/server-side-eval/negation-and-iteration.yml b/data/data-files/server-side-eval/negation-and-iteration.yml index 39706cf7..58a431a3 100644 --- a/data/data-files/server-side-eval/negation-and-iteration.yml +++ b/data/data-files/server-side-eval/negation-and-iteration.yml @@ -1,4 +1,5 @@ --- +# FLGEA 1.6.2.3: clause negation and value iteration name: clause negation and value iteration constants: diff --git a/data/data-files/server-side-eval/operators-date-bad-syntax.yml b/data/data-files/server-side-eval/operators-date-bad-syntax.yml index b2f35fb2..11a5f787 100644 --- a/data/data-files/server-side-eval/operators-date-bad-syntax.yml +++ b/data/data-files/server-side-eval/operators-date-bad-syntax.yml @@ -1,4 +1,5 @@ --- +# FLGEA 1.6.2.4.3: date-time operators with invalid syntax name: operators - date - bad syntax constants: diff --git a/data/data-files/server-side-eval/operators-date-bad-type.yml b/data/data-files/server-side-eval/operators-date-bad-type.yml index 6c3e3e65..d97d7bd2 100644 --- a/data/data-files/server-side-eval/operators-date-bad-type.yml +++ b/data/data-files/server-side-eval/operators-date-bad-type.yml @@ -1,4 +1,5 @@ --- +# FLGEA 1.6.2.4.3: date-time operators with invalid type name: operators - date - bad type constants: diff --git a/data/data-files/server-side-eval/operators-date-equal.yml b/data/data-files/server-side-eval/operators-date-equal.yml index d986a2cf..61c2f124 100644 --- a/data/data-files/server-side-eval/operators-date-equal.yml +++ b/data/data-files/server-side-eval/operators-date-equal.yml @@ -1,4 +1,5 @@ --- +# FLGEA 1.6.2.4.3: date-time operators with equal values name: operators - date - equal constants: diff --git a/data/data-files/server-side-eval/operators-date-unequal.yml b/data/data-files/server-side-eval/operators-date-unequal.yml index be6a275a..4b5e2c66 100644 --- a/data/data-files/server-side-eval/operators-date-unequal.yml +++ b/data/data-files/server-side-eval/operators-date-unequal.yml @@ -1,4 +1,5 @@ --- +# FLGEA 1.6.2.4.3: date-time operators with unequal values (before, after) name: operators - date - unequal constants: diff --git a/data/data-files/server-side-eval/operators-equality.yml b/data/data-files/server-side-eval/operators-equality.yml index f2194449..5a689055 100644 --- a/data/data-files/server-side-eval/operators-equality.yml +++ b/data/data-files/server-side-eval/operators-equality.yml @@ -1,4 +1,5 @@ --- +# FLGEA 1.6.2.4.1: equality (in operator) name: operators - equality () constants: diff --git a/data/data-files/server-side-eval/operators-numeric.yml b/data/data-files/server-side-eval/operators-numeric.yml index 83cd2e43..138aa840 100644 --- a/data/data-files/server-side-eval/operators-numeric.yml +++ b/data/data-files/server-side-eval/operators-numeric.yml @@ -1,4 +1,5 @@ --- +# FLGEA 1.6.2.4: numeric operators (lessThan, lessThanOrEqual, greaterThan, greaterThanOrEqual) name: operators - numeric constants: diff --git a/data/data-files/server-side-eval/operators-semver-bad-syntax.yml b/data/data-files/server-side-eval/operators-semver-bad-syntax.yml index 31febb51..99e7ebbd 100644 --- a/data/data-files/server-side-eval/operators-semver-bad-syntax.yml +++ b/data/data-files/server-side-eval/operators-semver-bad-syntax.yml @@ -1,4 +1,5 @@ --- +# FLGEA 1.6.2.4.4: semantic version invalid syntax name: operators - semver - bad syntax constants: diff --git a/data/data-files/server-side-eval/operators-semver-bad-type.yml b/data/data-files/server-side-eval/operators-semver-bad-type.yml index ffb6c61c..53c4efe4 100644 --- a/data/data-files/server-side-eval/operators-semver-bad-type.yml +++ b/data/data-files/server-side-eval/operators-semver-bad-type.yml @@ -1,4 +1,5 @@ --- +# FLGEA 1.6.2.4.4: semantic version invalid type name: operators - semver - bad type constants: diff --git a/data/data-files/server-side-eval/operators-semver-equal.yml b/data/data-files/server-side-eval/operators-semver-equal.yml index 5f4e7c16..c2a3159b 100644 --- a/data/data-files/server-side-eval/operators-semver-equal.yml +++ b/data/data-files/server-side-eval/operators-semver-equal.yml @@ -1,4 +1,5 @@ --- +# FLGEA 1.6.2.4.4: semantic version equality (semVerEqual) name: operators - semver - equal constants: diff --git a/data/data-files/server-side-eval/operators-semver-unequal.yml b/data/data-files/server-side-eval/operators-semver-unequal.yml index a3dacbc2..20e00f56 100644 --- a/data/data-files/server-side-eval/operators-semver-unequal.yml +++ b/data/data-files/server-side-eval/operators-semver-unequal.yml @@ -1,4 +1,5 @@ --- +# FLGEA 1.6.2.4.4: semantic version comparison (semVerLessThan, semVerGreaterThan) name: operators - semver - unequal constants: diff --git a/data/data-files/server-side-eval/operators-string.yml b/data/data-files/server-side-eval/operators-string.yml index 35838df2..3eecacf1 100644 --- a/data/data-files/server-side-eval/operators-string.yml +++ b/data/data-files/server-side-eval/operators-string.yml @@ -1,4 +1,5 @@ --- +# FLGEA 1.6.2.4: string operators (startsWith, endsWith, contains, matches) name: operators - string constants: diff --git a/data/data-files/server-side-eval/prerequisites.yml b/data/data-files/server-side-eval/prerequisites.yml index 7c747044..d853990e 100644 --- a/data/data-files/server-side-eval/prerequisites.yml +++ b/data/data-files/server-side-eval/prerequisites.yml @@ -1,4 +1,5 @@ --- +# FLGEA 1.6.2.1: prerequisite evaluation name: prerequisites constants: diff --git a/data/data-files/server-side-eval/rollout-or-experiment.yml b/data/data-files/server-side-eval/rollout-or-experiment.yml index 8e2fac3e..fdfecca2 100644 --- a/data/data-files/server-side-eval/rollout-or-experiment.yml +++ b/data/data-files/server-side-eval/rollout-or-experiment.yml @@ -1,4 +1,5 @@ --- +# FLGEA 1.6.2.6: rollouts and experiments name: rollout or experiment - constants: diff --git a/data/data-files/server-side-eval/rule-match.yml b/data/data-files/server-side-eval/rule-match.yml index 6cbf362c..81326881 100644 --- a/data/data-files/server-side-eval/rule-match.yml +++ b/data/data-files/server-side-eval/rule-match.yml @@ -1,4 +1,5 @@ --- +# FLGEA 1.6.2.3: flag rule evaluation name: rule match () constants: diff --git a/data/data-files/server-side-eval/segment-match.yml b/data/data-files/server-side-eval/segment-match.yml index f9f5fc6f..8c224f4a 100644 --- a/data/data-files/server-side-eval/segment-match.yml +++ b/data/data-files/server-side-eval/segment-match.yml @@ -1,4 +1,5 @@ --- + # FLGEA 1.6.2.5: segment matching name: segment match constants: diff --git a/data/data-files/server-side-eval/segment-recursion.yml b/data/data-files/server-side-eval/segment-recursion.yml index 8648da18..19677c95 100644 --- a/data/data-files/server-side-eval/segment-recursion.yml +++ b/data/data-files/server-side-eval/segment-recursion.yml @@ -1,4 +1,5 @@ --- +# FLGEA 1.6.2.5: segment matching with circular reference detection name: segment recursion constants: diff --git a/data/data-files/server-side-eval/target-match.yml b/data/data-files/server-side-eval/target-match.yml index b446a075..16734c6a 100644 --- a/data/data-files/server-side-eval/target-match.yml +++ b/data/data-files/server-side-eval/target-match.yml @@ -1,4 +1,5 @@ --- +# FLGEA 1.6.2.2: individual targeting (targets and contextTargets) name: target match constants: diff --git a/data/data-files/server-side-eval/variation-value-types.yml b/data/data-files/server-side-eval/variation-value-types.yml index f9e783d4..cd972c01 100644 --- a/data/data-files/server-side-eval/variation-value-types.yml +++ b/data/data-files/server-side-eval/variation-value-types.yml @@ -1,4 +1,5 @@ --- + # FLGEA 1.6: evaluation algorithm across variation value types name: variation value types () parameters: diff --git a/sdktests/client_side_auto_env_attributes.go b/sdktests/client_side_auto_env_attributes.go index 3bc9f2dd..4a7d989a 100644 --- a/sdktests/client_side_auto_env_attributes.go +++ b/sdktests/client_side_auto_env_attributes.go @@ -34,7 +34,10 @@ func doClientSideAutoEnvAttributesRequestingTests(t *ldtest.T) { t.Run("collisions", doClientSideAutoEnvAttributesRequestingCollisionsTests) } -// Start tests for events +// AUTOENVATTR 1.2.1.1: SDK must require explicit opt-in/opt-out decision for environment attributes +// AUTOENVATTR 1.2.1.2: SDK must not add environment attributes without explicit opt-in +// AUTOENVATTR 1.2.2.5: SDK must include envAttributesVersion in auto-env contexts +// AUTOENVATTR 1.2.2.7: All auto-env contexts must be added to the evaluation context func doClientSideAutoEnvAttributesEventsNoCollisionsTests(t *ldtest.T) { base := newCommonTestsBase(t, "doClientSideAutoEnvAttributesEventsNoCollisionsTests") dataSystem := NewSDKDataSystem(t, nil) @@ -112,6 +115,8 @@ func doClientSideAutoEnvAttributesEventsNoCollisionsTests(t *ldtest.T) { }) } +// AUTOENVATTR 1.2.2.6: Auto-env context kinds must not be added if user-specified context of that kind exists +// AUTOENVATTR 1.2.5.1: SDK must not alter any portion of a customer-provided context kind func doClientSideAutoEnvAttributesEventsCollisionsTests(t *ldtest.T) { base := newCommonTestsBase(t, "doClientSideAutoEnvAttributesEventsCollisionsTests") dataSystem := NewSDKDataSystem(t, nil) @@ -175,7 +180,8 @@ func doClientSideAutoEnvAttributesEventsCollisionsTests(t *ldtest.T) { // end tests for events -// start tests for streaming/polling +// AUTOENVATTR 1.2.2.5: SDK must include envAttributesVersion in auto-env contexts +// AUTOENVATTR 1.2.2.7: All auto-env contexts must be added to the evaluation context func doClientSideAutoEnvAttributesRequestingNoCollisionsTests(t *ldtest.T) { base := newCommonTestsBase(t, "doClientSideAutoEnvAttributesPollNoCollisionsTests") dsos := []SDKDataSystemOption{DataSystemOptionPolling(), DataSystemOptionStreaming()} @@ -206,6 +212,8 @@ func doClientSideAutoEnvAttributesRequestingNoCollisionsTests(t *ldtest.T) { } } +// AUTOENVATTR 1.2.2.6: Auto-env context kinds must not be added if user-specified context of that kind exists +// AUTOENVATTR 1.2.5.1: SDK must not alter any portion of a customer-provided context kind func doClientSideAutoEnvAttributesRequestingCollisionsTests(t *ldtest.T) { base := newCommonTestsBase(t, "doClientSideAutoEnvAttributesPollCollisionsTests") dsos := []SDKDataSystemOption{DataSystemOptionPolling(), DataSystemOptionStreaming()} @@ -245,7 +253,7 @@ func doClientSideAutoEnvAttributesRequestingCollisionsTests(t *ldtest.T) { // end tests for streaming/polling -// start tests for headers +// AUTOENVATTR 1.2.5.2: ld_application id, version, and versionName sourced by priority (config > platform > SDK info) func doClientSideAutoEnvAttributesHeaderTests(t *ldtest.T) { t.RequireCapability(servicedef.CapabilityTags) base := newCommonTestsBase(t, "doClientSideAutoEnvAttributesEventsCollisionsTests") diff --git a/sdktests/client_side_eval.go b/sdktests/client_side_eval.go index 9e86a009..8ede4d01 100644 --- a/sdktests/client_side_eval.go +++ b/sdktests/client_side_eval.go @@ -23,6 +23,9 @@ func doClientSideEvalTests(t *ldtest.T) { t.Run("parameterized", runParameterizedClientSideEvalTests) } +// FLGEDETAIL 1.1.7.1: client-side reason field may be absent when not requested +// FLGEDETAIL 1.4.4.1: client-side *variationDetail accepts (flagKey, defaultValue) +// FLGEDETAIL 1.4.4.2: client-side *variationDetail returns synchronously func runParameterizedClientSideEvalTests(t *ldtest.T) { // For client-side SDKs, you have to tell the SDK at initialization time whether we'll be using evaluation // reasons or not. The main effect of that is the client will add a withReasons query string parameter to diff --git a/sdktests/client_side_events_eval.go b/sdktests/client_side_events_eval.go index 33c85602..81ee5ec8 100644 --- a/sdktests/client_side_events_eval.go +++ b/sdktests/client_side_events_eval.go @@ -254,6 +254,9 @@ func doClientSideFeatureEventTests(t *ldtest.T) { }) } +// CSPE 1.2.1: prerequisite events emitted equivalent to direct evaluation +// CSPE 1.2.3: prerequisites processed in order before the dependent flag's event +// CSPE 1.2.4: recursive prerequisites are handled (prereq1 depends on prereq2) func doClientSideInOrderPrereqEventTests(t *ldtest.T) { dataBuilder := mockld.NewClientSDKDataBuilder() dataBuilder. diff --git a/sdktests/client_side_events_summary.go b/sdktests/client_side_events_summary.go index 1918543f..6551a839 100644 --- a/sdktests/client_side_events_summary.go +++ b/sdktests/client_side_events_summary.go @@ -34,6 +34,8 @@ func doClientSideSummaryEventTests(t *ldtest.T) { }) } +// CSSE 2.1.2.1: counters group evaluations by flag key, variation, and version +// CSSE 2.1.1: accumulator tracks default value, count, and resulting value per counter func doClientSideSummaryEventBasicTest(t *ldtest.T) { flag1Key := "flag1" flag1Result1 := mockld.ClientSDKFlag{ @@ -112,6 +114,8 @@ func doClientSideSummaryEventBasicTest(t *ldtest.T) { ) } +// CSSE 1.1.1.2: unique accumulator per unique context produces separate summary events +// CSSE 2.1.3.1: getSummary populates context on per-context summary events func doClientSidePerContextSummaryEventBasicTest(t *ldtest.T) { flag1Key := "flag1" flag1Result1 := mockld.ClientSDKFlag{ @@ -198,6 +202,7 @@ func doClientSidePerContextSummaryEventBasicTest(t *ldtest.T) { ) } +// CSSE 2.1.3.1: features include contextKinds from all evaluating context kinds func doClientSideSummaryEventContextKindsTest(t *ldtest.T) { flag1Key := "flag1" flag1Result := mockld.ClientSDKFlag{ @@ -270,6 +275,8 @@ func doClientSideSummaryEventContextKindsTest(t *ldtest.T) { ) } +// CSSE 1.1.1.2: unique accumulator per unique multi-kind context +// CSSE 2.1.3.1: contextKinds populated for per-context summary events func doClientSidePerContxtSummaryEventContextKindsTest(t *ldtest.T) { flag1Key := "flag1" flag1Result := mockld.ClientSDKFlag{ @@ -345,6 +352,7 @@ func doClientSidePerContxtSummaryEventContextKindsTest(t *ldtest.T) { ) } +// CSSE 2.1.3.1: unknown flag counter sets unknown=true with no version or variation func doClientSideSummaryEventUnknownFlagTest(t *ldtest.T) { unknownKey := "flag-x" context := ldcontext.New("user-key") @@ -380,6 +388,8 @@ func doClientSideSummaryEventUnknownFlagTest(t *ldtest.T) { ) } +// CSSE 1.1.2.1: accumulators cleared after getSummaries; new flush has fresh counts +// CSSE 3.1.1: summaries included in flushed event batch func doClientSideSummaryEventResetTest(t *ldtest.T) { flagKey := "flag1" flag1Result1 := mockld.ClientSDKFlag{ @@ -454,6 +464,9 @@ func doClientSideSummaryEventResetTest(t *ldtest.T) { ) } +// CSPE 1.2.2: summary counters updated equivalent to direct evaluation of prerequisites +// CSPE 1.2.4: recursive prerequisites generate correct summary counts +// CSSE 2.1.2.1: prerequisite evaluations increment counters in summary event func doClientSideSummaryBasicPrereqTest(t *ldtest.T) { topLevelKey := "flag1" topLevelResult := mockld.ClientSDKFlag{ @@ -551,6 +564,8 @@ func doClientSideSummaryBasicPrereqTest(t *ldtest.T) { ) } +// CSPE 1.2.1: unknown prerequisites generate unknown flag summary events (Note 1.2.1) +// CSSE 2.1.3.1: unknown prerequisite flag sets unknown=true in counter func doClientSideSummaryPrereqUnknownFlagTest(t *ldtest.T) { topLevelKey := "flag1" topLevelResult := mockld.ClientSDKFlag{ diff --git a/sdktests/common_tests_eval.go b/sdktests/common_tests_eval.go index bbe8d716..48971ab2 100644 --- a/sdktests/common_tests_eval.go +++ b/sdktests/common_tests_eval.go @@ -99,6 +99,12 @@ func (c CommonEvalParameterizedTestRunner[SDKDataType]) runTestSuite( } } +// FLGEDETAIL 1.1.1: EvaluationDetail returned by *variationDetail methods +// FLGEDETAIL 1.1.2: EvaluationDetail carries value field +// FLGEDETAIL 1.1.3: EvaluationDetail carries variationIndex field +// FLGEDETAIL 1.1.5: EvaluationDetail carries reason field +// FLGEDETAIL 1.2.1: EvaluationReason type describing evaluation result +// FLGEDETAIL 1.4.1: typed *variationDetail methods for bool, string, int, double, JSON func (c CommonEvalParameterizedTestRunner[SDKDataType]) runTestEval( t *ldtest.T, suite testmodel.EvalTestSuite[SDKDataType], diff --git a/sdktests/common_tests_events_contexts.go b/sdktests/common_tests_events_contexts.go index 5d94cb7a..942b452b 100644 --- a/sdktests/common_tests_events_contexts.go +++ b/sdktests/common_tests_events_contexts.go @@ -25,6 +25,11 @@ type eventContextTestParams struct { redactedShouldBe redactedAttrsByKind } +// CONTEXT 1.4.2: custom attribute value types in event context output +// CONTEXT 1.7.1: private attributes do not affect flag evaluations +// CONTEXT 1.7.2: kind, key, anonymous cannot be designated private +// CONTEXT 1.10.1.2: _meta.privateAttributes redaction in event output +// CONTEXT 1.10.1.5: non-built-in properties treated as custom attributes in events func makeEventContextTestParams() []eventContextTestParams { ret := []eventContextTestParams{ // Note that in the output matchers, we can't just check for JSON equality with an entire @@ -136,6 +141,12 @@ func makeEventContextTestParams() []eventContextTestParams { } return ret } + +// CONTEXT 1.7.1: private attributes do not affect flag evaluations +// CONTEXT 1.7.2: kind, key, anonymous not redacted even with allAttributesPrivate +// CONTEXT 1.10.1.2: _meta.privateAttributes redaction in event output +// CONTEXT 1.10.2.1: multi-kind contexts in event output +// CONTEXT 1.10.2.2: multi-kind sub-objects in event context output func (c CommonEventTests) EventContexts(t *ldtest.T) { // Flags to use for "feature" and "debug" event tests // The flag variation/value is irrelevant. diff --git a/sdktests/common_tests_events_request.go b/sdktests/common_tests_events_request.go index 22a7b5aa..a8c4f026 100644 --- a/sdktests/common_tests_events_request.go +++ b/sdktests/common_tests_events_request.go @@ -1,3 +1,19 @@ +// EVENTS spec traceability (EVENTS-event-payloads v4.0) +// +// The EVENTS spec defines 8 requirements (1.1.1–1.7.2.1), each stating that a +// particular event type "MUST pass validation against" its JSON schema. The +// harness does not run JSON Schema validators; instead it tests event shapes +// behaviorally in other files (common_tests_events_*.go). This file validates +// the HTTP request envelope — method, headers, URL path, payload IDs, gzip, +// TLS, and proxy — which are prerequisites for event delivery but are not +// individually called out by EVENTS requirement IDs. +// +// The currentEventSchema constant below corresponds to the spec version +// directory (v4.0) and is used in schema-version header assertions elsewhere. +// +// Coverage model: indirect/behavioral. Individual event-type schema conformance +// is tested across multiple files rather than by a JSON schema validator. + package sdktests import ( @@ -19,6 +35,8 @@ import ( const currentEventSchema = "4" const phpLegacyEventSchema = "2" +// EVENTS (all): validates POST method, auth header, content-type, and gzip +// encoding — transport-level prerequisites for delivering schema-conformant payloads. func (c CommonEventTests) RequestMethodAndHeaders(t *ldtest.T, credential string, headersMatcher m.Matcher) { t.Run("method and headers", func(t *ldtest.T) { for _, transport := range c.withAvailableTransports(t) { @@ -66,6 +84,8 @@ func (c CommonEventTests) RequestMethodAndHeaders(t *ldtest.T, credential string }) } +// EVENTS (all): validates correct URL path construction for the /bulk endpoint, +// ensuring events reach the schema-validated ingestion path. func (c CommonEventTests) RequestURLPath(t *ldtest.T, pathMatcher m.Matcher) { t.Run("URL path is computed correctly", func(t *ldtest.T) { for _, trailingSlash := range []bool{false, true} { @@ -94,6 +114,8 @@ func (c CommonEventTests) RequestURLPath(t *ldtest.T, pathMatcher m.Matcher) { }) } +// EVENTS (all): validates unique X-LaunchDarkly-Payload-Id header per flush, +// supporting deduplication of schema-conformant event payloads. func (c CommonEventTests) UniquePayloadIDs(t *ldtest.T) { t.Run("new payload ID for each post", func(t *ldtest.T) { dataSystem := NewSDKDataSystem(t, nil) @@ -121,6 +143,8 @@ func (c CommonEventTests) UniquePayloadIDs(t *ldtest.T) { }) } +// EVENTS (all): validates event delivery through an HTTP proxy, ensuring +// schema-conformant payloads are transmitted correctly in proxied environments. func (c CommonEventTests) HTTPProxy(t *ldtest.T) { t.Run("http proxy", func(t *ldtest.T) { events := NewSDKEventSink(t) diff --git a/sdktests/common_tests_hooks.go b/sdktests/common_tests_hooks.go index 4aa5f433..a8168e2a 100644 --- a/sdktests/common_tests_hooks.go +++ b/sdktests/common_tests_hooks.go @@ -157,6 +157,8 @@ func trackTestParams(context o.Maybe[ldcontext.Context]) []TrackParameters { } } +// HOOK 1.2.1: beforeEvaluation stage accepts EvaluationSeriesContext and returns EvaluationSeriesData +// HOOK 1.2.1.2: beforeEvaluation executes before the flag value has been determined func executesBeforeEvaluationStageDetail(t *ldtest.T, detail bool) { testParams := variationTestParams(detail) @@ -197,6 +199,7 @@ func executesBeforeEvaluationStageDetail(t *ldtest.T, detail bool) { } } +// HOOK 1.2.1: beforeEvaluation stage fires for migration variations func executesBeforeEvaluationStageMigration(t *ldtest.T) { hookName := "executesBeforeEvaluationStageMigration" client, hooks := createClientForHooks(t, []string{hookName}, nil) @@ -222,6 +225,8 @@ func executesBeforeEvaluationStageMigration(t *ldtest.T) { }) } +// HOOK 1.2.2: afterEvaluation stage accepts EvaluationSeriesContext, EvaluationSeriesData, and EvaluationDetail +// HOOK 1.2.2.2: afterEvaluation executes after the flag detail has been determined func executesAfterEvaluationStageDetail(t *ldtest.T, detail bool) { testParams := variationTestParams(detail) @@ -269,6 +274,7 @@ func executesAfterEvaluationStageDetail(t *ldtest.T, detail bool) { } } +// HOOK 1.2.2: afterEvaluation stage fires for migration variations with correct EvaluationDetail func executesAfterEvaluationStageMigration(t *ldtest.T) { hookName := "executesBeforeEvaluationStageMigration" client, hooks := createClientForHooks(t, []string{hookName}, nil) @@ -296,6 +302,8 @@ func executesAfterEvaluationStageMigration(t *ldtest.T) { }) } +// HOOK 1.2.2.3: afterEvaluation receives the EvaluationSeriesData returned by beforeEvaluation +// HOOK 1.3.6: series data propagates between stages on a per-hook basis func beforeEvaluationDataPropagatesToAfterDetail(t *ldtest.T, detail bool) { testParams := variationTestParams(detail) @@ -339,6 +347,7 @@ func beforeEvaluationDataPropagatesToAfterDetail(t *ldtest.T, detail bool) { } } +// HOOK 1.2.2.3, 1.3.6: series data propagation also works for migration variations func beforeEvaluationDataPropagatesToAfterMigration(t *ldtest.T) { hookName := "beforeEvaluationDataPropagatesToAfterDetail" hookData := make(map[servicedef.HookStage]servicedef.SDKConfigEvaluationHookData) @@ -367,9 +376,8 @@ func beforeEvaluationDataPropagatesToAfterMigration(t *ldtest.T) { }) } -// This test is meant to check Requirement HOOKS:1.3.7: -// The client MUST handle exceptions which are thrown (or errors returned, if idiomatic for the language) -// during the execution of a stage or handler allowing operations to complete unaffected. +// HOOK 1.3.7: client handles stage exceptions without affecting operations +// HOOK 1.3.7.1: when error prevents stage from returning data, use empty data func errorInBeforeStageDoesNotAffectAfterStage(t *ldtest.T) { const numHooks = 3 @@ -420,6 +428,8 @@ func errorInBeforeStageDoesNotAffectAfterStage(t *ldtest.T) { } } +// HOOK 1.6.1: afterTrack stage accepts TrackSeriesContext (key, context, metricValue, data) +// HOOK 1.6.2: afterTrack executes after the custom event has been enqueued func executesAfterTrackStage(t *ldtest.T) { hookName := "executesAfterTrackStage" context := ldcontext.New("user-key") @@ -455,6 +465,7 @@ func executesAfterTrackStage(t *ldtest.T) { } } +// HOOK 1.3.7: client handles hook errors without crashing; afterTrack not called on error func errorInHookPreventsAfterTrackStage(t *ldtest.T) { hookName := "doesNotExecuteAfterTrackStage" context := ldcontext.New("user-key") @@ -516,8 +527,7 @@ func observedHookOrder(t *ldtest.T, hooks *Hooks, names []string, stage serviced return out } -// afterTrack must execute in the order of hook registration (forward), -// unlike afterEvaluation/afterIdentify which run in reverse-registration order. +// HOOK 1.6.3: afterTrack executes hooks in registration order (not reversed) func executesAfterTrackHooksInRegistrationOrder(t *ldtest.T) { names := hookOrderTestNames("afterTrackOrderHook", 3) @@ -542,7 +552,7 @@ func executesAfterTrackHooksInRegistrationOrder(t *ldtest.T) { "afterTrack hooks must execute in the order of hook registration") } -// beforeEvaluation must execute in the order of hook registration. +// HOOK 1.3.2: beforeEvaluation executes hooks in registration order func executesBeforeEvaluationHooksInRegistrationOrder(t *ldtest.T) { names := hookOrderTestNames("beforeEvalOrderHook", 3) @@ -569,7 +579,7 @@ func executesBeforeEvaluationHooksInRegistrationOrder(t *ldtest.T) { "beforeEvaluation hooks must execute in the order of hook registration") } -// afterEvaluation must execute in the reverse of the order of hook registration. +// HOOK 1.3.3: afterEvaluation executes hooks in reverse registration order func executesAfterEvaluationHooksInReverseRegistrationOrder(t *ldtest.T) { names := hookOrderTestNames("afterEvalOrderHook", 3) diff --git a/sdktests/common_tests_poll_request.go b/sdktests/common_tests_poll_request.go index f7e8a6cb..a4a94dfc 100644 --- a/sdktests/common_tests_poll_request.go +++ b/sdktests/common_tests_poll_request.go @@ -106,6 +106,10 @@ func (c CommonPollingTests) LargePayloads(t *ldtest.T) { }) } +// ENVFILTER 1.2.1: request URL joins base URI with filter query parameter +// ENVFILTER 1.2.2: exactly one filter parameter is included +// ENVFILTER 1.2.3: filter parameter value is URL encoded +// ENVFILTER 1.3.1: filter parameter omitted for invalid filter key func (c CommonPollingTests) RequestURLPath(t *ldtest.T, pathMatcher func(flagRequestMethod) m.Matcher) { t.Run("URL path is computed correctly", func(t *ldtest.T) { for _, filter := range c.environmentFilters() { diff --git a/sdktests/common_tests_stream_fdv2.go b/sdktests/common_tests_stream_fdv2.go index 7a2ff228..4d16365d 100644 --- a/sdktests/common_tests_stream_fdv2.go +++ b/sdktests/common_tests_stream_fdv2.go @@ -85,6 +85,7 @@ func (c CommonStreamingTests) StateTransitions(t *ldtest.T) { t.Run("updates previously known state", c.UpdatesPreviouslyKnownState) } +// DATASYSTEM 1.1.1: SDK initializes from a single synchronizer with a full basis func (c CommonStreamingTests) InitializeFromEmptyState(t *ldtest.T) { dataSystem, configurers := c.setupDataSystems(t, c.makeSDKDataWithFlag(1, initialValue)) client := NewSDKClient(t, c.baseSDKConfigurationPlus(configurers...)...) @@ -93,6 +94,7 @@ func (c CommonStreamingTests) InitializeFromEmptyState(t *ldtest.T) { validatePayloadReceived(t, dataSystem.Synchronizers[0].endpoint, client, "", expectedEvaluations) } +// DATASYSTEM 1.1.2: initializers run in order; synchronizer receives basis state func (c CommonStreamingTests) InitializeFromPollingInitializer(t *ldtest.T) { dataBefore := mockld.NewServerSDKDataBuilder().Flag(c.makeServerSideFlag("flag-key", 1, initialValue)).Build() dataAfter := mockld.NewServerSDKDataBuilder().IntentCode("none").IntentReason("up-to-date").Build() @@ -107,6 +109,7 @@ func (c CommonStreamingTests) InitializeFromPollingInitializer(t *ldtest.T) { validatePayloadReceived(t, dataSystem.Synchronizers[0].Endpoint(), client, "initial", expectedEvaluations) } +// DATASYSTEM 1.1.5: initializer basis + synchronizer xfer-changes merged at startup func (c CommonStreamingTests) InitializeFromPollingInitializerWithStreamingUpdates(t *ldtest.T) { dataBefore := mockld.NewServerSDKDataBuilder(). Flag(c.makeServerSideFlag("flag-key", 1, initialValue)). @@ -128,6 +131,7 @@ func (c CommonStreamingTests) InitializeFromPollingInitializerWithStreamingUpdat validatePayloadReceived(t, dataSystem.Synchronizers[0].Endpoint(), client, "initial", expectedEvaluations) } +// DATASYSTEM 1.1.2, 1.1.3: first initializer failure falls through to next initializer func (c CommonStreamingTests) InitializeFromTwoPollingInitializers(t *ldtest.T) { emptyPayload := mockld.NewServerSDKDataBuilder(). Build() @@ -158,6 +162,7 @@ func (c CommonStreamingTests) InitializeFromTwoPollingInitializers(t *ldtest.T) validatePayloadReceived(t, dataSystem.Synchronizers[0].Endpoint(), client, "expected-state", expectedEvaluations) } +// DATASYSTEM 1.2.7: recoverable error triggers fallback to next synchronizer func (c CommonStreamingTests) RecoverableFallbackToSecondarySynchronizer(t *ldtest.T) { t.LongRunning() @@ -197,6 +202,7 @@ func (c CommonStreamingTests) RecoverableFallbackToSecondarySynchronizer(t *ldte validatePayloadReceived(t, secondaryEndpoint, client, "", expectedEvaluations) } +// DATASYSTEM 1.2.7: non-recoverable 4xx error permanently removes synchronizer, falls back immediately func (c CommonStreamingTests) PermanentFallbackToSecondarySynchronizer(t *ldtest.T) { // First synchronizer returns 401 Unauthorized, which is a non-recoverable error. // Non-recoverable 4xx errors (all except 400, 408, 429) cause the synchronizer to be @@ -226,6 +232,8 @@ func (c CommonStreamingTests) PermanentFallbackToSecondarySynchronizer(t *ldtest validatePayloadReceived(t, secondaryEndpoint, client, "", expectedEvaluations) } +// DATASYSTEM 1.2.8: after recovery period, SDK attempts reconnection to earlier synchronizers +// CSFDV2 8.2.1: client-side recovery timeout default is 300 seconds (VALID condition) func (c CommonStreamingTests) RecoverableFallbackWithRecovery(t *ldtest.T) { t.LongRunning() @@ -309,6 +317,8 @@ func (c CommonStreamingTests) RecoverableFallbackWithRecovery(t *ldtest.T) { validatePayloadReceived(t, firstEndpoint, client, "", expectedEvaluations) } +// DATASYSTEM 1.2.8: permanently removed synchronizers excluded from recovery; recoverable ones revisited +// CSFDV2 8.2.1: client-side recovery timeout default is 300 seconds (VALID condition) func (c CommonStreamingTests) PermanentFallbackWithRecovery(t *ldtest.T) { t.LongRunning() @@ -428,6 +438,11 @@ func (c CommonStreamingTests) FDv1FallbackDirective(t *ldtest.T) { // serves it. The FDv1 endpoint therefore delivers a distinctive flag value, and the // test asserts evaluations return that value — proving FDv1 actually became the // active data source, not just that its URL was hit. +// +// DATASYSTEM 1.6.1: X-LD-FD-Fallback header on error engages FDv1 Fallback Synchronizer +// DATASYSTEM 1.6.3: FDv2 synchronizer chain halted when directive engages FDv1 +// CSFDV2 8.1.1: x-ld-fd-fallback header triggers client-side FDv1 fallback +// CSFDV2 8.1.3: FDv2 synchronizers disabled, not removed func (c CommonStreamingTests) DirectiveOnStreamingErrorEngagesFDv1(t *ldtest.T) { streamHandler, _ := httphelpers.RecordingHandler(httphelpers.HandlerWithResponse( 403, http.Header{"X-LD-FD-Fallback": []string{"true"}}, nil)) @@ -509,6 +524,11 @@ func (c CommonStreamingTests) DirectiveOnStreamingErrorEngagesFDv1(t *ldtest.T) // request whose validators match (RFC 9111 §4.3.4); browser-based runtimes in // particular translate an unsolicited 304 into a 200 against their cache, which // would defeat the "FDv1 supplies no fresh data" property this test relies on. +// +// DATASYSTEM 1.6.2: payload applied to Memory Store before transitioning to FDv1 +// DATASYSTEM 1.6.3: FDv2 Primary Synchronizer stopped when directive engages FDv1 +// CSFDV2 8.1.1: x-ld-fd-fallback header triggers client-side FDv1 fallback +// CSFDV2 8.1.3: FDv2 synchronizers disabled, not removed func (c CommonStreamingTests) DirectiveOnStreamingSuccessAppliesPayload(t *ldtest.T) { streamingValue := ldvalue.String("value-from-streaming-payload") streamingData := c.makeSDKDataWithFlag(1, streamingValue) @@ -595,6 +615,10 @@ func (c CommonStreamingTests) DirectiveOnStreamingSuccessAppliesPayload(t *ldtes // initializer-then-synchronizer flow. In particular, the configured FDv2 Primary // Synchronizer must never be started; the SDK transitions directly to the FDv1 // Fallback Synchronizer. +// +// DATASYSTEM 1.6.3: initializer directive skips FDv2 synchronizers entirely +// CSFDV2 8.1.1: x-ld-fd-fallback header triggers client-side FDv1 fallback +// CSFDV2 8.1.3: FDv2 synchronizers never started after directive func (c CommonStreamingTests) DirectiveOnPollingInitializerSkipsSynchronizers(t *ldtest.T) { // Initializer endpoint: 500 + directive. No payload accompanies the directive in // this variant, so there is nothing to apply beforehand (1.6.2 is a no-op here). @@ -688,6 +712,9 @@ func (c CommonStreamingTests) DirectiveOnPollingInitializerSkipsSynchronizers(t // *ignored* the directive, we would see retries on the streaming endpoint; absence of // those retries is the positive signal that the directive caused a halt rather than // ordinary permanent removal (which would fire on a 4xx instead). +// +// DATASYSTEM 1.6.3: directive without FDv1 configured halts data system entirely +// CSFDV2 8.1.1: x-ld-fd-fallback header recognized on client-side func (c CommonStreamingTests) DirectiveWithoutFDv1ConfiguredHaltsDataSystem(t *ldtest.T) { handler, channel := httphelpers.RecordingHandler(httphelpers.HandlerWithResponse( 500, http.Header{"X-LD-FD-Fallback": []string{"true"}}, nil)) @@ -751,6 +778,10 @@ func (c CommonStreamingTests) DirectiveWithoutFDv1ConfiguredHaltsDataSystem(t *l // applies. We prove this by keeping the FDv1 endpoint healthy and watching a long // enough window that the FDv2 Recovery Condition (5 minutes in the default config) // would normally fire if the SDK were still treating this as a heuristic fallback. +// +// DATASYSTEM 1.6.4: directed fallback is terminal; SDK never returns to FDv2 synchronizers +// CSFDV2 8.1.1: x-ld-fd-fallback triggers client-side FDv1 fallback +// CSFDV2 8.1.3: FDv2 synchronizers remain disabled indefinitely func (c CommonStreamingTests) DirectedFallbackIsTerminal(t *ldtest.T) { t.LongRunning() @@ -838,6 +869,7 @@ func (c CommonStreamingTests) DirectedFallbackIsTerminal(t *ldtest.T) { "flag-key should still be served from the FDv1 fallback after the recovery window elapses") } +// DATASYSTEM 1.2.1: synchronizer reconnects with previously known state (basis query param) func (c CommonStreamingTests) SavesPreviouslyKnownState(t *ldtest.T) { dataBefore := c.makeSDKDataWithFlag(1, initialValue) dataAfter := mockld.NewServerSDKDataBuilder().IntentCode("none").IntentReason("up-to-date").Build() @@ -852,6 +884,7 @@ func (c CommonStreamingTests) SavesPreviouslyKnownState(t *ldtest.T) { validatePayloadReceived(t, streamEndpoint, client, "initial", expectedEvaluations) } +// DATASYSTEM 1.2.1: xfer-full intent replaces entire store contents on reconnect func (c CommonStreamingTests) ReplacesPreviouslyKnownState(t *ldtest.T) { dataBefore := c.makeSDKDataWithFlag(1, initialValue) dataAfter := mockld.NewServerSDKDataBuilder(). @@ -874,6 +907,7 @@ func (c CommonStreamingTests) ReplacesPreviouslyKnownState(t *ldtest.T) { validatePayloadReceived(t, streamEndpoint, client, "initial", expectedEvaluations) } +// DATASYSTEM 1.2.1: xfer-changes intent applies deltas to existing store on reconnect func (c CommonStreamingTests) UpdatesPreviouslyKnownState(t *ldtest.T) { dataBefore := c.makeSDKDataWithFlag(1, initialValue) dataAfter := mockld.NewServerSDKDataBuilder(). @@ -894,6 +928,7 @@ func (c CommonStreamingTests) UpdatesPreviouslyKnownState(t *ldtest.T) { validatePayloadReceived(t, streamEndpoint, client, "initial", expectedEvaluations) } +// DATASYSTEM 1.2.10: updates buffered until payload-transferred event confirms completion func (c CommonStreamingTests) UpdatesAreNotCompleteUntilPayloadTransferredIsSent(t *ldtest.T) { dataSystem, configurers := c.setupDataSystems(t, c.makeSDKDataWithFlag(1, initialValue)) client := NewSDKClient(t, c.baseSDKConfigurationPlus(configurers...)...) @@ -931,6 +966,7 @@ func (c CommonStreamingTests) UpdatesAreNotCompleteUntilPayloadTransferredIsSent pollUntilFlagValueUpdated(t, client, "new-flag-key", context, defaultValue, newInitialValue, defaultValue) } +// DATASYSTEM 1.2.10: multiple sequential payloads each applied after payload-transferred func (c CommonStreamingTests) HandlesMultipleUpdates(t *ldtest.T) { dataSystem, configurers := c.setupDataSystems(t, c.makeSDKDataWithFlag(1, initialValue)) client := NewSDKClient(t, c.baseSDKConfigurationPlus(configurers...)...) @@ -957,6 +993,7 @@ func (c CommonStreamingTests) HandlesMultipleUpdates(t *ldtest.T) { pollUntilFlagValueUpdated(t, client, "new-flag-key", context, newInitialValue, defaultValue, defaultValue) } +// DATASYSTEM 1.2.10: SDK trusts aggregate state version, ignores individual model version func (c CommonStreamingTests) IgnoresModelVersion(t *ldtest.T) { dataSystem, configurers := c.setupDataSystems(t, c.makeSDKDataWithFlag(100, initialValue)) client := NewSDKClient(t, c.baseSDKConfigurationPlus(configurers...)...) @@ -976,6 +1013,7 @@ func (c CommonStreamingTests) IgnoresModelVersion(t *ldtest.T) { pollUntilFlagValueUpdated(t, client, "flag-key", context, initialValue, updatedValue, defaultValue) } +// DATASYSTEM 1.2.2: heartbeat events ignored, do not affect payload processing func (c CommonStreamingTests) IgnoresHeartBeat(t *ldtest.T) { dataSystem, configurers := c.setupDataSystems(t, c.makeSDKDataWithFlag(1, initialValue)) client := NewSDKClient(t, c.baseSDKConfigurationPlus(configurers...)...) @@ -1046,6 +1084,7 @@ func (c CommonStreamingTests) IgnoresUnknownEventStartOfPayload(t *ldtest.T) { pollUntilFlagValueUpdated(t, client, "flag-key", context, initialValue, updatedValue, defaultValue) } +// DATASYSTEM 1.2.3: error event discards preceding buffered updates; subsequent payload applied func (c CommonStreamingTests) CanDiscardPartialEventsOnError(t *ldtest.T) { dataSystem, configurers := c.setupDataSystems(t, c.makeSDKDataWithFlag(1, initialValue)) client := NewSDKClient(t, c.baseSDKConfigurationPlus(configurers...)...) @@ -1078,6 +1117,7 @@ func (c CommonStreamingTests) CanDiscardPartialEventsOnError(t *ldtest.T) { require.Equal(t, initialValue, value) } +// DATASYSTEM 1.2.3: error event discards updates; xfer-full intent replaces store entirely func (c CommonStreamingTests) CanDiscardFullEventsOnError(t *ldtest.T) { dataSystem, configurers := c.setupDataSystems(t, c.makeSDKDataWithFlag(1, initialValue)) client := NewSDKClient(t, c.baseSDKConfigurationPlus(configurers...)...) @@ -1102,6 +1142,7 @@ func (c CommonStreamingTests) CanDiscardFullEventsOnError(t *ldtest.T) { pollUntilFlagValueUpdated(t, client, "new-flag-key", context, defaultValue, newInitialValue, defaultValue) } +// DATASYSTEM 1.2.2: goodbye event causes SDK to disconnect and discard pending updates func (c CommonStreamingTests) DisconnectsOnGoodbye(t *ldtest.T) { dataBefore := c.makeSDKDataWithFlag(1, initialValue) dataAfter := mockld.NewServerSDKDataBuilder().IntentCode("none").IntentReason("up-to-date").Build() diff --git a/sdktests/common_tests_stream_request.go b/sdktests/common_tests_stream_request.go index ef110b0e..934c514f 100644 --- a/sdktests/common_tests_stream_request.go +++ b/sdktests/common_tests_stream_request.go @@ -53,6 +53,10 @@ func (c CommonStreamingTests) RequestMethodAndHeaders(t *ldtest.T, credential st }) } +// ENVFILTER 1.2.1: request URL joins base URI with filter query parameter +// ENVFILTER 1.2.2: exactly one filter parameter is included +// ENVFILTER 1.2.3: filter parameter value is URL encoded +// ENVFILTER 1.3.1: filter parameter omitted for invalid filter key func (c CommonStreamingTests) RequestURLPath(t *ldtest.T, pathMatcher func(flagRequestMethod) m.Matcher) { t.Run("URL path is computed correctly", func(t *ldtest.T) { for _, filter := range c.environmentFilters() { diff --git a/sdktests/sdk_context_type.go b/sdktests/sdk_context_type.go index bcd35727..7e075b15 100644 --- a/sdktests/sdk_context_type.go +++ b/sdktests/sdk_context_type.go @@ -32,6 +32,22 @@ func doSDKContextTypeTests(t *ldtest.T) { // a context object-- the current test harness architecture requires all test service commands to be directed // at a client instance. +// CTXBLD 1.2.3: kind operation takes kind string and optional key, returns LDAttributesBuilder +// CTXBLD 1.2.6: build operation returns LDContext +// CTXBLD 1.2.7: one kind = individual context, more than one = multi-context +// CTXBLD 1.4.1: dedicated name operation accepts string +// CTXBLD 1.4.2: dedicated anonymous operation accepts boolean +// CTXBLD 1.4.3: unset anonymous behaves as false +// CTXBLD 1.5.1: generic attribute setter stores custom attributes +// CTXBLD 1.5.3: null value removes custom attribute +// CTXBLD 1.5.4: non-null value stores custom attribute +// CTXBLD 1.6.4: add private attribute reference strings +// CONTEXT 1.2.3.2: anonymous false treated as absent +// CONTEXT 1.4.2: custom attribute value types (boolean, number, string, array, object) +// CONTEXT 1.5.6: null attribute value treated as absent +// CONTEXT 1.10.1.1: single-kind JSON includes kind and key +// CONTEXT 1.10.2.1: multi-kind JSON includes kind:"multi" +// CONTEXT 1.10.2.2: multi-kind sub-objects follow single-kind schema without kind property func doSDKContextBuildTests(t *ldtest.T) { dataSystem := NewSDKDataSystem(t, nil) client := NewSDKClient(t, dataSystem) @@ -112,6 +128,8 @@ func doSDKContextBuildTests(t *ldtest.T) { }) } +// CONTEXT 1.12.4: SDK supports converting context to/from JSON +// CONTEXT 1.12.5: SDK supports deserializing legacy JSON format func doSDKContextConvertTests(t *ldtest.T) { dataSystem := NewSDKDataSystem(t, nil) client := NewSDKClient(t, dataSystem) @@ -123,6 +141,16 @@ func doSDKContextConvertTests(t *ldtest.T) { return fmt.Sprintf(`{"kind": "org", "key": "x"%s}`, extraProps) } + // CONTEXT 1.1.2: kind contains only allowed characters (ASCII alphanumerics, ., -, _) + // CONTEXT 1.2.1.1: key present on every single-kind context + // CONTEXT 1.2.2.1: name is a string if set + // CONTEXT 1.2.3.1: anonymous is a boolean if set + // CONTEXT 1.4.2: custom attribute value types + // CONTEXT 1.10.1.1: single-kind includes kind and key as top-level properties + // CONTEXT 1.10.1.2: _meta.privateAttributes format + // CONTEXT 1.10.1.5: non-built-in top-level properties are custom attributes + // CONTEXT 1.10.2.1: multi-kind includes kind:"multi" + // CONTEXT 1.10.2.2: multi-kind sub-objects without kind property t.Run("valid, no changes", func(t *ldtest.T) { singleKindProps := []string{ ``, @@ -166,6 +194,8 @@ func doSDKContextConvertTests(t *ldtest.T) { } }) + // CONTEXT 1.5.6: null attribute value treated as absent + // CONTEXT 1.5.7: JSON null property treated as omitting property t.Run("unnecessary properties are dropped", func(t *ldtest.T) { expected := json.RawMessage(basicInputPlusProps("")) @@ -186,6 +216,12 @@ func doSDKContextConvertTests(t *ldtest.T) { } }) + // CONTEXT 1.10.3.1: no kind property means kind "user" + // CONTEXT 1.10.3.2: legacy key and name treated same as regular context + // CONTEXT 1.10.3.3: privateAttributeNames treated as _meta.privateAttributes + // CONTEXT 1.10.3.4: legacy string attrs (firstName, lastName, email, country, ip, avatar) + // CONTEXT 1.10.3.5: custom property treated as additional custom attributes + // CONTEXT 1.12.5: SDK deserializes legacy JSON format t.Run("old user to context", func(t *ldtest.T) { t.RequireCapability(servicedef.CapabilityUserType) type contextConversionParams struct { @@ -235,6 +271,7 @@ func doSDKContextConvertTests(t *ldtest.T) { } }) + // CTXBLD 1.2.7: multi-context with single kind normalizes to individual context t.Run("multi-kind with only one kind becomes single-kind", func(t *ldtest.T) { singleKindJSON := `{"kind": "org", "key": "a", "name": "b"}` multiKindJSON := `{"kind": "multi", "org": {"key": "a", "name": "b"}}` @@ -243,6 +280,14 @@ func doSDKContextConvertTests(t *ldtest.T) { m.In(t).Assert(json.RawMessage(resp.Output), m.JSONEqual(json.RawMessage(singleKindJSON))) }) + // CONTEXT 1.1.1: kind must be non-empty string + // CONTEXT 1.1.2: kind must contain only ASCII alphanumerics, ., -, _ + // CONTEXT 1.1.3: kind must not be "kind" + // CONTEXT 1.13.1.1: invalid kind (empty, bad chars, or equals "kind") + // CONTEXT 1.13.1.3: multi-kind with no contexts is invalid + // CONTEXT 1.13.2.1: kind null or not a string is invalid + // CONTEXT 1.13.2.2: single-kind type/schema validation errors + // CONTEXT 1.13.4.1: deserialization must fail on invalid conditions t.Run("invalid context", func(t *ldtest.T) { inputs := []string{ ``, @@ -298,6 +343,7 @@ func doSDKContextConvertTests(t *ldtest.T) { }) }) + // CONTEXT 1.13.2.4: legacy format type/schema validation errors t.Run("invalid old user", func(t *ldtest.T) { t.RequireCapability(servicedef.CapabilityUserType) inputs := make([]string, 0, 12) @@ -320,6 +366,8 @@ func doSDKContextConvertTests(t *ldtest.T) { }) } +// CTXBLD 1.8.4: equal contexts have same kinds, keys, names, anonymous, custom attrs, private attrs +// CONTEXT 1.10.1.4: order of privateAttributes entries is not significant func doSDKContextComparisonTests(t *ldtest.T) { dataSystem := NewSDKDataSystem(t, nil) client := NewSDKClient(t, dataSystem) diff --git a/sdktests/server_side_big_segments.go b/sdktests/server_side_big_segments.go index 875e279a..71c8746d 100644 --- a/sdktests/server_side_big_segments.go +++ b/sdktests/server_side_big_segments.go @@ -37,6 +37,13 @@ func doServerSideBigSegmentsTests(t *ldtest.T) { t.Run("error handling", doBigSegmentsErrorHandlingTests) } +// BIGSEG 1.3.1: context keys hashed as base64(sha256(key)) before store query +// BIGSEG 1.9.2.1: included/excluded/includedContexts/excludedContexts fields ignored for big segments +// BIGSEG 1.9.2.2: store queried when unbounded=true and generation has a value +// BIGSEG 1.9.2.5: membership tested using ".g" reference format +// BIGSEG 1.9.2.6: true→match, false→non-match, absent→fall through to rules +// BIGSEG 1.9.1.3: no query if context kind doesn't match unboundedContextKind +// BIGSEG 1.9.4.1: bigSegmentsStatus added to evaluation reason func doBigSegmentsEvaluateSegment(t *ldtest.T) { otherContext := ldcontext.New("other-user-key") otherKind := ldcontext.Kind("other") @@ -179,6 +186,13 @@ func doBigSegmentsEvaluateSegment(t *ldtest.T) { }) } +// BIGSEG 1.9.5.1: store queried at most once per context key within a single evaluation +// BIGSEG 1.9.1.2: individual context extracted by unboundedContextKind +// BIGSEG 1.9.1.4: store query uses context key alone, not (kind, key) pair +// BIGSEG 1.5.2, 1.6.1: wrapper checks LRU cache before querying store +// BIGSEG 1.6.2: cache keyed by unhashed context key +// BIGSEG 1.6.3, 1.4.2: cache holds at most configured max entries (LRU eviction) +// BIGSEG 1.6.4, 1.4.3: cache entries expire after configured TTL func doBigSegmentsMembershipCachingTests(t *ldtest.T) { user1, user2, user3 := ldcontext.New("user1"), ldcontext.New("user2"), ldcontext.New("user3") otherKind := ldcontext.Kind("other") @@ -404,6 +418,10 @@ func doBigSegmentsMembershipCachingTests(t *ldtest.T) { }) } +// BIGSEG 1.8.4, 1.4.5: background task polls store metadata at configured interval +// BIGSEG 1.8.5: polling calls GetMetadata and updates status (available + stale) +// BIGSEG 1.8.6: listeners notified only when status actually changes +// BIGSEG 1.8.9: evaluations do not trigger additional metadata polls func doBigSegmentsStatusPollingTests(t *ldtest.T) { dataSystem := NewSDKDataSystem(t, mockld.EmptyServerSDKData()) @@ -519,6 +537,9 @@ func doBigSegmentsStatusPollingTests(t *ldtest.T) { } } +// BIGSEG 1.9.2.4, 1.5.4: no store configured → NOT_CONFIGURED status, segment is non-match +// BIGSEG 1.9.2.3: unbounded=true but no generation → NOT_CONFIGURED, segment is non-match +// BIGSEG 1.5.5, 1.9.3.1, 1.9.3.3: store error → STORE_ERROR status, segment is non-match, eval doesn't fail func doBigSegmentsErrorHandlingTests(t *ldtest.T) { t.Run("big segment store was not configured", func(t *ldtest.T) { segment := ldbuilders.NewSegmentBuilder("segment-key").Version(1). diff --git a/sdktests/server_side_eval.go b/sdktests/server_side_eval.go index 982bb693..bfd40708 100644 --- a/sdktests/server_side_eval.go +++ b/sdktests/server_side_eval.go @@ -21,6 +21,8 @@ func doServerSideEvalTests(t *ldtest.T) { t.Run("client not ready", runParameterizedServerSideClientNotReadyEvalTests) } +// FLGEDETAIL 1.1.6.1: server-side reason field is always present and non-null +// FLGEDETAIL 1.4.3.1: server-side *variationDetail accepts (flagKey, context, defaultValue) func runParameterizedServerSideEvalTests(t *ldtest.T) { parameterizedTests := CommonEvalParameterizedTestRunner[mockld.ServerSDKData]{ SDKConfigurers: func(testSuite testmodel.EvalTestSuite[mockld.ServerSDKData]) []SDKConfigurer { return nil }, @@ -30,6 +32,8 @@ func runParameterizedServerSideEvalTests(t *ldtest.T) { parameterizedTests.RunAll(t, "server-side-eval") } +// FLGEDETAIL 1.3.1: errorKind CLIENT_NOT_READY when SDK not initialized +// FLGEDETAIL 1.1.3: variationIndex absent when default value returned func runParameterizedServerSideClientNotReadyEvalTests(t *ldtest.T) { defaultValues := data.MakeValueFactoryBySDKValueType() flagKey := "some-flag" diff --git a/sdktests/server_side_eval_all_flags.go b/sdktests/server_side_eval_all_flags.go index 55148e2e..0ca182fe 100644 --- a/sdktests/server_side_eval_all_flags.go +++ b/sdktests/server_side_eval_all_flags.go @@ -40,6 +40,14 @@ func runServerSideEvalAllFlagsTests(t *ldtest.T) { }) } +// FLGMES 1.1.1: AllFlagsState returns evaluation results for all flags for a context +// FLGMES 1.3.1.5: Flags state has valid=true when evaluation succeeds +// FLGMES 1.3.2.5: Flag version set to the flag's version property +// FLGMES 1.3.2.6: Flag value set to the value returned by the evaluation algorithm +// FLGMES 1.3.2.7: Variation index set to the evaluation result variation index +// FLGMES 1.3.2.9: Track events true when flag's trackEvents property is true +// FLGMES 1.3.2.11: Debug events until date set from flag's debugEventsUntilDate property +// FLGMES 1.4.1: Omits null/false properties in bootstrapping JSON func doServerSideAllFlagsBasicTest(t *ldtest.T) { flag1 := ldbuilders.NewFlagBuilder("flag1").Version(100). Variations(dummyValue0, ldvalue.String("value1")). @@ -104,6 +112,8 @@ func doServerSideAllFlagsBasicTest(t *ldtest.T) { m.In(t).Assert(resultJSON, m.JSONStrEqual(expectedJSON)) } +// FLGEDETAIL 1.2.2: reason kind OFF and FALLTHROUGH returned in all-flags state +// FLGMES 1.3.2.8: Reason included when with-reasons option is true func doServerSideAllFlagsWithReasonsTest(t *ldtest.T) { t.RequireCapability(servicedef.CapabilityAllFlagsWithReasons) @@ -147,6 +157,10 @@ func doServerSideAllFlagsWithReasonsTest(t *ldtest.T) { m.In(t).Assert(resultJSON, m.JSONStrEqual(expectedJSON)) } +// FLGEDETAIL 1.2.3: inExperiment field present when kind is FALLTHROUGH or RULE_MATCH +// FLGMES 1.3.2.8: Reason included when evaluation involves an experiment +// FLGMES 1.3.2.9: Track events true when evaluation involves an experiment +// FLGMES 1.3.2.10: Track reason true when evaluation involves an experiment func doServerSideAllFlagsExperimentationTest(t *ldtest.T) { // flag1 has experiment behavior because it's a fallthrough and has trackEventsFallthrough=true flag1 := ldbuilders.NewFlagBuilder("flag1").Version(100). @@ -191,6 +205,10 @@ func doServerSideAllFlagsExperimentationTest(t *ldtest.T) { m.In(t).Assert(resultJSON, m.JSONStrEqual(expectedJSON)) } +// FLGEDETAIL 1.2.2: reason kind ERROR with errorKind in all-flags state +// FLGEDETAIL 1.3.1: errorKind MALFORMED_FLAG for invalid flag data +// FLGMES 1.6.1.1: Failed flag still included with null value and no variation +// FLGMES 1.3.2.8: Error reason included when with-reasons is true func doServerSideAllFlagsErrorInFlagTest(t *ldtest.T) { // This test verifies that 1. an error in evaluation of one flag does not prevent evaluation // of the rest of the flags, and 2. the failed flag is still included in the results, with a @@ -262,6 +280,7 @@ func doServerSideAllFlagsErrorInFlagTest(t *ldtest.T) { }) } +// FLGMES 1.3.2.1: Flags skipped when client-side only and usingEnvironmentId is false func doServerSideAllFlagsClientSideOnlyTest(t *ldtest.T) { t.RequireCapability(servicedef.CapabilityAllFlagsClientSideOnly) @@ -289,6 +308,9 @@ func doServerSideAllFlagsClientSideOnlyTest(t *ldtest.T) { assert.NotContains(t, result.State, flag2.Key) } +// FLGEDETAIL 1.1.3: variationIndex always present even when reason/version omitted +// FLGEDETAIL 1.2.2: reason kind OFF included only for tracked flags +// FLGMES 1.3.2.8: Reason and version omitted for untracked flags when details-only-for-tracked is true func doServerSideAllFlagsDetailsOnlyForTrackedFlagsTest(t *ldtest.T) { // Note that it's only "version" and "reason" that are omitted for untracked flags in this mode. // The variation index always must be included, because it's necessary for summary events. The @@ -353,6 +375,8 @@ func doServerSideAllFlagsDetailsOnlyForTrackedFlagsTest(t *ldtest.T) { m.In(t).Assert(resultJSON, m.JSONStrEqual(expectedJSON)) } +// FLGEDETAIL 1.3.1: errorKind CLIENT_NOT_READY in all-flags when SDK not initialized +// FLGMES 1.3.1.1: Uninitialized SDK returns empty flags state with valid=false func doServerSideAllFlagsClientNotReadyTest(t *ldtest.T) { dataSystem := NewSDKDataSystem(t, mockld.BlockingUnavailableSDKData(mockld.ServerSideSDK)) client := NewSDKClient(t, @@ -372,6 +396,7 @@ func doServerSideAllFlagsClientNotReadyTest(t *ldtest.T) { m.In(t).Assert(resultJSON, m.JSONStrEqual(expectedJSON)) } +// FLGMES 1.4.1: SDKs must omit null/false properties in bootstrapping JSON func doServerSideAllFlagsCompactRepresentationsTest(t *ldtest.T) { t.NonCritical(`If this failed but the other 'all flags' tests passed, the SDK is including null-valued` + ` properties within the $flagsState part of the representation. To save bandwidth, it's desirable` + @@ -406,6 +431,8 @@ func doServerSideAllFlagsCompactRepresentationsTest(t *ldtest.T) { m.In(t).Assert(resultJSON, m.JSONStrEqual(expectedMetadata)) } +// CSPE 1.1.1: $flagsState includes prerequisites field for flags with prerequisite evaluations +// CSPE 1.1.1.2: prerequisites is an ordered list of evaluated direct prerequisites func doServerSideAllFlagsIncludesToplevelPreqrequisitesTest(t *ldtest.T) { topLevel := ldbuilders.NewFlagBuilder("topLevel").Version(100). Variations(ldvalue.String("value1")).On(true).FallthroughVariation(0). @@ -459,6 +486,8 @@ func doServerSideAllFlagsIncludesToplevelPreqrequisitesTest(t *ldtest.T) { m.In(t).Assert(resultJSON, m.JSONStrEqual(expectedJSON)) } +// CSPE 1.1.1.1: prerequisites field omitted when flag is off or not evaluated +// CSPE 1.1.1.2: only evaluated prerequisites included; short-circuits on failure func doServerSideAllFlagsIgnoresPrereqsIfNotEvaluatedTest(t *ldtest.T) { flagOn := ldbuilders.NewFlagBuilder("flagOn").Version(100). Variations(ldvalue.String("value1")).On(true).FallthroughVariation(0). @@ -528,6 +557,7 @@ func doServerSideAllFlagsIgnoresPrereqsIfNotEvaluatedTest(t *ldtest.T) { m.In(t).Assert(resultJSON, m.JSONStrEqual(expectedJSON)) } +// CSPE 1.1.1: prerequisites listed regardless of client-side visibility (Note 1.1.1) func doServerSideAllFlagsIgnoresClientSideOnlyForPrereqKeys(t *ldtest.T) { flag := ldbuilders.NewFlagBuilder("flag").Version(100). ClientSideUsingEnvironmentID(true). diff --git a/sdktests/server_side_events_eval.go b/sdktests/server_side_events_eval.go index bc3ec7ae..cee52dbf 100644 --- a/sdktests/server_side_events_eval.go +++ b/sdktests/server_side_events_eval.go @@ -483,6 +483,9 @@ func doDebugEventTestCases( }) } +// CSPE 1.2.1: prerequisite events emitted equivalent to direct evaluation (server-side) +// CSPE 1.2.3: prerequisite events emitted before dependent flag's event +// CSPE 1.2.4: recursive prerequisites generate events (flag1→flag2→flag3) func doServerSideFeaturePrerequisiteEventTests(t *ldtest.T) { // The test logic for this is *almost* exactly the same for PHP as for other server-side SDKs // (the only difference is the absence of index and summary events), so we reuse the same diff --git a/sdktests/server_side_events_summary.go b/sdktests/server_side_events_summary.go index d54e48b1..01f54662 100644 --- a/sdktests/server_side_events_summary.go +++ b/sdktests/server_side_events_summary.go @@ -28,6 +28,8 @@ func doServerSideSummaryEventTests(t *ldtest.T) { t.Run("flag versions", doServerSideSummaryEventVersionTest) } +// CSSE 2.1.2.1: counters group evaluations by flag key, variation, and version +// CSSE 2.1.1: accumulator tracks default value, count, and resulting value per counter func doServerSideSummaryEventBasicTest(t *ldtest.T) { flag1 := ldbuilders.NewFlagBuilder("flag1").Version(100). Variations(ldvalue.String("value1a"), ldvalue.String("value1b")). @@ -99,6 +101,7 @@ func doServerSideSummarySamplingRatioTests(t *ldtest.T) { t.Run("prereq can be excluded", flagPreqIsExcludedFromSummaries) } +// CSSE 1.1.1.2.1: excludeFromSummaries=true causes flag to be omitted from summary func flagIsExcludedFromSummaries(t *ldtest.T) { flag1 := ldbuilders.NewFlagBuilder("flag1"). On(true). @@ -155,6 +158,7 @@ func flagIsExcludedFromSummaries(t *ldtest.T) { ) } +// CSSE 1.1.1.2.1: prerequisite with excludeFromSummaries=true omitted from summary func flagPreqIsExcludedFromSummaries(t *ldtest.T) { flag1 := ldbuilders.NewFlagBuilder("flag1"). On(true). @@ -199,6 +203,7 @@ func flagPreqIsExcludedFromSummaries(t *ldtest.T) { ) } +// CSSE 2.1.3.1: features include contextKinds from all evaluating context kinds func doServerSideSummaryEventContextKindsTest(t *ldtest.T) { flag1 := ldbuilders.NewFlagBuilder("flag1").Version(100). Variations(ldvalue.String("value1a"), ldvalue.String("value1b")). @@ -266,6 +271,7 @@ func doServerSideSummaryEventContextKindsTest(t *ldtest.T) { ) } +// CSSE 2.1.3.1: unknown flag counter sets unknown=true with no version or variation func doServerSideSummaryEventUnknownFlagTest(t *ldtest.T) { unknownKey := "flag-x" context := ldcontext.New("user-key") @@ -301,6 +307,8 @@ func doServerSideSummaryEventUnknownFlagTest(t *ldtest.T) { ) } +// CSSE 1.1.2.1: accumulators cleared after getSummaries; new flush has fresh counts +// CSSE 3.1.1: summaries included in flushed event batch func doServerSideSummaryEventResetTest(t *ldtest.T) { flag := ldbuilders.NewFlagBuilder("flag1").Version(100). Variations(ldvalue.String("value-a"), ldvalue.String("value-b")). @@ -370,6 +378,7 @@ func doServerSideSummaryEventResetTest(t *ldtest.T) { ))) } +// CSSE 2.1.2.1: prerequisite evaluations increment counters in summary event func doServerSideSummaryEventPrerequisitesTest(t *ldtest.T) { context := ldcontext.New("user-key") expectedValue1 := ldvalue.String("value1") @@ -436,6 +445,7 @@ func doServerSideSummaryEventPrerequisitesTest(t *ldtest.T) { ))) } +// CSSE 2.1.2.1: different flag versions tracked separately in counters func doServerSideSummaryEventVersionTest(t *ldtest.T) { // This test verifies that if the version of a flag changes within the timespan of one event payload, // evaluations for each version are tracked separately. We do this by evaluating the flag in its diff --git a/sdktests/server_side_migrations.go b/sdktests/server_side_migrations.go index b472ac76..a52c3ee6 100644 --- a/sdktests/server_side_migrations.go +++ b/sdktests/server_side_migrations.go @@ -61,6 +61,8 @@ func withExecutionOrders(test func(*ldtest.T, ldmigration.ExecutionOrder)) func( } } +// MIGRATIONS 1.1.1: MigrationVariation returns current stage for a migration flag key +// MIGRATIONS 1.1.1.1: returns one of off/dualwrite/shadow/live/rampdown/complete func identifyCorrectStageFromStringFlag(t *ldtest.T) { stages := []ldmigration.Stage{ldmigration.Off, ldmigration.DualWrite, ldmigration.Shadow, ldmigration.Live, ldmigration.RampDown, ldmigration.Complete} @@ -82,6 +84,8 @@ func identifyCorrectStageFromStringFlag(t *ldtest.T) { } } +// MIGRATIONS 1.1.1.2: invalid default stage falls back to off +// MIGRATIONS 1.1.1.3: invalid eval result returns default stage with WRONG_TYPE reason func usesDefaultWhenAppropriate(t *ldtest.T) { stages := []ldmigration.Stage{ldmigration.Off, ldmigration.DualWrite, ldmigration.Shadow, ldmigration.Live, ldmigration.RampDown, ldmigration.Complete} scenarios := []struct { @@ -114,6 +118,7 @@ func usesDefaultWhenAppropriate(t *ldtest.T) { } } +// MIGRATIONS 1.3.2.2: write operations execute origins in table-specified order, not parallel func executesOriginsInCorrectOrder(t *ldtest.T) { testParams := []struct { Operation ldmigration.Operation @@ -184,6 +189,7 @@ func executesOriginsInCorrectOrder(t *ldtest.T) { } } +// MIGRATIONS 1.3.1.3: read operations invoke old/new/both origins per stage table func executesReads(t *ldtest.T) { testParams := []struct { Operation ldmigration.Operation @@ -254,6 +260,7 @@ func executesReads(t *ldtest.T) { } } +// MIGRATIONS 1.3.1.5, 1.3.2.4: payload parameter forwarded to read and write methods func payloadsArePassedThrough(t *ldtest.T) { testParams := []struct { Operation ldmigration.Operation @@ -338,6 +345,8 @@ func payloadsArePassedThrough(t *ldtest.T) { } } +// MIGRATIONS 1.1.3.3: tracker records which origins were invoked +// MIGRATIONS 1.1.3.1: tracker event includes evaluation detail from MigrationVariation func tracksInvoked(t *ldtest.T, order ldmigration.ExecutionOrder) { onlyOld := []m.Matcher{m.JSONOptProperty("old").Should(m.Not(m.BeNil())), m.JSONOptProperty("new").Should(m.BeNil())} both := []m.Matcher{m.JSONOptProperty("old").Should(m.Not(m.BeNil())), m.JSONOptProperty("new").Should(m.Not(m.BeNil()))} @@ -430,6 +439,9 @@ func tracksInvoked(t *ldtest.T, order ldmigration.ExecutionOrder) { } } +// MIGRATIONS 1.1.3.5: tracker records latency measurements per origin +// MIGRATIONS 1.2.3: latency tracking is opt-in +// //nolint:dupl // Invokes and latency happen to share the same setup, but should be tested independently. func tracksLatency(t *ldtest.T, order ldmigration.ExecutionOrder) { onlyOld := []m.Matcher{m.JSONOptProperty("old").Should(m.Not(m.BeNil())), m.JSONOptProperty("new").Should(m.BeNil())} @@ -543,6 +555,9 @@ func tracksLatency(t *ldtest.T, order ldmigration.ExecutionOrder) { } } +// MIGRATIONS 1.1.3.4: tracker records error occurrences per origin +// MIGRATIONS 1.2.4: error tracking is opt-in +// MIGRATIONS 1.3.2.3: execution halts on first error func writeFailuresShouldGenerateErrorMetrics(t *ldtest.T, order ldmigration.ExecutionOrder) { hasError := func(label string) m.Matcher { return m.JSONOptProperty(label).Should(m.Equal(true)) } isMissingOrNoError := func(label string) m.Matcher { return JSONPropertyNullOrAbsentOrEqualTo(label, false) } @@ -639,6 +654,7 @@ func writeFailuresShouldGenerateErrorMetrics(t *ldtest.T, order ldmigration.Exec } } +// MIGRATIONS 1.2.4: no error measurement emitted when all handlers succeed func successfulHandlersShouldNotGenerateErrorMetrics(t *ldtest.T, order ldmigration.ExecutionOrder) { successfulHandler := func(w http.ResponseWriter, req *http.Request) { w.WriteHeader(http.StatusOK) } @@ -715,6 +731,7 @@ func successfulHandlersShouldNotGenerateErrorMetrics(t *ldtest.T, order ldmigrat } } +// MIGRATIONS 1.1.3.8: validation — missing flag generates error event with FLAG_NOT_FOUND func itHandlesMigrationEventsForMissingFlags(t *ldtest.T) { successfulHandler := func(w http.ResponseWriter, req *http.Request) { w.WriteHeader(http.StatusOK) } @@ -794,6 +811,7 @@ func itHandlesMigrationEventsForMissingFlags(t *ldtest.T) { } } +// MIGRATIONS 1.1.3.1: migration op event includes context (redacted when anonymous) func itRedactsAnonymousContextAttributes(t *ldtest.T) { t.RequireCapability(servicedef.CapabilityAnonymousRedaction) successfulHandler := func(w http.ResponseWriter, req *http.Request) { w.WriteHeader(http.StatusOK) } @@ -860,6 +878,7 @@ func itRedactsAnonymousContextAttributes(t *ldtest.T) { ) } +// MIGRATIONS 1.1.1.3: non-migration flag returns default stage with WRONG_TYPE reason func itHandlesNonMigrationFlags(t *ldtest.T) { successfulHandler := func(w http.ResponseWriter, req *http.Request) { w.WriteHeader(http.StatusOK) } @@ -938,12 +957,15 @@ func itHandlesNonMigrationFlags(t *ldtest.T) { } } +// MIGRATIONS 1.1.3.6: tracker records consistency check results +// MIGRATIONS 1.3.1.4: consistency tracked only for shadow/live stages when both reads succeed func trackConsistency(t *ldtest.T) { t.Run("checks for correct stage", withExecutionOrders(tracksConsistencyCorrectlyBasedOnStage)) t.Run("check ratio can disable", withExecutionOrders(tracksConsistencyIsDisabledByCheckRatio)) t.Run("unless callbacks fail", withExecutionOrders(tracksConsistencyIsDisabledIfCallbackFails)) } +// MIGRATIONS 1.5: samplingRatio of 0 suppresses op event emission func disableOpEventWithSamplingRatio(t *ldtest.T) { t.RequireCapability(servicedef.CapabilityEventSampling) @@ -1005,6 +1027,7 @@ func disableOpEventWithSamplingRatio(t *ldtest.T) { } } +// MIGRATIONS 1.3.1.4: consistency checked only for shadow/live read stages func tracksConsistencyCorrectlyBasedOnStage(t *ldtest.T, order ldmigration.ExecutionOrder) { handler := func(response string) func(w http.ResponseWriter, req *http.Request) { return func(w http.ResponseWriter, req *http.Request) { @@ -1117,6 +1140,7 @@ func tracksConsistencyCorrectlyBasedOnStage(t *ldtest.T, order ldmigration.Execu } } +// MIGRATIONS 1.1.3.6: checkRatio of 0 disables consistency check invocation func tracksConsistencyIsDisabledByCheckRatio(t *ldtest.T, order ldmigration.ExecutionOrder) { handler := func(w http.ResponseWriter, req *http.Request) { w.WriteHeader(http.StatusOK) } @@ -1193,6 +1217,7 @@ func tracksConsistencyIsDisabledByCheckRatio(t *ldtest.T, order ldmigration.Exec } } +// MIGRATIONS 1.3.1.4: consistency not tracked when read callbacks fail func tracksConsistencyIsDisabledIfCallbackFails(t *ldtest.T, order ldmigration.ExecutionOrder) { handler := func(w http.ResponseWriter, req *http.Request) { w.WriteHeader(http.StatusConflict) } diff --git a/sdktests/server_side_persistence_base.go b/sdktests/server_side_persistence_base.go index 27cfc56e..6027a43b 100644 --- a/sdktests/server_side_persistence_base.go +++ b/sdktests/server_side_persistence_base.go @@ -41,6 +41,7 @@ const ( persistenceInitedKey = "$inited" ) +// PS 1.3, 1.4: dispatches persistence tests across Redis, Consul, and DynamoDB backends func doServerSidePersistentTests(t *ldtest.T) { ranAtLeastOnce := false @@ -145,6 +146,14 @@ func newServerSidePersistentTests( } } +// PS 1.1.2: cache TTL modes (off, positive, infinite) +// PS 1.1.3: store initialized with Init() containing all flag data +// PS 1.1.4: Get retrieves items from persistence when not cached +// PS 1.1.6: Upsert writes items to persistent store with version checking +// PS 1.1.6.1: successful upsert updates item cache (non-infinite TTL) +// PS 1.1.6.5: successful upsert updates item and all-items cache (infinite TTL) +// PS 1.1.7: IsInitialized gates reads until store is initialized +// PS 1.2.1: cached items expire after TTL elapses func (s *ServerSidePersistentTests) Run(t *ldtest.T) { s.runWithEmptyStore(t, "uses default prefix", func(t *ldtest.T) { require.NoError(t, s.persistentStore.WriteMap(s.defaultPrefix, "features", s.initialFlags)) @@ -615,6 +624,7 @@ func (s *ServerSidePersistentTests) runWithEmptyStore(t *ldtest.T, testName stri }) } +// PS 1.1.7: validates the $inited key is set in the persistent store func (s *ServerSidePersistentTests) eventuallyRequireDataStoreInit(t *ldtest.T, prefix string) { h.RequireEventually(t, func() bool { value, _ := s.persistentStore.Get(prefix, persistenceInitedKey) @@ -622,6 +632,7 @@ func (s *ServerSidePersistentTests) eventuallyRequireDataStoreInit(t *ldtest.T, }, time.Second, time.Millisecond*20, persistenceInitedKey+" key was not set") } +// PS 1.1.3, 1.1.6: validates flag data written to persistent store matches expectations func (s *ServerSidePersistentTests) eventuallyValidateFlagData( t *ldtest.T, prefix string, matchers map[string]m.Matcher) { h.RequireEventually(t, func() bool { @@ -634,6 +645,7 @@ func (s *ServerSidePersistentTests) eventuallyValidateFlagData( }, time.Second, time.Millisecond*20, "flag data did not match") } +// PS 1.1.6: validates flag data is NOT written when version check rejects update func (s *ServerSidePersistentTests) neverValidateFlagData(t *ldtest.T, prefix string, matchers map[string]m.Matcher) { h.RequireNever(t, func() bool { data, err := s.persistentStore.GetMap(prefix, "features") @@ -645,6 +657,7 @@ func (s *ServerSidePersistentTests) neverValidateFlagData(t *ldtest.T, prefix st }, time.Second, time.Millisecond*20, "flag data did not match") } +// PS 1.1.3: builds matcher for flag key, version, and variations in store func basicFlagValidationMatcher(key string, version int, value string) m.Matcher { return m.AllOf( m.JSONProperty("key").Should(m.Equal(key)), @@ -653,6 +666,7 @@ func basicFlagValidationMatcher(key string, version int, value string) m.Matcher ) } +// PS 1.1.6: builds matcher for deleted flag tombstone in store func basicDeletedFlagValidationMatcher(key string, version int) m.Matcher { return m.AllOf( m.AnyOf(