diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..5bbafa70 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,58 @@ +# SDK Test Harness -- AI Agent Guide + +## Spec traceability + +Every contract test that exercises a formal SDK specification requirement **must** call +`t.Specification(specID, number, summary)` to record the link at runtime. + +### Rules + +1. **Placement.** `t.Specification()` calls go first in the test function body, before + `t.LongRunning()`, `t.RequireCapability()`, or any other `t.*` method. +2. **Multiple specs.** A single test may call `t.Specification()` more than once if it + exercises multiple requirements. +3. **YAML eval suites.** Use the `specifications` field at suite level instead of Go calls. + The parameterized test runner propagates these to `t.Specification()` automatically. +4. **Spec source.** Spec IDs (e.g. `HOOK`, `DATASYSTEM`, `FDV2PL`) and requirement numbers + come from the `sdk-specs` repo's `main` branch. +5. **Line length.** Keep the summary string short enough that the full `t.Specification()` + call stays under 120 characters. Break onto a second line if needed. +6. **Helper functions.** Functions that don't accept `*ldtest.T` cannot call + `t.Specification()`. Use a `// SPEC_ID X.Y.Z: summary` comment as a fallback. + +### Why + +Spec references are surfaced as `` elements in JUnit XML output, making +specification coverage visible in CI dashboards and test reports. This is the Go equivalent +of OpenFeature's `@Specification` annotation (Java) / `[Specification]` attribute (.NET). + +### Examples + +Go test: + +```go +func (c CommonStreamingTests) RecoverableFallback(t *ldtest.T) { + t.Specification("DATASYSTEM", "1.2.6", "recoverable error triggers fallback") + t.Specification("CSFDV2", "8.2.1", "recovery timeout default is 300 seconds") + t.LongRunning() + // ... test body ... +} +``` + +YAML eval suite: + +```yaml +name: attribute references +specifications: + - spec: "ATREF" + number: "1.2.1" + summary: "plain references treated as literal attribute name" + - spec: "ATREF" + number: "1.2.2" + summary: "~1 escape sequence interpreted as /" +``` + +## Linting + +Run `golangci-lint` after modifying Go files. The repo enforces a 120-character line limit +(`lll`) among other checks. See `.golangci.yml` for the full configuration. diff --git a/data/data-files/client-side-eval/basic-values.yml b/data/data-files/client-side-eval/basic-values.yml index 2f4bf3b6..62f4ce8a 100644 --- a/data/data-files/client-side-eval/basic-values.yml +++ b/data/data-files/client-side-eval/basic-values.yml @@ -1,5 +1,12 @@ --- name: basic values - +specifications: + - spec: "FLGEA" + number: "1.1.2" + summary: "client-side SDKs surface pre-evaluated outputs (value, variationIndex, reason)" + - spec: "FLGEA" + number: "1.6.3" + summary: "post-evaluation returns correct variation value or default" parameters: - TYPE_NAME: bool diff --git a/data/data-files/client-side-eval/errors-wrong-type.yml b/data/data-files/client-side-eval/errors-wrong-type.yml index 9c37a0ec..ed8c65b7 100644 --- a/data/data-files/client-side-eval/errors-wrong-type.yml +++ b/data/data-files/client-side-eval/errors-wrong-type.yml @@ -3,6 +3,19 @@ name: wrong type errors # These test are only relevant for SDKs that have variation methods for specific types requireCapability: strongly-typed +specifications: + - spec: "FLGEA" + number: "1.6.3" + summary: "post-evaluation type mismatch returns WRONG_TYPE error and default value" + - spec: "FLGEA" + number: "1.3" + summary: "evaluation reason ERROR with errorKind WRONG_TYPE" + - spec: "FLGEDETAIL" + number: "1.3.1" + summary: "errorKind WRONG_TYPE when flag value does not match requested type" + - spec: "FLGEDETAIL" + number: "1.1.2" + summary: "default value returned on type mismatch" # There is no value type expectation when evaluating all flags skipEvaluateAllFlags: true diff --git a/data/data-files/client-side-eval/errors.yml b/data/data-files/client-side-eval/errors.yml index 302e828b..95d4be6c 100644 --- a/data/data-files/client-side-eval/errors.yml +++ b/data/data-files/client-side-eval/errors.yml @@ -1,5 +1,18 @@ --- name: errors - +specifications: + - spec: "FLGEA" + number: "1.6.1" + summary: "flag not found returns ERROR with errorKind FLAG_NOT_FOUND" + - spec: "FLGEDETAIL" + number: "1.2.2" + summary: "reason kind ERROR when flag could not be evaluated" + - spec: "FLGEDETAIL" + number: "1.3.1" + summary: "errorKind FLAG_NOT_FOUND when flag key unknown" + - spec: "FLGEDETAIL" + number: "1.1.2" + summary: "default value returned when evaluation fails" skipEvaluateAllFlags: true diff --git a/data/data-files/client-side-eval/reasons.yml b/data/data-files/client-side-eval/reasons.yml index 64025b48..b1cb10f5 100644 --- a/data/data-files/client-side-eval/reasons.yml +++ b/data/data-files/client-side-eval/reasons.yml @@ -1,5 +1,18 @@ --- name: evaluation reasons - +specifications: + - spec: "FLGEA" + number: "1.3" + summary: "all evaluation reason kinds (OFF, FALLTHROUGH, TARGET_MATCH, RULE_MATCH, PREREQUISITE_FAILED, ERROR)" + - spec: "FLGEDETAIL" + number: "1.2.2" + summary: "reason kind values OFF, FALLTHROUGH, TARGET_MATCH, RULE_MATCH, PREREQUISITE_FAILED, ERROR" + - spec: "FLGEDETAIL" + number: "1.2.3" + summary: "ruleIndex, ruleId fields for RULE_MATCH; prerequisiteKey for PREREQUISITE_FAILED; inExperiment for experiments" + - spec: "FLGEDETAIL" + number: "1.3.1" + summary: "errorKind MALFORMED_FLAG and EXCEPTION" parameters: - NAME: off diff --git a/data/data-files/server-side-eval/attribute-references.yml b/data/data-files/server-side-eval/attribute-references.yml index d0388ad3..04b0e091 100644 --- a/data/data-files/server-side-eval/attribute-references.yml +++ b/data/data-files/server-side-eval/attribute-references.yml @@ -1,5 +1,15 @@ --- name: attribute references +specifications: + - spec: "ATREF" + number: "1.2.1" + summary: "plain references (no leading slash) treated as literal attribute name" + - spec: "ATREF" + number: "1.2.2" + summary: "~1 escape sequence interpreted as /" + - spec: "ATREF" + number: "1.2.3" + summary: "~0 escape sequence interpreted as ~" constants: IS_MATCH: diff --git a/data/data-files/server-side-eval/builtin-attrs.yml b/data/data-files/server-side-eval/builtin-attrs.yml index 3c72d667..2c23035e 100644 --- a/data/data-files/server-side-eval/builtin-attrs.yml +++ b/data/data-files/server-side-eval/builtin-attrs.yml @@ -1,5 +1,9 @@ --- name: built-in attributes other than kind +specifications: + - spec: "FLGEA" + number: "1.2" + summary: "context attribute lookup for built-in attributes" constants: IS_MATCH: diff --git a/data/data-files/server-side-eval/clause-kind-matching.yml b/data/data-files/server-side-eval/clause-kind-matching.yml index 1229ab2e..5148b7a9 100644 --- a/data/data-files/server-side-eval/clause-kind-matching.yml +++ b/data/data-files/server-side-eval/clause-kind-matching.yml @@ -1,5 +1,9 @@ --- name: clause kind matching +specifications: + - spec: "FLGEA" + number: "1.6.2.6" + summary: "clause evaluation for kind attribute" constants: IS_MATCH: diff --git a/data/data-files/server-side-eval/errors-bad-attribute-reference.yml b/data/data-files/server-side-eval/errors-bad-attribute-reference.yml index 2f70e438..c2dc8d31 100644 --- a/data/data-files/server-side-eval/errors-bad-attribute-reference.yml +++ b/data/data-files/server-side-eval/errors-bad-attribute-reference.yml @@ -1,5 +1,9 @@ --- name: bad attribute reference errors - +specifications: + - spec: "ATREF" + number: "1.1.1" + summary: "non-conforming strings are not valid attribute references" parameters: - NAME: clause with no attribute diff --git a/data/data-files/server-side-eval/errors-wrong-type.yml b/data/data-files/server-side-eval/errors-wrong-type.yml index bc6d453b..b02524d6 100644 --- a/data/data-files/server-side-eval/errors-wrong-type.yml +++ b/data/data-files/server-side-eval/errors-wrong-type.yml @@ -3,6 +3,16 @@ name: wrong type errors # These test are only relevant for SDKs that have variation methods for specific types requireCapability: strongly-typed +specifications: + - spec: "FLGEA" + number: "1.6.3.3" + summary: "post-evaluation WRONG_TYPE error" + - spec: "FLGEDETAIL" + number: "1.3.1" + summary: "errorKind WRONG_TYPE when flag value does not match requested type" + - spec: "FLGEDETAIL" + number: "1.1.2" + summary: "default value returned on type mismatch" # There is no value type expectation when evaluating all flags skipEvaluateAllFlags: true diff --git a/data/data-files/server-side-eval/errors.yml b/data/data-files/server-side-eval/errors.yml index f2c649d3..c8985d8e 100644 --- a/data/data-files/server-side-eval/errors.yml +++ b/data/data-files/server-side-eval/errors.yml @@ -1,5 +1,27 @@ --- name: evaluation failures () +specifications: + - spec: "FLGEA" + number: "1.6.1" + summary: "FLAG_NOT_FOUND error" + - spec: "FLGEA" + number: "1.6.2.3" + summary: "MALFORMED_FLAG for invalid variation index" + - spec: "FLGEA" + number: "1.6.2" + summary: "off with undefined offVariation returns default" + - spec: "FLGEDETAIL" + number: "1.2.2" + summary: "reason kind ERROR when flag could not be evaluated" + - spec: "FLGEDETAIL" + number: "1.3.1" + summary: "errorKind FLAG_NOT_FOUND and MALFORMED_FLAG" + - spec: "FLGEDETAIL" + number: "1.1.2" + summary: "default value returned when evaluation fails" + - spec: "FLGEDETAIL" + number: "1.1.3" + summary: "variationIndex absent when default value returned" constants: CONTEXT: { kind: "user", key: "user-key" } diff --git a/data/data-files/server-side-eval/json-variations.yml b/data/data-files/server-side-eval/json-variations.yml index 10307706..5e3a1da8 100644 --- a/data/data-files/server-side-eval/json-variations.yml +++ b/data/data-files/server-side-eval/json-variations.yml @@ -1,5 +1,9 @@ --- name: JSON variations +specifications: + - spec: "FLGEA" + number: "1.1.2" + summary: "evaluation outputs with JSON variation values" constants: CONTEXT: { kind: "user", key: "x"} diff --git a/data/data-files/server-side-eval/negation-and-iteration.yml b/data/data-files/server-side-eval/negation-and-iteration.yml index 39706cf7..fd87c1de 100644 --- a/data/data-files/server-side-eval/negation-and-iteration.yml +++ b/data/data-files/server-side-eval/negation-and-iteration.yml @@ -1,5 +1,9 @@ --- name: clause negation and value iteration +specifications: + - spec: "FLGEA" + number: "1.6.2.6" + summary: "clause negation and value iteration" constants: IS_MATCH: diff --git a/data/data-files/server-side-eval/operators-date-bad-syntax.yml b/data/data-files/server-side-eval/operators-date-bad-syntax.yml index b2f35fb2..221d00b0 100644 --- a/data/data-files/server-side-eval/operators-date-bad-syntax.yml +++ b/data/data-files/server-side-eval/operators-date-bad-syntax.yml @@ -1,5 +1,9 @@ --- name: operators - date - bad syntax +specifications: + - spec: "FLGEA" + number: "1.6.2.7.3" + summary: "date-time operators with invalid syntax" constants: IS_NOT_MATCH: diff --git a/data/data-files/server-side-eval/operators-date-bad-type.yml b/data/data-files/server-side-eval/operators-date-bad-type.yml index 6c3e3e65..3273017a 100644 --- a/data/data-files/server-side-eval/operators-date-bad-type.yml +++ b/data/data-files/server-side-eval/operators-date-bad-type.yml @@ -1,5 +1,9 @@ --- name: operators - date - bad type +specifications: + - spec: "FLGEA" + number: "1.6.2.7.3" + summary: "date-time operators with invalid type" constants: IS_NOT_MATCH: diff --git a/data/data-files/server-side-eval/operators-date-equal.yml b/data/data-files/server-side-eval/operators-date-equal.yml index d986a2cf..15376df2 100644 --- a/data/data-files/server-side-eval/operators-date-equal.yml +++ b/data/data-files/server-side-eval/operators-date-equal.yml @@ -1,5 +1,9 @@ --- name: operators - date - equal +specifications: + - spec: "FLGEA" + number: "1.6.2.7.3" + summary: "date-time operators with equal values" constants: IS_NOT_MATCH: diff --git a/data/data-files/server-side-eval/operators-date-unequal.yml b/data/data-files/server-side-eval/operators-date-unequal.yml index be6a275a..7a78b55d 100644 --- a/data/data-files/server-side-eval/operators-date-unequal.yml +++ b/data/data-files/server-side-eval/operators-date-unequal.yml @@ -1,5 +1,9 @@ --- name: operators - date - unequal +specifications: + - spec: "FLGEA" + number: "1.6.2.7.3" + summary: "date-time operators with unequal values (before, after)" constants: IS_MATCH: diff --git a/data/data-files/server-side-eval/operators-equality.yml b/data/data-files/server-side-eval/operators-equality.yml index f2194449..33141d6c 100644 --- a/data/data-files/server-side-eval/operators-equality.yml +++ b/data/data-files/server-side-eval/operators-equality.yml @@ -1,5 +1,9 @@ --- name: operators - equality () +specifications: + - spec: "FLGEA" + number: "1.6.2.7.1" + summary: "equality (in operator)" constants: IS_MATCH: diff --git a/data/data-files/server-side-eval/operators-numeric.yml b/data/data-files/server-side-eval/operators-numeric.yml index 83cd2e43..1d10a81c 100644 --- a/data/data-files/server-side-eval/operators-numeric.yml +++ b/data/data-files/server-side-eval/operators-numeric.yml @@ -1,5 +1,9 @@ --- name: operators - numeric +specifications: + - spec: "FLGEA" + number: "1.6.2.7" + summary: "numeric operators (lessThan, lessThanOrEqual, greaterThan, greaterThanOrEqual)" constants: IS_MATCH: diff --git a/data/data-files/server-side-eval/operators-semver-bad-syntax.yml b/data/data-files/server-side-eval/operators-semver-bad-syntax.yml index 31febb51..4f383a3a 100644 --- a/data/data-files/server-side-eval/operators-semver-bad-syntax.yml +++ b/data/data-files/server-side-eval/operators-semver-bad-syntax.yml @@ -1,5 +1,9 @@ --- name: operators - semver - bad syntax +specifications: + - spec: "FLGEA" + number: "1.6.2.7.4" + summary: "semantic version invalid syntax" constants: IS_NOT_MATCH: diff --git a/data/data-files/server-side-eval/operators-semver-bad-type.yml b/data/data-files/server-side-eval/operators-semver-bad-type.yml index ffb6c61c..80ba5007 100644 --- a/data/data-files/server-side-eval/operators-semver-bad-type.yml +++ b/data/data-files/server-side-eval/operators-semver-bad-type.yml @@ -1,5 +1,9 @@ --- name: operators - semver - bad type +specifications: + - spec: "FLGEA" + number: "1.6.2.7.4" + summary: "semantic version invalid type" constants: IS_NOT_MATCH: diff --git a/data/data-files/server-side-eval/operators-semver-equal.yml b/data/data-files/server-side-eval/operators-semver-equal.yml index 5f4e7c16..dad5bc2d 100644 --- a/data/data-files/server-side-eval/operators-semver-equal.yml +++ b/data/data-files/server-side-eval/operators-semver-equal.yml @@ -1,5 +1,9 @@ --- name: operators - semver - equal +specifications: + - spec: "FLGEA" + number: "1.6.2.7.4" + summary: "semantic version equality (semVerEqual)" constants: IS_MATCH: diff --git a/data/data-files/server-side-eval/operators-semver-unequal.yml b/data/data-files/server-side-eval/operators-semver-unequal.yml index a3dacbc2..d35bf1a3 100644 --- a/data/data-files/server-side-eval/operators-semver-unequal.yml +++ b/data/data-files/server-side-eval/operators-semver-unequal.yml @@ -1,5 +1,9 @@ --- name: operators - semver - unequal +specifications: + - spec: "FLGEA" + number: "1.6.2.7.4" + summary: "semantic version comparison (semVerLessThan, semVerGreaterThan)" constants: IS_MATCH: diff --git a/data/data-files/server-side-eval/operators-string.yml b/data/data-files/server-side-eval/operators-string.yml index 35838df2..3d5c8e79 100644 --- a/data/data-files/server-side-eval/operators-string.yml +++ b/data/data-files/server-side-eval/operators-string.yml @@ -1,5 +1,9 @@ --- name: operators - string +specifications: + - spec: "FLGEA" + number: "1.6.2.7" + summary: "string operators (startsWith, endsWith, contains, matches)" constants: IS_MATCH: diff --git a/data/data-files/server-side-eval/prerequisites.yml b/data/data-files/server-side-eval/prerequisites.yml index 7c747044..59c32388 100644 --- a/data/data-files/server-side-eval/prerequisites.yml +++ b/data/data-files/server-side-eval/prerequisites.yml @@ -1,5 +1,9 @@ --- name: prerequisites +specifications: + - spec: "FLGEA" + number: "1.6.2.4" + summary: "prerequisite evaluation" constants: CONTEXT: { key: "user-key" } diff --git a/data/data-files/server-side-eval/rollout-or-experiment.yml b/data/data-files/server-side-eval/rollout-or-experiment.yml index 8e2fac3e..602e3fda 100644 --- a/data/data-files/server-side-eval/rollout-or-experiment.yml +++ b/data/data-files/server-side-eval/rollout-or-experiment.yml @@ -1,5 +1,9 @@ --- name: rollout or experiment - +specifications: + - spec: "FLGEA" + number: "1.6.2.9" + summary: "rollouts and experiments" constants: MATCH_FALLTHROUGH_VARIATION_A: diff --git a/data/data-files/server-side-eval/rule-match.yml b/data/data-files/server-side-eval/rule-match.yml index 6cbf362c..06bdb59d 100644 --- a/data/data-files/server-side-eval/rule-match.yml +++ b/data/data-files/server-side-eval/rule-match.yml @@ -1,5 +1,9 @@ --- name: rule match () +specifications: + - spec: "FLGEA" + number: "1.6.2.6" + summary: "flag rule evaluation" constants: VALUE_OFF: "off" diff --git a/data/data-files/server-side-eval/segment-match.yml b/data/data-files/server-side-eval/segment-match.yml index f9f5fc6f..b739ba44 100644 --- a/data/data-files/server-side-eval/segment-match.yml +++ b/data/data-files/server-side-eval/segment-match.yml @@ -1,5 +1,9 @@ --- name: segment match + specifications: + - spec: "FLGEA" + number: "1.6.2.8" + summary: "segment matching" constants: IS_MATCH: diff --git a/data/data-files/server-side-eval/segment-recursion.yml b/data/data-files/server-side-eval/segment-recursion.yml index 8648da18..c0d91bc0 100644 --- a/data/data-files/server-side-eval/segment-recursion.yml +++ b/data/data-files/server-side-eval/segment-recursion.yml @@ -1,5 +1,9 @@ --- name: segment recursion +specifications: + - spec: "FLGEA" + number: "1.6.2.8" + summary: "segment matching with circular reference detection" constants: IS_MATCH: diff --git a/data/data-files/server-side-eval/target-match.yml b/data/data-files/server-side-eval/target-match.yml index b446a075..dc172184 100644 --- a/data/data-files/server-side-eval/target-match.yml +++ b/data/data-files/server-side-eval/target-match.yml @@ -1,5 +1,9 @@ --- name: target match +specifications: + - spec: "FLGEA" + number: "1.6.2.5" + summary: "individual targeting (targets and contextTargets)" constants: VALUE_OFF: "off" diff --git a/data/data-files/server-side-eval/variation-value-types.yml b/data/data-files/server-side-eval/variation-value-types.yml index f9e783d4..f05cb616 100644 --- a/data/data-files/server-side-eval/variation-value-types.yml +++ b/data/data-files/server-side-eval/variation-value-types.yml @@ -1,5 +1,9 @@ --- name: variation value types () + specifications: + - spec: "FLGEA" + number: "1.6" + summary: "evaluation algorithm across variation value types" parameters: - TYPE_NAME: bool diff --git a/data/testmodel/eval.go b/data/testmodel/eval.go index 5b7d230d..31ce42cf 100644 --- a/data/testmodel/eval.go +++ b/data/testmodel/eval.go @@ -11,10 +11,18 @@ import ( const DefaultForAllTypes servicedef.ValueType = "allDefaults" +// TestSpecReference links a test or test suite to a formal specification requirement. +type TestSpecReference struct { + Spec string `json:"spec"` + Number string `json:"number"` + Summary string `json:"summary"` +} + type EvalTestSuite[SDKDataType any] struct { Name string `json:"name"` RequireCapability string `json:"requireCapability"` SkipEvaluateAllFlags bool `json:"skipEvaluateAllFlags"` + Specifications []TestSpecReference `json:"specifications,omitempty"` SDKData SDKDataType `json:"sdkData"` Context o.Maybe[ldcontext.Context] `json:"context"` // used only for client-side tests Evaluations []EvalTest `json:"evaluations"` diff --git a/framework/ldtest/junit_test_logger.go b/framework/ldtest/junit_test_logger.go index f2356ea8..c4403786 100644 --- a/framework/ldtest/junit_test_logger.go +++ b/framework/ldtest/junit_test_logger.go @@ -24,12 +24,13 @@ type JUnitTestLogger struct { } type jUnitTestStatus struct { - failures []error - skipped o.Maybe[string] - nonCritical bool - output string - startTime time.Time - duration time.Duration + failures []error + skipped o.Maybe[string] + nonCritical bool + output string + startTime time.Time + duration time.Duration + specifications []SpecReference } // Struct definitions for the JUnit XML schema - see https://github.com/jstemmer/go-junit-report @@ -54,6 +55,7 @@ type jUnitXMLTestCase struct { Classname string `xml:"classname,attr"` Name string `xml:"name,attr"` Time string `xml:"time,attr"` + Properties []jUnitXMLProperty `xml:"properties>property,omitempty"` SkipMessage *jUnitXMLSkipMessage `xml:"skipped,omitempty"` Failure *jUnitXMLFailure `xml:"failure,omitempty"` } @@ -110,6 +112,7 @@ func (j *JUnitTestLogger) TestFinished(id TestID, result TestResult, debugOutput status.output = debugOutput.ToString("") status.duration = time.Since(status.startTime) status.nonCritical = result.NonCritical + status.specifications = result.Specifications j.tests[id.String()] = status } @@ -163,6 +166,12 @@ func (j *JUnitTestLogger) EndLog(results Results) error { Name: testID.String(), Time: jUnitDurationString(status.duration), } + for _, spec := range status.specifications { + testCase.Properties = append(testCase.Properties, jUnitXMLProperty{ + Name: "specification", + Value: spec.String(), + }) + } if status.nonCritical { testCase.Name += " (non-critical)" } diff --git a/framework/ldtest/result.go b/framework/ldtest/result.go index e7b44915..f309c34b 100644 --- a/framework/ldtest/result.go +++ b/framework/ldtest/result.go @@ -12,10 +12,11 @@ type Results struct { } type TestResult struct { - TestID TestID - Errors []error - NonCritical bool - Explanation string + TestID TestID + Errors []error + NonCritical bool + Explanation string + Specifications []SpecReference } func (r Results) OK() bool { diff --git a/framework/ldtest/spec.go b/framework/ldtest/spec.go new file mode 100644 index 00000000..6b260422 --- /dev/null +++ b/framework/ldtest/spec.go @@ -0,0 +1,18 @@ +package ldtest + +import "fmt" + +// SpecReference links a test to a formal specification requirement. +// It is the Go equivalent of the OpenFeature @Specification annotation (Java) +// or [Specification] attribute (.NET). +type SpecReference struct { + SpecID string // e.g. "HOOK", "DATASYSTEM" + Number string // e.g. "1.2.1" + Summary string // e.g. "beforeEvaluation stage receives hook context" +} + +// String returns a compact representation of the spec reference suitable for +// display in test output and JUnit XML properties. +func (s SpecReference) String() string { + return fmt.Sprintf("%s %s: %s", s.SpecID, s.Number, s.Summary) +} diff --git a/framework/ldtest/test_scope.go b/framework/ldtest/test_scope.go index cecef0a9..0d752247 100644 --- a/framework/ldtest/test_scope.go +++ b/framework/ldtest/test_scope.go @@ -16,16 +16,17 @@ type environment struct { // T represents a test scope. It is very similar to Go's testing.T type. type T struct { - env *environment - id TestID - debugLogger framework.CapturingLogger - nonCritical string - failed bool - skipped bool - skipReason string - cleanups []func() - errors []error - helperFns []string + env *environment + id TestID + debugLogger framework.CapturingLogger + nonCritical string + failed bool + skipped bool + skipReason string + cleanups []func() + errors []error + helperFns []string + specifications []SpecReference } // TestConfiguration contains options for the entire test run. @@ -84,6 +85,7 @@ func (t *T) run(action func(*T)) (result TestResult) { } } result.Errors = t.errors + result.Specifications = t.specifications if t.failed { if t.nonCritical == "" { t.env.results.Failures = append(t.env.results.Failures, result) @@ -237,6 +239,18 @@ func (t *T) RequireCapabilities(names ...string) { } } +// Specification records that this test exercises a formal specification requirement. +// It is the Go equivalent of the OpenFeature @Specification annotation (Java) or +// [Specification] attribute (.NET). A test may call Specification multiple times to +// link itself to multiple requirements. +func (t *T) Specification(specID, number, summary string) { + t.specifications = append(t.specifications, SpecReference{ + SpecID: specID, + Number: number, + Summary: summary, + }) +} + // Helper marks the function that calls it as a test helper that shouldn't appear in stacktraces. // Equivalent to Go's testing.T.Helper(). func (t *T) Helper() { diff --git a/sdktests/client_side_auto_env_attributes.go b/sdktests/client_side_auto_env_attributes.go index 3bc9f2dd..a66e484c 100644 --- a/sdktests/client_side_auto_env_attributes.go +++ b/sdktests/client_side_auto_env_attributes.go @@ -34,8 +34,12 @@ func doClientSideAutoEnvAttributesRequestingTests(t *ldtest.T) { t.Run("collisions", doClientSideAutoEnvAttributesRequestingCollisionsTests) } -// Start tests for events func doClientSideAutoEnvAttributesEventsNoCollisionsTests(t *ldtest.T) { + t.Specification("AUTOENVATTR", "1.2.1.1", + "SDK must require explicit opt-in/opt-out decision for environment attributes") + t.Specification("AUTOENVATTR", "1.2.1.2", "SDK must not add environment attributes without explicit opt-in") + t.Specification("AUTOENVATTR", "1.2.2.4", "SDK must include envAttributesVersion in auto-env contexts") + t.Specification("AUTOENVATTR", "1.2.2.6", "All auto-env contexts must be added to the evaluation context") base := newCommonTestsBase(t, "doClientSideAutoEnvAttributesEventsNoCollisionsTests") dataSystem := NewSDKDataSystem(t, nil) contextFactories := data.NewContextFactoriesForSingleAndMultiKind(base.contextFactory.Prefix()) @@ -113,6 +117,9 @@ func doClientSideAutoEnvAttributesEventsNoCollisionsTests(t *ldtest.T) { } func doClientSideAutoEnvAttributesEventsCollisionsTests(t *ldtest.T) { + t.Specification("AUTOENVATTR", "1.2.2.5", + "Auto-env context kinds not added if user-specified context of that kind exists") + t.Specification("AUTOENVATTR", "1.2.4.1", "SDK must not alter any portion of a customer-provided context kind") base := newCommonTestsBase(t, "doClientSideAutoEnvAttributesEventsCollisionsTests") dataSystem := NewSDKDataSystem(t, nil) contextFactories := data.NewContextFactoriesForSingleAndMultiKind(base.contextFactory.Prefix()) @@ -175,8 +182,9 @@ func doClientSideAutoEnvAttributesEventsCollisionsTests(t *ldtest.T) { // end tests for events -// start tests for streaming/polling func doClientSideAutoEnvAttributesRequestingNoCollisionsTests(t *ldtest.T) { + t.Specification("AUTOENVATTR", "1.2.2.4", "SDK must include envAttributesVersion in auto-env contexts") + t.Specification("AUTOENVATTR", "1.2.2.6", "All auto-env contexts must be added to the evaluation context") base := newCommonTestsBase(t, "doClientSideAutoEnvAttributesPollNoCollisionsTests") dsos := []SDKDataSystemOption{DataSystemOptionPolling(), DataSystemOptionStreaming()} for _, dso := range dsos { @@ -207,6 +215,9 @@ func doClientSideAutoEnvAttributesRequestingNoCollisionsTests(t *ldtest.T) { } func doClientSideAutoEnvAttributesRequestingCollisionsTests(t *ldtest.T) { + t.Specification("AUTOENVATTR", "1.2.2.5", + "Auto-env context kinds not added if user-specified context of that kind exists") + t.Specification("AUTOENVATTR", "1.2.4.1", "SDK must not alter any portion of a customer-provided context kind") base := newCommonTestsBase(t, "doClientSideAutoEnvAttributesPollCollisionsTests") dsos := []SDKDataSystemOption{DataSystemOptionPolling(), DataSystemOptionStreaming()} for _, dso := range dsos { @@ -245,8 +256,9 @@ func doClientSideAutoEnvAttributesRequestingCollisionsTests(t *ldtest.T) { // end tests for streaming/polling -// start tests for headers func doClientSideAutoEnvAttributesHeaderTests(t *ldtest.T) { + t.Specification("AUTOENVATTR", "1.2.4.2", + "ld_application id/version/versionName sourced by priority (config > platform > SDK)") t.RequireCapability(servicedef.CapabilityTags) base := newCommonTestsBase(t, "doClientSideAutoEnvAttributesEventsCollisionsTests") diff --git a/sdktests/client_side_eval.go b/sdktests/client_side_eval.go index 9e86a009..48bf4246 100644 --- a/sdktests/client_side_eval.go +++ b/sdktests/client_side_eval.go @@ -24,6 +24,9 @@ func doClientSideEvalTests(t *ldtest.T) { } func runParameterizedClientSideEvalTests(t *ldtest.T) { + t.Specification("FLGEDETAIL", "1.1.7.1", "client-side reason field may be absent when not requested") + t.Specification("FLGEDETAIL", "1.4.4.1", "client-side *variationDetail accepts (flagKey, defaultValue)") + t.Specification("FLGEDETAIL", "1.4.4.2", "client-side *variationDetail returns synchronously") // For client-side SDKs, you have to tell the SDK at initialization time whether we'll be using evaluation // reasons or not. The main effect of that is the client will add a withReasons query string parameter to // its requests-- which we're not actually checking for here; we're just configuring the polling service diff --git a/sdktests/client_side_events_eval.go b/sdktests/client_side_events_eval.go index 33c85602..6033e067 100644 --- a/sdktests/client_side_events_eval.go +++ b/sdktests/client_side_events_eval.go @@ -255,6 +255,10 @@ func doClientSideFeatureEventTests(t *ldtest.T) { } func doClientSideInOrderPrereqEventTests(t *ldtest.T) { + t.Specification("CSPE", "1.2.1", "prerequisite events emitted equivalent to direct evaluation") + t.Specification("CSPE", "1.2.3", "prerequisites processed in order before the dependent flag's event") + t.Specification("CSPE", "1.2.4", "recursive prerequisites are handled (prereq1 depends on prereq2)") + dataBuilder := mockld.NewClientSDKDataBuilder() dataBuilder. Flag("topLevel", mockld.ClientSDKFlag{ diff --git a/sdktests/client_side_events_summary.go b/sdktests/client_side_events_summary.go index 1918543f..eb5f84b0 100644 --- a/sdktests/client_side_events_summary.go +++ b/sdktests/client_side_events_summary.go @@ -35,6 +35,9 @@ func doClientSideSummaryEventTests(t *ldtest.T) { } func doClientSideSummaryEventBasicTest(t *ldtest.T) { + t.Specification("CSSE", "2.1.2.1", "counters group evaluations by flag key, variation, and version") + t.Specification("CSSE", "2.1.1", "accumulator tracks default value, count, and resulting value per counter") + flag1Key := "flag1" flag1Result1 := mockld.ClientSDKFlag{ Value: ldvalue.String("value1-a"), @@ -113,6 +116,9 @@ func doClientSideSummaryEventBasicTest(t *ldtest.T) { } func doClientSidePerContextSummaryEventBasicTest(t *ldtest.T) { + t.Specification("CSSE", "1.1.1.2", "unique accumulator per unique context produces separate summary events") + t.Specification("CSSE", "2.1.3.1", "getSummary populates context on per-context summary events") + flag1Key := "flag1" flag1Result1 := mockld.ClientSDKFlag{ Value: ldvalue.String("value1-a"), @@ -199,6 +205,8 @@ func doClientSidePerContextSummaryEventBasicTest(t *ldtest.T) { } func doClientSideSummaryEventContextKindsTest(t *ldtest.T) { + t.Specification("CSSE", "2.1.3.1", "features include contextKinds from all evaluating context kinds") + flag1Key := "flag1" flag1Result := mockld.ClientSDKFlag{ Value: ldvalue.String("value1-a"), @@ -271,6 +279,9 @@ func doClientSideSummaryEventContextKindsTest(t *ldtest.T) { } func doClientSidePerContxtSummaryEventContextKindsTest(t *ldtest.T) { + t.Specification("CSSE", "1.1.1.2", "unique accumulator per unique multi-kind context") + t.Specification("CSSE", "2.1.3.1", "contextKinds populated for per-context summary events") + flag1Key := "flag1" flag1Result := mockld.ClientSDKFlag{ Value: ldvalue.String("value1-a"), @@ -346,6 +357,8 @@ func doClientSidePerContxtSummaryEventContextKindsTest(t *ldtest.T) { } func doClientSideSummaryEventUnknownFlagTest(t *ldtest.T) { + t.Specification("CSSE", "2.1.3.1", "unknown flag counter sets unknown=true with no version or variation") + unknownKey := "flag-x" context := ldcontext.New("user-key") default1 := ldvalue.String("default1") @@ -381,6 +394,9 @@ func doClientSideSummaryEventUnknownFlagTest(t *ldtest.T) { } func doClientSideSummaryEventResetTest(t *ldtest.T) { + t.Specification("CSSE", "1.1.2.1", "accumulators cleared after getSummaries; new flush has fresh counts") + t.Specification("CSSE", "3.1.1", "summaries included in flushed event batch") + flagKey := "flag1" flag1Result1 := mockld.ClientSDKFlag{ Value: ldvalue.String("value-a"), @@ -455,6 +471,10 @@ func doClientSideSummaryEventResetTest(t *ldtest.T) { } func doClientSideSummaryBasicPrereqTest(t *ldtest.T) { + t.Specification("CSPE", "1.2.2", "summary counters updated equivalent to direct evaluation of prerequisites") + t.Specification("CSPE", "1.2.4", "recursive prerequisites generate correct summary counts") + t.Specification("CSSE", "2.1.2.1", "prerequisite evaluations increment counters in summary event") + topLevelKey := "flag1" topLevelResult := mockld.ClientSDKFlag{ Value: ldvalue.String("value1-a"), @@ -552,6 +572,9 @@ func doClientSideSummaryBasicPrereqTest(t *ldtest.T) { } func doClientSideSummaryPrereqUnknownFlagTest(t *ldtest.T) { + t.Specification("CSPE", "1.2.1", "unknown prerequisites generate unknown flag summary events (Note 1.2.1)") + t.Specification("CSSE", "2.1.3.1", "unknown prerequisite flag sets unknown=true in counter") + topLevelKey := "flag1" topLevelResult := mockld.ClientSDKFlag{ Value: ldvalue.String("value1-a"), diff --git a/sdktests/common_tests_eval.go b/sdktests/common_tests_eval.go index bbe8d716..f7931d34 100644 --- a/sdktests/common_tests_eval.go +++ b/sdktests/common_tests_eval.go @@ -62,6 +62,9 @@ func (c CommonEvalParameterizedTestRunner[SDKDataType]) runTestSuite( t *ldtest.T, suite testmodel.EvalTestSuite[SDKDataType], ) { + for _, spec := range suite.Specifications { + t.Specification(spec.Spec, spec.Number, spec.Summary) + } if suite.RequireCapability != "" { t.RequireCapability(suite.RequireCapability) } @@ -106,6 +109,12 @@ func (c CommonEvalParameterizedTestRunner[SDKDataType]) runTestEval( sdkData SDKDataType, client *SDKClient, ) { + t.Specification("FLGEDETAIL", "1.1.1", "EvaluationDetail returned by *variationDetail methods") + t.Specification("FLGEDETAIL", "1.1.2", "EvaluationDetail carries value field") + t.Specification("FLGEDETAIL", "1.1.3", "EvaluationDetail carries variationIndex field") + t.Specification("FLGEDETAIL", "1.1.5", "EvaluationDetail carries reason field") + t.Specification("FLGEDETAIL", "1.2.1", "EvaluationReason type describing evaluation result") + t.Specification("FLGEDETAIL", "1.4.1", "typed *variationDetail methods for bool, string, int, double, JSON") name := test.Name if name == "" { name = test.FlagKey diff --git a/sdktests/common_tests_events_contexts.go b/sdktests/common_tests_events_contexts.go index 5d94cb7a..070681ad 100644 --- a/sdktests/common_tests_events_contexts.go +++ b/sdktests/common_tests_events_contexts.go @@ -136,7 +136,16 @@ func makeEventContextTestParams() []eventContextTestParams { } return ret } + func (c CommonEventTests) EventContexts(t *ldtest.T) { + t.Specification("CONTEXT", "1.4.2", "custom attribute value types in event context output") + t.Specification("CONTEXT", "1.7.1", "private attributes do not affect flag evaluations") + t.Specification("CONTEXT", "1.7.2", "kind, key, anonymous not redacted even with allAttributesPrivate") + t.Specification("CONTEXT", "1.10.1.2", "_meta.privateAttributes redaction in event output") + t.Specification("CONTEXT", "1.10.1.5", "non-built-in properties treated as custom attributes in events") + t.Specification("CONTEXT", "1.10.2.1", "multi-kind contexts in event output") + t.Specification("CONTEXT", "1.10.2.2", "multi-kind sub-objects in event context output") + // Flags to use for "feature" and "debug" event tests // The flag variation/value is irrelevant. var flagKey, debuggedFlagKey string diff --git a/sdktests/common_tests_hooks.go b/sdktests/common_tests_hooks.go index 4aa5f433..85b708e0 100644 --- a/sdktests/common_tests_hooks.go +++ b/sdktests/common_tests_hooks.go @@ -158,6 +158,9 @@ func trackTestParams(context o.Maybe[ldcontext.Context]) []TrackParameters { } func executesBeforeEvaluationStageDetail(t *ldtest.T, detail bool) { + t.Specification("HOOK", "1.2.1", + "beforeEvaluation stage accepts EvaluationSeriesContext and returns EvaluationSeriesData") + t.Specification("HOOK", "1.2.1.1", "beforeEvaluation executes before the flag value has been determined") testParams := variationTestParams(detail) hookName := "executesBeforeEvaluationStage" @@ -198,6 +201,7 @@ func executesBeforeEvaluationStageDetail(t *ldtest.T, detail bool) { } func executesBeforeEvaluationStageMigration(t *ldtest.T) { + t.Specification("HOOK", "1.2.1", "beforeEvaluation stage fires for migration variations") hookName := "executesBeforeEvaluationStageMigration" client, hooks := createClientForHooks(t, []string{hookName}, nil) defer hooks.Close() @@ -223,6 +227,9 @@ func executesBeforeEvaluationStageMigration(t *ldtest.T) { } func executesAfterEvaluationStageDetail(t *ldtest.T, detail bool) { + t.Specification("HOOK", "1.2.2", + "afterEvaluation accepts EvaluationSeriesContext, EvaluationSeriesData, and EvaluationDetail") + t.Specification("HOOK", "1.2.2.1", "afterEvaluation executes after the flag detail has been determined") testParams := variationTestParams(detail) hookName := "executesAfterEvaluationStage" @@ -270,6 +277,7 @@ func executesAfterEvaluationStageDetail(t *ldtest.T, detail bool) { } func executesAfterEvaluationStageMigration(t *ldtest.T) { + t.Specification("HOOK", "1.2.2", "afterEvaluation stage fires for migration variations with correct EvaluationDetail") hookName := "executesBeforeEvaluationStageMigration" client, hooks := createClientForHooks(t, []string{hookName}, nil) defer hooks.Close() @@ -297,6 +305,8 @@ func executesAfterEvaluationStageMigration(t *ldtest.T) { } func beforeEvaluationDataPropagatesToAfterDetail(t *ldtest.T, detail bool) { + t.Specification("HOOK", "1.2.2.2", "afterEvaluation receives the EvaluationSeriesData returned by beforeEvaluation") + t.Specification("HOOK", "1.3.6", "series data propagates between stages on a per-hook basis") testParams := variationTestParams(detail) hookName := "beforeEvaluationDataPropagatesToAfterDetail" @@ -340,6 +350,8 @@ func beforeEvaluationDataPropagatesToAfterDetail(t *ldtest.T, detail bool) { } func beforeEvaluationDataPropagatesToAfterMigration(t *ldtest.T) { + t.Specification("HOOK", "1.2.2.2", "series data propagation also works for migration variations") + t.Specification("HOOK", "1.3.6", "series data propagation also works for migration variations") hookName := "beforeEvaluationDataPropagatesToAfterDetail" hookData := make(map[servicedef.HookStage]servicedef.SDKConfigEvaluationHookData) hookData[servicedef.BeforeEvaluation] = make(servicedef.SDKConfigEvaluationHookData) @@ -367,10 +379,9 @@ func beforeEvaluationDataPropagatesToAfterMigration(t *ldtest.T) { }) } -// This test is meant to check Requirement HOOKS:1.3.7: -// The client MUST handle exceptions which are thrown (or errors returned, if idiomatic for the language) -// during the execution of a stage or handler allowing operations to complete unaffected. func errorInBeforeStageDoesNotAffectAfterStage(t *ldtest.T) { + t.Specification("HOOK", "1.3.7", "client handles stage exceptions without affecting operations") + t.Specification("HOOK", "1.3.7.1", "when error prevents stage from returning data, use empty data") const numHooks = 3 // We're configuring the beforeEvaluation stage with some data, but we don't expect @@ -421,6 +432,8 @@ func errorInBeforeStageDoesNotAffectAfterStage(t *ldtest.T) { } func executesAfterTrackStage(t *ldtest.T) { + t.Specification("HOOK", "1.6.1", "afterTrack stage accepts TrackSeriesContext (key, context, metricValue, data)") + t.Specification("HOOK", "1.6.2", "afterTrack executes after the custom event has been enqueued") hookName := "executesAfterTrackStage" context := ldcontext.New("user-key") flagContext := o.Some(context) @@ -456,6 +469,7 @@ func executesAfterTrackStage(t *ldtest.T) { } func errorInHookPreventsAfterTrackStage(t *ldtest.T) { + t.Specification("HOOK", "1.3.7", "client handles hook errors without crashing; afterTrack not called on error") hookName := "doesNotExecuteAfterTrackStage" context := ldcontext.New("user-key") flagContext := o.Some(context) @@ -516,9 +530,8 @@ func observedHookOrder(t *ldtest.T, hooks *Hooks, names []string, stage serviced return out } -// afterTrack must execute in the order of hook registration (forward), -// unlike afterEvaluation/afterIdentify which run in reverse-registration order. func executesAfterTrackHooksInRegistrationOrder(t *ldtest.T) { + t.Specification("HOOK", "1.6.3", "afterTrack executes hooks in registration order (not reversed)") names := hookOrderTestNames("afterTrackOrderHook", 3) context := ldcontext.New("user-key") @@ -542,8 +555,8 @@ func executesAfterTrackHooksInRegistrationOrder(t *ldtest.T) { "afterTrack hooks must execute in the order of hook registration") } -// beforeEvaluation must execute in the order of hook registration. func executesBeforeEvaluationHooksInRegistrationOrder(t *ldtest.T) { + t.Specification("HOOK", "1.3.2", "beforeEvaluation executes hooks in registration order") names := hookOrderTestNames("beforeEvalOrderHook", 3) context := ldcontext.New("user-key") @@ -569,8 +582,8 @@ func executesBeforeEvaluationHooksInRegistrationOrder(t *ldtest.T) { "beforeEvaluation hooks must execute in the order of hook registration") } -// afterEvaluation must execute in the reverse of the order of hook registration. func executesAfterEvaluationHooksInReverseRegistrationOrder(t *ldtest.T) { + t.Specification("HOOK", "1.3.3", "afterEvaluation executes hooks in reverse registration order") names := hookOrderTestNames("afterEvalOrderHook", 3) context := ldcontext.New("user-key") diff --git a/sdktests/common_tests_poll_request.go b/sdktests/common_tests_poll_request.go index f7e8a6cb..8fca63d2 100644 --- a/sdktests/common_tests_poll_request.go +++ b/sdktests/common_tests_poll_request.go @@ -107,6 +107,10 @@ func (c CommonPollingTests) LargePayloads(t *ldtest.T) { } func (c CommonPollingTests) RequestURLPath(t *ldtest.T, pathMatcher func(flagRequestMethod) m.Matcher) { + t.Specification("ENVFILTER", "1.2.1", "request URL joins base URI with filter query parameter") + t.Specification("ENVFILTER", "1.2.2", "exactly one filter parameter is included") + t.Specification("ENVFILTER", "1.2.3", "filter parameter value is URL encoded") + t.Specification("ENVFILTER", "1.3.1", "filter parameter omitted for invalid filter key") t.Run("URL path is computed correctly", func(t *ldtest.T) { for _, filter := range c.environmentFilters() { t.Run(h.IfElse(filter.IsDefined(), filter.String(), "no environment filter"), func(t *ldtest.T) { diff --git a/sdktests/common_tests_stream_fdv2.go b/sdktests/common_tests_stream_fdv2.go index 7a2ff228..04718a16 100644 --- a/sdktests/common_tests_stream_fdv2.go +++ b/sdktests/common_tests_stream_fdv2.go @@ -86,6 +86,7 @@ func (c CommonStreamingTests) StateTransitions(t *ldtest.T) { } func (c CommonStreamingTests) InitializeFromEmptyState(t *ldtest.T) { + t.Specification("DATASYSTEM", "1.1.1", "SDK initializes from a single synchronizer with a full basis") dataSystem, configurers := c.setupDataSystems(t, c.makeSDKDataWithFlag(1, initialValue)) client := NewSDKClient(t, c.baseSDKConfigurationPlus(configurers...)...) @@ -94,6 +95,7 @@ func (c CommonStreamingTests) InitializeFromEmptyState(t *ldtest.T) { } func (c CommonStreamingTests) InitializeFromPollingInitializer(t *ldtest.T) { + t.Specification("DATASYSTEM", "1.1.2", "initializers run in order; synchronizer receives basis state") dataBefore := mockld.NewServerSDKDataBuilder().Flag(c.makeServerSideFlag("flag-key", 1, initialValue)).Build() dataAfter := mockld.NewServerSDKDataBuilder().IntentCode("none").IntentReason("up-to-date").Build() dataSystem := NewSDKDataSystem(t, dataAfter, DataSystemOptionPollingInitializer(dataBefore)) @@ -108,6 +110,7 @@ func (c CommonStreamingTests) InitializeFromPollingInitializer(t *ldtest.T) { } func (c CommonStreamingTests) InitializeFromPollingInitializerWithStreamingUpdates(t *ldtest.T) { + t.Specification("DATASYSTEM", "1.1.5", "initializer basis + synchronizer xfer-changes merged at startup") dataBefore := mockld.NewServerSDKDataBuilder(). Flag(c.makeServerSideFlag("flag-key", 1, initialValue)). Build() @@ -129,6 +132,9 @@ func (c CommonStreamingTests) InitializeFromPollingInitializerWithStreamingUpdat } func (c CommonStreamingTests) InitializeFromTwoPollingInitializers(t *ldtest.T) { + t.Specification("DATASYSTEM", "1.1.2", "initializers called in configuration order") + t.Specification("DATASYSTEM", "1.1.3", + "initialization continues when first initializer fails") emptyPayload := mockld.NewServerSDKDataBuilder(). Build() initialStatefulData := mockld.NewServerSDKDataBuilder(). @@ -159,6 +165,7 @@ func (c CommonStreamingTests) InitializeFromTwoPollingInitializers(t *ldtest.T) } func (c CommonStreamingTests) RecoverableFallbackToSecondarySynchronizer(t *ldtest.T) { + t.Specification("DATASYSTEM", "1.2.6", "recoverable error triggers fallback to next synchronizer") t.LongRunning() // First synchronizer hangs (never responds) to trigger initialization timeout fallback. @@ -198,6 +205,8 @@ func (c CommonStreamingTests) RecoverableFallbackToSecondarySynchronizer(t *ldte } func (c CommonStreamingTests) PermanentFallbackToSecondarySynchronizer(t *ldtest.T) { + t.Specification("DATASYSTEM", "1.2.6", + "non-recoverable 4xx error permanently removes synchronizer, falls back immediately") // First synchronizer returns 401 Unauthorized, which is a non-recoverable error. // Non-recoverable 4xx errors (all except 400, 408, 429) cause the synchronizer to be // permanently removed from the list, and the SDK immediately falls back to the secondary. @@ -227,6 +236,8 @@ func (c CommonStreamingTests) PermanentFallbackToSecondarySynchronizer(t *ldtest } func (c CommonStreamingTests) RecoverableFallbackWithRecovery(t *ldtest.T) { + t.Specification("DATASYSTEM", "1.2.7", "after recovery period, SDK attempts reconnection to earlier synchronizers") + t.Specification("CSFDV2", "8.2.1", "client-side recovery timeout default is 300 seconds (VALID condition)") t.LongRunning() // This test verifies that after a recoverable fallback, the SDK will attempt to @@ -310,6 +321,9 @@ func (c CommonStreamingTests) RecoverableFallbackWithRecovery(t *ldtest.T) { } func (c CommonStreamingTests) PermanentFallbackWithRecovery(t *ldtest.T) { + t.Specification("DATASYSTEM", "1.2.7", + "permanently removed synchronizers excluded from recovery; recoverable ones revisited") + t.Specification("CSFDV2", "8.2.1", "client-side recovery timeout default is 300 seconds (VALID condition)") t.LongRunning() // This test verifies that after a permanent removal (non-recoverable error), the SDK @@ -429,6 +443,10 @@ func (c CommonStreamingTests) FDv1FallbackDirective(t *ldtest.T) { // test asserts evaluations return that value — proving FDv1 actually became the // active data source, not just that its URL was hit. func (c CommonStreamingTests) DirectiveOnStreamingErrorEngagesFDv1(t *ldtest.T) { + t.Specification("DATASYSTEM", "1.6.1", "X-LD-FD-Fallback header on error engages FDv1 Fallback Synchronizer") + t.Specification("DATASYSTEM", "1.6.3", "FDv2 synchronizer chain halted when directive engages FDv1") + t.Specification("CSFDV2", "8.1.1", "x-ld-fd-fallback header triggers client-side FDv1 fallback") + t.Specification("CSFDV2", "8.1.3", "FDv2 synchronizers disabled, not removed") streamHandler, _ := httphelpers.RecordingHandler(httphelpers.HandlerWithResponse( 403, http.Header{"X-LD-FD-Fallback": []string{"true"}}, nil)) streamEndpoint := requireContext(t).harness.NewMockEndpoint(streamHandler, t.DebugLogger(), @@ -510,6 +528,10 @@ func (c CommonStreamingTests) DirectiveOnStreamingErrorEngagesFDv1(t *ldtest.T) // particular translate an unsolicited 304 into a 200 against their cache, which // would defeat the "FDv1 supplies no fresh data" property this test relies on. func (c CommonStreamingTests) DirectiveOnStreamingSuccessAppliesPayload(t *ldtest.T) { + t.Specification("DATASYSTEM", "1.6.2", "payload applied to Memory Store before transitioning to FDv1") + t.Specification("DATASYSTEM", "1.6.3", "FDv2 Primary Synchronizer stopped when directive engages FDv1") + t.Specification("CSFDV2", "8.1.1", "x-ld-fd-fallback header triggers client-side FDv1 fallback") + t.Specification("CSFDV2", "8.1.3", "FDv2 synchronizers disabled, not removed") streamingValue := ldvalue.String("value-from-streaming-payload") streamingData := c.makeSDKDataWithFlag(1, streamingValue) streamingService := mockld.NewStreamingService(streamingData, requireContext(t).sdkKind, t.DebugLogger()) @@ -596,6 +618,9 @@ func (c CommonStreamingTests) DirectiveOnStreamingSuccessAppliesPayload(t *ldtes // Synchronizer must never be started; the SDK transitions directly to the FDv1 // Fallback Synchronizer. func (c CommonStreamingTests) DirectiveOnPollingInitializerSkipsSynchronizers(t *ldtest.T) { + t.Specification("DATASYSTEM", "1.6.3", "initializer directive skips FDv2 synchronizers entirely") + t.Specification("CSFDV2", "8.1.1", "x-ld-fd-fallback header triggers client-side FDv1 fallback") + t.Specification("CSFDV2", "8.1.3", "FDv2 synchronizers never started after directive") // Initializer endpoint: 500 + directive. No payload accompanies the directive in // this variant, so there is nothing to apply beforehand (1.6.2 is a no-op here). initHandler, _ := httphelpers.RecordingHandler(httphelpers.HandlerWithResponse( @@ -689,6 +714,8 @@ func (c CommonStreamingTests) DirectiveOnPollingInitializerSkipsSynchronizers(t // those retries is the positive signal that the directive caused a halt rather than // ordinary permanent removal (which would fire on a 4xx instead). func (c CommonStreamingTests) DirectiveWithoutFDv1ConfiguredHaltsDataSystem(t *ldtest.T) { + t.Specification("DATASYSTEM", "1.6.3", "directive without FDv1 configured halts data system entirely") + t.Specification("CSFDV2", "8.1.1", "x-ld-fd-fallback header recognized on client-side") handler, channel := httphelpers.RecordingHandler(httphelpers.HandlerWithResponse( 500, http.Header{"X-LD-FD-Fallback": []string{"true"}}, nil)) endpoint := requireContext(t).harness.NewMockEndpoint(handler, t.DebugLogger(), @@ -752,6 +779,9 @@ func (c CommonStreamingTests) DirectiveWithoutFDv1ConfiguredHaltsDataSystem(t *l // enough window that the FDv2 Recovery Condition (5 minutes in the default config) // would normally fire if the SDK were still treating this as a heuristic fallback. func (c CommonStreamingTests) DirectedFallbackIsTerminal(t *ldtest.T) { + t.Specification("DATASYSTEM", "1.6.4", "directed fallback is terminal; SDK never returns to FDv2 synchronizers") + t.Specification("CSFDV2", "8.1.1", "x-ld-fd-fallback triggers client-side FDv1 fallback") + t.Specification("CSFDV2", "8.1.3", "FDv2 synchronizers remain disabled indefinitely") t.LongRunning() // FDv2 streaming endpoint: 500 + directive on every request. 500 is normally a @@ -839,6 +869,7 @@ func (c CommonStreamingTests) DirectedFallbackIsTerminal(t *ldtest.T) { } func (c CommonStreamingTests) SavesPreviouslyKnownState(t *ldtest.T) { + t.Specification("DATASYSTEM", "1.2.1", "synchronizer reconnects with previously known state (basis query param)") dataBefore := c.makeSDKDataWithFlag(1, initialValue) dataAfter := mockld.NewServerSDKDataBuilder().IntentCode("none").IntentReason("up-to-date").Build() streamEndpoint, _ := makeSequentialStreamHandler(t, dataBefore, dataAfter) @@ -853,6 +884,7 @@ func (c CommonStreamingTests) SavesPreviouslyKnownState(t *ldtest.T) { } func (c CommonStreamingTests) ReplacesPreviouslyKnownState(t *ldtest.T) { + t.Specification("DATASYSTEM", "1.2.1", "xfer-full intent replaces entire store contents on reconnect") dataBefore := c.makeSDKDataWithFlag(1, initialValue) dataAfter := mockld.NewServerSDKDataBuilder(). IntentCode("xfer-full"). @@ -875,6 +907,7 @@ func (c CommonStreamingTests) ReplacesPreviouslyKnownState(t *ldtest.T) { } func (c CommonStreamingTests) UpdatesPreviouslyKnownState(t *ldtest.T) { + t.Specification("DATASYSTEM", "1.2.1", "xfer-changes intent applies deltas to existing store on reconnect") dataBefore := c.makeSDKDataWithFlag(1, initialValue) dataAfter := mockld.NewServerSDKDataBuilder(). IntentCode("xfer-changes"). @@ -895,6 +928,7 @@ func (c CommonStreamingTests) UpdatesPreviouslyKnownState(t *ldtest.T) { } func (c CommonStreamingTests) UpdatesAreNotCompleteUntilPayloadTransferredIsSent(t *ldtest.T) { + t.Specification("DATASYSTEM", "1.2.9", "updates buffered until payload-transferred event confirms completion") dataSystem, configurers := c.setupDataSystems(t, c.makeSDKDataWithFlag(1, initialValue)) client := NewSDKClient(t, c.baseSDKConfigurationPlus(configurers...)...) @@ -932,6 +966,7 @@ func (c CommonStreamingTests) UpdatesAreNotCompleteUntilPayloadTransferredIsSent } func (c CommonStreamingTests) HandlesMultipleUpdates(t *ldtest.T) { + t.Specification("DATASYSTEM", "1.2.9", "multiple sequential payloads each applied after payload-transferred") dataSystem, configurers := c.setupDataSystems(t, c.makeSDKDataWithFlag(1, initialValue)) client := NewSDKClient(t, c.baseSDKConfigurationPlus(configurers...)...) context := ldcontext.New("context-key") @@ -958,6 +993,7 @@ func (c CommonStreamingTests) HandlesMultipleUpdates(t *ldtest.T) { } func (c CommonStreamingTests) IgnoresModelVersion(t *ldtest.T) { + t.Specification("DATASYSTEM", "1.2.9", "SDK trusts aggregate state version, ignores individual model version") dataSystem, configurers := c.setupDataSystems(t, c.makeSDKDataWithFlag(100, initialValue)) client := NewSDKClient(t, c.baseSDKConfigurationPlus(configurers...)...) @@ -977,6 +1013,8 @@ func (c CommonStreamingTests) IgnoresModelVersion(t *ldtest.T) { } func (c CommonStreamingTests) IgnoresHeartBeat(t *ldtest.T) { + t.Specification("FDV2PL", "4.3.9", + "heartbeat events silently ignored") dataSystem, configurers := c.setupDataSystems(t, c.makeSDKDataWithFlag(1, initialValue)) client := NewSDKClient(t, c.baseSDKConfigurationPlus(configurers...)...) @@ -1047,6 +1085,7 @@ func (c CommonStreamingTests) IgnoresUnknownEventStartOfPayload(t *ldtest.T) { } func (c CommonStreamingTests) CanDiscardPartialEventsOnError(t *ldtest.T) { + t.Specification("DATASYSTEM", "1.2.3", "error event discards preceding buffered updates; subsequent payload applied") dataSystem, configurers := c.setupDataSystems(t, c.makeSDKDataWithFlag(1, initialValue)) client := NewSDKClient(t, c.baseSDKConfigurationPlus(configurers...)...) @@ -1079,6 +1118,7 @@ func (c CommonStreamingTests) CanDiscardPartialEventsOnError(t *ldtest.T) { } func (c CommonStreamingTests) CanDiscardFullEventsOnError(t *ldtest.T) { + t.Specification("DATASYSTEM", "1.2.3", "error event discards updates; xfer-full intent replaces store entirely") dataSystem, configurers := c.setupDataSystems(t, c.makeSDKDataWithFlag(1, initialValue)) client := NewSDKClient(t, c.baseSDKConfigurationPlus(configurers...)...) @@ -1103,6 +1143,8 @@ func (c CommonStreamingTests) CanDiscardFullEventsOnError(t *ldtest.T) { } func (c CommonStreamingTests) DisconnectsOnGoodbye(t *ldtest.T) { + t.Specification("FDV2PL", "4.3.5", + "goodbye event logged and triggers disconnection") dataBefore := c.makeSDKDataWithFlag(1, initialValue) dataAfter := mockld.NewServerSDKDataBuilder().IntentCode("none").IntentReason("up-to-date").Build() streamEndpoint, dataSystems := makeSequentialStreamHandler(t, dataBefore, dataAfter) diff --git a/sdktests/common_tests_stream_request.go b/sdktests/common_tests_stream_request.go index ef110b0e..0b358bb7 100644 --- a/sdktests/common_tests_stream_request.go +++ b/sdktests/common_tests_stream_request.go @@ -54,6 +54,10 @@ func (c CommonStreamingTests) RequestMethodAndHeaders(t *ldtest.T, credential st } func (c CommonStreamingTests) RequestURLPath(t *ldtest.T, pathMatcher func(flagRequestMethod) m.Matcher) { + t.Specification("ENVFILTER", "1.2.1", "request URL joins base URI with filter query parameter") + t.Specification("ENVFILTER", "1.2.2", "exactly one filter parameter is included") + t.Specification("ENVFILTER", "1.2.3", "filter parameter value is URL encoded") + t.Specification("ENVFILTER", "1.3.1", "filter parameter omitted for invalid filter key") t.Run("URL path is computed correctly", func(t *ldtest.T) { for _, filter := range c.environmentFilters() { t.Run(h.IfElse(filter.IsDefined(), filter.String(), "no environment filter"), func(t *ldtest.T) { diff --git a/sdktests/sdk_context_type.go b/sdktests/sdk_context_type.go index bcd35727..2b25a557 100644 --- a/sdktests/sdk_context_type.go +++ b/sdktests/sdk_context_type.go @@ -33,6 +33,23 @@ func doSDKContextTypeTests(t *ldtest.T) { // at a client instance. func doSDKContextBuildTests(t *ldtest.T) { + t.Specification("CTXBLD", "1.2.3", "kind operation takes kind string and optional key, returns LDAttributesBuilder") + t.Specification("CTXBLD", "1.2.6", "build operation returns LDContext") + t.Specification("CTXBLD", "1.2.7", "one kind = individual context, more than one = multi-context") + t.Specification("CTXBLD", "1.4.1", "dedicated name operation accepts string") + t.Specification("CTXBLD", "1.4.2", "dedicated anonymous operation accepts boolean") + t.Specification("CTXBLD", "1.4.3", "unset anonymous behaves as false") + t.Specification("CTXBLD", "1.5.1", "generic attribute setter stores custom attributes") + t.Specification("CTXBLD", "1.5.3", "null value removes custom attribute") + t.Specification("CTXBLD", "1.5.4", "non-null value stores custom attribute") + t.Specification("CTXBLD", "1.6.4", "add private attribute reference strings") + t.Specification("CONTEXT", "1.2.3.2", "anonymous false treated as absent") + t.Specification("CONTEXT", "1.4.2", "custom attribute value types (boolean, number, string, array, object)") + t.Specification("CONTEXT", "1.5.6", "null attribute value treated as absent") + t.Specification("CONTEXT", "1.10.1.1", "single-kind JSON includes kind and key") + t.Specification("CONTEXT", "1.10.2.1", "multi-kind JSON includes kind:\"multi\"") + t.Specification("CONTEXT", "1.10.2.2", "multi-kind sub-objects follow single-kind schema without kind property") + dataSystem := NewSDKDataSystem(t, nil) client := NewSDKClient(t, dataSystem) @@ -113,6 +130,9 @@ func doSDKContextBuildTests(t *ldtest.T) { } func doSDKContextConvertTests(t *ldtest.T) { + t.Specification("CONTEXT", "1.12.4", "SDK supports converting context to/from JSON") + t.Specification("CONTEXT", "1.12.5", "SDK supports deserializing legacy JSON format") + dataSystem := NewSDKDataSystem(t, nil) client := NewSDKClient(t, dataSystem) @@ -124,6 +144,17 @@ func doSDKContextConvertTests(t *ldtest.T) { } t.Run("valid, no changes", func(t *ldtest.T) { + t.Specification("CONTEXT", "1.1.2", "kind contains only allowed characters (ASCII alphanumerics, ., -, _)") + t.Specification("CONTEXT", "1.2.1.1", "key present on every single-kind context") + t.Specification("CONTEXT", "1.2.2.1", "name is a string if set") + t.Specification("CONTEXT", "1.2.3.1", "anonymous is a boolean if set") + t.Specification("CONTEXT", "1.4.2", "custom attribute value types") + t.Specification("CONTEXT", "1.10.1.1", "single-kind includes kind and key as top-level properties") + t.Specification("CONTEXT", "1.10.1.2", "_meta.privateAttributes format") + t.Specification("CONTEXT", "1.10.1.5", "non-built-in top-level properties are custom attributes") + t.Specification("CONTEXT", "1.10.2.1", "multi-kind includes kind:\"multi\"") + t.Specification("CONTEXT", "1.10.2.2", "multi-kind sub-objects without kind property") + singleKindProps := []string{ ``, `"name": "b"`, @@ -167,6 +198,9 @@ func doSDKContextConvertTests(t *ldtest.T) { }) t.Run("unnecessary properties are dropped", func(t *ldtest.T) { + t.Specification("CONTEXT", "1.5.6", "null attribute value treated as absent") + t.Specification("CONTEXT", "1.5.7", "JSON null property treated as omitting property") + expected := json.RawMessage(basicInputPlusProps("")) for _, extraProps := range []string{ @@ -187,6 +221,13 @@ func doSDKContextConvertTests(t *ldtest.T) { }) t.Run("old user to context", func(t *ldtest.T) { + t.Specification("CONTEXT", "1.10.3.1", "no kind property means kind \"user\"") + t.Specification("CONTEXT", "1.10.3.2", "legacy key and name treated same as regular context") + t.Specification("CONTEXT", "1.10.3.3", "privateAttributeNames treated as _meta.privateAttributes") + t.Specification("CONTEXT", "1.10.3.4", "legacy string attrs (firstName, lastName, email, country, ip, avatar)") + t.Specification("CONTEXT", "1.10.3.5", "custom property treated as additional custom attributes") + t.Specification("CONTEXT", "1.12.5", "SDK deserializes legacy JSON format") + t.RequireCapability(servicedef.CapabilityUserType) type contextConversionParams struct { in, out string // out only needs to be set if it's different from in @@ -236,6 +277,7 @@ func doSDKContextConvertTests(t *ldtest.T) { }) t.Run("multi-kind with only one kind becomes single-kind", func(t *ldtest.T) { + t.Specification("CTXBLD", "1.2.7", "multi-context with single kind normalizes to individual context") singleKindJSON := `{"kind": "org", "key": "a", "name": "b"}` multiKindJSON := `{"kind": "multi", "org": {"key": "a", "name": "b"}}` resp := client.ContextConvert(t, servicedef.ContextConvertParams{Input: multiKindJSON}) @@ -244,6 +286,15 @@ func doSDKContextConvertTests(t *ldtest.T) { }) t.Run("invalid context", func(t *ldtest.T) { + t.Specification("CONTEXT", "1.1.1", "kind must be non-empty string") + t.Specification("CONTEXT", "1.1.2", "kind must contain only ASCII alphanumerics, ., -, _") + t.Specification("CONTEXT", "1.1.3", "kind must not be \"kind\"") + t.Specification("CONTEXT", "1.13.1.1", "invalid kind (empty, bad chars, or equals \"kind\")") + t.Specification("CONTEXT", "1.13.1.3", "multi-kind with no contexts is invalid") + t.Specification("CONTEXT", "1.13.2.1", "kind null or not a string is invalid") + t.Specification("CONTEXT", "1.13.2.2", "single-kind type/schema validation errors") + t.Specification("CONTEXT", "1.13.4.1", "deserialization must fail on invalid conditions") + inputs := []string{ ``, `{`, // malformed JSON @@ -299,6 +350,8 @@ func doSDKContextConvertTests(t *ldtest.T) { }) t.Run("invalid old user", func(t *ldtest.T) { + t.Specification("CONTEXT", "1.13.2.4", "legacy format type/schema validation errors") + t.RequireCapability(servicedef.CapabilityUserType) inputs := make([]string, 0, 12) inputs = append(inputs, @@ -321,6 +374,9 @@ func doSDKContextConvertTests(t *ldtest.T) { } func doSDKContextComparisonTests(t *ldtest.T) { + t.Specification("CTXBLD", "1.8.4", + "equal contexts have same kinds, keys, names, anonymous, custom attrs, private attrs") + t.Specification("CONTEXT", "1.10.1.4", "order of privateAttributes entries is not significant") dataSystem := NewSDKDataSystem(t, nil) client := NewSDKClient(t, dataSystem) address := ldvalue.ObjectBuild().SetString("street", "123 Easy St").SetString("city", "Anytown").Build() diff --git a/sdktests/server_side_big_segments.go b/sdktests/server_side_big_segments.go index 875e279a..c1608a25 100644 --- a/sdktests/server_side_big_segments.go +++ b/sdktests/server_side_big_segments.go @@ -38,6 +38,14 @@ func doServerSideBigSegmentsTests(t *ldtest.T) { } func doBigSegmentsEvaluateSegment(t *ldtest.T) { + t.Specification("BIGSEG", "1.3.1", "context keys hashed as base64(sha256(key)) before store query") + t.Specification("BIGSEG", "1.9.2.1", + "included/excluded/includedContexts/excludedContexts fields ignored for big segments") + t.Specification("BIGSEG", "1.9.2.2", "store queried when unbounded=true and generation has a value") + t.Specification("BIGSEG", "1.9.2.5", "membership tested using \".g\" reference format") + t.Specification("BIGSEG", "1.9.2.6", "true→match, false→non-match, absent→fall through to rules") + t.Specification("BIGSEG", "1.9.1.3", "no query if context kind doesn't match unboundedContextKind") + t.Specification("BIGSEG", "1.9.4.1", "bigSegmentsStatus added to evaluation reason") otherContext := ldcontext.New("other-user-key") otherKind := ldcontext.Kind("other") @@ -180,6 +188,16 @@ func doBigSegmentsEvaluateSegment(t *ldtest.T) { } func doBigSegmentsMembershipCachingTests(t *ldtest.T) { + t.Specification("BIGSEG", "1.9.5.1", "store queried at most once per context key within a single evaluation") + t.Specification("BIGSEG", "1.9.1.2", "individual context extracted by unboundedContextKind") + t.Specification("BIGSEG", "1.9.1.4", "store query uses context key alone, not (kind, key) pair") + t.Specification("BIGSEG", "1.5.2", "wrapper checks LRU cache before querying store") + t.Specification("BIGSEG", "1.6.1", "wrapper checks LRU cache before querying store") + t.Specification("BIGSEG", "1.6.2", "cache keyed by unhashed context key") + t.Specification("BIGSEG", "1.6.3", "cache holds at most configured max entries (LRU eviction)") + t.Specification("BIGSEG", "1.4.2", "cache holds at most configured max entries (LRU eviction)") + t.Specification("BIGSEG", "1.6.4", "cache entries expire after configured TTL") + t.Specification("BIGSEG", "1.4.3", "cache entries expire after configured TTL") user1, user2, user3 := ldcontext.New("user1"), ldcontext.New("user2"), ldcontext.New("user3") otherKind := ldcontext.Kind("other") expectedUserHash1, expectedUserHash2, expectedUserHash3 := "CgQblGLKpKMbrDVn4Lbm/ZEAeH2yq0M9lvbReMq/zpA=", @@ -405,6 +423,11 @@ func doBigSegmentsMembershipCachingTests(t *ldtest.T) { } func doBigSegmentsStatusPollingTests(t *ldtest.T) { + t.Specification("BIGSEG", "1.8.4", "background task polls store metadata at configured interval") + t.Specification("BIGSEG", "1.4.5", "background task polls store metadata at configured interval") + t.Specification("BIGSEG", "1.8.5", "polling calls GetMetadata and updates status (available + stale)") + t.Specification("BIGSEG", "1.8.6", "listeners notified only when status actually changes") + t.Specification("BIGSEG", "1.8.9", "evaluations do not trigger additional metadata polls") dataSystem := NewSDKDataSystem(t, mockld.EmptyServerSDKData()) // PHP SDKs only support second-level granularity @@ -520,6 +543,12 @@ func doBigSegmentsStatusPollingTests(t *ldtest.T) { } func doBigSegmentsErrorHandlingTests(t *ldtest.T) { + t.Specification("BIGSEG", "1.9.2.4", "no store configured → NOT_CONFIGURED status, segment is non-match") + t.Specification("BIGSEG", "1.5.4", "no store configured → NOT_CONFIGURED status, segment is non-match") + t.Specification("BIGSEG", "1.9.2.3", "unbounded=true but no generation → NOT_CONFIGURED, segment is non-match") + t.Specification("BIGSEG", "1.5.5", "store error → STORE_ERROR status, segment is non-match, eval doesn't fail") + t.Specification("BIGSEG", "1.9.3.1", "store error → STORE_ERROR status, segment is non-match, eval doesn't fail") + t.Specification("BIGSEG", "1.9.3.3", "store error → STORE_ERROR status, segment is non-match, eval doesn't fail") t.Run("big segment store was not configured", func(t *ldtest.T) { segment := ldbuilders.NewSegmentBuilder("segment-key").Version(1). Included(bigSegmentsContext.Key()). // regular include list should be ignored if unbounded=true diff --git a/sdktests/server_side_eval.go b/sdktests/server_side_eval.go index 982bb693..b3e81ac0 100644 --- a/sdktests/server_side_eval.go +++ b/sdktests/server_side_eval.go @@ -22,6 +22,8 @@ func doServerSideEvalTests(t *ldtest.T) { } func runParameterizedServerSideEvalTests(t *ldtest.T) { + t.Specification("FLGEDETAIL", "1.1.6.1", "server-side reason field is always present and non-null") + t.Specification("FLGEDETAIL", "1.4.3.1", "server-side *variationDetail accepts (flagKey, context, defaultValue)") parameterizedTests := CommonEvalParameterizedTestRunner[mockld.ServerSDKData]{ SDKConfigurers: func(testSuite testmodel.EvalTestSuite[mockld.ServerSDKData]) []SDKConfigurer { return nil }, FilterSDKData: nil, @@ -31,6 +33,8 @@ func runParameterizedServerSideEvalTests(t *ldtest.T) { } func runParameterizedServerSideClientNotReadyEvalTests(t *ldtest.T) { + t.Specification("FLGEDETAIL", "1.3.1", "errorKind CLIENT_NOT_READY when SDK not initialized") + t.Specification("FLGEDETAIL", "1.1.3", "variationIndex absent when default value returned") defaultValues := data.MakeValueFactoryBySDKValueType() flagKey := "some-flag" context := ldcontext.New("user-key") diff --git a/sdktests/server_side_eval_all_flags.go b/sdktests/server_side_eval_all_flags.go index 55148e2e..03b3b652 100644 --- a/sdktests/server_side_eval_all_flags.go +++ b/sdktests/server_side_eval_all_flags.go @@ -41,6 +41,14 @@ func runServerSideEvalAllFlagsTests(t *ldtest.T) { } func doServerSideAllFlagsBasicTest(t *ldtest.T) { + t.Specification("FLGMES", "1.1.1", "AllFlagsState returns evaluation results for all flags for a context") + t.Specification("FLGMES", "1.3.1.5", "Flags state has valid=true when evaluation succeeds") + t.Specification("FLGMES", "1.3.2.5", "Flag version set to the flag's version property") + t.Specification("FLGMES", "1.3.2.6", "Flag value set to the value returned by the evaluation algorithm") + t.Specification("FLGMES", "1.3.2.7", "Variation index set to the evaluation result variation index") + t.Specification("FLGMES", "1.3.2.9", "Track events true when flag's trackEvents property is true") + t.Specification("FLGMES", "1.3.2.11", "Debug events until date set from flag's debugEventsUntilDate property") + t.Specification("FLGMES", "1.4.1", "Omits null/false properties in bootstrapping JSON") flag1 := ldbuilders.NewFlagBuilder("flag1").Version(100). Variations(dummyValue0, ldvalue.String("value1")). On(false).OffVariation(1). @@ -106,6 +114,8 @@ func doServerSideAllFlagsBasicTest(t *ldtest.T) { func doServerSideAllFlagsWithReasonsTest(t *ldtest.T) { t.RequireCapability(servicedef.CapabilityAllFlagsWithReasons) + t.Specification("FLGEDETAIL", "1.2.2", "reason kind OFF and FALLTHROUGH returned in all-flags state") + t.Specification("FLGMES", "1.3.2.8", "Reason included when with-reasons option is true") // flag1 has reason "OFF" flag1 := ldbuilders.NewFlagBuilder("flag1").Version(100). @@ -148,6 +158,10 @@ func doServerSideAllFlagsWithReasonsTest(t *ldtest.T) { } func doServerSideAllFlagsExperimentationTest(t *ldtest.T) { + t.Specification("FLGEDETAIL", "1.2.3", "inExperiment field present when kind is FALLTHROUGH or RULE_MATCH") + t.Specification("FLGMES", "1.3.2.8", "Reason included when evaluation involves an experiment") + t.Specification("FLGMES", "1.3.2.9", "Track events true when evaluation involves an experiment") + t.Specification("FLGMES", "1.3.2.10", "Track reason true when evaluation involves an experiment") // flag1 has experiment behavior because it's a fallthrough and has trackEventsFallthrough=true flag1 := ldbuilders.NewFlagBuilder("flag1").Version(100). Variations(dummyValue0, ldvalue.String("value1")). @@ -192,6 +206,10 @@ func doServerSideAllFlagsExperimentationTest(t *ldtest.T) { } func doServerSideAllFlagsErrorInFlagTest(t *ldtest.T) { + t.Specification("FLGEDETAIL", "1.2.2", "reason kind ERROR with errorKind in all-flags state") + t.Specification("FLGEDETAIL", "1.3.1", "errorKind MALFORMED_FLAG for invalid flag data") + t.Specification("FLGMES", "1.6.1.1", "Failed flag still included with null value and no variation") + t.Specification("FLGMES", "1.3.2.8", "Error reason included when with-reasons is true") // This test verifies that 1. an error in evaluation of one flag does not prevent evaluation // of the rest of the flags, and 2. the failed flag is still included in the results, with a // value of null (and, if reasons are present, a reason that explains the error) @@ -264,6 +282,7 @@ func doServerSideAllFlagsErrorInFlagTest(t *ldtest.T) { func doServerSideAllFlagsClientSideOnlyTest(t *ldtest.T) { t.RequireCapability(servicedef.CapabilityAllFlagsClientSideOnly) + t.Specification("FLGMES", "1.3.2.1", "Flags skipped when client-side only and usingEnvironmentId is false") flag1 := ldbuilders.NewFlagBuilder("server-side-1").Build() flag2 := ldbuilders.NewFlagBuilder("server-side-2").Build() @@ -297,6 +316,10 @@ func doServerSideAllFlagsDetailsOnlyForTrackedFlagsTest(t *ldtest.T) { // appropriate. t.RequireCapability(servicedef.CapabilityAllFlagsDetailsOnlyForTrackedFlags) + t.Specification("FLGEDETAIL", "1.1.3", "variationIndex always present even when reason/version omitted") + t.Specification("FLGEDETAIL", "1.2.2", "reason kind OFF included only for tracked flags") + t.Specification("FLGMES", "1.3.2.8", + "Reason and version omitted for untracked flags when details-only-for-tracked is true") // flag1 will have details removed because it's not in any of the other categories below flag1 := ldbuilders.NewFlagBuilder("flag1").Version(100). @@ -354,6 +377,8 @@ func doServerSideAllFlagsDetailsOnlyForTrackedFlagsTest(t *ldtest.T) { } func doServerSideAllFlagsClientNotReadyTest(t *ldtest.T) { + t.Specification("FLGEDETAIL", "1.3.1", "errorKind CLIENT_NOT_READY in all-flags when SDK not initialized") + t.Specification("FLGMES", "1.3.1.1", "Uninitialized SDK returns empty flags state with valid=false") dataSystem := NewSDKDataSystem(t, mockld.BlockingUnavailableSDKData(mockld.ServerSideSDK)) client := NewSDKClient(t, WithConfig(servicedef.SDKConfigParams{StartWaitTimeMS: o.Some(ldtime.UnixMillisecondTime(1)), @@ -373,6 +398,7 @@ func doServerSideAllFlagsClientNotReadyTest(t *ldtest.T) { } func doServerSideAllFlagsCompactRepresentationsTest(t *ldtest.T) { + t.Specification("FLGMES", "1.4.1", "SDKs must omit null/false properties in bootstrapping JSON") t.NonCritical(`If this failed but the other 'all flags' tests passed, the SDK is including null-valued` + ` properties within the $flagsState part of the representation. To save bandwidth, it's desirable` + ` to omit such properties.`) @@ -407,6 +433,9 @@ func doServerSideAllFlagsCompactRepresentationsTest(t *ldtest.T) { } func doServerSideAllFlagsIncludesToplevelPreqrequisitesTest(t *ldtest.T) { + t.Specification("CSPE", "1.1.1", "$flagsState includes prerequisites field for flags with prerequisite evaluations") + t.Specification("CSPE", "1.1.1.2", "prerequisites is an ordered list of evaluated direct prerequisites") + topLevel := ldbuilders.NewFlagBuilder("topLevel").Version(100). Variations(ldvalue.String("value1")).On(true).FallthroughVariation(0). AddPrerequisite("directPrereq1", 0). @@ -460,6 +489,9 @@ func doServerSideAllFlagsIncludesToplevelPreqrequisitesTest(t *ldtest.T) { } func doServerSideAllFlagsIgnoresPrereqsIfNotEvaluatedTest(t *ldtest.T) { + t.Specification("CSPE", "1.1.1.1", "prerequisites field omitted when flag is off or not evaluated") + t.Specification("CSPE", "1.1.1.2", "only evaluated prerequisites included; short-circuits on failure") + flagOn := ldbuilders.NewFlagBuilder("flagOn").Version(100). Variations(ldvalue.String("value1")).On(true).FallthroughVariation(0). AddPrerequisite("prereq1", 0). @@ -529,6 +561,8 @@ func doServerSideAllFlagsIgnoresPrereqsIfNotEvaluatedTest(t *ldtest.T) { } func doServerSideAllFlagsIgnoresClientSideOnlyForPrereqKeys(t *ldtest.T) { + t.Specification("CSPE", "1.1.1", "prerequisites listed regardless of client-side visibility (Note 1.1.1)") + flag := ldbuilders.NewFlagBuilder("flag").Version(100). ClientSideUsingEnvironmentID(true). Variations(ldvalue.String("value1")).On(true).FallthroughVariation(0). diff --git a/sdktests/server_side_events_eval.go b/sdktests/server_side_events_eval.go index bc3ec7ae..492283ea 100644 --- a/sdktests/server_side_events_eval.go +++ b/sdktests/server_side_events_eval.go @@ -484,6 +484,10 @@ func doDebugEventTestCases( } func doServerSideFeaturePrerequisiteEventTests(t *ldtest.T) { + t.Specification("CSPE", "1.2.1", "prerequisite events emitted equivalent to direct evaluation (server-side)") + t.Specification("CSPE", "1.2.3", "prerequisite events emitted before dependent flag's event") + t.Specification("CSPE", "1.2.4", "recursive prerequisites generate events (flag1→flag2→flag3)") + // The test logic for this is *almost* exactly the same for PHP as for other server-side SDKs // (the only difference is the absence of index and summary events), so we reuse the same // function. diff --git a/sdktests/server_side_events_summary.go b/sdktests/server_side_events_summary.go index d54e48b1..5c3d2e46 100644 --- a/sdktests/server_side_events_summary.go +++ b/sdktests/server_side_events_summary.go @@ -29,6 +29,9 @@ func doServerSideSummaryEventTests(t *ldtest.T) { } func doServerSideSummaryEventBasicTest(t *ldtest.T) { + t.Specification("CSSE", "2.1.2.1", "counters group evaluations by flag key, variation, and version") + t.Specification("CSSE", "2.1.1", "accumulator tracks default value, count, and resulting value per counter") + flag1 := ldbuilders.NewFlagBuilder("flag1").Version(100). Variations(ldvalue.String("value1a"), ldvalue.String("value1b")). On(true).FallthroughVariation(0). @@ -100,6 +103,8 @@ func doServerSideSummarySamplingRatioTests(t *ldtest.T) { } func flagIsExcludedFromSummaries(t *ldtest.T) { + t.Specification("CSSE", "1.1.1.2.1", "excludeFromSummaries=true causes flag to be omitted from summary") + flag1 := ldbuilders.NewFlagBuilder("flag1"). On(true). Variations(ldvalue.String("value1a"), ldvalue.String("value1b")). @@ -156,6 +161,8 @@ func flagIsExcludedFromSummaries(t *ldtest.T) { } func flagPreqIsExcludedFromSummaries(t *ldtest.T) { + t.Specification("CSSE", "1.1.1.2.1", "prerequisite with excludeFromSummaries=true omitted from summary") + flag1 := ldbuilders.NewFlagBuilder("flag1"). On(true). Variations(ldvalue.Bool(true), ldvalue.Bool(false)). @@ -200,6 +207,8 @@ func flagPreqIsExcludedFromSummaries(t *ldtest.T) { } func doServerSideSummaryEventContextKindsTest(t *ldtest.T) { + t.Specification("CSSE", "2.1.3.1", "features include contextKinds from all evaluating context kinds") + flag1 := ldbuilders.NewFlagBuilder("flag1").Version(100). Variations(ldvalue.String("value1a"), ldvalue.String("value1b")). On(true).FallthroughVariation(0). @@ -267,6 +276,8 @@ func doServerSideSummaryEventContextKindsTest(t *ldtest.T) { } func doServerSideSummaryEventUnknownFlagTest(t *ldtest.T) { + t.Specification("CSSE", "2.1.3.1", "unknown flag counter sets unknown=true with no version or variation") + unknownKey := "flag-x" context := ldcontext.New("user-key") default1 := ldvalue.String("default1") @@ -302,6 +313,9 @@ func doServerSideSummaryEventUnknownFlagTest(t *ldtest.T) { } func doServerSideSummaryEventResetTest(t *ldtest.T) { + t.Specification("CSSE", "1.1.2.1", "accumulators cleared after getSummaries; new flush has fresh counts") + t.Specification("CSSE", "3.1.1", "summaries included in flushed event batch") + flag := ldbuilders.NewFlagBuilder("flag1").Version(100). Variations(ldvalue.String("value-a"), ldvalue.String("value-b")). On(true).FallthroughVariation(0). @@ -371,6 +385,8 @@ func doServerSideSummaryEventResetTest(t *ldtest.T) { } func doServerSideSummaryEventPrerequisitesTest(t *ldtest.T) { + t.Specification("CSSE", "2.1.2.1", "prerequisite evaluations increment counters in summary event") + context := ldcontext.New("user-key") expectedValue1 := ldvalue.String("value1") expectedPrereqValue2 := ldvalue.String("ok2") @@ -437,6 +453,8 @@ func doServerSideSummaryEventPrerequisitesTest(t *ldtest.T) { } func doServerSideSummaryEventVersionTest(t *ldtest.T) { + t.Specification("CSSE", "2.1.2.1", "different flag versions tracked separately in counters") + // This test verifies that if the version of a flag changes within the timespan of one event payload, // evaluations for each version are tracked separately. We do this by evaluating the flag in its // original version, then pushing a stream update and polling until the SDK reports the updated diff --git a/sdktests/server_side_migrations.go b/sdktests/server_side_migrations.go index b472ac76..30590903 100644 --- a/sdktests/server_side_migrations.go +++ b/sdktests/server_side_migrations.go @@ -62,6 +62,9 @@ func withExecutionOrders(test func(*ldtest.T, ldmigration.ExecutionOrder)) func( } func identifyCorrectStageFromStringFlag(t *ldtest.T) { + t.Specification("MIGRATIONS", "1.1.1", "MigrationVariation returns current stage for a migration flag key") + t.Specification("MIGRATIONS", "1.1.1.1", "returns one of off/dualwrite/shadow/live/rampdown/complete") + stages := []ldmigration.Stage{ldmigration.Off, ldmigration.DualWrite, ldmigration.Shadow, ldmigration.Live, ldmigration.RampDown, ldmigration.Complete} for _, stage := range stages { @@ -83,6 +86,9 @@ func identifyCorrectStageFromStringFlag(t *ldtest.T) { } func usesDefaultWhenAppropriate(t *ldtest.T) { + t.Specification("MIGRATIONS", "1.1.1.2", "invalid default stage falls back to off") + t.Specification("MIGRATIONS", "1.1.1.3", "invalid eval result returns default stage with WRONG_TYPE reason") + stages := []ldmigration.Stage{ldmigration.Off, ldmigration.DualWrite, ldmigration.Shadow, ldmigration.Live, ldmigration.RampDown, ldmigration.Complete} scenarios := []struct { key string @@ -115,6 +121,8 @@ func usesDefaultWhenAppropriate(t *ldtest.T) { } func executesOriginsInCorrectOrder(t *ldtest.T) { + t.Specification("MIGRATIONS", "1.3.2.2", "write operations execute origins in table-specified order, not parallel") + testParams := []struct { Operation ldmigration.Operation Stage ldmigration.Stage @@ -185,6 +193,8 @@ func executesOriginsInCorrectOrder(t *ldtest.T) { } func executesReads(t *ldtest.T) { + t.Specification("MIGRATIONS", "1.3.1.3", "read operations invoke old/new/both origins per stage table") + testParams := []struct { Operation ldmigration.Operation Stage ldmigration.Stage @@ -255,6 +265,9 @@ func executesReads(t *ldtest.T) { } func payloadsArePassedThrough(t *ldtest.T) { + t.Specification("MIGRATIONS", "1.3.1.5", "payload parameter forwarded to read and write methods") + t.Specification("MIGRATIONS", "1.3.2.4", "payload parameter forwarded to read and write methods") + testParams := []struct { Operation ldmigration.Operation Stage ldmigration.Stage @@ -339,6 +352,9 @@ func payloadsArePassedThrough(t *ldtest.T) { } func tracksInvoked(t *ldtest.T, order ldmigration.ExecutionOrder) { + t.Specification("MIGRATIONS", "1.1.3.3", "tracker records which origins were invoked") + t.Specification("MIGRATIONS", "1.1.3.1", "tracker event includes evaluation detail from MigrationVariation") + onlyOld := []m.Matcher{m.JSONOptProperty("old").Should(m.Not(m.BeNil())), m.JSONOptProperty("new").Should(m.BeNil())} both := []m.Matcher{m.JSONOptProperty("old").Should(m.Not(m.BeNil())), m.JSONOptProperty("new").Should(m.Not(m.BeNil()))} onlyNew := []m.Matcher{m.JSONOptProperty("old").Should(m.BeNil()), m.JSONOptProperty("new").Should(m.Not(m.BeNil()))} @@ -432,6 +448,9 @@ func tracksInvoked(t *ldtest.T, order ldmigration.ExecutionOrder) { //nolint:dupl // Invokes and latency happen to share the same setup, but should be tested independently. func tracksLatency(t *ldtest.T, order ldmigration.ExecutionOrder) { + t.Specification("MIGRATIONS", "1.1.3.5", "tracker records latency measurements per origin") + t.Specification("MIGRATIONS", "1.2.3", "latency tracking is opt-in") + onlyOld := []m.Matcher{m.JSONOptProperty("old").Should(m.Not(m.BeNil())), m.JSONOptProperty("new").Should(m.BeNil())} both := []m.Matcher{m.JSONOptProperty("old").Should(m.Not(m.BeNil())), m.JSONOptProperty("new").Should(m.Not(m.BeNil()))} onlyNew := []m.Matcher{m.JSONOptProperty("old").Should(m.BeNil()), m.JSONOptProperty("new").Should(m.Not(m.BeNil()))} @@ -544,6 +563,10 @@ func tracksLatency(t *ldtest.T, order ldmigration.ExecutionOrder) { } func writeFailuresShouldGenerateErrorMetrics(t *ldtest.T, order ldmigration.ExecutionOrder) { + t.Specification("MIGRATIONS", "1.1.3.4", "tracker records error occurrences per origin") + t.Specification("MIGRATIONS", "1.2.4", "error tracking is opt-in") + t.Specification("MIGRATIONS", "1.3.2.3", "execution halts on first error") + hasError := func(label string) m.Matcher { return m.JSONOptProperty(label).Should(m.Equal(true)) } isMissingOrNoError := func(label string) m.Matcher { return JSONPropertyNullOrAbsentOrEqualTo(label, false) } @@ -640,6 +663,8 @@ func writeFailuresShouldGenerateErrorMetrics(t *ldtest.T, order ldmigration.Exec } func successfulHandlersShouldNotGenerateErrorMetrics(t *ldtest.T, order ldmigration.ExecutionOrder) { + t.Specification("MIGRATIONS", "1.2.4", "no error measurement emitted when all handlers succeed") + successfulHandler := func(w http.ResponseWriter, req *http.Request) { w.WriteHeader(http.StatusOK) } testParams := []struct { @@ -716,6 +741,8 @@ func successfulHandlersShouldNotGenerateErrorMetrics(t *ldtest.T, order ldmigrat } func itHandlesMigrationEventsForMissingFlags(t *ldtest.T) { + t.Specification("MIGRATIONS", "1.1.3.8", "validation — missing flag generates error event with FLAG_NOT_FOUND") + successfulHandler := func(w http.ResponseWriter, req *http.Request) { w.WriteHeader(http.StatusOK) } testParams := []struct { @@ -796,6 +823,8 @@ func itHandlesMigrationEventsForMissingFlags(t *ldtest.T) { func itRedactsAnonymousContextAttributes(t *ldtest.T) { t.RequireCapability(servicedef.CapabilityAnonymousRedaction) + t.Specification("MIGRATIONS", "1.1.3.1", "migration op event includes context (redacted when anonymous)") + successfulHandler := func(w http.ResponseWriter, req *http.Request) { w.WriteHeader(http.StatusOK) } // Variation index does not matter for this test. @@ -861,6 +890,8 @@ func itRedactsAnonymousContextAttributes(t *ldtest.T) { } func itHandlesNonMigrationFlags(t *ldtest.T) { + t.Specification("MIGRATIONS", "1.1.1.3", "non-migration flag returns default stage with WRONG_TYPE reason") + successfulHandler := func(w http.ResponseWriter, req *http.Request) { w.WriteHeader(http.StatusOK) } testParams := []struct { @@ -939,6 +970,9 @@ func itHandlesNonMigrationFlags(t *ldtest.T) { } func trackConsistency(t *ldtest.T) { + t.Specification("MIGRATIONS", "1.1.3.6", "tracker records consistency check results") + t.Specification("MIGRATIONS", "1.3.1.4", "consistency tracked only for shadow/live stages when both reads succeed") + t.Run("checks for correct stage", withExecutionOrders(tracksConsistencyCorrectlyBasedOnStage)) t.Run("check ratio can disable", withExecutionOrders(tracksConsistencyIsDisabledByCheckRatio)) t.Run("unless callbacks fail", withExecutionOrders(tracksConsistencyIsDisabledIfCallbackFails)) @@ -946,6 +980,7 @@ func trackConsistency(t *ldtest.T) { func disableOpEventWithSamplingRatio(t *ldtest.T) { t.RequireCapability(servicedef.CapabilityEventSampling) + t.Specification("MIGRATIONS", "1.5", "samplingRatio of 0 suppresses op event emission") handler := func(w http.ResponseWriter, req *http.Request) { w.WriteHeader(http.StatusOK) @@ -1006,6 +1041,8 @@ func disableOpEventWithSamplingRatio(t *ldtest.T) { } func tracksConsistencyCorrectlyBasedOnStage(t *ldtest.T, order ldmigration.ExecutionOrder) { + t.Specification("MIGRATIONS", "1.3.1.4", "consistency checked only for shadow/live read stages") + handler := func(response string) func(w http.ResponseWriter, req *http.Request) { return func(w http.ResponseWriter, req *http.Request) { w.WriteHeader(http.StatusOK) @@ -1118,6 +1155,8 @@ func tracksConsistencyCorrectlyBasedOnStage(t *ldtest.T, order ldmigration.Execu } func tracksConsistencyIsDisabledByCheckRatio(t *ldtest.T, order ldmigration.ExecutionOrder) { + t.Specification("MIGRATIONS", "1.1.3.6", "checkRatio of 0 disables consistency check invocation") + handler := func(w http.ResponseWriter, req *http.Request) { w.WriteHeader(http.StatusOK) } testParams := []struct { @@ -1194,6 +1233,8 @@ func tracksConsistencyIsDisabledByCheckRatio(t *ldtest.T, order ldmigration.Exec } func tracksConsistencyIsDisabledIfCallbackFails(t *ldtest.T, order ldmigration.ExecutionOrder) { + t.Specification("MIGRATIONS", "1.3.1.4", "consistency not tracked when read callbacks fail") + handler := func(w http.ResponseWriter, req *http.Request) { w.WriteHeader(http.StatusConflict) } testParams := []struct { diff --git a/sdktests/server_side_persistence_base.go b/sdktests/server_side_persistence_base.go index 27cfc56e..7a614aca 100644 --- a/sdktests/server_side_persistence_base.go +++ b/sdktests/server_side_persistence_base.go @@ -42,6 +42,9 @@ const ( ) func doServerSidePersistentTests(t *ldtest.T) { + t.Specification("PS", "1.3", "dispatches persistence tests across Redis, Consul, and DynamoDB backends") + t.Specification("PS", "1.4", "dispatches persistence tests across Redis, Consul, and DynamoDB backends") + ranAtLeastOnce := false if t.Capabilities().Has(servicedef.CapabilityPersistentDataStoreRedis) { @@ -146,6 +149,15 @@ func newServerSidePersistentTests( } func (s *ServerSidePersistentTests) Run(t *ldtest.T) { + t.Specification("PS", "1.1.2", "cache TTL modes (off, positive, infinite)") + t.Specification("PS", "1.1.3", "store initialized with Init() containing all flag data") + t.Specification("PS", "1.1.4", "Get retrieves items from persistence when not cached") + t.Specification("PS", "1.1.6", "Upsert writes items to persistent store with version checking") + t.Specification("PS", "1.1.6.1", "successful upsert updates item cache (non-infinite TTL)") + t.Specification("PS", "1.1.6.5", "successful upsert updates item and all-items cache (infinite TTL)") + t.Specification("PS", "1.1.7", "IsInitialized gates reads until store is initialized") + t.Specification("PS", "1.2.1", "cached items expire after TTL elapses") + s.runWithEmptyStore(t, "uses default prefix", func(t *ldtest.T) { require.NoError(t, s.persistentStore.WriteMap(s.defaultPrefix, "features", s.initialFlags)) @@ -615,6 +627,7 @@ func (s *ServerSidePersistentTests) runWithEmptyStore(t *ldtest.T, testName stri }) } +// PS 1.1.7: validates the $inited key is set in the persistent store func (s *ServerSidePersistentTests) eventuallyRequireDataStoreInit(t *ldtest.T, prefix string) { h.RequireEventually(t, func() bool { value, _ := s.persistentStore.Get(prefix, persistenceInitedKey) @@ -622,6 +635,8 @@ func (s *ServerSidePersistentTests) eventuallyRequireDataStoreInit(t *ldtest.T, }, time.Second, time.Millisecond*20, persistenceInitedKey+" key was not set") } +// PS 1.1.3: validates flag data written to persistent store matches expectations +// PS 1.1.6: validates flag data written to persistent store matches expectations func (s *ServerSidePersistentTests) eventuallyValidateFlagData( t *ldtest.T, prefix string, matchers map[string]m.Matcher) { h.RequireEventually(t, func() bool { @@ -634,6 +649,7 @@ func (s *ServerSidePersistentTests) eventuallyValidateFlagData( }, time.Second, time.Millisecond*20, "flag data did not match") } +// PS 1.1.6: validates flag data is NOT written when version check rejects update func (s *ServerSidePersistentTests) neverValidateFlagData(t *ldtest.T, prefix string, matchers map[string]m.Matcher) { h.RequireNever(t, func() bool { data, err := s.persistentStore.GetMap(prefix, "features") @@ -645,6 +661,7 @@ func (s *ServerSidePersistentTests) neverValidateFlagData(t *ldtest.T, prefix st }, time.Second, time.Millisecond*20, "flag data did not match") } +// PS 1.1.3: builds matcher for flag key, version, and variations in store func basicFlagValidationMatcher(key string, version int, value string) m.Matcher { return m.AllOf( m.JSONProperty("key").Should(m.Equal(key)), @@ -653,6 +670,7 @@ func basicFlagValidationMatcher(key string, version int, value string) m.Matcher ) } +// PS 1.1.6: builds matcher for deleted flag tombstone in store func basicDeletedFlagValidationMatcher(key string, version int) m.Matcher { return m.AllOf( m.AnyOf(