From 7f0e345adb7f78552d457739097baf928d5d312e Mon Sep 17 00:00:00 2001 From: Aaron Zielstorff Date: Fri, 5 Jun 2026 21:13:25 +0200 Subject: [PATCH 1/5] Adds WORM evidence storage for history events and snapshots - Add WriteHistoryEvidence function to verify history ranges and store event/snapshot artifacts. - Introduce EventArtifactCandidate and SnapshotArtifactCandidate structures for managing artifacts. - Implement functions to load event and snapshot artifact candidates from the database. - Create HistoryManifest and associated functions for building and validating manifests. - Add ManifestJWSSigner for signing manifests with RSA keys. - Implement artifact storage and retrieval mechanisms for both events and snapshots. - Ensure proper error handling and validation throughout the evidence storage process. --- .github/workflows/common-tests.yml | 3 + .github/workflows/examples-smoke.yml | 6 + cmd/aasenvironmentservice/main.go | 3 + cmd/aasregistryservice/main.go | 3 + cmd/aasrepositoryservice/main.go | 3 + cmd/basyxconfigurationservice/main.go | 3 +- .../main.go | 3 + cmd/digitaltwinregistryservice/main.go | 3 + cmd/historyevidenceverifier/main.go | 254 +++++++++ cmd/submodelrepositoryservice/main.go | 3 + database/patches/1_1_3.sql | 141 +++++ docu/developer/aas_v3_2_runtime.md | 65 ++- docu/user/aas_api_v3_2.md | 59 +- .../BaSyxHistoryAuditGuardedExample/README.md | 96 +++- .../docker-compose.yml | 44 ++ go.mod | 18 + go.sum | 36 ++ internal/common/configuration.go | 276 +++++++++- internal/common/configuration_test.go | 118 ++++ internal/common/database.go | 2 +- internal/common/history/append.go | 26 +- internal/common/history/append_test.go | 223 ++++++++ internal/common/history/config.go | 96 ++++ internal/common/history/evidence_catalog.go | 310 +++++++++++ internal/common/history/evidence_hash.go | 43 ++ internal/common/history/evidence_store_s3.go | 342 ++++++++++++ internal/common/history/evidence_test.go | 266 +++++++++ internal/common/history/evidence_types.go | 178 ++++++ internal/common/history/evidence_verifier.go | 508 ++++++++++++++++++ internal/common/history/evidence_writer.go | 234 ++++++++ .../common/history/history_event_artifact.go | 301 +++++++++++ internal/common/history/manifest.go | 185 +++++++ internal/common/history/manifest_artifact.go | 204 +++++++ internal/common/history/snapshot_artifact.go | 215 ++++++++ internal/common/jws/jws.go | 40 ++ 35 files changed, 4268 insertions(+), 42 deletions(-) create mode 100644 cmd/historyevidenceverifier/main.go create mode 100644 database/patches/1_1_3.sql create mode 100644 internal/common/history/evidence_catalog.go create mode 100644 internal/common/history/evidence_hash.go create mode 100644 internal/common/history/evidence_store_s3.go create mode 100644 internal/common/history/evidence_test.go create mode 100644 internal/common/history/evidence_types.go create mode 100644 internal/common/history/evidence_verifier.go create mode 100644 internal/common/history/evidence_writer.go create mode 100644 internal/common/history/history_event_artifact.go create mode 100644 internal/common/history/manifest.go create mode 100644 internal/common/history/manifest_artifact.go create mode 100644 internal/common/history/snapshot_artifact.go diff --git a/.github/workflows/common-tests.yml b/.github/workflows/common-tests.yml index 427a157c2..64889c4eb 100644 --- a/.github/workflows/common-tests.yml +++ b/.github/workflows/common-tests.yml @@ -41,6 +41,9 @@ jobs: - name: Run common root package tests run: go test -count=1 -v -cover ./internal/common + - name: Run common history tests + run: go test -count=1 -v -cover ./internal/common/history + - name: Run common model grammar tests run: go test github.com/eclipse-basyx/basyx-go-components/internal/common/model/grammar -count=1 -v -cover diff --git a/.github/workflows/examples-smoke.yml b/.github/workflows/examples-smoke.yml index 8e2cf5b54..ccdd321b8 100644 --- a/.github/workflows/examples-smoke.yml +++ b/.github/workflows/examples-smoke.yml @@ -72,6 +72,12 @@ jobs: image_builds: | eclipsebasyx/companylookup-go:SNAPSHOT|./cmd/companylookupservice/Dockerfile eclipsebasyx/basyxconfigurationservice-go:SNAPSHOT|./cmd/basyxconfigurationservice/Dockerfile + - target_name: History Audit Guarded Evidence Example + compose_files: | + examples/BaSyxHistoryAuditGuardedExample/docker-compose.yml + image_builds: | + eclipsebasyx/aasenvironment-go:SNAPSHOT|./cmd/aasenvironmentservice/Dockerfile + eclipsebasyx/basyxconfigurationservice-go:SNAPSHOT|./cmd/basyxconfigurationservice/Dockerfile steps: - name: Checkout code diff --git a/cmd/aasenvironmentservice/main.go b/cmd/aasenvironmentservice/main.go index a1838bdc0..710fa4eb4 100644 --- a/cmd/aasenvironmentservice/main.go +++ b/cmd/aasenvironmentservice/main.go @@ -88,6 +88,9 @@ func runServer(ctx context.Context, configPath string) error { Immutability: cfg.History.Immutability, AuditIdentityMode: cfg.History.AuditIdentityMode, }) + if err = history.ConfigureEvidence(ctx, cfg.History.Evidence); err != nil { + return err + } registrySyncConfig, err := aasenvironment.NewRegistrySyncConfig( cfg.General.AASRegistryIntegration, diff --git a/cmd/aasregistryservice/main.go b/cmd/aasregistryservice/main.go index 1b3784663..813a2c40d 100644 --- a/cmd/aasregistryservice/main.go +++ b/cmd/aasregistryservice/main.go @@ -68,6 +68,9 @@ func runServer(ctx context.Context, configPath string) error { Immutability: cfg.History.Immutability, AuditIdentityMode: cfg.History.AuditIdentityMode, }) + if err = history.ConfigureEvidence(ctx, cfg.History.Evidence); err != nil { + return err + } commonmodel.SetSupportsSingularSupplementalSemanticId(cfg.General.SupportsSingularSupplementalSemanticId) r := chi.NewRouter() diff --git a/cmd/aasrepositoryservice/main.go b/cmd/aasrepositoryservice/main.go index e53e57567..0d1633f70 100644 --- a/cmd/aasrepositoryservice/main.go +++ b/cmd/aasrepositoryservice/main.go @@ -48,6 +48,9 @@ func runServer(ctx context.Context, configPath string) error { Immutability: cfg.History.Immutability, AuditIdentityMode: cfg.History.AuditIdentityMode, }) + if err = history.ConfigureEvidence(ctx, cfg.History.Evidence); err != nil { + return err + } if err = aasenvironment.ValidateStandaloneAASRepositoryRegistrySyncConfig(cfg); err != nil { return err diff --git a/cmd/basyxconfigurationservice/main.go b/cmd/basyxconfigurationservice/main.go index 76d279364..47d1d5e2e 100644 --- a/cmd/basyxconfigurationservice/main.go +++ b/cmd/basyxconfigurationservice/main.go @@ -60,7 +60,8 @@ func main() { schemInit.Register(sequences.NewSchemaPatch(execCtx, filepath.Join(patchBasePath, "1_0_2.sql"), "v1.0.2")) schemInit.Register(sequences.NewSchemaPatch(execCtx, filepath.Join(patchBasePath, "1_1_0.sql"), "v1.1.0")) schemInit.Register(sequences.NewSchemaPatch(execCtx, filepath.Join(patchBasePath, "1_1_1.sql"), "v1.1.1")) - schemInit.Register(sequences.NewSchemaPatch(execCtx, filepath.Join(patchBasePath, "1_1_2.sql"), common.CURRENT_DATABASE_VERSION)) + schemInit.Register(sequences.NewSchemaPatch(execCtx, filepath.Join(patchBasePath, "1_1_2.sql"), "v1.1.2")) + schemInit.Register(sequences.NewSchemaPatch(execCtx, filepath.Join(patchBasePath, "1_1_3.sql"), common.CURRENT_DATABASE_VERSION)) if err := schemInit.Execute(); err != nil { log.Printf("BASYXCFG-MAIN-EXECUTE: %v", err) diff --git a/cmd/conceptdescriptionrepositoryservice/main.go b/cmd/conceptdescriptionrepositoryservice/main.go index 47bed7c16..15e967c9b 100644 --- a/cmd/conceptdescriptionrepositoryservice/main.go +++ b/cmd/conceptdescriptionrepositoryservice/main.go @@ -67,6 +67,9 @@ func runServer(ctx context.Context, configPath string) error { Immutability: cfg.History.Immutability, AuditIdentityMode: cfg.History.AuditIdentityMode, }) + if err = history.ConfigureEvidence(ctx, cfg.History.Evidence); err != nil { + return err + } // Create Chi router r := chi.NewRouter() diff --git a/cmd/digitaltwinregistryservice/main.go b/cmd/digitaltwinregistryservice/main.go index 114473407..ac71b06d0 100644 --- a/cmd/digitaltwinregistryservice/main.go +++ b/cmd/digitaltwinregistryservice/main.go @@ -72,6 +72,9 @@ func runServer(ctx context.Context, configPath string) error { Immutability: cfg.History.Immutability, AuditIdentityMode: cfg.History.AuditIdentityMode, }) + if err = history.ConfigureEvidence(ctx, cfg.History.Evidence); err != nil { + return err + } commonmodel.SetSupportsSingularSupplementalSemanticId(cfg.General.SupportsSingularSupplementalSemanticId) // Digital Twin Registry always enables discovery integration. diff --git a/cmd/historyevidenceverifier/main.go b/cmd/historyevidenceverifier/main.go new file mode 100644 index 000000000..a220c2547 --- /dev/null +++ b/cmd/historyevidenceverifier/main.go @@ -0,0 +1,254 @@ +/******************************************************************************* +* Copyright (C) 2026 the Eclipse BaSyx Authors and Fraunhofer IESE +* +* Permission is hereby granted, free of charge, to any person obtaining +* a copy of this software and associated documentation files (the +* "Software"), to deal in the Software without restriction, including +* without limitation the rights to use, copy, modify, merge, publish, +* distribute, sublicense, and/or sell copies of the Software, and to +* permit persons to whom the Software is furnished to do so, subject to +* the following conditions: +* +* The above copyright notice and this permission notice shall be +* included in all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +* +* SPDX-License-Identifier: MIT +******************************************************************************/ + +// Package main implements the history evidence verifier and publisher CLI. +package main + +import ( + "context" + "database/sql" + "encoding/json" + "flag" + "fmt" + "os" + "os/signal" + "strings" + "syscall" + + "github.com/eclipse-basyx/basyx-go-components/internal/common" + "github.com/eclipse-basyx/basyx-go-components/internal/common/history" +) + +type cliOptions struct { + configPath string + historyTable string + identifier string + firstHistoryID int64 + lastHistoryID int64 + writeEvidence bool + manifestObjectKey string + manifestVersionID string + manifestSHA256 string + signerKeyID string +} + +func main() { + options := parseFlags() + flag.Parse() + ctx, stop := signal.NotifyContext(context.TODO(), os.Interrupt, syscall.SIGTERM) + err := run(ctx, *options) + stop() + if err != nil { + _, _ = fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func parseFlags() *cliOptions { + options := &cliOptions{} + flag.StringVar(&options.configPath, "config", "", "Path to BaSyx config YAML") + flag.StringVar(&options.historyTable, "table", "", "History table, for example aas_history or submodel_history") + flag.StringVar(&options.identifier, "identifier", "", "Optional entity identifier scope") + flag.Int64Var(&options.firstHistoryID, "from", 0, "First history_id in the range") + flag.Int64Var(&options.lastHistoryID, "to", 0, "Last history_id in the range") + flag.BoolVar(&options.writeEvidence, "write", false, "Write snapshot and manifest artifacts before verifying") + flag.StringVar(&options.manifestObjectKey, "manifest-object-key", "", "Stored manifest object key to verify") + flag.StringVar(&options.manifestVersionID, "manifest-version-id", "", "Stored manifest object version ID") + flag.StringVar(&options.manifestSHA256, "manifest-sha256", "", "Expected SHA-256 for the stored manifest object") + flag.StringVar(&options.signerKeyID, "signer-key-id", "", "Optional manifest signer key id") + return options +} + +func run(ctx context.Context, options cliOptions) error { + if err := validateCLIOptions(options); err != nil { + return err + } + cfg, err := common.LoadConfig(options.configPath, common.QUIET) + if err != nil { + return err + } + db, err := openDatabase(cfg) + if err != nil { + return err + } + defer func() { + _ = db.Close() + }() + if options.writeEvidence { + return writeEvidence(ctx, cfg, db, options) + } + return verifyEvidence(ctx, cfg, db, options) +} + +func validateCLIOptions(options cliOptions) error { + if strings.TrimSpace(options.historyTable) == "" { + return fmt.Errorf("HISTORY-EVIDENCE-CLI-TABLE -table is required") + } + if options.firstHistoryID < 1 { + return fmt.Errorf("HISTORY-EVIDENCE-CLI-FROM -from must be positive") + } + if options.lastHistoryID < options.firstHistoryID { + return fmt.Errorf("HISTORY-EVIDENCE-CLI-TO -to must be greater than or equal to -from") + } + return nil +} + +func openDatabase(cfg *common.Config) (*sql.DB, error) { + dsn := common.BuildPostgresDSN(cfg.Postgres) + if err := common.ValidateSchemaVersionByDSN(dsn, common.CURRENT_DATABASE_VERSION); err != nil { + return nil, err + } + db, err := common.NewDatabaseConnection(dsn) + if err != nil { + return nil, err + } + if cfg.Postgres.MaxOpenConnections > 0 { + db.SetMaxOpenConns(cfg.Postgres.MaxOpenConnections) + } + if cfg.Postgres.MaxIdleConnections > 0 { + db.SetMaxIdleConns(cfg.Postgres.MaxIdleConnections) + } + return db, nil +} + +func writeEvidence(ctx context.Context, cfg *common.Config, db *sql.DB, options cliOptions) error { + if !cfg.History.Evidence.Enabled { + return fmt.Errorf("HISTORY-EVIDENCE-CLI-DISABLED history.evidence.enabled must be true for -write") + } + store, err := newS3EvidenceStore(ctx, cfg) + if err != nil { + return err + } + signer, err := newManifestSigner(cfg, options.signerKeyID) + if err != nil { + return err + } + result, err := history.WriteHistoryEvidence(ctx, history.WriteHistoryEvidenceOptions{ + DB: db, + Store: store, + HistoryTable: options.historyTable, + Identifier: options.identifier, + FirstHistoryID: options.firstHistoryID, + LastHistoryID: options.lastHistoryID, + Signer: signer, + }) + if err != nil { + return err + } + return printJSON(result) +} + +func verifyEvidence(ctx context.Context, cfg *common.Config, db *sql.DB, options cliOptions) error { + verifyOptions := history.VerifyHistoryRangeOptions{ + HistoryTable: options.historyTable, + Identifier: options.identifier, + FirstHistoryID: options.firstHistoryID, + LastHistoryID: options.lastHistoryID, + } + store, err := optionalS3EvidenceStore(ctx, cfg) + if err != nil { + return err + } + verifyOptions.EvidenceStore = store + if strings.TrimSpace(options.manifestObjectKey) != "" { + if store == nil { + store, err = newS3EvidenceStore(ctx, cfg) + if err != nil { + return err + } + verifyOptions.EvidenceStore = store + } + ref := history.EvidenceReference{ + Provider: history.EvidenceProviderS3, + Bucket: cfg.History.Evidence.Bucket, + ObjectKey: strings.TrimSpace(options.manifestObjectKey), + VersionID: strings.TrimSpace(options.manifestVersionID), + } + object, err := store.GetArtifact(ctx, ref) + if err != nil { + return err + } + manifest, _, err := history.DecodeManifestArtifact(object.Data, object.ContentType) + if err != nil { + return err + } + verifyOptions.Manifest = &manifest + verifyOptions.EvidenceStore = store + verifyOptions.ManifestArtifactRef = ref + verifyOptions.ManifestArtifactHash = strings.TrimSpace(options.manifestSHA256) + } + report, err := history.VerifyHistoryRange(ctx, db, verifyOptions) + if err != nil { + return err + } + return printJSON(report) +} + +func newS3EvidenceStore(ctx context.Context, cfg *common.Config) (*history.S3EvidenceStore, error) { + if strings.ToLower(strings.TrimSpace(cfg.History.Evidence.Provider)) != history.EvidenceProviderS3 { + return nil, fmt.Errorf("HISTORY-EVIDENCE-CLI-PROVIDER history.evidence.provider must be s3") + } + return history.NewS3EvidenceStore(ctx, history.S3EvidenceStoreConfig{ + Bucket: cfg.History.Evidence.Bucket, + Prefix: cfg.History.Evidence.Prefix, + Region: cfg.History.Evidence.Region, + Endpoint: cfg.History.Evidence.Endpoint, + AccessKeyID: cfg.History.Evidence.AccessKeyID, + SecretAccessKey: cfg.History.Evidence.SecretAccessKey, + UsePathStyle: cfg.History.Evidence.UsePathStyle, + RetentionMode: cfg.History.Evidence.RetentionMode, + RetentionDays: cfg.History.Evidence.RetentionDays, + }) +} + +func optionalS3EvidenceStore(ctx context.Context, cfg *common.Config) (*history.S3EvidenceStore, error) { + if strings.ToLower(strings.TrimSpace(cfg.History.Evidence.Provider)) != history.EvidenceProviderS3 { + return nil, nil + } + if strings.TrimSpace(cfg.History.Evidence.Bucket) == "" { + return nil, nil + } + return newS3EvidenceStore(ctx, cfg) +} + +func newManifestSigner(cfg *common.Config, keyID string) (*history.ManifestJWSSigner, error) { + keyPath := strings.TrimSpace(cfg.History.Evidence.Signing.PrivateKeyPath) + if keyPath == "" { + keyPath = strings.TrimSpace(cfg.JWS.PrivateKeyPath) + } + if keyPath == "" { + return nil, nil + } + return history.NewManifestJWSSignerFromKeyFile(keyPath, keyID) +} + +func printJSON(value any) error { + encoded, err := json.MarshalIndent(value, "", " ") + if err != nil { + return fmt.Errorf("HISTORY-EVIDENCE-CLI-PRINTJSON %w", err) + } + _, err = fmt.Fprintln(os.Stdout, string(encoded)) + return err +} diff --git a/cmd/submodelrepositoryservice/main.go b/cmd/submodelrepositoryservice/main.go index 86b6c7ebe..ad377b6bb 100644 --- a/cmd/submodelrepositoryservice/main.go +++ b/cmd/submodelrepositoryservice/main.go @@ -49,6 +49,9 @@ func runServer(ctx context.Context, configPath string) error { Immutability: cfg.History.Immutability, AuditIdentityMode: cfg.History.AuditIdentityMode, }) + if err = history.ConfigureEvidence(ctx, cfg.History.Evidence); err != nil { + return err + } if err = aasenvironment.ValidateStandaloneSubmodelRepositoryRegistrySyncConfig(cfg); err != nil { return err diff --git a/database/patches/1_1_3.sql b/database/patches/1_1_3.sql new file mode 100644 index 000000000..70f8d16ef --- /dev/null +++ b/database/patches/1_1_3.sql @@ -0,0 +1,141 @@ +/******************************************************************************* +* Copyright (C) 2026 the Eclipse BaSyx Authors and Fraunhofer IESE +* +* Permission is hereby granted, free of charge, to any person obtaining +* a copy of this software and associated documentation files (the +* "Software"), to deal in the Software without restriction, including +* without limitation the rights to use, copy, modify, merge, publish, +* distribute, sublicense, and/or sell copies of the Software, and to +* permit persons to whom the Software is furnished to do so, subject to +* the following conditions: +* +* The above copyright notice and this permission notice shall be +* included in all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +* +* SPDX-License-Identifier: MIT +******************************************************************************/ + +-- ============================================================================ +-- Project : Eclipse BaSyx +-- Organization : Fraunhofer IESE +-- File Type : SQL Patch Script +-- Patch Version : 1.1.3 +-- Metamodel Ver. : 3.2 +-- ---------------------------------------------------------------------------- +-- Description: +-- Database patch script for WORM history evidence manifest catalogs. +-- +-- Copyright (c) Eclipse BaSyx Authors and Fraunhofer IESE +-- SPDX-License-Identifier: MIT +-- ============================================================================ + +-- ------------------------------------------ +-- History evidence manifests and artifacts +-- ------------------------------------------ + +CREATE TABLE IF NOT EXISTS history_evidence_manifests ( + manifest_id BIGSERIAL PRIMARY KEY, + manifest_version TEXT NOT NULL, + history_table TEXT NOT NULL, + identifier TEXT, + first_history_id BIGINT NOT NULL, + last_history_id BIGINT NOT NULL, + first_row_hash TEXT NOT NULL, + last_row_hash TEXT NOT NULL, + row_count BIGINT NOT NULL CHECK (row_count > 0), + range_digest TEXT NOT NULL, + generated_at TIMESTAMPTZ NOT NULL, + signature_state TEXT NOT NULL CHECK (signature_state IN ('signed', 'unsigned')), + signer_key_id TEXT, + signer_algorithm TEXT, + snapshot_reference_count INTEGER NOT NULL DEFAULT 0 CHECK (snapshot_reference_count >= 0), + provider TEXT NOT NULL, + bucket TEXT, + manifest_object_key TEXT NOT NULL, + manifest_object_version_id TEXT, + manifest_sha256 TEXT NOT NULL, + retention_mode TEXT, + retain_until TIMESTAMPTZ, + legal_hold BOOLEAN NOT NULL DEFAULT FALSE, + artifact_metadata JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CHECK (last_history_id >= first_history_id) +); + +CREATE TABLE IF NOT EXISTS history_evidence_artifacts ( + artifact_id BIGSERIAL PRIMARY KEY, + manifest_id BIGINT REFERENCES history_evidence_manifests(manifest_id), + artifact_type TEXT NOT NULL CHECK (artifact_type IN ('manifest', 'snapshot', 'history_event')), + history_table TEXT NOT NULL, + identifier TEXT, + history_id BIGINT, + row_hash TEXT, + content_hash TEXT, + provider TEXT NOT NULL, + bucket TEXT, + object_key TEXT NOT NULL, + object_version_id TEXT, + sha256 TEXT NOT NULL, + size_bytes BIGINT NOT NULL CHECK (size_bytes >= 0), + content_type TEXT NOT NULL, + retention_mode TEXT, + retain_until TIMESTAMPTZ, + legal_hold BOOLEAN NOT NULL DEFAULT FALSE, + artifact_metadata JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE UNIQUE INDEX IF NOT EXISTS ux_history_evidence_manifest_range + ON history_evidence_manifests( + history_table, + COALESCE(identifier, ''), + first_history_id, + last_history_id, + range_digest + ); + +CREATE UNIQUE INDEX IF NOT EXISTS ux_history_evidence_artifact_object + ON history_evidence_artifacts( + provider, + COALESCE(bucket, ''), + object_key, + COALESCE(object_version_id, '') + ); + +CREATE INDEX IF NOT EXISTS ix_history_evidence_manifest_table_identifier + ON history_evidence_manifests(history_table, identifier, last_history_id DESC); + +CREATE INDEX IF NOT EXISTS ix_history_evidence_artifact_manifest + ON history_evidence_artifacts(manifest_id, artifact_type); + +CREATE INDEX IF NOT EXISTS ix_history_evidence_artifact_history_row + ON history_evidence_artifacts(history_table, identifier, history_id); + +CREATE UNIQUE INDEX IF NOT EXISTS ux_history_evidence_artifact_history_event + ON history_evidence_artifacts( + history_table, + COALESCE(identifier, ''), + history_id, + COALESCE(row_hash, '') + ) + WHERE artifact_type = 'history_event'; + +-- Mark the schema as upgraded only after all schema objects completed +-- successfully. +UPDATE basyxsystem +SET schema_version = 'v1.1.3', + state = 'clean' +WHERE identifier = ( + SELECT identifier + FROM basyxsystem + ORDER BY identifier ASC + LIMIT 1 +); diff --git a/docu/developer/aas_v3_2_runtime.md b/docu/developer/aas_v3_2_runtime.md index baedc4b64..ccb808e61 100644 --- a/docu/developer/aas_v3_2_runtime.md +++ b/docu/developer/aas_v3_2_runtime.md @@ -43,6 +43,8 @@ Implemented history, recent-change, and signing runtime areas from the v3.2 Open - AAS Environment: `/serialization`, `/upload`, `/shell-descriptors/$recent-changes`, `/shells/$recent-changes`, `/shells/{aasIdentifier}/$history`, `/shells/{aasIdentifier}/$signed`, `/submodels/$recent-changes`, `/submodels/{submodelIdentifier}/$history`, `/submodels/{submodelIdentifier}/$signed`, `/submodels/{submodelIdentifier}/$value/$signed`, `/concept-descriptions/$recent-changes`, and the composed asynchronous operation result/status endpoints. - Migration `1_1_0.sql`: adds v3.2 timestamp columns and the enum migration for `Batch`. - Migration `1_1_1.sql`: adds history metadata and payload tables, indexes, and PostgreSQL mutation guards. +- Migration `1_1_2.sql`: adds snapshot-checkpoint indexes for diff-backed restore. +- Migration `1_1_3.sql`: adds WORM evidence manifest and artifact receipt catalogs. The Submodel Registry does not have a recent-changes endpoint in the official v3.2 profile currently used here. @@ -117,6 +119,7 @@ sequenceDiagram participant Live as Current normalized tables participant Metadata as *_history table participant Payload as *_history_payload table + participant Evidence as WORM EvidenceStore API->>Live: Apply typical current-state mutation in transaction API->>Metadata: Advisory lock(table, identifier) Metadata-->>API: Latest history row @@ -124,6 +127,11 @@ sequenceDiagram API->>API: Apply scoped snapshot mutation API->>Metadata: INSERT event metadata with previous_hash API->>Payload: INSERT snapshot or RFC 6902 diff payload + opt history.evidence.enabled + API->>Evidence: PUT history-event artifact before commit + Evidence-->>API: Evidence receipt + API->>Metadata: INSERT evidence artifact receipt + end API->>Live: Commit transaction ``` @@ -228,6 +236,31 @@ flowchart TD The guard is enabled when history is active and `history.immutability` is `postgres_guarded`. It blocks direct maintenance mutations as well as accidental application mutations. Enabling is monotonic during normal service startup: a runtime service can enable the database-wide guard, but it cannot disable an enabled guard. A service configured as unguarded fails fast when it encounters an already-enabled database guard. Services sharing one database can start concurrently when their configuration is consistent. Disabling guarded mode requires an explicit operator maintenance action. The guard is not equivalent to WORM storage: sufficiently privileged PostgreSQL operators can alter schema objects. +### WORM Evidence Manifests + +The default stronger-integrity architecture is: + +```text +PostgreSQL history tables -> hash chain -> synchronous history-event artifact -> signed manifest -> S3-compatible WORM object storage +``` + +The HTTP APIs are unchanged. When `history.evidence.enabled` is active, `history.mode` must be `api` or `audit`. The history append path writes one WORM `history_event` artifact synchronously before the surrounding PostgreSQL transaction can commit. The artifact stores the same history payload selected for PostgreSQL: either a full snapshot or an RFC 6902 diff according to `history.fullSnapshotInterval`. If the WORM write fails, the history append returns an error and the caller rolls back the PostgreSQL transaction. + +The `cmd/historyevidenceverifier` tool can additionally publish signed range manifests, backfill per-row `history_event` artifacts for existing rows, and publish checkpoint artifacts using the shared `EvidenceStore` interface. The current implementation includes an S3-compatible `EvidenceStore`; MinIO can be used for local or CI-style object-lock testing, while production deployments should use an S3-compatible WORM service with versioning/object lock configured by operations. + +For a selected history range, evidence publication: + +- verifies PostgreSQL payload hashes and per-identifier row-hash chains first; +- writes canonical `history_event` artifacts for every snapshot and diff row in the range; +- writes full snapshot checkpoint artifacts for every `payload_type = snapshot` row in the range; +- builds a canonical `HistoryManifest` containing first/last `history_id`, first/last `row_hash`, row count, ordered range digest, timestamp, signature metadata, and snapshot artifact references; +- signs the canonical manifest as compact RS256 JWS when an evidence signing key is configured, otherwise stores canonical JSON with `signature_state = unsigned`; +- writes object-store receipts to `history_evidence_manifests` and `history_evidence_artifacts`. + +Per-row `history_event` artifacts provide recovery evidence for every acknowledged write while evidence is enabled. With `history.fullSnapshotInterval: 5`, recovery from WORM replays up to four WORM-stored diff payloads after the latest WORM-stored snapshot event. Use `history.fullSnapshotInterval: 1` when each individual WORM event must be recoverable without replaying diffs. Automated PostgreSQL restore from WORM artifacts is not implemented in this ticket. + +Verification can compare PostgreSQL rows against the hash chain, verify every per-row `history_event` receipt and object hash, compare a stored manifest against the live range digest, and verify an object-store artifact hash. This detects missing, modified, reordered, or overwritten records when the expected receipts and object hashes are known. + ### Configuration Status | Setting | Current runtime behavior | @@ -239,11 +272,39 @@ The guard is enabled when history is active and `history.immutability` is `postg | `history.fullSnapshotInterval` | `1` stores all payloads as snapshots. Values greater than `1` store one full checkpoint plus up to `N-1` diff rows, with earlier checkpoints when the diff payload is not smaller than the snapshot payload. | | `history.immutability: none` | Keep PostgreSQL mutation guards disabled. | | `history.immutability: postgres_guarded` | Enable PostgreSQL mutation guards at service startup. | -| `history.immutability: external_anchor` | Fail fast until an external anchoring worker is implemented. | +| `history.immutability: external_anchor` | Reserved for a future `IntegrityAnchor` backend and still fails fast unless a real provider is implemented. | +| `history.evidence.enabled` | Enables fail-closed WORM history-event artifact writes. It does not change HTTP response shapes, but mutating requests fail if the evidence artifact cannot be stored. | +| `history.evidence.provider: s3` | Configures the S3-compatible `EvidenceStore`. Requires bucket, region, retention mode, and positive retention days. Endpoint override and path-style mode support MinIO-style tests. | +| `history.evidence.writeTimeoutSeconds` | Bounds synchronous WORM writes while the PostgreSQL transaction is open. Default is `10`. | +| `history.evidence.signing.privateKeyPath` | Optional RS256 manifest signing key. Falls back to `jws.privateKeyPath` when empty. | +| `history.integrityAnchor.provider: none` | Default. Non-`none` providers such as immudb, Rekor, Trillian, or timestamping services are reserved for later work. | | `history.auditIdentityMode` | Must remain `none`. Identity enrichment modes fail fast until middleware populates audit context. | | Active eventing, configured event sinks, or enabled outbox processing | Fail fast until outbox publishing is implemented. | -`AuditContext`, `ChangeEvent`, and the anchor interfaces remain extension points. Current runtime middleware does not populate `AuditContext`, and no external anchor client is invoked by the append path yet. +`AuditContext`, `ChangeEvent`, `EvidenceStore`, and `IntegrityAnchor` remain extension points. Current runtime middleware does not populate `AuditContext`, and no external ledger anchor client is invoked by the append path yet. + +Example verifier/publisher usage: + +```sh +go run ./cmd/historyevidenceverifier \ + -config ./config.yaml \ + -table submodel_history \ + -identifier 'https://example.com/submodels/1' \ + -from 1 \ + -to 25 \ + -write +``` + +```sh +go run ./cmd/historyevidenceverifier \ + -config ./config.yaml \ + -table submodel_history \ + -identifier 'https://example.com/submodels/1' \ + -from 1 \ + -to 25 \ + -manifest-object-key 'history-evidence/history-manifests/submodel_history/https:%2F%2Fexample.com%2Fsubmodels%2F1/1-25-...json' \ + -manifest-sha256 '' +``` ### Diff-Backed Storage diff --git a/docu/user/aas_api_v3_2.md b/docu/user/aas_api_v3_2.md index b2c87022b..6c3b38648 100644 --- a/docu/user/aas_api_v3_2.md +++ b/docu/user/aas_api_v3_2.md @@ -75,6 +75,14 @@ Environment variables: - `BASYX_HISTORY_FULL_SNAPSHOT_INTERVAL`: `1` stores every history row as a complete snapshot. Values greater than `1` store one full checkpoint followed by up to `N-1` RFC 6902 diff rows, with earlier checkpoints when the diff payload is not smaller than the snapshot payload. - `BASYX_HISTORY_IMMUTABILITY`: `none` or `postgres_guarded`. Default is `none`. - `BASYX_AUDIT_IDENTITY_MODE`: must remain `none`. Automatic identity enrichment is not implemented yet. +- `BASYX_HISTORY_EVIDENCE_ENABLED`: `true` enables fail-closed WORM history-event artifact writes. Requires `BASYX_HISTORY_MODE=api` or `audit`; mutating requests fail when the configured evidence artifact cannot be stored. +- `BASYX_HISTORY_EVIDENCE_PROVIDER`: `s3` for the S3-compatible evidence backend. +- `BASYX_HISTORY_EVIDENCE_BUCKET`, `BASYX_HISTORY_EVIDENCE_PREFIX`, `BASYX_HISTORY_EVIDENCE_REGION`, `BASYX_HISTORY_EVIDENCE_ENDPOINT`: object-store target settings. `endpoint` is useful for MinIO tests. +- `BASYX_HISTORY_EVIDENCE_ACCESS_KEY_ID`, `BASYX_HISTORY_EVIDENCE_SECRET_ACCESS_KEY`, `BASYX_HISTORY_EVIDENCE_PATH_STYLE`: optional S3-compatible credentials and path-style addressing. +- `BASYX_HISTORY_EVIDENCE_RETENTION_MODE`, `BASYX_HISTORY_EVIDENCE_RETENTION_DAYS`: required object-lock retention settings when evidence is enabled, such as `governance` plus a positive day count. +- `BASYX_HISTORY_EVIDENCE_WRITE_TIMEOUT_SECONDS`: timeout for synchronous WORM writes while the PostgreSQL transaction is open. Default is `10`. +- `BASYX_HISTORY_EVIDENCE_SIGNING_PRIVATE_KEY_PATH`: optional manifest signing key. When empty, evidence signing can use `jws.privateKeyPath`. +- `BASYX_HISTORY_INTEGRITY_ANCHOR_PROVIDER`: must remain `none` in this release. Ledger/timestamping providers are reserved for later work. Equivalent YAML: @@ -85,6 +93,24 @@ history: fullSnapshotInterval: 1 immutability: none auditIdentityMode: none + evidence: + enabled: false + provider: none + bucket: "" + prefix: basyx-history-evidence + region: us-east-1 + endpoint: "" + accessKeyId: "" + secretAccessKey: "" + pathStyle: false + retentionMode: "" + retentionDays: 0 + writeTimeoutSeconds: 10 + signing: + privateKeyPath: "" + required: false + integrityAnchor: + provider: none ``` Mode semantics: @@ -97,16 +123,45 @@ Current implementation status: - Runtime history rows are append-only, hash-chained event rows in `api` and `audit` mode. - Schema migration installs PostgreSQL guard triggers. `postgres_guarded` enables them at service startup. When enabled, `UPDATE`, `DELETE`, and `TRUNCATE` on history metadata and payload tables fail with `history tables are append-only`. -- `external_anchor`, non-zero `retentionDays`, and identity enrichment modes currently fail fast during configuration loading. Their runtime implementations are planned for later work. +- When WORM evidence is enabled, each acknowledged history append synchronously stores a `history_event` artifact in S3-compatible object storage. The artifact contains the same snapshot or diff payload that PostgreSQL stores. +- WORM evidence manifests, backfilled `history_event` artifacts, and additional snapshot checkpoint artifacts can be published to S3-compatible object storage with `cmd/historyevidenceverifier`. +- `external_anchor`, non-zero `retentionDays`, non-`none` integrity anchor providers, and identity enrichment modes currently fail fast during configuration loading. Their runtime implementations are planned for later work. - Audit metadata columns are present, but regular API middleware does not populate the audit context yet. - The implementation supports compliance-oriented deployments, but it does not by itself make a deployment legally compliant with any specific regulation. -Guarded PostgreSQL mode protects against normal accidental or unauthorized mutations through the application database user. PostgreSQL superusers or operators with permissions to alter triggers/functions can still bypass or remove this protection. Stronger guarantees require external anchoring, WORM storage, or a transparency-log style system. +Guarded PostgreSQL mode protects against normal accidental or unauthorized mutations through the application database user. PostgreSQL superusers or operators with permissions to alter triggers/functions can still bypass or remove this protection. Stronger tamper evidence and recovery evidence are provided by storing per-row history-event artifacts and signed history manifests in S3-compatible WORM storage such as AWS S3 Object Lock. MinIO Object Lock is useful for tests and local examples, but production deployments should use an operated WORM-capable object store with versioning and retention policy controls. + +WORM history-event artifacts provide recoverability for acknowledged writes while evidence is enabled. With `fullSnapshotInterval: 5`, for example, recovery from WORM starts from the latest WORM-stored snapshot event and replays up to four WORM-stored diff payloads. Use `fullSnapshotInterval: 1` when every acknowledged history row must be recoverable as a full WORM snapshot without diff replay. Automated PostgreSQL restore from WORM artifacts is not implemented in this release. The guard switch is database-wide. Configure all BaSyx services that share one database with the same history immutability mode. Runtime services may enable guarded mode, but normal service startup cannot disable an enabled database guard. A service configured as unguarded fails during startup when it encounters an already-enabled database guard. Disabling guarded mode is an explicit operator maintenance action. See `examples/BaSyxHistoryAuditGuardedExample` for a Docker Compose setup with audit history and `postgres_guarded` enabled. +Example evidence publication: + +```sh +go run ./cmd/historyevidenceverifier \ + -config ./config.yaml \ + -table submodel_history \ + -identifier 'https://example.com/submodels/1' \ + -from 1 \ + -to 25 \ + -write +``` + +Example verification against a stored manifest object: + +```sh +go run ./cmd/historyevidenceverifier \ + -config ./config.yaml \ + -table submodel_history \ + -identifier 'https://example.com/submodels/1' \ + -from 1 \ + -to 25 \ + -manifest-object-key '' \ + -manifest-sha256 '' +``` + ## What Activating Versioning Means When versioning is active, each supported identifiable create, update, or delete appends a new row to a dedicated history table. With `fullSnapshotInterval: 1`, every row stores a complete identifiable snapshot. With a larger interval, the runtime stores a full checkpoint followed by up to `N-1` RFC 6902 diffs against the previous reconstructed snapshot. diff --git a/examples/BaSyxHistoryAuditGuardedExample/README.md b/examples/BaSyxHistoryAuditGuardedExample/README.md index 067ae768f..707d5d079 100644 --- a/examples/BaSyxHistoryAuditGuardedExample/README.md +++ b/examples/BaSyxHistoryAuditGuardedExample/README.md @@ -5,7 +5,7 @@ SPDX-License-Identifier: MIT # BaSyx History Audit Guarded Example -This example shows how to start the AAS Environment Service with AAS v3.2 audit history and PostgreSQL mutation guards enabled. +This example shows how to start the AAS Environment Service with AAS v3.2 audit history, PostgreSQL mutation guards, and an S3-compatible WORM evidence test backend enabled. It also includes the BaSyx UI and preloads one AASX package at startup. For the complete feature description, see: @@ -17,9 +17,16 @@ The important settings are: - `BASYX_HISTORY_MODE=audit` - `BASYX_HISTORY_RETENTION_DAYS=0` -- `BASYX_HISTORY_FULL_SNAPSHOT_INTERVAL=1` +- `BASYX_HISTORY_FULL_SNAPSHOT_INTERVAL=5` - `BASYX_HISTORY_IMMUTABILITY=postgres_guarded` - `BASYX_AUDIT_IDENTITY_MODE=none` +- `BASYX_HISTORY_EVIDENCE_ENABLED=true` +- `BASYX_HISTORY_EVIDENCE_PROVIDER=s3` +- `BASYX_HISTORY_EVIDENCE_BUCKET=basyx-history-evidence` +- `BASYX_HISTORY_EVIDENCE_RETENTION_MODE=governance` +- `BASYX_HISTORY_EVIDENCE_RETENTION_DAYS=7` +- `BASYX_HISTORY_EVIDENCE_WRITE_TIMEOUT_SECONDS=10` +- `BASYX_HISTORY_INTEGRITY_ANCHOR_PROVIDER=none` ## Start @@ -32,14 +39,17 @@ This example is configured for local development: - `aas-environment` and `basyx_configuration` use local Docker `build` entries. - The published image lines are kept as comments so you can switch back quickly. -The configuration service initializes the `v1.1.1` schema first. The AAS Environment Service then enables the history guard switch at startup. +The configuration service initializes the current schema first. The AAS Environment Service then enables the history guard switch at startup. The MinIO init sidecar creates a versioned object-lock bucket for local evidence tests. -`BASYX_HISTORY_RETENTION_DAYS=0` is accepted as configuration, but automatic retention cleanup is not implemented yet. `BASYX_HISTORY_FULL_SNAPSHOT_INTERVAL=1` keeps the full-snapshot behavior; values greater than `1` store periodic full checkpoints with RFC 6902 diff rows between them. +`BASYX_HISTORY_RETENTION_DAYS=0` is accepted as configuration, but automatic retention cleanup is not implemented yet. `BASYX_HISTORY_FULL_SNAPSHOT_INTERVAL=5` stores periodic full checkpoints with RFC 6902 diff rows between them. +When evidence is enabled, history mode must be `api` or `audit`. Each successful history append writes a WORM `history_event` artifact to MinIO before PostgreSQL commits. If MinIO is unavailable or rejects the object write, the API mutation fails and the PostgreSQL transaction rolls back. ## UI And Preconfigured Data - UI: [http://localhost:3000](http://localhost:3000) - Backend: [http://localhost:8082](http://localhost:8082) +- MinIO API: [http://localhost:9000](http://localhost:9000) +- MinIO Console: [http://localhost:9001](http://localhost:9001), login `minioadmin` / `minioadmin` - Preconfiguration path: `./aas` mounted to `/app/preconfiguration` The file `aas/IESEDriveMotorDM3000.aasx` is imported automatically via `GENERAL_AAS_PRECONFIG_PATHS=/app/preconfiguration`. @@ -71,9 +81,85 @@ Normal API writes still append new history rows. The guard switch is database-wide. Configure every BaSyx runtime service sharing this database consistently. Normal service startup can enable the guard but cannot disable it. A service configured with `BASYX_HISTORY_IMMUTABILITY=none` fails fast when the database guard is already enabled. +## Inspect, Publish, And Verify WORM Evidence + +MinIO is included only as a local/test S3-compatible Object Lock backend. Production deployments should use an operated S3-compatible WORM service, such as [AWS S3 Object Lock](https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-lock.html), with versioning and retention configured by operations. MinIO Object Lock behavior is documented in the [MinIO object retention guide](https://minio.community/community/minio-object-store/administration/object-management/object-retention.html). + +Create or update an AAS or Submodel through the UI or API, then inspect the history rows: + +```bash +docker compose exec db psql -U admin -d basyxTestDB -c \ + "SELECT history_id, identifier, payload_type, row_hash FROM submodel_history ORDER BY history_id" +``` + +Inspect the automatic WORM history-event receipts. These are written during the API transaction and exist independently from later manifest publication: + +```bash +docker compose exec db psql -U admin -d basyxTestDB -c \ + "SELECT artifact_type, history_table, identifier, history_id, object_key, sha256 FROM history_evidence_artifacts WHERE artifact_type = 'history_event' ORDER BY artifact_id" +``` + +Publish a signed or unsigned manifest for a Submodel history range from the repository root. This verifies the PostgreSQL hash chain, writes range manifest/checkpoint artifacts, and records their receipts. Adjust `-identifier`, `-from`, and `-to` to values from the query above: + +```bash +POSTGRES_HOST=localhost \ +POSTGRES_PORT=5432 \ +POSTGRES_USER=admin \ +POSTGRES_PASSWORD=admin123 \ +POSTGRES_DBNAME=basyxTestDB \ +BASYX_HISTORY_MODE=audit \ +BASYX_HISTORY_IMMUTABILITY=postgres_guarded \ +BASYX_HISTORY_EVIDENCE_ENABLED=true \ +BASYX_HISTORY_EVIDENCE_PROVIDER=s3 \ +BASYX_HISTORY_EVIDENCE_BUCKET=basyx-history-evidence \ +BASYX_HISTORY_EVIDENCE_PREFIX=history-evidence \ +BASYX_HISTORY_EVIDENCE_REGION=us-east-1 \ +BASYX_HISTORY_EVIDENCE_ENDPOINT=http://localhost:9000 \ +BASYX_HISTORY_EVIDENCE_ACCESS_KEY_ID=minioadmin \ +BASYX_HISTORY_EVIDENCE_SECRET_ACCESS_KEY=minioadmin \ +BASYX_HISTORY_EVIDENCE_PATH_STYLE=true \ +BASYX_HISTORY_EVIDENCE_RETENTION_MODE=governance \ +BASYX_HISTORY_EVIDENCE_RETENTION_DAYS=7 \ +BASYX_HISTORY_EVIDENCE_WRITE_TIMEOUT_SECONDS=10 \ +go run ./cmd/historyevidenceverifier \ + -table submodel_history \ + -identifier '' \ + -from 1 \ + -to 5 \ + -write +``` + +The command prints the manifest, object-store receipt, per-row history-event receipts, snapshot checkpoint receipts, and PostgreSQL catalog id. The same object receipts are also stored in `history_evidence_manifests` and `history_evidence_artifacts`. + +Verify PostgreSQL against the per-row WORM history-event artifacts and, when supplied, a stored manifest: + +```bash +POSTGRES_HOST=localhost \ +POSTGRES_PORT=5432 \ +POSTGRES_USER=admin \ +POSTGRES_PASSWORD=admin123 \ +POSTGRES_DBNAME=basyxTestDB \ +BASYX_HISTORY_EVIDENCE_PROVIDER=s3 \ +BASYX_HISTORY_EVIDENCE_BUCKET=basyx-history-evidence \ +BASYX_HISTORY_EVIDENCE_REGION=us-east-1 \ +BASYX_HISTORY_EVIDENCE_ENDPOINT=http://localhost:9000 \ +BASYX_HISTORY_EVIDENCE_ACCESS_KEY_ID=minioadmin \ +BASYX_HISTORY_EVIDENCE_SECRET_ACCESS_KEY=minioadmin \ +BASYX_HISTORY_EVIDENCE_PATH_STYLE=true \ +go run ./cmd/historyevidenceverifier \ + -table submodel_history \ + -identifier '' \ + -from 1 \ + -to 5 \ + -manifest-object-key '' \ + -manifest-sha256 '' +``` + +History-event artifacts provide recovery evidence for acknowledged writes while evidence is enabled. With `BASYX_HISTORY_FULL_SNAPSHOT_INTERVAL=5`, recovery from WORM starts from the latest WORM-stored snapshot event and replays up to four WORM-stored diff payloads. Use `BASYX_HISTORY_FULL_SNAPSHOT_INTERVAL=1` when every history row must be recoverable as a full WORM snapshot without diff replay. Automated PostgreSQL restore from WORM artifacts is not implemented in this example. + ## Limitations -This mode protects against accidental or unauthorized mutation through normal database access. PostgreSQL superusers or operators with enough permissions can still alter triggers, functions, or schema objects. Use external anchoring, WORM storage, or transparency-log infrastructure when stronger tamper-resistance is required. +This mode protects against accidental or unauthorized mutation through normal database access. PostgreSQL superusers or operators with enough permissions can still alter triggers, functions, or schema objects. Signed manifests and WORM object storage strengthen tamper evidence, but they must be operated with appropriate retention, key management, monitoring, and backup procedures. For local development or tests that truncate tables, use `BASYX_HISTORY_IMMUTABILITY=none` from the start. To disable an already-enabled guard for maintenance, stop the guarded services and explicitly update `history_guard_config` as a database operator before cleanup. diff --git a/examples/BaSyxHistoryAuditGuardedExample/docker-compose.yml b/examples/BaSyxHistoryAuditGuardedExample/docker-compose.yml index 7fcea2a5a..0da5077d7 100644 --- a/examples/BaSyxHistoryAuditGuardedExample/docker-compose.yml +++ b/examples/BaSyxHistoryAuditGuardedExample/docker-compose.yml @@ -31,6 +31,19 @@ services: - BASYX_HISTORY_FULL_SNAPSHOT_INTERVAL=5 - BASYX_HISTORY_IMMUTABILITY=postgres_guarded - BASYX_AUDIT_IDENTITY_MODE=none + - BASYX_HISTORY_EVIDENCE_ENABLED=true + - BASYX_HISTORY_EVIDENCE_PROVIDER=s3 + - BASYX_HISTORY_EVIDENCE_BUCKET=basyx-history-evidence + - BASYX_HISTORY_EVIDENCE_PREFIX=history-evidence + - BASYX_HISTORY_EVIDENCE_REGION=us-east-1 + - BASYX_HISTORY_EVIDENCE_ENDPOINT=http://minio:9000 + - BASYX_HISTORY_EVIDENCE_ACCESS_KEY_ID=minioadmin + - BASYX_HISTORY_EVIDENCE_SECRET_ACCESS_KEY=minioadmin + - BASYX_HISTORY_EVIDENCE_PATH_STYLE=true + - BASYX_HISTORY_EVIDENCE_RETENTION_MODE=governance + - BASYX_HISTORY_EVIDENCE_RETENTION_DAYS=7 + - BASYX_HISTORY_EVIDENCE_WRITE_TIMEOUT_SECONDS=10 + - BASYX_HISTORY_INTEGRITY_ANCHOR_PROVIDER=none - ABAC_ENABLED=false - GENERAL_AASREGISTRYINTEGRATION=true - GENERAL_SUBMODELREGISTRYINTEGRATION=true @@ -42,6 +55,8 @@ services: depends_on: basyx_configuration: condition: service_completed_successfully + minio-init: + condition: service_completed_successfully restart: unless-stopped aas-ui: @@ -62,6 +77,8 @@ services: db: container_name: postgres_db_history_guarded image: postgres:18 + ports: + - "5432:5432" environment: POSTGRES_USER: admin POSTGRES_PASSWORD: admin123 @@ -93,3 +110,30 @@ services: depends_on: db: condition: service_healthy + + minio: + container_name: minio_history_evidence + image: quay.io/minio/minio:latest + command: ["server", "/data", "--console-address", ":9001"] + environment: + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin + ports: + - "9000:9000" + - "9001:9001" + restart: unless-stopped + + minio-init: + container_name: minio_history_evidence_init + image: quay.io/minio/mc:latest + depends_on: + minio: + condition: service_started + entrypoint: + - /bin/sh + - -c + - | + until mc alias set local http://minio:9000 minioadmin minioadmin; do sleep 2; done + mc mb --ignore-existing --with-lock local/basyx-history-evidence + mc version enable local/basyx-history-evidence + mc retention set --default governance 7d local/basyx-history-evidence diff --git a/go.mod b/go.mod index 12dff4009..a45644560 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,10 @@ require ( github.com/DATA-DOG/go-sqlmock v1.5.2 github.com/FriedJannik/aas-go-sdk v1.3.2 github.com/aas-core-works/aas-package3-golang v1.0.0 + github.com/aws/aws-sdk-go-v2 v1.41.12 + github.com/aws/aws-sdk-go-v2/config v1.32.23 + github.com/aws/aws-sdk-go-v2/credentials v1.19.22 + github.com/aws/aws-sdk-go-v2/service/s3 v1.103.2 github.com/coreos/go-oidc/v3 v3.18.0 github.com/doug-martin/goqu/v9 v9.19.0 github.com/go-chi/cors v1.2.2 @@ -20,6 +24,20 @@ require ( ) require ( + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.13 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.28 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.29 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.12 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.21 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.28 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.28 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.1.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.31.2 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.43.2 // indirect + github.com/aws/smithy-go v1.27.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/fsnotify/fsnotify v1.10.1 // indirect github.com/go-jose/go-jose/v4 v4.1.4 // indirect diff --git a/go.sum b/go.sum index 3f0e37dac..752284768 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,42 @@ github.com/FriedJannik/aas-go-sdk v1.3.2 h1:kb6TZc0ORINUoGPzmbCPagMRJZ/aMR747PIE github.com/FriedJannik/aas-go-sdk v1.3.2/go.mod h1:2Ui4RhCuJcLXyG26FhM7ti5WOALw9Iv3t7SnDu+tYu0= github.com/aas-core-works/aas-package3-golang v1.0.0 h1:Q+iLgs28eOYjRbZ9YbkxNLb4vjC/9CA2TZFwFagjbN4= github.com/aas-core-works/aas-package3-golang v1.0.0/go.mod h1:uDObjS4hNhBujH9L59QIAacokk7uAl9X0BbmNCkPtDs= +github.com/aws/aws-sdk-go-v2 v1.41.12 h1:DIKX2c31ekm9RA2D9FBj1EWXx++9AdAqRw+e78Tq2Ck= +github.com/aws/aws-sdk-go-v2 v1.41.12/go.mod h1:27+ACypSLljLAEKsCYOmrjKh83vuTRkuAe9Uv/3A4bg= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.13 h1:p1BBrg/Hhp6uK7zpejeI8QFXHJeC/mynzi04Sl03k9g= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.13/go.mod h1:8cIfkE9MDhkRZGpQ22aV6/lkYeYSozpz16Smrs5x4Ls= +github.com/aws/aws-sdk-go-v2/config v1.32.23 h1:PYDobtcsJXK6bQe9I8RQk6s19Bz3xa3xRU08Hy1Em3Y= +github.com/aws/aws-sdk-go-v2/config v1.32.23/go.mod h1:QID4dqUQVgEOYPKsPWd1sNWCCR2c5g7o3jeEtIXPOZU= +github.com/aws/aws-sdk-go-v2/credentials v1.19.22 h1:SHfH6wyPsEgG7fVsi5rQxWEt7tuIcN2PGhb1mTFv6tE= +github.com/aws/aws-sdk-go-v2/credentials v1.19.22/go.mod h1:54nO8lKD4aQPOntM/VTWjnR+DYzTwx0YkSMZMhAgewQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.28 h1:b+kcDejJrXc30zU/w8Tc9klISwaO5wh+6T0sMBdDoHM= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.28/go.mod h1:LnI62O9GnSv6GcuLXxOYqlq0C8EmxMcgnF6m7LdYuOY= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28 h1:Xf2j7NdVcUKomlZ4iihOP4AZ3Fzlr8h4yKpXeP+OFPg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28/go.mod h1:O8cDo1dW63jU7ki//kRe1z+tLGcpnD1jrouitsQddDw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28 h1:KqIfN9kpkKkcBqBbNpNGTIrXO6ExTUvFKvXkC+YAzVo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28/go.mod h1:uxtQiKvLtNS4iXVsH2McVD/ls8FKN/uUhe1hGxPjrw0= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.29 h1:VkE9FuzTQVjBBrnj4+oCdxCLFIz7aqLYKUCjtvxVcOs= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.29/go.mod h1:H32Z2Qth9b+9LqjyBsCnozMQ8H2N7YBUDVXwbs0iggg= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.12 h1:ZD2+BSw9vFsNlKYIasSNt3uDbjqqXIBcM13UJv/Lx2k= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.12/go.mod h1:Ms4zlcVBbXbiP7EVLhl+lgjvA/a7YphqQ3Ih3174EmI= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.21 h1:FsZxbPiVgEHYofziwfylouMki8b1Z7mI4CMU/7bhwBA= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.21/go.mod h1:Mmm30OV+JLXYQUcbSd84THnv3P5JtjhVDujLwMqRG0U= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.28 h1:axj4mEDletwKmTm/9jR+DkIMmCfcn5vE4jBMAAN+3Vg= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.28/go.mod h1:3Aaz69M0jqfSHLKqxgolgUBFT4hpwSNc7DzC95orEi8= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.28 h1:li8rTZAAb22g4UsxbjwMdaNVWbgVcDzPqI7nDTI+mF4= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.28/go.mod h1:/brXioSGIMEdcBFoubpSdmighSVp6poP+mma/wB7iHA= +github.com/aws/aws-sdk-go-v2/service/s3 v1.103.2 h1:b4ikkRk22T4xYkEgaWc3Voe+3xbt5YbbFhNehOWyUiY= +github.com/aws/aws-sdk-go-v2/service/s3 v1.103.2/go.mod h1:Gp7eHZ0NZ8ZK5RXpoIUp/C8OeAmJqpCgdwEK1D/QOek= +github.com/aws/aws-sdk-go-v2/service/signin v1.1.4 h1:YcpVyIPLCbiypN6KSphijN5fC7DDjX114SqA7prnnxg= +github.com/aws/aws-sdk-go-v2/service/signin v1.1.4/go.mod h1:5ZICS++oFTRPfa1GsBqFDWX/8WamZ/QQOcCzIuU/zLw= +github.com/aws/aws-sdk-go-v2/service/sso v1.31.2 h1:ySNWu7TPmj5fKFIa1GYvX+Ddxd5ccruqC20aMNuyWDM= +github.com/aws/aws-sdk-go-v2/service/sso v1.31.2/go.mod h1:A+U9luAOwFeB1kseyWCITVg7/NntoPebCFR9pQ4ch9A= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.5 h1:KSzGGqfk39O+WU3OEyYbx6F7sLDQCqxlOJ+2IksfK6U= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.5/go.mod h1:ATs88lXDeQB6CZOgQ5BIl9JbYS+EsCWUSDyff6L/oVo= +github.com/aws/aws-sdk-go-v2/service/sts v1.43.2 h1:RTO7mmGyedgnNmcPh3yQizNfc6GKoV5iqfdJavuf9vw= +github.com/aws/aws-sdk-go-v2/service/sts v1.43.2/go.mod h1:fBhUZXDin9YYqhcpOMjIcpdik25rVwWyxLdPH1RZd9s= +github.com/aws/smithy-go v1.27.1 h1:4T340VFndXtADGF52gYa1POyL7s9E4Z1OeZ1hCscIw8= +github.com/aws/smithy-go v1.27.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/coreos/go-oidc/v3 v3.18.0 h1:V9orjXynvu5wiC9SemFTWnG4F45v403aIcjWo0d41+A= github.com/coreos/go-oidc/v3 v3.18.0/go.mod h1:DYCf24+ncYi+XkIH97GY1+dqoRlbaSI26KVTCI9SrY4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/internal/common/configuration.go b/internal/common/configuration.go index dbf9bbd40..de0eaa52e 100644 --- a/internal/common/configuration.go +++ b/internal/common/configuration.go @@ -80,6 +80,21 @@ var DefaultConfig = struct { HistoryConfigFullSnapshotInterval int HistoryConfigImmutability string HistoryConfigAuditIdentityMode string + HistoryEvidenceEnabled bool + HistoryEvidenceProvider string + HistoryEvidenceBucket string + HistoryEvidencePrefix string + HistoryEvidenceRegion string + HistoryEvidenceEndpoint string + HistoryEvidenceAccessKeyID string + HistoryEvidenceSecretAccessKey string + HistoryEvidenceUsePathStyle bool + HistoryEvidenceRetentionMode string + HistoryEvidenceRetentionDays int + HistoryEvidenceWriteTimeoutSeconds int + HistoryEvidenceSigningPrivateKey string + HistoryEvidenceSigningRequired bool + HistoryIntegrityAnchorProvider string EventingEnabled bool EventingFormat string EventingSinks []string @@ -117,6 +132,21 @@ var DefaultConfig = struct { HistoryConfigFullSnapshotInterval: 1, HistoryConfigImmutability: "none", HistoryConfigAuditIdentityMode: "none", + HistoryEvidenceEnabled: false, + HistoryEvidenceProvider: "none", + HistoryEvidenceBucket: "", + HistoryEvidencePrefix: "basyx-history-evidence", + HistoryEvidenceRegion: "us-east-1", + HistoryEvidenceEndpoint: "", + HistoryEvidenceAccessKeyID: "", + HistoryEvidenceSecretAccessKey: "", + HistoryEvidenceUsePathStyle: false, + HistoryEvidenceRetentionMode: "", + HistoryEvidenceRetentionDays: 0, + HistoryEvidenceWriteTimeoutSeconds: 10, + HistoryEvidenceSigningPrivateKey: "", + HistoryEvidenceSigningRequired: false, + HistoryIntegrityAnchorProvider: "none", EventingEnabled: false, EventingFormat: "cloudevents", EventingSinks: []string{}, @@ -194,11 +224,41 @@ type JWSConfig struct { // HistoryConfig contains history and audit configuration. type HistoryConfig struct { - Mode string `mapstructure:"mode" yaml:"mode" json:"mode"` // off|api|audit - RetentionDays int `mapstructure:"retentionDays" yaml:"retentionDays" json:"retentionDays"` // 0 = keep forever - FullSnapshotInterval int `mapstructure:"fullSnapshotInterval" yaml:"fullSnapshotInterval" json:"fullSnapshotInterval"` // 1 = every history row is a full snapshot - Immutability string `mapstructure:"immutability" yaml:"immutability" json:"immutability"` // none|postgres_guarded|external_anchor - AuditIdentityMode string `mapstructure:"auditIdentityMode" yaml:"auditIdentityMode" json:"auditIdentityMode"` // none|minimal|extended + Mode string `mapstructure:"mode" yaml:"mode" json:"mode"` // off|api|audit + RetentionDays int `mapstructure:"retentionDays" yaml:"retentionDays" json:"retentionDays"` // 0 = keep forever + FullSnapshotInterval int `mapstructure:"fullSnapshotInterval" yaml:"fullSnapshotInterval" json:"fullSnapshotInterval"` // 1 = every history row is a full snapshot + Immutability string `mapstructure:"immutability" yaml:"immutability" json:"immutability"` // none|postgres_guarded|external_anchor + AuditIdentityMode string `mapstructure:"auditIdentityMode" yaml:"auditIdentityMode" json:"auditIdentityMode"` // none|minimal|extended + Evidence HistoryEvidenceConfig `mapstructure:"evidence" yaml:"evidence" json:"evidence"` + IntegrityAnchor HistoryIntegrityAnchorConfig `mapstructure:"integrityAnchor" yaml:"integrityAnchor" json:"integrityAnchor"` +} + +// HistoryEvidenceConfig configures WORM-compatible evidence artifact storage. +type HistoryEvidenceConfig struct { + Enabled bool `mapstructure:"enabled" yaml:"enabled" json:"enabled"` + Provider string `mapstructure:"provider" yaml:"provider" json:"provider"` // none|s3 + Bucket string `mapstructure:"bucket" yaml:"bucket" json:"bucket"` + Prefix string `mapstructure:"prefix" yaml:"prefix" json:"prefix"` + Region string `mapstructure:"region" yaml:"region" json:"region"` + Endpoint string `mapstructure:"endpoint" yaml:"endpoint" json:"endpoint"` + AccessKeyID string `mapstructure:"accessKeyId" yaml:"accessKeyId" json:"accessKeyId"` + SecretAccessKey string `mapstructure:"secretAccessKey" yaml:"secretAccessKey" json:"secretAccessKey"` + UsePathStyle bool `mapstructure:"pathStyle" yaml:"pathStyle" json:"pathStyle"` + RetentionMode string `mapstructure:"retentionMode" yaml:"retentionMode" json:"retentionMode"` // governance|compliance + RetentionDays int `mapstructure:"retentionDays" yaml:"retentionDays" json:"retentionDays"` + WriteTimeoutSec int `mapstructure:"writeTimeoutSeconds" yaml:"writeTimeoutSeconds" json:"writeTimeoutSeconds"` + Signing HistoryEvidenceSigningConfig `mapstructure:"signing" yaml:"signing" json:"signing"` +} + +// HistoryEvidenceSigningConfig configures optional manifest signing. +type HistoryEvidenceSigningConfig struct { + PrivateKeyPath string `mapstructure:"privateKeyPath" yaml:"privateKeyPath" json:"privateKeyPath"` + Required bool `mapstructure:"required" yaml:"required" json:"required"` +} + +// HistoryIntegrityAnchorConfig reserves future ledger/timestamping backends. +type HistoryIntegrityAnchorConfig struct { + Provider string `mapstructure:"provider" yaml:"provider" json:"provider"` // none today; immudb/Rekor/Trillian later } // EventingConfig reserves future-compatible eventing configuration. @@ -380,24 +440,57 @@ func applyHistoryEnvOverrides(cfg *Config) { if value, ok := lookupTrimmedEnv("BASYX_HISTORY_MODE"); ok { cfg.History.Mode = value } - if value, ok := lookupTrimmedEnv("BASYX_HISTORY_RETENTION_DAYS"); ok { - var retention int - if _, err := fmt.Sscanf(value, "%d", &retention); err == nil { - cfg.History.RetentionDays = retention - } - } - if value, ok := lookupTrimmedEnv("BASYX_HISTORY_FULL_SNAPSHOT_INTERVAL"); ok { - var interval int - if _, err := fmt.Sscanf(value, "%d", &interval); err == nil { - cfg.History.FullSnapshotInterval = interval - } - } + applyIntEnv("BASYX_HISTORY_RETENTION_DAYS", func(value int) { cfg.History.RetentionDays = value }) + applyIntEnv("BASYX_HISTORY_FULL_SNAPSHOT_INTERVAL", func(value int) { cfg.History.FullSnapshotInterval = value }) if value, ok := lookupTrimmedEnv("BASYX_HISTORY_IMMUTABILITY"); ok { cfg.History.Immutability = value } if value, ok := lookupTrimmedEnv("BASYX_AUDIT_IDENTITY_MODE"); ok { cfg.History.AuditIdentityMode = value } + applyHistoryEvidenceEnvOverrides(cfg) + if value, ok := lookupTrimmedEnv("BASYX_HISTORY_INTEGRITY_ANCHOR_PROVIDER"); ok { + cfg.History.IntegrityAnchor.Provider = value + } +} + +func applyHistoryEvidenceEnvOverrides(cfg *Config) { + applyBoolEnv("BASYX_HISTORY_EVIDENCE_ENABLED", func(value bool) { cfg.History.Evidence.Enabled = value }) + if value, ok := lookupTrimmedEnv("BASYX_HISTORY_EVIDENCE_PROVIDER"); ok { + cfg.History.Evidence.Provider = value + } + if value, ok := lookupTrimmedEnv("BASYX_HISTORY_EVIDENCE_BUCKET"); ok { + cfg.History.Evidence.Bucket = value + } + if value, ok := lookupTrimmedEnv("BASYX_HISTORY_EVIDENCE_PREFIX"); ok { + cfg.History.Evidence.Prefix = value + } + if value, ok := lookupTrimmedEnv("BASYX_HISTORY_EVIDENCE_REGION"); ok { + cfg.History.Evidence.Region = value + } + if value, ok := lookupTrimmedEnv("BASYX_HISTORY_EVIDENCE_ENDPOINT"); ok { + cfg.History.Evidence.Endpoint = value + } + if value, ok := lookupTrimmedEnv("BASYX_HISTORY_EVIDENCE_ACCESS_KEY_ID"); ok { + cfg.History.Evidence.AccessKeyID = value + } + if value, ok := lookupTrimmedEnv("BASYX_HISTORY_EVIDENCE_SECRET_ACCESS_KEY"); ok { + cfg.History.Evidence.SecretAccessKey = value + } + if value, ok := lookupTrimmedEnv("BASYX_HISTORY_EVIDENCE_SECRET_KEY"); ok { + cfg.History.Evidence.SecretAccessKey = value + } + applyBoolEnv("BASYX_HISTORY_EVIDENCE_PATH_STYLE", func(value bool) { cfg.History.Evidence.UsePathStyle = value }) + applyBoolEnv("BASYX_HISTORY_EVIDENCE_USE_PATH_STYLE", func(value bool) { cfg.History.Evidence.UsePathStyle = value }) + if value, ok := lookupTrimmedEnv("BASYX_HISTORY_EVIDENCE_RETENTION_MODE"); ok { + cfg.History.Evidence.RetentionMode = value + } + applyIntEnv("BASYX_HISTORY_EVIDENCE_RETENTION_DAYS", func(value int) { cfg.History.Evidence.RetentionDays = value }) + applyIntEnv("BASYX_HISTORY_EVIDENCE_WRITE_TIMEOUT_SECONDS", func(value int) { cfg.History.Evidence.WriteTimeoutSec = value }) + if value, ok := lookupTrimmedEnv("BASYX_HISTORY_EVIDENCE_SIGNING_PRIVATE_KEY_PATH"); ok { + cfg.History.Evidence.Signing.PrivateKeyPath = value + } + applyBoolEnv("BASYX_HISTORY_EVIDENCE_SIGNING_REQUIRED", func(value bool) { cfg.History.Evidence.Signing.Required = value }) } func applyEventingEnvOverrides(cfg *Config) { @@ -426,6 +519,13 @@ func validateHistoryAndEventingConfig(cfg *Config) error { return fmt.Errorf("CONFIG-HISTORY-NIL configuration must not be nil") } + if err := validateHistoryConfig(cfg); err != nil { + return err + } + return validateEventingConfig(cfg.Eventing) +} + +func validateHistoryConfig(cfg *Config) error { switch strings.ToLower(strings.TrimSpace(cfg.History.Mode)) { case "off", "api", "audit": default: @@ -440,7 +540,9 @@ func validateHistoryAndEventingConfig(cfg *Config) error { switch strings.ToLower(strings.TrimSpace(cfg.History.Immutability)) { case "none", "postgres_guarded": case "external_anchor": - return fmt.Errorf("CONFIG-HISTORY-ANCHOR history.immutability external_anchor is not implemented yet") + if normalizeProvider(cfg.History.IntegrityAnchor.Provider) == "none" { + return fmt.Errorf("CONFIG-HISTORY-ANCHOR history.immutability external_anchor requires a configured history.integrityAnchor.provider") + } default: return fmt.Errorf("CONFIG-HISTORY-IMMUTABILITY unsupported history.immutability %q", cfg.History.Immutability) } @@ -451,12 +553,100 @@ func validateHistoryAndEventingConfig(cfg *Config) error { default: return fmt.Errorf("CONFIG-HISTORY-AUDITIDENTITY unsupported history.auditIdentityMode %q", cfg.History.AuditIdentityMode) } - if cfg.Eventing.Enabled || cfg.Eventing.OutboxEnabled || len(cfg.Eventing.Sinks) > 0 { + if err := validateHistoryEvidenceConfig(cfg); err != nil { + return err + } + return validateIntegrityAnchorConfig(cfg.History.IntegrityAnchor) +} + +func validateHistoryEvidenceConfig(cfg *Config) error { + evidence := cfg.History.Evidence + provider := normalizeProvider(evidence.Provider) + switch provider { + case "none", "s3": + default: + return fmt.Errorf("CONFIG-HISTORY-EVIDENCE-PROVIDER unsupported history.evidence.provider %q", evidence.Provider) + } + if !evidence.Enabled { + return nil + } + if strings.EqualFold(strings.TrimSpace(cfg.History.Mode), "off") { + return fmt.Errorf("CONFIG-HISTORY-EVIDENCE-MODE history.evidence.enabled requires history.mode api or audit") + } + if provider == "none" { + return fmt.Errorf("CONFIG-HISTORY-EVIDENCE-PROVIDER history.evidence.enabled requires history.evidence.provider") + } + if provider != "s3" { + return fmt.Errorf("CONFIG-HISTORY-EVIDENCE-PROVIDER history.evidence.provider %q is not implemented", evidence.Provider) + } + if strings.TrimSpace(evidence.Bucket) == "" { + return fmt.Errorf("CONFIG-HISTORY-EVIDENCE-BUCKET history.evidence.bucket is required for S3 evidence") + } + if strings.TrimSpace(evidence.Region) == "" { + return fmt.Errorf("CONFIG-HISTORY-EVIDENCE-REGION history.evidence.region is required for S3 evidence") + } + if (strings.TrimSpace(evidence.AccessKeyID) == "") != (strings.TrimSpace(evidence.SecretAccessKey) == "") { + return fmt.Errorf("CONFIG-HISTORY-EVIDENCE-CREDENTIALS history.evidence accessKeyId and secretAccessKey must be configured together") + } + if evidence.RetentionDays < 0 { + return fmt.Errorf("CONFIG-HISTORY-EVIDENCE-RETENTIONDAYS history.evidence.retentionDays must not be negative") + } + retentionMode := strings.ToLower(strings.TrimSpace(evidence.RetentionMode)) + if retentionMode == "" { + return fmt.Errorf("CONFIG-HISTORY-EVIDENCE-RETENTIONMODE history.evidence.retentionMode is required when evidence is enabled") + } + switch retentionMode { + case "governance", "compliance": + default: + return fmt.Errorf("CONFIG-HISTORY-EVIDENCE-RETENTIONMODE unsupported history.evidence.retentionMode %q", evidence.RetentionMode) + } + if evidence.RetentionDays < 1 { + return fmt.Errorf("CONFIG-HISTORY-EVIDENCE-RETENTION history.evidence.retentionDays must be at least 1 when evidence is enabled") + } + if evidence.WriteTimeoutSec < 1 { + return fmt.Errorf("CONFIG-HISTORY-EVIDENCE-TIMEOUT history.evidence.writeTimeoutSeconds must be at least 1") + } + if evidence.Signing.Required && effectiveHistoryEvidenceSigningKeyPath(cfg) == "" { + return fmt.Errorf("CONFIG-HISTORY-EVIDENCE-SIGNING history.evidence.signing.required needs history.evidence.signing.privateKeyPath or jws.privateKeyPath") + } + return nil +} + +func validateIntegrityAnchorConfig(cfg HistoryIntegrityAnchorConfig) error { + switch normalizeProvider(cfg.Provider) { + case "none": + return nil + default: + return fmt.Errorf("CONFIG-HISTORY-INTEGRITYANCHOR-NOTIMPLEMENTED history.integrityAnchor.provider %q is reserved for a future backend", cfg.Provider) + } +} + +func validateEventingConfig(cfg EventingConfig) error { + if cfg.Enabled || cfg.OutboxEnabled || len(cfg.Sinks) > 0 { return fmt.Errorf("CONFIG-EVENTING-NOTIMPLEMENTED eventing publishing and outbox processing are not implemented yet") } return nil } +func normalizeProvider(provider string) string { + normalized := strings.ToLower(strings.TrimSpace(provider)) + if normalized == "" { + return "none" + } + return normalized +} + +func effectiveHistoryEvidenceSigningKeyPath(cfg *Config) string { + if cfg == nil { + return "" + } + keyPath := strings.TrimSpace(cfg.History.Evidence.Signing.PrivateKeyPath) + if keyPath != "" { + return keyPath + } + return strings.TrimSpace(cfg.JWS.PrivateKeyPath) +} + func lookupTrimmedEnv(key string) (string, bool) { value, ok := os.LookupEnv(key) if !ok { @@ -465,6 +655,25 @@ func lookupTrimmedEnv(key string) (string, bool) { return strings.TrimSpace(value), true } +func applyBoolEnv(key string, assign func(bool)) { + value, ok := lookupTrimmedEnv(key) + if !ok { + return + } + assign(strings.EqualFold(value, "true")) +} + +func applyIntEnv(key string, assign func(int)) { + value, ok := lookupTrimmedEnv(key) + if !ok { + return + } + var parsed int + if _, err := fmt.Sscanf(value, "%d", &parsed); err == nil { + assign(parsed) + } +} + func parseCommaSeparated(rawValue string) []string { parts := strings.Split(rawValue, ",") values := make([]string, 0, len(parts)) @@ -571,6 +780,21 @@ func setDefaults(v *viper.Viper) { v.SetDefault("history.fullSnapshotInterval", DefaultConfig.HistoryConfigFullSnapshotInterval) v.SetDefault("history.immutability", "none") v.SetDefault("history.auditIdentityMode", "none") + v.SetDefault("history.evidence.enabled", DefaultConfig.HistoryEvidenceEnabled) + v.SetDefault("history.evidence.provider", DefaultConfig.HistoryEvidenceProvider) + v.SetDefault("history.evidence.bucket", DefaultConfig.HistoryEvidenceBucket) + v.SetDefault("history.evidence.prefix", DefaultConfig.HistoryEvidencePrefix) + v.SetDefault("history.evidence.region", DefaultConfig.HistoryEvidenceRegion) + v.SetDefault("history.evidence.endpoint", DefaultConfig.HistoryEvidenceEndpoint) + v.SetDefault("history.evidence.accessKeyId", DefaultConfig.HistoryEvidenceAccessKeyID) + v.SetDefault("history.evidence.secretAccessKey", DefaultConfig.HistoryEvidenceSecretAccessKey) + v.SetDefault("history.evidence.pathStyle", DefaultConfig.HistoryEvidenceUsePathStyle) + v.SetDefault("history.evidence.retentionMode", DefaultConfig.HistoryEvidenceRetentionMode) + v.SetDefault("history.evidence.retentionDays", DefaultConfig.HistoryEvidenceRetentionDays) + v.SetDefault("history.evidence.writeTimeoutSeconds", DefaultConfig.HistoryEvidenceWriteTimeoutSeconds) + v.SetDefault("history.evidence.signing.privateKeyPath", DefaultConfig.HistoryEvidenceSigningPrivateKey) + v.SetDefault("history.evidence.signing.required", DefaultConfig.HistoryEvidenceSigningRequired) + v.SetDefault("history.integrityAnchor.provider", DefaultConfig.HistoryIntegrityAnchorProvider) // Eventing placeholders v.SetDefault("eventing.enabled", false) @@ -704,6 +928,20 @@ func PrintConfiguration(cfg *Config) { add("Full Snapshot Interval", cfg.History.FullSnapshotInterval, DefaultConfig.HistoryConfigFullSnapshotInterval) add("Immutability", cfg.History.Immutability, DefaultConfig.HistoryConfigImmutability) add("Audit Identity Mode", cfg.History.AuditIdentityMode, DefaultConfig.HistoryConfigAuditIdentityMode) + add("Evidence Enabled", cfg.History.Evidence.Enabled, DefaultConfig.HistoryEvidenceEnabled) + add("Evidence Provider", cfg.History.Evidence.Provider, DefaultConfig.HistoryEvidenceProvider) + if cfg.History.Evidence.Enabled { + add("Evidence Bucket", cfg.History.Evidence.Bucket, DefaultConfig.HistoryEvidenceBucket) + add("Evidence Prefix", cfg.History.Evidence.Prefix, DefaultConfig.HistoryEvidencePrefix) + add("Evidence Region", cfg.History.Evidence.Region, DefaultConfig.HistoryEvidenceRegion) + add("Evidence Endpoint", cfg.History.Evidence.Endpoint, DefaultConfig.HistoryEvidenceEndpoint) + add("Evidence Path Style", cfg.History.Evidence.UsePathStyle, DefaultConfig.HistoryEvidenceUsePathStyle) + add("Evidence Retention Mode", cfg.History.Evidence.RetentionMode, DefaultConfig.HistoryEvidenceRetentionMode) + add("Evidence Retention Days", cfg.History.Evidence.RetentionDays, DefaultConfig.HistoryEvidenceRetentionDays) + add("Evidence Write Timeout Seconds", cfg.History.Evidence.WriteTimeoutSec, DefaultConfig.HistoryEvidenceWriteTimeoutSeconds) + add("Evidence Signing Required", cfg.History.Evidence.Signing.Required, DefaultConfig.HistoryEvidenceSigningRequired) + } + add("Integrity Anchor Provider", cfg.History.IntegrityAnchor.Provider, DefaultConfig.HistoryIntegrityAnchorProvider) // Eventing lines = append(lines, "🔹 Eventing:") diff --git a/internal/common/configuration_test.go b/internal/common/configuration_test.go index 3ee4b5c7f..eb202160a 100644 --- a/internal/common/configuration_test.go +++ b/internal/common/configuration_test.go @@ -166,6 +166,12 @@ func TestLoadConfigAppliesHistoryAndEventingDefaults(t *testing.T) { withUnsetEnv(t, "BASYX_HISTORY_FULL_SNAPSHOT_INTERVAL") withUnsetEnv(t, "BASYX_HISTORY_IMMUTABILITY") withUnsetEnv(t, "BASYX_AUDIT_IDENTITY_MODE") + withUnsetEnv(t, "BASYX_HISTORY_EVIDENCE_ENABLED") + withUnsetEnv(t, "BASYX_HISTORY_EVIDENCE_PROVIDER") + withUnsetEnv(t, "BASYX_HISTORY_EVIDENCE_BUCKET") + withUnsetEnv(t, "BASYX_HISTORY_EVIDENCE_REGION") + withUnsetEnv(t, "BASYX_HISTORY_EVIDENCE_WRITE_TIMEOUT_SECONDS") + withUnsetEnv(t, "BASYX_HISTORY_INTEGRITY_ANCHOR_PROVIDER") withUnsetEnv(t, "BASYX_EVENTING_ENABLED") withUnsetEnv(t, "BASYX_EVENTING_FORMAT") withUnsetEnv(t, "BASYX_EVENTING_SINKS") @@ -193,6 +199,9 @@ func TestLoadConfigAppliesHistoryAndEventingDefaults(t *testing.T) { if cfg.History.AuditIdentityMode != "none" { t.Fatalf("expected default audit identity mode none, got %q", cfg.History.AuditIdentityMode) } + if cfg.History.Evidence.Enabled || cfg.History.Evidence.Provider != "none" || cfg.History.IntegrityAnchor.Provider != "none" { + t.Fatalf("unexpected history evidence defaults: %+v", cfg.History) + } if cfg.Eventing.Enabled || cfg.Eventing.Format != "cloudevents" || cfg.Eventing.TopicPrefix != "basyx" { t.Fatalf("unexpected eventing defaults: %+v", cfg.Eventing) } @@ -216,6 +225,38 @@ func TestLoadConfigAppliesSupportedBasyxHistoryEnvOverrides(t *testing.T) { } } +func TestLoadConfigAppliesHistoryEvidenceEnvOverrides(t *testing.T) { + t.Setenv("BASYX_HISTORY_MODE", "audit") + t.Setenv("BASYX_HISTORY_IMMUTABILITY", "postgres_guarded") + t.Setenv("BASYX_AUDIT_IDENTITY_MODE", "none") + t.Setenv("BASYX_HISTORY_EVIDENCE_ENABLED", "true") + t.Setenv("BASYX_HISTORY_EVIDENCE_PROVIDER", "s3") + t.Setenv("BASYX_HISTORY_EVIDENCE_BUCKET", "history-evidence") + t.Setenv("BASYX_HISTORY_EVIDENCE_PREFIX", "test-prefix") + t.Setenv("BASYX_HISTORY_EVIDENCE_REGION", "us-east-1") + t.Setenv("BASYX_HISTORY_EVIDENCE_ENDPOINT", "http://minio:9000") + t.Setenv("BASYX_HISTORY_EVIDENCE_ACCESS_KEY_ID", "minio") + t.Setenv("BASYX_HISTORY_EVIDENCE_SECRET_ACCESS_KEY", "minio123") + t.Setenv("BASYX_HISTORY_EVIDENCE_PATH_STYLE", "true") + t.Setenv("BASYX_HISTORY_EVIDENCE_RETENTION_MODE", "governance") + t.Setenv("BASYX_HISTORY_EVIDENCE_RETENTION_DAYS", "7") + t.Setenv("BASYX_HISTORY_EVIDENCE_WRITE_TIMEOUT_SECONDS", "12") + t.Setenv("BASYX_HISTORY_INTEGRITY_ANCHOR_PROVIDER", "none") + captureLogOutput(t) + + cfg, err := LoadConfig("", NORMAL) + if err != nil { + t.Fatalf("unexpected config load error: %v", err) + } + + if !cfg.History.Evidence.Enabled || cfg.History.Evidence.Provider != "s3" || cfg.History.Evidence.Bucket != "history-evidence" || cfg.History.Evidence.Prefix != "test-prefix" { + t.Fatalf("unexpected evidence env override result: %+v", cfg.History.Evidence) + } + if !cfg.History.Evidence.UsePathStyle || cfg.History.Evidence.RetentionMode != "governance" || cfg.History.Evidence.RetentionDays != 7 || cfg.History.Evidence.WriteTimeoutSec != 12 { + t.Fatalf("unexpected evidence retention/path-style result: %+v", cfg.History.Evidence) + } +} + func TestValidateHistoryAndEventingConfigAcceptsDiffBackedSnapshotInterval(t *testing.T) { cfg := Config{History: HistoryConfig{Mode: "api", FullSnapshotInterval: 10, Immutability: "none", AuditIdentityMode: "none"}} @@ -224,6 +265,33 @@ func TestValidateHistoryAndEventingConfigAcceptsDiffBackedSnapshotInterval(t *te } } +func TestValidateHistoryAndEventingConfigAcceptsCompleteS3EvidenceConfig(t *testing.T) { + cfg := Config{ + JWS: JWSConfig{PrivateKeyPath: "fallback-key.pem"}, + History: HistoryConfig{ + Mode: "audit", + FullSnapshotInterval: 5, + Immutability: "postgres_guarded", + AuditIdentityMode: "none", + Evidence: HistoryEvidenceConfig{ + Enabled: true, + Provider: "s3", + Bucket: "history-evidence", + Region: "us-east-1", + RetentionMode: "governance", + RetentionDays: 1, + WriteTimeoutSec: 10, + Signing: HistoryEvidenceSigningConfig{Required: true}, + }, + IntegrityAnchor: HistoryIntegrityAnchorConfig{Provider: "none"}, + }, + } + + if err := validateHistoryAndEventingConfig(&cfg); err != nil { + t.Fatalf("expected complete S3 evidence config to be accepted, got %v", err) + } +} + func TestValidateHistoryAndEventingConfigRejectsUnsupportedFeatures(t *testing.T) { tests := []struct { name string @@ -241,6 +309,56 @@ func TestValidateHistoryAndEventingConfigRejectsUnsupportedFeatures(t *testing.T name: "external anchor", config: Config{History: HistoryConfig{Mode: "api", FullSnapshotInterval: 1, Immutability: "external_anchor", AuditIdentityMode: "none"}}, }, + { + name: "incomplete evidence", + config: Config{History: HistoryConfig{ + Mode: "api", + FullSnapshotInterval: 1, + Immutability: "none", + AuditIdentityMode: "none", + Evidence: HistoryEvidenceConfig{Enabled: true, Provider: "s3", Region: "us-east-1"}, + }}, + }, + { + name: "evidence with history off", + config: Config{History: HistoryConfig{ + Mode: "off", + FullSnapshotInterval: 1, + Immutability: "none", + AuditIdentityMode: "none", + Evidence: HistoryEvidenceConfig{Enabled: true, Provider: "s3", Bucket: "history-evidence", Region: "us-east-1"}, + }}, + }, + { + name: "evidence without retention", + config: Config{History: HistoryConfig{ + Mode: "api", + FullSnapshotInterval: 1, + Immutability: "none", + AuditIdentityMode: "none", + Evidence: HistoryEvidenceConfig{Enabled: true, Provider: "s3", Bucket: "history-evidence", Region: "us-east-1", WriteTimeoutSec: 10}, + }}, + }, + { + name: "evidence without write timeout", + config: Config{History: HistoryConfig{ + Mode: "api", + FullSnapshotInterval: 1, + Immutability: "none", + AuditIdentityMode: "none", + Evidence: HistoryEvidenceConfig{Enabled: true, Provider: "s3", Bucket: "history-evidence", Region: "us-east-1", RetentionMode: "governance", RetentionDays: 1}, + }}, + }, + { + name: "reserved integrity anchor provider", + config: Config{History: HistoryConfig{ + Mode: "api", + FullSnapshotInterval: 1, + Immutability: "none", + AuditIdentityMode: "none", + IntegrityAnchor: HistoryIntegrityAnchorConfig{Provider: "immudb"}, + }}, + }, { name: "audit identity", config: Config{History: HistoryConfig{Mode: "api", FullSnapshotInterval: 1, Immutability: "none", AuditIdentityMode: "minimal"}}, diff --git a/internal/common/database.go b/internal/common/database.go index 235e81079..85287444f 100644 --- a/internal/common/database.go +++ b/internal/common/database.go @@ -13,7 +13,7 @@ import ( ) const ( - CURRENT_DATABASE_VERSION = "v1.1.2" + CURRENT_DATABASE_VERSION = "v1.1.3" cleanSchemaState = "clean" ) diff --git a/internal/common/history/append.go b/internal/common/history/append.go index c8eaec0c9..d95fece7a 100644 --- a/internal/common/history/append.go +++ b/internal/common/history/append.go @@ -86,16 +86,16 @@ func AppendVersionTx(ctx context.Context, tx *sql.Tx, table string, identifier s if hashErr != nil { return hashErr } - return appendSnapshotVersionWithPreviousHashTx(ctx, tx, table, identifier, changeType, snapshot, deleted, previousHash) + return appendSnapshotVersionWithPreviousHashTx(ctx, tx, table, identifier, changeType, snapshot, deleted, previousHash, cfg) } latest, err := latestVersionTx(ctx, tx, table, identifier) if err != nil && !common.IsErrNotFound(err) { return err } if common.IsErrNotFound(err) { - return appendVersionWithLatestTx(ctx, tx, table, identifier, changeType, snapshot, deleted, nil) + return appendVersionWithLatestTx(ctx, tx, table, identifier, changeType, snapshot, deleted, nil, cfg) } - return appendVersionWithLatestTx(ctx, tx, table, identifier, changeType, snapshot, deleted, &latest) + return appendVersionWithLatestTx(ctx, tx, table, identifier, changeType, snapshot, deleted, &latest, cfg) } // AppendMutatedVersionTx restores the latest snapshot, applies mutate, and appends the result. @@ -128,7 +128,8 @@ func AppendVersionTx(ctx context.Context, tx *sql.Tx, table string, identifier s // return err // } func AppendMutatedVersionTx(ctx context.Context, tx *sql.Tx, table string, identifier string, changeType string, mutate SnapshotMutator) error { - if ActiveConfig().Mode == ModeOff { + cfg := ActiveConfig() + if cfg.Mode == ModeOff { return nil } @@ -160,7 +161,7 @@ func AppendMutatedVersionTx(ctx context.Context, tx *sql.Tx, table string, ident } previousVersion := latest previousVersion.snapshot = previousSnapshot - return appendVersionWithLatestTx(ctx, tx, table, identifier, changeType, currentSnapshot, false, &previousVersion) + return appendVersionWithLatestTx(ctx, tx, table, identifier, changeType, currentSnapshot, false, &previousVersion, cfg) } func validateAppendInputs(tx *sql.Tx, identifier string) (string, error) { @@ -174,8 +175,8 @@ func validateAppendInputs(tx *sql.Tx, identifier string) (string, error) { return identifier, nil } -func appendVersionWithLatestTx(ctx context.Context, tx *sql.Tx, table string, identifier string, changeType string, snapshot map[string]any, deleted bool, latest *latestVersion) error { - payload, err := buildHistoryPayload(snapshot, latest, ActiveConfig()) +func appendVersionWithLatestTx(ctx context.Context, tx *sql.Tx, table string, identifier string, changeType string, snapshot map[string]any, deleted bool, latest *latestVersion, cfg Config) error { + payload, err := buildHistoryPayload(snapshot, latest, cfg) if err != nil { return err } @@ -183,18 +184,18 @@ func appendVersionWithLatestTx(ctx context.Context, tx *sql.Tx, table string, id if latest != nil { previousHash = latest.rowHash } - return insertHistoryVersionTx(ctx, tx, table, identifier, changeType, snapshot, deleted, payload, previousHash) + return insertHistoryVersionTx(ctx, tx, table, identifier, changeType, snapshot, deleted, payload, previousHash, cfg) } -func appendSnapshotVersionWithPreviousHashTx(ctx context.Context, tx *sql.Tx, table string, identifier string, changeType string, snapshot map[string]any, deleted bool, previousHash string) error { +func appendSnapshotVersionWithPreviousHashTx(ctx context.Context, tx *sql.Tx, table string, identifier string, changeType string, snapshot map[string]any, deleted bool, previousHash string, cfg Config) error { payload, err := buildSnapshotPayload(snapshot) if err != nil { return err } - return insertHistoryVersionTx(ctx, tx, table, identifier, changeType, snapshot, deleted, payload, previousHash) + return insertHistoryVersionTx(ctx, tx, table, identifier, changeType, snapshot, deleted, payload, previousHash, cfg) } -func insertHistoryVersionTx(ctx context.Context, tx *sql.Tx, table string, identifier string, changeType string, snapshot map[string]any, deleted bool, payload historyPayload, previousHash string) error { +func insertHistoryVersionTx(ctx context.Context, tx *sql.Tx, table string, identifier string, changeType string, snapshot map[string]any, deleted bool, payload historyPayload, previousHash string, cfg Config) error { payloadTable, err := historyPayloadTable(table) if err != nil { return err @@ -234,6 +235,7 @@ func insertHistoryVersionTx(ctx context.Context, tx *sql.Tx, table string, ident if err != nil { return common.NewInternalServerError("HISTORY-APPEND-ROWHASH " + err.Error()) } + event.RowHash = rowHash createdAt, updatedAt := administrationTimestamps(snapshot) insertQuery, insertArgs, err := goqu.Insert(table).Rows(goqu.Record{ "identifier": identifier, @@ -286,7 +288,7 @@ func insertHistoryVersionTx(ctx context.Context, tx *sql.Tx, table string, ident if _, err = tx.ExecContext(ctx, payloadQuery, payloadArgs...); err != nil { return common.NewInternalServerError("HISTORY-APPEND-EXECPAYLOADINSERT " + err.Error()) } - return nil + return publishHistoryEventEvidenceTx(ctx, tx, cfg, table, historyID, event, payload, createdAt, updatedAt) } func lockIdentifierTx(ctx context.Context, tx *sql.Tx, table string, identifier string) error { diff --git a/internal/common/history/append_test.go b/internal/common/history/append_test.go index 80728aa2c..de0d3de47 100644 --- a/internal/common/history/append_test.go +++ b/internal/common/history/append_test.go @@ -29,6 +29,7 @@ import ( "context" "database/sql" "encoding/json" + "errors" "strings" "testing" "time" @@ -99,6 +100,143 @@ func TestAppendVersionTxSnapshotIntervalOneUsesPreviousHashOnly(t *testing.T) { require.NoError(t, mock.ExpectationsWereMet()) } +func TestAppendVersionTxWritesEvidenceArtifactBeforeCommitWhenEnabled(t *testing.T) { + store := &recordingEvidenceStore{} + t.Cleanup(func() { + Configure(Config{Mode: ModeOff, Immutability: ImmutabilityNone, AuditIdentityMode: AuditIdentityNone}) + }) + Configure(Config{ + Mode: ModeAPI, + FullSnapshotInterval: 1, + Immutability: ImmutabilityNone, + AuditIdentityMode: AuditIdentityNone, + EvidenceEnabled: true, + EvidenceProvider: EvidenceProviderS3, + EvidenceStore: store, + }) + + db, mock, err := sqlmock.New() + require.NoError(t, err) + defer func() { + _ = db.Close() + }() + + mock.ExpectBegin() + mock.ExpectExec(`SELECT pg_advisory_xact_lock`). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectQuery(`SELECT "row_hash" FROM "aas_history"`). + WillReturnError(sql.ErrNoRows) + mock.ExpectQuery(`INSERT INTO "aas_history".*RETURNING "history_id"`). + WillReturnRows(sqlmock.NewRows([]string{"history_id"}).AddRow(1)) + mock.ExpectExec(`INSERT INTO "aas_history_payload".*"snapshot"`). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec(`INSERT INTO "history_evidence_artifacts"`). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectCommit() + + tx, err := db.Begin() + require.NoError(t, err) + err = AppendVersionTx(context.Background(), tx, TableAAS, "aas-1", ChangeCreated, map[string]any{"id": "aas-1"}, false) + require.NoError(t, err) + require.NoError(t, tx.Commit()) + + require.Len(t, store.artifacts, 1) + artifact := store.artifacts[0] + require.Equal(t, EvidenceArtifactHistoryEvent, artifact.ArtifactType) + require.Contains(t, artifact.ObjectKey, "history-events/aas_history/aas-1/1-") + var artifactPayload map[string]any + require.NoError(t, decodeJSONPreservingNumbers(artifact.Data, &artifactPayload)) + require.Equal(t, PayloadTypeSnapshot, artifactPayload["payload_type"]) + require.Equal(t, "aas-1", artifactPayload["identifier"]) + payload, ok := artifactPayload["payload"].(map[string]any) + require.True(t, ok) + require.Equal(t, "aas-1", payload["id"]) + require.NoError(t, mock.ExpectationsWereMet()) +} + +func TestAppendVersionTxRollsBackWhenEvidenceStoreFails(t *testing.T) { + store := &recordingEvidenceStore{err: errors.New("object storage unavailable")} + t.Cleanup(func() { + Configure(Config{Mode: ModeOff, Immutability: ImmutabilityNone, AuditIdentityMode: AuditIdentityNone}) + }) + Configure(Config{ + Mode: ModeAPI, + FullSnapshotInterval: 1, + Immutability: ImmutabilityNone, + AuditIdentityMode: AuditIdentityNone, + EvidenceEnabled: true, + EvidenceProvider: EvidenceProviderS3, + EvidenceStore: store, + }) + + db, mock, err := sqlmock.New() + require.NoError(t, err) + defer func() { + _ = db.Close() + }() + + mock.ExpectBegin() + mock.ExpectExec(`SELECT pg_advisory_xact_lock`). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectQuery(`SELECT "row_hash" FROM "aas_history"`). + WillReturnError(sql.ErrNoRows) + mock.ExpectQuery(`INSERT INTO "aas_history".*RETURNING "history_id"`). + WillReturnRows(sqlmock.NewRows([]string{"history_id"}).AddRow(1)) + mock.ExpectExec(`INSERT INTO "aas_history_payload".*"snapshot"`). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectRollback() + + tx, err := db.Begin() + require.NoError(t, err) + err = AppendVersionTx(context.Background(), tx, TableAAS, "aas-1", ChangeCreated, map[string]any{"id": "aas-1"}, false) + require.ErrorContains(t, err, "HISTORY-EVIDENCE-APPEND-PUTARTIFACT") + require.NoError(t, tx.Rollback()) + require.NoError(t, mock.ExpectationsWereMet()) +} + +func TestAppendVersionTxRollsBackWhenEvidenceReceiptCatalogFails(t *testing.T) { + store := &recordingEvidenceStore{} + t.Cleanup(func() { + Configure(Config{Mode: ModeOff, Immutability: ImmutabilityNone, AuditIdentityMode: AuditIdentityNone}) + }) + Configure(Config{ + Mode: ModeAPI, + FullSnapshotInterval: 1, + Immutability: ImmutabilityNone, + AuditIdentityMode: AuditIdentityNone, + EvidenceEnabled: true, + EvidenceProvider: EvidenceProviderS3, + EvidenceStore: store, + }) + + db, mock, err := sqlmock.New() + require.NoError(t, err) + defer func() { + _ = db.Close() + }() + + mock.ExpectBegin() + mock.ExpectExec(`SELECT pg_advisory_xact_lock`). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectQuery(`SELECT "row_hash" FROM "aas_history"`). + WillReturnError(sql.ErrNoRows) + mock.ExpectQuery(`INSERT INTO "aas_history".*RETURNING "history_id"`). + WillReturnRows(sqlmock.NewRows([]string{"history_id"}).AddRow(1)) + mock.ExpectExec(`INSERT INTO "aas_history_payload".*"snapshot"`). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec(`INSERT INTO "history_evidence_artifacts"`). + WillReturnError(errors.New("catalog insert failed")) + mock.ExpectRollback() + + tx, err := db.Begin() + require.NoError(t, err) + err = AppendVersionTx(context.Background(), tx, TableAAS, "aas-1", ChangeCreated, map[string]any{"id": "aas-1"}, false) + require.ErrorContains(t, err, "HISTORY-EVIDENCE-EVENTCATALOG-INSERT") + require.NoError(t, tx.Rollback()) + require.Len(t, store.artifacts, 1) + require.NoError(t, mock.ExpectationsWereMet()) +} + func TestBuildLockIdentifierQueryUsesPostgresPlaceholders(t *testing.T) { query, args, err := buildLockIdentifierQuery(TableAAS, "aas-1") @@ -210,6 +348,58 @@ func TestAppendVersionTxStoresDiffWhenIntervalAllows(t *testing.T) { require.NoError(t, mock.ExpectationsWereMet()) } +func TestAppendVersionTxWritesDiffEvidenceArtifactWhenDiffPayloadIsSelected(t *testing.T) { + store := &recordingEvidenceStore{} + t.Cleanup(func() { + Configure(Config{Mode: ModeOff, Immutability: ImmutabilityNone, AuditIdentityMode: AuditIdentityNone}) + }) + Configure(Config{ + Mode: ModeAPI, + FullSnapshotInterval: 3, + Immutability: ImmutabilityNone, + AuditIdentityMode: AuditIdentityNone, + EvidenceEnabled: true, + EvidenceProvider: EvidenceProviderS3, + EvidenceStore: store, + }) + + db, mock, err := sqlmock.New() + require.NoError(t, err) + defer func() { + _ = db.Close() + }() + + largeUnchangedValue := strings.Repeat("unchanged-", 40) + baseSnapshot := map[string]any{"id": "aas-1", "idShort": "before", "description": largeUnchangedValue} + + mock.ExpectBegin() + mock.ExpectExec(`SELECT pg_advisory_xact_lock`). + WillReturnResult(sqlmock.NewResult(0, 1)) + expectLatestSnapshotRestore(mock, TableAAS, "aas_history_payload", "aas-1", 1, baseSnapshot, false) + mock.ExpectQuery(`INSERT INTO "aas_history".*RETURNING "history_id"`). + WillReturnRows(sqlmock.NewRows([]string{"history_id"}).AddRow(2)) + mock.ExpectExec(`INSERT INTO "aas_history_payload".*"diff"`). + WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectExec(`INSERT INTO "history_evidence_artifacts"`). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectCommit() + + tx, err := db.Begin() + require.NoError(t, err) + err = AppendVersionTx(context.Background(), tx, TableAAS, "aas-1", ChangeUpdated, map[string]any{"id": "aas-1", "idShort": "after", "description": largeUnchangedValue}, false) + require.NoError(t, err) + require.NoError(t, tx.Commit()) + + require.Len(t, store.artifacts, 1) + var artifactPayload map[string]any + require.NoError(t, decodeJSONPreservingNumbers(store.artifacts[0].Data, &artifactPayload)) + require.Equal(t, PayloadTypeDiff, artifactPayload["payload_type"]) + diff, ok := artifactPayload["payload"].([]any) + require.True(t, ok) + require.NotEmpty(t, diff) + require.NoError(t, mock.ExpectationsWereMet()) +} + func TestAppendVersionTxStoresSnapshotWhenDiffWouldBeLarger(t *testing.T) { t.Cleanup(func() { Configure(Config{Mode: ModeOff, Immutability: ImmutabilityNone, AuditIdentityMode: AuditIdentityNone}) @@ -347,3 +537,36 @@ func TestNullableTimestampAcceptsSharedISO8601Formats(t *testing.T) { require.True(t, ok) require.Equal(t, time.Date(2026, 5, 28, 12, 30, 0, 123456789, time.UTC), utcTimestamp) } + +type recordingEvidenceStore struct { + err error + artifacts []EvidenceArtifact +} + +func (store *recordingEvidenceStore) PutArtifact(_ context.Context, artifact EvidenceArtifact) (*EvidenceReceipt, error) { + if store.err != nil { + return nil, store.err + } + store.artifacts = append(store.artifacts, artifact) + return &EvidenceReceipt{ + Reference: EvidenceReference{ + Provider: EvidenceProviderS3, + Bucket: "history-evidence", + ObjectKey: artifact.ObjectKey, + VersionID: "version-1", + }, + SHA256: SHA256Hex(artifact.Data), + SizeBytes: int64(len(artifact.Data)), + ContentType: artifact.ContentType, + StoredAt: time.Date(2026, 5, 28, 12, 0, 0, 0, time.UTC), + Metadata: artifact.Metadata, + }, nil +} + +func (store *recordingEvidenceStore) GetArtifact(_ context.Context, _ EvidenceReference) (*EvidenceObject, error) { + return nil, errors.New("not implemented") +} + +func (store *recordingEvidenceStore) VerifyArtifact(_ context.Context, _ EvidenceReference, _ string) (*EvidenceReceipt, error) { + return nil, errors.New("not implemented") +} diff --git a/internal/common/history/config.go b/internal/common/history/config.go index 2f709d417..7e4a2c998 100644 --- a/internal/common/history/config.go +++ b/internal/common/history/config.go @@ -26,8 +26,13 @@ package history import ( + "context" + "fmt" "strings" "sync" + "time" + + "github.com/eclipse-basyx/basyx-go-components/internal/common" ) var ( @@ -38,9 +43,14 @@ var ( FullSnapshotInterval: DefaultFullSnapshotInterval, Immutability: ImmutabilityNone, AuditIdentityMode: AuditIdentityNone, + EvidenceProvider: EvidenceProviderNone, + EvidenceWriteTimeout: DefaultEvidenceWriteTimeout, } ) +// DefaultEvidenceWriteTimeout bounds synchronous evidence writes inside database transactions. +const DefaultEvidenceWriteTimeout = 10 * time.Second + // Config controls process-local history and audit behavior. // // Services populate this from their common configuration during startup. Mode @@ -53,6 +63,10 @@ type Config struct { FullSnapshotInterval int Immutability string AuditIdentityMode string + EvidenceEnabled bool + EvidenceProvider string + EvidenceStore EvidenceStore + EvidenceWriteTimeout time.Duration } // Configure replaces the process-local history configuration. @@ -78,6 +92,56 @@ func Configure(cfg Config) { activeConfig = normalizeConfig(cfg) } +// ConfigureEvidence creates and activates the configured WORM evidence store. +// +// The store is used synchronously by history append operations while the +// PostgreSQL transaction is still open. A positive write timeout is therefore +// required so object-store stalls cannot hold history advisory locks forever. +// +// Parameters: +// - ctx: Startup context used while initializing provider clients. +// - cfg: Common history evidence configuration loaded from YAML and environment. +// +// Returns: +// - error: Error when evidence is enabled with an unsupported provider, +// incomplete S3 settings, disabled history mode, or an invalid write timeout. +func ConfigureEvidence(ctx context.Context, cfg common.HistoryEvidenceConfig) error { + provider := normalizeEvidenceProvider(cfg.Provider) + if !cfg.Enabled { + configureEvidenceStore(false, EvidenceProviderNone, nil, DefaultEvidenceWriteTimeout) + return nil + } + if provider == EvidenceProviderNone { + return fmt.Errorf("HISTORY-EVIDENCE-CONFIG-PROVIDER history.evidence.enabled requires an evidence provider") + } + if ActiveConfig().Mode == ModeOff { + return fmt.Errorf("HISTORY-EVIDENCE-CONFIG-MODE history.evidence.enabled requires history.mode api or audit") + } + if provider != EvidenceProviderS3 { + return fmt.Errorf("HISTORY-EVIDENCE-CONFIG-PROVIDER unsupported evidence provider %q", cfg.Provider) + } + writeTimeout := time.Duration(cfg.WriteTimeoutSec) * time.Second + if writeTimeout < time.Second { + return fmt.Errorf("HISTORY-EVIDENCE-CONFIG-TIMEOUT history.evidence.writeTimeoutSeconds must be at least 1") + } + store, err := NewS3EvidenceStore(ctx, S3EvidenceStoreConfig{ + Bucket: cfg.Bucket, + Prefix: cfg.Prefix, + Region: cfg.Region, + Endpoint: cfg.Endpoint, + AccessKeyID: cfg.AccessKeyID, + SecretAccessKey: cfg.SecretAccessKey, + UsePathStyle: cfg.UsePathStyle, + RetentionMode: cfg.RetentionMode, + RetentionDays: cfg.RetentionDays, + }) + if err != nil { + return fmt.Errorf("HISTORY-EVIDENCE-CONFIG-STORE %w", err) + } + configureEvidenceStore(true, provider, store, writeTimeout) + return nil +} + // ActiveConfig returns the normalized process-local history configuration. // // The returned value is a copy and can be read safely while other goroutines @@ -102,15 +166,39 @@ func normalizeConfig(cfg Config) Config { cfg.Mode = normalizeHistoryMode(cfg.Mode) cfg.Immutability = normalizeImmutability(cfg.Immutability) cfg.AuditIdentityMode = normalizeAuditIdentityMode(cfg.AuditIdentityMode) + cfg.EvidenceProvider = normalizeEvidenceProvider(cfg.EvidenceProvider) if cfg.RetentionDays < 0 { cfg.RetentionDays = 0 } if cfg.FullSnapshotInterval < 1 { cfg.FullSnapshotInterval = DefaultFullSnapshotInterval } + if cfg.EvidenceWriteTimeout < time.Second { + cfg.EvidenceWriteTimeout = DefaultEvidenceWriteTimeout + } + if !cfg.EvidenceEnabled { + cfg.EvidenceProvider = EvidenceProviderNone + cfg.EvidenceStore = nil + } return cfg } +func configureEvidenceStore(enabled bool, provider string, store EvidenceStore, writeTimeout time.Duration) { + configMu.Lock() + defer configMu.Unlock() + activeConfig.EvidenceEnabled = enabled + activeConfig.EvidenceProvider = normalizeEvidenceProvider(provider) + activeConfig.EvidenceStore = store + if writeTimeout < time.Second { + writeTimeout = DefaultEvidenceWriteTimeout + } + activeConfig.EvidenceWriteTimeout = writeTimeout + if !enabled { + activeConfig.EvidenceProvider = EvidenceProviderNone + activeConfig.EvidenceStore = nil + } +} + func normalizeHistoryMode(mode string) string { switch strings.ToLower(strings.TrimSpace(mode)) { case "", ModeOff: @@ -149,3 +237,11 @@ func normalizeAuditIdentityMode(mode string) string { return AuditIdentityMinimal } } + +func normalizeEvidenceProvider(provider string) string { + normalized := strings.ToLower(strings.TrimSpace(provider)) + if normalized == "" { + return EvidenceProviderNone + } + return normalized +} diff --git a/internal/common/history/evidence_catalog.go b/internal/common/history/evidence_catalog.go new file mode 100644 index 000000000..1b89638bc --- /dev/null +++ b/internal/common/history/evidence_catalog.go @@ -0,0 +1,310 @@ +/******************************************************************************* +* Copyright (C) 2026 the Eclipse BaSyx Authors and Fraunhofer IESE +* +* Permission is hereby granted, free of charge, to any person obtaining +* a copy of this software and associated documentation files (the +* "Software"), to deal in the Software without restriction, including +* without limitation the rights to use, copy, modify, merge, publish, +* distribute, sublicense, and/or sell copies of the Software, and to +* permit persons to whom the Software is furnished to do so, subject to +* the following conditions: +* +* The above copyright notice and this permission notice shall be +* included in all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +* +* SPDX-License-Identifier: MIT +******************************************************************************/ + +package history + +import ( + "context" + "database/sql" + "encoding/json" + "strings" + "time" + + "github.com/doug-martin/goqu/v9" + "github.com/eclipse-basyx/basyx-go-components/internal/common" +) + +// Evidence catalog table constants identify the PostgreSQL receipt tables. +const ( + TableHistoryEvidenceManifests = "history_evidence_manifests" + TableHistoryEvidenceArtifacts = "history_evidence_artifacts" +) + +// EvidenceCatalogSnapshotReceipt pairs a stored snapshot artifact receipt with its history row. +type EvidenceCatalogSnapshotReceipt struct { + Candidate SnapshotArtifactCandidate + Receipt EvidenceReceipt +} + +// EvidenceCatalogEventReceipt pairs a stored history-event artifact receipt with its history row. +type EvidenceCatalogEventReceipt struct { + Candidate EventArtifactCandidate + Receipt EvidenceReceipt +} + +// EvidenceCatalogRecord contains the manifest and artifact receipts to persist in PostgreSQL. +type EvidenceCatalogRecord struct { + Manifest HistoryManifest + ManifestReceipt EvidenceReceipt + EventReceipts []EvidenceCatalogEventReceipt + SnapshotReceipts []EvidenceCatalogSnapshotReceipt +} + +// EventEvidenceRecord contains one synchronously stored history event artifact receipt. +type EventEvidenceRecord struct { + HistoryTable string + Identifier string + HistoryID int64 + RowHash string + ContentHash string + Receipt EvidenceReceipt +} + +// RecordEvidenceCatalogTx records evidence object receipts in the PostgreSQL catalog. +// +// The catalog write is part of the caller's PostgreSQL transaction. It records +// the manifest receipt and all associated history-event and snapshot artifact +// receipts without mutating guarded history rows. +// +// Parameters: +// - ctx: Transaction context for catalog inserts. +// - tx: Open PostgreSQL transaction owned by the caller. +// - record: Manifest and artifact receipts to persist. +// +// Returns: +// - int64: Generated manifest catalog ID. +// - error: Error when the transaction is nil, the manifest is invalid, or a +// catalog insert fails. +func RecordEvidenceCatalogTx(ctx context.Context, tx *sql.Tx, record EvidenceCatalogRecord) (int64, error) { + if tx == nil { + return 0, common.NewErrBadRequest("HISTORY-EVIDENCE-CATALOG-NILTX transaction must not be nil") + } + if err := validateManifest(record.Manifest); err != nil { + return 0, err + } + manifestID, err := insertEvidenceManifestCatalogRow(ctx, tx, record) + if err != nil { + return 0, err + } + if err = insertEvidenceArtifactCatalogRows(ctx, tx, manifestID, record); err != nil { + return 0, err + } + return manifestID, nil +} + +// RecordHistoryEventEvidenceArtifactTx records one WORM history event artifact receipt. +// +// This is used by the synchronous history append path after the object-store +// write succeeded but before the surrounding PostgreSQL transaction commits. +// +// Parameters: +// - ctx: Transaction context for the receipt insert. +// - tx: Open PostgreSQL transaction owned by the history append operation. +// - record: History row identity and evidence object receipt. +// +// Returns: +// - error: Error when required receipt fields are missing or the insert fails. +func RecordHistoryEventEvidenceArtifactTx(ctx context.Context, tx *sql.Tx, record EventEvidenceRecord) error { + if tx == nil { + return common.NewErrBadRequest("HISTORY-EVIDENCE-EVENTCATALOG-NILTX transaction must not be nil") + } + if record.HistoryID < 1 { + return common.NewErrBadRequest("HISTORY-EVIDENCE-EVENTCATALOG-HISTORYID history_id must be positive") + } + if strings.TrimSpace(record.HistoryTable) == "" { + return common.NewErrBadRequest("HISTORY-EVIDENCE-EVENTCATALOG-TABLE history table is required") + } + if strings.TrimSpace(record.Receipt.Reference.ObjectKey) == "" { + return common.NewErrBadRequest("HISTORY-EVIDENCE-EVENTCATALOG-OBJECTKEY evidence object key is required") + } + query, args, err := goqu.Insert(TableHistoryEvidenceArtifacts). + Rows(historyEventArtifactCatalogRow(record)). + ToSQL() + if err != nil { + return common.NewInternalServerError("HISTORY-EVIDENCE-EVENTCATALOG-BUILD " + err.Error()) + } + if _, err = tx.ExecContext(ctx, query, args...); err != nil { + return common.NewInternalServerError("HISTORY-EVIDENCE-EVENTCATALOG-INSERT " + err.Error()) + } + return nil +} + +func insertEvidenceManifestCatalogRow(ctx context.Context, tx *sql.Tx, record EvidenceCatalogRecord) (int64, error) { + manifest := record.Manifest + receipt := record.ManifestReceipt + row := goqu.Record{ + "manifest_version": manifest.ManifestVersion, + "history_table": manifest.HistoryTable, + "identifier": nullableText(manifest.Identifier), + "first_history_id": manifest.FirstHistoryID, + "last_history_id": manifest.LastHistoryID, + "first_row_hash": manifest.FirstRowHash, + "last_row_hash": manifest.LastRowHash, + "row_count": manifest.RowCount, + "range_digest": manifest.RangeDigest, + "generated_at": manifest.GeneratedAt.UTC(), + "signature_state": manifest.SignatureState, + "signer_key_id": nullableText(manifestSignerKeyID(manifest)), + "signer_algorithm": nullableText(manifestSignerAlgorithm(manifest)), + "snapshot_reference_count": len(manifest.SnapshotReferences), + "provider": receipt.Reference.Provider, + "bucket": nullableText(receipt.Reference.Bucket), + "manifest_object_key": receipt.Reference.ObjectKey, + "manifest_object_version_id": nullableText(receipt.Reference.VersionID), + "manifest_sha256": receipt.SHA256, + "retention_mode": nullableText(receipt.RetentionMode), + "retain_until": nullableTime(receipt.RetainUntil), + "legal_hold": receipt.LegalHold, + "artifact_metadata": jsonbMetadata(receipt.Metadata), + } + query, args, err := goqu.Insert(TableHistoryEvidenceManifests). + Rows(row). + Returning(goqu.C("manifest_id")). + ToSQL() + if err != nil { + return 0, common.NewInternalServerError("HISTORY-EVIDENCE-CATALOG-BUILDMANIFEST " + err.Error()) + } + var manifestID int64 + if err = tx.QueryRowContext(ctx, query, args...).Scan(&manifestID); err != nil { + return 0, common.NewInternalServerError("HISTORY-EVIDENCE-CATALOG-INSERTMANIFEST " + err.Error()) + } + return manifestID, nil +} + +func insertEvidenceArtifactCatalogRows(ctx context.Context, tx *sql.Tx, manifestID int64, record EvidenceCatalogRecord) error { + rows := []goqu.Record{manifestArtifactCatalogRow(manifestID, record)} + for _, eventReceipt := range record.EventReceipts { + rows = append(rows, evidenceEventArtifactCatalogRow(eventReceipt)) + } + for _, snapshotReceipt := range record.SnapshotReceipts { + rows = append(rows, snapshotArtifactCatalogRow(manifestID, record.Manifest, snapshotReceipt)) + } + query, args, err := goqu.Insert(TableHistoryEvidenceArtifacts). + Rows(rows). + OnConflict(goqu.DoNothing()). + ToSQL() + if err != nil { + return common.NewInternalServerError("HISTORY-EVIDENCE-CATALOG-BUILDARTIFACTS " + err.Error()) + } + if _, err = tx.ExecContext(ctx, query, args...); err != nil { + return common.NewInternalServerError("HISTORY-EVIDENCE-CATALOG-INSERTARTIFACTS " + err.Error()) + } + return nil +} + +func manifestArtifactCatalogRow(manifestID int64, record EvidenceCatalogRecord) goqu.Record { + manifest := record.Manifest + receipt := record.ManifestReceipt + return baseArtifactCatalogRow(manifestID, EvidenceArtifactManifest, manifest.HistoryTable, manifest.Identifier, receipt, goqu.Record{ + "history_id": nil, + "row_hash": nil, + "content_hash": nil, + }) +} + +func snapshotArtifactCatalogRow(manifestID int64, manifest HistoryManifest, snapshotReceipt EvidenceCatalogSnapshotReceipt) goqu.Record { + candidate := snapshotReceipt.Candidate + row := baseArtifactCatalogRow(manifestID, EvidenceArtifactSnapshot, manifest.HistoryTable, candidate.Identifier, snapshotReceipt.Receipt, goqu.Record{ + "history_id": candidate.HistoryID, + "row_hash": nullableText(candidate.RowHash), + "content_hash": nullableText(candidate.ContentHash), + }) + return row +} + +func evidenceEventArtifactCatalogRow(eventReceipt EvidenceCatalogEventReceipt) goqu.Record { + candidate := eventReceipt.Candidate + return historyEventArtifactCatalogRow(EventEvidenceRecord{ + HistoryTable: candidate.HistoryTable, + Identifier: candidate.Identifier, + HistoryID: candidate.HistoryID, + RowHash: candidate.RowHash, + ContentHash: candidate.ContentHash, + Receipt: eventReceipt.Receipt, + }) +} + +func historyEventArtifactCatalogRow(record EventEvidenceRecord) goqu.Record { + row := baseArtifactCatalogRow(0, EvidenceArtifactHistoryEvent, record.HistoryTable, record.Identifier, record.Receipt, goqu.Record{ + "history_id": record.HistoryID, + "row_hash": nullableText(record.RowHash), + "content_hash": nullableText(record.ContentHash), + }) + row["manifest_id"] = nil + return row +} + +func baseArtifactCatalogRow(manifestID int64, artifactType string, table string, identifier string, receipt EvidenceReceipt, extra goqu.Record) goqu.Record { + row := goqu.Record{ + "manifest_id": manifestID, + "artifact_type": artifactType, + "history_table": table, + "identifier": nullableText(identifier), + "provider": receipt.Reference.Provider, + "bucket": nullableText(receipt.Reference.Bucket), + "object_key": receipt.Reference.ObjectKey, + "object_version_id": nullableText(receipt.Reference.VersionID), + "sha256": receipt.SHA256, + "size_bytes": receipt.SizeBytes, + "content_type": receipt.ContentType, + "retention_mode": nullableText(receipt.RetentionMode), + "retain_until": nullableTime(receipt.RetainUntil), + "legal_hold": receipt.LegalHold, + "artifact_metadata": jsonbMetadata(receipt.Metadata), + } + for key, value := range extra { + row[key] = value + } + return row +} + +func manifestSignerKeyID(manifest HistoryManifest) string { + if manifest.Signer == nil { + return "" + } + return manifest.Signer.KeyID +} + +func manifestSignerAlgorithm(manifest HistoryManifest) string { + if manifest.Signer == nil { + return "" + } + return manifest.Signer.Algorithm +} + +func nullableText(value string) any { + if value == "" { + return nil + } + return value +} + +func nullableTime(value *time.Time) any { + if value == nil { + return nil + } + return value.UTC() +} + +func jsonbMetadata(metadata map[string]string) any { + if metadata == nil { + metadata = map[string]string{} + } + encoded, err := json.Marshal(metadata) + if err != nil { + return goqu.L("?::jsonb", "{}") + } + return goqu.L("?::jsonb", string(encoded)) +} diff --git a/internal/common/history/evidence_hash.go b/internal/common/history/evidence_hash.go new file mode 100644 index 000000000..31fce3245 --- /dev/null +++ b/internal/common/history/evidence_hash.go @@ -0,0 +1,43 @@ +/******************************************************************************* +* Copyright (C) 2026 the Eclipse BaSyx Authors and Fraunhofer IESE +* +* Permission is hereby granted, free of charge, to any person obtaining +* a copy of this software and associated documentation files (the +* "Software"), to deal in the Software without restriction, including +* without limitation the rights to use, copy, modify, merge, publish, +* distribute, sublicense, and/or sell copies of the Software, and to +* permit persons to whom the Software is furnished to do so, subject to +* the following conditions: +* +* The above copyright notice and this permission notice shall be +* included in all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +* +* SPDX-License-Identifier: MIT +******************************************************************************/ + +package history + +import ( + "crypto/sha256" + "encoding/hex" +) + +// SHA256Hex returns the lowercase hexadecimal SHA-256 digest for binary evidence data. +// +// Parameters: +// - data: Artifact bytes to hash. +// +// Returns: +// - string: Lowercase hexadecimal SHA-256 digest. +func SHA256Hex(data []byte) string { + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) +} diff --git a/internal/common/history/evidence_store_s3.go b/internal/common/history/evidence_store_s3.go new file mode 100644 index 000000000..8a804a41c --- /dev/null +++ b/internal/common/history/evidence_store_s3.go @@ -0,0 +1,342 @@ +/******************************************************************************* +* Copyright (C) 2026 the Eclipse BaSyx Authors and Fraunhofer IESE +* +* Permission is hereby granted, free of charge, to any person obtaining +* a copy of this software and associated documentation files (the +* "Software"), to deal in the Software without restriction, including +* without limitation the rights to use, copy, modify, merge, publish, +* distribute, sublicense, and/or sell copies of the Software, and to +* permit persons to whom the Software is furnished to do so, subject to +* the following conditions: +* +* The above copyright notice and this permission notice shall be +* included in all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +* +* SPDX-License-Identifier: MIT +******************************************************************************/ + +package history + +import ( + "bytes" + "context" + "fmt" + "io" + "path" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" +) + +// S3EvidenceStoreConfig configures an S3-compatible WORM evidence backend. +// +// Bucket and region are required to create the client. RetentionMode and +// RetentionDays are required for PutArtifact calls because evidence writes must +// be protected by object-lock retention. +type S3EvidenceStoreConfig struct { + Bucket string + Prefix string + Region string + Endpoint string + AccessKeyID string + SecretAccessKey string + UsePathStyle bool + RetentionMode string + RetentionDays int +} + +// S3EvidenceStore stores evidence artifacts in AWS S3 or an S3-compatible backend. +type S3EvidenceStore struct { + client *s3.Client + cfg S3EvidenceStoreConfig + now func() time.Time +} + +// NewS3EvidenceStore creates an EvidenceStore backed by S3-compatible object storage. +// +// The store supports AWS S3 and endpoint-compatible backends such as MinIO. A +// bucket and region are always required. Retention settings may be omitted for +// read-only verifier usage, but writes fail unless object-lock retention is +// configured through RetentionMode and RetentionDays. +// +// Parameters: +// - ctx: Startup context used while loading AWS SDK configuration. +// - cfg: S3 bucket, endpoint, credential, prefix, and retention settings. +// +// Returns: +// - *S3EvidenceStore: Initialized evidence store. +// - error: Error when configuration is incomplete or AWS SDK setup fails. +func NewS3EvidenceStore(ctx context.Context, cfg S3EvidenceStoreConfig) (*S3EvidenceStore, error) { + normalized, err := normalizeS3EvidenceStoreConfig(cfg) + if err != nil { + return nil, err + } + loadOptions := []func(*config.LoadOptions) error{config.WithRegion(normalized.Region)} + if normalized.AccessKeyID != "" { + loadOptions = append(loadOptions, config.WithCredentialsProvider( + credentials.NewStaticCredentialsProvider(normalized.AccessKeyID, normalized.SecretAccessKey, ""), + )) + } + if normalized.Endpoint != "" { + loadOptions = append(loadOptions, config.WithBaseEndpoint(normalized.Endpoint)) + } + awsCfg, err := config.LoadDefaultConfig(ctx, loadOptions...) + if err != nil { + return nil, fmt.Errorf("HISTORY-EVIDENCE-S3-LOADCONFIG %w", err) + } + client := s3.NewFromConfig(awsCfg, func(options *s3.Options) { + options.UsePathStyle = normalized.UsePathStyle + }) + return &S3EvidenceStore{client: client, cfg: normalized, now: func() time.Time { return time.Now().UTC() }}, nil +} + +func normalizeS3EvidenceStoreConfig(cfg S3EvidenceStoreConfig) (S3EvidenceStoreConfig, error) { + cfg.Bucket = strings.TrimSpace(cfg.Bucket) + cfg.Prefix = strings.Trim(strings.TrimSpace(cfg.Prefix), "/") + cfg.Region = strings.TrimSpace(cfg.Region) + cfg.Endpoint = strings.TrimSpace(cfg.Endpoint) + cfg.AccessKeyID = strings.TrimSpace(cfg.AccessKeyID) + cfg.SecretAccessKey = strings.TrimSpace(cfg.SecretAccessKey) + cfg.RetentionMode = strings.ToLower(strings.TrimSpace(cfg.RetentionMode)) + if cfg.Bucket == "" { + return S3EvidenceStoreConfig{}, fmt.Errorf("HISTORY-EVIDENCE-S3-BUCKET bucket is required") + } + if cfg.Region == "" { + return S3EvidenceStoreConfig{}, fmt.Errorf("HISTORY-EVIDENCE-S3-REGION region is required") + } + if (cfg.AccessKeyID == "") != (cfg.SecretAccessKey == "") { + return S3EvidenceStoreConfig{}, fmt.Errorf("HISTORY-EVIDENCE-S3-CREDENTIALS access key and secret access key must be configured together") + } + switch cfg.RetentionMode { + case "", "governance", "compliance": + default: + return S3EvidenceStoreConfig{}, fmt.Errorf("HISTORY-EVIDENCE-S3-RETENTIONMODE unsupported retention mode %q", cfg.RetentionMode) + } + if cfg.RetentionDays < 0 { + return S3EvidenceStoreConfig{}, fmt.Errorf("HISTORY-EVIDENCE-S3-RETENTIONDAYS retention days must not be negative") + } + return cfg, nil +} + +// PutArtifact writes one immutable evidence artifact to S3. +// +// The object is written with object-lock retention metadata. Evidence writes +// fail closed when no retention mode and retain-until timestamp can be derived +// from the artifact or store configuration. +// +// Parameters: +// - ctx: Request context for the S3 PutObject call. +// - artifact: Canonical evidence bytes plus object key and metadata. +// +// Returns: +// - *EvidenceReceipt: Object reference, version, hash, retention, and metadata. +// - error: Error when the store is uninitialized, retention is missing, or S3 +// rejects the write. +func (store *S3EvidenceStore) PutArtifact(ctx context.Context, artifact EvidenceArtifact) (*EvidenceReceipt, error) { + if store == nil || store.client == nil { + return nil, fmt.Errorf("HISTORY-EVIDENCE-S3-NILSTORE evidence store is not initialized") + } + objectKey := store.objectKey(artifact.ObjectKey) + if objectKey == "" { + return nil, fmt.Errorf("HISTORY-EVIDENCE-S3-EMPTYKEY artifact object key is required") + } + receipt := store.receiptForArtifact(objectKey, artifact) + if receipt.RetentionMode == "" || receipt.RetainUntil == nil { + return nil, fmt.Errorf("HISTORY-EVIDENCE-S3-RETENTION retention mode and retain-until timestamp are required for evidence writes") + } + input := &s3.PutObjectInput{ + Bucket: aws.String(store.cfg.Bucket), + Key: aws.String(objectKey), + Body: bytes.NewReader(artifact.Data), + ContentType: aws.String(artifact.ContentType), + Metadata: cleanS3Metadata(receipt.Metadata), + } + applyS3Retention(input, receipt) + output, err := store.client.PutObject(ctx, input) + if err != nil { + return nil, fmt.Errorf("HISTORY-EVIDENCE-S3-PUTOBJECT %w", err) + } + if output.VersionId != nil { + receipt.Reference.VersionID = strings.TrimSpace(*output.VersionId) + } + return receipt, nil +} + +// GetArtifact reads an evidence artifact from S3. +// +// Parameters: +// - ctx: Request context for the S3 GetObject call. +// - ref: Provider object reference to read. VersionID is used when present. +// +// Returns: +// - *EvidenceObject: Downloaded bytes, content type, metadata, and resolved reference. +// - error: Error when the store is uninitialized, the reference is incomplete, +// or S3 cannot return the object. +func (store *S3EvidenceStore) GetArtifact(ctx context.Context, ref EvidenceReference) (*EvidenceObject, error) { + if store == nil || store.client == nil { + return nil, fmt.Errorf("HISTORY-EVIDENCE-S3-NILSTORE evidence store is not initialized") + } + objectKey := strings.TrimSpace(ref.ObjectKey) + if objectKey == "" { + return nil, fmt.Errorf("HISTORY-EVIDENCE-S3-EMPTYREF artifact object key is required") + } + input := &s3.GetObjectInput{Bucket: aws.String(store.cfg.Bucket), Key: aws.String(objectKey)} + if strings.TrimSpace(ref.VersionID) != "" { + input.VersionId = aws.String(strings.TrimSpace(ref.VersionID)) + } + output, err := store.client.GetObject(ctx, input) + if err != nil { + return nil, fmt.Errorf("HISTORY-EVIDENCE-S3-GETOBJECT %w", err) + } + defer func() { + _ = output.Body.Close() + }() + data, err := io.ReadAll(output.Body) + if err != nil { + return nil, fmt.Errorf("HISTORY-EVIDENCE-S3-READOBJECT %w", err) + } + versionID := strings.TrimSpace(ref.VersionID) + if output.VersionId != nil && strings.TrimSpace(*output.VersionId) != "" { + versionID = strings.TrimSpace(*output.VersionId) + } + contentType := "" + if output.ContentType != nil { + contentType = strings.TrimSpace(*output.ContentType) + } + return &EvidenceObject{ + Reference: EvidenceReference{Provider: EvidenceProviderS3, Bucket: store.cfg.Bucket, ObjectKey: objectKey, VersionID: versionID}, + Data: data, + ContentType: contentType, + Metadata: copyStringMap(output.Metadata), + }, nil +} + +// VerifyArtifact checks that the stored object bytes match expectedHash. +// +// Parameters: +// - ctx: Request context for reading the artifact. +// - ref: Provider object reference to verify. +// - expectedHash: Expected lowercase or uppercase hexadecimal SHA-256 digest. +// +// Returns: +// - *EvidenceReceipt: Receipt reconstructed from the downloaded object. +// - error: Error when the object cannot be read or its SHA-256 digest differs. +func (store *S3EvidenceStore) VerifyArtifact(ctx context.Context, ref EvidenceReference, expectedHash string) (*EvidenceReceipt, error) { + object, err := store.GetArtifact(ctx, ref) + if err != nil { + return nil, err + } + actualHash := SHA256Hex(object.Data) + if !strings.EqualFold(strings.TrimSpace(expectedHash), actualHash) { + return nil, fmt.Errorf("HISTORY-EVIDENCE-S3-HASHMISMATCH artifact hash mismatch") + } + return &EvidenceReceipt{ + Reference: object.Reference, + SHA256: actualHash, + SizeBytes: int64(len(object.Data)), + ContentType: object.ContentType, + StoredAt: store.now(), + Metadata: object.Metadata, + }, nil +} + +func (store *S3EvidenceStore) objectKey(rawKey string) string { + key := strings.Trim(strings.TrimSpace(rawKey), "/") + if key == "" { + return "" + } + if store.cfg.Prefix == "" || strings.HasPrefix(key, store.cfg.Prefix+"/") { + return key + } + return path.Join(store.cfg.Prefix, key) +} + +func (store *S3EvidenceStore) receiptForArtifact(objectKey string, artifact EvidenceArtifact) *EvidenceReceipt { + retentionMode, retainUntil := store.retention(artifact) + metadata := copyStringMap(artifact.Metadata) + metadata["sha256"] = SHA256Hex(artifact.Data) + metadata["artifact_type"] = strings.TrimSpace(artifact.ArtifactType) + return &EvidenceReceipt{ + Reference: EvidenceReference{ + Provider: EvidenceProviderS3, + Bucket: store.cfg.Bucket, + ObjectKey: objectKey, + }, + SHA256: metadata["sha256"], + SizeBytes: int64(len(artifact.Data)), + ContentType: strings.TrimSpace(artifact.ContentType), + RetentionMode: retentionMode, + RetainUntil: retainUntil, + LegalHold: artifact.LegalHold, + StoredAt: store.now(), + Metadata: metadata, + } +} + +func (store *S3EvidenceStore) retention(artifact EvidenceArtifact) (string, *time.Time) { + mode := strings.ToLower(strings.TrimSpace(artifact.RetentionMode)) + if mode == "" { + mode = store.cfg.RetentionMode + } + if mode == "" { + return "", nil + } + if !artifact.RetainUntil.IsZero() { + retainUntil := artifact.RetainUntil.UTC() + return mode, &retainUntil + } + if store.cfg.RetentionDays < 1 { + return "", nil + } + retainUntil := store.now().AddDate(0, 0, store.cfg.RetentionDays).UTC() + return mode, &retainUntil +} + +func applyS3Retention(input *s3.PutObjectInput, receipt *EvidenceReceipt) { + if receipt.LegalHold { + input.ObjectLockLegalHoldStatus = types.ObjectLockLegalHoldStatusOn + } + if receipt.RetainUntil == nil || receipt.RetentionMode == "" { + return + } + input.ObjectLockRetainUntilDate = receipt.RetainUntil + switch strings.ToLower(receipt.RetentionMode) { + case "compliance": + input.ObjectLockMode = types.ObjectLockModeCompliance + case "governance": + input.ObjectLockMode = types.ObjectLockModeGovernance + } +} + +func cleanS3Metadata(metadata map[string]string) map[string]string { + cleaned := make(map[string]string, len(metadata)) + for key, value := range metadata { + cleanKey := strings.ToLower(strings.TrimSpace(key)) + if cleanKey == "" { + continue + } + cleaned[cleanKey] = strings.TrimSpace(value) + } + return cleaned +} + +func copyStringMap(values map[string]string) map[string]string { + copied := make(map[string]string, len(values)) + for key, value := range values { + copied[key] = value + } + return copied +} diff --git a/internal/common/history/evidence_test.go b/internal/common/history/evidence_test.go new file mode 100644 index 000000000..eb4a93931 --- /dev/null +++ b/internal/common/history/evidence_test.go @@ -0,0 +1,266 @@ +/******************************************************************************* +* Copyright (C) 2026 the Eclipse BaSyx Authors and Fraunhofer IESE +* +* Permission is hereby granted, free of charge, to any person obtaining +* a copy of this software and associated documentation files (the +* "Software"), to deal in the Software without restriction, including +* without limitation the rights to use, copy, modify, merge, publish, +* distribute, sublicense, and/or sell copies of the Software, and to +* permit persons to whom the Software is furnished to do so, subject to +* the following conditions: +* +* The above copyright notice and this permission notice shall be +* included in all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +* +* SPDX-License-Identifier: MIT +******************************************************************************/ + +package history + +import ( + "crypto/rand" + "crypto/rsa" + "strings" + "testing" + "time" + + sqlmock "github.com/DATA-DOG/go-sqlmock" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/stretchr/testify/require" +) + +func TestBuildHistoryManifestComputesDeterministicRangeDigest(t *testing.T) { + generatedAt := time.Date(2026, 6, 5, 12, 0, 0, 0, time.UTC) + rows := []ManifestRangeRow{ + {HistoryID: 1, RowHash: strings.Repeat("a", 64)}, + {HistoryID: 2, RowHash: strings.Repeat("b", 64)}, + } + + first, err := BuildHistoryManifest(HistoryManifestOptions{HistoryTable: TableAAS, Identifier: "aas-1", Rows: rows, GeneratedAt: generatedAt}) + require.NoError(t, err) + second, err := BuildHistoryManifest(HistoryManifestOptions{HistoryTable: TableAAS, Identifier: "aas-1", Rows: rows, GeneratedAt: generatedAt}) + require.NoError(t, err) + + require.Equal(t, first.RangeDigest, second.RangeDigest) + require.Equal(t, int64(2), first.RowCount) + require.Equal(t, rows[0].RowHash, first.FirstRowHash) + require.Equal(t, rows[1].RowHash, first.LastRowHash) + firstJSON, err := CanonicalManifestJSON(first) + require.NoError(t, err) + secondJSON, err := CanonicalManifestJSON(second) + require.NoError(t, err) + require.Equal(t, firstJSON, secondJSON) +} + +func TestBuildHistoryManifestRejectsReorderedRows(t *testing.T) { + _, err := BuildHistoryManifest(HistoryManifestOptions{ + HistoryTable: TableAAS, + Rows: []ManifestRangeRow{ + {HistoryID: 2, RowHash: strings.Repeat("b", 64)}, + {HistoryID: 1, RowHash: strings.Repeat("a", 64)}, + }, + }) + + require.ErrorContains(t, err, "HISTORY-MANIFEST-ORDER") +} + +func TestBuildManifestEvidenceArtifactSignsCanonicalManifest(t *testing.T) { + privateKey, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + manifest, err := BuildHistoryManifest(HistoryManifestOptions{ + HistoryTable: TableSubmodel, + Identifier: "submodel-1", + Rows: []ManifestRangeRow{ + {HistoryID: 10, RowHash: strings.Repeat("c", 64)}, + }, + GeneratedAt: time.Date(2026, 6, 5, 12, 0, 0, 0, time.UTC), + }) + require.NoError(t, err) + + artifact, signedManifest, err := BuildManifestEvidenceArtifact(manifest, "prefix", &ManifestJWSSigner{PrivateKey: privateKey, KeyID: "test-key"}) + require.NoError(t, err) + decoded, signed, err := DecodeManifestArtifact(artifact.Data, artifact.ContentType) + require.NoError(t, err) + + require.True(t, signed) + require.Equal(t, manifestJWSContentType, artifact.ContentType) + require.Equal(t, SignatureStateSigned, signedManifest.SignatureState) + require.Equal(t, "test-key", signedManifest.Signer.KeyID) + require.Equal(t, signedManifest.RangeDigest, decoded.RangeDigest) + require.True(t, strings.HasPrefix(artifact.ObjectKey, "prefix/history-manifests/")) +} + +func TestS3EvidenceStoreReceiptAppliesPrefixAndRetention(t *testing.T) { + now := time.Date(2026, 6, 5, 12, 0, 0, 0, time.UTC) + store := &S3EvidenceStore{ + cfg: S3EvidenceStoreConfig{ + Bucket: "evidence", + Prefix: "tenant-a", + RetentionMode: "governance", + RetentionDays: 7, + }, + now: func() time.Time { return now }, + } + + key := store.objectKey("manifests/one.json") + receipt := store.receiptForArtifact(key, EvidenceArtifact{ArtifactType: EvidenceArtifactManifest, ContentType: manifestJSONContentType, Data: []byte(`{"ok":true}`)}) + + require.Equal(t, "tenant-a/manifests/one.json", receipt.Reference.ObjectKey) + require.Equal(t, "governance", receipt.RetentionMode) + require.NotNil(t, receipt.RetainUntil) + require.Equal(t, now.AddDate(0, 0, 7), *receipt.RetainUntil) + require.Equal(t, SHA256Hex([]byte(`{"ok":true}`)), receipt.SHA256) +} + +func TestS3EvidenceStorePutRequiresRetention(t *testing.T) { + store := &S3EvidenceStore{ + client: &s3.Client{}, + cfg: S3EvidenceStoreConfig{Bucket: "evidence", Region: "us-east-1"}, + now: func() time.Time { return time.Date(2026, 6, 5, 12, 0, 0, 0, time.UTC) }, + } + + _, err := store.PutArtifact(t.Context(), EvidenceArtifact{ArtifactType: EvidenceArtifactHistoryEvent, ObjectKey: "events/one.json", ContentType: manifestJSONContentType, Data: []byte(`{}`)}) + + require.ErrorContains(t, err, "HISTORY-EVIDENCE-S3-RETENTION") +} + +func TestStoreHistoryEventArtifactsBackfillsSnapshotAndDiffRows(t *testing.T) { + db, mock, err := sqlmock.New() + require.NoError(t, err) + defer func() { + _ = db.Close() + }() + + store := &recordingEvidenceStore{} + baseSnapshot := map[string]any{"id": "aas-1", "idShort": "before", "description": strings.Repeat("unchanged-", 40)} + nextSnapshot := map[string]any{"id": "aas-1", "idShort": "after", "description": strings.Repeat("unchanged-", 40)} + patch, err := BuildJSONPatch(baseSnapshot, nextSnapshot) + require.NoError(t, err) + operationTime := time.Date(2026, 6, 5, 12, 0, 0, 0, time.UTC) + + mock.ExpectQuery(`SELECT .*FROM "aas_history" AS "history" INNER JOIN "aas_history_payload" AS "payload"`). + WillReturnRows(newHistoryChainRows(TableAAS, + historyChainRowSpec{HistoryID: 1, Identifier: "aas-1", ChangeType: ChangeCreated, PayloadType: PayloadTypeSnapshot, Snapshot: baseSnapshot, Deleted: false, OperationTime: operationTime}, + historyChainRowSpec{HistoryID: 2, Identifier: "aas-1", ChangeType: ChangeUpdated, PayloadType: PayloadTypeDiff, Patch: patch, ContentSnapshot: nextSnapshot, Deleted: false, OperationTime: operationTime}, + )) + + receipts, err := storeHistoryEventArtifacts(t.Context(), WriteHistoryEvidenceOptions{ + DB: db, + Store: store, + HistoryTable: TableAAS, + Identifier: "aas-1", + FirstHistoryID: 1, + LastHistoryID: 2, + }) + + require.NoError(t, err) + require.Len(t, receipts, 2) + require.Len(t, store.artifacts, 2) + require.Equal(t, EvidenceArtifactHistoryEvent, store.artifacts[0].ArtifactType) + require.Equal(t, EvidenceArtifactHistoryEvent, store.artifacts[1].ArtifactType) + require.Contains(t, string(store.artifacts[1].Data), `"payload_type":"diff"`) + require.NoError(t, mock.ExpectationsWereMet()) +} + +func TestVerifyHistoryRangeReportsMissingHistoryEventArtifact(t *testing.T) { + db, mock, err := sqlmock.New() + require.NoError(t, err) + defer func() { + _ = db.Close() + }() + + snapshot := map[string]any{"id": "aas-1"} + spec := historyChainRowSpec{HistoryID: 1, Identifier: "aas-1", ChangeType: ChangeCreated, PayloadType: PayloadTypeSnapshot, Snapshot: snapshot, Deleted: false, OperationTime: time.Date(2026, 6, 5, 12, 0, 0, 0, time.UTC)} + _, rowHash := historyChainRow(TableAAS, spec, "") + + mock.ExpectQuery(`SELECT "history_id", "identifier", "row_hash" FROM "aas_history"`). + WillReturnRows(sqlmock.NewRows([]string{"history_id", "identifier", "row_hash"}).AddRow(1, "aas-1", rowHash)) + mock.ExpectQuery(`SELECT "history_id" FROM "aas_history".*"payload_type" = 'snapshot'`). + WillReturnRows(sqlmock.NewRows([]string{"history_id"}).AddRow(1)) + mock.ExpectQuery(`SELECT .*FROM "aas_history" AS "history" INNER JOIN "aas_history_payload" AS "payload"`). + WillReturnRows(newHistoryChainRows(TableAAS, spec)) + mock.ExpectQuery(`SELECT .*FROM "aas_history" AS "history" INNER JOIN "aas_history_payload" AS "payload"`). + WillReturnRows(newHistoryChainRows(TableAAS, spec)) + mock.ExpectQuery(`SELECT .*FROM "history_evidence_artifacts"`). + WillReturnRows(sqlmock.NewRows(eventArtifactReceiptColumns())) + + report, err := VerifyHistoryRange(t.Context(), db, VerifyHistoryRangeOptions{ + HistoryTable: TableAAS, + Identifier: "aas-1", + FirstHistoryID: 1, + LastHistoryID: 1, + }) + + require.NoError(t, err) + require.False(t, report.Valid) + require.Contains(t, verificationFindingCodes(report), "HISTORY-EVIDENCE-VERIFY-EVENTMISSING") + require.NoError(t, mock.ExpectationsWereMet()) +} + +func TestVerifyManifestRangeReportsDigestMismatch(t *testing.T) { + report := &HistoryEvidenceVerificationReport{ + Valid: true, + HistoryTable: TableAAS, + Identifier: "aas-1", + FirstHistoryID: 1, + LastHistoryID: 2, + FirstRowHash: "first", + LastRowHash: "last", + RowCount: 2, + RangeDigest: "actual", + } + manifest := &HistoryManifest{ + HistoryTable: TableAAS, + Identifier: "aas-1", + FirstHistoryID: 1, + LastHistoryID: 2, + FirstRowHash: "first", + LastRowHash: "last", + RowCount: 2, + RangeDigest: "expected", + SignatureState: SignatureStateUnsigned, + ManifestVersion: HistoryManifestVersion, + } + + verifyManifestRange(report, manifest) + + require.False(t, report.Valid) + require.Len(t, report.Findings, 1) + require.Equal(t, "HISTORY-EVIDENCE-VERIFY-MANIFESTDIGEST", report.Findings[0].Code) +} + +func eventArtifactReceiptColumns() []string { + return []string{ + "artifact_id", + "identifier", + "history_id", + "row_hash", + "content_hash", + "provider", + "bucket", + "object_key", + "object_version_id", + "sha256", + "size_bytes", + "content_type", + "retention_mode", + "retain_until", + "legal_hold", + } +} + +func verificationFindingCodes(report *HistoryEvidenceVerificationReport) []string { + codes := make([]string, 0, len(report.Findings)) + for _, finding := range report.Findings { + codes = append(codes, finding.Code) + } + return codes +} diff --git a/internal/common/history/evidence_types.go b/internal/common/history/evidence_types.go new file mode 100644 index 000000000..6aed01990 --- /dev/null +++ b/internal/common/history/evidence_types.go @@ -0,0 +1,178 @@ +/******************************************************************************* +* Copyright (C) 2026 the Eclipse BaSyx Authors and Fraunhofer IESE +* +* Permission is hereby granted, free of charge, to any person obtaining +* a copy of this software and associated documentation files (the +* "Software"), to deal in the Software without restriction, including +* without limitation the rights to use, copy, modify, merge, publish, +* distribute, sublicense, and/or sell copies of the Software, and to +* permit persons to whom the Software is furnished to do so, subject to +* the following conditions: +* +* The above copyright notice and this permission notice shall be +* included in all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +* +* SPDX-License-Identifier: MIT +******************************************************************************/ + +package history + +import ( + "context" + "time" +) + +// Evidence and integrity constants define supported providers, artifact types, and signature states. +const ( + EvidenceProviderNone = "none" + EvidenceProviderS3 = "s3" + + IntegrityAnchorProviderNone = "none" + + EvidenceArtifactManifest = "manifest" + EvidenceArtifactSnapshot = "snapshot" + EvidenceArtifactHistoryEvent = "history_event" + + SignatureStateUnsigned = "unsigned" + SignatureStateSigned = "signed" + + HistoryManifestVersion = "basyx-history-manifest-v1" + historyRangeContract = "basyx-history-range-v1" +) + +// EvidenceStore writes immutable evidence artifacts and can later re-read them for verification. +type EvidenceStore interface { + PutArtifact(ctx context.Context, artifact EvidenceArtifact) (*EvidenceReceipt, error) + GetArtifact(ctx context.Context, ref EvidenceReference) (*EvidenceObject, error) + VerifyArtifact(ctx context.Context, ref EvidenceReference, expectedHash string) (*EvidenceReceipt, error) +} + +// EvidenceArtifact is a byte artifact destined for WORM-compatible object storage. +type EvidenceArtifact struct { + ArtifactType string + ObjectKey string + ContentType string + Data []byte + Metadata map[string]string + RetentionMode string + RetainUntil time.Time + LegalHold bool +} + +// EvidenceReference identifies a stored evidence object. +type EvidenceReference struct { + Provider string `json:"provider"` + Bucket string `json:"bucket,omitempty"` + ObjectKey string `json:"object_key"` + VersionID string `json:"version_id,omitempty"` +} + +// EvidenceReceipt records immutable object-store metadata returned after a write or verification. +type EvidenceReceipt struct { + Reference EvidenceReference `json:"reference"` + SHA256 string `json:"sha256"` + SizeBytes int64 `json:"size_bytes"` + ContentType string `json:"content_type"` + RetentionMode string `json:"retention_mode,omitempty"` + RetainUntil *time.Time `json:"retain_until,omitempty"` + LegalHold bool `json:"legal_hold"` + StoredAt time.Time `json:"stored_at"` + Metadata map[string]string `json:"metadata,omitempty"` +} + +// EvidenceObject is the downloaded representation of a stored evidence artifact. +type EvidenceObject struct { + Reference EvidenceReference + Data []byte + ContentType string + Metadata map[string]string +} + +// IntegrityAnchor is reserved for optional external ledgers or timestamping services. +type IntegrityAnchor interface { + AnchorIntegrity(ctx context.Context, request IntegrityAnchorRequest) (*IntegrityAnchorReceipt, error) +} + +// IntegrityAnchorRequest is the deterministic digest submitted to future anchor backends. +type IntegrityAnchorRequest struct { + Provider string + HistoryTable string + Identifier string + RangeDigest string + ManifestHash string + ManifestRef EvidenceReference + GeneratedAt time.Time + AdditionalData map[string]string +} + +// IntegrityAnchorReceipt captures future ledger/timestamping provider metadata. +type IntegrityAnchorReceipt struct { + Provider string + AnchorID string + AnchorTime time.Time + Proof map[string]string +} + +// NoopIntegrityAnchor is the default anchor implementation when no ledger backend is configured. +type NoopIntegrityAnchor struct{} + +// AnchorIntegrity intentionally performs no external write and returns no receipt. +// +// Parameters: +// - ctx: Unused context accepted to satisfy IntegrityAnchor. +// - request: Unused anchor request accepted to satisfy IntegrityAnchor. +// +// Returns: +// - *IntegrityAnchorReceipt: Always nil. +// - error: Always nil. +func (NoopIntegrityAnchor) AnchorIntegrity(_ context.Context, _ IntegrityAnchorRequest) (*IntegrityAnchorReceipt, error) { + return nil, nil +} + +// HistoryManifest covers a deterministic, ordered range of history rows. +// +//revive:disable-next-line:exported +type HistoryManifest struct { + ManifestVersion string `json:"manifest_version"` + HistoryTable string `json:"history_table"` + Identifier string `json:"identifier,omitempty"` + FirstHistoryID int64 `json:"first_history_id"` + LastHistoryID int64 `json:"last_history_id"` + FirstRowHash string `json:"first_row_hash"` + LastRowHash string `json:"last_row_hash"` + RowCount int64 `json:"row_count"` + RangeDigest string `json:"range_digest"` + GeneratedAt time.Time `json:"generated_at"` + SignatureState string `json:"signature_state"` + Signer *ManifestSignerInfo `json:"signer,omitempty"` + SnapshotReferences []SnapshotArtifactReference `json:"snapshot_references,omitempty"` +} + +// ManifestSignerInfo identifies the key material used for a signed manifest artifact. +type ManifestSignerInfo struct { + KeyID string `json:"key_id,omitempty"` + Algorithm string `json:"algorithm"` +} + +// ManifestRangeRow is the ordered hash-chain row input for a manifest range digest. +type ManifestRangeRow struct { + HistoryID int64 `json:"history_id"` + RowHash string `json:"row_hash"` +} + +// SnapshotArtifactReference links a recovery checkpoint snapshot artifact to a manifest. +type SnapshotArtifactReference struct { + HistoryID int64 `json:"history_id"` + RowHash string `json:"row_hash"` + ContentHash string `json:"content_hash,omitempty"` + SHA256 string `json:"sha256"` + Reference EvidenceReference `json:"reference"` +} diff --git a/internal/common/history/evidence_verifier.go b/internal/common/history/evidence_verifier.go new file mode 100644 index 000000000..48323c9d7 --- /dev/null +++ b/internal/common/history/evidence_verifier.go @@ -0,0 +1,508 @@ +/******************************************************************************* +* Copyright (C) 2026 the Eclipse BaSyx Authors and Fraunhofer IESE +* +* Permission is hereby granted, free of charge, to any person obtaining +* a copy of this software and associated documentation files (the +* "Software"), to deal in the Software without restriction, including +* without limitation the rights to use, copy, modify, merge, publish, +* distribute, sublicense, and/or sell copies of the Software, and to +* permit persons to whom the Software is furnished to do so, subject to +* the following conditions: +* +* The above copyright notice and this permission notice shall be +* included in all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +* +* SPDX-License-Identifier: MIT +******************************************************************************/ + +package history + +import ( + "context" + "database/sql" + "fmt" + "strings" + + "github.com/doug-martin/goqu/v9" + "github.com/eclipse-basyx/basyx-go-components/internal/common" +) + +// Verification severity constants classify verifier findings. +const ( + VerificationSeverityError = "error" + VerificationSeverityInfo = "info" +) + +// VerifyHistoryRangeOptions selects the history range and optional evidence object to verify. +// +// SkipEventArtifacts is intended for publisher preflight checks before existing +// PostgreSQL rows have been backfilled into WORM history_event artifacts. Normal +// verification should leave it false so missing or modified event artifacts are +// reported. +type VerifyHistoryRangeOptions struct { + HistoryTable string + Identifier string + FirstHistoryID int64 + LastHistoryID int64 + Manifest *HistoryManifest + EvidenceStore EvidenceStore + ManifestArtifactRef EvidenceReference + ManifestArtifactHash string + SkipEventArtifacts bool +} + +// HistoryEvidenceVerificationReport summarizes hash-chain, manifest, and artifact verification. +// +//revive:disable-next-line:exported +type HistoryEvidenceVerificationReport struct { + Valid bool `json:"valid"` + HistoryTable string `json:"history_table"` + Identifier string `json:"identifier,omitempty"` + FirstHistoryID int64 `json:"first_history_id"` + LastHistoryID int64 `json:"last_history_id"` + FirstRowHash string `json:"first_row_hash,omitempty"` + LastRowHash string `json:"last_row_hash,omitempty"` + RowCount int64 `json:"row_count"` + RangeDigest string `json:"range_digest,omitempty"` + Findings []VerificationFinding `json:"findings,omitempty"` +} + +// VerificationFinding records one verifier observation. +type VerificationFinding struct { + Severity string `json:"severity"` + Code string `json:"code"` + Message string `json:"message"` + Identifier string `json:"identifier,omitempty"` + HistoryID int64 `json:"history_id,omitempty"` +} + +type manifestDBRow struct { + HistoryID int64 + Identifier string + RowHash string +} + +type eventArtifactReceiptRow struct { + ArtifactID int64 + Identifier string + HistoryID int64 + RowHash string + ContentHash string + Receipt EvidenceReceipt +} + +// VerifyHistoryRange verifies PostgreSQL history rows against the hash chain and optional WORM evidence. +// +// The verifier reports missing, modified, reordered, or overwritten history +// records by checking row hashes, chain continuity, range manifests, per-row +// history_event receipts, and object-store artifact hashes when an EvidenceStore +// is configured. +// +// Parameters: +// - ctx: Request context for PostgreSQL and optional evidence-store reads. +// - db: Database handle connected to the BaSyx PostgreSQL database. +// - options: History range plus optional manifest and evidence-store references. +// +// Returns: +// - *HistoryEvidenceVerificationReport: Verification status and findings. +// - error: Error when inputs are invalid or required rows cannot be loaded. +func VerifyHistoryRange(ctx context.Context, db *sql.DB, options VerifyHistoryRangeOptions) (*HistoryEvidenceVerificationReport, error) { + if db == nil { + return nil, common.NewErrBadRequest("HISTORY-EVIDENCE-VERIFY-NILDB database handle must not be nil") + } + if err := validateVerifyHistoryRangeOptions(options); err != nil { + return nil, err + } + rows, err := loadManifestRangeDBRows(ctx, db, options.HistoryTable, options.Identifier, options.FirstHistoryID, options.LastHistoryID) + if err != nil { + return nil, err + } + report := newVerificationReport(options, rows) + if len(rows) == 0 { + report.addFinding(VerificationSeverityError, "HISTORY-EVIDENCE-VERIFY-EMPTYRANGE", "no history rows exist in the requested range", "", 0) + return report, nil + } + verifyRangeDigest(report, rows) + verifyManifestRange(report, options.Manifest) + verifyChainsByIdentifier(ctx, db, report, options.HistoryTable, rows) + if !options.SkipEventArtifacts { + verifyHistoryEventArtifacts(ctx, db, options, report) + } + verifyManifestArtifact(ctx, options, report) + report.Valid = len(report.Findings) == 0 + return report, nil +} + +// LoadManifestRangeRows returns ordered row-hash inputs for manifest generation. +// +// Parameters: +// - ctx: Request context for reading history metadata. +// - db: Database handle connected to the BaSyx PostgreSQL database. +// - table: History table to read. +// - identifier: Optional entity identifier scope. Empty means all identifiers. +// - firstHistoryID: Inclusive lower history_id bound. +// - lastHistoryID: Inclusive upper history_id bound. +// +// Returns: +// - []ManifestRangeRow: Ordered history_id and row_hash pairs. +// - error: Error when the table is unsupported or rows cannot be read. +func LoadManifestRangeRows(ctx context.Context, db *sql.DB, table string, identifier string, firstHistoryID int64, lastHistoryID int64) ([]ManifestRangeRow, error) { + rows, err := loadManifestRangeDBRows(ctx, db, table, identifier, firstHistoryID, lastHistoryID) + if err != nil { + return nil, err + } + manifestRows := make([]ManifestRangeRow, 0, len(rows)) + for _, row := range rows { + manifestRows = append(manifestRows, ManifestRangeRow{HistoryID: row.HistoryID, RowHash: row.RowHash}) + } + return manifestRows, nil +} + +func validateVerifyHistoryRangeOptions(options VerifyHistoryRangeOptions) error { + if _, err := historyPayloadTable(options.HistoryTable); err != nil { + return err + } + if options.FirstHistoryID < 1 { + return common.NewErrBadRequest("HISTORY-EVIDENCE-VERIFY-FIRSTID first history_id must be positive") + } + if options.LastHistoryID < options.FirstHistoryID { + return common.NewErrBadRequest("HISTORY-EVIDENCE-VERIFY-LASTID last history_id must be greater than or equal to first history_id") + } + return nil +} + +func loadManifestRangeDBRows(ctx context.Context, queryer historyQueryer, table string, identifier string, firstHistoryID int64, lastHistoryID int64) ([]manifestDBRow, error) { + if _, err := historyPayloadTable(table); err != nil { + return nil, err + } + dataset := goqu.From(table). + Select(goqu.C("history_id"), goqu.C("identifier"), goqu.C("row_hash")). + Where( + goqu.C("history_id").Gte(firstHistoryID), + goqu.C("history_id").Lte(lastHistoryID), + ) + if strings.TrimSpace(identifier) != "" { + dataset = dataset.Where(goqu.C("identifier").Eq(strings.TrimSpace(identifier))) + } + query, args, err := dataset.Order(goqu.C("history_id").Asc()).ToSQL() + if err != nil { + return nil, common.NewInternalServerError("HISTORY-EVIDENCE-VERIFY-BUILDRANGE " + err.Error()) + } + sqlRows, err := queryer.QueryContext(ctx, query, args...) + if err != nil { + return nil, common.NewInternalServerError("HISTORY-EVIDENCE-VERIFY-READRANGE " + err.Error()) + } + defer func() { + _ = sqlRows.Close() + }() + return scanManifestDBRows(sqlRows) +} + +func scanManifestDBRows(sqlRows *sql.Rows) ([]manifestDBRow, error) { + rows := make([]manifestDBRow, 0) + for sqlRows.Next() { + var row manifestDBRow + var rowHash sql.NullString + if err := sqlRows.Scan(&row.HistoryID, &row.Identifier, &rowHash); err != nil { + return nil, common.NewInternalServerError("HISTORY-EVIDENCE-VERIFY-SCANRANGE " + err.Error()) + } + row.RowHash = strings.TrimSpace(nullStringValue(rowHash)) + rows = append(rows, row) + } + if err := sqlRows.Err(); err != nil { + return nil, common.NewInternalServerError("HISTORY-EVIDENCE-VERIFY-RANGEROWS " + err.Error()) + } + return rows, nil +} + +func newVerificationReport(options VerifyHistoryRangeOptions, rows []manifestDBRow) *HistoryEvidenceVerificationReport { + report := &HistoryEvidenceVerificationReport{ + Valid: true, + HistoryTable: options.HistoryTable, + Identifier: strings.TrimSpace(options.Identifier), + FirstHistoryID: options.FirstHistoryID, + LastHistoryID: options.LastHistoryID, + RowCount: int64(len(rows)), + } + if len(rows) > 0 { + report.FirstRowHash = rows[0].RowHash + report.LastRowHash = rows[len(rows)-1].RowHash + } + return report +} + +func verifyRangeDigest(report *HistoryEvidenceVerificationReport, rows []manifestDBRow) { + manifestRows := make([]ManifestRangeRow, 0, len(rows)) + for _, row := range rows { + if row.RowHash == "" { + report.addFinding(VerificationSeverityError, "HISTORY-EVIDENCE-VERIFY-MISSINGROWHASH", "row_hash is missing", row.Identifier, row.HistoryID) + } + manifestRows = append(manifestRows, ManifestRangeRow{HistoryID: row.HistoryID, RowHash: row.RowHash}) + } + rangeDigest, err := ComputeRangeDigest(manifestRows) + if err != nil { + report.addFinding(VerificationSeverityError, "HISTORY-EVIDENCE-VERIFY-RANGEDIGEST", err.Error(), "", 0) + return + } + report.RangeDigest = rangeDigest +} + +func verifyManifestRange(report *HistoryEvidenceVerificationReport, manifest *HistoryManifest) { + if manifest == nil { + return + } + if manifest.HistoryTable != report.HistoryTable { + report.addFinding(VerificationSeverityError, "HISTORY-EVIDENCE-VERIFY-MANIFESTTABLE", "manifest history table does not match requested table", "", 0) + } + if strings.TrimSpace(manifest.Identifier) != strings.TrimSpace(report.Identifier) { + report.addFinding(VerificationSeverityError, "HISTORY-EVIDENCE-VERIFY-MANIFESTIDENTIFIER", "manifest identifier does not match requested identifier", "", 0) + } + if manifest.FirstHistoryID != report.FirstHistoryID || manifest.LastHistoryID != report.LastHistoryID { + report.addFinding(VerificationSeverityError, "HISTORY-EVIDENCE-VERIFY-MANIFESTRANGE", "manifest history_id range does not match requested range", "", 0) + } + if manifest.RowCount != report.RowCount { + report.addFinding(VerificationSeverityError, "HISTORY-EVIDENCE-VERIFY-MANIFESTROWCOUNT", "manifest row count does not match PostgreSQL rows", "", 0) + } + if manifest.FirstRowHash != report.FirstRowHash || manifest.LastRowHash != report.LastRowHash { + report.addFinding(VerificationSeverityError, "HISTORY-EVIDENCE-VERIFY-MANIFESTROWHASH", "manifest first or last row hash does not match PostgreSQL rows", "", 0) + } + if manifest.RangeDigest != report.RangeDigest { + report.addFinding(VerificationSeverityError, "HISTORY-EVIDENCE-VERIFY-MANIFESTDIGEST", "manifest range digest does not match PostgreSQL rows", "", 0) + } +} + +func verifyChainsByIdentifier(ctx context.Context, queryer historyQueryer, report *HistoryEvidenceVerificationReport, table string, rows []manifestDBRow) { + ranges := identifierRanges(rows) + for identifier, historyRange := range ranges { + if err := verifyIdentifierChain(ctx, queryer, table, identifier, historyRange.firstID, historyRange.lastID); err != nil { + report.addFinding(VerificationSeverityError, "HISTORY-EVIDENCE-VERIFY-CHAIN", err.Error(), identifier, historyRange.lastID) + } + } +} + +type identifierHistoryRange struct { + firstID int64 + lastID int64 +} + +func identifierRanges(rows []manifestDBRow) map[string]identifierHistoryRange { + ranges := make(map[string]identifierHistoryRange) + for _, row := range rows { + current, exists := ranges[row.Identifier] + if !exists { + ranges[row.Identifier] = identifierHistoryRange{firstID: row.HistoryID, lastID: row.HistoryID} + continue + } + if row.HistoryID < current.firstID { + current.firstID = row.HistoryID + } + if row.HistoryID > current.lastID { + current.lastID = row.HistoryID + } + ranges[row.Identifier] = current + } + return ranges +} + +func verifyIdentifierChain(ctx context.Context, queryer historyQueryer, table string, identifier string, firstHistoryID int64, lastHistoryID int64) error { + payloadTable, err := historyPayloadTable(table) + if err != nil { + return err + } + checkpointID, err := nearestSnapshotHistoryID(ctx, queryer, table, identifier, firstHistoryID) + if err != nil { + return err + } + rows, err := loadVersionChain(ctx, queryer, table, payloadTable, identifier, checkpointID, lastHistoryID) + if err != nil { + return err + } + if _, err = restoreVersionChainRows(rows); err != nil { + return err + } + return nil +} + +func verifyManifestArtifact(ctx context.Context, options VerifyHistoryRangeOptions, report *HistoryEvidenceVerificationReport) { + if options.EvidenceStore == nil || strings.TrimSpace(options.ManifestArtifactRef.ObjectKey) == "" || strings.TrimSpace(options.ManifestArtifactHash) == "" { + return + } + if _, err := options.EvidenceStore.VerifyArtifact(ctx, options.ManifestArtifactRef, options.ManifestArtifactHash); err != nil { + report.addFinding(VerificationSeverityError, "HISTORY-EVIDENCE-VERIFY-ARTIFACT", fmt.Sprintf("manifest artifact verification failed: %v", err), "", 0) + } +} + +func verifyHistoryEventArtifacts(ctx context.Context, db *sql.DB, options VerifyHistoryRangeOptions, report *HistoryEvidenceVerificationReport) { + candidates, err := LoadEventArtifactCandidates(ctx, db, options.HistoryTable, options.Identifier, options.FirstHistoryID, options.LastHistoryID) + if err != nil { + report.addFinding(VerificationSeverityError, "HISTORY-EVIDENCE-VERIFY-EVENTLOAD", err.Error(), "", 0) + return + } + receipts, err := loadEventArtifactReceiptRows(ctx, db, options.HistoryTable, options.Identifier, options.FirstHistoryID, options.LastHistoryID) + if err != nil { + report.addFinding(VerificationSeverityError, "HISTORY-EVIDENCE-VERIFY-EVENTRECEIPTS", err.Error(), "", 0) + return + } + receiptIndex := indexEventArtifactReceipts(receipts, report) + for _, candidate := range candidates { + verifyEventArtifactCandidate(ctx, options, report, candidate, receiptIndex[eventArtifactKey(candidate.Identifier, candidate.HistoryID, candidate.RowHash)]) + } +} + +func verifyEventArtifactCandidate(ctx context.Context, options VerifyHistoryRangeOptions, report *HistoryEvidenceVerificationReport, candidate EventArtifactCandidate, receipt *eventArtifactReceiptRow) { + if receipt == nil { + report.addFinding(VerificationSeverityError, "HISTORY-EVIDENCE-VERIFY-EVENTMISSING", "history_event artifact receipt is missing", candidate.Identifier, candidate.HistoryID) + return + } + expectedSHA := SHA256Hex(candidate.Artifact.Data) + if !strings.EqualFold(receipt.Receipt.SHA256, expectedSHA) { + report.addFinding(VerificationSeverityError, "HISTORY-EVIDENCE-VERIFY-EVENTSHA", "history_event receipt SHA-256 does not match PostgreSQL row artifact", candidate.Identifier, candidate.HistoryID) + } + if receipt.ContentHash != "" && receipt.ContentHash != candidate.ContentHash { + report.addFinding(VerificationSeverityError, "HISTORY-EVIDENCE-VERIFY-EVENTCONTENTHASH", "history_event receipt content_hash does not match PostgreSQL row", candidate.Identifier, candidate.HistoryID) + } + if receipt.Receipt.RetentionMode == "" || receipt.Receipt.RetainUntil == nil { + report.addFinding(VerificationSeverityError, "HISTORY-EVIDENCE-VERIFY-EVENTRETENTION", "history_event receipt is missing WORM retention metadata", candidate.Identifier, candidate.HistoryID) + } + if options.EvidenceStore == nil { + return + } + if _, err := options.EvidenceStore.VerifyArtifact(ctx, receipt.Receipt.Reference, receipt.Receipt.SHA256); err != nil { + report.addFinding(VerificationSeverityError, "HISTORY-EVIDENCE-VERIFY-EVENTOBJECT", fmt.Sprintf("history_event artifact verification failed: %v", err), candidate.Identifier, candidate.HistoryID) + } +} + +func loadEventArtifactReceiptRows(ctx context.Context, queryer historyQueryer, table string, identifier string, firstHistoryID int64, lastHistoryID int64) ([]eventArtifactReceiptRow, error) { + dataset := goqu.From(TableHistoryEvidenceArtifacts). + Select( + goqu.C("artifact_id"), + goqu.C("identifier"), + goqu.C("history_id"), + goqu.C("row_hash"), + goqu.C("content_hash"), + goqu.C("provider"), + goqu.C("bucket"), + goqu.C("object_key"), + goqu.C("object_version_id"), + goqu.C("sha256"), + goqu.C("size_bytes"), + goqu.C("content_type"), + goqu.C("retention_mode"), + goqu.C("retain_until"), + goqu.C("legal_hold"), + ). + Where( + goqu.C("artifact_type").Eq(EvidenceArtifactHistoryEvent), + goqu.C("history_table").Eq(table), + goqu.C("history_id").Gte(firstHistoryID), + goqu.C("history_id").Lte(lastHistoryID), + ) + if strings.TrimSpace(identifier) != "" { + dataset = dataset.Where(goqu.C("identifier").Eq(strings.TrimSpace(identifier))) + } + query, args, err := dataset.Order(goqu.C("history_id").Asc(), goqu.C("artifact_id").Asc()).ToSQL() + if err != nil { + return nil, common.NewInternalServerError("HISTORY-EVIDENCE-VERIFY-BUILDEVENTRECEIPTS " + err.Error()) + } + rows, err := queryer.QueryContext(ctx, query, args...) + if err != nil { + return nil, common.NewInternalServerError("HISTORY-EVIDENCE-VERIFY-READEVENTRECEIPTS " + err.Error()) + } + defer func() { + _ = rows.Close() + }() + return scanEventArtifactReceiptRows(rows) +} + +func scanEventArtifactReceiptRows(rows *sql.Rows) ([]eventArtifactReceiptRow, error) { + receipts := make([]eventArtifactReceiptRow, 0) + for rows.Next() { + receipt, err := scanEventArtifactReceiptRow(rows) + if err != nil { + return nil, err + } + receipts = append(receipts, receipt) + } + if err := rows.Err(); err != nil { + return nil, common.NewInternalServerError("HISTORY-EVIDENCE-VERIFY-EVENTRECEIPTROWS " + err.Error()) + } + return receipts, nil +} + +func scanEventArtifactReceiptRow(rows *sql.Rows) (eventArtifactReceiptRow, error) { + var row eventArtifactReceiptRow + var identifier sql.NullString + var rowHash sql.NullString + var contentHash sql.NullString + var bucket sql.NullString + var versionID sql.NullString + var retentionMode sql.NullString + var retainUntil sql.NullTime + if err := rows.Scan( + &row.ArtifactID, + &identifier, + &row.HistoryID, + &rowHash, + &contentHash, + &row.Receipt.Reference.Provider, + &bucket, + &row.Receipt.Reference.ObjectKey, + &versionID, + &row.Receipt.SHA256, + &row.Receipt.SizeBytes, + &row.Receipt.ContentType, + &retentionMode, + &retainUntil, + &row.Receipt.LegalHold, + ); err != nil { + return eventArtifactReceiptRow{}, common.NewInternalServerError("HISTORY-EVIDENCE-VERIFY-SCANEVENTRECEIPT " + err.Error()) + } + row.Identifier = nullStringValue(identifier) + row.RowHash = nullStringValue(rowHash) + row.ContentHash = nullStringValue(contentHash) + row.Receipt.Reference.Bucket = nullStringValue(bucket) + row.Receipt.Reference.VersionID = nullStringValue(versionID) + row.Receipt.RetentionMode = nullStringValue(retentionMode) + if retainUntil.Valid { + retainUntilUTC := retainUntil.Time.UTC() + row.Receipt.RetainUntil = &retainUntilUTC + } + return row, nil +} + +func indexEventArtifactReceipts(receipts []eventArtifactReceiptRow, report *HistoryEvidenceVerificationReport) map[string]*eventArtifactReceiptRow { + index := make(map[string]*eventArtifactReceiptRow, len(receipts)) + for i := range receipts { + receipt := &receipts[i] + key := eventArtifactKey(receipt.Identifier, receipt.HistoryID, receipt.RowHash) + if existing := index[key]; existing != nil { + report.addFinding(VerificationSeverityError, "HISTORY-EVIDENCE-VERIFY-EVENTDUPLICATE", "multiple history_event receipts exist for one history row", receipt.Identifier, receipt.HistoryID) + continue + } + index[key] = receipt + } + return index +} + +func eventArtifactKey(identifier string, historyID int64, rowHash string) string { + return strings.TrimSpace(identifier) + "\x00" + fmt.Sprintf("%d", historyID) + "\x00" + strings.TrimSpace(rowHash) +} + +func (report *HistoryEvidenceVerificationReport) addFinding(severity string, code string, message string, identifier string, historyID int64) { + report.Valid = false + report.Findings = append(report.Findings, VerificationFinding{ + Severity: severity, + Code: code, + Message: message, + Identifier: identifier, + HistoryID: historyID, + }) +} diff --git a/internal/common/history/evidence_writer.go b/internal/common/history/evidence_writer.go new file mode 100644 index 000000000..209dc2429 --- /dev/null +++ b/internal/common/history/evidence_writer.go @@ -0,0 +1,234 @@ +/******************************************************************************* +* Copyright (C) 2026 the Eclipse BaSyx Authors and Fraunhofer IESE +* +* Permission is hereby granted, free of charge, to any person obtaining +* a copy of this software and associated documentation files (the +* "Software"), to deal in the Software without restriction, including +* without limitation the rights to use, copy, modify, merge, publish, +* distribute, sublicense, and/or sell copies of the Software, and to +* permit persons to whom the Software is furnished to do so, subject to +* the following conditions: +* +* The above copyright notice and this permission notice shall be +* included in all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +* +* SPDX-License-Identifier: MIT +******************************************************************************/ + +package history + +import ( + "context" + "database/sql" + "fmt" + "strings" + "time" + + "github.com/eclipse-basyx/basyx-go-components/internal/common" +) + +// WriteHistoryEvidenceOptions selects the history range to publish as WORM evidence. +type WriteHistoryEvidenceOptions struct { + DB *sql.DB + Store EvidenceStore + HistoryTable string + Identifier string + FirstHistoryID int64 + LastHistoryID int64 + GeneratedAt time.Time + Signer *ManifestJWSSigner +} + +// WriteHistoryEvidenceResult summarizes the evidence publication. +type WriteHistoryEvidenceResult struct { + Manifest HistoryManifest + ManifestReceipt EvidenceReceipt + EventReceipts []EvidenceCatalogEventReceipt + SnapshotReceipts []EvidenceCatalogSnapshotReceipt + ManifestID int64 + VerificationReport *HistoryEvidenceVerificationReport +} + +// WriteHistoryEvidence verifies a history range, stores event/snapshot/manifest artifacts, and records receipts. +// +// The function first verifies PostgreSQL hashes, then backfills every row in the +// requested range as a history_event artifact, stores snapshot checkpoints and a +// manifest artifact, and finally records all receipts in PostgreSQL. It is used +// by tooling and does not change the HTTP history API. +// +// Parameters: +// - ctx: Request context for database and evidence-store operations. +// - options: History range, evidence store, database handle, timestamp, and signer. +// +// Returns: +// - *WriteHistoryEvidenceResult: Stored manifest, artifact receipts, catalog ID, +// and PostgreSQL verification report. +// - error: Error when PostgreSQL verification fails, WORM writes fail, or +// catalog persistence fails. +func WriteHistoryEvidence(ctx context.Context, options WriteHistoryEvidenceOptions) (*WriteHistoryEvidenceResult, error) { + if err := validateWriteHistoryEvidenceOptions(options); err != nil { + return nil, err + } + report, err := VerifyHistoryRange(ctx, options.DB, VerifyHistoryRangeOptions{ + HistoryTable: options.HistoryTable, + Identifier: options.Identifier, + FirstHistoryID: options.FirstHistoryID, + LastHistoryID: options.LastHistoryID, + SkipEventArtifacts: true, + }) + if err != nil { + return nil, err + } + if !report.Valid { + return nil, fmt.Errorf("HISTORY-EVIDENCE-WRITE-VERIFYFAILED PostgreSQL history range is not verifiable: %s", firstFindingMessage(report)) + } + rows, err := LoadManifestRangeRows(ctx, options.DB, options.HistoryTable, options.Identifier, options.FirstHistoryID, options.LastHistoryID) + if err != nil { + return nil, err + } + eventReceipts, err := storeHistoryEventArtifacts(ctx, options) + if err != nil { + return nil, err + } + snapshotReceipts, snapshotRefs, err := storeSnapshotArtifacts(ctx, options) + if err != nil { + return nil, err + } + manifest, err := BuildHistoryManifest(HistoryManifestOptions{ + HistoryTable: options.HistoryTable, + Identifier: options.Identifier, + Rows: rows, + SnapshotReferences: snapshotRefs, + GeneratedAt: options.GeneratedAt, + }) + if err != nil { + return nil, err + } + manifestReceipt, manifest, err := storeManifestArtifact(ctx, options.Store, manifest, options.Signer) + if err != nil { + return nil, err + } + manifestID, err := recordEvidenceCatalog(ctx, options.DB, manifest, manifestReceipt, eventReceipts, snapshotReceipts) + if err != nil { + return nil, err + } + return &WriteHistoryEvidenceResult{ + Manifest: manifest, + ManifestReceipt: *manifestReceipt, + EventReceipts: eventReceipts, + SnapshotReceipts: snapshotReceipts, + ManifestID: manifestID, + VerificationReport: report, + }, nil +} + +func validateWriteHistoryEvidenceOptions(options WriteHistoryEvidenceOptions) error { + if options.DB == nil { + return common.NewErrBadRequest("HISTORY-EVIDENCE-WRITE-NILDB database handle must not be nil") + } + if options.Store == nil { + return common.NewErrBadRequest("HISTORY-EVIDENCE-WRITE-NILSTORE evidence store must not be nil") + } + return validateVerifyHistoryRangeOptions(VerifyHistoryRangeOptions{ + HistoryTable: options.HistoryTable, + Identifier: options.Identifier, + FirstHistoryID: options.FirstHistoryID, + LastHistoryID: options.LastHistoryID, + }) +} + +func storeHistoryEventArtifacts(ctx context.Context, options WriteHistoryEvidenceOptions) ([]EvidenceCatalogEventReceipt, error) { + candidates, err := LoadEventArtifactCandidates(ctx, options.DB, options.HistoryTable, options.Identifier, options.FirstHistoryID, options.LastHistoryID) + if err != nil { + return nil, err + } + receipts := make([]EvidenceCatalogEventReceipt, 0, len(candidates)) + for _, candidate := range candidates { + receipt, err := options.Store.PutArtifact(ctx, candidate.Artifact) + if err != nil { + return nil, fmt.Errorf("HISTORY-EVIDENCE-WRITE-PUTEVENT %w", err) + } + if receipt == nil { + return nil, fmt.Errorf("HISTORY-EVIDENCE-WRITE-NILEVENTRECEIPT evidence store returned nil receipt") + } + receipts = append(receipts, EvidenceCatalogEventReceipt{Candidate: candidate, Receipt: *receipt}) + } + return receipts, nil +} + +func storeSnapshotArtifacts(ctx context.Context, options WriteHistoryEvidenceOptions) ([]EvidenceCatalogSnapshotReceipt, []SnapshotArtifactReference, error) { + candidates, err := LoadSnapshotArtifactCandidates(ctx, options.DB, options.HistoryTable, options.Identifier, options.FirstHistoryID, options.LastHistoryID) + if err != nil { + return nil, nil, err + } + receipts := make([]EvidenceCatalogSnapshotReceipt, 0, len(candidates)) + refs := make([]SnapshotArtifactReference, 0, len(candidates)) + for _, candidate := range candidates { + receipt, err := options.Store.PutArtifact(ctx, candidate.Artifact) + if err != nil { + return nil, nil, fmt.Errorf("HISTORY-EVIDENCE-WRITE-PUTSNAPSHOT %w", err) + } + ref, err := candidate.SnapshotReference(receipt) + if err != nil { + return nil, nil, err + } + receipts = append(receipts, EvidenceCatalogSnapshotReceipt{Candidate: candidate, Receipt: *receipt}) + refs = append(refs, ref) + } + return receipts, refs, nil +} + +func storeManifestArtifact(ctx context.Context, store EvidenceStore, manifest HistoryManifest, signer *ManifestJWSSigner) (*EvidenceReceipt, HistoryManifest, error) { + artifact, finalManifest, err := BuildManifestEvidenceArtifact(manifest, "", signer) + if err != nil { + return nil, HistoryManifest{}, err + } + receipt, err := store.PutArtifact(ctx, artifact) + if err != nil { + return nil, HistoryManifest{}, fmt.Errorf("HISTORY-EVIDENCE-WRITE-PUTMANIFEST %w", err) + } + return receipt, finalManifest, nil +} + +func recordEvidenceCatalog(ctx context.Context, db *sql.DB, manifest HistoryManifest, manifestReceipt *EvidenceReceipt, eventReceipts []EvidenceCatalogEventReceipt, snapshotReceipts []EvidenceCatalogSnapshotReceipt) (int64, error) { + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return 0, common.NewInternalServerError("HISTORY-EVIDENCE-WRITE-BEGINTX " + err.Error()) + } + committed := false + defer func() { + if !committed { + _ = tx.Rollback() + } + }() + manifestID, err := RecordEvidenceCatalogTx(ctx, tx, EvidenceCatalogRecord{ + Manifest: manifest, + ManifestReceipt: *manifestReceipt, + EventReceipts: eventReceipts, + SnapshotReceipts: snapshotReceipts, + }) + if err != nil { + return 0, err + } + if err = tx.Commit(); err != nil { + return 0, common.NewInternalServerError("HISTORY-EVIDENCE-WRITE-COMMIT " + err.Error()) + } + committed = true + return manifestID, nil +} + +func firstFindingMessage(report *HistoryEvidenceVerificationReport) string { + if report == nil || len(report.Findings) == 0 { + return "unknown verifier failure" + } + finding := report.Findings[0] + return strings.TrimSpace(finding.Code + " " + finding.Message) +} diff --git a/internal/common/history/history_event_artifact.go b/internal/common/history/history_event_artifact.go new file mode 100644 index 000000000..2d7fdd58c --- /dev/null +++ b/internal/common/history/history_event_artifact.go @@ -0,0 +1,301 @@ +/******************************************************************************* +* Copyright (C) 2026 the Eclipse BaSyx Authors and Fraunhofer IESE +* +* Permission is hereby granted, free of charge, to any person obtaining +* a copy of this software and associated documentation files (the +* "Software"), to deal in the Software without restriction, including +* without limitation the rights to use, copy, modify, merge, publish, +* distribute, sublicense, and/or sell copies of the Software, and to +* permit persons to whom the Software is furnished to do so, subject to +* the following conditions: +* +* The above copyright notice and this permission notice shall be +* included in all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +* +* SPDX-License-Identifier: MIT +******************************************************************************/ + +package history + +import ( + "context" + "database/sql" + "fmt" + "net/url" + "path" + "strings" + "time" + + "github.com/doug-martin/goqu/v9" + "github.com/eclipse-basyx/basyx-go-components/internal/common" +) + +const historyEventArtifactVersion = "basyx-history-event-v1" + +// EventArtifactCandidate is one history row prepared for WORM event storage. +type EventArtifactCandidate struct { + Artifact EvidenceArtifact + HistoryTable string + Identifier string + HistoryID int64 + RowHash string + ContentHash string +} + +// LoadEventArtifactCandidates loads snapshot and diff rows as WORM history-event artifacts. +// +// The returned artifacts use the same canonical representation as the +// synchronous append path. Callers can use this to backfill existing PostgreSQL +// history rows into WORM evidence storage before publishing a range manifest. +// +// Parameters: +// - ctx: Request context for reading history rows. +// - db: Database handle connected to the BaSyx PostgreSQL database. +// - table: History table to read, for example aas_history. +// - identifier: Optional entity identifier scope. Empty means all identifiers. +// - firstHistoryID: Inclusive lower history_id bound. +// - lastHistoryID: Inclusive upper history_id bound. +// +// Returns: +// - []EventArtifactCandidate: Canonical per-row artifacts in history_id order. +// - error: Error when the table is unsupported, hashes do not validate, or +// rows cannot be read. +func LoadEventArtifactCandidates(ctx context.Context, db *sql.DB, table string, identifier string, firstHistoryID int64, lastHistoryID int64) ([]EventArtifactCandidate, error) { + if db == nil { + return nil, common.NewErrBadRequest("HISTORY-EVIDENCE-EVENTARTIFACT-NILDB database handle must not be nil") + } + payloadTable, err := historyPayloadTable(table) + if err != nil { + return nil, err + } + query, args, err := historyEventArtifactCandidateQuery(table, payloadTable, identifier, firstHistoryID, lastHistoryID) + if err != nil { + return nil, err + } + rows, err := db.QueryContext(ctx, query, args...) + if err != nil { + return nil, common.NewInternalServerError("HISTORY-EVIDENCE-EVENTARTIFACT-EXECQUERY " + err.Error()) + } + defer func() { + _ = rows.Close() + }() + return scanEventArtifactCandidates(rows, table) +} + +func buildHistoryEventEvidenceArtifact(table string, historyID int64, event ChangeEvent, payload historyPayload, createdAt string, updatedAt string) (EvidenceArtifact, error) { + payloadValue, err := decodeHistoryEventPayload(payload) + if err != nil { + return EvidenceArtifact{}, err + } + rowHash := strings.TrimSpace(event.RowHash) + if rowHash == "" { + return EvidenceArtifact{}, common.NewInternalServerError("HISTORY-EVIDENCE-EVENTARTIFACT-ROWHASH row hash is required") + } + data, err := CanonicalJSON(map[string]any{ + "artifact_version": historyEventArtifactVersion, + "hash_contract": historyRowHashContract, + "history_table": table, + "history_id": historyID, + "identifier": event.Identifier, + "change_type": event.ChangeType, + "deleted": event.Deleted, + "valid_from": event.Timestamp.UTC().Format(time.RFC3339Nano), + "operation_time": event.Timestamp.UTC().Format(time.RFC3339Nano), + "administration": map[string]any{ + "created_at_text": createdAt, + "updated_at_text": updatedAt, + }, + "payload_type": payload.payloadType, + "payload": payloadValue, + "content_hash": event.ContentHash, + "payload_hash": event.PayloadHash, + "previous_hash": event.PreviousHash, + "row_hash": rowHash, + "audit": map[string]any{ + "request_id": event.RequestID, + "correlation_id": event.CorrelationID, + "actor_subject": event.ActorSubject, + "actor_issuer": event.ActorIssuer, + "client_id": event.ClientID, + "authorization_result": event.AuthorizationResult, + "policy_id": event.PolicyID, + "matched_rule_id": event.MatchedRuleID, + "source_ip": event.SourceIP, + "user_agent": event.UserAgent, + "operation": event.Operation, + "endpoint": event.Endpoint, + "http_method": event.HTTPMethod, + }, + }) + if err != nil { + return EvidenceArtifact{}, common.NewInternalServerError("HISTORY-EVIDENCE-EVENTARTIFACT-CANONICAL " + err.Error()) + } + return EvidenceArtifact{ + ArtifactType: EvidenceArtifactHistoryEvent, + ObjectKey: historyEventObjectKey(table, event.Identifier, historyID, rowHash), + ContentType: manifestJSONContentType, + Data: data, + Metadata: historyEventMetadata(table, event.Identifier, historyID, rowHash), + }, nil +} + +func publishHistoryEventEvidenceTx(ctx context.Context, tx *sql.Tx, cfg Config, table string, historyID int64, event ChangeEvent, payload historyPayload, createdAt string, updatedAt string) error { + if !cfg.EvidenceEnabled { + return nil + } + if cfg.EvidenceStore == nil { + return common.NewInternalServerError("HISTORY-EVIDENCE-APPEND-NILSTORE evidence store is not initialized") + } + artifact, err := buildHistoryEventEvidenceArtifact(table, historyID, event, payload, createdAt, updatedAt) + if err != nil { + return err + } + writeCtx := ctx + cancel := func() {} + if cfg.EvidenceWriteTimeout > 0 { + writeCtx, cancel = context.WithTimeout(ctx, cfg.EvidenceWriteTimeout) + } + defer cancel() + receipt, err := cfg.EvidenceStore.PutArtifact(writeCtx, artifact) + if err != nil { + return common.NewInternalServerError("HISTORY-EVIDENCE-APPEND-PUTARTIFACT " + err.Error()) + } + if receipt == nil { + return common.NewInternalServerError("HISTORY-EVIDENCE-APPEND-NILRECEIPT evidence store returned nil receipt") + } + return RecordHistoryEventEvidenceArtifactTx(ctx, tx, EventEvidenceRecord{ + HistoryTable: table, + Identifier: event.Identifier, + HistoryID: historyID, + RowHash: event.RowHash, + ContentHash: event.ContentHash, + Receipt: *receipt, + }) +} + +func decodeHistoryEventPayload(payload historyPayload) (any, error) { + var payloadValue any + if err := decodeJSONPreservingNumbers(payload.json, &payloadValue); err != nil { + return nil, common.NewInternalServerError("HISTORY-EVIDENCE-EVENTARTIFACT-DECODE " + err.Error()) + } + return payloadValue, nil +} + +func historyEventObjectKey(table string, identifier string, historyID int64, rowHash string) string { + return path.Join( + "history-events", + url.PathEscape(table), + url.PathEscape(identifier), + fmt.Sprintf("%d-%s.json", historyID, strings.TrimSpace(rowHash)), + ) +} + +func historyEventMetadata(table string, identifier string, historyID int64, rowHash string) map[string]string { + return map[string]string{ + "artifact_type": EvidenceArtifactHistoryEvent, + "history_table": table, + "identifier": identifier, + "history_id": fmt.Sprintf("%d", historyID), + "row_hash": strings.TrimSpace(rowHash), + } +} + +func historyEventArtifactCandidateQuery(table string, payloadTable string, identifier string, firstHistoryID int64, lastHistoryID int64) (string, []any, error) { + historyAlias := goqu.T(table).As("history") + payloadAlias := goqu.T(payloadTable).As("payload") + dataset := baseVersionChainQuery(historyAlias, payloadAlias). + Where( + historyAlias.Col("history_id").Gte(firstHistoryID), + historyAlias.Col("history_id").Lte(lastHistoryID), + ) + if strings.TrimSpace(identifier) != "" { + dataset = dataset.Where(historyAlias.Col("identifier").Eq(strings.TrimSpace(identifier))) + } + return dataset.Order(historyAlias.Col("history_id").Asc()).ToSQL() +} + +func scanEventArtifactCandidates(rows *sql.Rows, table string) ([]EventArtifactCandidate, error) { + storedRows, err := scanStoredHistoryRows(rows, table) + if err != nil { + return nil, err + } + candidates := make([]EventArtifactCandidate, 0, len(storedRows)) + for _, row := range storedRows { + candidate, err := historyEventArtifactCandidateFromStoredRow(row) + if err != nil { + return nil, err + } + candidates = append(candidates, candidate) + } + return candidates, nil +} + +func historyEventArtifactCandidateFromStoredRow(row storedHistoryRow) (EventArtifactCandidate, error) { + payload, err := historyPayloadFromStoredRow(row) + if err != nil { + return EventArtifactCandidate{}, err + } + artifact, err := buildHistoryEventEvidenceArtifact( + row.EntityType, + row.HistoryID, + historyRowHashEvent(row), + payload, + nullStringValue(row.CreatedAt), + nullStringValue(row.UpdatedAt), + ) + if err != nil { + return EventArtifactCandidate{}, err + } + return EventArtifactCandidate{ + Artifact: artifact, + HistoryTable: row.EntityType, + Identifier: row.Identifier, + HistoryID: row.HistoryID, + RowHash: nullStringValue(row.RowHash), + ContentHash: nullStringValue(row.ContentHash), + }, nil +} + +func historyPayloadFromStoredRow(row storedHistoryRow) (historyPayload, error) { + payloadJSON, err := storedPayloadJSON(row) + if err != nil { + return historyPayload{}, err + } + var payloadValue any + if err = decodeJSONPreservingNumbers(payloadJSON, &payloadValue); err != nil { + return historyPayload{}, common.NewInternalServerError("HISTORY-EVIDENCE-EVENTARTIFACT-DECODESTORED " + err.Error()) + } + if err = verifyCanonicalHash(payloadValue, row.PayloadHash, "HISTORY-EVIDENCE-EVENTARTIFACT-PAYLOADHASH"); err != nil { + return historyPayload{}, err + } + return historyPayload{ + payloadType: row.PayloadType, + json: payloadJSON, + hash: nullStringValue(row.PayloadHash), + }, nil +} + +func storedPayloadJSON(row storedHistoryRow) ([]byte, error) { + switch row.PayloadType { + case PayloadTypeSnapshot: + if !row.Snapshot.Valid { + return nil, common.NewInternalServerError("HISTORY-EVIDENCE-EVENTARTIFACT-MISSINGSNAPSHOT snapshot payload is missing") + } + return []byte(row.Snapshot.String), nil + case PayloadTypeDiff: + if !row.Diff.Valid { + return nil, common.NewInternalServerError("HISTORY-EVIDENCE-EVENTARTIFACT-MISSINGDIFF diff payload is missing") + } + return []byte(row.Diff.String), nil + default: + return nil, common.NewInternalServerError("HISTORY-EVIDENCE-EVENTARTIFACT-PAYLOADTYPE unsupported payload type '" + row.PayloadType + "'") + } +} diff --git a/internal/common/history/manifest.go b/internal/common/history/manifest.go new file mode 100644 index 000000000..2ba400de8 --- /dev/null +++ b/internal/common/history/manifest.go @@ -0,0 +1,185 @@ +/******************************************************************************* +* Copyright (C) 2026 the Eclipse BaSyx Authors and Fraunhofer IESE +* +* Permission is hereby granted, free of charge, to any person obtaining +* a copy of this software and associated documentation files (the +* "Software"), to deal in the Software without restriction, including +* without limitation the rights to use, copy, modify, merge, publish, +* distribute, sublicense, and/or sell copies of the Software, and to +* permit persons to whom the Software is furnished to do so, subject to +* the following conditions: +* +* The above copyright notice and this permission notice shall be +* included in all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +* +* SPDX-License-Identifier: MIT +******************************************************************************/ + +package history + +import ( + "fmt" + "strings" + "time" +) + +// HistoryManifestOptions contains the ordered row range used to build a manifest. +// +//revive:disable-next-line:exported +type HistoryManifestOptions struct { + HistoryTable string + Identifier string + Rows []ManifestRangeRow + SnapshotReferences []SnapshotArtifactReference + GeneratedAt time.Time + SignatureState string + Signer *ManifestSignerInfo +} + +// BuildHistoryManifest creates a canonical manifest for an ordered hash-chain range. +// +// Parameters: +// - options: History table, ordered row hashes, optional snapshot references, +// generation timestamp, and signer metadata. +// +// Returns: +// - HistoryManifest: Deterministic range manifest ready for signing or storage. +// - error: Error when the range is empty, unordered, or missing required hashes. +func BuildHistoryManifest(options HistoryManifestOptions) (HistoryManifest, error) { + rows, err := validateManifestRows(options.Rows) + if err != nil { + return HistoryManifest{}, err + } + historyTable := strings.TrimSpace(options.HistoryTable) + if historyTable == "" { + return HistoryManifest{}, fmt.Errorf("HISTORY-MANIFEST-EMPTYTABLE history table is required") + } + rangeDigest, err := ComputeRangeDigest(rows) + if err != nil { + return HistoryManifest{}, err + } + generatedAt := options.GeneratedAt.UTC() + if generatedAt.IsZero() { + generatedAt = time.Now().UTC() + } + signatureState := normalizeSignatureState(options.SignatureState) + return HistoryManifest{ + ManifestVersion: HistoryManifestVersion, + HistoryTable: historyTable, + Identifier: strings.TrimSpace(options.Identifier), + FirstHistoryID: rows[0].HistoryID, + LastHistoryID: rows[len(rows)-1].HistoryID, + FirstRowHash: rows[0].RowHash, + LastRowHash: rows[len(rows)-1].RowHash, + RowCount: int64(len(rows)), + RangeDigest: rangeDigest, + GeneratedAt: generatedAt, + SignatureState: signatureState, + Signer: options.Signer, + SnapshotReferences: cloneSnapshotReferences(options.SnapshotReferences), + }, nil +} + +// ComputeRangeDigest hashes an ordered list of row hashes and their sequence metadata. +// +// Parameters: +// - rows: Strictly increasing history_id and row_hash pairs. +// +// Returns: +// - string: Canonical SHA-256 range digest. +// - error: Error when rows are empty, unordered, or contain empty row hashes. +func ComputeRangeDigest(rows []ManifestRangeRow) (string, error) { + orderedRows, err := validateManifestRows(rows) + if err != nil { + return "", err + } + return CanonicalJSONHash(map[string]any{ + "hashContract": historyRangeContract, + "rows": orderedRows, + }) +} + +// CanonicalManifestJSON returns the canonical JSON artifact bytes for a manifest. +// +// Parameters: +// - manifest: Valid history manifest. +// +// Returns: +// - []byte: Canonical JSON representation used for hashing and signing. +// - error: Error when manifest validation or canonical serialization fails. +func CanonicalManifestJSON(manifest HistoryManifest) ([]byte, error) { + if err := validateManifest(manifest); err != nil { + return nil, err + } + return CanonicalJSON(manifest) +} + +func validateManifest(manifest HistoryManifest) error { + if strings.TrimSpace(manifest.ManifestVersion) != HistoryManifestVersion { + return fmt.Errorf("HISTORY-MANIFEST-VERSION unsupported manifest version %q", manifest.ManifestVersion) + } + if strings.TrimSpace(manifest.HistoryTable) == "" { + return fmt.Errorf("HISTORY-MANIFEST-EMPTYTABLE history table is required") + } + if manifest.RowCount < 1 { + return fmt.Errorf("HISTORY-MANIFEST-ROWCOUNT row count must be positive") + } + if manifest.FirstHistoryID < 1 || manifest.LastHistoryID < manifest.FirstHistoryID { + return fmt.Errorf("HISTORY-MANIFEST-RANGE invalid history_id range") + } + if strings.TrimSpace(manifest.FirstRowHash) == "" || strings.TrimSpace(manifest.LastRowHash) == "" { + return fmt.Errorf("HISTORY-MANIFEST-ROWHASH first and last row hashes are required") + } + if strings.TrimSpace(manifest.RangeDigest) == "" { + return fmt.Errorf("HISTORY-MANIFEST-RANGEDIGEST range digest is required") + } + if normalizeSignatureState(manifest.SignatureState) != manifest.SignatureState { + return fmt.Errorf("HISTORY-MANIFEST-SIGNATURESTATE unsupported signature state %q", manifest.SignatureState) + } + return nil +} + +func validateManifestRows(rows []ManifestRangeRow) ([]ManifestRangeRow, error) { + if len(rows) == 0 { + return nil, fmt.Errorf("HISTORY-MANIFEST-EMPTYRANGE at least one history row is required") + } + cloned := make([]ManifestRangeRow, 0, len(rows)) + previousID := int64(0) + for _, row := range rows { + if row.HistoryID <= previousID { + return nil, fmt.Errorf("HISTORY-MANIFEST-ORDER history rows must be strictly increasing by history_id") + } + if strings.TrimSpace(row.RowHash) == "" { + return nil, fmt.Errorf("HISTORY-MANIFEST-EMPTYROWHASH row hash is required") + } + cloned = append(cloned, ManifestRangeRow{HistoryID: row.HistoryID, RowHash: strings.TrimSpace(row.RowHash)}) + previousID = row.HistoryID + } + return cloned, nil +} + +func normalizeSignatureState(state string) string { + switch strings.ToLower(strings.TrimSpace(state)) { + case SignatureStateSigned: + return SignatureStateSigned + default: + return SignatureStateUnsigned + } +} + +func cloneSnapshotReferences(refs []SnapshotArtifactReference) []SnapshotArtifactReference { + if len(refs) == 0 { + return nil + } + cloned := make([]SnapshotArtifactReference, len(refs)) + copy(cloned, refs) + return cloned +} diff --git a/internal/common/history/manifest_artifact.go b/internal/common/history/manifest_artifact.go new file mode 100644 index 000000000..ef675d7e7 --- /dev/null +++ b/internal/common/history/manifest_artifact.go @@ -0,0 +1,204 @@ +/******************************************************************************* +* Copyright (C) 2026 the Eclipse BaSyx Authors and Fraunhofer IESE +* +* Permission is hereby granted, free of charge, to any person obtaining +* a copy of this software and associated documentation files (the +* "Software"), to deal in the Software without restriction, including +* without limitation the rights to use, copy, modify, merge, publish, +* distribute, sublicense, and/or sell copies of the Software, and to +* permit persons to whom the Software is furnished to do so, subject to +* the following conditions: +* +* The above copyright notice and this permission notice shall be +* included in all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +* +* SPDX-License-Identifier: MIT +******************************************************************************/ + +package history + +import ( + "crypto/rsa" + "encoding/json" + "fmt" + "net/url" + "path" + "strings" + + commonjws "github.com/eclipse-basyx/basyx-go-components/internal/common/jws" + jose "gopkg.in/go-jose/go-jose.v2" +) + +const ( + manifestJSONContentType = "application/json" + manifestJWSContentType = "application/jose" + manifestSigningAlgRS256 = "RS256" +) + +// ManifestJWSSigner signs canonical manifests with a configured RSA key. +type ManifestJWSSigner struct { + PrivateKey *rsa.PrivateKey + KeyID string +} + +// NewManifestJWSSignerFromKeyFile loads a JWS signer for evidence manifests. +// +// Parameters: +// - privateKeyPath: Path to a PEM encoded RSA private key. +// - keyID: Optional key identifier to embed in manifest signer metadata. +// +// Returns: +// - *ManifestJWSSigner: Signer configured for RS256 compact JWS manifests. +// - error: Error when the key path is empty or the key cannot be loaded. +func NewManifestJWSSignerFromKeyFile(privateKeyPath string, keyID string) (*ManifestJWSSigner, error) { + privateKeyPath = strings.TrimSpace(privateKeyPath) + if privateKeyPath == "" { + return nil, fmt.Errorf("HISTORY-MANIFESTSIGNER-EMPTYPATH private key path is required") + } + privateKey, err := commonjws.LoadPrivateKey(privateKeyPath) + if err != nil { + return nil, fmt.Errorf("HISTORY-MANIFESTSIGNER-LOADKEY %w", err) + } + return &ManifestJWSSigner{PrivateKey: privateKey, KeyID: strings.TrimSpace(keyID)}, nil +} + +// BuildManifestEvidenceArtifact returns the signed or unsigned manifest object-store artifact. +// +// When signer is nil, the artifact contains unsigned canonical JSON and the +// returned manifest is marked unsigned. When signer is present, the artifact +// contains compact JWS over canonical manifest JSON and the returned manifest +// includes signer metadata. +// +// Parameters: +// - manifest: Manifest range metadata to serialize. +// - objectPrefix: Optional object key prefix for the evidence store. +// - signer: Optional JWS signer. +// +// Returns: +// - EvidenceArtifact: Manifest artifact ready for WORM storage. +// - HistoryManifest: Manifest with final signature state and signer metadata. +// - error: Error when canonical serialization or signing fails. +func BuildManifestEvidenceArtifact(manifest HistoryManifest, objectPrefix string, signer *ManifestJWSSigner) (EvidenceArtifact, HistoryManifest, error) { + if signer == nil { + return buildUnsignedManifestArtifact(manifest, objectPrefix) + } + return buildSignedManifestArtifact(manifest, objectPrefix, signer) +} + +func buildUnsignedManifestArtifact(manifest HistoryManifest, objectPrefix string) (EvidenceArtifact, HistoryManifest, error) { + manifest.SignatureState = SignatureStateUnsigned + manifest.Signer = nil + data, err := CanonicalManifestJSON(manifest) + if err != nil { + return EvidenceArtifact{}, HistoryManifest{}, err + } + return EvidenceArtifact{ + ArtifactType: EvidenceArtifactManifest, + ObjectKey: manifestObjectKey(objectPrefix, manifest, ".json"), + ContentType: manifestJSONContentType, + Data: data, + Metadata: manifestMetadata(manifest), + }, manifest, nil +} + +func buildSignedManifestArtifact(manifest HistoryManifest, objectPrefix string, signer *ManifestJWSSigner) (EvidenceArtifact, HistoryManifest, error) { + if signer.PrivateKey == nil { + return EvidenceArtifact{}, HistoryManifest{}, fmt.Errorf("HISTORY-MANIFESTSIGNER-NILKEY private key must not be nil") + } + manifest.SignatureState = SignatureStateSigned + manifest.Signer = &ManifestSignerInfo{KeyID: signer.KeyID, Algorithm: manifestSigningAlgRS256} + payload, err := CanonicalManifestJSON(manifest) + if err != nil { + return EvidenceArtifact{}, HistoryManifest{}, err + } + signed, err := commonjws.SignPayload(signer.PrivateKey, payload) + if err != nil { + return EvidenceArtifact{}, HistoryManifest{}, fmt.Errorf("HISTORY-MANIFESTSIGNER-SIGN %w", err) + } + return EvidenceArtifact{ + ArtifactType: EvidenceArtifactManifest, + ObjectKey: manifestObjectKey(objectPrefix, manifest, ".jws"), + ContentType: manifestJWSContentType, + Data: []byte(signed), + Metadata: manifestMetadata(manifest), + }, manifest, nil +} + +func manifestObjectKey(objectPrefix string, manifest HistoryManifest, extension string) string { + prefix := strings.Trim(strings.TrimSpace(objectPrefix), "/") + identifier := strings.TrimSpace(manifest.Identifier) + if identifier == "" { + identifier = "_all" + } + key := path.Join( + "history-manifests", + url.PathEscape(manifest.HistoryTable), + url.PathEscape(identifier), + fmt.Sprintf("%d-%d-%s%s", manifest.FirstHistoryID, manifest.LastHistoryID, manifest.RangeDigest, extension), + ) + if prefix == "" { + return key + } + return path.Join(prefix, key) +} + +func manifestMetadata(manifest HistoryManifest) map[string]string { + return map[string]string{ + "artifact_type": EvidenceArtifactManifest, + "history_table": manifest.HistoryTable, + "identifier": manifest.Identifier, + "first_history_id": fmt.Sprintf("%d", manifest.FirstHistoryID), + "last_history_id": fmt.Sprintf("%d", manifest.LastHistoryID), + "range_digest": manifest.RangeDigest, + "signature_state": manifest.SignatureState, + } +} + +// DecodeManifestArtifact extracts a manifest from unsigned JSON or compact JWS evidence bytes. +// +// JWS payloads are decoded without cryptographic signature verification. Callers +// that require signer trust validation must verify the JWS signature with their +// configured public key before accepting the manifest as signed evidence. +// +// Parameters: +// - data: Stored manifest artifact bytes. +// - contentType: Artifact content type returned by the evidence store. +// +// Returns: +// - HistoryManifest: Decoded and structurally validated manifest. +// - bool: True when the artifact was encoded as compact JWS. +// - error: Error when the artifact cannot be decoded or manifest validation fails. +func DecodeManifestArtifact(data []byte, contentType string) (HistoryManifest, bool, error) { + payload := data + signed := isJWSManifestArtifact(data, contentType) + if signed { + jws, err := jose.ParseSigned(strings.TrimSpace(string(data))) + if err != nil { + return HistoryManifest{}, false, fmt.Errorf("HISTORY-MANIFEST-DECODE-PARSEJWS %w", err) + } + payload = jws.UnsafePayloadWithoutVerification() + } + var manifest HistoryManifest + if err := json.Unmarshal(payload, &manifest); err != nil { + return HistoryManifest{}, signed, fmt.Errorf("HISTORY-MANIFEST-DECODE-JSON %w", err) + } + if err := validateManifest(manifest); err != nil { + return HistoryManifest{}, signed, err + } + return manifest, signed, nil +} + +func isJWSManifestArtifact(data []byte, contentType string) bool { + if strings.Contains(strings.ToLower(contentType), "jose") { + return true + } + return strings.Count(strings.TrimSpace(string(data)), ".") == 2 +} diff --git a/internal/common/history/snapshot_artifact.go b/internal/common/history/snapshot_artifact.go new file mode 100644 index 000000000..70493aca3 --- /dev/null +++ b/internal/common/history/snapshot_artifact.go @@ -0,0 +1,215 @@ +/******************************************************************************* +* Copyright (C) 2026 the Eclipse BaSyx Authors and Fraunhofer IESE +* +* Permission is hereby granted, free of charge, to any person obtaining +* a copy of this software and associated documentation files (the +* "Software"), to deal in the Software without restriction, including +* without limitation the rights to use, copy, modify, merge, publish, +* distribute, sublicense, and/or sell copies of the Software, and to +* permit persons to whom the Software is furnished to do so, subject to +* the following conditions: +* +* The above copyright notice and this permission notice shall be +* included in all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +* +* SPDX-License-Identifier: MIT +******************************************************************************/ + +package history + +import ( + "context" + "database/sql" + "fmt" + "net/url" + "path" + "strings" + + "github.com/doug-martin/goqu/v9" + "github.com/eclipse-basyx/basyx-go-components/internal/common" +) + +const snapshotArtifactVersion = "basyx-history-snapshot-v1" + +// SnapshotArtifactCandidate is a full-snapshot recovery checkpoint ready for evidence storage. +type SnapshotArtifactCandidate struct { + Artifact EvidenceArtifact + HistoryID int64 + Identifier string + RowHash string + ContentHash string +} + +// SnapshotReference builds a manifest reference after the snapshot artifact was stored. +// +// Parameters: +// - receipt: Object-store receipt returned after writing the snapshot artifact. +// +// Returns: +// - SnapshotArtifactReference: Manifest-ready pointer to the recovery snapshot. +// - error: Error when receipt is nil. +func (candidate SnapshotArtifactCandidate) SnapshotReference(receipt *EvidenceReceipt) (SnapshotArtifactReference, error) { + if receipt == nil { + return SnapshotArtifactReference{}, fmt.Errorf("HISTORY-EVIDENCE-SNAPSHOT-NILRECEIPT evidence receipt must not be nil") + } + return SnapshotArtifactReference{ + HistoryID: candidate.HistoryID, + RowHash: candidate.RowHash, + ContentHash: candidate.ContentHash, + SHA256: receipt.SHA256, + Reference: receipt.Reference, + }, nil +} + +// LoadSnapshotArtifactCandidates loads full snapshot rows that can be archived for bounded recovery. +// +// Parameters: +// - ctx: Request context for reading history rows. +// - db: Database handle connected to the BaSyx PostgreSQL database. +// - table: History table to read, for example submodel_history. +// - identifier: Optional entity identifier scope. Empty means all identifiers. +// - firstHistoryID: Inclusive lower history_id bound. +// - lastHistoryID: Inclusive upper history_id bound. +// +// Returns: +// - []SnapshotArtifactCandidate: Canonical full-snapshot artifacts in history_id order. +// - error: Error when the table is unsupported, snapshot hashes do not validate, +// or rows cannot be read. +func LoadSnapshotArtifactCandidates(ctx context.Context, db *sql.DB, table string, identifier string, firstHistoryID int64, lastHistoryID int64) ([]SnapshotArtifactCandidate, error) { + if db == nil { + return nil, common.NewErrBadRequest("HISTORY-EVIDENCE-SNAPSHOT-NILDB database handle must not be nil") + } + payloadTable, err := historyPayloadTable(table) + if err != nil { + return nil, err + } + query, args, err := snapshotArtifactQuery(table, payloadTable, identifier, firstHistoryID, lastHistoryID) + if err != nil { + return nil, err + } + rows, err := db.QueryContext(ctx, query, args...) + if err != nil { + return nil, common.NewInternalServerError("HISTORY-EVIDENCE-SNAPSHOT-EXECQUERY " + err.Error()) + } + defer func() { + _ = rows.Close() + }() + return scanSnapshotArtifactCandidates(rows, table) +} + +func snapshotArtifactQuery(table string, payloadTable string, identifier string, firstHistoryID int64, lastHistoryID int64) (string, []any, error) { + historyAlias := goqu.T(table).As("history") + payloadAlias := goqu.T(payloadTable).As("payload") + dataset := goqu.From(historyAlias). + InnerJoin(payloadAlias, goqu.On(historyAlias.Col("history_id").Eq(payloadAlias.Col("history_id")))). + Select( + historyAlias.Col("history_id"), + historyAlias.Col("identifier"), + historyAlias.Col("content_hash"), + historyAlias.Col("payload_hash"), + historyAlias.Col("row_hash"), + goqu.L(`"payload"."snapshot"::text`), + ). + Where( + historyAlias.Col("history_id").Gte(firstHistoryID), + historyAlias.Col("history_id").Lte(lastHistoryID), + historyAlias.Col("payload_type").Eq(PayloadTypeSnapshot), + ) + if strings.TrimSpace(identifier) != "" { + dataset = dataset.Where(historyAlias.Col("identifier").Eq(strings.TrimSpace(identifier))) + } + return dataset.Order(historyAlias.Col("history_id").Asc()).ToSQL() +} + +func scanSnapshotArtifactCandidates(rows *sql.Rows, table string) ([]SnapshotArtifactCandidate, error) { + candidates := make([]SnapshotArtifactCandidate, 0) + for rows.Next() { + candidate, err := scanSnapshotArtifactCandidate(rows, table) + if err != nil { + return nil, err + } + candidates = append(candidates, candidate) + } + if err := rows.Err(); err != nil { + return nil, common.NewInternalServerError("HISTORY-EVIDENCE-SNAPSHOT-ROWS " + err.Error()) + } + return candidates, nil +} + +func scanSnapshotArtifactCandidate(rows *sql.Rows, table string) (SnapshotArtifactCandidate, error) { + var historyID int64 + var identifier string + var contentHash sql.NullString + var payloadHash sql.NullString + var rowHash sql.NullString + var snapshotText sql.NullString + if err := rows.Scan(&historyID, &identifier, &contentHash, &payloadHash, &rowHash, &snapshotText); err != nil { + return SnapshotArtifactCandidate{}, common.NewInternalServerError("HISTORY-EVIDENCE-SNAPSHOT-SCAN " + err.Error()) + } + if !snapshotText.Valid { + return SnapshotArtifactCandidate{}, common.NewInternalServerError("HISTORY-EVIDENCE-SNAPSHOT-MISSING snapshot payload is missing") + } + var snapshot map[string]any + if err := decodeJSONPreservingNumbers([]byte(snapshotText.String), &snapshot); err != nil { + return SnapshotArtifactCandidate{}, common.NewInternalServerError("HISTORY-EVIDENCE-SNAPSHOT-DECODE " + err.Error()) + } + if err := verifyCanonicalHash(snapshot, contentHash, "HISTORY-EVIDENCE-SNAPSHOT-CONTENTHASH"); err != nil { + return SnapshotArtifactCandidate{}, err + } + if err := verifyCanonicalHash(snapshot, payloadHash, "HISTORY-EVIDENCE-SNAPSHOT-PAYLOADHASH"); err != nil { + return SnapshotArtifactCandidate{}, err + } + data, err := CanonicalJSON(map[string]any{ + "artifact_version": snapshotArtifactVersion, + "history_table": table, + "identifier": identifier, + "history_id": historyID, + "row_hash": nullStringValue(rowHash), + "content_hash": nullStringValue(contentHash), + "payload_hash": nullStringValue(payloadHash), + "snapshot": snapshot, + }) + if err != nil { + return SnapshotArtifactCandidate{}, common.NewInternalServerError("HISTORY-EVIDENCE-SNAPSHOT-CANONICAL " + err.Error()) + } + return SnapshotArtifactCandidate{ + Artifact: EvidenceArtifact{ + ArtifactType: EvidenceArtifactSnapshot, + ObjectKey: snapshotObjectKey(table, identifier, historyID, nullStringValue(rowHash)), + ContentType: manifestJSONContentType, + Data: data, + Metadata: snapshotMetadata(table, identifier, historyID, nullStringValue(rowHash)), + }, + HistoryID: historyID, + Identifier: identifier, + RowHash: nullStringValue(rowHash), + ContentHash: nullStringValue(contentHash), + }, nil +} + +func snapshotObjectKey(table string, identifier string, historyID int64, rowHash string) string { + return path.Join( + "history-snapshots", + url.PathEscape(table), + url.PathEscape(identifier), + fmt.Sprintf("%d-%s.json", historyID, strings.TrimSpace(rowHash)), + ) +} + +func snapshotMetadata(table string, identifier string, historyID int64, rowHash string) map[string]string { + return map[string]string{ + "artifact_type": EvidenceArtifactSnapshot, + "history_table": table, + "identifier": identifier, + "history_id": fmt.Sprintf("%d", historyID), + "row_hash": strings.TrimSpace(rowHash), + } +} diff --git a/internal/common/jws/jws.go b/internal/common/jws/jws.go index ea6337d28..65d390247 100644 --- a/internal/common/jws/jws.go +++ b/internal/common/jws/jws.go @@ -32,9 +32,20 @@ import ( "encoding/pem" "fmt" "os" + + jose "gopkg.in/go-jose/go-jose.v2" ) // LoadPrivateKey reads and parses an RSA private key from a PEM file. +// +// PKCS#8 keys are tried first and PKCS#1 RSA keys are accepted as a fallback. +// +// Parameters: +// - path: Filesystem path to the PEM encoded private key. +// +// Returns: +// - *rsa.PrivateKey: Parsed RSA private key. +// - error: Error when the file cannot be read, decoded, or parsed as RSA. func LoadPrivateKey(path string) (*rsa.PrivateKey, error) { //nolint:all // Ignore linter warnings for this function as it deals with cryptographic key loading. keyData, err := os.ReadFile(path) @@ -64,3 +75,32 @@ func LoadPrivateKey(path string) (*rsa.PrivateKey, error) { } return rsaKey, nil } + +// SignPayload returns a compact RS256 JWS over payload. +// +// Parameters: +// - privateKey: RSA private key used for RS256 signing. +// - payload: Canonical payload bytes to sign. +// +// Returns: +// - string: Compact serialized JWS. +// - error: Error when the key is nil, the signer cannot be created, or +// serialization fails. +func SignPayload(privateKey *rsa.PrivateKey, payload []byte) (string, error) { + if privateKey == nil { + return "", fmt.Errorf("JWS-SIGN-NILKEY private key must not be nil") + } + signer, err := jose.NewSigner(jose.SigningKey{Algorithm: jose.RS256, Key: privateKey}, nil) + if err != nil { + return "", fmt.Errorf("JWS-SIGN-NEWSIGNER %w", err) + } + jws, err := signer.Sign(payload) + if err != nil { + return "", fmt.Errorf("JWS-SIGN-PAYLOAD %w", err) + } + signed, err := jws.CompactSerialize() + if err != nil { + return "", fmt.Errorf("JWS-SIGN-SERIALIZE %w", err) + } + return signed, nil +} From 88a8051e3bd531e6620e4ab0ddb84565e30271cc Mon Sep 17 00:00:00 2001 From: Aaron Zielstorff Date: Fri, 5 Jun 2026 21:38:39 +0200 Subject: [PATCH 2/5] Adresses copilot remarks --- internal/common/history/evidence_test.go | 81 ++++++++++++++++++++ internal/common/history/evidence_writer.go | 6 ++ internal/common/history/manifest_artifact.go | 26 ++++++- internal/common/history/snapshot_artifact.go | 12 ++- 4 files changed, 119 insertions(+), 6 deletions(-) diff --git a/internal/common/history/evidence_test.go b/internal/common/history/evidence_test.go index eb4a93931..d84d2dd30 100644 --- a/internal/common/history/evidence_test.go +++ b/internal/common/history/evidence_test.go @@ -26,6 +26,7 @@ package history import ( + "context" "crypto/rand" "crypto/rsa" "strings" @@ -98,6 +99,44 @@ func TestBuildManifestEvidenceArtifactSignsCanonicalManifest(t *testing.T) { require.True(t, strings.HasPrefix(artifact.ObjectKey, "prefix/history-manifests/")) } +func TestDecodeManifestArtifactTreatsJSONContentTypeAsJSONWhenPayloadContainsDots(t *testing.T) { + manifest, err := BuildHistoryManifest(HistoryManifestOptions{ + HistoryTable: TableAAS, + Identifier: "urn.example.component", + Rows: []ManifestRangeRow{ + {HistoryID: 1, RowHash: strings.Repeat("a", 64)}, + }, + GeneratedAt: time.Date(2026, 6, 5, 12, 0, 0, 0, time.UTC), + }) + require.NoError(t, err) + artifact, finalManifest, err := BuildManifestEvidenceArtifact(manifest, "", nil) + require.NoError(t, err) + require.Equal(t, manifestJSONContentType, artifact.ContentType) + require.Equal(t, 2, strings.Count(string(artifact.Data), ".")) + + decoded, signed, err := DecodeManifestArtifact(artifact.Data, manifestJSONContentType) + + require.NoError(t, err) + require.False(t, signed) + require.Equal(t, finalManifest.Identifier, decoded.Identifier) + require.Equal(t, finalManifest.RangeDigest, decoded.RangeDigest) +} + +func TestStoreManifestArtifactRejectsNilReceipt(t *testing.T) { + manifest, err := BuildHistoryManifest(HistoryManifestOptions{ + HistoryTable: TableAAS, + Rows: []ManifestRangeRow{ + {HistoryID: 1, RowHash: strings.Repeat("a", 64)}, + }, + }) + require.NoError(t, err) + + receipt, _, err := storeManifestArtifact(t.Context(), nilReceiptEvidenceStore{}, manifest, nil) + + require.Nil(t, receipt) + require.ErrorContains(t, err, "HISTORY-EVIDENCE-WRITE-NILMANIFESTRECEIPT") +} + func TestS3EvidenceStoreReceiptAppliesPrefixAndRetention(t *testing.T) { now := time.Date(2026, 6, 5, 12, 0, 0, 0, time.UTC) store := &S3EvidenceStore{ @@ -170,6 +209,34 @@ func TestStoreHistoryEventArtifactsBackfillsSnapshotAndDiffRows(t *testing.T) { require.NoError(t, mock.ExpectationsWereMet()) } +func TestLoadSnapshotArtifactCandidatesRejectsEmptyRowHash(t *testing.T) { + db, mock, err := sqlmock.New() + require.NoError(t, err) + defer func() { + _ = db.Close() + }() + + snapshot := map[string]any{"id": "aas-1"} + snapshotData, err := CanonicalJSON(snapshot) + require.NoError(t, err) + snapshotHash := SHA256Hex(snapshotData) + mock.ExpectQuery(`SELECT .*FROM "aas_history" AS "history" INNER JOIN "aas_history_payload" AS "payload".*"payload_type" = 'snapshot'`). + WillReturnRows(sqlmock.NewRows([]string{ + "history_id", + "identifier", + "content_hash", + "payload_hash", + "row_hash", + "snapshot", + }).AddRow(int64(1), "aas-1", snapshotHash, snapshotHash, "", string(snapshotData))) + + candidates, err := LoadSnapshotArtifactCandidates(t.Context(), db, TableAAS, "aas-1", 1, 1) + + require.Nil(t, candidates) + require.ErrorContains(t, err, "HISTORY-EVIDENCE-SNAPSHOT-ROWHASH") + require.NoError(t, mock.ExpectationsWereMet()) +} + func TestVerifyHistoryRangeReportsMissingHistoryEventArtifact(t *testing.T) { db, mock, err := sqlmock.New() require.NoError(t, err) @@ -264,3 +331,17 @@ func verificationFindingCodes(report *HistoryEvidenceVerificationReport) []strin } return codes } + +type nilReceiptEvidenceStore struct{} + +func (nilReceiptEvidenceStore) PutArtifact(_ context.Context, _ EvidenceArtifact) (*EvidenceReceipt, error) { + return nil, nil +} + +func (nilReceiptEvidenceStore) GetArtifact(_ context.Context, _ EvidenceReference) (*EvidenceObject, error) { + return nil, nil +} + +func (nilReceiptEvidenceStore) VerifyArtifact(_ context.Context, _ EvidenceReference, _ string) (*EvidenceReceipt, error) { + return nil, nil +} diff --git a/internal/common/history/evidence_writer.go b/internal/common/history/evidence_writer.go index 209dc2429..e3d14a99a 100644 --- a/internal/common/history/evidence_writer.go +++ b/internal/common/history/evidence_writer.go @@ -195,10 +195,16 @@ func storeManifestArtifact(ctx context.Context, store EvidenceStore, manifest Hi if err != nil { return nil, HistoryManifest{}, fmt.Errorf("HISTORY-EVIDENCE-WRITE-PUTMANIFEST %w", err) } + if receipt == nil { + return nil, HistoryManifest{}, fmt.Errorf("HISTORY-EVIDENCE-WRITE-NILMANIFESTRECEIPT evidence store returned nil receipt") + } return receipt, finalManifest, nil } func recordEvidenceCatalog(ctx context.Context, db *sql.DB, manifest HistoryManifest, manifestReceipt *EvidenceReceipt, eventReceipts []EvidenceCatalogEventReceipt, snapshotReceipts []EvidenceCatalogSnapshotReceipt) (int64, error) { + if manifestReceipt == nil { + return 0, common.NewInternalServerError("HISTORY-EVIDENCE-WRITE-NILMANIFESTRECEIPT evidence store returned nil receipt") + } tx, err := db.BeginTx(ctx, nil) if err != nil { return 0, common.NewInternalServerError("HISTORY-EVIDENCE-WRITE-BEGINTX " + err.Error()) diff --git a/internal/common/history/manifest_artifact.go b/internal/common/history/manifest_artifact.go index ef675d7e7..45f72a1d5 100644 --- a/internal/common/history/manifest_artifact.go +++ b/internal/common/history/manifest_artifact.go @@ -27,6 +27,7 @@ package history import ( "crypto/rsa" + "encoding/base64" "encoding/json" "fmt" "net/url" @@ -197,8 +198,29 @@ func DecodeManifestArtifact(data []byte, contentType string) (HistoryManifest, b } func isJWSManifestArtifact(data []byte, contentType string) bool { - if strings.Contains(strings.ToLower(contentType), "jose") { + normalizedContentType := strings.ToLower(strings.TrimSpace(contentType)) + if strings.Contains(normalizedContentType, "jose") { return true } - return strings.Count(strings.TrimSpace(string(data)), ".") == 2 + if strings.Contains(normalizedContentType, "json") { + return false + } + return isCompactJWS(data) +} + +func isCompactJWS(data []byte) bool { + parts := strings.Split(strings.TrimSpace(string(data)), ".") + if len(parts) != 3 || strings.TrimSpace(parts[1]) == "" || strings.TrimSpace(parts[2]) == "" { + return false + } + protectedHeader, err := base64.RawURLEncoding.DecodeString(parts[0]) + if err != nil { + return false + } + var header map[string]any + if err = json.Unmarshal(protectedHeader, &header); err != nil { + return false + } + _, hasAlgorithm := header["alg"].(string) + return hasAlgorithm } diff --git a/internal/common/history/snapshot_artifact.go b/internal/common/history/snapshot_artifact.go index 70493aca3..7b3280a8c 100644 --- a/internal/common/history/snapshot_artifact.go +++ b/internal/common/history/snapshot_artifact.go @@ -157,6 +157,10 @@ func scanSnapshotArtifactCandidate(rows *sql.Rows, table string) (SnapshotArtifa if !snapshotText.Valid { return SnapshotArtifactCandidate{}, common.NewInternalServerError("HISTORY-EVIDENCE-SNAPSHOT-MISSING snapshot payload is missing") } + rowHashValue := strings.TrimSpace(nullStringValue(rowHash)) + if rowHashValue == "" { + return SnapshotArtifactCandidate{}, common.NewInternalServerError("HISTORY-EVIDENCE-SNAPSHOT-ROWHASH row hash is required") + } var snapshot map[string]any if err := decodeJSONPreservingNumbers([]byte(snapshotText.String), &snapshot); err != nil { return SnapshotArtifactCandidate{}, common.NewInternalServerError("HISTORY-EVIDENCE-SNAPSHOT-DECODE " + err.Error()) @@ -172,7 +176,7 @@ func scanSnapshotArtifactCandidate(rows *sql.Rows, table string) (SnapshotArtifa "history_table": table, "identifier": identifier, "history_id": historyID, - "row_hash": nullStringValue(rowHash), + "row_hash": rowHashValue, "content_hash": nullStringValue(contentHash), "payload_hash": nullStringValue(payloadHash), "snapshot": snapshot, @@ -183,14 +187,14 @@ func scanSnapshotArtifactCandidate(rows *sql.Rows, table string) (SnapshotArtifa return SnapshotArtifactCandidate{ Artifact: EvidenceArtifact{ ArtifactType: EvidenceArtifactSnapshot, - ObjectKey: snapshotObjectKey(table, identifier, historyID, nullStringValue(rowHash)), + ObjectKey: snapshotObjectKey(table, identifier, historyID, rowHashValue), ContentType: manifestJSONContentType, Data: data, - Metadata: snapshotMetadata(table, identifier, historyID, nullStringValue(rowHash)), + Metadata: snapshotMetadata(table, identifier, historyID, rowHashValue), }, HistoryID: historyID, Identifier: identifier, - RowHash: nullStringValue(rowHash), + RowHash: rowHashValue, ContentHash: nullStringValue(contentHash), }, nil } From df09070892f3606d049566f6bf3fe8e59f622213 Mon Sep 17 00:00:00 2001 From: Aaron Zielstorff Date: Fri, 5 Jun 2026 23:40:12 +0200 Subject: [PATCH 3/5] Enhances WORM evidence storage by adding effective_diff field to history_event artifacts for improved audit attribution --- docu/developer/aas_v3_2_runtime.md | 4 +- docu/user/aas_api_v3_2.md | 4 +- .../BaSyxHistoryAuditGuardedExample/README.md | 4 +- internal/common/history/append.go | 107 +++++++++--- internal/common/history/append_test.go | 75 ++++++++- internal/common/history/evidence_test.go | 9 +- .../common/history/history_event_artifact.go | 154 ++++++++++++++++-- 7 files changed, 306 insertions(+), 51 deletions(-) diff --git a/docu/developer/aas_v3_2_runtime.md b/docu/developer/aas_v3_2_runtime.md index ccb808e61..56816e8a5 100644 --- a/docu/developer/aas_v3_2_runtime.md +++ b/docu/developer/aas_v3_2_runtime.md @@ -244,7 +244,7 @@ The default stronger-integrity architecture is: PostgreSQL history tables -> hash chain -> synchronous history-event artifact -> signed manifest -> S3-compatible WORM object storage ``` -The HTTP APIs are unchanged. When `history.evidence.enabled` is active, `history.mode` must be `api` or `audit`. The history append path writes one WORM `history_event` artifact synchronously before the surrounding PostgreSQL transaction can commit. The artifact stores the same history payload selected for PostgreSQL: either a full snapshot or an RFC 6902 diff according to `history.fullSnapshotInterval`. If the WORM write fails, the history append returns an error and the caller rolls back the PostgreSQL transaction. +The HTTP APIs are unchanged. When `history.evidence.enabled` is active, `history.mode` must be `api` or `audit`. The history append path writes one WORM `history_event` artifact synchronously before the surrounding PostgreSQL transaction can commit. The artifact stores the same history payload selected for PostgreSQL: either a full snapshot or an RFC 6902 diff according to `history.fullSnapshotInterval`. It also stores `effective_diff`, an RFC 6902 JSON Patch from the previous reconstructed version to the current version. If the WORM write fails, the history append returns an error and the caller rolls back the PostgreSQL transaction. The `cmd/historyevidenceverifier` tool can additionally publish signed range manifests, backfill per-row `history_event` artifacts for existing rows, and publish checkpoint artifacts using the shared `EvidenceStore` interface. The current implementation includes an S3-compatible `EvidenceStore`; MinIO can be used for local or CI-style object-lock testing, while production deployments should use an S3-compatible WORM service with versioning/object lock configured by operations. @@ -257,7 +257,7 @@ For a selected history range, evidence publication: - signs the canonical manifest as compact RS256 JWS when an evidence signing key is configured, otherwise stores canonical JSON with `signature_state = unsigned`; - writes object-store receipts to `history_evidence_manifests` and `history_evidence_artifacts`. -Per-row `history_event` artifacts provide recovery evidence for every acknowledged write while evidence is enabled. With `history.fullSnapshotInterval: 5`, recovery from WORM replays up to four WORM-stored diff payloads after the latest WORM-stored snapshot event. Use `history.fullSnapshotInterval: 1` when each individual WORM event must be recoverable without replaying diffs. Automated PostgreSQL restore from WORM artifacts is not implemented in this ticket. +Per-row `history_event` artifacts provide recovery evidence for every acknowledged write while evidence is enabled. With `history.fullSnapshotInterval: 5`, recovery from WORM replays up to four WORM-stored diff payloads after the latest WORM-stored snapshot event. Use `history.fullSnapshotInterval: 1` when each individual WORM event must be recoverable without replaying diffs. The `effective_diff` field is the attribution trail: for snapshot checkpoint rows it prevents a full recovery snapshot from being mistaken for the set of fields changed by the actor. Automated PostgreSQL restore from WORM artifacts is not implemented in this ticket. Verification can compare PostgreSQL rows against the hash chain, verify every per-row `history_event` receipt and object hash, compare a stored manifest against the live range digest, and verify an object-store artifact hash. This detects missing, modified, reordered, or overwritten records when the expected receipts and object hashes are known. diff --git a/docu/user/aas_api_v3_2.md b/docu/user/aas_api_v3_2.md index 6c3b38648..d3409e425 100644 --- a/docu/user/aas_api_v3_2.md +++ b/docu/user/aas_api_v3_2.md @@ -123,7 +123,7 @@ Current implementation status: - Runtime history rows are append-only, hash-chained event rows in `api` and `audit` mode. - Schema migration installs PostgreSQL guard triggers. `postgres_guarded` enables them at service startup. When enabled, `UPDATE`, `DELETE`, and `TRUNCATE` on history metadata and payload tables fail with `history tables are append-only`. -- When WORM evidence is enabled, each acknowledged history append synchronously stores a `history_event` artifact in S3-compatible object storage. The artifact contains the same snapshot or diff payload that PostgreSQL stores. +- When WORM evidence is enabled, each acknowledged history append synchronously stores a `history_event` artifact in S3-compatible object storage. The artifact contains the same snapshot or diff payload that PostgreSQL stores plus an `effective_diff` that records what changed compared with the previous reconstructed version. - WORM evidence manifests, backfilled `history_event` artifacts, and additional snapshot checkpoint artifacts can be published to S3-compatible object storage with `cmd/historyevidenceverifier`. - `external_anchor`, non-zero `retentionDays`, non-`none` integrity anchor providers, and identity enrichment modes currently fail fast during configuration loading. Their runtime implementations are planned for later work. - Audit metadata columns are present, but regular API middleware does not populate the audit context yet. @@ -131,7 +131,7 @@ Current implementation status: Guarded PostgreSQL mode protects against normal accidental or unauthorized mutations through the application database user. PostgreSQL superusers or operators with permissions to alter triggers/functions can still bypass or remove this protection. Stronger tamper evidence and recovery evidence are provided by storing per-row history-event artifacts and signed history manifests in S3-compatible WORM storage such as AWS S3 Object Lock. MinIO Object Lock is useful for tests and local examples, but production deployments should use an operated WORM-capable object store with versioning and retention policy controls. -WORM history-event artifacts provide recoverability for acknowledged writes while evidence is enabled. With `fullSnapshotInterval: 5`, for example, recovery from WORM starts from the latest WORM-stored snapshot event and replays up to four WORM-stored diff payloads. Use `fullSnapshotInterval: 1` when every acknowledged history row must be recoverable as a full WORM snapshot without diff replay. Automated PostgreSQL restore from WORM artifacts is not implemented in this release. +WORM history-event artifacts provide recoverability for acknowledged writes while evidence is enabled. With `fullSnapshotInterval: 5`, for example, recovery from WORM starts from the latest WORM-stored snapshot event and replays up to four WORM-stored diff payloads. Use `fullSnapshotInterval: 1` when every acknowledged history row must be recoverable as a full WORM snapshot without diff replay. Even when the stored payload is a full snapshot checkpoint, `effective_diff` records the actual JSON Patch relative to the previous version so audit analysis can distinguish recovery checkpoints from fields changed by the actor. Automated PostgreSQL restore from WORM artifacts is not implemented in this release. The guard switch is database-wide. Configure all BaSyx services that share one database with the same history immutability mode. Runtime services may enable guarded mode, but normal service startup cannot disable an enabled database guard. A service configured as unguarded fails during startup when it encounters an already-enabled database guard. Disabling guarded mode is an explicit operator maintenance action. diff --git a/examples/BaSyxHistoryAuditGuardedExample/README.md b/examples/BaSyxHistoryAuditGuardedExample/README.md index 707d5d079..42eea3915 100644 --- a/examples/BaSyxHistoryAuditGuardedExample/README.md +++ b/examples/BaSyxHistoryAuditGuardedExample/README.md @@ -42,7 +42,7 @@ This example is configured for local development: The configuration service initializes the current schema first. The AAS Environment Service then enables the history guard switch at startup. The MinIO init sidecar creates a versioned object-lock bucket for local evidence tests. `BASYX_HISTORY_RETENTION_DAYS=0` is accepted as configuration, but automatic retention cleanup is not implemented yet. `BASYX_HISTORY_FULL_SNAPSHOT_INTERVAL=5` stores periodic full checkpoints with RFC 6902 diff rows between them. -When evidence is enabled, history mode must be `api` or `audit`. Each successful history append writes a WORM `history_event` artifact to MinIO before PostgreSQL commits. If MinIO is unavailable or rejects the object write, the API mutation fails and the PostgreSQL transaction rolls back. +When evidence is enabled, history mode must be `api` or `audit`. Each successful history append writes a WORM `history_event` artifact to MinIO before PostgreSQL commits. The artifact stores the recovery payload plus `effective_diff`, which records the actual JSON Patch relative to the previous version for audit attribution. If MinIO is unavailable or rejects the object write, the API mutation fails and the PostgreSQL transaction rolls back. ## UI And Preconfigured Data @@ -155,7 +155,7 @@ go run ./cmd/historyevidenceverifier \ -manifest-sha256 '' ``` -History-event artifacts provide recovery evidence for acknowledged writes while evidence is enabled. With `BASYX_HISTORY_FULL_SNAPSHOT_INTERVAL=5`, recovery from WORM starts from the latest WORM-stored snapshot event and replays up to four WORM-stored diff payloads. Use `BASYX_HISTORY_FULL_SNAPSHOT_INTERVAL=1` when every history row must be recoverable as a full WORM snapshot without diff replay. Automated PostgreSQL restore from WORM artifacts is not implemented in this example. +History-event artifacts provide recovery evidence for acknowledged writes while evidence is enabled. With `BASYX_HISTORY_FULL_SNAPSHOT_INTERVAL=5`, recovery from WORM starts from the latest WORM-stored snapshot event and replays up to four WORM-stored diff payloads. Use `BASYX_HISTORY_FULL_SNAPSHOT_INTERVAL=1` when every history row must be recoverable as a full WORM snapshot without diff replay. For audit attribution, inspect `effective_diff`: a full snapshot event can be a recovery checkpoint, while `effective_diff` shows what the request actually changed. Automated PostgreSQL restore from WORM artifacts is not implemented in this example. ## Limitations diff --git a/internal/common/history/append.go b/internal/common/history/append.go index d95fece7a..bbba76a34 100644 --- a/internal/common/history/append.go +++ b/internal/common/history/append.go @@ -82,6 +82,16 @@ func AppendVersionTx(ctx context.Context, tx *sql.Tx, table string, identifier s return err } if cfg.FullSnapshotInterval == DefaultFullSnapshotInterval { + if cfg.EvidenceEnabled { + latest, latestErr := latestVersionTx(ctx, tx, table, identifier) + if latestErr != nil && !common.IsErrNotFound(latestErr) { + return latestErr + } + if common.IsErrNotFound(latestErr) { + return appendVersionWithLatestTx(ctx, tx, table, identifier, changeType, snapshot, deleted, nil, cfg) + } + return appendVersionWithLatestTx(ctx, tx, table, identifier, changeType, snapshot, deleted, &latest, cfg) + } previousHash, hashErr := latestRowHashTx(ctx, tx, table, identifier) if hashErr != nil { return hashErr @@ -184,7 +194,21 @@ func appendVersionWithLatestTx(ctx context.Context, tx *sql.Tx, table string, id if latest != nil { previousHash = latest.rowHash } - return insertHistoryVersionTx(ctx, tx, table, identifier, changeType, snapshot, deleted, payload, previousHash, cfg) + effectiveDiff, err := buildEffectiveDiff(snapshot, latest) + if err != nil { + return err + } + return insertHistoryVersionTx(ctx, tx, historyVersionInsert{ + table: table, + identifier: identifier, + changeType: changeType, + snapshot: snapshot, + deleted: deleted, + payload: payload, + effectiveDiff: effectiveDiff, + previousHash: previousHash, + cfg: cfg, + }) } func appendSnapshotVersionWithPreviousHashTx(ctx context.Context, tx *sql.Tx, table string, identifier string, changeType string, snapshot map[string]any, deleted bool, previousHash string, cfg Config) error { @@ -192,31 +216,52 @@ func appendSnapshotVersionWithPreviousHashTx(ctx context.Context, tx *sql.Tx, ta if err != nil { return err } - return insertHistoryVersionTx(ctx, tx, table, identifier, changeType, snapshot, deleted, payload, previousHash, cfg) + return insertHistoryVersionTx(ctx, tx, historyVersionInsert{ + table: table, + identifier: identifier, + changeType: changeType, + snapshot: snapshot, + deleted: deleted, + payload: payload, + previousHash: previousHash, + cfg: cfg, + }) } -func insertHistoryVersionTx(ctx context.Context, tx *sql.Tx, table string, identifier string, changeType string, snapshot map[string]any, deleted bool, payload historyPayload, previousHash string, cfg Config) error { - payloadTable, err := historyPayloadTable(table) +type historyVersionInsert struct { + table string + identifier string + changeType string + snapshot map[string]any + deleted bool + payload historyPayload + effectiveDiff []map[string]any + previousHash string + cfg Config +} + +func insertHistoryVersionTx(ctx context.Context, tx *sql.Tx, version historyVersionInsert) error { + payloadTable, err := historyPayloadTable(version.table) if err != nil { return err } now := databaseTimestamp(time.Now()) - contentHash, err := CanonicalJSONHash(snapshot) + contentHash, err := CanonicalJSONHash(version.snapshot) if err != nil { return common.NewInternalServerError("HISTORY-APPEND-CONTENTHASH " + err.Error()) } audit := FromContext(ctx) event := ChangeEvent{ - EntityType: table, - Identifier: identifier, - ChangeType: changeType, + EntityType: version.table, + Identifier: version.identifier, + ChangeType: version.changeType, Timestamp: now, - Snapshot: snapshot, - Deleted: deleted, - PayloadType: payload.payloadType, + Snapshot: version.snapshot, + Deleted: version.deleted, + PayloadType: version.payload.payloadType, ContentHash: contentHash, - PayloadHash: payload.hash, - PreviousHash: previousHash, + PayloadHash: version.payload.hash, + PreviousHash: version.previousHash, RequestID: audit.RequestID, CorrelationID: audit.CorrelationID, ActorSubject: audit.ActorSubject, @@ -236,21 +281,21 @@ func insertHistoryVersionTx(ctx context.Context, tx *sql.Tx, table string, ident return common.NewInternalServerError("HISTORY-APPEND-ROWHASH " + err.Error()) } event.RowHash = rowHash - createdAt, updatedAt := administrationTimestamps(snapshot) - insertQuery, insertArgs, err := goqu.Insert(table).Rows(goqu.Record{ - "identifier": identifier, - "change_type": changeType, - "deleted": deleted, + createdAt, updatedAt := administrationTimestamps(version.snapshot) + insertQuery, insertArgs, err := goqu.Insert(version.table).Rows(goqu.Record{ + "identifier": version.identifier, + "change_type": version.changeType, + "deleted": version.deleted, "valid_from": now, "operation_time": now, "administration_created_at_text": createdAt, "administration_updated_at_text": updatedAt, "administration_created_at": nullableTimestamp(createdAt), "administration_updated_at": nullableTimestamp(updatedAt), - "payload_type": payload.payloadType, + "payload_type": version.payload.payloadType, "content_hash": contentHash, - "payload_hash": payload.hash, - "previous_hash": previousHash, + "payload_hash": version.payload.hash, + "previous_hash": version.previousHash, "row_hash": rowHash, "actor_subject": audit.ActorSubject, "actor_issuer": audit.ActorIssuer, @@ -276,10 +321,10 @@ func insertHistoryVersionTx(ctx context.Context, tx *sql.Tx, table string, ident payloadRecord := goqu.Record{ "history_id": historyID, } - if payload.payloadType == PayloadTypeSnapshot { - payloadRecord["snapshot"] = goqu.L("?::jsonb", string(payload.json)) + if version.payload.payloadType == PayloadTypeSnapshot { + payloadRecord["snapshot"] = goqu.L("?::jsonb", string(version.payload.json)) } else { - payloadRecord["diff"] = goqu.L("?::jsonb", string(payload.json)) + payloadRecord["diff"] = goqu.L("?::jsonb", string(version.payload.json)) } payloadQuery, payloadArgs, err := goqu.Insert(payloadTable).Rows(payloadRecord).ToSQL() if err != nil { @@ -288,7 +333,7 @@ func insertHistoryVersionTx(ctx context.Context, tx *sql.Tx, table string, ident if _, err = tx.ExecContext(ctx, payloadQuery, payloadArgs...); err != nil { return common.NewInternalServerError("HISTORY-APPEND-EXECPAYLOADINSERT " + err.Error()) } - return publishHistoryEventEvidenceTx(ctx, tx, cfg, table, historyID, event, payload, createdAt, updatedAt) + return publishHistoryEventEvidenceTx(ctx, tx, version.cfg, version.table, historyID, event, version.payload, version.effectiveDiff, createdAt, updatedAt) } func lockIdentifierTx(ctx context.Context, tx *sql.Tx, table string, identifier string) error { @@ -369,6 +414,18 @@ func buildDiffPayloadWithJSON(patch []map[string]any, payloadJSON []byte) (histo return historyPayload{payloadType: PayloadTypeDiff, json: payloadJSON, hash: payloadHash}, nil } +func buildEffectiveDiff(snapshot map[string]any, latest *latestVersion) ([]map[string]any, error) { + base := map[string]any{} + if latest != nil { + base = latest.snapshot + } + patch, err := BuildJSONPatch(base, snapshot) + if err != nil { + return nil, common.NewInternalServerError("HISTORY-APPEND-EFFECTIVEDIFF " + err.Error()) + } + return patch, nil +} + func buildSnapshotPayload(snapshot map[string]any) (historyPayload, error) { payloadJSON, err := json.Marshal(snapshot) if err != nil { diff --git a/internal/common/history/append_test.go b/internal/common/history/append_test.go index de0d3de47..a2f02ff19 100644 --- a/internal/common/history/append_test.go +++ b/internal/common/history/append_test.go @@ -124,7 +124,7 @@ func TestAppendVersionTxWritesEvidenceArtifactBeforeCommitWhenEnabled(t *testing mock.ExpectBegin() mock.ExpectExec(`SELECT pg_advisory_xact_lock`). WillReturnResult(sqlmock.NewResult(0, 1)) - mock.ExpectQuery(`SELECT "row_hash" FROM "aas_history"`). + mock.ExpectQuery(`SELECT "history_id" FROM "aas_history"`). WillReturnError(sql.ErrNoRows) mock.ExpectQuery(`INSERT INTO "aas_history".*RETURNING "history_id"`). WillReturnRows(sqlmock.NewRows([]string{"history_id"}).AddRow(1)) @@ -151,6 +151,14 @@ func TestAppendVersionTxWritesEvidenceArtifactBeforeCommitWhenEnabled(t *testing payload, ok := artifactPayload["payload"].(map[string]any) require.True(t, ok) require.Equal(t, "aas-1", payload["id"]) + effectiveDiff, ok := artifactPayload["effective_diff"].([]any) + require.True(t, ok) + require.Len(t, effectiveDiff, 1) + operation, ok := effectiveDiff[0].(map[string]any) + require.True(t, ok) + require.Equal(t, "add", operation["op"]) + require.Equal(t, "/id", operation["path"]) + require.Equal(t, "aas-1", operation["value"]) require.NoError(t, mock.ExpectationsWereMet()) } @@ -178,7 +186,7 @@ func TestAppendVersionTxRollsBackWhenEvidenceStoreFails(t *testing.T) { mock.ExpectBegin() mock.ExpectExec(`SELECT pg_advisory_xact_lock`). WillReturnResult(sqlmock.NewResult(0, 1)) - mock.ExpectQuery(`SELECT "row_hash" FROM "aas_history"`). + mock.ExpectQuery(`SELECT "history_id" FROM "aas_history"`). WillReturnError(sql.ErrNoRows) mock.ExpectQuery(`INSERT INTO "aas_history".*RETURNING "history_id"`). WillReturnRows(sqlmock.NewRows([]string{"history_id"}).AddRow(1)) @@ -218,7 +226,7 @@ func TestAppendVersionTxRollsBackWhenEvidenceReceiptCatalogFails(t *testing.T) { mock.ExpectBegin() mock.ExpectExec(`SELECT pg_advisory_xact_lock`). WillReturnResult(sqlmock.NewResult(0, 1)) - mock.ExpectQuery(`SELECT "row_hash" FROM "aas_history"`). + mock.ExpectQuery(`SELECT "history_id" FROM "aas_history"`). WillReturnError(sql.ErrNoRows) mock.ExpectQuery(`INSERT INTO "aas_history".*RETURNING "history_id"`). WillReturnRows(sqlmock.NewRows([]string{"history_id"}).AddRow(1)) @@ -397,6 +405,67 @@ func TestAppendVersionTxWritesDiffEvidenceArtifactWhenDiffPayloadIsSelected(t *t diff, ok := artifactPayload["payload"].([]any) require.True(t, ok) require.NotEmpty(t, diff) + effectiveDiff, ok := artifactPayload["effective_diff"].([]any) + require.True(t, ok) + require.Equal(t, diff, effectiveDiff) + require.NoError(t, mock.ExpectationsWereMet()) +} + +func TestAppendVersionTxSnapshotEvidenceIncludesEffectiveDiffOnlyForChangedFields(t *testing.T) { + store := &recordingEvidenceStore{} + t.Cleanup(func() { + Configure(Config{Mode: ModeOff, Immutability: ImmutabilityNone, AuditIdentityMode: AuditIdentityNone}) + }) + Configure(Config{ + Mode: ModeAPI, + FullSnapshotInterval: DefaultFullSnapshotInterval, + Immutability: ImmutabilityNone, + AuditIdentityMode: AuditIdentityNone, + EvidenceEnabled: true, + EvidenceProvider: EvidenceProviderS3, + EvidenceStore: store, + }) + + db, mock, err := sqlmock.New() + require.NoError(t, err) + defer func() { + _ = db.Close() + }() + + largeUnchangedValue := strings.Repeat("unchanged-", 40) + baseSnapshot := map[string]any{"id": "aas-1", "idShort": "before", "description": largeUnchangedValue} + targetSnapshot := map[string]any{"id": "aas-1", "idShort": "after", "description": largeUnchangedValue} + + mock.ExpectBegin() + mock.ExpectExec(`SELECT pg_advisory_xact_lock`). + WillReturnResult(sqlmock.NewResult(0, 1)) + expectLatestSnapshotRestore(mock, TableAAS, "aas_history_payload", "aas-1", 1, baseSnapshot, false) + mock.ExpectQuery(`INSERT INTO "aas_history".*RETURNING "history_id"`). + WillReturnRows(sqlmock.NewRows([]string{"history_id"}).AddRow(2)) + mock.ExpectExec(`INSERT INTO "aas_history_payload".*"snapshot"`). + WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectExec(`INSERT INTO "history_evidence_artifacts"`). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectCommit() + + tx, err := db.Begin() + require.NoError(t, err) + err = AppendVersionTx(context.Background(), tx, TableAAS, "aas-1", ChangeUpdated, targetSnapshot, false) + require.NoError(t, err) + require.NoError(t, tx.Commit()) + + require.Len(t, store.artifacts, 1) + var artifactPayload map[string]any + require.NoError(t, decodeJSONPreservingNumbers(store.artifacts[0].Data, &artifactPayload)) + require.Equal(t, PayloadTypeSnapshot, artifactPayload["payload_type"]) + effectiveDiff, ok := artifactPayload["effective_diff"].([]any) + require.True(t, ok) + require.Len(t, effectiveDiff, 1) + operation, ok := effectiveDiff[0].(map[string]any) + require.True(t, ok) + require.Equal(t, "replace", operation["op"]) + require.Equal(t, "/idShort", operation["path"]) + require.Equal(t, "after", operation["value"]) require.NoError(t, mock.ExpectationsWereMet()) } diff --git a/internal/common/history/evidence_test.go b/internal/common/history/evidence_test.go index d84d2dd30..cd951d25d 100644 --- a/internal/common/history/evidence_test.go +++ b/internal/common/history/evidence_test.go @@ -205,7 +205,14 @@ func TestStoreHistoryEventArtifactsBackfillsSnapshotAndDiffRows(t *testing.T) { require.Len(t, store.artifacts, 2) require.Equal(t, EvidenceArtifactHistoryEvent, store.artifacts[0].ArtifactType) require.Equal(t, EvidenceArtifactHistoryEvent, store.artifacts[1].ArtifactType) - require.Contains(t, string(store.artifacts[1].Data), `"payload_type":"diff"`) + var diffArtifact map[string]any + require.NoError(t, decodeJSONPreservingNumbers(store.artifacts[1].Data, &diffArtifact)) + require.Equal(t, PayloadTypeDiff, diffArtifact["payload_type"]) + payloadDiff, ok := diffArtifact["payload"].([]any) + require.True(t, ok) + effectiveDiff, ok := diffArtifact["effective_diff"].([]any) + require.True(t, ok) + require.Equal(t, payloadDiff, effectiveDiff) require.NoError(t, mock.ExpectationsWereMet()) } diff --git a/internal/common/history/history_event_artifact.go b/internal/common/history/history_event_artifact.go index 2d7fdd58c..d47620b63 100644 --- a/internal/common/history/history_event_artifact.go +++ b/internal/common/history/history_event_artifact.go @@ -28,6 +28,7 @@ package history import ( "context" "database/sql" + "errors" "fmt" "net/url" "path" @@ -87,14 +88,18 @@ func LoadEventArtifactCandidates(ctx context.Context, db *sql.DB, table string, defer func() { _ = rows.Close() }() - return scanEventArtifactCandidates(rows, table) + return scanEventArtifactCandidates(ctx, db, rows, table) } -func buildHistoryEventEvidenceArtifact(table string, historyID int64, event ChangeEvent, payload historyPayload, createdAt string, updatedAt string) (EvidenceArtifact, error) { +func buildHistoryEventEvidenceArtifact(table string, historyID int64, event ChangeEvent, payload historyPayload, effectiveDiff []map[string]any, createdAt string, updatedAt string) (EvidenceArtifact, error) { payloadValue, err := decodeHistoryEventPayload(payload) if err != nil { return EvidenceArtifact{}, err } + effectiveDiffValue, err := canonicalEffectiveDiffValue(effectiveDiff) + if err != nil { + return EvidenceArtifact{}, err + } rowHash := strings.TrimSpace(event.RowHash) if rowHash == "" { return EvidenceArtifact{}, common.NewInternalServerError("HISTORY-EVIDENCE-EVENTARTIFACT-ROWHASH row hash is required") @@ -113,12 +118,13 @@ func buildHistoryEventEvidenceArtifact(table string, historyID int64, event Chan "created_at_text": createdAt, "updated_at_text": updatedAt, }, - "payload_type": payload.payloadType, - "payload": payloadValue, - "content_hash": event.ContentHash, - "payload_hash": event.PayloadHash, - "previous_hash": event.PreviousHash, - "row_hash": rowHash, + "payload_type": payload.payloadType, + "payload": payloadValue, + "effective_diff": effectiveDiffValue, + "content_hash": event.ContentHash, + "payload_hash": event.PayloadHash, + "previous_hash": event.PreviousHash, + "row_hash": rowHash, "audit": map[string]any{ "request_id": event.RequestID, "correlation_id": event.CorrelationID, @@ -147,14 +153,14 @@ func buildHistoryEventEvidenceArtifact(table string, historyID int64, event Chan }, nil } -func publishHistoryEventEvidenceTx(ctx context.Context, tx *sql.Tx, cfg Config, table string, historyID int64, event ChangeEvent, payload historyPayload, createdAt string, updatedAt string) error { +func publishHistoryEventEvidenceTx(ctx context.Context, tx *sql.Tx, cfg Config, table string, historyID int64, event ChangeEvent, payload historyPayload, effectiveDiff []map[string]any, createdAt string, updatedAt string) error { if !cfg.EvidenceEnabled { return nil } if cfg.EvidenceStore == nil { return common.NewInternalServerError("HISTORY-EVIDENCE-APPEND-NILSTORE evidence store is not initialized") } - artifact, err := buildHistoryEventEvidenceArtifact(table, historyID, event, payload, createdAt, updatedAt) + artifact, err := buildHistoryEventEvidenceArtifact(table, historyID, event, payload, effectiveDiff, createdAt, updatedAt) if err != nil { return err } @@ -189,6 +195,33 @@ func decodeHistoryEventPayload(payload historyPayload) (any, error) { return payloadValue, nil } +func canonicalEffectiveDiffValue(effectiveDiff []map[string]any) ([]map[string]any, error) { + if effectiveDiff == nil { + return []map[string]any{}, nil + } + cloned, err := cloneJSONValue(effectiveDiff) + if err != nil { + return nil, common.NewInternalServerError("HISTORY-EVIDENCE-EVENTARTIFACT-EFFECTIVEDIFF " + err.Error()) + } + typed, ok := cloned.([]map[string]any) + if ok { + return typed, nil + } + asAny, ok := cloned.([]any) + if !ok { + return nil, common.NewInternalServerError("HISTORY-EVIDENCE-EVENTARTIFACT-EFFECTIVEDIFF effective diff must be an array") + } + typed = make([]map[string]any, 0, len(asAny)) + for _, item := range asAny { + operation, ok := item.(map[string]any) + if !ok { + return nil, common.NewInternalServerError("HISTORY-EVIDENCE-EVENTARTIFACT-EFFECTIVEDIFF effective diff operation must be an object") + } + typed = append(typed, operation) + } + return typed, nil +} + func historyEventObjectKey(table string, identifier string, historyID int64, rowHash string) string { return path.Join( "history-events", @@ -222,37 +255,44 @@ func historyEventArtifactCandidateQuery(table string, payloadTable string, ident return dataset.Order(historyAlias.Col("history_id").Asc()).ToSQL() } -func scanEventArtifactCandidates(rows *sql.Rows, table string) ([]EventArtifactCandidate, error) { +func scanEventArtifactCandidates(ctx context.Context, db *sql.DB, rows *sql.Rows, table string) ([]EventArtifactCandidate, error) { storedRows, err := scanStoredHistoryRows(rows, table) if err != nil { return nil, err } candidates := make([]EventArtifactCandidate, 0, len(storedRows)) + latestByIdentifier := make(map[string]map[string]any) for _, row := range storedRows { - candidate, err := historyEventArtifactCandidateFromStoredRow(row) + candidate, currentSnapshot, err := historyEventArtifactCandidateFromStoredRow(ctx, db, row, latestByIdentifier[row.Identifier]) if err != nil { return nil, err } + latestByIdentifier[row.Identifier] = currentSnapshot candidates = append(candidates, candidate) } return candidates, nil } -func historyEventArtifactCandidateFromStoredRow(row storedHistoryRow) (EventArtifactCandidate, error) { +func historyEventArtifactCandidateFromStoredRow(ctx context.Context, db *sql.DB, row storedHistoryRow, previousSnapshot map[string]any) (EventArtifactCandidate, map[string]any, error) { payload, err := historyPayloadFromStoredRow(row) if err != nil { - return EventArtifactCandidate{}, err + return EventArtifactCandidate{}, nil, err + } + effectiveDiff, currentSnapshot, err := effectiveDiffFromStoredRow(ctx, db, row, payload, previousSnapshot) + if err != nil { + return EventArtifactCandidate{}, nil, err } artifact, err := buildHistoryEventEvidenceArtifact( row.EntityType, row.HistoryID, historyRowHashEvent(row), payload, + effectiveDiff, nullStringValue(row.CreatedAt), nullStringValue(row.UpdatedAt), ) if err != nil { - return EventArtifactCandidate{}, err + return EventArtifactCandidate{}, nil, err } return EventArtifactCandidate{ Artifact: artifact, @@ -261,7 +301,89 @@ func historyEventArtifactCandidateFromStoredRow(row storedHistoryRow) (EventArti HistoryID: row.HistoryID, RowHash: nullStringValue(row.RowHash), ContentHash: nullStringValue(row.ContentHash), - }, nil + }, currentSnapshot, nil +} + +func effectiveDiffFromStoredRow(ctx context.Context, db *sql.DB, row storedHistoryRow, payload historyPayload, previousSnapshot map[string]any) ([]map[string]any, map[string]any, error) { + switch row.PayloadType { + case PayloadTypeSnapshot: + return effectiveDiffFromSnapshotRow(ctx, db, row, previousSnapshot) + case PayloadTypeDiff: + return effectiveDiffFromDiffRow(ctx, db, row, payload, previousSnapshot) + default: + return nil, nil, common.NewInternalServerError("HISTORY-EVIDENCE-EVENTARTIFACT-PAYLOADTYPE unsupported payload type '" + row.PayloadType + "'") + } +} + +func effectiveDiffFromSnapshotRow(ctx context.Context, db *sql.DB, row storedHistoryRow, previousSnapshot map[string]any) ([]map[string]any, map[string]any, error) { + currentSnapshot, err := restoreSnapshotPayload(row) + if err != nil { + return nil, nil, err + } + if err = verifyCanonicalHash(currentSnapshot, row.ContentHash, "HISTORY-EVIDENCE-EVENTARTIFACT-CONTENTHASH"); err != nil { + return nil, nil, err + } + baseSnapshot := previousSnapshot + if baseSnapshot == nil && strings.TrimSpace(nullStringValue(row.PreviousHash)) != "" { + previousVersion, previousErr := previousVersionBeforeHistoryID(ctx, db, row.EntityType, row.Identifier, row.HistoryID) + if previousErr != nil { + return nil, nil, previousErr + } + baseSnapshot = previousVersion.snapshot + } + if baseSnapshot == nil { + baseSnapshot = map[string]any{} + } + effectiveDiff, err := BuildJSONPatch(baseSnapshot, currentSnapshot) + if err != nil { + return nil, nil, common.NewInternalServerError("HISTORY-EVIDENCE-EVENTARTIFACT-EFFECTIVEDIFF " + err.Error()) + } + return effectiveDiff, currentSnapshot, nil +} + +func effectiveDiffFromDiffRow(ctx context.Context, db *sql.DB, row storedHistoryRow, payload historyPayload, previousSnapshot map[string]any) ([]map[string]any, map[string]any, error) { + var effectiveDiff []map[string]any + if err := decodeJSONPreservingNumbers(payload.json, &effectiveDiff); err != nil { + return nil, nil, common.NewInternalServerError("HISTORY-EVIDENCE-EVENTARTIFACT-EFFECTIVEDIFFDECODE " + err.Error()) + } + if previousSnapshot != nil { + currentSnapshot, err := restoreDiffPayload(previousSnapshot, row) + if err != nil { + return nil, nil, err + } + if err = verifyCanonicalHash(currentSnapshot, row.ContentHash, "HISTORY-EVIDENCE-EVENTARTIFACT-CONTENTHASH"); err != nil { + return nil, nil, err + } + return effectiveDiff, currentSnapshot, nil + } + currentVersion, err := restoreVersionByHistoryID(ctx, db, row.EntityType, row.Identifier, row.HistoryID) + if err != nil { + return nil, nil, err + } + return effectiveDiff, currentVersion.snapshot, nil +} + +func previousVersionBeforeHistoryID(ctx context.Context, queryer historyQueryer, table string, identifier string, historyID int64) (latestVersion, error) { + query, args, err := goqu.From(table). + Select(goqu.C("history_id")). + Where( + goqu.C("identifier").Eq(identifier), + goqu.C("history_id").Lt(historyID), + ). + Order(goqu.C("history_id").Desc()). + Limit(1). + ToSQL() + if err != nil { + return latestVersion{}, common.NewInternalServerError("HISTORY-EVIDENCE-EVENTARTIFACT-BUILDPREVIOUS " + err.Error()) + } + var previousHistoryID int64 + if err = queryer.QueryRowContext(ctx, query, args...).Scan(&previousHistoryID); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return latestVersion{}, common.NewInternalServerError("HISTORY-EVIDENCE-EVENTARTIFACT-MISSINGPREVIOUS previous hash exists but no previous row was found") + } + return latestVersion{}, common.NewInternalServerError("HISTORY-EVIDENCE-EVENTARTIFACT-READPREVIOUS " + err.Error()) + } + return restoreVersionByHistoryID(ctx, queryer, table, identifier, previousHistoryID) } func historyPayloadFromStoredRow(row storedHistoryRow) (historyPayload, error) { From 529a12325dbd52e3fc8b752f9b977b6812b07221 Mon Sep 17 00:00:00 2001 From: Aaron Zielstorff Date: Sat, 6 Jun 2026 08:33:49 +0200 Subject: [PATCH 4/5] Adresses further remark --- database/patches/1_1_3.sql | 31 ++++++++++++++++--- .../docker-compose.yml | 4 +-- internal/common/history/evidence_catalog.go | 9 +++--- internal/common/history/evidence_test.go | 31 +++++++++++++++++++ internal/common/history/evidence_verifier.go | 22 +++++++------ 5 files changed, 77 insertions(+), 20 deletions(-) diff --git a/database/patches/1_1_3.sql b/database/patches/1_1_3.sql index 70f8d16ef..05239c009 100644 --- a/database/patches/1_1_3.sql +++ b/database/patches/1_1_3.sql @@ -102,13 +102,24 @@ CREATE UNIQUE INDEX IF NOT EXISTS ux_history_evidence_manifest_range range_digest ); -CREATE UNIQUE INDEX IF NOT EXISTS ux_history_evidence_artifact_object +CREATE UNIQUE INDEX IF NOT EXISTS ux_history_evidence_artifact_object_manifest ON history_evidence_artifacts( + manifest_id, provider, COALESCE(bucket, ''), object_key, COALESCE(object_version_id, '') - ); + ) + WHERE manifest_id IS NOT NULL; + +CREATE UNIQUE INDEX IF NOT EXISTS ux_history_evidence_artifact_object_unlinked + ON history_evidence_artifacts( + provider, + COALESCE(bucket, ''), + object_key, + COALESCE(object_version_id, '') + ) + WHERE manifest_id IS NULL; CREATE INDEX IF NOT EXISTS ix_history_evidence_manifest_table_identifier ON history_evidence_manifests(history_table, identifier, last_history_id DESC); @@ -119,14 +130,26 @@ CREATE INDEX IF NOT EXISTS ix_history_evidence_artifact_manifest CREATE INDEX IF NOT EXISTS ix_history_evidence_artifact_history_row ON history_evidence_artifacts(history_table, identifier, history_id); -CREATE UNIQUE INDEX IF NOT EXISTS ux_history_evidence_artifact_history_event +CREATE UNIQUE INDEX IF NOT EXISTS ux_history_evidence_artifact_history_event_manifest + ON history_evidence_artifacts( + manifest_id, + history_table, + COALESCE(identifier, ''), + history_id, + COALESCE(row_hash, '') + ) + WHERE artifact_type = 'history_event' + AND manifest_id IS NOT NULL; + +CREATE UNIQUE INDEX IF NOT EXISTS ux_history_evidence_artifact_history_event_unlinked ON history_evidence_artifacts( history_table, COALESCE(identifier, ''), history_id, COALESCE(row_hash, '') ) - WHERE artifact_type = 'history_event'; + WHERE artifact_type = 'history_event' + AND manifest_id IS NULL; -- Mark the schema as upgraded only after all schema objects completed -- successfully. diff --git a/examples/BaSyxHistoryAuditGuardedExample/docker-compose.yml b/examples/BaSyxHistoryAuditGuardedExample/docker-compose.yml index 0da5077d7..1296441c0 100644 --- a/examples/BaSyxHistoryAuditGuardedExample/docker-compose.yml +++ b/examples/BaSyxHistoryAuditGuardedExample/docker-compose.yml @@ -40,7 +40,7 @@ services: - BASYX_HISTORY_EVIDENCE_ACCESS_KEY_ID=minioadmin - BASYX_HISTORY_EVIDENCE_SECRET_ACCESS_KEY=minioadmin - BASYX_HISTORY_EVIDENCE_PATH_STYLE=true - - BASYX_HISTORY_EVIDENCE_RETENTION_MODE=governance + - BASYX_HISTORY_EVIDENCE_RETENTION_MODE=compliance - BASYX_HISTORY_EVIDENCE_RETENTION_DAYS=7 - BASYX_HISTORY_EVIDENCE_WRITE_TIMEOUT_SECONDS=10 - BASYX_HISTORY_INTEGRITY_ANCHOR_PROVIDER=none @@ -136,4 +136,4 @@ services: until mc alias set local http://minio:9000 minioadmin minioadmin; do sleep 2; done mc mb --ignore-existing --with-lock local/basyx-history-evidence mc version enable local/basyx-history-evidence - mc retention set --default governance 7d local/basyx-history-evidence + mc retention set --default compliance 7d local/basyx-history-evidence diff --git a/internal/common/history/evidence_catalog.go b/internal/common/history/evidence_catalog.go index 1b89638bc..dd12453f3 100644 --- a/internal/common/history/evidence_catalog.go +++ b/internal/common/history/evidence_catalog.go @@ -186,14 +186,13 @@ func insertEvidenceManifestCatalogRow(ctx context.Context, tx *sql.Tx, record Ev func insertEvidenceArtifactCatalogRows(ctx context.Context, tx *sql.Tx, manifestID int64, record EvidenceCatalogRecord) error { rows := []goqu.Record{manifestArtifactCatalogRow(manifestID, record)} for _, eventReceipt := range record.EventReceipts { - rows = append(rows, evidenceEventArtifactCatalogRow(eventReceipt)) + rows = append(rows, evidenceEventArtifactCatalogRow(manifestID, eventReceipt)) } for _, snapshotReceipt := range record.SnapshotReceipts { rows = append(rows, snapshotArtifactCatalogRow(manifestID, record.Manifest, snapshotReceipt)) } query, args, err := goqu.Insert(TableHistoryEvidenceArtifacts). Rows(rows). - OnConflict(goqu.DoNothing()). ToSQL() if err != nil { return common.NewInternalServerError("HISTORY-EVIDENCE-CATALOG-BUILDARTIFACTS " + err.Error()) @@ -224,9 +223,9 @@ func snapshotArtifactCatalogRow(manifestID int64, manifest HistoryManifest, snap return row } -func evidenceEventArtifactCatalogRow(eventReceipt EvidenceCatalogEventReceipt) goqu.Record { +func evidenceEventArtifactCatalogRow(manifestID int64, eventReceipt EvidenceCatalogEventReceipt) goqu.Record { candidate := eventReceipt.Candidate - return historyEventArtifactCatalogRow(EventEvidenceRecord{ + row := historyEventArtifactCatalogRow(EventEvidenceRecord{ HistoryTable: candidate.HistoryTable, Identifier: candidate.Identifier, HistoryID: candidate.HistoryID, @@ -234,6 +233,8 @@ func evidenceEventArtifactCatalogRow(eventReceipt EvidenceCatalogEventReceipt) g ContentHash: candidate.ContentHash, Receipt: eventReceipt.Receipt, }) + row["manifest_id"] = manifestID + return row } func historyEventArtifactCatalogRow(record EventEvidenceRecord) goqu.Record { diff --git a/internal/common/history/evidence_test.go b/internal/common/history/evidence_test.go index cd951d25d..72d8ebaab 100644 --- a/internal/common/history/evidence_test.go +++ b/internal/common/history/evidence_test.go @@ -137,6 +137,37 @@ func TestStoreManifestArtifactRejectsNilReceipt(t *testing.T) { require.ErrorContains(t, err, "HISTORY-EVIDENCE-WRITE-NILMANIFESTRECEIPT") } +func TestEvidenceEventArtifactCatalogRowLinksPublishedEventsToManifest(t *testing.T) { + receipt := EvidenceReceipt{ + Reference: EvidenceReference{ + Provider: EvidenceProviderS3, + Bucket: "history-evidence", + ObjectKey: "history-events/aas_history/aas-1/1-rowhash.json", + VersionID: "version-1", + }, + SHA256: strings.Repeat("a", 64), + SizeBytes: 42, + ContentType: manifestJSONContentType, + StoredAt: time.Date(2026, 6, 5, 12, 0, 0, 0, time.UTC), + } + eventReceipt := EvidenceCatalogEventReceipt{ + Candidate: EventArtifactCandidate{ + HistoryTable: TableAAS, + Identifier: "aas-1", + HistoryID: 1, + RowHash: "rowhash", + ContentHash: "contenthash", + }, + Receipt: receipt, + } + + row := evidenceEventArtifactCatalogRow(99, eventReceipt) + + require.Equal(t, int64(99), row["manifest_id"]) + require.Equal(t, EvidenceArtifactHistoryEvent, row["artifact_type"]) + require.Equal(t, int64(1), row["history_id"]) +} + func TestS3EvidenceStoreReceiptAppliesPrefixAndRetention(t *testing.T) { now := time.Date(2026, 6, 5, 12, 0, 0, 0, time.UTC) store := &S3EvidenceStore{ diff --git a/internal/common/history/evidence_verifier.go b/internal/common/history/evidence_verifier.go index 48323c9d7..487605d49 100644 --- a/internal/common/history/evidence_verifier.go +++ b/internal/common/history/evidence_verifier.go @@ -351,18 +351,24 @@ func verifyHistoryEventArtifacts(ctx context.Context, db *sql.DB, options Verify report.addFinding(VerificationSeverityError, "HISTORY-EVIDENCE-VERIFY-EVENTRECEIPTS", err.Error(), "", 0) return } - receiptIndex := indexEventArtifactReceipts(receipts, report) + receiptIndex := indexEventArtifactReceipts(receipts) for _, candidate := range candidates { verifyEventArtifactCandidate(ctx, options, report, candidate, receiptIndex[eventArtifactKey(candidate.Identifier, candidate.HistoryID, candidate.RowHash)]) } } -func verifyEventArtifactCandidate(ctx context.Context, options VerifyHistoryRangeOptions, report *HistoryEvidenceVerificationReport, candidate EventArtifactCandidate, receipt *eventArtifactReceiptRow) { - if receipt == nil { +func verifyEventArtifactCandidate(ctx context.Context, options VerifyHistoryRangeOptions, report *HistoryEvidenceVerificationReport, candidate EventArtifactCandidate, receipts []*eventArtifactReceiptRow) { + if len(receipts) == 0 { report.addFinding(VerificationSeverityError, "HISTORY-EVIDENCE-VERIFY-EVENTMISSING", "history_event artifact receipt is missing", candidate.Identifier, candidate.HistoryID) return } expectedSHA := SHA256Hex(candidate.Artifact.Data) + for _, receipt := range receipts { + verifyEventArtifactReceipt(ctx, options, report, candidate, receipt, expectedSHA) + } +} + +func verifyEventArtifactReceipt(ctx context.Context, options VerifyHistoryRangeOptions, report *HistoryEvidenceVerificationReport, candidate EventArtifactCandidate, receipt *eventArtifactReceiptRow, expectedSHA string) { if !strings.EqualFold(receipt.Receipt.SHA256, expectedSHA) { report.addFinding(VerificationSeverityError, "HISTORY-EVIDENCE-VERIFY-EVENTSHA", "history_event receipt SHA-256 does not match PostgreSQL row artifact", candidate.Identifier, candidate.HistoryID) } @@ -478,16 +484,12 @@ func scanEventArtifactReceiptRow(rows *sql.Rows) (eventArtifactReceiptRow, error return row, nil } -func indexEventArtifactReceipts(receipts []eventArtifactReceiptRow, report *HistoryEvidenceVerificationReport) map[string]*eventArtifactReceiptRow { - index := make(map[string]*eventArtifactReceiptRow, len(receipts)) +func indexEventArtifactReceipts(receipts []eventArtifactReceiptRow) map[string][]*eventArtifactReceiptRow { + index := make(map[string][]*eventArtifactReceiptRow, len(receipts)) for i := range receipts { receipt := &receipts[i] key := eventArtifactKey(receipt.Identifier, receipt.HistoryID, receipt.RowHash) - if existing := index[key]; existing != nil { - report.addFinding(VerificationSeverityError, "HISTORY-EVIDENCE-VERIFY-EVENTDUPLICATE", "multiple history_event receipts exist for one history row", receipt.Identifier, receipt.HistoryID) - continue - } - index[key] = receipt + index[key] = append(index[key], receipt) } return index } From 9a59bd39ec06ba632720494f18610c06c52e62c1 Mon Sep 17 00:00:00 2001 From: Aaron Zielstorff Date: Mon, 8 Jun 2026 13:15:39 +0200 Subject: [PATCH 5/5] Addresses Martins remarks --- cmd/historyevidenceverifier/main.go | 11 ++ cmd/historyevidenceverifier/main_test.go | 43 ++++++ internal/common/error_handler.go | 30 ++++ internal/common/error_handler_test.go | 21 +++ internal/common/history/append_test.go | 1 + internal/common/history/evidence_store_s3.go | 94 ++++++++++- internal/common/history/evidence_test.go | 146 ++++++++++++++++++ internal/common/history/evidence_types.go | 9 ++ internal/common/history/evidence_verifier.go | 22 ++- .../common/history/history_event_artifact.go | 2 +- 10 files changed, 376 insertions(+), 3 deletions(-) create mode 100644 cmd/historyevidenceverifier/main_test.go diff --git a/cmd/historyevidenceverifier/main.go b/cmd/historyevidenceverifier/main.go index a220c2547..aa2f2b60a 100644 --- a/cmd/historyevidenceverifier/main.go +++ b/cmd/historyevidenceverifier/main.go @@ -112,6 +112,17 @@ func validateCLIOptions(options cliOptions) error { if options.lastHistoryID < options.firstHistoryID { return fmt.Errorf("HISTORY-EVIDENCE-CLI-TO -to must be greater than or equal to -from") } + hasManifestObjectKey := strings.TrimSpace(options.manifestObjectKey) != "" + hasManifestSHA256 := strings.TrimSpace(options.manifestSHA256) != "" + if hasManifestObjectKey && !hasManifestSHA256 { + return fmt.Errorf("HISTORY-EVIDENCE-CLI-MANIFESTHASH -manifest-sha256 is required when -manifest-object-key is set") + } + if hasManifestSHA256 && !hasManifestObjectKey { + return fmt.Errorf("HISTORY-EVIDENCE-CLI-MANIFESTOBJECT -manifest-object-key is required when -manifest-sha256 is set") + } + if strings.TrimSpace(options.manifestVersionID) != "" && !hasManifestObjectKey { + return fmt.Errorf("HISTORY-EVIDENCE-CLI-MANIFESTVERSION -manifest-object-key is required when -manifest-version-id is set") + } return nil } diff --git a/cmd/historyevidenceverifier/main_test.go b/cmd/historyevidenceverifier/main_test.go new file mode 100644 index 000000000..0d6d06dff --- /dev/null +++ b/cmd/historyevidenceverifier/main_test.go @@ -0,0 +1,43 @@ +/******************************************************************************* +* Copyright (C) 2026 the Eclipse BaSyx Authors and Fraunhofer IESE +* +* Permission is hereby granted, free of charge, to any person obtaining +* a copy of this software and associated documentation files (the +* "Software"), to deal in the Software without restriction, including +* without limitation the rights to use, copy, modify, merge, publish, +* distribute, sublicense, and/or sell copies of the Software, and to +* permit persons to whom the Software is furnished to do so, subject to +* the following conditions: +* +* The above copyright notice and this permission notice shall be +* included in all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +* +* SPDX-License-Identifier: MIT +******************************************************************************/ + +package main + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestValidateCLIOptionsRejectsManifestObjectKeyWithoutHash(t *testing.T) { + err := validateCLIOptions(cliOptions{ + historyTable: "aas_history", + firstHistoryID: 1, + lastHistoryID: 10, + manifestObjectKey: "history-evidence/manifests/aas_history/1-10.json", + }) + + require.ErrorContains(t, err, "HISTORY-EVIDENCE-CLI-MANIFESTHASH") +} diff --git a/internal/common/error_handler.go b/internal/common/error_handler.go index 0167fdb89..623e78a6f 100644 --- a/internal/common/error_handler.go +++ b/internal/common/error_handler.go @@ -104,6 +104,22 @@ func NewInternalServerError(message string) error { return errors.New("500 Internal Server Error: " + message) } +// NewErrServiceUnavailable creates a standardized "503 Service Unavailable" error. +// +// Parameters: +// - message: Description of the temporarily unavailable dependency or service. +// +// Returns: +// - error: An error with message format "503 Service Unavailable: " +// +// Example: +// +// err := NewErrServiceUnavailable("object store unavailable") +// // Returns error: "503 Service Unavailable: object store unavailable" +func NewErrServiceUnavailable(message string) error { + return errors.New("503 Service Unavailable: " + message) +} + // NewErrMethodNotAllowed creates a standardized "405 Method Not Allowed" error. // // Parameters: @@ -201,6 +217,17 @@ func IsInternalServerError(err error) bool { return hasErrorPrefix(err, "500 Internal Server Error: ") } +// IsErrServiceUnavailable checks if the given error is a "503 Service Unavailable" error. +// +// Parameters: +// - err: The error to check +// +// Returns: +// - bool: true if the error is a 503 Service Unavailable error, false otherwise +func IsErrServiceUnavailable(err error) bool { + return hasErrorPrefix(err, "503 Service Unavailable: ") +} + // IsErrConflict checks if the given error is a "409 Conflict" error. // // Parameters: @@ -276,6 +303,9 @@ func hasErrorPrefix(err error, prefix string) bool { // response := NewErrorResponse(err, 404, "submodel", "GetSubmodel", "invalidID") // // Creates response with correlation ID: "submodel-404-GetSubmodel-NotFound-invalidID" func NewErrorResponse(err error, errorCode int, component string, function string, info string) model.ImplResponse { + if IsErrServiceUnavailable(err) { + errorCode = http.StatusServiceUnavailable + } codeStr := strconv.Itoa(errorCode) statusText := strings.ReplaceAll(http.StatusText(errorCode), " ", "") internalCode := fmt.Sprintf("%s-%s-%s-%s-%s", component, codeStr, function, statusText, info) diff --git a/internal/common/error_handler_test.go b/internal/common/error_handler_test.go index 65db60ac2..8aa19a944 100644 --- a/internal/common/error_handler_test.go +++ b/internal/common/error_handler_test.go @@ -1,7 +1,9 @@ package common import ( + "errors" "fmt" + "net/http" "testing" ) @@ -26,6 +28,11 @@ func TestErrorClassifiers_RecognizeWrappedErrors(t *testing.T) { err: fmt.Errorf("outer: %w", NewInternalServerError("x")), assert: IsInternalServerError, }, + { + name: "service unavailable", + err: fmt.Errorf("outer: %w", NewErrServiceUnavailable("x")), + assert: IsErrServiceUnavailable, + }, { name: "conflict", err: fmt.Errorf("outer: %w", NewErrConflict("x")), @@ -52,3 +59,17 @@ func TestErrorClassifiers_RecognizeWrappedErrors(t *testing.T) { }) } } + +func TestNewErrorResponsePreservesExplicitServiceUnavailable(t *testing.T) { + response := NewErrorResponse( + errors.New("503 Service Unavailable: object storage unavailable"), + http.StatusInternalServerError, + "SMREPO", + "PatchSubmodelElement", + "EvidenceStore", + ) + + if response.Code != http.StatusServiceUnavailable { + t.Fatalf("expected status %d, got %d", http.StatusServiceUnavailable, response.Code) + } +} diff --git a/internal/common/history/append_test.go b/internal/common/history/append_test.go index a2f02ff19..8d7981564 100644 --- a/internal/common/history/append_test.go +++ b/internal/common/history/append_test.go @@ -198,6 +198,7 @@ func TestAppendVersionTxRollsBackWhenEvidenceStoreFails(t *testing.T) { require.NoError(t, err) err = AppendVersionTx(context.Background(), tx, TableAAS, "aas-1", ChangeCreated, map[string]any{"id": "aas-1"}, false) require.ErrorContains(t, err, "HISTORY-EVIDENCE-APPEND-PUTARTIFACT") + require.ErrorContains(t, err, "503 Service Unavailable") require.NoError(t, tx.Rollback()) require.NoError(t, mock.ExpectationsWereMet()) } diff --git a/internal/common/history/evidence_store_s3.go b/internal/common/history/evidence_store_s3.go index 8a804a41c..f78529421 100644 --- a/internal/common/history/evidence_store_s3.go +++ b/internal/common/history/evidence_store_s3.go @@ -169,9 +169,14 @@ func (store *S3EvidenceStore) PutArtifact(ctx context.Context, artifact Evidence if err != nil { return nil, fmt.Errorf("HISTORY-EVIDENCE-S3-PUTOBJECT %w", err) } + versionID := "" if output.VersionId != nil { - receipt.Reference.VersionID = strings.TrimSpace(*output.VersionId) + versionID = strings.TrimSpace(*output.VersionId) + } + if versionID == "" { + return nil, fmt.Errorf("HISTORY-EVIDENCE-S3-VERSIONID object storage did not return a version ID; ensure bucket versioning and object lock are enabled") } + receipt.Reference.VersionID = versionID return receipt, nil } @@ -253,6 +258,93 @@ func (store *S3EvidenceStore) VerifyArtifact(ctx context.Context, ref EvidenceRe }, nil } +// VerifyArtifactRetention verifies S3 Object Lock retention and legal hold state. +// +// Parameters: +// - ctx: Request context for S3 Object Lock API calls. +// - ref: Versioned S3 object reference to verify. +// - expected: PostgreSQL receipt containing expected retention and legal hold. +// +// Returns: +// - error: Error when the object reference is incomplete, S3 cannot return +// retention/legal-hold state, or backend state differs from the receipt. +func (store *S3EvidenceStore) VerifyArtifactRetention(ctx context.Context, ref EvidenceReference, expected EvidenceReceipt) error { + if store == nil || store.client == nil { + return fmt.Errorf("HISTORY-EVIDENCE-S3-NILSTORE evidence store is not initialized") + } + objectKey := strings.TrimSpace(ref.ObjectKey) + if objectKey == "" { + return fmt.Errorf("HISTORY-EVIDENCE-S3-EMPTYREF artifact object key is required") + } + versionID := strings.TrimSpace(ref.VersionID) + if versionID == "" { + return fmt.Errorf("HISTORY-EVIDENCE-S3-RETENTIONVERSION object version ID is required for retention verification") + } + retention, err := store.getObjectRetention(ctx, objectKey, versionID) + if err != nil { + return err + } + if err = compareS3Retention(retention, expected); err != nil { + return err + } + legalHold, err := store.getObjectLegalHold(ctx, objectKey, versionID) + if err != nil { + return err + } + if legalHold != expected.LegalHold { + return fmt.Errorf("HISTORY-EVIDENCE-S3-LEGALHOLDMISMATCH expected legal hold %t, got %t", expected.LegalHold, legalHold) + } + return nil +} + +func (store *S3EvidenceStore) getObjectRetention(ctx context.Context, objectKey string, versionID string) (*types.ObjectLockRetention, error) { + output, err := store.client.GetObjectRetention(ctx, &s3.GetObjectRetentionInput{ + Bucket: aws.String(store.cfg.Bucket), + Key: aws.String(objectKey), + VersionId: aws.String(versionID), + }) + if err != nil { + return nil, fmt.Errorf("HISTORY-EVIDENCE-S3-GETRETENTION %w", err) + } + if output.Retention == nil { + return nil, fmt.Errorf("HISTORY-EVIDENCE-S3-RETENTIONMISSING object retention is missing") + } + return output.Retention, nil +} + +func compareS3Retention(actual *types.ObjectLockRetention, expected EvidenceReceipt) error { + expectedMode := strings.ToLower(strings.TrimSpace(expected.RetentionMode)) + if expectedMode == "" || expected.RetainUntil == nil { + return fmt.Errorf("HISTORY-EVIDENCE-S3-RETENTIONEXPECTED expected retention metadata is missing") + } + actualMode := strings.ToLower(string(actual.Mode)) + if actualMode != expectedMode { + return fmt.Errorf("HISTORY-EVIDENCE-S3-RETENTIONMODEMISMATCH expected retention mode %s, got %s", expectedMode, actualMode) + } + if actual.RetainUntilDate == nil { + return fmt.Errorf("HISTORY-EVIDENCE-S3-RETAINUNTILMISSING retain-until timestamp is missing") + } + if actual.RetainUntilDate.Before(expected.RetainUntil.Add(-time.Second)) { + return fmt.Errorf("HISTORY-EVIDENCE-S3-RETAINUNTILMISMATCH actual retain-until is earlier than expected") + } + return nil +} + +func (store *S3EvidenceStore) getObjectLegalHold(ctx context.Context, objectKey string, versionID string) (bool, error) { + output, err := store.client.GetObjectLegalHold(ctx, &s3.GetObjectLegalHoldInput{ + Bucket: aws.String(store.cfg.Bucket), + Key: aws.String(objectKey), + VersionId: aws.String(versionID), + }) + if err != nil { + return false, fmt.Errorf("HISTORY-EVIDENCE-S3-GETLEGALHOLD %w", err) + } + if output.LegalHold == nil { + return false, fmt.Errorf("HISTORY-EVIDENCE-S3-LEGALHOLDMISSING object legal-hold state is missing") + } + return output.LegalHold.Status == types.ObjectLockLegalHoldStatusOn, nil +} + func (store *S3EvidenceStore) objectKey(rawKey string) string { key := strings.Trim(strings.TrimSpace(rawKey), "/") if key == "" { diff --git a/internal/common/history/evidence_test.go b/internal/common/history/evidence_test.go index 72d8ebaab..383a1b3de 100644 --- a/internal/common/history/evidence_test.go +++ b/internal/common/history/evidence_test.go @@ -29,11 +29,16 @@ import ( "context" "crypto/rand" "crypto/rsa" + "errors" + "io" + "net/http" "strings" "testing" "time" sqlmock "github.com/DATA-DOG/go-sqlmock" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/stretchr/testify/require" ) @@ -202,6 +207,72 @@ func TestS3EvidenceStorePutRequiresRetention(t *testing.T) { require.ErrorContains(t, err, "HISTORY-EVIDENCE-S3-RETENTION") } +func TestS3EvidenceStorePutRequiresVersionID(t *testing.T) { + client := s3.NewFromConfig(aws.Config{ + Region: "us-east-1", + Credentials: credentials.NewStaticCredentialsProvider("access", "secret", ""), + HTTPClient: httpClientFunc(func(*http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusOK, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader("")), + }, nil + }), + }) + store := &S3EvidenceStore{ + client: client, + cfg: S3EvidenceStoreConfig{ + Bucket: "evidence", + RetentionMode: "governance", + RetentionDays: 7, + }, + now: func() time.Time { return time.Date(2026, 6, 5, 12, 0, 0, 0, time.UTC) }, + } + + receipt, err := store.PutArtifact(t.Context(), EvidenceArtifact{ + ArtifactType: EvidenceArtifactHistoryEvent, + ObjectKey: "events/one.json", + ContentType: manifestJSONContentType, + Data: []byte(`{}`), + }) + + require.Nil(t, receipt) + require.ErrorContains(t, err, "HISTORY-EVIDENCE-S3-VERSIONID") +} + +func TestS3EvidenceStoreVerifiesRetentionState(t *testing.T) { + retainUntil := time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC) + client := s3.NewFromConfig(aws.Config{ + Region: "us-east-1", + Credentials: credentials.NewStaticCredentialsProvider("access", "secret", ""), + HTTPClient: httpClientFunc(func(request *http.Request) (*http.Response, error) { + body := `OFF` + if strings.Contains(request.URL.RawQuery, "retention") { + body = `GOVERNANCE2026-06-12T12:00:00Z` + } + return &http.Response{ + StatusCode: http.StatusOK, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader(body)), + }, nil + }), + }) + store := &S3EvidenceStore{client: client, cfg: S3EvidenceStoreConfig{Bucket: "evidence"}} + + err := store.VerifyArtifactRetention(t.Context(), EvidenceReference{ + Provider: EvidenceProviderS3, + Bucket: "evidence", + ObjectKey: "history-events/aas_history/aas-1/1-row.json", + VersionID: "version-1", + }, EvidenceReceipt{ + RetentionMode: "governance", + RetainUntil: &retainUntil, + LegalHold: false, + }) + + require.NoError(t, err) +} + func TestStoreHistoryEventArtifactsBackfillsSnapshotAndDiffRows(t *testing.T) { db, mock, err := sqlmock.New() require.NoError(t, err) @@ -342,6 +413,55 @@ func TestVerifyManifestRangeReportsDigestMismatch(t *testing.T) { require.Equal(t, "HISTORY-EVIDENCE-VERIFY-MANIFESTDIGEST", report.Findings[0].Code) } +func TestVerifyManifestArtifactReportsPartialConfiguration(t *testing.T) { + report := &HistoryEvidenceVerificationReport{Valid: true} + + verifyManifestArtifact(t.Context(), VerifyHistoryRangeOptions{ + EvidenceStore: nilReceiptEvidenceStore{}, + ManifestArtifactRef: EvidenceReference{Provider: EvidenceProviderS3, Bucket: "evidence", ObjectKey: "history-evidence/manifests/aas_history/1-10.json"}, + }, report) + + require.False(t, report.Valid) + require.Contains(t, verificationFindingCodes(report), "HISTORY-EVIDENCE-VERIFY-MANIFESTARTIFACTCONFIG") +} + +func TestVerifyEventArtifactReceiptChecksSourceRetentionState(t *testing.T) { + retainUntil := time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC) + candidate := EventArtifactCandidate{ + Identifier: "aas-1", + HistoryID: 1, + RowHash: strings.Repeat("a", 64), + ContentHash: strings.Repeat("b", 64), + Artifact: EvidenceArtifact{ + Data: []byte(`{"artifact_version":"basyx-history-event-v1"}`), + }, + } + receipt := &eventArtifactReceiptRow{ + Identifier: candidate.Identifier, + HistoryID: candidate.HistoryID, + RowHash: candidate.RowHash, + ContentHash: candidate.ContentHash, + Receipt: EvidenceReceipt{ + Reference: EvidenceReference{ + Provider: EvidenceProviderS3, + Bucket: "evidence", + ObjectKey: "history-events/aas_history/aas-1/1-row.json", + VersionID: "version-1", + }, + SHA256: SHA256Hex(candidate.Artifact.Data), + RetentionMode: "governance", + RetainUntil: &retainUntil, + }, + } + report := &HistoryEvidenceVerificationReport{Valid: true} + store := &retentionCheckingEvidenceStore{retentionErr: errors.New("retention state mismatch")} + + verifyEventArtifactReceipt(t.Context(), VerifyHistoryRangeOptions{EvidenceStore: store}, report, candidate, receipt, receipt.Receipt.SHA256) + + require.False(t, report.Valid) + require.Contains(t, verificationFindingCodes(report), "HISTORY-EVIDENCE-VERIFY-EVENTRETENTIONSTATE") +} + func eventArtifactReceiptColumns() []string { return []string{ "artifact_id", @@ -383,3 +503,29 @@ func (nilReceiptEvidenceStore) GetArtifact(_ context.Context, _ EvidenceReferenc func (nilReceiptEvidenceStore) VerifyArtifact(_ context.Context, _ EvidenceReference, _ string) (*EvidenceReceipt, error) { return nil, nil } + +type retentionCheckingEvidenceStore struct { + retentionErr error +} + +func (store *retentionCheckingEvidenceStore) PutArtifact(_ context.Context, _ EvidenceArtifact) (*EvidenceReceipt, error) { + return nil, nil +} + +func (store *retentionCheckingEvidenceStore) GetArtifact(_ context.Context, _ EvidenceReference) (*EvidenceObject, error) { + return nil, nil +} + +func (store *retentionCheckingEvidenceStore) VerifyArtifact(_ context.Context, _ EvidenceReference, _ string) (*EvidenceReceipt, error) { + return &EvidenceReceipt{}, nil +} + +func (store *retentionCheckingEvidenceStore) VerifyArtifactRetention(_ context.Context, _ EvidenceReference, _ EvidenceReceipt) error { + return store.retentionErr +} + +type httpClientFunc func(*http.Request) (*http.Response, error) + +func (fn httpClientFunc) Do(request *http.Request) (*http.Response, error) { + return fn(request) +} diff --git a/internal/common/history/evidence_types.go b/internal/common/history/evidence_types.go index 6aed01990..5c43773b2 100644 --- a/internal/common/history/evidence_types.go +++ b/internal/common/history/evidence_types.go @@ -55,6 +55,15 @@ type EvidenceStore interface { VerifyArtifact(ctx context.Context, ref EvidenceReference, expectedHash string) (*EvidenceReceipt, error) } +// EvidenceRetentionVerifier verifies WORM retention state from the evidence backend. +// +// Implementations use provider-specific APIs to compare the stored object version +// against the PostgreSQL receipt. The S3 implementation checks Object Lock +// retention and legal hold state for the referenced version. +type EvidenceRetentionVerifier interface { + VerifyArtifactRetention(ctx context.Context, ref EvidenceReference, expected EvidenceReceipt) error +} + // EvidenceArtifact is a byte artifact destined for WORM-compatible object storage. type EvidenceArtifact struct { ArtifactType string diff --git a/internal/common/history/evidence_verifier.go b/internal/common/history/evidence_verifier.go index 487605d49..fc07abecb 100644 --- a/internal/common/history/evidence_verifier.go +++ b/internal/common/history/evidence_verifier.go @@ -332,7 +332,13 @@ func verifyIdentifierChain(ctx context.Context, queryer historyQueryer, table st } func verifyManifestArtifact(ctx context.Context, options VerifyHistoryRangeOptions, report *HistoryEvidenceVerificationReport) { - if options.EvidenceStore == nil || strings.TrimSpace(options.ManifestArtifactRef.ObjectKey) == "" || strings.TrimSpace(options.ManifestArtifactHash) == "" { + hasObjectKey := strings.TrimSpace(options.ManifestArtifactRef.ObjectKey) != "" + hasHash := strings.TrimSpace(options.ManifestArtifactHash) != "" + if !hasObjectKey && !hasHash { + return + } + if options.EvidenceStore == nil || !hasObjectKey || !hasHash { + report.addFinding(VerificationSeverityError, "HISTORY-EVIDENCE-VERIFY-MANIFESTARTIFACTCONFIG", "manifest artifact verification requires evidence store, object key, and SHA-256", "", 0) return } if _, err := options.EvidenceStore.VerifyArtifact(ctx, options.ManifestArtifactRef, options.ManifestArtifactHash); err != nil { @@ -384,6 +390,20 @@ func verifyEventArtifactReceipt(ctx context.Context, options VerifyHistoryRangeO if _, err := options.EvidenceStore.VerifyArtifact(ctx, receipt.Receipt.Reference, receipt.Receipt.SHA256); err != nil { report.addFinding(VerificationSeverityError, "HISTORY-EVIDENCE-VERIFY-EVENTOBJECT", fmt.Sprintf("history_event artifact verification failed: %v", err), candidate.Identifier, candidate.HistoryID) } + verifyEventArtifactRetentionState(ctx, options.EvidenceStore, report, candidate, receipt) +} + +func verifyEventArtifactRetentionState(ctx context.Context, store EvidenceStore, report *HistoryEvidenceVerificationReport, candidate EventArtifactCandidate, receipt *eventArtifactReceiptRow) { + if receipt.Receipt.RetentionMode == "" || receipt.Receipt.RetainUntil == nil { + return + } + retentionVerifier, ok := store.(EvidenceRetentionVerifier) + if !ok { + return + } + if err := retentionVerifier.VerifyArtifactRetention(ctx, receipt.Receipt.Reference, receipt.Receipt); err != nil { + report.addFinding(VerificationSeverityError, "HISTORY-EVIDENCE-VERIFY-EVENTRETENTIONSTATE", fmt.Sprintf("history_event artifact retention verification failed: %v", err), candidate.Identifier, candidate.HistoryID) + } } func loadEventArtifactReceiptRows(ctx context.Context, queryer historyQueryer, table string, identifier string, firstHistoryID int64, lastHistoryID int64) ([]eventArtifactReceiptRow, error) { diff --git a/internal/common/history/history_event_artifact.go b/internal/common/history/history_event_artifact.go index d47620b63..b70be4625 100644 --- a/internal/common/history/history_event_artifact.go +++ b/internal/common/history/history_event_artifact.go @@ -172,7 +172,7 @@ func publishHistoryEventEvidenceTx(ctx context.Context, tx *sql.Tx, cfg Config, defer cancel() receipt, err := cfg.EvidenceStore.PutArtifact(writeCtx, artifact) if err != nil { - return common.NewInternalServerError("HISTORY-EVIDENCE-APPEND-PUTARTIFACT " + err.Error()) + return common.NewErrServiceUnavailable("HISTORY-EVIDENCE-APPEND-PUTARTIFACT " + err.Error()) } if receipt == nil { return common.NewInternalServerError("HISTORY-EVIDENCE-APPEND-NILRECEIPT evidence store returned nil receipt")