From d4abf061d5dbf5a69689e1f283f175891a5db146 Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 14:20:12 +0200 Subject: [PATCH 01/77] build(crypto): scaffold SquidStd.Crypto module over PgpCore --- SquidStd.slnx | 1 + src/SquidStd.Crypto/SquidStd.Crypto.csproj | 19 +++++++++++++++++++ tests/SquidStd.Tests/SquidStd.Tests.csproj | 1 + 3 files changed, 21 insertions(+) create mode 100644 src/SquidStd.Crypto/SquidStd.Crypto.csproj diff --git a/SquidStd.slnx b/SquidStd.slnx index c970f273..55f1af76 100644 --- a/SquidStd.slnx +++ b/SquidStd.slnx @@ -37,6 +37,7 @@ + diff --git a/src/SquidStd.Crypto/SquidStd.Crypto.csproj b/src/SquidStd.Crypto/SquidStd.Crypto.csproj new file mode 100644 index 00000000..f7054b8e --- /dev/null +++ b/src/SquidStd.Crypto/SquidStd.Crypto.csproj @@ -0,0 +1,19 @@ + + + + net10.0 + enable + enable + true + + + + + + + + + + + + diff --git a/tests/SquidStd.Tests/SquidStd.Tests.csproj b/tests/SquidStd.Tests/SquidStd.Tests.csproj index 810f8e0c..82c89143 100644 --- a/tests/SquidStd.Tests/SquidStd.Tests.csproj +++ b/tests/SquidStd.Tests/SquidStd.Tests.csproj @@ -70,6 +70,7 @@ + From fd3abe8f25dc709e606c56f481b94b42c226cda9 Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 14:21:02 +0200 Subject: [PATCH 02/77] feat(crypto): add PGP key model, options, and result DTOs --- .../Pgp/Data/PgpDecryptionResult.cs | 7 +++++++ src/SquidStd.Crypto/Pgp/Data/PgpKey.cs | 21 +++++++++++++++++++ src/SquidStd.Crypto/Pgp/Data/PgpKeyOptions.cs | 18 ++++++++++++++++ .../Pgp/Data/PgpVerificationResult.cs | 7 +++++++ .../Pgp/Types/PgpKeyAlgorithm.cs | 10 +++++++++ 5 files changed, 63 insertions(+) create mode 100644 src/SquidStd.Crypto/Pgp/Data/PgpDecryptionResult.cs create mode 100644 src/SquidStd.Crypto/Pgp/Data/PgpKey.cs create mode 100644 src/SquidStd.Crypto/Pgp/Data/PgpKeyOptions.cs create mode 100644 src/SquidStd.Crypto/Pgp/Data/PgpVerificationResult.cs create mode 100644 src/SquidStd.Crypto/Pgp/Types/PgpKeyAlgorithm.cs diff --git a/src/SquidStd.Crypto/Pgp/Data/PgpDecryptionResult.cs b/src/SquidStd.Crypto/Pgp/Data/PgpDecryptionResult.cs new file mode 100644 index 00000000..417836b4 --- /dev/null +++ b/src/SquidStd.Crypto/Pgp/Data/PgpDecryptionResult.cs @@ -0,0 +1,7 @@ +namespace SquidStd.Crypto.Pgp.Data; + +/// +/// Outcome of decrypt-and-verify: the recovered plaintext, whether the message carried a signature, and +/// whether that signature validated against a keyring public key. +/// +public sealed record PgpDecryptionResult(byte[] Data, bool IsSigned, bool IsValid); diff --git a/src/SquidStd.Crypto/Pgp/Data/PgpKey.cs b/src/SquidStd.Crypto/Pgp/Data/PgpKey.cs new file mode 100644 index 00000000..467f18dd --- /dev/null +++ b/src/SquidStd.Crypto/Pgp/Data/PgpKey.cs @@ -0,0 +1,21 @@ +using SquidStd.Crypto.Pgp.Types; + +namespace SquidStd.Crypto.Pgp.Data; + +/// +/// An OpenPGP key as held in the keyring: identity, key id, fingerprint, the armored public block, and +/// optionally the armored secret block. Metadata is derived from the public key material. +/// +public sealed record PgpKey( + string Identity, + string KeyId, + string Fingerprint, + string PublicArmored, + string? PrivateArmored, + DateTimeOffset CreatedUtc, + DateTimeOffset? ExpiresUtc, + PgpKeyAlgorithm Algorithm) +{ + /// Whether this key carries secret material (can decrypt and sign). + public bool HasSecret => PrivateArmored is not null; +} diff --git a/src/SquidStd.Crypto/Pgp/Data/PgpKeyOptions.cs b/src/SquidStd.Crypto/Pgp/Data/PgpKeyOptions.cs new file mode 100644 index 00000000..69f62e86 --- /dev/null +++ b/src/SquidStd.Crypto/Pgp/Data/PgpKeyOptions.cs @@ -0,0 +1,18 @@ +using SquidStd.Crypto.Pgp.Types; + +namespace SquidStd.Crypto.Pgp.Data; + +/// +/// Options controlling key generation: algorithm, key size, and an optional validity period. +/// +public sealed class PgpKeyOptions +{ + /// Public-key algorithm family. Defaults to RSA. + public PgpKeyAlgorithm Algorithm { get; init; } = PgpKeyAlgorithm.Rsa; + + /// RSA key size in bits. Defaults to 2048. + public int KeySizeBits { get; init; } = 2048; + + /// Optional validity period from creation; null means the key does not expire. + public TimeSpan? ExpiresAfter { get; init; } +} diff --git a/src/SquidStd.Crypto/Pgp/Data/PgpVerificationResult.cs b/src/SquidStd.Crypto/Pgp/Data/PgpVerificationResult.cs new file mode 100644 index 00000000..7fe6fc3d --- /dev/null +++ b/src/SquidStd.Crypto/Pgp/Data/PgpVerificationResult.cs @@ -0,0 +1,7 @@ +namespace SquidStd.Crypto.Pgp.Data; + +/// +/// Outcome of verifying a signed message: whether the signature validated against a keyring public key, +/// plus the recovered content. PgpCore reports pass/fail only — no signer attribution is exposed. +/// +public sealed record PgpVerificationResult(bool IsValid, byte[] Data); diff --git a/src/SquidStd.Crypto/Pgp/Types/PgpKeyAlgorithm.cs b/src/SquidStd.Crypto/Pgp/Types/PgpKeyAlgorithm.cs new file mode 100644 index 00000000..f98a5288 --- /dev/null +++ b/src/SquidStd.Crypto/Pgp/Types/PgpKeyAlgorithm.cs @@ -0,0 +1,10 @@ +namespace SquidStd.Crypto.Pgp.Types; + +/// +/// OpenPGP public-key algorithm family for a generated key. Extensible (ECC may be added later). +/// +public enum PgpKeyAlgorithm +{ + /// RSA key pair. + Rsa +} From 22bc121c6a0da19c108f2a70b1b687ba912c3578 Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 14:21:02 +0200 Subject: [PATCH 03/77] feat(crypto): add PGP service, keyring, and key store contracts --- .../Pgp/Interfaces/IPgpKeyStore.cs | 15 +++++++ .../Pgp/Interfaces/IPgpKeyring.cs | 31 ++++++++++++++ .../Pgp/Interfaces/IPgpService.cs | 40 +++++++++++++++++++ 3 files changed, 86 insertions(+) create mode 100644 src/SquidStd.Crypto/Pgp/Interfaces/IPgpKeyStore.cs create mode 100644 src/SquidStd.Crypto/Pgp/Interfaces/IPgpKeyring.cs create mode 100644 src/SquidStd.Crypto/Pgp/Interfaces/IPgpService.cs diff --git a/src/SquidStd.Crypto/Pgp/Interfaces/IPgpKeyStore.cs b/src/SquidStd.Crypto/Pgp/Interfaces/IPgpKeyStore.cs new file mode 100644 index 00000000..b5f4ebbb --- /dev/null +++ b/src/SquidStd.Crypto/Pgp/Interfaces/IPgpKeyStore.cs @@ -0,0 +1,15 @@ +using SquidStd.Crypto.Pgp.Data; + +namespace SquidStd.Crypto.Pgp.Interfaces; + +/// +/// Persistence backend for a set of PGP keys. Implementations decide the on-disk representation. +/// +public interface IPgpKeyStore +{ + /// Persists the supplied keys, replacing any previously stored set. + Task SaveAsync(IReadOnlyCollection keys, CancellationToken cancellationToken = default); + + /// Loads all persisted keys. Returns an empty list when nothing is stored. + Task> LoadAsync(CancellationToken cancellationToken = default); +} diff --git a/src/SquidStd.Crypto/Pgp/Interfaces/IPgpKeyring.cs b/src/SquidStd.Crypto/Pgp/Interfaces/IPgpKeyring.cs new file mode 100644 index 00000000..bd2917f6 --- /dev/null +++ b/src/SquidStd.Crypto/Pgp/Interfaces/IPgpKeyring.cs @@ -0,0 +1,31 @@ +using SquidStd.Crypto.Pgp.Data; + +namespace SquidStd.Crypto.Pgp.Interfaces; + +/// +/// In-memory collection of PGP keys, indexed by identity, key id, and fingerprint. Crypto operations look +/// recipients and signers up here by identity. +/// +public interface IPgpKeyring +{ + /// All keys currently held. + IReadOnlyCollection Keys { get; } + + /// Imports an armored public or secret key block (auto-detected) and returns the parsed key. + PgpKey Import(string armored); + + /// Removes a key matched by identity, key id, or fingerprint. Returns whether one was removed. + bool Remove(string identityOrKeyIdOrFingerprint); + + /// Finds a key by identity, key id, or fingerprint; null when absent. + PgpKey? Find(string identityOrKeyIdOrFingerprint); + + /// Whether a key matching identity, key id, or fingerprint is present. + bool Contains(string identityOrKeyIdOrFingerprint); + + /// Replaces the keyring contents with the key store's persisted keys. + Task LoadAsync(IPgpKeyStore store, CancellationToken cancellationToken = default); + + /// Persists the keyring contents to the key store. + Task SaveAsync(IPgpKeyStore store, CancellationToken cancellationToken = default); +} diff --git a/src/SquidStd.Crypto/Pgp/Interfaces/IPgpService.cs b/src/SquidStd.Crypto/Pgp/Interfaces/IPgpService.cs new file mode 100644 index 00000000..a3ffa871 --- /dev/null +++ b/src/SquidStd.Crypto/Pgp/Interfaces/IPgpService.cs @@ -0,0 +1,40 @@ +using SquidStd.Crypto.Pgp.Data; + +namespace SquidStd.Crypto.Pgp.Interfaces; + +/// +/// OpenPGP operations over the keyring: key generation, encrypt/decrypt, sign/verify, and the combined +/// encrypt+sign / decrypt+verify flows. Recipients and signers are resolved from the keyring by identity. +/// +public interface IPgpService +{ + /// Generates a new key pair for protected by . + PgpKey GenerateKey(string identity, string passphrase, PgpKeyOptions? options = null); + + /// Encrypts for the recipient, returning an armored message. + Task EncryptForAsync(string recipientIdentity, byte[] data, CancellationToken cancellationToken = default); + + /// Encrypts a stream for the recipient, writing an armored message to . + Task EncryptForAsync(string recipientIdentity, Stream input, Stream output, CancellationToken cancellationToken = default); + + /// Decrypts an armored message using the matching keyring secret key and passphrase. + Task DecryptAsync(string armored, string passphrase, CancellationToken cancellationToken = default); + + /// Decrypts a stream using the matching keyring secret key and passphrase. + Task DecryptAsync(Stream input, Stream output, string passphrase, CancellationToken cancellationToken = default); + + /// Encrypts for the recipient and signs it with the signer's secret key. + Task EncryptAndSignForAsync(string recipientIdentity, byte[] data, string signerIdentity, string signerPassphrase, CancellationToken cancellationToken = default); + + /// Encrypts and signs a stream for the recipient. + Task EncryptAndSignForAsync(string recipientIdentity, Stream input, Stream output, string signerIdentity, string signerPassphrase, CancellationToken cancellationToken = default); + + /// Decrypts an armored message and reports whether it was signed and whether the signature validated. + Task DecryptAndVerifyAsync(string armored, string passphrase, CancellationToken cancellationToken = default); + + /// Produces an armored signed message that embeds . + Task SignAsync(byte[] data, string signerIdentity, string passphrase, CancellationToken cancellationToken = default); + + /// Verifies an armored signed message against the keyring, recovering the embedded content. + Task VerifyAsync(string signedMessage, CancellationToken cancellationToken = default); +} From d0ff4b3f9626d4c8678ade6ed3d43c4278230e47 Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 14:22:31 +0200 Subject: [PATCH 04/77] feat(crypto): parse PGP key metadata from armored material --- .../Pgp/Internal/PgpKeyFactory.cs | 62 +++++++++++++++++++ src/SquidStd.Crypto/SquidStd.Crypto.csproj | 4 ++ .../Crypto/Pgp/PgpKeyFactoryTests.cs | 37 +++++++++++ .../Crypto/Pgp/Support/PgpKeysCollection.cs | 7 +++ .../Crypto/Pgp/Support/PgpTestKeys.cs | 44 +++++++++++++ 5 files changed, 154 insertions(+) create mode 100644 src/SquidStd.Crypto/Pgp/Internal/PgpKeyFactory.cs create mode 100644 tests/SquidStd.Tests/Crypto/Pgp/PgpKeyFactoryTests.cs create mode 100644 tests/SquidStd.Tests/Crypto/Pgp/Support/PgpKeysCollection.cs create mode 100644 tests/SquidStd.Tests/Crypto/Pgp/Support/PgpTestKeys.cs diff --git a/src/SquidStd.Crypto/Pgp/Internal/PgpKeyFactory.cs b/src/SquidStd.Crypto/Pgp/Internal/PgpKeyFactory.cs new file mode 100644 index 00000000..5838b962 --- /dev/null +++ b/src/SquidStd.Crypto/Pgp/Internal/PgpKeyFactory.cs @@ -0,0 +1,62 @@ +using System.Globalization; +using Org.BouncyCastle.Bcpg; +using Org.BouncyCastle.Bcpg.OpenPgp; +using PgpCore; +using SquidStd.Crypto.Pgp.Data; +using SquidStd.Crypto.Pgp.Types; + +namespace SquidStd.Crypto.Pgp.Internal; + +/// +/// Builds a from armored key material by reading metadata off the public master key. +/// Shared by key generation, keyring import, and key-store loading. +/// +internal static class PgpKeyFactory +{ + public static PgpKey FromArmored(string publicArmored, string? privateArmored) + { + var master = new EncryptionKeys(publicArmored).MasterKey; + + var identity = FirstUserId(master); + var keyId = master.KeyId.ToString("X16", CultureInfo.InvariantCulture); + var fingerprint = Convert.ToHexString(master.GetFingerprint()); + var createdUtc = new DateTimeOffset(DateTime.SpecifyKind(master.CreationTime, DateTimeKind.Utc)); + + var validSeconds = master.GetValidSeconds(); + DateTimeOffset? expiresUtc = validSeconds > 0 ? createdUtc.AddSeconds(validSeconds) : null; + + return new PgpKey( + identity, + keyId, + fingerprint, + publicArmored, + privateArmored, + createdUtc, + expiresUtc, + MapAlgorithm(master.Algorithm) + ); + } + + private static string FirstUserId(PgpPublicKey master) + { + foreach (var userId in master.GetUserIds()) + { + if (userId is string text && !string.IsNullOrWhiteSpace(text)) + { + return text; + } + } + + return string.Empty; + } + + private static PgpKeyAlgorithm MapAlgorithm(PublicKeyAlgorithmTag tag) + { + return tag switch + { + PublicKeyAlgorithmTag.RsaGeneral or PublicKeyAlgorithmTag.RsaEncrypt or PublicKeyAlgorithmTag.RsaSign => + PgpKeyAlgorithm.Rsa, + _ => PgpKeyAlgorithm.Rsa + }; + } +} diff --git a/src/SquidStd.Crypto/SquidStd.Crypto.csproj b/src/SquidStd.Crypto/SquidStd.Crypto.csproj index f7054b8e..f81170ff 100644 --- a/src/SquidStd.Crypto/SquidStd.Crypto.csproj +++ b/src/SquidStd.Crypto/SquidStd.Crypto.csproj @@ -11,6 +11,10 @@ + + + + diff --git a/tests/SquidStd.Tests/Crypto/Pgp/PgpKeyFactoryTests.cs b/tests/SquidStd.Tests/Crypto/Pgp/PgpKeyFactoryTests.cs new file mode 100644 index 00000000..765f2ea8 --- /dev/null +++ b/tests/SquidStd.Tests/Crypto/Pgp/PgpKeyFactoryTests.cs @@ -0,0 +1,37 @@ +using SquidStd.Crypto.Pgp.Internal; +using SquidStd.Crypto.Pgp.Types; +using SquidStd.Tests.Crypto.Pgp.Support; + +namespace SquidStd.Tests.Crypto.Pgp; + +[Collection("PgpKeys")] +public class PgpKeyFactoryTests +{ + private readonly PgpTestKeys _keys; + + public PgpKeyFactoryTests(PgpTestKeys keys) + { + _keys = keys; + } + + [Fact] + public void FromArmored_PublicOnly_ParsesMetadataWithoutSecret() + { + var key = PgpKeyFactory.FromArmored(_keys.AlicePublic, null); + + Assert.Equal(PgpTestKeys.AliceIdentity, key.Identity); + Assert.False(string.IsNullOrWhiteSpace(key.KeyId)); + Assert.False(string.IsNullOrWhiteSpace(key.Fingerprint)); + Assert.Equal(PgpKeyAlgorithm.Rsa, key.Algorithm); + Assert.False(key.HasSecret); + } + + [Fact] + public void FromArmored_WithSecret_SetsHasSecret() + { + var key = PgpKeyFactory.FromArmored(_keys.AlicePublic, _keys.AlicePrivate); + + Assert.True(key.HasSecret); + Assert.Equal(_keys.AlicePrivate, key.PrivateArmored); + } +} diff --git a/tests/SquidStd.Tests/Crypto/Pgp/Support/PgpKeysCollection.cs b/tests/SquidStd.Tests/Crypto/Pgp/Support/PgpKeysCollection.cs new file mode 100644 index 00000000..93185bd8 --- /dev/null +++ b/tests/SquidStd.Tests/Crypto/Pgp/Support/PgpKeysCollection.cs @@ -0,0 +1,7 @@ +namespace SquidStd.Tests.Crypto.Pgp.Support; + +/// Shares one instance across the PGP test classes. +[CollectionDefinition("PgpKeys")] +public sealed class PgpKeysCollection : ICollectionFixture +{ +} diff --git a/tests/SquidStd.Tests/Crypto/Pgp/Support/PgpTestKeys.cs b/tests/SquidStd.Tests/Crypto/Pgp/Support/PgpTestKeys.cs new file mode 100644 index 00000000..996b6a0d --- /dev/null +++ b/tests/SquidStd.Tests/Crypto/Pgp/Support/PgpTestKeys.cs @@ -0,0 +1,44 @@ +using PgpCore; + +namespace SquidStd.Tests.Crypto.Pgp.Support; + +/// +/// Generates two throwaway armored RSA-2048 key pairs once for the whole assembly. Key generation is +/// expensive, so tests share this fixture via . +/// +public sealed class PgpTestKeys +{ + public const string AlicePassphrase = "alice-pw"; + public const string BobPassphrase = "bob-pw"; + public const string AliceIdentity = "alice@squidstd.test"; + public const string BobIdentity = "bob@squidstd.test"; + + public PgpTestKeys() + { + (AlicePublic, AlicePrivate) = Generate(AliceIdentity, AlicePassphrase); + (BobPublic, BobPrivate) = Generate(BobIdentity, BobPassphrase); + } + + public string AlicePublic { get; } + + public string AlicePrivate { get; } + + public string BobPublic { get; } + + public string BobPrivate { get; } + + private static (string Public, string Private) Generate(string identity, string passphrase) + { + var pgp = new PGP(); + using var pub = new MemoryStream(); + using var priv = new MemoryStream(); + pgp.GenerateKey(pub, priv, identity, passphrase, 2048); + + return (ReadAll(pub), ReadAll(priv)); + } + + private static string ReadAll(MemoryStream stream) + { + return System.Text.Encoding.UTF8.GetString(stream.ToArray()); + } +} From 2799b415fc50ed0e34ee96ab4a7e66ddd9f5d5d0 Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 14:24:12 +0200 Subject: [PATCH 05/77] feat(crypto): add in-memory PGP keyring indexed by identity, key id, fingerprint --- .../Pgp/Services/PgpKeyring.cs | 114 ++++++++++++++++++ .../Crypto/Pgp/PgpKeyringTests.cs | 49 ++++++++ 2 files changed, 163 insertions(+) create mode 100644 src/SquidStd.Crypto/Pgp/Services/PgpKeyring.cs create mode 100644 tests/SquidStd.Tests/Crypto/Pgp/PgpKeyringTests.cs diff --git a/src/SquidStd.Crypto/Pgp/Services/PgpKeyring.cs b/src/SquidStd.Crypto/Pgp/Services/PgpKeyring.cs new file mode 100644 index 00000000..8d258635 --- /dev/null +++ b/src/SquidStd.Crypto/Pgp/Services/PgpKeyring.cs @@ -0,0 +1,114 @@ +using System.Collections.Concurrent; +using System.Text; +using Org.BouncyCastle.Bcpg; +using Org.BouncyCastle.Bcpg.OpenPgp; +using SquidStd.Crypto.Pgp.Data; +using SquidStd.Crypto.Pgp.Interfaces; +using SquidStd.Crypto.Pgp.Internal; + +namespace SquidStd.Crypto.Pgp.Services; + +/// +/// Thread-safe in-memory keyring indexed by identity, key id, and fingerprint. +/// +public sealed class PgpKeyring : IPgpKeyring +{ + private const string SecretHeader = "BEGIN PGP PRIVATE KEY BLOCK"; + private readonly ConcurrentDictionary _byKeyId = new(StringComparer.OrdinalIgnoreCase); + + /// + public IReadOnlyCollection Keys => _byKeyId.Values.ToArray(); + + /// + public PgpKey Import(string armored) + { + ArgumentException.ThrowIfNullOrWhiteSpace(armored); + + var key = armored.Contains(SecretHeader, StringComparison.Ordinal) + ? PgpKeyFactory.FromArmored(ExportPublic(armored), armored) + : PgpKeyFactory.FromArmored(armored, null); + + _byKeyId[key.KeyId] = key; + + return key; + } + + /// + public bool Remove(string identityOrKeyIdOrFingerprint) + { + ArgumentException.ThrowIfNullOrWhiteSpace(identityOrKeyIdOrFingerprint); + + var match = Find(identityOrKeyIdOrFingerprint); + + return match is not null && _byKeyId.TryRemove(match.KeyId, out _); + } + + /// + public PgpKey? Find(string identityOrKeyIdOrFingerprint) + { + ArgumentException.ThrowIfNullOrWhiteSpace(identityOrKeyIdOrFingerprint); + + if (_byKeyId.TryGetValue(identityOrKeyIdOrFingerprint, out var byId)) + { + return byId; + } + + foreach (var key in _byKeyId.Values) + { + if (string.Equals(key.Identity, identityOrKeyIdOrFingerprint, StringComparison.OrdinalIgnoreCase) || + string.Equals(key.Fingerprint, identityOrKeyIdOrFingerprint, StringComparison.OrdinalIgnoreCase)) + { + return key; + } + } + + return null; + } + + /// + public bool Contains(string identityOrKeyIdOrFingerprint) + { + return Find(identityOrKeyIdOrFingerprint) is not null; + } + + /// + public async Task LoadAsync(IPgpKeyStore store, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(store); + + var loaded = await store.LoadAsync(cancellationToken).ConfigureAwait(false); + _byKeyId.Clear(); + + foreach (var key in loaded) + { + _byKeyId[key.KeyId] = key; + } + } + + /// + public Task SaveAsync(IPgpKeyStore store, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(store); + + return store.SaveAsync(Keys, cancellationToken); + } + + private static string ExportPublic(string secretArmored) + { + using var input = PgpUtilities.GetDecoderStream( + new MemoryStream(Encoding.UTF8.GetBytes(secretArmored)) + ); + var ring = new PgpSecretKeyRingBundle(input).GetKeyRings().Cast().First(); + + using var output = new MemoryStream(); + using (var armor = new ArmoredOutputStream(output)) + { + foreach (PgpSecretKey secretKey in ring.GetSecretKeys()) + { + secretKey.PublicKey.Encode(armor); + } + } + + return Encoding.UTF8.GetString(output.ToArray()); + } +} diff --git a/tests/SquidStd.Tests/Crypto/Pgp/PgpKeyringTests.cs b/tests/SquidStd.Tests/Crypto/Pgp/PgpKeyringTests.cs new file mode 100644 index 00000000..b85859ca --- /dev/null +++ b/tests/SquidStd.Tests/Crypto/Pgp/PgpKeyringTests.cs @@ -0,0 +1,49 @@ +using SquidStd.Crypto.Pgp.Services; +using SquidStd.Tests.Crypto.Pgp.Support; + +namespace SquidStd.Tests.Crypto.Pgp; + +[Collection("PgpKeys")] +public class PgpKeyringTests +{ + private readonly PgpTestKeys _keys; + + public PgpKeyringTests(PgpTestKeys keys) + { + _keys = keys; + } + + [Fact] + public void Import_PublicBlock_FindableByIdentityKeyIdAndFingerprint() + { + var keyring = new PgpKeyring(); + var key = keyring.Import(_keys.AlicePublic); + + Assert.Same(key, keyring.Find(PgpTestKeys.AliceIdentity)); + Assert.Same(key, keyring.Find(key.KeyId)); + Assert.Same(key, keyring.Find(key.Fingerprint)); + Assert.True(keyring.Contains(PgpTestKeys.AliceIdentity)); + Assert.False(key.HasSecret); + } + + [Fact] + public void Import_SecretBlock_SetsHasSecret() + { + var keyring = new PgpKeyring(); + + var key = keyring.Import(_keys.AlicePrivate); + + Assert.True(key.HasSecret); + } + + [Fact] + public void Remove_ByIdentity_RemovesKey() + { + var keyring = new PgpKeyring(); + keyring.Import(_keys.AlicePublic); + + Assert.True(keyring.Remove(PgpTestKeys.AliceIdentity)); + Assert.Null(keyring.Find(PgpTestKeys.AliceIdentity)); + Assert.Empty(keyring.Keys); + } +} From 2b4c75459c3d596ea3175f29a94efbc747e48580 Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 14:26:11 +0200 Subject: [PATCH 06/77] feat(crypto): PGP key generation and encrypt/decrypt over the keyring --- .../Pgp/Services/PgpService.cs | 270 ++++++++++++++++++ .../Crypto/Pgp/PgpServiceEncryptionTests.cs | 61 ++++ .../Pgp/PgpServiceKeyGenerationTests.cs | 20 ++ 3 files changed, 351 insertions(+) create mode 100644 src/SquidStd.Crypto/Pgp/Services/PgpService.cs create mode 100644 tests/SquidStd.Tests/Crypto/Pgp/PgpServiceEncryptionTests.cs create mode 100644 tests/SquidStd.Tests/Crypto/Pgp/PgpServiceKeyGenerationTests.cs diff --git a/src/SquidStd.Crypto/Pgp/Services/PgpService.cs b/src/SquidStd.Crypto/Pgp/Services/PgpService.cs new file mode 100644 index 00000000..abe90dd7 --- /dev/null +++ b/src/SquidStd.Crypto/Pgp/Services/PgpService.cs @@ -0,0 +1,270 @@ +using System.Globalization; +using System.Text; +using PgpCore; +using SquidStd.Crypto.Pgp.Data; +using SquidStd.Crypto.Pgp.Interfaces; +using SquidStd.Crypto.Pgp.Internal; +using BcPgpException = Org.BouncyCastle.Bcpg.OpenPgp.PgpException; + +namespace SquidStd.Crypto.Pgp.Services; + +/// +/// OpenPGP operations over a keyring, implemented with PgpCore. Every byte/armored-string operation +/// round-trips through so binary payloads survive intact. +/// +public sealed class PgpService : IPgpService +{ + private readonly IPgpKeyring _keyring; + + public PgpService(IPgpKeyring keyring) + { + _keyring = keyring; + } + + /// + public PgpKey GenerateKey(string identity, string passphrase, PgpKeyOptions? options = null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(identity); + ArgumentException.ThrowIfNullOrWhiteSpace(passphrase); + + var settings = options ?? new PgpKeyOptions(); + var expirationSeconds = settings.ExpiresAfter is { } span ? (long)span.TotalSeconds : 0L; + + var pgp = new PGP(); + using var publicStream = new MemoryStream(); + using var privateStream = new MemoryStream(); + pgp.GenerateKey( + publicStream, + privateStream, + identity, + passphrase, + settings.KeySizeBits, + armor: true, + keyExpirationInSeconds: expirationSeconds + ); + + var publicArmored = ReadAll(publicStream); + var privateArmored = ReadAll(privateStream); + + return PgpKeyFactory.FromArmored(publicArmored, privateArmored); + } + + /// + public async Task EncryptForAsync(string recipientIdentity, byte[] data, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(data); + + var recipient = RequireKey(recipientIdentity); + var pgp = new PGP(new EncryptionKeys(recipient.PublicArmored)); + + using var input = new MemoryStream(data); + using var output = new MemoryStream(); + await pgp.EncryptAsync(input, output).ConfigureAwait(false); + + return Encoding.UTF8.GetString(output.ToArray()); + } + + /// + public async Task EncryptForAsync(string recipientIdentity, Stream input, Stream output, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(input); + ArgumentNullException.ThrowIfNull(output); + + var recipient = RequireKey(recipientIdentity); + var pgp = new PGP(new EncryptionKeys(recipient.PublicArmored)); + await pgp.EncryptAsync(input, output).ConfigureAwait(false); + } + + /// + public async Task DecryptAsync(string armored, string passphrase, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(armored); + + var secret = RequireSecretFor(armored); + var pgp = new PGP(new EncryptionKeys(secret.PrivateArmored!, passphrase)); + + using var input = new MemoryStream(Encoding.UTF8.GetBytes(armored)); + using var output = new MemoryStream(); + await pgp.DecryptAsync(input, output).ConfigureAwait(false); + + return output.ToArray(); + } + + /// + public async Task DecryptAsync(Stream input, Stream output, string passphrase, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(input); + ArgumentNullException.ThrowIfNull(output); + + // Buffer to a seekable copy so the recipient key id can be read before decrypting. + using var buffer = new MemoryStream(); + await input.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false); + var armored = Encoding.UTF8.GetString(buffer.ToArray()); + + var secret = RequireSecretFor(armored); + var pgp = new PGP(new EncryptionKeys(secret.PrivateArmored!, passphrase)); + + buffer.Position = 0; + await pgp.DecryptAsync(buffer, output).ConfigureAwait(false); + } + + /// + public async Task EncryptAndSignForAsync(string recipientIdentity, byte[] data, string signerIdentity, string signerPassphrase, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(data); + + var pgp = BuildEncryptAndSign(recipientIdentity, signerIdentity, signerPassphrase); + + using var input = new MemoryStream(data); + using var output = new MemoryStream(); + await pgp.EncryptAndSignAsync(input, output).ConfigureAwait(false); + + return Encoding.UTF8.GetString(output.ToArray()); + } + + /// + public async Task EncryptAndSignForAsync(string recipientIdentity, Stream input, Stream output, string signerIdentity, string signerPassphrase, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(input); + ArgumentNullException.ThrowIfNull(output); + + var pgp = BuildEncryptAndSign(recipientIdentity, signerIdentity, signerPassphrase); + await pgp.EncryptAndSignAsync(input, output).ConfigureAwait(false); + } + + /// + public async Task DecryptAndVerifyAsync(string armored, string passphrase, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(armored); + + var secret = RequireSecretFor(armored); + var publicKeys = _keyring.Keys.Select(key => key.PublicArmored).ToArray(); + var pgp = new PGP(new EncryptionKeys(publicKeys, secret.PrivateArmored!, passphrase)); + + var isSigned = pgp.Inspect(armored).IsSigned; + + if (!isSigned) + { + var plain = await DecryptWith(pgp, armored).ConfigureAwait(false); + + return new PgpDecryptionResult(plain, false, false); + } + + try + { + using var input = new MemoryStream(Encoding.UTF8.GetBytes(armored)); + using var output = new MemoryStream(); + await pgp.DecryptAndVerifyAsync(input, output).ConfigureAwait(false); + + return new PgpDecryptionResult(output.ToArray(), true, true); + } + catch (Exception ex) when (ex is BcPgpException or InvalidOperationException or ArgumentException or IOException) + { + var plain = await DecryptWith(pgp, armored).ConfigureAwait(false); + + return new PgpDecryptionResult(plain, true, false); + } + } + + /// + public async Task SignAsync(byte[] data, string signerIdentity, string passphrase, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(data); + + var signer = RequireKey(signerIdentity); + var pgp = new PGP(new EncryptionKeys(signer.PrivateArmored!, passphrase)); + + using var input = new MemoryStream(data); + using var output = new MemoryStream(); + await pgp.SignAsync(input, output).ConfigureAwait(false); + + return Encoding.UTF8.GetString(output.ToArray()); + } + + /// + public async Task VerifyAsync(string signedMessage, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(signedMessage); + + var recovered = Array.Empty(); + + foreach (var key in _keyring.Keys) + { + using var input = new MemoryStream(Encoding.UTF8.GetBytes(signedMessage)); + using var output = new MemoryStream(); + var pgp = new PGP(new EncryptionKeys(key.PublicArmored)); + + bool ok; + try + { + ok = await pgp.VerifyAsync(input, output).ConfigureAwait(false); + } + catch (Exception ex) when (ex is BcPgpException or InvalidOperationException or ArgumentException or IOException) + { + ok = false; + } + + if (output.Length > 0) + { + recovered = output.ToArray(); + } + + if (ok) + { + return new PgpVerificationResult(true, recovered); + } + } + + return new PgpVerificationResult(false, recovered); + } + + private PGP BuildEncryptAndSign(string recipientIdentity, string signerIdentity, string signerPassphrase) + { + var recipient = RequireKey(recipientIdentity); + var signer = RequireKey(signerIdentity); + + return new PGP(new EncryptionKeys(recipient.PublicArmored, signer.PrivateArmored!, signerPassphrase)); + } + + private static async Task DecryptWith(PGP pgp, string armored) + { + using var input = new MemoryStream(Encoding.UTF8.GetBytes(armored)); + using var output = new MemoryStream(); + await pgp.DecryptAsync(input, output).ConfigureAwait(false); + + return output.ToArray(); + } + + private PgpKey RequireKey(string identity) + { + ArgumentException.ThrowIfNullOrWhiteSpace(identity); + + return _keyring.Find(identity) + ?? throw new KeyNotFoundException($"No key for identity '{identity}' in the keyring."); + } + + private PgpKey RequireSecretFor(string armored) + { + var recipientIds = new PGP().GetRecipients(armored).ToHashSet(); + + foreach (var key in _keyring.Keys) + { + if (key.HasSecret && recipientIds.Contains(ParseKeyId(key.KeyId))) + { + return key; + } + } + + throw new KeyNotFoundException("No secret key in the keyring matches the message recipients."); + } + + private static long ParseKeyId(string keyId) + { + return long.Parse(keyId, NumberStyles.HexNumber, CultureInfo.InvariantCulture); + } + + private static string ReadAll(MemoryStream stream) + { + return Encoding.UTF8.GetString(stream.ToArray()); + } +} diff --git a/tests/SquidStd.Tests/Crypto/Pgp/PgpServiceEncryptionTests.cs b/tests/SquidStd.Tests/Crypto/Pgp/PgpServiceEncryptionTests.cs new file mode 100644 index 00000000..069d9a1d --- /dev/null +++ b/tests/SquidStd.Tests/Crypto/Pgp/PgpServiceEncryptionTests.cs @@ -0,0 +1,61 @@ +using System.Text; +using SquidStd.Crypto.Pgp.Services; +using SquidStd.Tests.Crypto.Pgp.Support; + +namespace SquidStd.Tests.Crypto.Pgp; + +[Collection("PgpKeys")] +public class PgpServiceEncryptionTests +{ + private readonly PgpTestKeys _keys; + + public PgpServiceEncryptionTests(PgpTestKeys keys) + { + _keys = keys; + } + + [Fact] + public async Task EncryptFor_ThenDecrypt_RoundTripsBytes() + { + var keyring = new PgpKeyring(); + keyring.Import(_keys.AlicePublic); // recipient public only for encryption + keyring.Import(_keys.AlicePrivate); // secret for decryption (replaces same key id, now HasSecret) + var service = new PgpService(keyring); + var payload = Encoding.UTF8.GetBytes("squid secret message"); + + var armored = await service.EncryptForAsync(PgpTestKeys.AliceIdentity, payload); + var plain = await service.DecryptAsync(armored, PgpTestKeys.AlicePassphrase); + + Assert.StartsWith("-----BEGIN PGP MESSAGE-----", armored); + Assert.Equal(payload, plain); + } + + [Fact] + public async Task EncryptFor_UnknownRecipient_ThrowsKeyNotFound() + { + var service = new PgpService(new PgpKeyring()); + + await Assert.ThrowsAsync( + () => service.EncryptForAsync("nobody@squidstd.test", Encoding.UTF8.GetBytes("x")) + ); + } + + [Fact] + public async Task EncryptFor_ThenDecrypt_StreamOverloadsRoundTripLargePayload() + { + var keyring = new PgpKeyring(); + keyring.Import(_keys.AlicePrivate); + var service = new PgpService(keyring); + var payload = Encoding.UTF8.GetBytes(new string('z', 8192)); + + using var plaintext = new MemoryStream(payload); + using var ciphertext = new MemoryStream(); + await service.EncryptForAsync(PgpTestKeys.AliceIdentity, plaintext, ciphertext); + + ciphertext.Position = 0; + using var recovered = new MemoryStream(); + await service.DecryptAsync(ciphertext, recovered, PgpTestKeys.AlicePassphrase); + + Assert.Equal(payload, recovered.ToArray()); + } +} diff --git a/tests/SquidStd.Tests/Crypto/Pgp/PgpServiceKeyGenerationTests.cs b/tests/SquidStd.Tests/Crypto/Pgp/PgpServiceKeyGenerationTests.cs new file mode 100644 index 00000000..5b7d63cf --- /dev/null +++ b/tests/SquidStd.Tests/Crypto/Pgp/PgpServiceKeyGenerationTests.cs @@ -0,0 +1,20 @@ +using SquidStd.Crypto.Pgp.Services; + +namespace SquidStd.Tests.Crypto.Pgp; + +public class PgpServiceKeyGenerationTests +{ + [Fact] + public void GenerateKey_ReturnsKeyWithMetadataAndSecret() + { + var keyring = new PgpKeyring(); + var service = new PgpService(keyring); + + var key = service.GenerateKey("carol@squidstd.test", "carol-pw"); + + Assert.Equal("carol@squidstd.test", key.Identity); + Assert.False(string.IsNullOrWhiteSpace(key.KeyId)); + Assert.False(string.IsNullOrWhiteSpace(key.Fingerprint)); + Assert.True(key.HasSecret); + } +} From 164e76190ca73d8bac9f37b7d5b4692dab742552 Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 14:26:11 +0200 Subject: [PATCH 07/77] feat(crypto): PGP sign/verify and encrypt-and-sign/decrypt-and-verify --- .../Crypto/Pgp/PgpServiceSigningTests.cs | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 tests/SquidStd.Tests/Crypto/Pgp/PgpServiceSigningTests.cs diff --git a/tests/SquidStd.Tests/Crypto/Pgp/PgpServiceSigningTests.cs b/tests/SquidStd.Tests/Crypto/Pgp/PgpServiceSigningTests.cs new file mode 100644 index 00000000..c6a9f9ae --- /dev/null +++ b/tests/SquidStd.Tests/Crypto/Pgp/PgpServiceSigningTests.cs @@ -0,0 +1,105 @@ +using System.Text; +using SquidStd.Crypto.Pgp.Services; +using SquidStd.Tests.Crypto.Pgp.Support; + +namespace SquidStd.Tests.Crypto.Pgp; + +[Collection("PgpKeys")] +public class PgpServiceSigningTests +{ + private readonly PgpTestKeys _keys; + + public PgpServiceSigningTests(PgpTestKeys keys) + { + _keys = keys; + } + + [Fact] + public async Task Sign_ThenVerify_ValidAndRecoversData() + { + var keyring = new PgpKeyring(); + keyring.Import(_keys.AlicePrivate); + var service = new PgpService(keyring); + var payload = Encoding.UTF8.GetBytes("signed by alice"); + + var signed = await service.SignAsync(payload, PgpTestKeys.AliceIdentity, PgpTestKeys.AlicePassphrase); + var result = await service.VerifyAsync(signed); + + Assert.True(result.IsValid); + Assert.Equal(payload, result.Data); + } + + [Fact] + public async Task Verify_TamperedMessage_IsInvalid() + { + var keyring = new PgpKeyring(); + keyring.Import(_keys.AlicePrivate); + var service = new PgpService(keyring); + + var signed = await service.SignAsync(Encoding.UTF8.GetBytes("original"), PgpTestKeys.AliceIdentity, PgpTestKeys.AlicePassphrase); + var tampered = MutateBody(signed); + + var result = await service.VerifyAsync(tampered); + + Assert.False(result.IsValid); + } + + [Fact] + public async Task EncryptAndSign_ThenDecryptAndVerify_RoundTripsAndValidates() + { + var keyring = new PgpKeyring(); + keyring.Import(_keys.AlicePrivate); // recipient (has secret to decrypt) + keyring.Import(_keys.BobPrivate); // signer + var service = new PgpService(keyring); + var payload = Encoding.UTF8.GetBytes("confidential and signed"); + + var armored = await service.EncryptAndSignForAsync( + PgpTestKeys.AliceIdentity, payload, PgpTestKeys.BobIdentity, PgpTestKeys.BobPassphrase + ); + var result = await service.DecryptAndVerifyAsync(armored, PgpTestKeys.AlicePassphrase); + + Assert.Equal(payload, result.Data); + Assert.True(result.IsSigned); + Assert.True(result.IsValid); + } + + [Fact] + public async Task DecryptAndVerify_SignerNotInKeyring_RecoversDataButInvalid() + { + // Sign with Bob, then decrypt on a keyring that lacks Bob's public key. + var signingKeyring = new PgpKeyring(); + signingKeyring.Import(_keys.AlicePrivate); + signingKeyring.Import(_keys.BobPrivate); + var signingService = new PgpService(signingKeyring); + var payload = Encoding.UTF8.GetBytes("signed by an unknown party"); + var armored = await signingService.EncryptAndSignForAsync( + PgpTestKeys.AliceIdentity, payload, PgpTestKeys.BobIdentity, PgpTestKeys.BobPassphrase + ); + + var recipientOnly = new PgpKeyring(); + recipientOnly.Import(_keys.AlicePrivate); // no Bob public key + var service = new PgpService(recipientOnly); + + var result = await service.DecryptAndVerifyAsync(armored, PgpTestKeys.AlicePassphrase); + + Assert.Equal(payload, result.Data); + Assert.True(result.IsSigned); + Assert.False(result.IsValid); + } + + private static string MutateBody(string signed) + { + // Flip a character in the middle of the armored body to corrupt the signed payload. + var lines = signed.Split('\n'); + var mid = lines.Length / 2; + + if (lines[mid].Length > 4) + { + var chars = lines[mid].ToCharArray(); + chars[2] = chars[2] == 'A' ? 'B' : 'A'; + lines[mid] = new string(chars); + } + + return string.Join('\n', lines); + } +} From 33a5e94851e16d5e8d2901b96384275035a5dbad Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 14:26:54 +0200 Subject: [PATCH 08/77] feat(crypto): add file-backed PGP key store (armored .asc files) --- .../Pgp/Services/FilePgpKeyStore.cs | 82 +++++++++++++++++++ .../Crypto/Pgp/FilePgpKeyStoreTests.cs | 45 ++++++++++ 2 files changed, 127 insertions(+) create mode 100644 src/SquidStd.Crypto/Pgp/Services/FilePgpKeyStore.cs create mode 100644 tests/SquidStd.Tests/Crypto/Pgp/FilePgpKeyStoreTests.cs diff --git a/src/SquidStd.Crypto/Pgp/Services/FilePgpKeyStore.cs b/src/SquidStd.Crypto/Pgp/Services/FilePgpKeyStore.cs new file mode 100644 index 00000000..6a1c3041 --- /dev/null +++ b/src/SquidStd.Crypto/Pgp/Services/FilePgpKeyStore.cs @@ -0,0 +1,82 @@ +using System.Text; +using SquidStd.Crypto.Pgp.Data; +using SquidStd.Crypto.Pgp.Interfaces; +using SquidStd.Crypto.Pgp.Internal; + +namespace SquidStd.Crypto.Pgp.Services; + +/// +/// Key store backed by a directory of armored .asc files (one public, optionally one secret, per +/// key). gpg-interoperable. +/// +public sealed class FilePgpKeyStore : IPgpKeyStore +{ + private const string PublicSuffix = ".pub.asc"; + private const string SecretSuffix = ".sec.asc"; + private readonly string _directory; + + public FilePgpKeyStore(string directory) + { + ArgumentException.ThrowIfNullOrWhiteSpace(directory); + _directory = directory; + } + + /// + public async Task SaveAsync(IReadOnlyCollection keys, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(keys); + Directory.CreateDirectory(_directory); + + foreach (var key in keys) + { + var stem = Stem(key); + await File.WriteAllTextAsync( + Path.Combine(_directory, stem + PublicSuffix), key.PublicArmored, cancellationToken + ).ConfigureAwait(false); + + if (key.PrivateArmored is not null) + { + await File.WriteAllTextAsync( + Path.Combine(_directory, stem + SecretSuffix), key.PrivateArmored, cancellationToken + ).ConfigureAwait(false); + } + } + } + + /// + public async Task> LoadAsync(CancellationToken cancellationToken = default) + { + if (!Directory.Exists(_directory)) + { + return []; + } + + var result = new List(); + + foreach (var publicPath in Directory.EnumerateFiles(_directory, "*" + PublicSuffix)) + { + var stem = Path.GetFileName(publicPath)[..^PublicSuffix.Length]; + var publicArmored = await File.ReadAllTextAsync(publicPath, cancellationToken).ConfigureAwait(false); + + var secretPath = Path.Combine(_directory, stem + SecretSuffix); + string? secretArmored = File.Exists(secretPath) + ? await File.ReadAllTextAsync(secretPath, cancellationToken).ConfigureAwait(false) + : null; + + result.Add(PgpKeyFactory.FromArmored(publicArmored, secretArmored)); + } + + return result; + } + + private static string Stem(PgpKey key) + { + var safeIdentity = new StringBuilder(key.Identity.Length); + foreach (var ch in key.Identity) + { + safeIdentity.Append(char.IsLetterOrDigit(ch) ? ch : '_'); + } + + return safeIdentity + "." + key.KeyId; + } +} diff --git a/tests/SquidStd.Tests/Crypto/Pgp/FilePgpKeyStoreTests.cs b/tests/SquidStd.Tests/Crypto/Pgp/FilePgpKeyStoreTests.cs new file mode 100644 index 00000000..f905de9a --- /dev/null +++ b/tests/SquidStd.Tests/Crypto/Pgp/FilePgpKeyStoreTests.cs @@ -0,0 +1,45 @@ +using SquidStd.Crypto.Pgp.Services; +using SquidStd.Tests.Crypto.Pgp.Support; + +namespace SquidStd.Tests.Crypto.Pgp; + +[Collection("PgpKeys")] +public class FilePgpKeyStoreTests +{ + private readonly PgpTestKeys _keys; + + public FilePgpKeyStoreTests(PgpTestKeys keys) + { + _keys = keys; + } + + [Fact] + public async Task SaveThenLoad_RestoresPublicAndSecretKeys() + { + var dir = Path.Combine(Path.GetTempPath(), "squidstd-pgp-" + Guid.NewGuid().ToString("N")); + + try + { + var source = new PgpKeyring(); + source.Import(_keys.AlicePrivate); // has secret + source.Import(_keys.BobPublic); // public only + + var store = new FilePgpKeyStore(dir); + await source.SaveAsync(store); + + var restored = new PgpKeyring(); + await restored.LoadAsync(store); + + Assert.Equal(2, restored.Keys.Count); + Assert.True(restored.Find(PgpTestKeys.AliceIdentity)!.HasSecret); + Assert.False(restored.Find(PgpTestKeys.BobIdentity)!.HasSecret); + } + finally + { + if (Directory.Exists(dir)) + { + Directory.Delete(dir, recursive: true); + } + } + } +} From a560a698ce89a633eeca8181ea523842ac7892d4 Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 14:27:48 +0200 Subject: [PATCH 09/77] feat(crypto): add AES-GCM encrypted PGP key store via ISecretProtector --- .../Pgp/Internal/PgpKeyStoreCodec.cs | 46 ++++++++++++++++ .../Pgp/Services/AesGcmPgpKeyStore.cs | 54 +++++++++++++++++++ .../Crypto/Pgp/AesGcmPgpKeyStoreTests.cs | 48 +++++++++++++++++ 3 files changed, 148 insertions(+) create mode 100644 src/SquidStd.Crypto/Pgp/Internal/PgpKeyStoreCodec.cs create mode 100644 src/SquidStd.Crypto/Pgp/Services/AesGcmPgpKeyStore.cs create mode 100644 tests/SquidStd.Tests/Crypto/Pgp/AesGcmPgpKeyStoreTests.cs diff --git a/src/SquidStd.Crypto/Pgp/Internal/PgpKeyStoreCodec.cs b/src/SquidStd.Crypto/Pgp/Internal/PgpKeyStoreCodec.cs new file mode 100644 index 00000000..24d3fcad --- /dev/null +++ b/src/SquidStd.Crypto/Pgp/Internal/PgpKeyStoreCodec.cs @@ -0,0 +1,46 @@ +using System.Text; +using SquidStd.Crypto.Pgp.Data; + +namespace SquidStd.Crypto.Pgp.Internal; + +/// +/// Encodes a set of keys into a single byte blob (one base64 record per line) and back. Only armored +/// material is stored; metadata is re-derived on load. +/// +internal static class PgpKeyStoreCodec +{ + public static byte[] Encode(IReadOnlyCollection keys) + { + var builder = new StringBuilder(); + + foreach (var key in keys) + { + var pub = Convert.ToBase64String(Encoding.UTF8.GetBytes(key.PublicArmored)); + var sec = key.PrivateArmored is null + ? string.Empty + : Convert.ToBase64String(Encoding.UTF8.GetBytes(key.PrivateArmored)); + builder.Append(pub).Append('\t').Append(sec).Append('\n'); + } + + return Encoding.UTF8.GetBytes(builder.ToString()); + } + + public static IReadOnlyList Decode(byte[] blob) + { + var text = Encoding.UTF8.GetString(blob); + var result = new List(); + + foreach (var line in text.Split('\n', StringSplitOptions.RemoveEmptyEntries)) + { + var parts = line.Split('\t'); + var publicArmored = Encoding.UTF8.GetString(Convert.FromBase64String(parts[0])); + string? privateArmored = parts.Length > 1 && parts[1].Length > 0 + ? Encoding.UTF8.GetString(Convert.FromBase64String(parts[1])) + : null; + + result.Add(PgpKeyFactory.FromArmored(publicArmored, privateArmored)); + } + + return result; + } +} diff --git a/src/SquidStd.Crypto/Pgp/Services/AesGcmPgpKeyStore.cs b/src/SquidStd.Crypto/Pgp/Services/AesGcmPgpKeyStore.cs new file mode 100644 index 00000000..f595a789 --- /dev/null +++ b/src/SquidStd.Crypto/Pgp/Services/AesGcmPgpKeyStore.cs @@ -0,0 +1,54 @@ +using SquidStd.Core.Interfaces.Secrets; +using SquidStd.Crypto.Pgp.Data; +using SquidStd.Crypto.Pgp.Interfaces; +using SquidStd.Crypto.Pgp.Internal; + +namespace SquidStd.Crypto.Pgp.Services; + +/// +/// Key store that serializes the keyring to a single file encrypted at rest with the application key via +/// . +/// +public sealed class AesGcmPgpKeyStore : IPgpKeyStore +{ + private readonly ISecretProtector _protector; + private readonly string _path; + + public AesGcmPgpKeyStore(ISecretProtector protector, string path) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + _protector = protector; + _path = path; + } + + /// + public async Task SaveAsync(IReadOnlyCollection keys, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(keys); + + var plaintext = PgpKeyStoreCodec.Encode(keys); + var protectedBytes = _protector.Protect(plaintext); + + var directory = Path.GetDirectoryName(_path); + if (!string.IsNullOrEmpty(directory)) + { + Directory.CreateDirectory(directory); + } + + await File.WriteAllBytesAsync(_path, protectedBytes, cancellationToken).ConfigureAwait(false); + } + + /// + public async Task> LoadAsync(CancellationToken cancellationToken = default) + { + if (!File.Exists(_path)) + { + return []; + } + + var protectedBytes = await File.ReadAllBytesAsync(_path, cancellationToken).ConfigureAwait(false); + var plaintext = _protector.Unprotect(protectedBytes); + + return PgpKeyStoreCodec.Decode(plaintext); + } +} diff --git a/tests/SquidStd.Tests/Crypto/Pgp/AesGcmPgpKeyStoreTests.cs b/tests/SquidStd.Tests/Crypto/Pgp/AesGcmPgpKeyStoreTests.cs new file mode 100644 index 00000000..d4e820db --- /dev/null +++ b/tests/SquidStd.Tests/Crypto/Pgp/AesGcmPgpKeyStoreTests.cs @@ -0,0 +1,48 @@ +using System.Text; +using SquidStd.Core.Data.Storage; +using SquidStd.Crypto.Pgp.Services; +using SquidStd.Services.Core.Services.Storage; +using SquidStd.Tests.Crypto.Pgp.Support; + +namespace SquidStd.Tests.Crypto.Pgp; + +[Collection("PgpKeys")] +public class AesGcmPgpKeyStoreTests +{ + private readonly PgpTestKeys _keys; + + public AesGcmPgpKeyStoreTests(PgpTestKeys keys) + { + _keys = keys; + } + + [Fact] + public async Task SaveThenLoad_RoundTripsAndBlobIsNotPlaintext() + { + var path = Path.Combine(Path.GetTempPath(), "squidstd-pgp-" + Guid.NewGuid().ToString("N") + ".bin"); + var protector = new AesGcmSecretProtector(new SecretsConfig()); + + try + { + var source = new PgpKeyring(); + source.Import(_keys.AlicePrivate); + var store = new AesGcmPgpKeyStore(protector, path); + await source.SaveAsync(store); + + var bytes = await File.ReadAllBytesAsync(path); + var asText = Encoding.UTF8.GetString(bytes); + Assert.DoesNotContain("BEGIN PGP", asText, StringComparison.Ordinal); + + var restored = new PgpKeyring(); + await restored.LoadAsync(store); + Assert.True(restored.Find(PgpTestKeys.AliceIdentity)!.HasSecret); + } + finally + { + if (File.Exists(path)) + { + File.Delete(path); + } + } + } +} From ee197da992bb2d0be8caf1555ff7ae1b47ecbe62 Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 14:28:24 +0200 Subject: [PATCH 10/77] feat(crypto): add RegisterPgp DI extension --- .../Pgp/Extensions/RegisterPgpExtensions.cs | 30 +++++++++++++++++++ .../Crypto/Pgp/RegisterPgpExtensionsTests.cs | 26 ++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 src/SquidStd.Crypto/Pgp/Extensions/RegisterPgpExtensions.cs create mode 100644 tests/SquidStd.Tests/Crypto/Pgp/RegisterPgpExtensionsTests.cs diff --git a/src/SquidStd.Crypto/Pgp/Extensions/RegisterPgpExtensions.cs b/src/SquidStd.Crypto/Pgp/Extensions/RegisterPgpExtensions.cs new file mode 100644 index 00000000..56153d0b --- /dev/null +++ b/src/SquidStd.Crypto/Pgp/Extensions/RegisterPgpExtensions.cs @@ -0,0 +1,30 @@ +using DryIoc; +using SquidStd.Crypto.Pgp.Interfaces; +using SquidStd.Crypto.Pgp.Services; + +namespace SquidStd.Crypto.Pgp.Extensions; + +/// DryIoc registration helpers for the PGP module. +public static class RegisterPgpExtensions +{ + /// Container that receives the PGP registrations. + extension(IContainer container) + { + /// + /// Registers the PGP keyring, service, and the chosen key store (all singletons). The keyring is not + /// auto-loaded; call at startup if persistence is desired. + /// + /// Builds the from the resolver. + /// The same container for chaining. + public IContainer RegisterPgp(Func keyStoreFactory) + { + ArgumentNullException.ThrowIfNull(keyStoreFactory); + + container.Register(Reuse.Singleton); + container.Register(Reuse.Singleton); + container.RegisterDelegate(keyStoreFactory, Reuse.Singleton); + + return container; + } + } +} diff --git a/tests/SquidStd.Tests/Crypto/Pgp/RegisterPgpExtensionsTests.cs b/tests/SquidStd.Tests/Crypto/Pgp/RegisterPgpExtensionsTests.cs new file mode 100644 index 00000000..47fd51b9 --- /dev/null +++ b/tests/SquidStd.Tests/Crypto/Pgp/RegisterPgpExtensionsTests.cs @@ -0,0 +1,26 @@ +using DryIoc; +using SquidStd.Crypto.Pgp.Extensions; +using SquidStd.Crypto.Pgp.Interfaces; +using SquidStd.Crypto.Pgp.Services; + +namespace SquidStd.Tests.Crypto.Pgp; + +public class RegisterPgpExtensionsTests +{ + [Fact] + public void RegisterPgp_RegistersKeyringServiceAndStoreAsSingletons() + { + using var container = new Container(); + var dir = Path.Combine(Path.GetTempPath(), "squidstd-pgp-di-" + Guid.NewGuid().ToString("N")); + + container.RegisterPgp(_ => new FilePgpKeyStore(dir)); + + var keyring = container.Resolve(); + var service = container.Resolve(); + var store = container.Resolve(); + + Assert.Same(keyring, container.Resolve()); + Assert.NotNull(service); + Assert.IsType(store); + } +} From 62fab4000269eedbf2fcf437680c2704c125eef5 Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 14:30:23 +0200 Subject: [PATCH 11/77] docs(crypto): add SquidStd.Crypto README and format module --- src/SquidStd.Crypto/Pgp/Data/PgpKey.cs | 3 +- .../Pgp/Interfaces/IPgpService.cs | 22 ++++-- .../Pgp/Services/FilePgpKeyStore.cs | 14 ++-- .../Pgp/Services/PgpService.cs | 30 ++++++-- src/SquidStd.Crypto/README.md | 69 +++++++++++++++++++ .../Crypto/Pgp/PgpServiceEncryptionTests.cs | 10 +-- .../Crypto/Pgp/PgpServiceSigningTests.cs | 20 ++++-- .../Crypto/Pgp/Support/PgpTestKeys.cs | 12 ++-- 8 files changed, 148 insertions(+), 32 deletions(-) create mode 100644 src/SquidStd.Crypto/README.md diff --git a/src/SquidStd.Crypto/Pgp/Data/PgpKey.cs b/src/SquidStd.Crypto/Pgp/Data/PgpKey.cs index 467f18dd..2fd65dcd 100644 --- a/src/SquidStd.Crypto/Pgp/Data/PgpKey.cs +++ b/src/SquidStd.Crypto/Pgp/Data/PgpKey.cs @@ -14,7 +14,8 @@ public sealed record PgpKey( string? PrivateArmored, DateTimeOffset CreatedUtc, DateTimeOffset? ExpiresUtc, - PgpKeyAlgorithm Algorithm) + PgpKeyAlgorithm Algorithm +) { /// Whether this key carries secret material (can decrypt and sign). public bool HasSecret => PrivateArmored is not null; diff --git a/src/SquidStd.Crypto/Pgp/Interfaces/IPgpService.cs b/src/SquidStd.Crypto/Pgp/Interfaces/IPgpService.cs index a3ffa871..cdf07fcc 100644 --- a/src/SquidStd.Crypto/Pgp/Interfaces/IPgpService.cs +++ b/src/SquidStd.Crypto/Pgp/Interfaces/IPgpService.cs @@ -15,7 +15,9 @@ public interface IPgpService Task EncryptForAsync(string recipientIdentity, byte[] data, CancellationToken cancellationToken = default); /// Encrypts a stream for the recipient, writing an armored message to . - Task EncryptForAsync(string recipientIdentity, Stream input, Stream output, CancellationToken cancellationToken = default); + Task EncryptForAsync( + string recipientIdentity, Stream input, Stream output, CancellationToken cancellationToken = default + ); /// Decrypts an armored message using the matching keyring secret key and passphrase. Task DecryptAsync(string armored, string passphrase, CancellationToken cancellationToken = default); @@ -24,16 +26,26 @@ public interface IPgpService Task DecryptAsync(Stream input, Stream output, string passphrase, CancellationToken cancellationToken = default); /// Encrypts for the recipient and signs it with the signer's secret key. - Task EncryptAndSignForAsync(string recipientIdentity, byte[] data, string signerIdentity, string signerPassphrase, CancellationToken cancellationToken = default); + Task EncryptAndSignForAsync( + string recipientIdentity, byte[] data, string signerIdentity, string signerPassphrase, + CancellationToken cancellationToken = default + ); /// Encrypts and signs a stream for the recipient. - Task EncryptAndSignForAsync(string recipientIdentity, Stream input, Stream output, string signerIdentity, string signerPassphrase, CancellationToken cancellationToken = default); + Task EncryptAndSignForAsync( + string recipientIdentity, Stream input, Stream output, string signerIdentity, string signerPassphrase, + CancellationToken cancellationToken = default + ); /// Decrypts an armored message and reports whether it was signed and whether the signature validated. - Task DecryptAndVerifyAsync(string armored, string passphrase, CancellationToken cancellationToken = default); + Task DecryptAndVerifyAsync( + string armored, string passphrase, CancellationToken cancellationToken = default + ); /// Produces an armored signed message that embeds . - Task SignAsync(byte[] data, string signerIdentity, string passphrase, CancellationToken cancellationToken = default); + Task SignAsync( + byte[] data, string signerIdentity, string passphrase, CancellationToken cancellationToken = default + ); /// Verifies an armored signed message against the keyring, recovering the embedded content. Task VerifyAsync(string signedMessage, CancellationToken cancellationToken = default); diff --git a/src/SquidStd.Crypto/Pgp/Services/FilePgpKeyStore.cs b/src/SquidStd.Crypto/Pgp/Services/FilePgpKeyStore.cs index 6a1c3041..e71d2c1c 100644 --- a/src/SquidStd.Crypto/Pgp/Services/FilePgpKeyStore.cs +++ b/src/SquidStd.Crypto/Pgp/Services/FilePgpKeyStore.cs @@ -31,14 +31,20 @@ public async Task SaveAsync(IReadOnlyCollection keys, CancellationToken { var stem = Stem(key); await File.WriteAllTextAsync( - Path.Combine(_directory, stem + PublicSuffix), key.PublicArmored, cancellationToken - ).ConfigureAwait(false); + Path.Combine(_directory, stem + PublicSuffix), + key.PublicArmored, + cancellationToken + ) + .ConfigureAwait(false); if (key.PrivateArmored is not null) { await File.WriteAllTextAsync( - Path.Combine(_directory, stem + SecretSuffix), key.PrivateArmored, cancellationToken - ).ConfigureAwait(false); + Path.Combine(_directory, stem + SecretSuffix), + key.PrivateArmored, + cancellationToken + ) + .ConfigureAwait(false); } } } diff --git a/src/SquidStd.Crypto/Pgp/Services/PgpService.cs b/src/SquidStd.Crypto/Pgp/Services/PgpService.cs index abe90dd7..6838551f 100644 --- a/src/SquidStd.Crypto/Pgp/Services/PgpService.cs +++ b/src/SquidStd.Crypto/Pgp/Services/PgpService.cs @@ -50,7 +50,9 @@ public PgpKey GenerateKey(string identity, string passphrase, PgpKeyOptions? opt } /// - public async Task EncryptForAsync(string recipientIdentity, byte[] data, CancellationToken cancellationToken = default) + public async Task EncryptForAsync( + string recipientIdentity, byte[] data, CancellationToken cancellationToken = default + ) { ArgumentNullException.ThrowIfNull(data); @@ -65,7 +67,9 @@ public async Task EncryptForAsync(string recipientIdentity, byte[] data, } /// - public async Task EncryptForAsync(string recipientIdentity, Stream input, Stream output, CancellationToken cancellationToken = default) + public async Task EncryptForAsync( + string recipientIdentity, Stream input, Stream output, CancellationToken cancellationToken = default + ) { ArgumentNullException.ThrowIfNull(input); ArgumentNullException.ThrowIfNull(output); @@ -91,7 +95,9 @@ public async Task DecryptAsync(string armored, string passphrase, Cancel } /// - public async Task DecryptAsync(Stream input, Stream output, string passphrase, CancellationToken cancellationToken = default) + public async Task DecryptAsync( + Stream input, Stream output, string passphrase, CancellationToken cancellationToken = default + ) { ArgumentNullException.ThrowIfNull(input); ArgumentNullException.ThrowIfNull(output); @@ -109,7 +115,10 @@ public async Task DecryptAsync(Stream input, Stream output, string passphrase, C } /// - public async Task EncryptAndSignForAsync(string recipientIdentity, byte[] data, string signerIdentity, string signerPassphrase, CancellationToken cancellationToken = default) + public async Task EncryptAndSignForAsync( + string recipientIdentity, byte[] data, string signerIdentity, string signerPassphrase, + CancellationToken cancellationToken = default + ) { ArgumentNullException.ThrowIfNull(data); @@ -123,7 +132,10 @@ public async Task EncryptAndSignForAsync(string recipientIdentity, byte[ } /// - public async Task EncryptAndSignForAsync(string recipientIdentity, Stream input, Stream output, string signerIdentity, string signerPassphrase, CancellationToken cancellationToken = default) + public async Task EncryptAndSignForAsync( + string recipientIdentity, Stream input, Stream output, string signerIdentity, string signerPassphrase, + CancellationToken cancellationToken = default + ) { ArgumentNullException.ThrowIfNull(input); ArgumentNullException.ThrowIfNull(output); @@ -133,7 +145,9 @@ public async Task EncryptAndSignForAsync(string recipientIdentity, Stream input, } /// - public async Task DecryptAndVerifyAsync(string armored, string passphrase, CancellationToken cancellationToken = default) + public async Task DecryptAndVerifyAsync( + string armored, string passphrase, CancellationToken cancellationToken = default + ) { ArgumentException.ThrowIfNullOrWhiteSpace(armored); @@ -167,7 +181,9 @@ public async Task DecryptAndVerifyAsync(string armored, str } /// - public async Task SignAsync(byte[] data, string signerIdentity, string passphrase, CancellationToken cancellationToken = default) + public async Task SignAsync( + byte[] data, string signerIdentity, string passphrase, CancellationToken cancellationToken = default + ) { ArgumentNullException.ThrowIfNull(data); diff --git a/src/SquidStd.Crypto/README.md b/src/SquidStd.Crypto/README.md new file mode 100644 index 00000000..69baf78c --- /dev/null +++ b/src/SquidStd.Crypto/README.md @@ -0,0 +1,69 @@ +

SquidStd.Crypto

+ +OpenPGP key management and operations for SquidStd, built on [PgpCore](https://github.com/mattosaurus/PgpCore) +(a maintained MIT wrapper over BouncyCastle). Provides key generation, encrypt/decrypt, sign/verify, and the +combined encrypt+sign / decrypt+verify flows over a stateful, indexed **keyring** with a pluggable persistence +backend. `SquidStd.Core` stays dependency-free; this module is the home for higher-level crypto features, with +PGP namespaced under `SquidStd.Crypto.Pgp` so future areas can coexist. + +## Install + +```bash +dotnet add package SquidStd.Crypto +``` + +## Usage + +```csharp +using DryIoc; +using SquidStd.Crypto.Pgp.Extensions; +using SquidStd.Crypto.Pgp.Interfaces; +using SquidStd.Crypto.Pgp.Services; + +// Register the keyring, service, and a key store (all singletons). +container.RegisterPgp(_ => new FilePgpKeyStore("/var/lib/app/pgp")); +// or encrypted at rest with the app key: +// container.RegisterPgp(r => new AesGcmPgpKeyStore(r.Resolve(), "/var/lib/app/pgp.bin")); + +var keyring = container.Resolve(); +var pgp = container.Resolve(); + +// Generate a key and import it into the keyring. +var alice = pgp.GenerateKey("alice@example.com", "passphrase"); +keyring.Import(alice.PrivateArmored!); +keyring.Import(bobPublicArmored); // a correspondent's public key + +// Encrypt for a recipient, decrypt with the held secret key. +string armored = await pgp.EncryptForAsync("bob@example.com", payloadBytes); +byte[] plaintext = await pgp.DecryptAsync(armoredFromBob, "passphrase"); + +// Sign and verify (signed message — the data is embedded in the armored block). +string signed = await pgp.SignAsync(payloadBytes, "alice@example.com", "passphrase"); +var verification = await pgp.VerifyAsync(signed); // verification.IsValid, verification.Data + +// Combined: encrypt + sign, then decrypt + verify. +string sealed = await pgp.EncryptAndSignForAsync("bob@example.com", payloadBytes, "alice@example.com", "passphrase"); +var result = await pgp.DecryptAndVerifyAsync(sealedFromBob, "passphrase"); // result.Data, result.IsSigned, result.IsValid + +// Persist / restore the keyring. +await keyring.SaveAsync(container.Resolve()); +await keyring.LoadAsync(container.Resolve()); +``` + +## Key stores + +- **`FilePgpKeyStore(directory)`** — one armored `.asc` per key (public, plus secret when held). gpg-interoperable. +- **`AesGcmPgpKeyStore(ISecretProtector, path)`** — the whole keyring serialized to a single file, encrypted at + rest with the application key via `SquidStd`'s `ISecretProtector`. + +## Notes + +- **Signatures are signed messages**, not detached signatures: `SignAsync` embeds the data in the armored block + and `VerifyAsync` recovers it. Verification is **pass/fail** — PgpCore does not expose the signer's key id or + identity, so the results carry no signer attribution. +- `DecryptAndVerifyAsync` never throws on a bad/absent signature when the ciphertext itself is valid: it always + recovers `Data` and reports `IsSigned` / `IsValid`. +- The streaming `DecryptAsync(Stream, Stream, …)` buffers its input internally so the recipient key id can be + read before decrypting; the encrypt and sign stream paths flow straight through. +- Passphrases are supplied per operation and never persisted (only passphrase-protected secret blocks are + stored). diff --git a/tests/SquidStd.Tests/Crypto/Pgp/PgpServiceEncryptionTests.cs b/tests/SquidStd.Tests/Crypto/Pgp/PgpServiceEncryptionTests.cs index 069d9a1d..0a1904c8 100644 --- a/tests/SquidStd.Tests/Crypto/Pgp/PgpServiceEncryptionTests.cs +++ b/tests/SquidStd.Tests/Crypto/Pgp/PgpServiceEncryptionTests.cs @@ -18,8 +18,8 @@ public PgpServiceEncryptionTests(PgpTestKeys keys) public async Task EncryptFor_ThenDecrypt_RoundTripsBytes() { var keyring = new PgpKeyring(); - keyring.Import(_keys.AlicePublic); // recipient public only for encryption - keyring.Import(_keys.AlicePrivate); // secret for decryption (replaces same key id, now HasSecret) + keyring.Import(_keys.AlicePublic); // recipient public only for encryption + keyring.Import(_keys.AlicePrivate); // secret for decryption (replaces same key id, now HasSecret) var service = new PgpService(keyring); var payload = Encoding.UTF8.GetBytes("squid secret message"); @@ -35,8 +35,10 @@ public async Task EncryptFor_UnknownRecipient_ThrowsKeyNotFound() { var service = new PgpService(new PgpKeyring()); - await Assert.ThrowsAsync( - () => service.EncryptForAsync("nobody@squidstd.test", Encoding.UTF8.GetBytes("x")) + await Assert.ThrowsAsync(() => service.EncryptForAsync( + "nobody@squidstd.test", + Encoding.UTF8.GetBytes("x") + ) ); } diff --git a/tests/SquidStd.Tests/Crypto/Pgp/PgpServiceSigningTests.cs b/tests/SquidStd.Tests/Crypto/Pgp/PgpServiceSigningTests.cs index c6a9f9ae..cad70f8d 100644 --- a/tests/SquidStd.Tests/Crypto/Pgp/PgpServiceSigningTests.cs +++ b/tests/SquidStd.Tests/Crypto/Pgp/PgpServiceSigningTests.cs @@ -36,7 +36,11 @@ public async Task Verify_TamperedMessage_IsInvalid() keyring.Import(_keys.AlicePrivate); var service = new PgpService(keyring); - var signed = await service.SignAsync(Encoding.UTF8.GetBytes("original"), PgpTestKeys.AliceIdentity, PgpTestKeys.AlicePassphrase); + var signed = await service.SignAsync( + Encoding.UTF8.GetBytes("original"), + PgpTestKeys.AliceIdentity, + PgpTestKeys.AlicePassphrase + ); var tampered = MutateBody(signed); var result = await service.VerifyAsync(tampered); @@ -48,13 +52,16 @@ public async Task Verify_TamperedMessage_IsInvalid() public async Task EncryptAndSign_ThenDecryptAndVerify_RoundTripsAndValidates() { var keyring = new PgpKeyring(); - keyring.Import(_keys.AlicePrivate); // recipient (has secret to decrypt) - keyring.Import(_keys.BobPrivate); // signer + keyring.Import(_keys.AlicePrivate); // recipient (has secret to decrypt) + keyring.Import(_keys.BobPrivate); // signer var service = new PgpService(keyring); var payload = Encoding.UTF8.GetBytes("confidential and signed"); var armored = await service.EncryptAndSignForAsync( - PgpTestKeys.AliceIdentity, payload, PgpTestKeys.BobIdentity, PgpTestKeys.BobPassphrase + PgpTestKeys.AliceIdentity, + payload, + PgpTestKeys.BobIdentity, + PgpTestKeys.BobPassphrase ); var result = await service.DecryptAndVerifyAsync(armored, PgpTestKeys.AlicePassphrase); @@ -73,7 +80,10 @@ public async Task DecryptAndVerify_SignerNotInKeyring_RecoversDataButInvalid() var signingService = new PgpService(signingKeyring); var payload = Encoding.UTF8.GetBytes("signed by an unknown party"); var armored = await signingService.EncryptAndSignForAsync( - PgpTestKeys.AliceIdentity, payload, PgpTestKeys.BobIdentity, PgpTestKeys.BobPassphrase + PgpTestKeys.AliceIdentity, + payload, + PgpTestKeys.BobIdentity, + PgpTestKeys.BobPassphrase ); var recipientOnly = new PgpKeyring(); diff --git a/tests/SquidStd.Tests/Crypto/Pgp/Support/PgpTestKeys.cs b/tests/SquidStd.Tests/Crypto/Pgp/Support/PgpTestKeys.cs index 996b6a0d..16b6e9ce 100644 --- a/tests/SquidStd.Tests/Crypto/Pgp/Support/PgpTestKeys.cs +++ b/tests/SquidStd.Tests/Crypto/Pgp/Support/PgpTestKeys.cs @@ -13,12 +13,6 @@ public sealed class PgpTestKeys public const string AliceIdentity = "alice@squidstd.test"; public const string BobIdentity = "bob@squidstd.test"; - public PgpTestKeys() - { - (AlicePublic, AlicePrivate) = Generate(AliceIdentity, AlicePassphrase); - (BobPublic, BobPrivate) = Generate(BobIdentity, BobPassphrase); - } - public string AlicePublic { get; } public string AlicePrivate { get; } @@ -27,6 +21,12 @@ public PgpTestKeys() public string BobPrivate { get; } + public PgpTestKeys() + { + (AlicePublic, AlicePrivate) = Generate(AliceIdentity, AlicePassphrase); + (BobPublic, BobPrivate) = Generate(BobIdentity, BobPassphrase); + } + private static (string Public, string Private) Generate(string identity, string passphrase) { var pgp = new PGP(); From 00ace0480501daf2fed8b5e3e0e5faee3671d88f Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 17:38:12 +0200 Subject: [PATCH 12/77] build(vfs): scaffold SquidStd.Vfs.Abstractions and SquidStd.Vfs modules --- SquidStd.slnx | 2 ++ src/SquidStd.Crypto/SquidStd.Crypto.csproj | 1 + .../SquidStd.Vfs.Abstractions.csproj | 10 ++++++++++ src/SquidStd.Vfs/SquidStd.Vfs.csproj | 19 +++++++++++++++++++ tests/SquidStd.Tests/SquidStd.Tests.csproj | 2 ++ 5 files changed, 34 insertions(+) create mode 100644 src/SquidStd.Vfs.Abstractions/SquidStd.Vfs.Abstractions.csproj create mode 100644 src/SquidStd.Vfs/SquidStd.Vfs.csproj diff --git a/SquidStd.slnx b/SquidStd.slnx index 55f1af76..a850a11c 100644 --- a/SquidStd.slnx +++ b/SquidStd.slnx @@ -38,6 +38,8 @@ + + diff --git a/src/SquidStd.Crypto/SquidStd.Crypto.csproj b/src/SquidStd.Crypto/SquidStd.Crypto.csproj index f81170ff..fa0605cd 100644 --- a/src/SquidStd.Crypto/SquidStd.Crypto.csproj +++ b/src/SquidStd.Crypto/SquidStd.Crypto.csproj @@ -18,6 +18,7 @@ + diff --git a/src/SquidStd.Vfs.Abstractions/SquidStd.Vfs.Abstractions.csproj b/src/SquidStd.Vfs.Abstractions/SquidStd.Vfs.Abstractions.csproj new file mode 100644 index 00000000..2f0e9b5b --- /dev/null +++ b/src/SquidStd.Vfs.Abstractions/SquidStd.Vfs.Abstractions.csproj @@ -0,0 +1,10 @@ + + + + net10.0 + enable + enable + true + + + diff --git a/src/SquidStd.Vfs/SquidStd.Vfs.csproj b/src/SquidStd.Vfs/SquidStd.Vfs.csproj new file mode 100644 index 00000000..34c96321 --- /dev/null +++ b/src/SquidStd.Vfs/SquidStd.Vfs.csproj @@ -0,0 +1,19 @@ + + + + net10.0 + enable + enable + true + + + + + + + + + + + + diff --git a/tests/SquidStd.Tests/SquidStd.Tests.csproj b/tests/SquidStd.Tests/SquidStd.Tests.csproj index 82c89143..0c7805d4 100644 --- a/tests/SquidStd.Tests/SquidStd.Tests.csproj +++ b/tests/SquidStd.Tests/SquidStd.Tests.csproj @@ -71,6 +71,8 @@ + + From 0efecc5c860d0c222b06910f561609bc583fdfa6 Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 17:38:40 +0200 Subject: [PATCH 13/77] feat(vfs): add virtual filesystem abstraction and lockable contract --- .../Data/VfsEntry.cs | 4 +++ .../Interfaces/ILockableFileSystem.cs | 14 ++++++++++ .../Interfaces/IVirtualFileSystem.cs | 28 +++++++++++++++++++ 3 files changed, 46 insertions(+) create mode 100644 src/SquidStd.Vfs.Abstractions/Data/VfsEntry.cs create mode 100644 src/SquidStd.Vfs.Abstractions/Interfaces/ILockableFileSystem.cs create mode 100644 src/SquidStd.Vfs.Abstractions/Interfaces/IVirtualFileSystem.cs diff --git a/src/SquidStd.Vfs.Abstractions/Data/VfsEntry.cs b/src/SquidStd.Vfs.Abstractions/Data/VfsEntry.cs new file mode 100644 index 00000000..b8c6f45a --- /dev/null +++ b/src/SquidStd.Vfs.Abstractions/Data/VfsEntry.cs @@ -0,0 +1,4 @@ +namespace SquidStd.Vfs.Abstractions.Data; + +/// An entry listed in a virtual filesystem: its logical path, byte size, and last-modified time. +public sealed record VfsEntry(string Path, long Size, DateTimeOffset ModifiedUtc); diff --git a/src/SquidStd.Vfs.Abstractions/Interfaces/ILockableFileSystem.cs b/src/SquidStd.Vfs.Abstractions/Interfaces/ILockableFileSystem.cs new file mode 100644 index 00000000..2edc5365 --- /dev/null +++ b/src/SquidStd.Vfs.Abstractions/Interfaces/ILockableFileSystem.cs @@ -0,0 +1,14 @@ +namespace SquidStd.Vfs.Abstractions.Interfaces; + +/// A virtual filesystem that is locked until unlocked with a passphrase (e.g. an encrypted vault). +public interface ILockableFileSystem : IVirtualFileSystem +{ + /// Whether the filesystem is currently unlocked. + bool IsUnlocked { get; } + + /// Derives the key from the passphrase and unlocks the filesystem. + void Unlock(string passphrase); + + /// Zeroes and drops the key, locking the filesystem. + void Lock(); +} diff --git a/src/SquidStd.Vfs.Abstractions/Interfaces/IVirtualFileSystem.cs b/src/SquidStd.Vfs.Abstractions/Interfaces/IVirtualFileSystem.cs new file mode 100644 index 00000000..5422b506 --- /dev/null +++ b/src/SquidStd.Vfs.Abstractions/Interfaces/IVirtualFileSystem.cs @@ -0,0 +1,28 @@ +using SquidStd.Vfs.Abstractions.Data; + +namespace SquidStd.Vfs.Abstractions.Interfaces; + +/// A path-based virtual filesystem over a pluggable backend (directory, zip, encrypted container). +public interface IVirtualFileSystem +{ + /// Whether a file exists at the logical path. + ValueTask ExistsAsync(string path, CancellationToken cancellationToken = default); + + /// Reads the whole file, or null when it does not exist. + ValueTask ReadAllBytesAsync(string path, CancellationToken cancellationToken = default); + + /// Writes the whole file, creating or overwriting it. + ValueTask WriteAllBytesAsync(string path, ReadOnlyMemory data, CancellationToken cancellationToken = default); + + /// Opens a readable stream over the file; throws when absent. + Task OpenReadAsync(string path, CancellationToken cancellationToken = default); + + /// Opens a writable stream that creates or overwrites the file on disposal. + Task OpenWriteAsync(string path, CancellationToken cancellationToken = default); + + /// Deletes the file. Returns whether one was removed. + ValueTask DeleteAsync(string path, CancellationToken cancellationToken = default); + + /// Lists entries, optionally filtered by logical path prefix. + IAsyncEnumerable ListAsync(string? prefix = null, CancellationToken cancellationToken = default); +} From e4f4e9e91e5dd30139251ce2fc2668bbfe0569e6 Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 17:39:46 +0200 Subject: [PATCH 14/77] feat(vfs): add path normalization and physical filesystem provider --- src/SquidStd.Vfs/Internal/VfsPath.cs | 23 ++++ .../Services/PhysicalFileSystem.cs | 106 ++++++++++++++++++ .../Vfs/PhysicalFileSystemTests.cs | 41 +++++++ tests/SquidStd.Tests/Vfs/VfsPathTests.cs | 23 ++++ 4 files changed, 193 insertions(+) create mode 100644 src/SquidStd.Vfs/Internal/VfsPath.cs create mode 100644 src/SquidStd.Vfs/Services/PhysicalFileSystem.cs create mode 100644 tests/SquidStd.Tests/Vfs/PhysicalFileSystemTests.cs create mode 100644 tests/SquidStd.Tests/Vfs/VfsPathTests.cs diff --git a/src/SquidStd.Vfs/Internal/VfsPath.cs b/src/SquidStd.Vfs/Internal/VfsPath.cs new file mode 100644 index 00000000..a9803ad6 --- /dev/null +++ b/src/SquidStd.Vfs/Internal/VfsPath.cs @@ -0,0 +1,23 @@ +namespace SquidStd.Vfs.Internal; + +/// Normalizes logical VFS paths to forward-slash, root-relative form and rejects traversal. +internal static class VfsPath +{ + public static string Normalize(string path) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + + var segments = path.Replace('\\', '/') + .Split('/', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + + foreach (var segment in segments) + { + if (segment is "." or "..") + { + throw new ArgumentException($"Path '{path}' must not contain relative segments.", nameof(path)); + } + } + + return string.Join('/', segments); + } +} diff --git a/src/SquidStd.Vfs/Services/PhysicalFileSystem.cs b/src/SquidStd.Vfs/Services/PhysicalFileSystem.cs new file mode 100644 index 00000000..f0514586 --- /dev/null +++ b/src/SquidStd.Vfs/Services/PhysicalFileSystem.cs @@ -0,0 +1,106 @@ +using System.Runtime.CompilerServices; +using SquidStd.Vfs.Abstractions.Data; +using SquidStd.Vfs.Abstractions.Interfaces; +using SquidStd.Vfs.Internal; + +namespace SquidStd.Vfs.Services; + +/// A virtual filesystem mapped onto a real directory tree. +public sealed class PhysicalFileSystem : IVirtualFileSystem +{ + private readonly string _root; + + public PhysicalFileSystem(string root) + { + ArgumentException.ThrowIfNullOrWhiteSpace(root); + _root = Path.GetFullPath(root); + Directory.CreateDirectory(_root); + } + + public ValueTask ExistsAsync(string path, CancellationToken cancellationToken = default) + { + return ValueTask.FromResult(File.Exists(Resolve(path))); + } + + public async ValueTask ReadAllBytesAsync(string path, CancellationToken cancellationToken = default) + { + var full = Resolve(path); + + return File.Exists(full) + ? await File.ReadAllBytesAsync(full, cancellationToken).ConfigureAwait(false) + : null; + } + + public async ValueTask WriteAllBytesAsync(string path, ReadOnlyMemory data, CancellationToken cancellationToken = default) + { + var full = Resolve(path); + Directory.CreateDirectory(Path.GetDirectoryName(full)!); + await File.WriteAllBytesAsync(full, data, cancellationToken).ConfigureAwait(false); + } + + public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) + { + var full = Resolve(path); + + if (!File.Exists(full)) + { + throw new FileNotFoundException($"No file at '{path}'.", path); + } + + return Task.FromResult(File.OpenRead(full)); + } + + public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) + { + var full = Resolve(path); + Directory.CreateDirectory(Path.GetDirectoryName(full)!); + + return Task.FromResult(File.Create(full)); + } + + public ValueTask DeleteAsync(string path, CancellationToken cancellationToken = default) + { + var full = Resolve(path); + + if (!File.Exists(full)) + { + return ValueTask.FromResult(false); + } + + File.Delete(full); + + return ValueTask.FromResult(true); + } + + public async IAsyncEnumerable ListAsync(string? prefix = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + if (!Directory.Exists(_root)) + { + yield break; + } + + var normalizedPrefix = string.IsNullOrEmpty(prefix) ? null : VfsPath.Normalize(prefix); + + foreach (var file in Directory.EnumerateFiles(_root, "*", SearchOption.AllDirectories)) + { + cancellationToken.ThrowIfCancellationRequested(); + + var logical = Path.GetRelativePath(_root, file).Replace('\\', '/'); + + if (normalizedPrefix is not null && !logical.StartsWith(normalizedPrefix, StringComparison.Ordinal)) + { + continue; + } + + var info = new FileInfo(file); + yield return new VfsEntry(logical, info.Length, info.LastWriteTimeUtc); + + await Task.CompletedTask; + } + } + + private string Resolve(string path) + { + return Path.Combine(_root, VfsPath.Normalize(path)); + } +} diff --git a/tests/SquidStd.Tests/Vfs/PhysicalFileSystemTests.cs b/tests/SquidStd.Tests/Vfs/PhysicalFileSystemTests.cs new file mode 100644 index 00000000..9d9a918b --- /dev/null +++ b/tests/SquidStd.Tests/Vfs/PhysicalFileSystemTests.cs @@ -0,0 +1,41 @@ +using System.Text; +using SquidStd.Vfs.Services; + +namespace SquidStd.Tests.Vfs; + +public class PhysicalFileSystemTests +{ + [Fact] + public async Task Write_Read_List_Delete_RoundTrips() + { + var root = Path.Combine(Path.GetTempPath(), "squidstd-vfs-" + Guid.NewGuid().ToString("N")); + + try + { + var fs = new PhysicalFileSystem(root); + await fs.WriteAllBytesAsync("docs/cv.pdf", Encoding.UTF8.GetBytes("hello")); + + Assert.True(await fs.ExistsAsync("docs/cv.pdf")); + Assert.Equal("hello", Encoding.UTF8.GetString((await fs.ReadAllBytesAsync("docs/cv.pdf"))!)); + + var entries = new List(); + await foreach (var e in fs.ListAsync()) + { + entries.Add(e.Path); + } + Assert.Contains("docs/cv.pdf", entries); + + Assert.True(await fs.DeleteAsync("docs/cv.pdf")); + Assert.False(await fs.ExistsAsync("docs/cv.pdf")); + Assert.Null(await fs.ReadAllBytesAsync("docs/cv.pdf")); + Assert.False(await fs.DeleteAsync("docs/cv.pdf")); + } + finally + { + if (Directory.Exists(root)) + { + Directory.Delete(root, recursive: true); + } + } + } +} diff --git a/tests/SquidStd.Tests/Vfs/VfsPathTests.cs b/tests/SquidStd.Tests/Vfs/VfsPathTests.cs new file mode 100644 index 00000000..a33151ea --- /dev/null +++ b/tests/SquidStd.Tests/Vfs/VfsPathTests.cs @@ -0,0 +1,23 @@ +using SquidStd.Vfs.Internal; + +namespace SquidStd.Tests.Vfs; + +public class VfsPathTests +{ + [Theory] + [InlineData("docs/cv.pdf", "docs/cv.pdf")] + [InlineData("/docs//cv.pdf", "docs/cv.pdf")] + [InlineData("docs\\cv.pdf", "docs/cv.pdf")] + public void Normalize_ProducesForwardSlashRelativePath(string input, string expected) + { + Assert.Equal(expected, VfsPath.Normalize(input)); + } + + [Theory] + [InlineData("../escape")] + [InlineData("a/../../b")] + public void Normalize_RejectsTraversal(string input) + { + Assert.Throws(() => VfsPath.Normalize(input)); + } +} From 02ff3ba4d6dff76ec82dab54a41d355e44a32184 Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 17:40:30 +0200 Subject: [PATCH 15/77] feat(vfs): add in-memory filesystem provider --- .../Services/InMemoryFileSystem.cs | 87 +++++++++++++++++++ .../Vfs/InMemoryFileSystemTests.cs | 27 ++++++ 2 files changed, 114 insertions(+) create mode 100644 src/SquidStd.Vfs/Services/InMemoryFileSystem.cs create mode 100644 tests/SquidStd.Tests/Vfs/InMemoryFileSystemTests.cs diff --git a/src/SquidStd.Vfs/Services/InMemoryFileSystem.cs b/src/SquidStd.Vfs/Services/InMemoryFileSystem.cs new file mode 100644 index 00000000..c7de6237 --- /dev/null +++ b/src/SquidStd.Vfs/Services/InMemoryFileSystem.cs @@ -0,0 +1,87 @@ +using System.Collections.Concurrent; +using System.Runtime.CompilerServices; +using SquidStd.Vfs.Abstractions.Data; +using SquidStd.Vfs.Abstractions.Interfaces; +using SquidStd.Vfs.Internal; + +namespace SquidStd.Vfs.Services; + +/// An in-memory virtual filesystem. Ephemeral; useful for tests and as a backend decorator target. +public sealed class InMemoryFileSystem : IVirtualFileSystem +{ + private readonly ConcurrentDictionary _files = new(StringComparer.Ordinal); + + public ValueTask ExistsAsync(string path, CancellationToken cancellationToken = default) + { + return ValueTask.FromResult(_files.ContainsKey(VfsPath.Normalize(path))); + } + + public ValueTask ReadAllBytesAsync(string path, CancellationToken cancellationToken = default) + { + return ValueTask.FromResult(_files.TryGetValue(VfsPath.Normalize(path), out var entry) ? entry.Data : null); + } + + public ValueTask WriteAllBytesAsync(string path, ReadOnlyMemory data, CancellationToken cancellationToken = default) + { + _files[VfsPath.Normalize(path)] = (data.ToArray(), DateTimeOffset.UtcNow); + + return ValueTask.CompletedTask; + } + + public async Task OpenReadAsync(string path, CancellationToken cancellationToken = default) + { + var data = await ReadAllBytesAsync(path, cancellationToken).ConfigureAwait(false) + ?? throw new FileNotFoundException($"No file at '{path}'.", path); + + return new MemoryStream(data, writable: false); + } + + public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) + { + return Task.FromResult(new WriteBackStream(this, path)); + } + + public ValueTask DeleteAsync(string path, CancellationToken cancellationToken = default) + { + return ValueTask.FromResult(_files.TryRemove(VfsPath.Normalize(path), out _)); + } + + public async IAsyncEnumerable ListAsync(string? prefix = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var normalizedPrefix = string.IsNullOrEmpty(prefix) ? null : VfsPath.Normalize(prefix); + + foreach (var (path, entry) in _files) + { + if (normalizedPrefix is not null && !path.StartsWith(normalizedPrefix, StringComparison.Ordinal)) + { + continue; + } + + yield return new VfsEntry(path, entry.Data.Length, entry.Modified); + + await Task.CompletedTask; + } + } + + private sealed class WriteBackStream : MemoryStream + { + private readonly InMemoryFileSystem _owner; + private readonly string _path; + + public WriteBackStream(InMemoryFileSystem owner, string path) + { + _owner = owner; + _path = path; + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + _owner._files[VfsPath.Normalize(_path)] = (ToArray(), DateTimeOffset.UtcNow); + } + + base.Dispose(disposing); + } + } +} diff --git a/tests/SquidStd.Tests/Vfs/InMemoryFileSystemTests.cs b/tests/SquidStd.Tests/Vfs/InMemoryFileSystemTests.cs new file mode 100644 index 00000000..db3e538c --- /dev/null +++ b/tests/SquidStd.Tests/Vfs/InMemoryFileSystemTests.cs @@ -0,0 +1,27 @@ +using System.Text; +using SquidStd.Vfs.Services; + +namespace SquidStd.Tests.Vfs; + +public class InMemoryFileSystemTests +{ + [Fact] + public async Task Write_Read_List_Delete_RoundTrips() + { + var fs = new InMemoryFileSystem(); + await fs.WriteAllBytesAsync("a/b.txt", Encoding.UTF8.GetBytes("x")); + + Assert.True(await fs.ExistsAsync("a/b.txt")); + Assert.Equal("x", Encoding.UTF8.GetString((await fs.ReadAllBytesAsync("a/b.txt"))!)); + + var paths = new List(); + await foreach (var e in fs.ListAsync("a")) + { + paths.Add(e.Path); + } + Assert.Equal(["a/b.txt"], paths); + + Assert.True(await fs.DeleteAsync("a/b.txt")); + Assert.Null(await fs.ReadAllBytesAsync("a/b.txt")); + } +} From fe8fe566d43c46aa303a9620345c35f711382e5c Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 17:41:31 +0200 Subject: [PATCH 16/77] feat(vfs): add zip-backed filesystem provider --- src/SquidStd.Vfs/Services/ZipFileSystem.cs | 130 ++++++++++++++++++ .../SquidStd.Tests/Vfs/ZipFileSystemTests.cs | 42 ++++++ 2 files changed, 172 insertions(+) create mode 100644 src/SquidStd.Vfs/Services/ZipFileSystem.cs create mode 100644 tests/SquidStd.Tests/Vfs/ZipFileSystemTests.cs diff --git a/src/SquidStd.Vfs/Services/ZipFileSystem.cs b/src/SquidStd.Vfs/Services/ZipFileSystem.cs new file mode 100644 index 00000000..294bb142 --- /dev/null +++ b/src/SquidStd.Vfs/Services/ZipFileSystem.cs @@ -0,0 +1,130 @@ +using System.IO.Compression; +using System.Runtime.CompilerServices; +using SquidStd.Vfs.Abstractions.Data; +using SquidStd.Vfs.Abstractions.Interfaces; +using SquidStd.Vfs.Internal; + +namespace SquidStd.Vfs.Services; + +/// A virtual filesystem backed by a single zip archive opened in update mode. +public sealed class ZipFileSystem : IVirtualFileSystem, IAsyncDisposable, IDisposable +{ + private readonly ZipArchive _archive; + private readonly FileStream _file; + + public ZipFileSystem(string path) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + _file = new FileStream(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None); + _archive = new ZipArchive(_file, ZipArchiveMode.Update, leaveOpen: true); + } + + public ValueTask ExistsAsync(string path, CancellationToken cancellationToken = default) + { + return ValueTask.FromResult(_archive.GetEntry(VfsPath.Normalize(path)) is not null); + } + + public async ValueTask ReadAllBytesAsync(string path, CancellationToken cancellationToken = default) + { + var entry = _archive.GetEntry(VfsPath.Normalize(path)); + + if (entry is null) + { + return null; + } + + await using var stream = entry.Open(); + using var buffer = new MemoryStream(); + await stream.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false); + + return buffer.ToArray(); + } + + public async ValueTask WriteAllBytesAsync(string path, ReadOnlyMemory data, CancellationToken cancellationToken = default) + { + var name = VfsPath.Normalize(path); + _archive.GetEntry(name)?.Delete(); + var entry = _archive.CreateEntry(name, CompressionLevel.Optimal); + + await using var stream = entry.Open(); + await stream.WriteAsync(data, cancellationToken).ConfigureAwait(false); + } + + public async Task OpenReadAsync(string path, CancellationToken cancellationToken = default) + { + var data = await ReadAllBytesAsync(path, cancellationToken).ConfigureAwait(false) + ?? throw new FileNotFoundException($"No file at '{path}'.", path); + + return new MemoryStream(data, writable: false); + } + + public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) + { + return Task.FromResult(new WriteBackStream(this, path)); + } + + public ValueTask DeleteAsync(string path, CancellationToken cancellationToken = default) + { + var entry = _archive.GetEntry(VfsPath.Normalize(path)); + + if (entry is null) + { + return ValueTask.FromResult(false); + } + + entry.Delete(); + + return ValueTask.FromResult(true); + } + + public async IAsyncEnumerable ListAsync(string? prefix = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var normalizedPrefix = string.IsNullOrEmpty(prefix) ? null : VfsPath.Normalize(prefix); + + foreach (var entry in _archive.Entries) + { + if (normalizedPrefix is not null && !entry.FullName.StartsWith(normalizedPrefix, StringComparison.Ordinal)) + { + continue; + } + + yield return new VfsEntry(entry.FullName, entry.Length, entry.LastWriteTime); + + await Task.CompletedTask; + } + } + + public async ValueTask DisposeAsync() + { + _archive.Dispose(); + await _file.DisposeAsync().ConfigureAwait(false); + } + + public void Dispose() + { + _archive.Dispose(); + _file.Dispose(); + } + + private sealed class WriteBackStream : MemoryStream + { + private readonly ZipFileSystem _owner; + private readonly string _path; + + public WriteBackStream(ZipFileSystem owner, string path) + { + _owner = owner; + _path = path; + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + _owner.WriteAllBytesAsync(_path, ToArray()).AsTask().GetAwaiter().GetResult(); + } + + base.Dispose(disposing); + } + } +} diff --git a/tests/SquidStd.Tests/Vfs/ZipFileSystemTests.cs b/tests/SquidStd.Tests/Vfs/ZipFileSystemTests.cs new file mode 100644 index 00000000..290a3983 --- /dev/null +++ b/tests/SquidStd.Tests/Vfs/ZipFileSystemTests.cs @@ -0,0 +1,42 @@ +using System.Text; +using SquidStd.Vfs.Services; + +namespace SquidStd.Tests.Vfs; + +public class ZipFileSystemTests +{ + [Fact] + public async Task Write_Reopen_Read_Delete_RoundTrips() + { + var path = Path.Combine(Path.GetTempPath(), "squidstd-vfs-" + Guid.NewGuid().ToString("N") + ".zip"); + + try + { + await using (var fs = new ZipFileSystem(path)) + { + await fs.WriteAllBytesAsync("docs/cv.pdf", Encoding.UTF8.GetBytes("hello")); + await fs.WriteAllBytesAsync("note.txt", Encoding.UTF8.GetBytes("hi")); + } + + await using (var fs = new ZipFileSystem(path)) + { + Assert.Equal("hello", Encoding.UTF8.GetString((await fs.ReadAllBytesAsync("docs/cv.pdf"))!)); + Assert.True(await fs.DeleteAsync("note.txt")); + Assert.False(await fs.ExistsAsync("note.txt")); + } + + await using (var fs = new ZipFileSystem(path)) + { + Assert.False(await fs.ExistsAsync("note.txt")); + Assert.True(await fs.ExistsAsync("docs/cv.pdf")); + } + } + finally + { + if (File.Exists(path)) + { + File.Delete(path); + } + } + } +} From 4b583c4e6953e976154d2fecb84cb969f6efad0c Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 17:42:20 +0200 Subject: [PATCH 17/77] feat(vfs): add VfsDirectories and RegisterVfs DI extension --- .../Extensions/RegisterVfsExtensions.cs | 21 +++++++++ src/SquidStd.Vfs/Services/VfsDirectories.cs | 43 +++++++++++++++++++ .../Vfs/RegisterVfsExtensionsTests.cs | 19 ++++++++ .../SquidStd.Tests/Vfs/VfsDirectoriesTests.cs | 23 ++++++++++ 4 files changed, 106 insertions(+) create mode 100644 src/SquidStd.Vfs/Extensions/RegisterVfsExtensions.cs create mode 100644 src/SquidStd.Vfs/Services/VfsDirectories.cs create mode 100644 tests/SquidStd.Tests/Vfs/RegisterVfsExtensionsTests.cs create mode 100644 tests/SquidStd.Tests/Vfs/VfsDirectoriesTests.cs diff --git a/src/SquidStd.Vfs/Extensions/RegisterVfsExtensions.cs b/src/SquidStd.Vfs/Extensions/RegisterVfsExtensions.cs new file mode 100644 index 00000000..c282b889 --- /dev/null +++ b/src/SquidStd.Vfs/Extensions/RegisterVfsExtensions.cs @@ -0,0 +1,21 @@ +using DryIoc; +using SquidStd.Vfs.Abstractions.Interfaces; + +namespace SquidStd.Vfs.Extensions; + +/// DryIoc registration helpers for the VFS module. +public static class RegisterVfsExtensions +{ + /// Container that receives the VFS registration. + extension(IContainer container) + { + /// Registers an singleton built by the factory. + public IContainer RegisterVfs(Func fileSystemFactory) + { + ArgumentNullException.ThrowIfNull(fileSystemFactory); + container.RegisterDelegate(fileSystemFactory, Reuse.Singleton); + + return container; + } + } +} diff --git a/src/SquidStd.Vfs/Services/VfsDirectories.cs b/src/SquidStd.Vfs/Services/VfsDirectories.cs new file mode 100644 index 00000000..a049e4ae --- /dev/null +++ b/src/SquidStd.Vfs/Services/VfsDirectories.cs @@ -0,0 +1,43 @@ +using SquidStd.Vfs.Abstractions.Interfaces; +using SquidStd.Vfs.Internal; + +namespace SquidStd.Vfs.Services; + +/// A VFS-backed analogue of DirectoriesConfig: named logical directories over any backend. +public sealed class VfsDirectories +{ + private readonly IVirtualFileSystem _fileSystem; + private readonly HashSet _directories; + + public VfsDirectories(IVirtualFileSystem fileSystem, IReadOnlyCollection directories) + { + ArgumentNullException.ThrowIfNull(directories); + _fileSystem = fileSystem; + _directories = directories.Select(VfsPath.Normalize).ToHashSet(StringComparer.Ordinal); + } + + /// The logical path of a named directory. + public string this[string directoryName] => GetPath(directoryName); + + /// The logical path of a named directory, by enum name. + public string this[Enum directoryType] => GetPath(directoryType.ToString()); + + /// Resolves the logical path of a named directory; throws when it was not declared. + public string GetPath(string directoryName) + { + var normalized = VfsPath.Normalize(directoryName); + + if (!_directories.Contains(normalized)) + { + throw new KeyNotFoundException($"Directory '{directoryName}' is not declared."); + } + + return normalized; + } + + /// Combines a named directory with a relative sub-path into a normalized logical path. + public string Combine(string directoryName, string relativePath) + { + return VfsPath.Normalize(GetPath(directoryName) + "/" + relativePath); + } +} diff --git a/tests/SquidStd.Tests/Vfs/RegisterVfsExtensionsTests.cs b/tests/SquidStd.Tests/Vfs/RegisterVfsExtensionsTests.cs new file mode 100644 index 00000000..4ef30132 --- /dev/null +++ b/tests/SquidStd.Tests/Vfs/RegisterVfsExtensionsTests.cs @@ -0,0 +1,19 @@ +using DryIoc; +using SquidStd.Vfs.Abstractions.Interfaces; +using SquidStd.Vfs.Extensions; +using SquidStd.Vfs.Services; + +namespace SquidStd.Tests.Vfs; + +public class RegisterVfsExtensionsTests +{ + [Fact] + public void RegisterVfs_RegistersSingletonFileSystem() + { + using var container = new Container(); + container.RegisterVfs(_ => new InMemoryFileSystem()); + + var first = container.Resolve(); + Assert.Same(first, container.Resolve()); + } +} diff --git a/tests/SquidStd.Tests/Vfs/VfsDirectoriesTests.cs b/tests/SquidStd.Tests/Vfs/VfsDirectoriesTests.cs new file mode 100644 index 00000000..a82c1ca8 --- /dev/null +++ b/tests/SquidStd.Tests/Vfs/VfsDirectoriesTests.cs @@ -0,0 +1,23 @@ +using System.Text; +using SquidStd.Vfs.Services; + +namespace SquidStd.Tests.Vfs; + +public class VfsDirectoriesTests +{ + [Fact] + public async Task ResolvesNamedDirectoriesAndWritesUnderThem() + { + var fs = new InMemoryFileSystem(); + var dirs = new VfsDirectories(fs, ["data", "logs"]); + + Assert.Equal("data", dirs["data"]); + Assert.Equal("logs", dirs["logs"]); + + var target = dirs.Combine("data", "cv.pdf"); + await fs.WriteAllBytesAsync(target, Encoding.UTF8.GetBytes("x")); + + Assert.Equal("data/cv.pdf", target); + Assert.True(await fs.ExistsAsync("data/cv.pdf")); + } +} From 057b93f08f1e27708a92740dddb0c51c4e637acc Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 17:43:35 +0200 Subject: [PATCH 18/77] feat(crypto): add vault options and Argon2id/HKDF key derivation --- src/SquidStd.Crypto/SquidStd.Crypto.csproj | 1 + .../Vfs/Data/CryptoVaultOptions.cs | 17 ++++++++ .../Vfs/Internal/VaultKeyDerivation.cs | 41 +++++++++++++++++++ .../Crypto/Vfs/VaultKeyDerivationTests.cs | 32 +++++++++++++++ 4 files changed, 91 insertions(+) create mode 100644 src/SquidStd.Crypto/Vfs/Data/CryptoVaultOptions.cs create mode 100644 src/SquidStd.Crypto/Vfs/Internal/VaultKeyDerivation.cs create mode 100644 tests/SquidStd.Tests/Crypto/Vfs/VaultKeyDerivationTests.cs diff --git a/src/SquidStd.Crypto/SquidStd.Crypto.csproj b/src/SquidStd.Crypto/SquidStd.Crypto.csproj index fa0605cd..818d1293 100644 --- a/src/SquidStd.Crypto/SquidStd.Crypto.csproj +++ b/src/SquidStd.Crypto/SquidStd.Crypto.csproj @@ -9,6 +9,7 @@ + diff --git a/src/SquidStd.Crypto/Vfs/Data/CryptoVaultOptions.cs b/src/SquidStd.Crypto/Vfs/Data/CryptoVaultOptions.cs new file mode 100644 index 00000000..7d9da9cd --- /dev/null +++ b/src/SquidStd.Crypto/Vfs/Data/CryptoVaultOptions.cs @@ -0,0 +1,17 @@ +namespace SquidStd.Crypto.Vfs.Data; + +/// Tunables for an encrypted vault: chunk size and Argon2id key-derivation cost parameters. +public sealed class CryptoVaultOptions +{ + /// Plaintext chunk size for per-entry AES-GCM, in bytes. + public int ChunkSize { get; init; } = 64 * 1024; + + /// Argon2id memory cost in KiB. + public int Argon2MemoryKib { get; init; } = 65536; + + /// Argon2id iteration (time) cost. + public int Argon2Iterations { get; init; } = 3; + + /// Argon2id parallelism (lanes). + public int Argon2Parallelism { get; init; } = 1; +} diff --git a/src/SquidStd.Crypto/Vfs/Internal/VaultKeyDerivation.cs b/src/SquidStd.Crypto/Vfs/Internal/VaultKeyDerivation.cs new file mode 100644 index 00000000..abb5212a --- /dev/null +++ b/src/SquidStd.Crypto/Vfs/Internal/VaultKeyDerivation.cs @@ -0,0 +1,41 @@ +using System.Text; +using Org.BouncyCastle.Crypto.Digests; +using Org.BouncyCastle.Crypto.Generators; +using Org.BouncyCastle.Crypto.Parameters; +using SquidStd.Crypto.Vfs.Data; + +namespace SquidStd.Crypto.Vfs.Internal; + +/// Derives the vault master key (Argon2id) and per-purpose subkeys (HKDF-SHA256). +internal static class VaultKeyDerivation +{ + public static byte[] DeriveMasterKey(string passphrase, byte[] salt, CryptoVaultOptions options) + { + var parameters = new Argon2Parameters.Builder(Argon2Parameters.Argon2id) + .WithVersion(Argon2Parameters.Version13) + .WithSalt(salt) + .WithIterations(options.Argon2Iterations) + .WithMemoryAsKB(options.Argon2MemoryKib) + .WithParallelism(options.Argon2Parallelism) + .Build(); + + var generator = new Argon2BytesGenerator(); + generator.Init(parameters); + + var key = new byte[32]; + generator.GenerateBytes(Encoding.UTF8.GetBytes(passphrase), key); + + return key; + } + + public static byte[] DeriveSubKey(byte[] masterKey, string label) + { + var hkdf = new HkdfBytesGenerator(new Sha256Digest()); + hkdf.Init(new HkdfParameters(masterKey, null, Encoding.UTF8.GetBytes(label))); + + var subKey = new byte[32]; + hkdf.GenerateBytes(subKey, 0, subKey.Length); + + return subKey; + } +} diff --git a/tests/SquidStd.Tests/Crypto/Vfs/VaultKeyDerivationTests.cs b/tests/SquidStd.Tests/Crypto/Vfs/VaultKeyDerivationTests.cs new file mode 100644 index 00000000..c42a5cab --- /dev/null +++ b/tests/SquidStd.Tests/Crypto/Vfs/VaultKeyDerivationTests.cs @@ -0,0 +1,32 @@ +using SquidStd.Crypto.Vfs.Data; +using SquidStd.Crypto.Vfs.Internal; + +namespace SquidStd.Tests.Crypto.Vfs; + +public class VaultKeyDerivationTests +{ + [Fact] + public void DeriveMasterKey_IsDeterministicForSameSaltAndPassphrase() + { + var options = new CryptoVaultOptions { Argon2MemoryKib = 8192, Argon2Iterations = 1 }; + var salt = new byte[16]; + + var a = VaultKeyDerivation.DeriveMasterKey("pw", salt, options); + var b = VaultKeyDerivation.DeriveMasterKey("pw", salt, options); + var c = VaultKeyDerivation.DeriveMasterKey("other", salt, options); + + Assert.Equal(32, a.Length); + Assert.Equal(a, b); + Assert.NotEqual(a, c); + } + + [Fact] + public void DeriveSubKey_DiffersByLabel() + { + var master = new byte[32]; + Assert.NotEqual( + VaultKeyDerivation.DeriveSubKey(master, "index"), + VaultKeyDerivation.DeriveSubKey(master, "entry:a1b2") + ); + } +} From 68afcc7a900072085237808698b86dc3d4c83a3f Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 17:44:34 +0200 Subject: [PATCH 19/77] feat(crypto): add chunked AES-GCM entry cipher --- .../Vfs/Internal/EntryCipher.cs | 119 ++++++++++++++++++ .../Crypto/Vfs/EntryCipherTests.cs | 40 ++++++ 2 files changed, 159 insertions(+) create mode 100644 src/SquidStd.Crypto/Vfs/Internal/EntryCipher.cs create mode 100644 tests/SquidStd.Tests/Crypto/Vfs/EntryCipherTests.cs diff --git a/src/SquidStd.Crypto/Vfs/Internal/EntryCipher.cs b/src/SquidStd.Crypto/Vfs/Internal/EntryCipher.cs new file mode 100644 index 00000000..f6005edd --- /dev/null +++ b/src/SquidStd.Crypto/Vfs/Internal/EntryCipher.cs @@ -0,0 +1,119 @@ +using System.Buffers.Binary; +using System.Security.Cryptography; + +namespace SquidStd.Crypto.Vfs.Internal; + +/// Streams a payload as chunked AES-GCM records: [len(4) | nonce(12) | ciphertext | tag(16)] per chunk. +internal sealed class EntryCipher : IDisposable +{ + private const int NonceSize = 12; + private const int TagSize = 16; + private readonly AesGcm _aes; + private readonly int _chunkSize; + + public EntryCipher(byte[] key, int chunkSize) + { + _aes = new AesGcm(key, TagSize); + _chunkSize = chunkSize; + } + + public async Task EncryptAsync(Stream plaintext, Stream output, CancellationToken cancellationToken = default) + { + var buffer = new byte[_chunkSize]; + + while (true) + { + var read = await ReadFullAsync(plaintext, buffer, cancellationToken).ConfigureAwait(false); + await WriteRecordAsync(output, buffer.AsMemory(0, read), cancellationToken).ConfigureAwait(false); + + if (read == 0) + { + break; // terminator record written + } + } + } + + public async Task DecryptAsync(Stream input, Stream output, CancellationToken cancellationToken = default) + { + var header = new byte[4 + NonceSize]; + + while (true) + { + await ReadExactAsync(input, header, cancellationToken).ConfigureAwait(false); + var length = BinaryPrimitives.ReadInt32BigEndian(header); + var nonce = header.AsMemory(4, NonceSize); + + var cipher = new byte[length]; + await ReadExactAsync(input, cipher, cancellationToken).ConfigureAwait(false); + var tag = new byte[TagSize]; + await ReadExactAsync(input, tag, cancellationToken).ConfigureAwait(false); + + if (length == 0) + { + _aes.Decrypt(nonce.Span, ReadOnlySpan.Empty, tag, Span.Empty); + break; + } + + var plain = new byte[length]; + _aes.Decrypt(nonce.Span, cipher, tag, plain); + await output.WriteAsync(plain, cancellationToken).ConfigureAwait(false); + } + } + + private async Task WriteRecordAsync(Stream output, ReadOnlyMemory plain, CancellationToken cancellationToken) + { + var nonce = RandomNumberGenerator.GetBytes(NonceSize); + var cipher = new byte[plain.Length]; + var tag = new byte[TagSize]; + _aes.Encrypt(nonce, plain.Span, cipher, tag); + + var header = new byte[4 + NonceSize]; + BinaryPrimitives.WriteInt32BigEndian(header, plain.Length); + nonce.CopyTo(header.AsSpan(4)); + + await output.WriteAsync(header, cancellationToken).ConfigureAwait(false); + await output.WriteAsync(cipher, cancellationToken).ConfigureAwait(false); + await output.WriteAsync(tag, cancellationToken).ConfigureAwait(false); + } + + private static async Task ReadFullAsync(Stream stream, byte[] buffer, CancellationToken cancellationToken) + { + var total = 0; + + while (total < buffer.Length) + { + var read = await stream.ReadAsync(buffer.AsMemory(total), cancellationToken).ConfigureAwait(false); + + if (read == 0) + { + break; + } + + total += read; + } + + return total; + } + + private static async Task ReadExactAsync(Stream stream, byte[] buffer, CancellationToken cancellationToken) + { + var total = 0; + + while (total < buffer.Length) + { + var read = await stream.ReadAsync(buffer.AsMemory(total), cancellationToken).ConfigureAwait(false); + + if (read == 0) + { + throw new EndOfStreamException("Encrypted entry is truncated."); + } + + total += read; + } + } + + public void Dispose() + { + _aes.Dispose(); + } +} diff --git a/tests/SquidStd.Tests/Crypto/Vfs/EntryCipherTests.cs b/tests/SquidStd.Tests/Crypto/Vfs/EntryCipherTests.cs new file mode 100644 index 00000000..1c13e880 --- /dev/null +++ b/tests/SquidStd.Tests/Crypto/Vfs/EntryCipherTests.cs @@ -0,0 +1,40 @@ +using System.Security.Cryptography; +using System.Text; +using SquidStd.Crypto.Vfs.Internal; + +namespace SquidStd.Tests.Crypto.Vfs; + +public class EntryCipherTests +{ + [Fact] + public async Task EncryptThenDecrypt_RoundTripsAcrossChunkBoundaries() + { + var key = RandomNumberGenerator.GetBytes(32); + var payload = RandomNumberGenerator.GetBytes(200_000); // > 3 chunks at 64 KiB + + using var cipher = new EntryCipher(key, chunkSize: 65536); + using var encrypted = new MemoryStream(); + await cipher.EncryptAsync(new MemoryStream(payload), encrypted); + + encrypted.Position = 0; + using var decrypted = new MemoryStream(); + await cipher.DecryptAsync(encrypted, decrypted); + + Assert.Equal(payload, decrypted.ToArray()); + } + + [Fact] + public async Task Decrypt_WrongKey_Throws() + { + var payload = Encoding.UTF8.GetBytes("secret"); + using var enc = new EntryCipher(RandomNumberGenerator.GetBytes(32), 65536); + using var encrypted = new MemoryStream(); + await enc.EncryptAsync(new MemoryStream(payload), encrypted); + + using var wrong = new EntryCipher(RandomNumberGenerator.GetBytes(32), 65536); + encrypted.Position = 0; + await Assert.ThrowsAsync( + () => wrong.DecryptAsync(encrypted, new MemoryStream()) + ); + } +} From 94f46e2a13c5d0cdb1e0224ee948164b20ac635c Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 17:45:27 +0200 Subject: [PATCH 20/77] feat(crypto): add vault header and encrypted-index models --- .../Vfs/Internal/VaultHeader.cs | 19 +++++++ .../Vfs/Internal/VaultIndex.cs | 52 +++++++++++++++++++ .../Crypto/Vfs/VaultHeaderTests.cs | 22 ++++++++ .../Crypto/Vfs/VaultIndexTests.cs | 18 +++++++ 4 files changed, 111 insertions(+) create mode 100644 src/SquidStd.Crypto/Vfs/Internal/VaultHeader.cs create mode 100644 src/SquidStd.Crypto/Vfs/Internal/VaultIndex.cs create mode 100644 tests/SquidStd.Tests/Crypto/Vfs/VaultHeaderTests.cs create mode 100644 tests/SquidStd.Tests/Crypto/Vfs/VaultIndexTests.cs diff --git a/src/SquidStd.Crypto/Vfs/Internal/VaultHeader.cs b/src/SquidStd.Crypto/Vfs/Internal/VaultHeader.cs new file mode 100644 index 00000000..e3c4ad21 --- /dev/null +++ b/src/SquidStd.Crypto/Vfs/Internal/VaultHeader.cs @@ -0,0 +1,19 @@ +using System.Text.Json; + +namespace SquidStd.Crypto.Vfs.Internal; + +/// Cleartext vault header: format magic/version, Argon2id salt + cost params, and chunk size. +internal sealed record VaultHeader( + string Magic, int Version, byte[] Salt, int MemoryKib, int Iterations, int Parallelism, int ChunkSize) +{ + public byte[] Serialize() + { + return JsonSerializer.SerializeToUtf8Bytes(this); + } + + public static VaultHeader Parse(byte[] data) + { + return JsonSerializer.Deserialize(data) + ?? throw new InvalidDataException("Vault header is empty or invalid."); + } +} diff --git a/src/SquidStd.Crypto/Vfs/Internal/VaultIndex.cs b/src/SquidStd.Crypto/Vfs/Internal/VaultIndex.cs new file mode 100644 index 00000000..0fb1bb1c --- /dev/null +++ b/src/SquidStd.Crypto/Vfs/Internal/VaultIndex.cs @@ -0,0 +1,52 @@ +using System.Text.Json; + +namespace SquidStd.Crypto.Vfs.Internal; + +/// A single index record: the opaque backing blob id, plaintext size, and modification time. +internal sealed record VaultIndexEntry(string BlobId, long Size, DateTimeOffset ModifiedUtc); + +/// The decrypted vault index: a map from logical path to its backing blob entry. +internal sealed class VaultIndex +{ + private readonly Dictionary _entries; + + public VaultIndex() + { + _entries = new Dictionary(StringComparer.Ordinal); + } + + private VaultIndex(Dictionary entries) + { + _entries = entries; + } + + public IReadOnlyDictionary Entries => _entries; + + public void Set(string path, VaultIndexEntry entry) + { + _entries[path] = entry; + } + + public bool TryGet(string path, out VaultIndexEntry? entry) + { + return _entries.TryGetValue(path, out entry); + } + + public bool Remove(string path, out VaultIndexEntry? entry) + { + return _entries.Remove(path, out entry); + } + + public byte[] Serialize() + { + return JsonSerializer.SerializeToUtf8Bytes(_entries); + } + + public static VaultIndex Parse(byte[] data) + { + var map = JsonSerializer.Deserialize>(data) + ?? new Dictionary(StringComparer.Ordinal); + + return new VaultIndex(new Dictionary(map, StringComparer.Ordinal)); + } +} diff --git a/tests/SquidStd.Tests/Crypto/Vfs/VaultHeaderTests.cs b/tests/SquidStd.Tests/Crypto/Vfs/VaultHeaderTests.cs new file mode 100644 index 00000000..a354f056 --- /dev/null +++ b/tests/SquidStd.Tests/Crypto/Vfs/VaultHeaderTests.cs @@ -0,0 +1,22 @@ +using SquidStd.Crypto.Vfs.Internal; + +namespace SquidStd.Tests.Crypto.Vfs; + +public class VaultHeaderTests +{ + [Fact] + public void Serialize_Parse_RoundTrips() + { + var header = new VaultHeader("SQVFS1", 1, [1, 2, 3, 4], MemoryKib: 8192, Iterations: 2, Parallelism: 1, ChunkSize: 65536); + + var parsed = VaultHeader.Parse(header.Serialize()); + + Assert.Equal(header.Magic, parsed.Magic); + Assert.Equal(header.Version, parsed.Version); + Assert.Equal(header.Salt, parsed.Salt); + Assert.Equal(header.MemoryKib, parsed.MemoryKib); + Assert.Equal(header.Iterations, parsed.Iterations); + Assert.Equal(header.Parallelism, parsed.Parallelism); + Assert.Equal(header.ChunkSize, parsed.ChunkSize); + } +} diff --git a/tests/SquidStd.Tests/Crypto/Vfs/VaultIndexTests.cs b/tests/SquidStd.Tests/Crypto/Vfs/VaultIndexTests.cs new file mode 100644 index 00000000..a7f0efd9 --- /dev/null +++ b/tests/SquidStd.Tests/Crypto/Vfs/VaultIndexTests.cs @@ -0,0 +1,18 @@ +using SquidStd.Crypto.Vfs.Internal; + +namespace SquidStd.Tests.Crypto.Vfs; + +public class VaultIndexTests +{ + [Fact] + public void Serialize_Parse_RoundTripsEntries() + { + var index = new VaultIndex(); + index.Set("docs/cv.pdf", new VaultIndexEntry("a1b2", 100, DateTimeOffset.UnixEpoch)); + + var parsed = VaultIndex.Parse(index.Serialize()); + + Assert.True(parsed.TryGet("docs/cv.pdf", out var entry)); + Assert.Equal("a1b2", entry!.BlobId); + } +} From 18b70a6366404a2eaac43f499d42b0c2b9c38e8a Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 17:48:41 +0200 Subject: [PATCH 21/77] feat(crypto): add crypto vault unlock/lock and index lifecycle Promote VfsPath to SquidStd.Vfs.Abstractions (public) so the crypto decorator and the VFS providers share identical path normalization. --- src/SquidStd.Crypto/Vfs/Internal/VaultBlob.cs | 40 +++ .../Vfs/Services/CryptoFileSystem.cs | 249 ++++++++++++++++++ src/SquidStd.Vfs.Abstractions/VfsPath.cs | 24 ++ .../Services/InMemoryFileSystem.cs | 2 +- .../Services/PhysicalFileSystem.cs | 2 +- src/SquidStd.Vfs/Services/VfsDirectories.cs | 2 +- src/SquidStd.Vfs/Services/ZipFileSystem.cs | 2 +- .../Crypto/Vfs/CryptoFileSystemLockTests.cs | 43 +++ tests/SquidStd.Tests/Vfs/VfsPathTests.cs | 2 +- 9 files changed, 361 insertions(+), 5 deletions(-) create mode 100644 src/SquidStd.Crypto/Vfs/Internal/VaultBlob.cs create mode 100644 src/SquidStd.Crypto/Vfs/Services/CryptoFileSystem.cs create mode 100644 src/SquidStd.Vfs.Abstractions/VfsPath.cs create mode 100644 tests/SquidStd.Tests/Crypto/Vfs/CryptoFileSystemLockTests.cs diff --git a/src/SquidStd.Crypto/Vfs/Internal/VaultBlob.cs b/src/SquidStd.Crypto/Vfs/Internal/VaultBlob.cs new file mode 100644 index 00000000..515f90d4 --- /dev/null +++ b/src/SquidStd.Crypto/Vfs/Internal/VaultBlob.cs @@ -0,0 +1,40 @@ +using System.Security.Cryptography; + +namespace SquidStd.Crypto.Vfs.Internal; + +/// Single-shot AES-GCM for small blobs (the index): layout [nonce(12) | ciphertext | tag(16)]. +internal static class VaultBlob +{ + private const int NonceSize = 12; + private const int TagSize = 16; + + public static byte[] Encrypt(byte[] key, byte[] plaintext) + { + var nonce = RandomNumberGenerator.GetBytes(NonceSize); + var cipher = new byte[plaintext.Length]; + var tag = new byte[TagSize]; + + using var aes = new AesGcm(key, TagSize); + aes.Encrypt(nonce, plaintext, cipher, tag); + + var output = new byte[NonceSize + cipher.Length + TagSize]; + nonce.CopyTo(output, 0); + cipher.CopyTo(output, NonceSize); + tag.CopyTo(output, NonceSize + cipher.Length); + + return output; + } + + public static byte[] Decrypt(byte[] key, byte[] blob) + { + var nonce = blob.AsSpan(0, NonceSize); + var tag = blob.AsSpan(blob.Length - TagSize, TagSize); + var cipher = blob.AsSpan(NonceSize, blob.Length - NonceSize - TagSize); + + var plaintext = new byte[cipher.Length]; + using var aes = new AesGcm(key, TagSize); + aes.Decrypt(nonce, cipher, tag, plaintext); + + return plaintext; + } +} diff --git a/src/SquidStd.Crypto/Vfs/Services/CryptoFileSystem.cs b/src/SquidStd.Crypto/Vfs/Services/CryptoFileSystem.cs new file mode 100644 index 00000000..eaf0ab90 --- /dev/null +++ b/src/SquidStd.Crypto/Vfs/Services/CryptoFileSystem.cs @@ -0,0 +1,249 @@ +using System.Runtime.CompilerServices; +using System.Security.Cryptography; +using SquidStd.Crypto.Vfs.Data; +using SquidStd.Crypto.Vfs.Internal; +using SquidStd.Vfs.Abstractions.Data; +using SquidStd.Vfs.Abstractions.Interfaces; +using SquidStd.Vfs.Abstractions; + +namespace SquidStd.Crypto.Vfs.Services; + +/// An encrypted, lockable virtual filesystem decorating any inner . +public sealed class CryptoFileSystem : ILockableFileSystem, IDisposable +{ + private const string HeaderPath = "_vfs_header"; + private const string IndexPath = "_vfs_index"; + private readonly IVirtualFileSystem _inner; + private readonly CryptoVaultOptions _options; + private byte[]? _masterKey; + private VaultIndex? _index; + + public CryptoFileSystem(IVirtualFileSystem inner, CryptoVaultOptions? options = null) + { + _inner = inner; + _options = options ?? new CryptoVaultOptions(); + } + + public bool IsUnlocked => _masterKey is not null; + + public void Unlock(string passphrase) + { + ArgumentException.ThrowIfNullOrWhiteSpace(passphrase); + + var headerBytes = _inner.ReadAllBytesAsync(HeaderPath).AsTask().GetAwaiter().GetResult(); + VaultHeader header; + + if (headerBytes is null) + { + header = new VaultHeader( + "SQVFS1", 1, RandomNumberGenerator.GetBytes(16), + _options.Argon2MemoryKib, _options.Argon2Iterations, _options.Argon2Parallelism, _options.ChunkSize + ); + _inner.WriteAllBytesAsync(HeaderPath, header.Serialize()).AsTask().GetAwaiter().GetResult(); + } + else + { + header = VaultHeader.Parse(headerBytes); + } + + var headerOptions = new CryptoVaultOptions + { + ChunkSize = header.ChunkSize, + Argon2MemoryKib = header.MemoryKib, + Argon2Iterations = header.Iterations, + Argon2Parallelism = header.Parallelism + }; + var master = VaultKeyDerivation.DeriveMasterKey(passphrase, header.Salt, headerOptions); + + var indexBytes = _inner.ReadAllBytesAsync(IndexPath).AsTask().GetAwaiter().GetResult(); + _index = indexBytes is null + ? new VaultIndex() + : VaultIndex.Parse(VaultBlob.Decrypt(VaultKeyDerivation.DeriveSubKey(master, "index"), indexBytes)); + + _masterKey = master; + } + + public void Lock() + { + if (_masterKey is null) + { + return; + } + + FlushIndex(); + PruneOrphans(); + CryptographicOperations.ZeroMemory(_masterKey); + _masterKey = null; + _index = null; + } + + public ValueTask ExistsAsync(string path, CancellationToken cancellationToken = default) + { + EnsureUnlocked(); + + return ValueTask.FromResult(_index!.TryGet(VfsPath.Normalize(path), out _)); + } + + public async ValueTask ReadAllBytesAsync(string path, CancellationToken cancellationToken = default) + { + EnsureUnlocked(); + + if (!_index!.TryGet(VfsPath.Normalize(path), out var entry)) + { + return null; + } + + var blob = await _inner.ReadAllBytesAsync(entry!.BlobId, cancellationToken).ConfigureAwait(false) + ?? throw new InvalidDataException($"Backing blob '{entry.BlobId}' is missing."); + + using var cipher = NewCipher(entry.BlobId); + using var input = new MemoryStream(blob); + using var output = new MemoryStream(); + await cipher.DecryptAsync(input, output, cancellationToken).ConfigureAwait(false); + + return output.ToArray(); + } + + public async ValueTask WriteAllBytesAsync(string path, ReadOnlyMemory data, CancellationToken cancellationToken = default) + { + EnsureUnlocked(); + var normalized = VfsPath.Normalize(path); + + var blobId = _index!.TryGet(normalized, out var existing) ? existing!.BlobId : NewBlobId(); + + using var cipher = NewCipher(blobId); + using var input = new MemoryStream(data.ToArray()); + using var output = new MemoryStream(); + await cipher.EncryptAsync(input, output, cancellationToken).ConfigureAwait(false); + + await _inner.WriteAllBytesAsync(blobId, output.ToArray(), cancellationToken).ConfigureAwait(false); + _index.Set(normalized, new VaultIndexEntry(blobId, data.Length, DateTimeOffset.UtcNow)); + } + + public async Task OpenReadAsync(string path, CancellationToken cancellationToken = default) + { + var data = await ReadAllBytesAsync(path, cancellationToken).ConfigureAwait(false) + ?? throw new FileNotFoundException($"No file at '{path}'.", path); + + return new MemoryStream(data, writable: false); + } + + public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) + { + EnsureUnlocked(); + + return Task.FromResult(new VaultWriteStream(this, path)); + } + + public async ValueTask DeleteAsync(string path, CancellationToken cancellationToken = default) + { + EnsureUnlocked(); + + if (!_index!.Remove(VfsPath.Normalize(path), out var entry)) + { + return false; + } + + await _inner.DeleteAsync(entry!.BlobId, cancellationToken).ConfigureAwait(false); + + return true; + } + + public async IAsyncEnumerable ListAsync(string? prefix = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + EnsureUnlocked(); + var normalizedPrefix = string.IsNullOrEmpty(prefix) ? null : VfsPath.Normalize(prefix); + + foreach (var (logicalPath, entry) in _index!.Entries) + { + if (normalizedPrefix is not null && !logicalPath.StartsWith(normalizedPrefix, StringComparison.Ordinal)) + { + continue; + } + + yield return new VfsEntry(logicalPath, entry.Size, entry.ModifiedUtc); + + await Task.CompletedTask; + } + } + + private EntryCipher NewCipher(string blobId) + { + return new EntryCipher(VaultKeyDerivation.DeriveSubKey(_masterKey!, "entry:" + blobId), _options.ChunkSize); + } + + private static string NewBlobId() + { + return Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16)); + } + + private void FlushIndex() + { + var indexKey = VaultKeyDerivation.DeriveSubKey(_masterKey!, "index"); + var encrypted = VaultBlob.Encrypt(indexKey, _index!.Serialize()); + _inner.WriteAllBytesAsync(IndexPath, encrypted).AsTask().GetAwaiter().GetResult(); + } + + private void PruneOrphans() + { + var keep = new HashSet(_index!.Entries.Values.Select(e => e.BlobId), StringComparer.Ordinal) + { + HeaderPath, + IndexPath + }; + + var paths = new List(); + var enumerator = _inner.ListAsync().GetAsyncEnumerator(); + try + { + while (enumerator.MoveNextAsync().AsTask().GetAwaiter().GetResult()) + { + paths.Add(enumerator.Current.Path); + } + } + finally + { + enumerator.DisposeAsync().AsTask().GetAwaiter().GetResult(); + } + + foreach (var path in paths.Where(p => !keep.Contains(p))) + { + _inner.DeleteAsync(path).AsTask().GetAwaiter().GetResult(); + } + } + + private void EnsureUnlocked() + { + if (_masterKey is null) + { + throw new InvalidOperationException("The vault is locked."); + } + } + + private sealed class VaultWriteStream : MemoryStream + { + private readonly CryptoFileSystem _owner; + private readonly string _path; + + public VaultWriteStream(CryptoFileSystem owner, string path) + { + _owner = owner; + _path = path; + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + _owner.WriteAllBytesAsync(_path, ToArray()).AsTask().GetAwaiter().GetResult(); + } + + base.Dispose(disposing); + } + } + + public void Dispose() + { + Lock(); + } +} diff --git a/src/SquidStd.Vfs.Abstractions/VfsPath.cs b/src/SquidStd.Vfs.Abstractions/VfsPath.cs new file mode 100644 index 00000000..4c3db181 --- /dev/null +++ b/src/SquidStd.Vfs.Abstractions/VfsPath.cs @@ -0,0 +1,24 @@ +namespace SquidStd.Vfs.Abstractions; + +/// Normalizes logical VFS paths to forward-slash, root-relative form and rejects traversal. +public static class VfsPath +{ + /// Normalizes a logical path: forward slashes, no leading/empty segments, no ./... + public static string Normalize(string path) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + + var segments = path.Replace('\\', '/') + .Split('/', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + + foreach (var segment in segments) + { + if (segment is "." or "..") + { + throw new ArgumentException($"Path '{path}' must not contain relative segments.", nameof(path)); + } + } + + return string.Join('/', segments); + } +} diff --git a/src/SquidStd.Vfs/Services/InMemoryFileSystem.cs b/src/SquidStd.Vfs/Services/InMemoryFileSystem.cs index c7de6237..f363aa54 100644 --- a/src/SquidStd.Vfs/Services/InMemoryFileSystem.cs +++ b/src/SquidStd.Vfs/Services/InMemoryFileSystem.cs @@ -2,7 +2,7 @@ using System.Runtime.CompilerServices; using SquidStd.Vfs.Abstractions.Data; using SquidStd.Vfs.Abstractions.Interfaces; -using SquidStd.Vfs.Internal; +using SquidStd.Vfs.Abstractions; namespace SquidStd.Vfs.Services; diff --git a/src/SquidStd.Vfs/Services/PhysicalFileSystem.cs b/src/SquidStd.Vfs/Services/PhysicalFileSystem.cs index f0514586..87bdbf8d 100644 --- a/src/SquidStd.Vfs/Services/PhysicalFileSystem.cs +++ b/src/SquidStd.Vfs/Services/PhysicalFileSystem.cs @@ -1,7 +1,7 @@ using System.Runtime.CompilerServices; using SquidStd.Vfs.Abstractions.Data; using SquidStd.Vfs.Abstractions.Interfaces; -using SquidStd.Vfs.Internal; +using SquidStd.Vfs.Abstractions; namespace SquidStd.Vfs.Services; diff --git a/src/SquidStd.Vfs/Services/VfsDirectories.cs b/src/SquidStd.Vfs/Services/VfsDirectories.cs index a049e4ae..f187d2d9 100644 --- a/src/SquidStd.Vfs/Services/VfsDirectories.cs +++ b/src/SquidStd.Vfs/Services/VfsDirectories.cs @@ -1,5 +1,5 @@ using SquidStd.Vfs.Abstractions.Interfaces; -using SquidStd.Vfs.Internal; +using SquidStd.Vfs.Abstractions; namespace SquidStd.Vfs.Services; diff --git a/src/SquidStd.Vfs/Services/ZipFileSystem.cs b/src/SquidStd.Vfs/Services/ZipFileSystem.cs index 294bb142..02067a46 100644 --- a/src/SquidStd.Vfs/Services/ZipFileSystem.cs +++ b/src/SquidStd.Vfs/Services/ZipFileSystem.cs @@ -2,7 +2,7 @@ using System.Runtime.CompilerServices; using SquidStd.Vfs.Abstractions.Data; using SquidStd.Vfs.Abstractions.Interfaces; -using SquidStd.Vfs.Internal; +using SquidStd.Vfs.Abstractions; namespace SquidStd.Vfs.Services; diff --git a/tests/SquidStd.Tests/Crypto/Vfs/CryptoFileSystemLockTests.cs b/tests/SquidStd.Tests/Crypto/Vfs/CryptoFileSystemLockTests.cs new file mode 100644 index 00000000..ec150d8f --- /dev/null +++ b/tests/SquidStd.Tests/Crypto/Vfs/CryptoFileSystemLockTests.cs @@ -0,0 +1,43 @@ +using System.Security.Cryptography; +using SquidStd.Crypto.Vfs.Data; +using SquidStd.Crypto.Vfs.Services; +using SquidStd.Vfs.Services; + +namespace SquidStd.Tests.Crypto.Vfs; + +public class CryptoFileSystemLockTests +{ + private static CryptoVaultOptions FastOptions() + { + return new CryptoVaultOptions { Argon2MemoryKib = 8192, Argon2Iterations = 1 }; + } + + [Fact] + public async Task LockedOperations_Throw_UntilUnlocked() + { + var vault = new CryptoFileSystem(new InMemoryFileSystem(), FastOptions()); + + Assert.False(vault.IsUnlocked); + await Assert.ThrowsAsync( + () => vault.WriteAllBytesAsync("a", new byte[] { 1 }).AsTask() + ); + + vault.Unlock("pw"); + Assert.True(vault.IsUnlocked); + + vault.Lock(); + Assert.False(vault.IsUnlocked); + } + + [Fact] + public void Unlock_WrongPassphrase_OnExistingVault_Throws() + { + var backend = new InMemoryFileSystem(); + var first = new CryptoFileSystem(backend, FastOptions()); + first.Unlock("correct"); + first.Lock(); // writes header + index + + var second = new CryptoFileSystem(backend, FastOptions()); + Assert.Throws(() => second.Unlock("wrong")); + } +} diff --git a/tests/SquidStd.Tests/Vfs/VfsPathTests.cs b/tests/SquidStd.Tests/Vfs/VfsPathTests.cs index a33151ea..903666fd 100644 --- a/tests/SquidStd.Tests/Vfs/VfsPathTests.cs +++ b/tests/SquidStd.Tests/Vfs/VfsPathTests.cs @@ -1,4 +1,4 @@ -using SquidStd.Vfs.Internal; +using SquidStd.Vfs.Abstractions; namespace SquidStd.Tests.Vfs; From e40349d06b110660092050ef9e4d5aa36a17dea0 Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 17:49:28 +0200 Subject: [PATCH 22/77] feat(crypto): implement encrypted per-entry read/write/delete/list --- .../Crypto/Vfs/CryptoFileSystemTests.cs | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 tests/SquidStd.Tests/Crypto/Vfs/CryptoFileSystemTests.cs diff --git a/tests/SquidStd.Tests/Crypto/Vfs/CryptoFileSystemTests.cs b/tests/SquidStd.Tests/Crypto/Vfs/CryptoFileSystemTests.cs new file mode 100644 index 00000000..a3beffa6 --- /dev/null +++ b/tests/SquidStd.Tests/Crypto/Vfs/CryptoFileSystemTests.cs @@ -0,0 +1,88 @@ +using System.Security.Cryptography; +using System.Text; +using SquidStd.Crypto.Vfs.Data; +using SquidStd.Crypto.Vfs.Services; +using SquidStd.Vfs.Services; + +namespace SquidStd.Tests.Crypto.Vfs; + +public class CryptoFileSystemTests +{ + private static CryptoVaultOptions FastOptions() + { + return new CryptoVaultOptions { Argon2MemoryKib = 8192, Argon2Iterations = 1 }; + } + + [Fact] + public async Task Write_Read_List_Delete_RoundTrips() + { + var vault = new CryptoFileSystem(new InMemoryFileSystem(), FastOptions()); + vault.Unlock("pw"); + + await vault.WriteAllBytesAsync("docs/cv.pdf", Encoding.UTF8.GetBytes("hello")); + Assert.Equal("hello", Encoding.UTF8.GetString((await vault.ReadAllBytesAsync("docs/cv.pdf"))!)); + + var paths = new List(); + await foreach (var e in vault.ListAsync()) + { + paths.Add(e.Path); + } + Assert.Equal(["docs/cv.pdf"], paths); + + Assert.True(await vault.DeleteAsync("docs/cv.pdf")); + Assert.Null(await vault.ReadAllBytesAsync("docs/cv.pdf")); + } + + [Fact] + public async Task Persists_AcrossLockUnlock() + { + var backend = new InMemoryFileSystem(); + var a = new CryptoFileSystem(backend, FastOptions()); + a.Unlock("pw"); + await a.WriteAllBytesAsync("a.txt", Encoding.UTF8.GetBytes("v1")); + a.Lock(); + + var b = new CryptoFileSystem(backend, FastOptions()); + b.Unlock("pw"); + Assert.Equal("v1", Encoding.UTF8.GetString((await b.ReadAllBytesAsync("a.txt"))!)); + } + + [Fact] + public async Task BackingStore_DoesNotLeakLogicalNames() + { + var backend = new InMemoryFileSystem(); + var vault = new CryptoFileSystem(backend, FastOptions()); + vault.Unlock("pw"); + await vault.WriteAllBytesAsync("docs/secret.pdf", Encoding.UTF8.GetBytes("x")); + vault.Lock(); + + await foreach (var e in backend.ListAsync()) + { + Assert.DoesNotContain("secret", e.Path, StringComparison.Ordinal); + Assert.DoesNotContain("docs", e.Path, StringComparison.Ordinal); + } + } + + [Fact] + public async Task LargePayload_StreamsThroughChunks() + { + var vault = new CryptoFileSystem(new InMemoryFileSystem(), FastOptions()); + vault.Unlock("pw"); + var payload = RandomNumberGenerator.GetBytes(300_000); + + await vault.WriteAllBytesAsync("big.bin", payload); + Assert.Equal(payload, await vault.ReadAllBytesAsync("big.bin")); + } + + [Fact] + public async Task Update_ReplacesContent() + { + var vault = new CryptoFileSystem(new InMemoryFileSystem(), FastOptions()); + vault.Unlock("pw"); + + await vault.WriteAllBytesAsync("a.txt", Encoding.UTF8.GetBytes("first")); + await vault.WriteAllBytesAsync("a.txt", Encoding.UTF8.GetBytes("second")); + + Assert.Equal("second", Encoding.UTF8.GetString((await vault.ReadAllBytesAsync("a.txt"))!)); + } +} From 750cd351b94bc747568179a74e0f4a4646ed2633 Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 17:50:33 +0200 Subject: [PATCH 23/77] feat(crypto): add RegisterCryptoVault DI extension --- src/SquidStd.Crypto/SquidStd.Crypto.csproj | 1 + .../RegisterCryptoVaultExtensions.cs | 31 +++++++++++++++++++ .../Vfs/RegisterCryptoVaultExtensionsTests.cs | 31 +++++++++++++++++++ 3 files changed, 63 insertions(+) create mode 100644 src/SquidStd.Crypto/Vfs/Extensions/RegisterCryptoVaultExtensions.cs create mode 100644 tests/SquidStd.Tests/Crypto/Vfs/RegisterCryptoVaultExtensionsTests.cs diff --git a/src/SquidStd.Crypto/SquidStd.Crypto.csproj b/src/SquidStd.Crypto/SquidStd.Crypto.csproj index 818d1293..02d903fa 100644 --- a/src/SquidStd.Crypto/SquidStd.Crypto.csproj +++ b/src/SquidStd.Crypto/SquidStd.Crypto.csproj @@ -20,6 +20,7 @@ + diff --git a/src/SquidStd.Crypto/Vfs/Extensions/RegisterCryptoVaultExtensions.cs b/src/SquidStd.Crypto/Vfs/Extensions/RegisterCryptoVaultExtensions.cs new file mode 100644 index 00000000..d87faadd --- /dev/null +++ b/src/SquidStd.Crypto/Vfs/Extensions/RegisterCryptoVaultExtensions.cs @@ -0,0 +1,31 @@ +using DryIoc; +using SquidStd.Crypto.Vfs.Data; +using SquidStd.Crypto.Vfs.Services; +using SquidStd.Vfs.Abstractions.Interfaces; +using SquidStd.Vfs.Services; + +namespace SquidStd.Crypto.Vfs.Extensions; + +/// DryIoc registration helpers for an encrypted single-file vault. +public static class RegisterCryptoVaultExtensions +{ + /// Container that receives the vault registration. + extension(IContainer container) + { + /// + /// Registers an singleton: a crypto vault over a single-file zip at + /// . The consumer calls at runtime. + /// + public IContainer RegisterCryptoVault(string path, CryptoVaultOptions? options = null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + + container.RegisterDelegate( + _ => new CryptoFileSystem(new ZipFileSystem(path), options), + Reuse.Singleton + ); + + return container; + } + } +} diff --git a/tests/SquidStd.Tests/Crypto/Vfs/RegisterCryptoVaultExtensionsTests.cs b/tests/SquidStd.Tests/Crypto/Vfs/RegisterCryptoVaultExtensionsTests.cs new file mode 100644 index 00000000..5c1a5ae1 --- /dev/null +++ b/tests/SquidStd.Tests/Crypto/Vfs/RegisterCryptoVaultExtensionsTests.cs @@ -0,0 +1,31 @@ +using DryIoc; +using SquidStd.Crypto.Vfs.Extensions; +using SquidStd.Vfs.Abstractions.Interfaces; + +namespace SquidStd.Tests.Crypto.Vfs; + +public class RegisterCryptoVaultExtensionsTests +{ + [Fact] + public void RegisterCryptoVault_RegistersLockableSingleton() + { + var path = Path.Combine(Path.GetTempPath(), "squidstd-vault-di-" + Guid.NewGuid().ToString("N") + ".dat"); + + try + { + using var container = new Container(); + container.RegisterCryptoVault(path); + + var vault = container.Resolve(); + Assert.False(vault.IsUnlocked); + Assert.Same(vault, container.Resolve()); + } + finally + { + if (File.Exists(path)) + { + File.Delete(path); + } + } + } +} From eab6e82115997f965ca4e71cec20ef2503a158cc Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 17:53:22 +0200 Subject: [PATCH 24/77] docs(vfs): add SquidStd.Vfs README and crypto vault docs; format module --- src/SquidStd.Crypto/README.md | 43 ++++++++++++++ .../Vfs/Internal/VaultHeader.cs | 9 ++- .../Vfs/Internal/VaultIndex.cs | 4 +- .../Vfs/Services/CryptoFileSystem.cs | 31 ++++++---- src/SquidStd.Vfs/README.md | 56 +++++++++++++++++++ .../Services/InMemoryFileSystem.cs | 12 +++- .../Services/PhysicalFileSystem.cs | 8 ++- src/SquidStd.Vfs/Services/VfsDirectories.cs | 12 ++-- src/SquidStd.Vfs/Services/ZipFileSystem.cs | 8 ++- .../Crypto/Vfs/CryptoFileSystemLockTests.cs | 3 +- .../Crypto/Vfs/CryptoFileSystemTests.cs | 1 + .../Crypto/Vfs/EntryCipherTests.cs | 3 +- .../Crypto/Vfs/VaultHeaderTests.cs | 10 +++- .../Vfs/InMemoryFileSystemTests.cs | 1 + .../Vfs/PhysicalFileSystemTests.cs | 1 + 15 files changed, 170 insertions(+), 32 deletions(-) create mode 100644 src/SquidStd.Vfs/README.md diff --git a/src/SquidStd.Crypto/README.md b/src/SquidStd.Crypto/README.md index 69baf78c..abec6e15 100644 --- a/src/SquidStd.Crypto/README.md +++ b/src/SquidStd.Crypto/README.md @@ -67,3 +67,46 @@ await keyring.LoadAsync(container.Resolve()); read before decrypting; the encrypt and sign stream paths flow straight through. - Passphrases are supplied per operation and never persisted (only passphrase-protected secret blocks are stored). + +## Crypto vault (encrypted virtual filesystem) + +`CryptoFileSystem` is an `ILockableFileSystem` that **decorates any `IVirtualFileSystem`** +(`SquidStd.Vfs`), encrypting file content and names. `Crypto(Zip("vault.dat"))` is a single-file encrypted +vault you unlock with a passphrase, write to, and lock again. + +```csharp +using DryIoc; +using SquidStd.Crypto.Vfs.Extensions; +using SquidStd.Vfs.Abstractions.Interfaces; + +// Single-file vault (crypto over a zip backend), registered as a singleton. +container.RegisterCryptoVault("/var/lib/app/vault.dat"); + +var vault = container.Resolve(); +vault.Unlock("my passphrase"); // derives the key (Argon2id) +await vault.WriteAllBytesAsync("docs/cv.pdf", bytes); +byte[]? back = await vault.ReadAllBytesAsync("docs/cv.pdf"); +vault.Lock(); // flushes, prunes, zeroes the key +``` + +Compose other backends directly: + +```csharp +using SquidStd.Crypto.Vfs.Services; +using SquidStd.Vfs.Services; + +var folderVault = new CryptoFileSystem(new PhysicalFileSystem("/secure/dir")); +``` + +### Notes + +- **Unlock with a passphrase**: a 256-bit key is derived with **Argon2id** (salt + cost params live in the + cleartext header). Per-purpose subkeys are derived with HKDF-SHA256. The passphrase is never persisted. +- **Per-entry encryption**: each file is stored as **chunked AES-GCM** (64 KiB chunks), so large files stream + with bounded memory and tampering/truncation is detected. +- **Encrypted name index**: logical paths and structure live in an encrypted index; backing entries use opaque + ids, so a locked vault leaks neither file names nor layout. +- **Read-write, lockable**: add/update/delete any time while unlocked; `Lock()`/`Dispose()` flush the encrypted + index, prune orphaned blobs, and zero the key with `CryptographicOperations.ZeroMemory`. +- A wrong passphrase fails the index authentication tag → `CryptographicException`; operations on a locked + vault throw `InvalidOperationException`. diff --git a/src/SquidStd.Crypto/Vfs/Internal/VaultHeader.cs b/src/SquidStd.Crypto/Vfs/Internal/VaultHeader.cs index e3c4ad21..289a9867 100644 --- a/src/SquidStd.Crypto/Vfs/Internal/VaultHeader.cs +++ b/src/SquidStd.Crypto/Vfs/Internal/VaultHeader.cs @@ -4,7 +4,14 @@ namespace SquidStd.Crypto.Vfs.Internal; /// Cleartext vault header: format magic/version, Argon2id salt + cost params, and chunk size. internal sealed record VaultHeader( - string Magic, int Version, byte[] Salt, int MemoryKib, int Iterations, int Parallelism, int ChunkSize) + string Magic, + int Version, + byte[] Salt, + int MemoryKib, + int Iterations, + int Parallelism, + int ChunkSize +) { public byte[] Serialize() { diff --git a/src/SquidStd.Crypto/Vfs/Internal/VaultIndex.cs b/src/SquidStd.Crypto/Vfs/Internal/VaultIndex.cs index 0fb1bb1c..d11447d1 100644 --- a/src/SquidStd.Crypto/Vfs/Internal/VaultIndex.cs +++ b/src/SquidStd.Crypto/Vfs/Internal/VaultIndex.cs @@ -10,6 +10,8 @@ internal sealed class VaultIndex { private readonly Dictionary _entries; + public IReadOnlyDictionary Entries => _entries; + public VaultIndex() { _entries = new Dictionary(StringComparer.Ordinal); @@ -20,8 +22,6 @@ private VaultIndex(Dictionary entries) _entries = entries; } - public IReadOnlyDictionary Entries => _entries; - public void Set(string path, VaultIndexEntry entry) { _entries[path] = entry; diff --git a/src/SquidStd.Crypto/Vfs/Services/CryptoFileSystem.cs b/src/SquidStd.Crypto/Vfs/Services/CryptoFileSystem.cs index eaf0ab90..044b6b00 100644 --- a/src/SquidStd.Crypto/Vfs/Services/CryptoFileSystem.cs +++ b/src/SquidStd.Crypto/Vfs/Services/CryptoFileSystem.cs @@ -18,14 +18,14 @@ public sealed class CryptoFileSystem : ILockableFileSystem, IDisposable private byte[]? _masterKey; private VaultIndex? _index; + public bool IsUnlocked => _masterKey is not null; + public CryptoFileSystem(IVirtualFileSystem inner, CryptoVaultOptions? options = null) { _inner = inner; _options = options ?? new CryptoVaultOptions(); } - public bool IsUnlocked => _masterKey is not null; - public void Unlock(string passphrase) { ArgumentException.ThrowIfNullOrWhiteSpace(passphrase); @@ -36,8 +36,13 @@ public void Unlock(string passphrase) if (headerBytes is null) { header = new VaultHeader( - "SQVFS1", 1, RandomNumberGenerator.GetBytes(16), - _options.Argon2MemoryKib, _options.Argon2Iterations, _options.Argon2Parallelism, _options.ChunkSize + "SQVFS1", + 1, + RandomNumberGenerator.GetBytes(16), + _options.Argon2MemoryKib, + _options.Argon2Iterations, + _options.Argon2Parallelism, + _options.ChunkSize ); _inner.WriteAllBytesAsync(HeaderPath, header.Serialize()).AsTask().GetAwaiter().GetResult(); } @@ -104,7 +109,9 @@ public ValueTask ExistsAsync(string path, CancellationToken cancellationTo return output.ToArray(); } - public async ValueTask WriteAllBytesAsync(string path, ReadOnlyMemory data, CancellationToken cancellationToken = default) + public async ValueTask WriteAllBytesAsync( + string path, ReadOnlyMemory data, CancellationToken cancellationToken = default + ) { EnsureUnlocked(); var normalized = VfsPath.Normalize(path); @@ -149,7 +156,9 @@ public async ValueTask DeleteAsync(string path, CancellationToken cancella return true; } - public async IAsyncEnumerable ListAsync(string? prefix = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + public async IAsyncEnumerable ListAsync( + string? prefix = null, [EnumeratorCancellation] CancellationToken cancellationToken = default + ) { EnsureUnlocked(); var normalizedPrefix = string.IsNullOrEmpty(prefix) ? null : VfsPath.Normalize(prefix); @@ -220,6 +229,11 @@ private void EnsureUnlocked() } } + public void Dispose() + { + Lock(); + } + private sealed class VaultWriteStream : MemoryStream { private readonly CryptoFileSystem _owner; @@ -241,9 +255,4 @@ protected override void Dispose(bool disposing) base.Dispose(disposing); } } - - public void Dispose() - { - Lock(); - } } diff --git a/src/SquidStd.Vfs/README.md b/src/SquidStd.Vfs/README.md new file mode 100644 index 00000000..b9b1f462 --- /dev/null +++ b/src/SquidStd.Vfs/README.md @@ -0,0 +1,56 @@ +

SquidStd.Vfs

+ +A path-based **virtual filesystem** abstraction for SquidStd with interchangeable backends. One interface +(`IVirtualFileSystem`) is implemented by real directories, a single zip archive, and an in-memory store — and +decorated by an encrypted vault in `SquidStd.Crypto`. `VfsDirectories` is a VFS-backed analogue of +`DirectoriesConfig`, so named directory layouts work over any backend (a zip or an encrypted container can +stand in for real folders). + +## Install + +```bash +dotnet add package SquidStd.Vfs +``` + +## Usage + +```csharp +using DryIoc; +using SquidStd.Vfs.Abstractions.Interfaces; +using SquidStd.Vfs.Extensions; +using SquidStd.Vfs.Services; + +// Register a backend as IVirtualFileSystem (singleton). +container.RegisterVfs(_ => new PhysicalFileSystem("/var/lib/app/data")); +// or: new ZipFileSystem("/var/lib/app/data.zip") +// or: new InMemoryFileSystem() + +var fs = container.Resolve(); + +await fs.WriteAllBytesAsync("docs/cv.pdf", bytes); +byte[]? data = await fs.ReadAllBytesAsync("docs/cv.pdf"); + +await foreach (var entry in fs.ListAsync("docs")) +{ + Console.WriteLine($"{entry.Path} ({entry.Size} bytes)"); +} + +await fs.DeleteAsync("docs/cv.pdf"); +``` + +### Named directories over any backend + +```csharp +var dirs = new VfsDirectories(fs, ["data", "logs"]); +var target = dirs.Combine("data", "report.csv"); // "data/report.csv" +await fs.WriteAllBytesAsync(target, csvBytes); +``` + +## Providers + +- **`PhysicalFileSystem(root)`** — maps logical paths onto a real directory tree. +- **`ZipFileSystem(path)`** — a single `.zip` archive opened in update mode; `IAsyncDisposable`. +- **`InMemoryFileSystem()`** — ephemeral, in-process; handy for tests and as a decorator target. + +Logical paths are normalized (forward slashes, root-relative) and reject `..` traversal via +`SquidStd.Vfs.Abstractions.VfsPath`. diff --git a/src/SquidStd.Vfs/Services/InMemoryFileSystem.cs b/src/SquidStd.Vfs/Services/InMemoryFileSystem.cs index f363aa54..31030dee 100644 --- a/src/SquidStd.Vfs/Services/InMemoryFileSystem.cs +++ b/src/SquidStd.Vfs/Services/InMemoryFileSystem.cs @@ -9,7 +9,9 @@ namespace SquidStd.Vfs.Services; /// An in-memory virtual filesystem. Ephemeral; useful for tests and as a backend decorator target. public sealed class InMemoryFileSystem : IVirtualFileSystem { - private readonly ConcurrentDictionary _files = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary _files = new( + StringComparer.Ordinal + ); public ValueTask ExistsAsync(string path, CancellationToken cancellationToken = default) { @@ -21,7 +23,9 @@ public ValueTask ExistsAsync(string path, CancellationToken cancellationTo return ValueTask.FromResult(_files.TryGetValue(VfsPath.Normalize(path), out var entry) ? entry.Data : null); } - public ValueTask WriteAllBytesAsync(string path, ReadOnlyMemory data, CancellationToken cancellationToken = default) + public ValueTask WriteAllBytesAsync( + string path, ReadOnlyMemory data, CancellationToken cancellationToken = default + ) { _files[VfsPath.Normalize(path)] = (data.ToArray(), DateTimeOffset.UtcNow); @@ -46,7 +50,9 @@ public ValueTask DeleteAsync(string path, CancellationToken cancellationTo return ValueTask.FromResult(_files.TryRemove(VfsPath.Normalize(path), out _)); } - public async IAsyncEnumerable ListAsync(string? prefix = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + public async IAsyncEnumerable ListAsync( + string? prefix = null, [EnumeratorCancellation] CancellationToken cancellationToken = default + ) { var normalizedPrefix = string.IsNullOrEmpty(prefix) ? null : VfsPath.Normalize(prefix); diff --git a/src/SquidStd.Vfs/Services/PhysicalFileSystem.cs b/src/SquidStd.Vfs/Services/PhysicalFileSystem.cs index 87bdbf8d..0231595d 100644 --- a/src/SquidStd.Vfs/Services/PhysicalFileSystem.cs +++ b/src/SquidStd.Vfs/Services/PhysicalFileSystem.cs @@ -31,7 +31,9 @@ public ValueTask ExistsAsync(string path, CancellationToken cancellationTo : null; } - public async ValueTask WriteAllBytesAsync(string path, ReadOnlyMemory data, CancellationToken cancellationToken = default) + public async ValueTask WriteAllBytesAsync( + string path, ReadOnlyMemory data, CancellationToken cancellationToken = default + ) { var full = Resolve(path); Directory.CreateDirectory(Path.GetDirectoryName(full)!); @@ -72,7 +74,9 @@ public ValueTask DeleteAsync(string path, CancellationToken cancellationTo return ValueTask.FromResult(true); } - public async IAsyncEnumerable ListAsync(string? prefix = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + public async IAsyncEnumerable ListAsync( + string? prefix = null, [EnumeratorCancellation] CancellationToken cancellationToken = default + ) { if (!Directory.Exists(_root)) { diff --git a/src/SquidStd.Vfs/Services/VfsDirectories.cs b/src/SquidStd.Vfs/Services/VfsDirectories.cs index f187d2d9..531a3a57 100644 --- a/src/SquidStd.Vfs/Services/VfsDirectories.cs +++ b/src/SquidStd.Vfs/Services/VfsDirectories.cs @@ -9,6 +9,12 @@ public sealed class VfsDirectories private readonly IVirtualFileSystem _fileSystem; private readonly HashSet _directories; + /// The logical path of a named directory. + public string this[string directoryName] => GetPath(directoryName); + + /// The logical path of a named directory, by enum name. + public string this[Enum directoryType] => GetPath(directoryType.ToString()); + public VfsDirectories(IVirtualFileSystem fileSystem, IReadOnlyCollection directories) { ArgumentNullException.ThrowIfNull(directories); @@ -16,12 +22,6 @@ public VfsDirectories(IVirtualFileSystem fileSystem, IReadOnlyCollection _directories = directories.Select(VfsPath.Normalize).ToHashSet(StringComparer.Ordinal); } - /// The logical path of a named directory. - public string this[string directoryName] => GetPath(directoryName); - - /// The logical path of a named directory, by enum name. - public string this[Enum directoryType] => GetPath(directoryType.ToString()); - /// Resolves the logical path of a named directory; throws when it was not declared. public string GetPath(string directoryName) { diff --git a/src/SquidStd.Vfs/Services/ZipFileSystem.cs b/src/SquidStd.Vfs/Services/ZipFileSystem.cs index 02067a46..742738d0 100644 --- a/src/SquidStd.Vfs/Services/ZipFileSystem.cs +++ b/src/SquidStd.Vfs/Services/ZipFileSystem.cs @@ -40,7 +40,9 @@ public ValueTask ExistsAsync(string path, CancellationToken cancellationTo return buffer.ToArray(); } - public async ValueTask WriteAllBytesAsync(string path, ReadOnlyMemory data, CancellationToken cancellationToken = default) + public async ValueTask WriteAllBytesAsync( + string path, ReadOnlyMemory data, CancellationToken cancellationToken = default + ) { var name = VfsPath.Normalize(path); _archive.GetEntry(name)?.Delete(); @@ -77,7 +79,9 @@ public ValueTask DeleteAsync(string path, CancellationToken cancellationTo return ValueTask.FromResult(true); } - public async IAsyncEnumerable ListAsync(string? prefix = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + public async IAsyncEnumerable ListAsync( + string? prefix = null, [EnumeratorCancellation] CancellationToken cancellationToken = default + ) { var normalizedPrefix = string.IsNullOrEmpty(prefix) ? null : VfsPath.Normalize(prefix); diff --git a/tests/SquidStd.Tests/Crypto/Vfs/CryptoFileSystemLockTests.cs b/tests/SquidStd.Tests/Crypto/Vfs/CryptoFileSystemLockTests.cs index ec150d8f..94353dfb 100644 --- a/tests/SquidStd.Tests/Crypto/Vfs/CryptoFileSystemLockTests.cs +++ b/tests/SquidStd.Tests/Crypto/Vfs/CryptoFileSystemLockTests.cs @@ -18,8 +18,7 @@ public async Task LockedOperations_Throw_UntilUnlocked() var vault = new CryptoFileSystem(new InMemoryFileSystem(), FastOptions()); Assert.False(vault.IsUnlocked); - await Assert.ThrowsAsync( - () => vault.WriteAllBytesAsync("a", new byte[] { 1 }).AsTask() + await Assert.ThrowsAsync(() => vault.WriteAllBytesAsync("a", new byte[] { 1 }).AsTask() ); vault.Unlock("pw"); diff --git a/tests/SquidStd.Tests/Crypto/Vfs/CryptoFileSystemTests.cs b/tests/SquidStd.Tests/Crypto/Vfs/CryptoFileSystemTests.cs index a3beffa6..394594c6 100644 --- a/tests/SquidStd.Tests/Crypto/Vfs/CryptoFileSystemTests.cs +++ b/tests/SquidStd.Tests/Crypto/Vfs/CryptoFileSystemTests.cs @@ -27,6 +27,7 @@ public async Task Write_Read_List_Delete_RoundTrips() { paths.Add(e.Path); } + Assert.Equal(["docs/cv.pdf"], paths); Assert.True(await vault.DeleteAsync("docs/cv.pdf")); diff --git a/tests/SquidStd.Tests/Crypto/Vfs/EntryCipherTests.cs b/tests/SquidStd.Tests/Crypto/Vfs/EntryCipherTests.cs index 1c13e880..fb67926a 100644 --- a/tests/SquidStd.Tests/Crypto/Vfs/EntryCipherTests.cs +++ b/tests/SquidStd.Tests/Crypto/Vfs/EntryCipherTests.cs @@ -33,8 +33,7 @@ public async Task Decrypt_WrongKey_Throws() using var wrong = new EntryCipher(RandomNumberGenerator.GetBytes(32), 65536); encrypted.Position = 0; - await Assert.ThrowsAsync( - () => wrong.DecryptAsync(encrypted, new MemoryStream()) + await Assert.ThrowsAsync(() => wrong.DecryptAsync(encrypted, new MemoryStream()) ); } } diff --git a/tests/SquidStd.Tests/Crypto/Vfs/VaultHeaderTests.cs b/tests/SquidStd.Tests/Crypto/Vfs/VaultHeaderTests.cs index a354f056..93ad01c0 100644 --- a/tests/SquidStd.Tests/Crypto/Vfs/VaultHeaderTests.cs +++ b/tests/SquidStd.Tests/Crypto/Vfs/VaultHeaderTests.cs @@ -7,7 +7,15 @@ public class VaultHeaderTests [Fact] public void Serialize_Parse_RoundTrips() { - var header = new VaultHeader("SQVFS1", 1, [1, 2, 3, 4], MemoryKib: 8192, Iterations: 2, Parallelism: 1, ChunkSize: 65536); + var header = new VaultHeader( + "SQVFS1", + 1, + [1, 2, 3, 4], + MemoryKib: 8192, + Iterations: 2, + Parallelism: 1, + ChunkSize: 65536 + ); var parsed = VaultHeader.Parse(header.Serialize()); diff --git a/tests/SquidStd.Tests/Vfs/InMemoryFileSystemTests.cs b/tests/SquidStd.Tests/Vfs/InMemoryFileSystemTests.cs index db3e538c..55cf8c16 100644 --- a/tests/SquidStd.Tests/Vfs/InMemoryFileSystemTests.cs +++ b/tests/SquidStd.Tests/Vfs/InMemoryFileSystemTests.cs @@ -19,6 +19,7 @@ public async Task Write_Read_List_Delete_RoundTrips() { paths.Add(e.Path); } + Assert.Equal(["a/b.txt"], paths); Assert.True(await fs.DeleteAsync("a/b.txt")); diff --git a/tests/SquidStd.Tests/Vfs/PhysicalFileSystemTests.cs b/tests/SquidStd.Tests/Vfs/PhysicalFileSystemTests.cs index 9d9a918b..f3efa01e 100644 --- a/tests/SquidStd.Tests/Vfs/PhysicalFileSystemTests.cs +++ b/tests/SquidStd.Tests/Vfs/PhysicalFileSystemTests.cs @@ -23,6 +23,7 @@ public async Task Write_Read_List_Delete_RoundTrips() { entries.Add(e.Path); } + Assert.Contains("docs/cv.pdf", entries); Assert.True(await fs.DeleteAsync("docs/cv.pdf")); From 8a4fbdeed110eb50306957de8660b1ee4471031c Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 18:57:40 +0200 Subject: [PATCH 25/77] feat(secrets): add ListNamesAsync to ISecretStore and FileSecretStore --- .../Interfaces/Secrets/ISecretStore.cs | 8 ++++ .../Services/Storage/FileSecretStore.cs | 18 +++++++ .../FileSecretStoreEnumerationTests.cs | 47 +++++++++++++++++++ 3 files changed, 73 insertions(+) create mode 100644 tests/SquidStd.Tests/Security/FileSecretStoreEnumerationTests.cs diff --git a/src/SquidStd.Core/Interfaces/Secrets/ISecretStore.cs b/src/SquidStd.Core/Interfaces/Secrets/ISecretStore.cs index 199f23ff..f499b937 100644 --- a/src/SquidStd.Core/Interfaces/Secrets/ISecretStore.cs +++ b/src/SquidStd.Core/Interfaces/Secrets/ISecretStore.cs @@ -36,4 +36,12 @@ public interface ISecretStore /// The secret value. /// Token used to cancel the operation. ValueTask SetAsync(string name, string value, CancellationToken cancellationToken = default); + + /// + /// Enumerates stored secret names, optionally filtered by prefix. + /// + /// Optional name prefix; null or empty returns all names. + /// Token used to cancel the enumeration. + /// An async sequence of secret names. + IAsyncEnumerable ListNamesAsync(string? prefix = null, CancellationToken cancellationToken = default); } diff --git a/src/SquidStd.Services.Core/Services/Storage/FileSecretStore.cs b/src/SquidStd.Services.Core/Services/Storage/FileSecretStore.cs index 6ec3513a..e497ea51 100644 --- a/src/SquidStd.Services.Core/Services/Storage/FileSecretStore.cs +++ b/src/SquidStd.Services.Core/Services/Storage/FileSecretStore.cs @@ -1,3 +1,4 @@ +using System.Runtime.CompilerServices; using System.Text; using SquidStd.Core.Data.Storage; using SquidStd.Core.Interfaces.Secrets; @@ -67,6 +68,23 @@ public async ValueTask SetAsync(string name, string value, CancellationToken can await _storageService.SaveAsync(ToStorageKey(name), protectedData, cancellationToken); } + /// + public async IAsyncEnumerable ListNamesAsync( + string? prefix = null, [EnumeratorCancellation] CancellationToken cancellationToken = default + ) + { + const string suffix = ".secret"; + var keyPrefix = string.IsNullOrEmpty(prefix) ? null : prefix; + + await foreach (var key in _storageService.ListKeysAsync(keyPrefix, cancellationToken).ConfigureAwait(false)) + { + if (key.EndsWith(suffix, StringComparison.Ordinal)) + { + yield return key[..^suffix.Length]; + } + } + } + private static string ToStorageKey(string name) { ArgumentException.ThrowIfNullOrWhiteSpace(name); diff --git a/tests/SquidStd.Tests/Security/FileSecretStoreEnumerationTests.cs b/tests/SquidStd.Tests/Security/FileSecretStoreEnumerationTests.cs new file mode 100644 index 00000000..d4887644 --- /dev/null +++ b/tests/SquidStd.Tests/Security/FileSecretStoreEnumerationTests.cs @@ -0,0 +1,47 @@ +using System.Security.Cryptography; +using SquidStd.Core.Data.Storage; +using SquidStd.Services.Core.Services.Storage; +using SquidStd.Tests.Support; + +namespace SquidStd.Tests.Security; + +public class FileSecretStoreEnumerationTests +{ + [Fact] + public async Task ListNamesAsync_ReturnsSetNames_AndHonoursPrefix() + { + using var temp = new TempDirectory(); + var variableName = "SQUIDSTD_TEST_SECRET_ENUM_KEY"; + var previous = Environment.GetEnvironmentVariable(variableName); + + try + { + Environment.SetEnvironmentVariable(variableName, Convert.ToBase64String(RandomNumberGenerator.GetBytes(32))); + var config = new SecretsConfig { RootDirectory = temp.Path, KeyEnvironmentVariable = variableName }; + var store = new FileSecretStore(config, new AesGcmSecretProtector(config)); + + await store.SetAsync("db/main", "a"); + await store.SetAsync("db/replica", "b"); + await store.SetAsync("api/key", "c"); + + var all = new List(); + await foreach (var name in store.ListNamesAsync()) + { + all.Add(name); + } + + var db = new List(); + await foreach (var name in store.ListNamesAsync("db/")) + { + db.Add(name); + } + + Assert.Equal(["api/key", "db/main", "db/replica"], all.OrderBy(x => x).ToArray()); + Assert.Equal(["db/main", "db/replica"], db.OrderBy(x => x).ToArray()); + } + finally + { + Environment.SetEnvironmentVariable(variableName, previous); + } + } +} From a225409d0a919c26c19f81864e774a5c17c4f0cd Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 18:59:00 +0200 Subject: [PATCH 26/77] build(secrets): scaffold SquidStd.Secrets.Aws module --- SquidStd.slnx | 1 + .../Data/AwsSecretsManagerOptions.cs | 13 +++++ .../Data/KmsSecretProtectorOptions.cs | 13 +++++ .../Internal/AwsClientFactory.cs | 52 +++++++++++++++++++ .../SquidStd.Secrets.Aws.csproj | 21 ++++++++ tests/SquidStd.Tests/SquidStd.Tests.csproj | 1 + 6 files changed, 101 insertions(+) create mode 100644 src/SquidStd.Secrets.Aws/Data/AwsSecretsManagerOptions.cs create mode 100644 src/SquidStd.Secrets.Aws/Data/KmsSecretProtectorOptions.cs create mode 100644 src/SquidStd.Secrets.Aws/Internal/AwsClientFactory.cs create mode 100644 src/SquidStd.Secrets.Aws/SquidStd.Secrets.Aws.csproj diff --git a/SquidStd.slnx b/SquidStd.slnx index a850a11c..3b463679 100644 --- a/SquidStd.slnx +++ b/SquidStd.slnx @@ -40,6 +40,7 @@ + diff --git a/src/SquidStd.Secrets.Aws/Data/AwsSecretsManagerOptions.cs b/src/SquidStd.Secrets.Aws/Data/AwsSecretsManagerOptions.cs new file mode 100644 index 00000000..a2520094 --- /dev/null +++ b/src/SquidStd.Secrets.Aws/Data/AwsSecretsManagerOptions.cs @@ -0,0 +1,13 @@ +using SquidStd.Aws.Abstractions.Data.Config; + +namespace SquidStd.Secrets.Aws.Data; + +/// Options for the AWS Secrets Manager store: AWS connection and an optional name prefix. +public sealed class AwsSecretsManagerOptions +{ + /// AWS-style connection config (region/credentials/endpoint override). + public AwsConfigEntry Aws { get; set; } = new(); + + /// Optional prefix applied to and stripped from secret names (e.g. myapp/). + public string? NamePrefix { get; set; } +} diff --git a/src/SquidStd.Secrets.Aws/Data/KmsSecretProtectorOptions.cs b/src/SquidStd.Secrets.Aws/Data/KmsSecretProtectorOptions.cs new file mode 100644 index 00000000..e8205ad9 --- /dev/null +++ b/src/SquidStd.Secrets.Aws/Data/KmsSecretProtectorOptions.cs @@ -0,0 +1,13 @@ +using SquidStd.Aws.Abstractions.Data.Config; + +namespace SquidStd.Secrets.Aws.Data; + +/// Options for the KMS-backed secret protector: AWS connection and the KMS key id/alias. +public sealed class KmsSecretProtectorOptions +{ + /// AWS-style connection config (region/credentials/endpoint override). + public AwsConfigEntry Aws { get; set; } = new(); + + /// The KMS key id or alias used to generate data keys (e.g. alias/app). + public string KeyId { get; set; } = string.Empty; +} diff --git a/src/SquidStd.Secrets.Aws/Internal/AwsClientFactory.cs b/src/SquidStd.Secrets.Aws/Internal/AwsClientFactory.cs new file mode 100644 index 00000000..eaaf6c8c --- /dev/null +++ b/src/SquidStd.Secrets.Aws/Internal/AwsClientFactory.cs @@ -0,0 +1,52 @@ +using Amazon; +using Amazon.KeyManagementService; +using Amazon.Runtime; +using Amazon.SecretsManager; +using SquidStd.Aws.Abstractions.Data.Config; + +namespace SquidStd.Secrets.Aws.Internal; + +/// Builds AWS SDK clients/credentials from a shared . +internal static class AwsClientFactory +{ + public static AWSCredentials Credentials(AwsConfigEntry aws) + { + if (!string.IsNullOrWhiteSpace(aws.AccessKey) && !string.IsNullOrWhiteSpace(aws.SecretKey)) + { + return string.IsNullOrWhiteSpace(aws.SessionToken) + ? new BasicAWSCredentials(aws.AccessKey, aws.SecretKey) + : new SessionAWSCredentials(aws.AccessKey, aws.SecretKey, aws.SessionToken); + } + + return FallbackCredentialsFactory.GetCredentials(); + } + + public static AmazonKeyManagementServiceConfig KmsConfig(AwsConfigEntry aws) + { + var config = new AmazonKeyManagementServiceConfig(); + Configure(config, aws); + + return config; + } + + public static AmazonSecretsManagerConfig SecretsManagerConfig(AwsConfigEntry aws) + { + var config = new AmazonSecretsManagerConfig(); + Configure(config, aws); + + return config; + } + + private static void Configure(ClientConfig config, AwsConfigEntry aws) + { + if (!string.IsNullOrWhiteSpace(aws.ServiceUrl)) + { + config.ServiceURL = aws.ServiceUrl; + config.AuthenticationRegion = aws.Region; + } + else + { + config.RegionEndpoint = RegionEndpoint.GetBySystemName(aws.Region); + } + } +} diff --git a/src/SquidStd.Secrets.Aws/SquidStd.Secrets.Aws.csproj b/src/SquidStd.Secrets.Aws/SquidStd.Secrets.Aws.csproj new file mode 100644 index 00000000..e9517927 --- /dev/null +++ b/src/SquidStd.Secrets.Aws/SquidStd.Secrets.Aws.csproj @@ -0,0 +1,21 @@ + + + + net10.0 + enable + enable + true + + + + + + + + + + + + + + diff --git a/tests/SquidStd.Tests/SquidStd.Tests.csproj b/tests/SquidStd.Tests/SquidStd.Tests.csproj index 0c7805d4..a2127844 100644 --- a/tests/SquidStd.Tests/SquidStd.Tests.csproj +++ b/tests/SquidStd.Tests/SquidStd.Tests.csproj @@ -73,6 +73,7 @@ + From 8478e4878f327dc2c35343b81f9a6970cd14a30d Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 19:02:21 +0200 Subject: [PATCH 27/77] feat(secrets): add KMS envelope-encryption secret protector - KmsSecretProtector implements ISecretProtector via AWS KMS data keys - KmsEnvelope frames [wrappedKeyLen|wrappedKey|nonce|tag|ciphertext] with AES-GCM - zero the plaintext data key after sealing/opening; dispose owns KMS client - LocalStack-backed integration test round-trips a >4KB payload --- .../Internal/KmsEnvelope.cs | 60 +++++++++++++++ .../Services/KmsSecretProtector.cs | 75 +++++++++++++++++++ .../Secrets/Aws/KmsSecretProtectorTests.cs | 44 +++++++++++ .../Aws/Support/LocalStackSecretsFixture.cs | 35 +++++++++ 4 files changed, 214 insertions(+) create mode 100644 src/SquidStd.Secrets.Aws/Internal/KmsEnvelope.cs create mode 100644 src/SquidStd.Secrets.Aws/Services/KmsSecretProtector.cs create mode 100644 tests/SquidStd.Tests/Secrets/Aws/KmsSecretProtectorTests.cs create mode 100644 tests/SquidStd.Tests/Secrets/Aws/Support/LocalStackSecretsFixture.cs diff --git a/src/SquidStd.Secrets.Aws/Internal/KmsEnvelope.cs b/src/SquidStd.Secrets.Aws/Internal/KmsEnvelope.cs new file mode 100644 index 00000000..5728b7f3 --- /dev/null +++ b/src/SquidStd.Secrets.Aws/Internal/KmsEnvelope.cs @@ -0,0 +1,60 @@ +using System.Buffers.Binary; +using System.Security.Cryptography; + +namespace SquidStd.Secrets.Aws.Internal; + +/// Frames an AES-GCM envelope: [int32 wrappedKeyLen | wrappedKey | nonce(12) | tag(16) | ciphertext]. +internal static class KmsEnvelope +{ + private const int NonceSize = 12; + private const int TagSize = 16; + + public static byte[] Seal(byte[] dataKey, byte[] wrappedKey, byte[] plaintext) + { + var nonce = RandomNumberGenerator.GetBytes(NonceSize); + var cipher = new byte[plaintext.Length]; + var tag = new byte[TagSize]; + + using (var aes = new AesGcm(dataKey, TagSize)) + { + aes.Encrypt(nonce, plaintext, cipher, tag); + } + + var output = new byte[4 + wrappedKey.Length + NonceSize + TagSize + cipher.Length]; + var span = output.AsSpan(); + BinaryPrimitives.WriteInt32BigEndian(span, wrappedKey.Length); + wrappedKey.CopyTo(span[4..]); + nonce.CopyTo(span[(4 + wrappedKey.Length)..]); + tag.CopyTo(span[(4 + wrappedKey.Length + NonceSize)..]); + cipher.CopyTo(span[(4 + wrappedKey.Length + NonceSize + TagSize)..]); + + return output; + } + + public static byte[] ReadWrappedKey(byte[] blob) + { + var wrappedLen = BinaryPrimitives.ReadInt32BigEndian(blob); + + if (wrappedLen <= 0 || 4 + wrappedLen + NonceSize + TagSize > blob.Length) + { + throw new ArgumentException("Malformed KMS envelope.", nameof(blob)); + } + + return blob[4..(4 + wrappedLen)]; + } + + public static byte[] Open(byte[] dataKey, byte[] blob) + { + var wrappedLen = BinaryPrimitives.ReadInt32BigEndian(blob); + var offset = 4 + wrappedLen; + var nonce = blob.AsSpan(offset, NonceSize); + var tag = blob.AsSpan(offset + NonceSize, TagSize); + var cipher = blob.AsSpan(offset + NonceSize + TagSize); + + var plaintext = new byte[cipher.Length]; + using var aes = new AesGcm(dataKey, TagSize); + aes.Decrypt(nonce, cipher, tag, plaintext); + + return plaintext; + } +} diff --git a/src/SquidStd.Secrets.Aws/Services/KmsSecretProtector.cs b/src/SquidStd.Secrets.Aws/Services/KmsSecretProtector.cs new file mode 100644 index 00000000..cff331ff --- /dev/null +++ b/src/SquidStd.Secrets.Aws/Services/KmsSecretProtector.cs @@ -0,0 +1,75 @@ +using System.Security.Cryptography; +using Amazon.KeyManagementService; +using Amazon.KeyManagementService.Model; +using SquidStd.Core.Interfaces.Secrets; +using SquidStd.Secrets.Aws.Data; +using SquidStd.Secrets.Aws.Internal; + +namespace SquidStd.Secrets.Aws.Services; + +/// An that envelope-encrypts with AWS KMS data keys. +public sealed class KmsSecretProtector : ISecretProtector, IDisposable +{ + private readonly AmazonKeyManagementServiceClient _kms; + private readonly string _keyId; + + public KmsSecretProtector(KmsSecretProtectorOptions options) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentException.ThrowIfNullOrWhiteSpace(options.KeyId); + + _keyId = options.KeyId; + _kms = new AmazonKeyManagementServiceClient( + AwsClientFactory.Credentials(options.Aws), AwsClientFactory.KmsConfig(options.Aws) + ); + } + + /// + public byte[] Protect(byte[] plaintext) + { + ArgumentNullException.ThrowIfNull(plaintext); + + var generated = _kms.GenerateDataKeyAsync( + new GenerateDataKeyRequest { KeyId = _keyId, KeySpec = DataKeySpec.AES_256 } + ).GetAwaiter().GetResult(); + + var dataKey = generated.Plaintext.ToArray(); + + try + { + return KmsEnvelope.Seal(dataKey, generated.CiphertextBlob.ToArray(), plaintext); + } + finally + { + CryptographicOperations.ZeroMemory(dataKey); + } + } + + /// + public byte[] Unprotect(byte[] protectedData) + { + ArgumentNullException.ThrowIfNull(protectedData); + + var wrappedKey = KmsEnvelope.ReadWrappedKey(protectedData); + var decrypted = _kms.DecryptAsync( + new DecryptRequest { CiphertextBlob = new MemoryStream(wrappedKey) } + ).GetAwaiter().GetResult(); + + var dataKey = decrypted.Plaintext.ToArray(); + + try + { + return KmsEnvelope.Open(dataKey, protectedData); + } + finally + { + CryptographicOperations.ZeroMemory(dataKey); + } + } + + /// + public void Dispose() + { + _kms.Dispose(); + } +} diff --git a/tests/SquidStd.Tests/Secrets/Aws/KmsSecretProtectorTests.cs b/tests/SquidStd.Tests/Secrets/Aws/KmsSecretProtectorTests.cs new file mode 100644 index 00000000..5fb3106b --- /dev/null +++ b/tests/SquidStd.Tests/Secrets/Aws/KmsSecretProtectorTests.cs @@ -0,0 +1,44 @@ +using System.Text; +using Amazon.KeyManagementService; +using Amazon.KeyManagementService.Model; +using SquidStd.Secrets.Aws.Data; +using SquidStd.Secrets.Aws.Services; +using SquidStd.Tests.Secrets.Aws.Support; + +namespace SquidStd.Tests.Secrets.Aws; + +[Collection(LocalStackSecretsCollection.Name)] +public class KmsSecretProtectorTests +{ + private readonly LocalStackSecretsFixture _localStack; + + public KmsSecretProtectorTests(LocalStackSecretsFixture localStack) + { + _localStack = localStack; + } + + [Fact] + public async Task Protect_Unprotect_RoundTripsLargePayload_WithoutPlaintext() + { + var keyId = await CreateKeyAsync(); + var protector = new KmsSecretProtector(new KmsSecretProtectorOptions { Aws = _localStack.Aws, KeyId = keyId }); + var plaintext = Encoding.UTF8.GetBytes(new string('s', 10_000)); // > 4 KB → exercises envelope + + var blob = protector.Protect(plaintext); + var roundTrip = protector.Unprotect(blob); + + Assert.Equal(plaintext, roundTrip); + Assert.DoesNotContain("ssss", Encoding.UTF8.GetString(blob), StringComparison.Ordinal); + } + + private async Task CreateKeyAsync() + { + using var kms = new AmazonKeyManagementServiceClient( + new Amazon.Runtime.BasicAWSCredentials("test", "test"), + new AmazonKeyManagementServiceConfig { ServiceURL = _localStack.Aws.ServiceUrl, AuthenticationRegion = "us-east-1" } + ); + var created = await kms.CreateKeyAsync(new CreateKeyRequest()); + + return created.KeyMetadata.KeyId; + } +} diff --git a/tests/SquidStd.Tests/Secrets/Aws/Support/LocalStackSecretsFixture.cs b/tests/SquidStd.Tests/Secrets/Aws/Support/LocalStackSecretsFixture.cs new file mode 100644 index 00000000..ed07dbe2 --- /dev/null +++ b/tests/SquidStd.Tests/Secrets/Aws/Support/LocalStackSecretsFixture.cs @@ -0,0 +1,35 @@ +using SquidStd.Aws.Abstractions.Data.Config; +using Testcontainers.LocalStack; + +namespace SquidStd.Tests.Secrets.Aws.Support; + +/// Starts a LocalStack container (KMS + Secrets Manager) once for the collection. +public sealed class LocalStackSecretsFixture : IAsyncLifetime +{ + private readonly LocalStackContainer _container = + new LocalStackBuilder().WithImage("localstack/localstack:3").Build(); + + public AwsConfigEntry Aws => new() + { + Region = "us-east-1", + AccessKey = "test", + SecretKey = "test", + ServiceUrl = _container.GetConnectionString() + }; + + public Task DisposeAsync() + { + return _container.DisposeAsync().AsTask(); + } + + public Task InitializeAsync() + { + return _container.StartAsync(); + } +} + +[CollectionDefinition(Name)] +public sealed class LocalStackSecretsCollection : ICollectionFixture +{ + public const string Name = "LocalStackSecrets"; +} From 0a2cffdeaf22bd559609a861a753d96066913586 Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 19:04:23 +0200 Subject: [PATCH 28/77] feat(secrets): add AWS Secrets Manager store - AwsSecretsManagerStore implements ISecretStore over AWS Secrets Manager - Set upserts via PutSecretValue, falling back to CreateSecret when absent - Delete gates on existence so a missing secret returns false on both real AWS and LocalStack - ListNamesAsync paginates ListSecrets and strips the configured NamePrefix - store disposes its Secrets Manager client - LocalStack-backed integration test covers the full set/get/exists/list/delete cycle --- .../Services/AwsSecretsManagerStore.cs | 141 ++++++++++++++++++ .../Aws/AwsSecretsManagerStoreTests.cs | 43 ++++++ 2 files changed, 184 insertions(+) create mode 100644 src/SquidStd.Secrets.Aws/Services/AwsSecretsManagerStore.cs create mode 100644 tests/SquidStd.Tests/Secrets/Aws/AwsSecretsManagerStoreTests.cs diff --git a/src/SquidStd.Secrets.Aws/Services/AwsSecretsManagerStore.cs b/src/SquidStd.Secrets.Aws/Services/AwsSecretsManagerStore.cs new file mode 100644 index 00000000..6af97fb1 --- /dev/null +++ b/src/SquidStd.Secrets.Aws/Services/AwsSecretsManagerStore.cs @@ -0,0 +1,141 @@ +using System.Runtime.CompilerServices; +using Amazon.SecretsManager; +using Amazon.SecretsManager.Model; +using SquidStd.Core.Interfaces.Secrets; +using SquidStd.Secrets.Aws.Data; +using SquidStd.Secrets.Aws.Internal; + +namespace SquidStd.Secrets.Aws.Services; + +/// An backed by AWS Secrets Manager. +public sealed class AwsSecretsManagerStore : ISecretStore, IDisposable +{ + private readonly AmazonSecretsManagerClient _client; + private readonly string _prefix; + + public AwsSecretsManagerStore(AwsSecretsManagerOptions options) + { + ArgumentNullException.ThrowIfNull(options); + + _prefix = options.NamePrefix ?? string.Empty; + _client = new AmazonSecretsManagerClient( + AwsClientFactory.Credentials(options.Aws), AwsClientFactory.SecretsManagerConfig(options.Aws) + ); + } + + /// + public async ValueTask DeleteAsync(string name, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(name); + + if (!await ExistsAsync(name, cancellationToken).ConfigureAwait(false)) + { + return false; + } + + try + { + await _client.DeleteSecretAsync( + new DeleteSecretRequest { SecretId = _prefix + name, ForceDeleteWithoutRecovery = true }, + cancellationToken + ).ConfigureAwait(false); + + return true; + } + catch (ResourceNotFoundException) + { + return false; + } + } + + /// + public async ValueTask ExistsAsync(string name, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(name); + + try + { + await _client.DescribeSecretAsync( + new DescribeSecretRequest { SecretId = _prefix + name }, cancellationToken + ).ConfigureAwait(false); + + return true; + } + catch (ResourceNotFoundException) + { + return false; + } + } + + /// + public async ValueTask GetAsync(string name, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(name); + + try + { + var response = await _client.GetSecretValueAsync( + new GetSecretValueRequest { SecretId = _prefix + name }, cancellationToken + ).ConfigureAwait(false); + + return response.SecretString; + } + catch (ResourceNotFoundException) + { + return null; + } + } + + /// + public async ValueTask SetAsync(string name, string value, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(name); + ArgumentNullException.ThrowIfNull(value); + + var secretId = _prefix + name; + + try + { + await _client.PutSecretValueAsync( + new PutSecretValueRequest { SecretId = secretId, SecretString = value }, cancellationToken + ).ConfigureAwait(false); + } + catch (ResourceNotFoundException) + { + await _client.CreateSecretAsync( + new CreateSecretRequest { Name = secretId, SecretString = value }, cancellationToken + ).ConfigureAwait(false); + } + } + + /// + public async IAsyncEnumerable ListNamesAsync(string? prefix = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var fullPrefix = _prefix + (prefix ?? string.Empty); + string? token = null; + + do + { + var response = await _client.ListSecretsAsync( + new ListSecretsRequest { NextToken = token, MaxResults = 100 }, cancellationToken + ).ConfigureAwait(false); + + foreach (var secret in response.SecretList) + { + if (secret.Name.StartsWith(fullPrefix, StringComparison.Ordinal)) + { + yield return secret.Name[_prefix.Length..]; + } + } + + token = response.NextToken; + } + while (!string.IsNullOrEmpty(token)); + } + + /// + public void Dispose() + { + _client.Dispose(); + } +} diff --git a/tests/SquidStd.Tests/Secrets/Aws/AwsSecretsManagerStoreTests.cs b/tests/SquidStd.Tests/Secrets/Aws/AwsSecretsManagerStoreTests.cs new file mode 100644 index 00000000..a7f74aef --- /dev/null +++ b/tests/SquidStd.Tests/Secrets/Aws/AwsSecretsManagerStoreTests.cs @@ -0,0 +1,43 @@ +using SquidStd.Secrets.Aws.Data; +using SquidStd.Secrets.Aws.Services; +using SquidStd.Tests.Secrets.Aws.Support; + +namespace SquidStd.Tests.Secrets.Aws; + +[Collection(LocalStackSecretsCollection.Name)] +public class AwsSecretsManagerStoreTests +{ + private readonly LocalStackSecretsFixture _localStack; + + public AwsSecretsManagerStoreTests(LocalStackSecretsFixture localStack) + { + _localStack = localStack; + } + + [Fact] + public async Task Set_Get_Exists_List_Delete_RoundTrips() + { + var prefix = "test-" + Guid.NewGuid().ToString("N") + "/"; + var store = new AwsSecretsManagerStore(new AwsSecretsManagerOptions { Aws = _localStack.Aws, NamePrefix = prefix }); + + Assert.Null(await store.GetAsync("db/main")); + Assert.False(await store.ExistsAsync("db/main")); + + await store.SetAsync("db/main", "secret-1"); + await store.SetAsync("db/main", "secret-2"); // update existing + + Assert.Equal("secret-2", await store.GetAsync("db/main")); + Assert.True(await store.ExistsAsync("db/main")); + + var names = new List(); + await foreach (var name in store.ListNamesAsync()) + { + names.Add(name); + } + Assert.Contains("db/main", names); + + Assert.True(await store.DeleteAsync("db/main")); + Assert.Null(await store.GetAsync("db/main")); + Assert.False(await store.DeleteAsync("db/main")); + } +} From ed5a6d79075cc852d744afd4e38cd8c5a0f7c5cc Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 19:05:01 +0200 Subject: [PATCH 29/77] feat(secrets): add RegisterKmsSecretProtector and RegisterAwsSecretsManagerStore DI extensions - RegisterKmsSecretProtector wires KmsSecretProtector as the singleton ISecretProtector - RegisterAwsSecretsManagerStore wires AwsSecretsManagerStore as the singleton ISecretStore - both use the Action configure pattern over settable options DTOs - offline tests verify singleton registration without any AWS call --- ...egisterAwsSecretsManagerStoreExtensions.cs | 26 ++++++++++++++++ .../RegisterKmsSecretProtectorExtensions.cs | 26 ++++++++++++++++ .../Aws/RegisterSecretsAwsExtensionsTests.cs | 31 +++++++++++++++++++ 3 files changed, 83 insertions(+) create mode 100644 src/SquidStd.Secrets.Aws/Extensions/RegisterAwsSecretsManagerStoreExtensions.cs create mode 100644 src/SquidStd.Secrets.Aws/Extensions/RegisterKmsSecretProtectorExtensions.cs create mode 100644 tests/SquidStd.Tests/Secrets/Aws/RegisterSecretsAwsExtensionsTests.cs diff --git a/src/SquidStd.Secrets.Aws/Extensions/RegisterAwsSecretsManagerStoreExtensions.cs b/src/SquidStd.Secrets.Aws/Extensions/RegisterAwsSecretsManagerStoreExtensions.cs new file mode 100644 index 00000000..152c604d --- /dev/null +++ b/src/SquidStd.Secrets.Aws/Extensions/RegisterAwsSecretsManagerStoreExtensions.cs @@ -0,0 +1,26 @@ +using DryIoc; +using SquidStd.Core.Interfaces.Secrets; +using SquidStd.Secrets.Aws.Data; +using SquidStd.Secrets.Aws.Services; + +namespace SquidStd.Secrets.Aws.Extensions; + +/// DryIoc registration helper for the AWS Secrets Manager store. +public static class RegisterAwsSecretsManagerStoreExtensions +{ + /// Container that receives the store registration. + extension(IContainer container) + { + /// Registers as the singleton . + public IContainer RegisterAwsSecretsManagerStore(Action configure) + { + ArgumentNullException.ThrowIfNull(configure); + + var options = new AwsSecretsManagerOptions(); + configure(options); + container.RegisterInstance(new AwsSecretsManagerStore(options)); + + return container; + } + } +} diff --git a/src/SquidStd.Secrets.Aws/Extensions/RegisterKmsSecretProtectorExtensions.cs b/src/SquidStd.Secrets.Aws/Extensions/RegisterKmsSecretProtectorExtensions.cs new file mode 100644 index 00000000..6d6da03a --- /dev/null +++ b/src/SquidStd.Secrets.Aws/Extensions/RegisterKmsSecretProtectorExtensions.cs @@ -0,0 +1,26 @@ +using DryIoc; +using SquidStd.Core.Interfaces.Secrets; +using SquidStd.Secrets.Aws.Data; +using SquidStd.Secrets.Aws.Services; + +namespace SquidStd.Secrets.Aws.Extensions; + +/// DryIoc registration helper for the KMS secret protector. +public static class RegisterKmsSecretProtectorExtensions +{ + /// Container that receives the protector registration. + extension(IContainer container) + { + /// Registers as the singleton . + public IContainer RegisterKmsSecretProtector(Action configure) + { + ArgumentNullException.ThrowIfNull(configure); + + var options = new KmsSecretProtectorOptions(); + configure(options); + container.RegisterInstance(new KmsSecretProtector(options)); + + return container; + } + } +} diff --git a/tests/SquidStd.Tests/Secrets/Aws/RegisterSecretsAwsExtensionsTests.cs b/tests/SquidStd.Tests/Secrets/Aws/RegisterSecretsAwsExtensionsTests.cs new file mode 100644 index 00000000..0dcae842 --- /dev/null +++ b/tests/SquidStd.Tests/Secrets/Aws/RegisterSecretsAwsExtensionsTests.cs @@ -0,0 +1,31 @@ +using DryIoc; +using SquidStd.Core.Interfaces.Secrets; +using SquidStd.Secrets.Aws.Extensions; +using SquidStd.Secrets.Aws.Services; + +namespace SquidStd.Tests.Secrets.Aws; + +public class RegisterSecretsAwsExtensionsTests +{ + [Fact] + public void RegisterKmsSecretProtector_RegistersSingletonProtector() + { + using var container = new Container(); + container.RegisterKmsSecretProtector(o => o.KeyId = "alias/test"); + + var protector = container.Resolve(); + Assert.IsType(protector); + Assert.Same(protector, container.Resolve()); + } + + [Fact] + public void RegisterAwsSecretsManagerStore_RegistersSingletonStore() + { + using var container = new Container(); + container.RegisterAwsSecretsManagerStore(o => { }); + + var store = container.Resolve(); + Assert.IsType(store); + Assert.Same(store, container.Resolve()); + } +} From 0670eb3bed6eb2eda51b6e190c4ebf2a348d69a6 Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 19:07:11 +0200 Subject: [PATCH 30/77] docs(secrets): add SquidStd.Secrets.Aws README and format module - README documents the two AWS secret seams, registration, usage, and notes - apply SquidStd cleanup profile across the new module and changed secret files --- src/SquidStd.Secrets.Aws/README.md | 64 +++++++++++++++++++ .../Services/AwsSecretsManagerStore.cs | 47 +++++++++----- .../Services/KmsSecretProtector.cs | 15 +++-- .../Aws/AwsSecretsManagerStoreTests.cs | 1 + .../Secrets/Aws/KmsSecretProtectorTests.cs | 3 +- 5 files changed, 107 insertions(+), 23 deletions(-) create mode 100644 src/SquidStd.Secrets.Aws/README.md diff --git a/src/SquidStd.Secrets.Aws/README.md b/src/SquidStd.Secrets.Aws/README.md new file mode 100644 index 00000000..01ba2da8 --- /dev/null +++ b/src/SquidStd.Secrets.Aws/README.md @@ -0,0 +1,64 @@ +

SquidStd.Secrets.Aws

+ +AWS adapters for the two SquidStd secret seams: an `ISecretProtector` that envelope-encrypts payloads +with **AWS KMS** data keys, and an `ISecretStore` backed by **AWS Secrets Manager**. Drop either into the +container in place of the file-backed defaults to push key custody and secret storage into AWS-managed +services, keeping the rest of the application unchanged. + +## Install + +```bash +dotnet add package SquidStd.Secrets.Aws +``` + +## Usage + +```csharp +using SquidStd.Secrets.Aws.Extensions; + +// KMS-backed ISecretProtector (envelope encryption) +container.RegisterKmsSecretProtector(o => +{ + o.KeyId = "alias/my-app"; // KMS key id, ARN, or alias + o.Aws.Region = "eu-west-1"; +}); + +// Secrets Manager-backed ISecretStore +container.RegisterAwsSecretsManagerStore(o => +{ + o.NamePrefix = "my-app/"; // optional logical namespace + o.Aws.Region = "eu-west-1"; +}); +``` + +Both seams are then resolved like any other SquidStd secret service: + +```csharp +var protector = container.Resolve(); +byte[] sealed = protector.Protect(payload); +byte[] restored = protector.Unprotect(sealed); + +var store = container.Resolve(); +await store.SetAsync("db/main", connectionString); +string? value = await store.GetAsync("db/main"); +await foreach (var name in store.ListNamesAsync("db/")) { /* ... */ } +``` + +## Notes + +- **Envelope encryption** — `KmsSecretProtector` calls `GenerateDataKey` per `Protect`, encrypts the payload + locally with AES-256-GCM, and frames the KMS-wrapped data key alongside the ciphertext. The plaintext + data key is zeroed immediately after use; `Unprotect` calls `Decrypt` to unwrap it. KMS never sees the + payload, and large payloads are not bound by the 4 KB KMS direct-encrypt limit. +- **NamePrefix** — `AwsSecretsManagerStore` prepends `NamePrefix` to every secret id, giving each + application its own namespace inside a shared account. `ListNamesAsync` strips the prefix so callers + always see logical names. `Delete` returns `false` for a missing secret. +- **Credentials** — when `Aws.AccessKey` / `Aws.SecretKey` are omitted, the AWS SDK default credential + chain is used (environment, shared profile, EC2/ECS role). Set `Aws.ServiceUrl` to target LocalStack + or another compatible endpoint. +- **Tested against LocalStack** — the KMS and Secrets Manager adapters are covered by integration tests + running on a `localstack/localstack` container. + +## License + +MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). diff --git a/src/SquidStd.Secrets.Aws/Services/AwsSecretsManagerStore.cs b/src/SquidStd.Secrets.Aws/Services/AwsSecretsManagerStore.cs index 6af97fb1..f1f48c04 100644 --- a/src/SquidStd.Secrets.Aws/Services/AwsSecretsManagerStore.cs +++ b/src/SquidStd.Secrets.Aws/Services/AwsSecretsManagerStore.cs @@ -19,7 +19,8 @@ public AwsSecretsManagerStore(AwsSecretsManagerOptions options) _prefix = options.NamePrefix ?? string.Empty; _client = new AmazonSecretsManagerClient( - AwsClientFactory.Credentials(options.Aws), AwsClientFactory.SecretsManagerConfig(options.Aws) + AwsClientFactory.Credentials(options.Aws), + AwsClientFactory.SecretsManagerConfig(options.Aws) ); } @@ -36,9 +37,10 @@ public async ValueTask DeleteAsync(string name, CancellationToken cancella try { await _client.DeleteSecretAsync( - new DeleteSecretRequest { SecretId = _prefix + name, ForceDeleteWithoutRecovery = true }, - cancellationToken - ).ConfigureAwait(false); + new DeleteSecretRequest { SecretId = _prefix + name, ForceDeleteWithoutRecovery = true }, + cancellationToken + ) + .ConfigureAwait(false); return true; } @@ -56,8 +58,10 @@ public async ValueTask ExistsAsync(string name, CancellationToken cancella try { await _client.DescribeSecretAsync( - new DescribeSecretRequest { SecretId = _prefix + name }, cancellationToken - ).ConfigureAwait(false); + new DescribeSecretRequest { SecretId = _prefix + name }, + cancellationToken + ) + .ConfigureAwait(false); return true; } @@ -75,8 +79,10 @@ await _client.DescribeSecretAsync( try { var response = await _client.GetSecretValueAsync( - new GetSecretValueRequest { SecretId = _prefix + name }, cancellationToken - ).ConfigureAwait(false); + new GetSecretValueRequest { SecretId = _prefix + name }, + cancellationToken + ) + .ConfigureAwait(false); return response.SecretString; } @@ -97,19 +103,25 @@ public async ValueTask SetAsync(string name, string value, CancellationToken can try { await _client.PutSecretValueAsync( - new PutSecretValueRequest { SecretId = secretId, SecretString = value }, cancellationToken - ).ConfigureAwait(false); + new PutSecretValueRequest { SecretId = secretId, SecretString = value }, + cancellationToken + ) + .ConfigureAwait(false); } catch (ResourceNotFoundException) { await _client.CreateSecretAsync( - new CreateSecretRequest { Name = secretId, SecretString = value }, cancellationToken - ).ConfigureAwait(false); + new CreateSecretRequest { Name = secretId, SecretString = value }, + cancellationToken + ) + .ConfigureAwait(false); } } /// - public async IAsyncEnumerable ListNamesAsync(string? prefix = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + public async IAsyncEnumerable ListNamesAsync( + string? prefix = null, [EnumeratorCancellation] CancellationToken cancellationToken = default + ) { var fullPrefix = _prefix + (prefix ?? string.Empty); string? token = null; @@ -117,8 +129,10 @@ public async IAsyncEnumerable ListNamesAsync(string? prefix = null, [Enu do { var response = await _client.ListSecretsAsync( - new ListSecretsRequest { NextToken = token, MaxResults = 100 }, cancellationToken - ).ConfigureAwait(false); + new ListSecretsRequest { NextToken = token, MaxResults = 100 }, + cancellationToken + ) + .ConfigureAwait(false); foreach (var secret in response.SecretList) { @@ -129,8 +143,7 @@ public async IAsyncEnumerable ListNamesAsync(string? prefix = null, [Enu } token = response.NextToken; - } - while (!string.IsNullOrEmpty(token)); + } while (!string.IsNullOrEmpty(token)); } /// diff --git a/src/SquidStd.Secrets.Aws/Services/KmsSecretProtector.cs b/src/SquidStd.Secrets.Aws/Services/KmsSecretProtector.cs index cff331ff..32388dac 100644 --- a/src/SquidStd.Secrets.Aws/Services/KmsSecretProtector.cs +++ b/src/SquidStd.Secrets.Aws/Services/KmsSecretProtector.cs @@ -20,7 +20,8 @@ public KmsSecretProtector(KmsSecretProtectorOptions options) _keyId = options.KeyId; _kms = new AmazonKeyManagementServiceClient( - AwsClientFactory.Credentials(options.Aws), AwsClientFactory.KmsConfig(options.Aws) + AwsClientFactory.Credentials(options.Aws), + AwsClientFactory.KmsConfig(options.Aws) ); } @@ -30,8 +31,10 @@ public byte[] Protect(byte[] plaintext) ArgumentNullException.ThrowIfNull(plaintext); var generated = _kms.GenerateDataKeyAsync( - new GenerateDataKeyRequest { KeyId = _keyId, KeySpec = DataKeySpec.AES_256 } - ).GetAwaiter().GetResult(); + new GenerateDataKeyRequest { KeyId = _keyId, KeySpec = DataKeySpec.AES_256 } + ) + .GetAwaiter() + .GetResult(); var dataKey = generated.Plaintext.ToArray(); @@ -52,8 +55,10 @@ public byte[] Unprotect(byte[] protectedData) var wrappedKey = KmsEnvelope.ReadWrappedKey(protectedData); var decrypted = _kms.DecryptAsync( - new DecryptRequest { CiphertextBlob = new MemoryStream(wrappedKey) } - ).GetAwaiter().GetResult(); + new DecryptRequest { CiphertextBlob = new MemoryStream(wrappedKey) } + ) + .GetAwaiter() + .GetResult(); var dataKey = decrypted.Plaintext.ToArray(); diff --git a/tests/SquidStd.Tests/Secrets/Aws/AwsSecretsManagerStoreTests.cs b/tests/SquidStd.Tests/Secrets/Aws/AwsSecretsManagerStoreTests.cs index a7f74aef..e3a148cd 100644 --- a/tests/SquidStd.Tests/Secrets/Aws/AwsSecretsManagerStoreTests.cs +++ b/tests/SquidStd.Tests/Secrets/Aws/AwsSecretsManagerStoreTests.cs @@ -34,6 +34,7 @@ public async Task Set_Get_Exists_List_Delete_RoundTrips() { names.Add(name); } + Assert.Contains("db/main", names); Assert.True(await store.DeleteAsync("db/main")); diff --git a/tests/SquidStd.Tests/Secrets/Aws/KmsSecretProtectorTests.cs b/tests/SquidStd.Tests/Secrets/Aws/KmsSecretProtectorTests.cs index 5fb3106b..86319911 100644 --- a/tests/SquidStd.Tests/Secrets/Aws/KmsSecretProtectorTests.cs +++ b/tests/SquidStd.Tests/Secrets/Aws/KmsSecretProtectorTests.cs @@ -35,7 +35,8 @@ private async Task CreateKeyAsync() { using var kms = new AmazonKeyManagementServiceClient( new Amazon.Runtime.BasicAWSCredentials("test", "test"), - new AmazonKeyManagementServiceConfig { ServiceURL = _localStack.Aws.ServiceUrl, AuthenticationRegion = "us-east-1" } + new AmazonKeyManagementServiceConfig + { ServiceURL = _localStack.Aws.ServiceUrl, AuthenticationRegion = "us-east-1" } ); var created = await kms.CreateKeyAsync(new CreateKeyRequest()); From 1479482500826da91e206740f4f872bf0f8a7485 Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 20:41:04 +0200 Subject: [PATCH 31/77] docs: group package list by domain in README - split the single flat package table into 13 domain-grouped subtables - add the missing Actors, Crypto, Secrets.Aws, Vfs and Vfs.Abstractions packages --- README.md | 95 +++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 81 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 3b420dd6..af215a2c 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,8 @@ block until cancellation for long-running hosts. ## Packages +### Core & hosting + | Package | Description | Links | |---------|-------------|-------| | `SquidStd.Core` | Foundational contracts & utilities (config, event bus, jobs, metrics, serialization, YAML/JSON, Serilog sink). | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Core/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Core.svg)](https://www.nuget.org/packages/SquidStd.Core/) | @@ -73,38 +75,103 @@ block until cancellation for long-running hosts. | `SquidStd.Generators` | Roslyn source generators for event listener, service, config, worker, and Lua registration helpers. | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Generators/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Generators.svg)](https://www.nuget.org/packages/SquidStd.Generators/) | | `SquidStd.Services.Core` | Concrete services: config, event bus, jobs, timer/cron scheduler, dispatcher, metrics, health checks, secrets. | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Services.Core/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Services.Core.svg)](https://www.nuget.org/packages/SquidStd.Services.Core/) | | `SquidStd.AspNetCore` | ASP.NET Core host integration for the SquidStd service stack. | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.AspNetCore/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.AspNetCore.svg)](https://www.nuget.org/packages/SquidStd.AspNetCore/) | +| `SquidStd.Plugin.Abstractions` | Plugin contracts (`ISquidStdPlugin`, metadata, context). | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Plugin.Abstractions/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Plugin.Abstractions.svg)](https://www.nuget.org/packages/SquidStd.Plugin.Abstractions/) | + +### Networking + +| Package | Description | Links | +|---------|-------------|-------| | `SquidStd.Network` | TCP/UDP servers & clients, sessions, framing/middleware pipeline, span readers/writers. | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Network/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Network.svg)](https://www.nuget.org/packages/SquidStd.Network/) | + +### Messaging & actors + +| Package | Description | Links | +|---------|-------------|-------| +| `SquidStd.Messaging.Abstractions` | Messaging contracts (`IMessageQueue`, `IQueueProvider`, serializer/metrics, listeners). | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Messaging.Abstractions/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Messaging.Abstractions.svg)](https://www.nuget.org/packages/SquidStd.Messaging.Abstractions/) | +| `SquidStd.Messaging` | In-memory messaging transport (`AddInMemoryMessaging`). | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Messaging/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Messaging.svg)](https://www.nuget.org/packages/SquidStd.Messaging/) | +| `SquidStd.Messaging.RabbitMq` | RabbitMQ messaging transport (`AddRabbitMqMessaging`). | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Messaging.RabbitMq/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Messaging.RabbitMq.svg)](https://www.nuget.org/packages/SquidStd.Messaging.RabbitMq/) | +| `SquidStd.Messaging.Sqs` | AWS SQS/SNS transport: `IQueueProvider` over SQS (redrive→DLQ) and `ITopicProvider` via SNS+SQS fan-out (`AddSqsMessaging`). | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Messaging.Sqs/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Messaging.Sqs.svg)](https://www.nuget.org/packages/SquidStd.Messaging.Sqs/) | +| `SquidStd.Actors` | Ordered, single-threaded, lock-free per-entity message processing on TPL Dataflow (`Actor`, `TellAsync`/`AskAsync`). | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Actors/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Actors.svg)](https://www.nuget.org/packages/SquidStd.Actors/) | + +### Persistence & database + +| Package | Description | Links | +|---------|-------------|-------| | `SquidStd.Persistence.Abstractions` | Binary-persistence contracts (`IEntityStore`, `IPersistenceService`, journal/snapshot/registry, `PersistenceConfig`). | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Persistence.Abstractions/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Persistence.Abstractions.svg)](https://www.nuget.org/packages/SquidStd.Persistence.Abstractions/) | | `SquidStd.Persistence` | In-memory entity store with durable binary snapshot + journal (WAL); serializer-agnostic engine. | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Persistence/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Persistence.svg)](https://www.nuget.org/packages/SquidStd.Persistence/) | | `SquidStd.Persistence.MessagePack` | MessagePack-backed binary `IDataSerializer` for SquidStd.Persistence (recommended default). | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Persistence.MessagePack/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Persistence.MessagePack.svg)](https://www.nuget.org/packages/SquidStd.Persistence.MessagePack/) | -| `SquidStd.Plugin.Abstractions` | Plugin contracts (`ISquidStdPlugin`, metadata, context). | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Plugin.Abstractions/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Plugin.Abstractions.svg)](https://www.nuget.org/packages/SquidStd.Plugin.Abstractions/) | | `SquidStd.Database.Abstractions` | Provider-agnostic data-access contracts (`IDataAccess`, `BaseEntity`, `PagedResultData`). | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Database.Abstractions/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Database.Abstractions.svg)](https://www.nuget.org/packages/SquidStd.Database.Abstractions/) | | `SquidStd.Database` | FreeSql-backed data access (CRUD/bulk/paging, URI connection strings, ZLinq helpers). | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Database/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Database.svg)](https://www.nuget.org/packages/SquidStd.Database/) | -| `SquidStd.Messaging.Abstractions` | Messaging contracts (`IMessageQueue`, `IQueueProvider`, serializer/metrics, listeners). | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Messaging.Abstractions/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Messaging.Abstractions.svg)](https://www.nuget.org/packages/SquidStd.Messaging.Abstractions/) | -| `SquidStd.Messaging` | In-memory messaging transport (`AddInMemoryMessaging`). | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Messaging/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Messaging.svg)](https://www.nuget.org/packages/SquidStd.Messaging/) | -| `SquidStd.Messaging.RabbitMq` | RabbitMQ messaging transport (`AddRabbitMqMessaging`). | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Messaging.RabbitMq/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Messaging.RabbitMq.svg)](https://www.nuget.org/packages/SquidStd.Messaging.RabbitMq/) | -| `SquidStd.Messaging.Sqs` | AWS SQS/SNS transport: `IQueueProvider` over SQS (redrive→DLQ) and `ITopicProvider` via SNS+SQS fan-out (`AddSqsMessaging`). | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Messaging.Sqs/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Messaging.Sqs.svg)](https://www.nuget.org/packages/SquidStd.Messaging.Sqs/) | -| `SquidStd.Aws.Abstractions` | Shared AWS connection config (`AwsConfigEntry`: region, credentials, endpoint override) for AWS-SDK providers. | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Aws.Abstractions/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Aws.Abstractions.svg)](https://www.nuget.org/packages/SquidStd.Aws.Abstractions/) | -| `SquidStd.Telemetry.Abstractions` | Shared telemetry config (`TelemetryOptions`, OTLP protocol, `SquidStd.*` ActivitySource convention). | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Telemetry.Abstractions/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Telemetry.Abstractions.svg)](https://www.nuget.org/packages/SquidStd.Telemetry.Abstractions/) | -| `SquidStd.Telemetry.OpenTelemetry` | OpenTelemetry tracing + metrics export (OTLP/console), standard instrumentation, metrics-snapshot bridge (`AddSquidStdTelemetry`). | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Telemetry.OpenTelemetry/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Telemetry.OpenTelemetry.svg)](https://www.nuget.org/packages/SquidStd.Telemetry.OpenTelemetry/) | + +### Caching + +| Package | Description | Links | +|---------|-------------|-------| | `SquidStd.Caching.Abstractions` | Caching contracts (`ICacheService`, `ICacheProvider`, `CacheService` facade, metrics, connection string). | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Caching.Abstractions/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Caching.Abstractions.svg)](https://www.nuget.org/packages/SquidStd.Caching.Abstractions/) | | `SquidStd.Caching` | In-memory cache backend (`AddInMemoryCache`). | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Caching/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Caching.svg)](https://www.nuget.org/packages/SquidStd.Caching/) | | `SquidStd.Caching.Redis` | Redis cache backend (`AddRedisCache`). | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Caching.Redis/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Caching.Redis.svg)](https://www.nuget.org/packages/SquidStd.Caching.Redis/) | + +### Storage & virtual filesystem + +| Package | Description | Links | +|---------|-------------|-------| | `SquidStd.Storage.Abstractions` | Storage contracts (`IStorageService`, `IObjectStorageService`, `StorageConfig`, `ListKeysAsync`). | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Storage.Abstractions/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Storage.Abstractions.svg)](https://www.nuget.org/packages/SquidStd.Storage.Abstractions/) | | `SquidStd.Storage` | Local file storage backend (`AddFileStorage`). | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Storage/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Storage.svg)](https://www.nuget.org/packages/SquidStd.Storage/) | | `SquidStd.Storage.S3` | S3/MinIO storage backend (`AddS3Storage`). | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Storage.S3/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Storage.S3.svg)](https://www.nuget.org/packages/SquidStd.Storage.S3/) | -| `SquidStd.Scripting.Lua` | Lua scripting engine with attribute-based modules and event bridging. | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Scripting.Lua/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Scripting.Lua.svg)](https://www.nuget.org/packages/SquidStd.Scripting.Lua/) | -| `SquidStd.Templating` | Scriban templating with a named-template registry and `templates/*.tmpl` auto-load (`AddTemplating`). | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Templating/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Templating.svg)](https://www.nuget.org/packages/SquidStd.Templating/) | -| `SquidStd.Workers.Abstractions` | Worker/manager shared contracts (`JobRequest`, `WorkerHeartbeat`, `WorkerInfo`, `WorkerChannels`). | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Workers.Abstractions/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Workers.Abstractions.svg)](https://www.nuget.org/packages/SquidStd.Workers.Abstractions/) | -| `SquidStd.Workers` | Worker runtime: consume jobs, dispatch to `IJobHandler`s, publish heartbeats (`AddWorkers`). | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Workers/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Workers.svg)](https://www.nuget.org/packages/SquidStd.Workers/) | -| `SquidStd.Workers.Manager` | Job enqueue, heartbeat registry, offline sweep, opt-in ASP.NET endpoints (`AddWorkerManager`). | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Workers.Manager/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Workers.Manager.svg)](https://www.nuget.org/packages/SquidStd.Workers.Manager/) | -| `SquidStd.Templates` | `dotnet new` templates for scaffolding SquidStd projects (host, ASP.NET, worker, manager). | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Templates/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Templates.svg)](https://www.nuget.org/packages/SquidStd.Templates/) | +| `SquidStd.Vfs.Abstractions` | Virtual filesystem contracts (`IVirtualFileSystem`, `ILockableFileSystem`, `VfsPath`). | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Vfs.Abstractions/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Vfs.Abstractions.svg)](https://www.nuget.org/packages/SquidStd.Vfs.Abstractions/) | +| `SquidStd.Vfs` | Virtual filesystem providers — physical, in-memory, and zip — plus `VfsDirectories`, a VFS-backed `DirectoriesConfig`. | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Vfs/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Vfs.svg)](https://www.nuget.org/packages/SquidStd.Vfs/) | + +### Security — crypto & secrets + +| Package | Description | Links | +|---------|-------------|-------| +| `SquidStd.Crypto` | OpenPGP key management/operations over an indexed keyring (`SquidStd.Crypto.Pgp`, `RegisterPgp`) plus the encrypted VFS vault decorator (Argon2id + per-entry AES-GCM). | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Crypto/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Crypto.svg)](https://www.nuget.org/packages/SquidStd.Crypto/) | +| `SquidStd.Secrets.Aws` | AWS adapters for the secret seams — KMS envelope `ISecretProtector` and Secrets Manager `ISecretStore`. | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Secrets.Aws/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Secrets.Aws.svg)](https://www.nuget.org/packages/SquidStd.Secrets.Aws/) | + +### Search + +| Package | Description | Links | +|---------|-------------|-------| | `SquidStd.Search.Abstractions` | Search/indexing contracts (`IIndexableEntity`, `[SearchIndex]`, `ISearchService`). | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Search.Abstractions/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Search.Abstractions.svg)](https://www.nuget.org/packages/SquidStd.Search.Abstractions/) | | `SquidStd.Search.Elasticsearch` | Elasticsearch indexing + constrained LINQ query provider (`AddElasticsearch`). | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Search.Elasticsearch/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Search.Elasticsearch.svg)](https://www.nuget.org/packages/SquidStd.Search.Elasticsearch/) | + +### Mail + +| Package | Description | Links | +|---------|-------------|-------| | `SquidStd.Mail.Abstractions` | Mail contracts (`MailMessage`, `MailReceivedEvent`, `IMailReader`, `MailOptions`). | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Mail.Abstractions/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Mail.Abstractions.svg)](https://www.nuget.org/packages/SquidStd.Mail.Abstractions/) | | `SquidStd.Mail.MailKit` | IMAP/POP3 mail poller that publishes `MailReceivedEvent` (`AddMail`). | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Mail.MailKit/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Mail.MailKit.svg)](https://www.nuget.org/packages/SquidStd.Mail.MailKit/) | | `SquidStd.Mail.Queue` | Outbound mail send queue over the messaging queue (`AddMailQueue`, `IMailQueue`). | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Mail.Queue/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Mail.Queue.svg)](https://www.nuget.org/packages/SquidStd.Mail.Queue/) | +### Workers + +| Package | Description | Links | +|---------|-------------|-------| +| `SquidStd.Workers.Abstractions` | Worker/manager shared contracts (`JobRequest`, `WorkerHeartbeat`, `WorkerInfo`, `WorkerChannels`). | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Workers.Abstractions/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Workers.Abstractions.svg)](https://www.nuget.org/packages/SquidStd.Workers.Abstractions/) | +| `SquidStd.Workers` | Worker runtime: consume jobs, dispatch to `IJobHandler`s, publish heartbeats (`AddWorkers`). | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Workers/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Workers.svg)](https://www.nuget.org/packages/SquidStd.Workers/) | +| `SquidStd.Workers.Manager` | Job enqueue, heartbeat registry, offline sweep, opt-in ASP.NET endpoints (`AddWorkerManager`). | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Workers.Manager/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Workers.Manager.svg)](https://www.nuget.org/packages/SquidStd.Workers.Manager/) | + +### Scripting & templating + +| Package | Description | Links | +|---------|-------------|-------| +| `SquidStd.Scripting.Lua` | Lua scripting engine with attribute-based modules and event bridging. | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Scripting.Lua/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Scripting.Lua.svg)](https://www.nuget.org/packages/SquidStd.Scripting.Lua/) | +| `SquidStd.Templating` | Scriban templating with a named-template registry and `templates/*.tmpl` auto-load (`AddTemplating`). | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Templating/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Templating.svg)](https://www.nuget.org/packages/SquidStd.Templating/) | + +### Telemetry + +| Package | Description | Links | +|---------|-------------|-------| +| `SquidStd.Telemetry.Abstractions` | Shared telemetry config (`TelemetryOptions`, OTLP protocol, `SquidStd.*` ActivitySource convention). | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Telemetry.Abstractions/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Telemetry.Abstractions.svg)](https://www.nuget.org/packages/SquidStd.Telemetry.Abstractions/) | +| `SquidStd.Telemetry.OpenTelemetry` | OpenTelemetry tracing + metrics export (OTLP/console), standard instrumentation, metrics-snapshot bridge (`AddSquidStdTelemetry`). | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Telemetry.OpenTelemetry/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Telemetry.OpenTelemetry.svg)](https://www.nuget.org/packages/SquidStd.Telemetry.OpenTelemetry/) | + +### Shared & tooling + +| Package | Description | Links | +|---------|-------------|-------| +| `SquidStd.Aws.Abstractions` | Shared AWS connection config (`AwsConfigEntry`: region, credentials, endpoint override) for AWS-SDK providers. | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Aws.Abstractions/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Aws.Abstractions.svg)](https://www.nuget.org/packages/SquidStd.Aws.Abstractions/) | +| `SquidStd.Templates` | `dotnet new` templates for scaffolding SquidStd projects (host, ASP.NET, worker, manager). | [![readme](https://img.shields.io/badge/readme-1390A3.svg)](src/SquidStd.Templates/README.md) · [![NuGet](https://img.shields.io/nuget/v/SquidStd.Templates.svg)](https://www.nuget.org/packages/SquidStd.Templates/) | + ## Related projects - **[Felix Network](https://github.com/tgiachi/SquidStd-Felix)** — a standalone secure binary From c00e43982b1ce96a9d0dcad7741e688a26a81eb8 Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 21:03:26 +0200 Subject: [PATCH 32/77] docs: expand root README with TOC, templates, fuller example and acknowledgements - add a Contents table of contents - broaden the Overview to cover all current domains (crypto, persistence, VFS, actors, search, mail, workers, telemetry, AWS adapters) - correct the Testcontainers list (RabbitMQ, Redis, Elasticsearch, MinIO, LocalStack) in Requirements and Test - add an event bus + provider example alongside the cache quick start - add a Project templates section (dotnet new install + short names) - add a Built on acknowledgements section --- README.md | 111 ++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 100 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index af215a2c..ff64646b 100644 --- a/README.md +++ b/README.md @@ -12,20 +12,43 @@ license

+## Contents + +- [Overview](#overview) +- [Requirements](#requirements) +- [Quick Start](#quick-start) +- [Project templates](#project-templates) +- [Packages](#packages) +- [Related projects](#related-projects) +- [Architecture](#architecture) +- [Documentation](#documentation) +- [Build](#build) +- [Test](#test) +- [Contributing](#contributing) +- [Versioning & Releases](#versioning--releases) +- [Built on](#built-on) +- [License](#license) + ## Overview **squid-std** is a batteries-included standard library for .NET, distilled from years and years of building real-world server software. Instead of re-solving the same problems on every project, it bundles the foundations you reach for again and again behind small, well-defined contracts: -- **Security & hashing** — password hashing and verification (`HashUtils`), AES-GCM secret - protection and a pluggable secret store (`ISecretProtector` / `ISecretStore`). -- **Configuration** — a YAML-backed config manager with section registration and environment-variable expansion. -- **Serialization** — unified JSON/YAML utilities (`JsonUtils`, `YamlUtils`) and a shared `IDataSerializer` / `IDataDeserializer`. -- **String & platform helpers** — case converters (camel/kebab/snake/pascal/…), network, version, platform and resource utilities. -- **Runtime services** — DI bootstrap, event bus, job system, timer/cron scheduler, metrics and storage. -- **Infrastructure modules** — messaging (in-memory + RabbitMQ), caching (in-memory + Redis), - database access, networking (TCP/UDP) and Lua scripting. +- **Security & crypto** — password hashing (`HashUtils`), AES-GCM secret protection and a pluggable + secret store (`ISecretProtector` / `ISecretStore`), an OpenPGP keyring (`SquidStd.Crypto`), and + AWS KMS / Secrets Manager adapters. +- **Configuration & serialization** — a YAML-backed config manager with section registration and + environment-variable expansion; unified JSON/YAML utilities and a shared `IDataSerializer`. +- **Runtime services** — DI bootstrap, event bus, command dispatcher, actors (ordered per-entity + mailboxes), job system, timer/cron scheduler, metrics and health checks. +- **Messaging & caching** — in-memory, RabbitMQ and AWS SQS/SNS transports; in-memory and Redis caches. +- **Data & storage** — FreeSql data access, binary persistence (snapshot + WAL), local / S3 / MinIO + object storage, and a virtual filesystem (physical / zip / in-memory + an encrypted vault). +- **Search, mail & workers** — Elasticsearch indexing with a constrained LINQ provider, IMAP/POP3 + mail polling and an outbound mail queue, and a worker / manager runtime. +- **Networking, scripting & observability** — TCP/UDP servers with a framing/middleware pipeline, + Lua scripting and Scriban templating, and OpenTelemetry tracing + metrics export. Everything is modular: take only the packages you need, each behind a clean abstraction with an in-memory implementation for tests and an external backend for production. @@ -33,7 +56,7 @@ in-memory implementation for tests and an external backend for production. ## Requirements - [.NET 10 SDK](https://dotnet.microsoft.com/download) -- [Docker](https://www.docker.com/) — only for running the integration tests (Testcontainers spin up RabbitMQ and Redis). +- [Docker](https://www.docker.com/) — only for the integration tests (Testcontainers spin up RabbitMQ, Redis, Elasticsearch, MinIO and LocalStack on demand). ## Quick Start @@ -64,6 +87,59 @@ await bootstrap.StopAsync(); `StartAsync` / `StopAsync` lifecycle of every registered `ISquidStdService`. Use `RunAsync` to block until cancellation for long-running hosts. +### A fuller example: event bus + a provider + +```csharp +using SquidStd.Core.Interfaces.Events; +using SquidStd.Caching.Abstractions.Interfaces; +using SquidStd.Caching.Extensions; +using SquidStd.Services.Core.Services.Bootstrap; + +public sealed record OrderPlaced(string OrderId) : IEvent; + +var bootstrap = SquidStdBootstrap.Create() + .ConfigureServices(container => container.AddInMemoryCache()); + +await bootstrap.StartAsync(); + +var eventBus = bootstrap.Resolve(); // core service, always available +var cache = bootstrap.Resolve(); + +// React to domain events… +using var subscription = eventBus.Subscribe(async (e, ct) => + await cache.SetAsync($"order:{e.OrderId}", DateTimeOffset.UtcNow, TimeSpan.FromHours(1))); + +// …and publish them. +await eventBus.PublishAsync(new OrderPlaced("A-1001")); + +await bootstrap.StopAsync(); +``` + +The event bus, config manager, job system, scheduler and metrics are wired by the bootstrapper out +of the box; modules such as caching are opted in through `ConfigureServices`. + +## Project templates + +Scaffold a ready-to-run project with the `dotnet new` template pack: + +```bash +dotnet new install SquidStd.Templates +``` + +| Template | Short name | What you get | +|---------------------|-----------------------|-----------------------------------------------------------------| +| Console host | `squidstd-host` | A `SquidStdBootstrap` console host. | +| ASP.NET minimal API | `squidstd-aspnetcore` | `UseSquidStd` + health checks + a sample endpoint + Dockerfile. | +| Worker microservice | `squidstd-worker` | `AddWorkers` + a sample `IJobHandler` + Dockerfile. | +| Worker manager | `squidstd-manager` | `AddWorkerManager` + `MapWorkerManagerEndpoints` + Dockerfile. | + +```bash +dotnet new squidstd-worker -n Acme.Resizer +dotnet new squidstd-manager -n Acme.Manager --messaging inmemory +``` + +`--messaging` (`rabbitmq` default | `inmemory`) is available on the worker and manager templates. + ## Packages ### Core & hosting @@ -215,8 +291,8 @@ dotnet build SquidStd.slnx dotnet test SquidStd.slnx ``` -Integration tests (RabbitMQ, Redis) need Docker running; they use Testcontainers to start -disposable containers automatically. +Integration tests need Docker running; Testcontainers starts disposable RabbitMQ, Redis, +Elasticsearch, MinIO and LocalStack containers automatically. ## Contributing @@ -232,6 +308,19 @@ Releases are automated with [semantic-release](https://semantic-release.gitbook. numbers and the changelog are derived from the conventional-commit history, and the NuGet packages are published on release. Package versions follow [Semantic Versioning](https://semver.org/). +## Built on + +squid-std stands on a small set of well-established libraries: + +- **DI & logging** — [DryIoc](https://github.com/dadhi/DryIoc), [Serilog](https://serilog.net/) +- **Data** — [FreeSql](https://github.com/dotnetcore/FreeSql), [MessagePack](https://github.com/MessagePack-CSharp/MessagePack-CSharp), [YamlDotNet](https://github.com/aaubry/YamlDotNet) +- **Messaging & cache** — [RabbitMQ.Client](https://github.com/rabbitmq/rabbitmq-dotnet-client), [StackExchange.Redis](https://github.com/StackExchange/StackExchange.Redis), [AWS SDK for .NET](https://github.com/aws/aws-sdk-net) +- **Search, mail & storage** — [Elastic.Clients.Elasticsearch](https://github.com/elastic/elasticsearch-net), [MailKit](https://github.com/jstedfast/MailKit), [Minio](https://github.com/minio/minio-dotnet) +- **Crypto** — [PgpCore](https://github.com/mattosaurus/PgpCore) over [BouncyCastle](https://www.bouncycastle.org/) +- **Scripting & templating** — [MoonSharp](https://www.moonsharp.org/) (Lua), [Scriban](https://github.com/scriban/scriban) +- **Observability** — [OpenTelemetry](https://opentelemetry.io/) +- **Scheduling** — [Cronos](https://github.com/HangfireIO/Cronos) + ## License MIT - see [LICENSE](LICENSE). From ec6a445fa2b646c041bf2e1d66819833d5b7aaef Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 21:08:28 +0200 Subject: [PATCH 33/77] =?UTF-8?q?docs:=20rewrite=20landing=20page=20with?= =?UTF-8?q?=20full=20surface=20and=20Di=C3=A1taxis=20entry=20points?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/index.md | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/docs/index.md b/docs/index.md index 54cd303d..ebd366c9 100644 --- a/docs/index.md +++ b/docs/index.md @@ -8,11 +8,19 @@ _layout: landing # SquidStd -A modular .NET toolkit: foundational service contracts and utilities, a DryIoc-based service stack, -networking, plugins, data access, messaging, and Lua scripting — published as focused NuGet packages. +A batteries-included, modular standard library for .NET 10 — distilled from years of building +real-world server software. Each capability ships behind a small contract with an in-memory +implementation for tests and a production backend, published as a focused NuGet package. -- **[Getting started](articles/getting-started.md)** — install and bootstrap the core services. -- **[API reference](api/index.md)** — the full type/member documentation. -- **Packages** — see the per-package guides under [Articles](articles/getting-started.md). -- **[Felix Network](articles/felix.md)** — companion secure binary mesh-networking library - (.NET + C/ESP32). +Security & crypto, configuration, persistence, messaging, caching, storage, a virtual filesystem, +search, mail, workers, actors, telemetry, scripting — take only what you need. + +## Start here + +- **[Tutorials](tutorials/index.md)** — learn by building: bootstrap, caching, messaging, workers, + crypto, persistence, and more. +- **[Guides](articles/guides/configuration.md)** — task-focused how-to and "which provider" decision guides. +- **[Concepts](articles/concepts/architecture.md)** — the architecture and the ideas behind it. +- **[Packages](articles/getting-started.md)** — per-package reference. +- **[API reference](api/index.md)** — full type/member documentation. +- **[Felix Network](articles/felix.md)** — companion secure binary mesh-networking library (.NET + C/ESP32). From d895bf82ef89d7f2812ab5cc0783fc59b7d54ce8 Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 21:08:28 +0200 Subject: [PATCH 34/77] docs: refresh getting-started with current bootstrap API --- docs/articles/getting-started.md | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/docs/articles/getting-started.md b/docs/articles/getting-started.md index 88533d39..94412372 100644 --- a/docs/articles/getting-started.md +++ b/docs/articles/getting-started.md @@ -7,14 +7,19 @@ dotnet add package SquidStd.Services.Core ``` ```csharp -using DryIoc; -using SquidStd.Services.Core.Extensions; +using SquidStd.Core.Data.Bootstrap; +using SquidStd.Services.Core.Services.Bootstrap; -var container = new Container(); +// Core services wired automatically: config manager, event bus, command dispatcher, +// job system, timer/cron scheduler, metrics, health checks, storage and secrets. +var bootstrap = SquidStdBootstrap.Create( + new SquidStdOptions { ConfigName = "squidstd", RootDirectory = AppContext.BaseDirectory }); -// config manager + event bus + jobs + timer wheel + dispatcher + metrics + storage + secrets -container.RegisterCoreServices("squidstd", Directory.GetCurrentDirectory()); +await bootstrap.StartAsync(); +// … resolve services, opt into modules with bootstrap.ConfigureServices(…) … +await bootstrap.StopAsync(); ``` -From here, add focused packages as needed — see the per-package guides in the sidebar and the -[API reference](../api/index.md). +Opt into modules with `ConfigureServices`, e.g. `container.AddInMemoryCache()`. See the +[Concepts](concepts/bootstrap-lifecycle.md) for the full lifecycle and +[Packages](getting-started.md) for per-module reference. From 4dbca359484f4d5aca05d1e95dc012cc0e774ae1 Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 21:12:36 +0200 Subject: [PATCH 35/77] docs: add reference articles for Actors, Crypto, Vfs, Secrets.Aws and Persistence --- docs/articles/actors.md | 1 + docs/articles/crypto.md | 1 + docs/articles/persistence-abstractions.md | 1 + docs/articles/persistence-messagepack.md | 1 + docs/articles/persistence.md | 1 + docs/articles/secrets-aws.md | 1 + docs/articles/toc.yml | 16 ++++++++++++++ docs/articles/vfs-abstractions.md | 1 + docs/articles/vfs.md | 1 + src/SquidStd.Vfs.Abstractions/README.md | 26 +++++++++++++++++++++++ 10 files changed, 50 insertions(+) create mode 100644 docs/articles/actors.md create mode 100644 docs/articles/crypto.md create mode 100644 docs/articles/persistence-abstractions.md create mode 100644 docs/articles/persistence-messagepack.md create mode 100644 docs/articles/persistence.md create mode 100644 docs/articles/secrets-aws.md create mode 100644 docs/articles/vfs-abstractions.md create mode 100644 docs/articles/vfs.md create mode 100644 src/SquidStd.Vfs.Abstractions/README.md diff --git a/docs/articles/actors.md b/docs/articles/actors.md new file mode 100644 index 00000000..ee50eff9 --- /dev/null +++ b/docs/articles/actors.md @@ -0,0 +1 @@ +[!include[](../../src/SquidStd.Actors/README.md)] diff --git a/docs/articles/crypto.md b/docs/articles/crypto.md new file mode 100644 index 00000000..3ba608c8 --- /dev/null +++ b/docs/articles/crypto.md @@ -0,0 +1 @@ +[!include[](../../src/SquidStd.Crypto/README.md)] diff --git a/docs/articles/persistence-abstractions.md b/docs/articles/persistence-abstractions.md new file mode 100644 index 00000000..96f9fb84 --- /dev/null +++ b/docs/articles/persistence-abstractions.md @@ -0,0 +1 @@ +[!include[](../../src/SquidStd.Persistence.Abstractions/README.md)] diff --git a/docs/articles/persistence-messagepack.md b/docs/articles/persistence-messagepack.md new file mode 100644 index 00000000..a3bc077d --- /dev/null +++ b/docs/articles/persistence-messagepack.md @@ -0,0 +1 @@ +[!include[](../../src/SquidStd.Persistence.MessagePack/README.md)] diff --git a/docs/articles/persistence.md b/docs/articles/persistence.md new file mode 100644 index 00000000..2104287c --- /dev/null +++ b/docs/articles/persistence.md @@ -0,0 +1 @@ +[!include[](../../src/SquidStd.Persistence/README.md)] diff --git a/docs/articles/secrets-aws.md b/docs/articles/secrets-aws.md new file mode 100644 index 00000000..2418ebfa --- /dev/null +++ b/docs/articles/secrets-aws.md @@ -0,0 +1 @@ +[!include[](../../src/SquidStd.Secrets.Aws/README.md)] diff --git a/docs/articles/toc.yml b/docs/articles/toc.yml index 2669ef5b..c80a43c9 100644 --- a/docs/articles/toc.yml +++ b/docs/articles/toc.yml @@ -18,6 +18,12 @@ href: database-abstractions.md - name: SquidStd.Database href: database.md +- name: SquidStd.Persistence.Abstractions + href: persistence-abstractions.md +- name: SquidStd.Persistence + href: persistence.md +- name: SquidStd.Persistence.MessagePack + href: persistence-messagepack.md - name: SquidStd.Messaging.Abstractions href: messaging-abstractions.md - name: SquidStd.Messaging @@ -26,6 +32,8 @@ href: messaging-rabbitmq.md - name: SquidStd.Messaging.Sqs href: messaging-sqs.md +- name: SquidStd.Actors + href: actors.md - name: SquidStd.Caching.Abstractions href: caching-abstractions.md - name: SquidStd.Caching @@ -38,6 +46,10 @@ href: storage.md - name: SquidStd.Storage.S3 href: storage-s3.md +- name: SquidStd.Vfs.Abstractions + href: vfs-abstractions.md +- name: SquidStd.Vfs + href: vfs.md - name: SquidStd.Scripting.Lua href: scripting-lua.md - name: SquidStd.Templating @@ -72,5 +84,9 @@ href: telemetry-opentelemetry.md - name: SquidStd.Aws.Abstractions href: aws-abstractions.md +- name: SquidStd.Crypto + href: crypto.md +- name: SquidStd.Secrets.Aws + href: secrets-aws.md - name: Felix Network href: felix.md diff --git a/docs/articles/vfs-abstractions.md b/docs/articles/vfs-abstractions.md new file mode 100644 index 00000000..6de38df3 --- /dev/null +++ b/docs/articles/vfs-abstractions.md @@ -0,0 +1 @@ +[!include[](../../src/SquidStd.Vfs.Abstractions/README.md)] diff --git a/docs/articles/vfs.md b/docs/articles/vfs.md new file mode 100644 index 00000000..247ccdc1 --- /dev/null +++ b/docs/articles/vfs.md @@ -0,0 +1 @@ +[!include[](../../src/SquidStd.Vfs/README.md)] diff --git a/src/SquidStd.Vfs.Abstractions/README.md b/src/SquidStd.Vfs.Abstractions/README.md new file mode 100644 index 00000000..a68e0c3b --- /dev/null +++ b/src/SquidStd.Vfs.Abstractions/README.md @@ -0,0 +1,26 @@ +

SquidStd.Vfs.Abstractions

+ +Virtual filesystem contracts for SquidStd: a path-based file/directory abstraction implemented by physical, in-memory and zip backends. + +## Install + +```bash +dotnet add package SquidStd.Vfs.Abstractions +``` + +## Key types + +| Type | Purpose | +|------|---------| +| `IVirtualFileSystem` | Path-based filesystem over a pluggable backend (directory, zip, encrypted container): exists, read/write bytes, open read/write streams, delete and list entries. | +| `ILockableFileSystem` | An `IVirtualFileSystem` that stays locked until unlocked with a passphrase (e.g. an encrypted vault), exposing `IsUnlocked`, `Unlock` and `Lock`. | +| `VfsPath` | Static helper that normalizes logical paths to forward-slash, root-relative form and rejects `.`/`..` traversal segments. | +| `VfsEntry` | Record describing a listed entry: its logical path, byte size and last-modified UTC timestamp. | + +## Related + +- Tutorial: [Virtual filesystem](https://tgiachi.github.io/squid-std/tutorials/vfs.html) + +## License + +MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). From a7ba8d5410cb668f7cecef944e12de91eaf223aa Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 21:21:44 +0200 Subject: [PATCH 36/77] docs: normalise core & hosting package READMEs --- src/SquidStd.Abstractions/README.md | 15 +++------- src/SquidStd.AspNetCore/README.md | 13 +++------ src/SquidStd.Core/README.md | 19 +++---------- src/SquidStd.Generators/README.md | 33 +++++++++++++++++----- src/SquidStd.Plugin.Abstractions/README.md | 10 +++---- src/SquidStd.Services.Core/README.md | 18 ++++-------- 6 files changed, 47 insertions(+), 61 deletions(-) diff --git a/src/SquidStd.Abstractions/README.md b/src/SquidStd.Abstractions/README.md index 3a83a2d9..d4829145 100644 --- a/src/SquidStd.Abstractions/README.md +++ b/src/SquidStd.Abstractions/README.md @@ -21,17 +21,6 @@ discoverable way (tracked through ordered registration lists). dotnet add package SquidStd.Abstractions ``` -## Features - -- `ISquidStdService` — a `StartAsync`/`StopAsync` lifecycle contract for managed services. -- `RegisterEventListenerAttribute` — mark `IEventListener` classes for generated registration. -- `RegisterStdServiceAttribute` — mark service implementations for generated lifecycle registration. -- `RegisterConfigSectionAttribute` — mark config models for generated config-section registration. -- `RegisterStdService()` — register a singleton service and record it in the - ordered service list (with optional priority). -- `RegisterConfigSection(sectionName)` — register a YAML config section for the config manager. -- `AddToRegisterTypedList(...)` — maintain ordered registration lists in the container. - ## Usage ```csharp @@ -59,6 +48,10 @@ container.RegisterConfigSection("my"); | `ServiceRegistrationData` | Ordered service registration record. | | `ConfigRegistrationData` | Config section registration record. | +## Related + +- Tutorial: [Source generators: registration](https://tgiachi.github.io/squid-std/tutorials/source-generators-registration.html) + ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). diff --git a/src/SquidStd.AspNetCore/README.md b/src/SquidStd.AspNetCore/README.md index 2c621eb4..0d12ec48 100644 --- a/src/SquidStd.AspNetCore/README.md +++ b/src/SquidStd.AspNetCore/README.md @@ -21,15 +21,6 @@ DryIoc container into the web host and registers a hosted service that starts an dotnet add package SquidStd.AspNetCore ``` -## Features - -- `WebApplicationBuilder.UseSquidStd(...)` — plug the SquidStd container and services into a web app. -- Configures DryIoc as the host's service-provider factory. -- Registers `SquidStdHostedService` to start/stop SquidStd services with the host. -- Optional `SquidStdOptions` configuration callback. -- `WebApplicationBuilder.AddSquidStdHealthChecks()` — bridges every SquidStd `IHealthCheck` into the standard ASP.NET Core - health-check system (one entry per check), exposed via the standard `app.MapHealthChecks(...)`. - ## Usage ```csharp @@ -71,6 +62,10 @@ Each registered `IHealthCheck` appears as its own entry in the report. Check nam | `SquidStdHostedService` | Hosted service bridging SquidStd service lifecycle to the host. | | `SquidStdHealthChecksExtensions` | `AddSquidStdHealthChecks(...)` — bridge to ASP.NET Core health checks. | +## Related + +- Tutorial: [Build an ASP.NET Core app](https://tgiachi.github.io/squid-std/tutorials/aspnetcore-app.html) + ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). diff --git a/src/SquidStd.Core/README.md b/src/SquidStd.Core/README.md index 711bedae..b96c1c09 100644 --- a/src/SquidStd.Core/README.md +++ b/src/SquidStd.Core/README.md @@ -22,21 +22,6 @@ SquidStd packages build on. dotnet add package SquidStd.Core ``` -## Features - -- Configuration contracts: `IConfigEntry` (a YAML section) and `IConfigManagerService`. -- In-process messaging: `IEventBus` with `ISyncEventListener` / `IAsyncEventListener` over `IEvent`. -- Command dispatch: `ICommandDispatcher` with `ICommandHandler`, fan-out, fault isolation, and a `CommandDispatchResult`; `ICommandContextFactory` builds the context from a seed for `ISeededCommandDispatcher`. -- Background work & timing: `IJobSystem`, `ITimerService`, `IMainThreadDispatcher`. -- Metrics & secrets: `IMetricProvider` and secret-protection contracts. -- Serialization: `IDataSerializer` / `IDataDeserializer` (default `JsonDataSerializer`), plus `YamlUtils` / `JsonUtils`. -- File watching: `IFileWatcherService` / `FileWatcherService` — recursive, debounced watchers that publish `FileChangedEvent` on the event bus. -- Object pooling: `ObjectPool` — thread-safe, non-blocking, factory-based reuse with optional reset. -- Cryptography: `CryptoUtils` (AES-GCM authenticated encrypt/decrypt + key generation), `EncryptString`/`DecryptString` string helpers, base64 extensions, and `SslUtils` for loading PEM/PFX TLS certificates. -- Randomness: `BuiltInRng` (seedable ambient RNG), `RandomUtils` (dice, coin flips), and collection `Shuffle`/`RandomElement`/`RandomSample` extensions. -- Utilities: a Serilog `EventSink`, and string/env/directory extensions. -- Shared domain enums under `Types` (e.g. `LogLevelType`, `PlatformType`, `FileChangeKind`). - ## Usage ```csharp @@ -88,6 +73,10 @@ pool.Return(builder); | `ObjectPool` | Thread-safe, non-blocking object pool. | | `ICommandDispatcher` | Typed protocol command dispatch with context. | +## Related + +- Tutorial: [Events, jobs & scheduling](https://tgiachi.github.io/squid-std/tutorials/events-jobs-scheduling.html) + ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). diff --git a/src/SquidStd.Generators/README.md b/src/SquidStd.Generators/README.md index 07567ef1..dc4de0d9 100644 --- a/src/SquidStd.Generators/README.md +++ b/src/SquidStd.Generators/README.md @@ -1,4 +1,15 @@ -# SquidStd.Generators +

+ SquidStd +

+ +

SquidStd.Generators

+ +

+ NuGet + Downloads + docs + license +

Roslyn source generators for SquidStd compile-time registration helpers. @@ -8,7 +19,7 @@ Roslyn source generators for SquidStd compile-time registration helpers. dotnet add package SquidStd.Generators ``` -## Event listeners +## Usage The event listener generator discovers concrete `IEventListener` implementations marked with `[RegisterEventListener]` and generates a DryIoc registration extension: @@ -26,17 +37,25 @@ public sealed class PingListener : IEventListener container.RegisterGeneratedEventListeners(); ``` -The generated method reuses the normal `RegisterEventListener()` runtime path, so listener activation stays compatible with `SquidStd.Services.Core`. - -## Other generated registrations +The generated method reuses the normal `RegisterEventListener()` runtime path, so listener activation stays compatible with `SquidStd.Services.Core`. Each registration family has its own marker attribute and generated extension method, all calling the same runtime APIs as manual registration (`RegisterStdService`, `RegisterConfigSection`, `AddJobHandler`, `RegisterScriptModule`). -Each registration family has its own marker attribute and generated extension method: +## Key types | Marker attribute | Generated method | |------------------|------------------| +| `[RegisterEventListener]` | `RegisterGeneratedEventListeners()` | | `[RegisterStdService(typeof(IMyService), Priority = 10)]` | `RegisterGeneratedStdServices()` | | `[RegisterConfigSection("workers", Priority = -50)]` | `RegisterGeneratedConfigSections()` | | `[RegisterJobHandler]` | `RegisterGeneratedJobHandlers()` | | `[RegisterScriptModule]` with `[ScriptModule("name")]` | `RegisterGeneratedScriptModules()` | -The generated methods call the same runtime APIs as manual registration: `RegisterStdService`, `RegisterConfigSection`, `AddJobHandler`, and `RegisterScriptModule`. +## Related + +- Tutorial: [Source generators: event listeners](https://tgiachi.github.io/squid-std/tutorials/source-generators-event-listeners.html) +- Tutorial: [Source generators: registration](https://tgiachi.github.io/squid-std/tutorials/source-generators-registration.html) + +## License + +MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). + + diff --git a/src/SquidStd.Plugin.Abstractions/README.md b/src/SquidStd.Plugin.Abstractions/README.md index 57af815d..0952933b 100644 --- a/src/SquidStd.Plugin.Abstractions/README.md +++ b/src/SquidStd.Plugin.Abstractions/README.md @@ -21,12 +21,6 @@ with shared boot data. dotnet add package SquidStd.Plugin.Abstractions ``` -## Features - -- `ISquidStdPlugin` — the plugin entry point: `Metadata` + `Configure(IContainer, PluginContext)`. -- `PluginMetadata` — id, name, `Version`, author, optional description, and dependency declarations. -- `PluginContext` — a typed bag of boot data shared with the plugin (`GetData(key)`). - ## Usage ```csharp @@ -59,6 +53,10 @@ public sealed class MyPlugin : ISquidStdPlugin | `PluginMetadata` | Plugin identity and dependency declarations. | | `PluginContext` | Shared boot data passed to the plugin. | +## Related + +- Tutorial: [Plugins](https://tgiachi.github.io/squid-std/tutorials/plugins.html) + ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). diff --git a/src/SquidStd.Services.Core/README.md b/src/SquidStd.Services.Core/README.md index d541f658..e009f7fb 100644 --- a/src/SquidStd.Services.Core/README.md +++ b/src/SquidStd.Services.Core/README.md @@ -21,18 +21,6 @@ main-thread dispatcher, metrics collection, storage, and secrets services. dotnet add package SquidStd.Services.Core ``` -## Features - -- One-line bootstrap: `container.RegisterCoreServices()` registers the full default service set. -- `ConfigManagerService` — loads/saves YAML config sections and substitutes `$ENV_VAR` tokens. -- `EventBusService` — in-process publish/subscribe over `IEvent`. -- `CommandDispatcher` — typed protocol command dispatch with fan-out, fault isolation, and a `CommandDispatchResult` (`RegisterCommandDispatcher` / `RegisterCommandHandler` / `RegisterSeededCommandDispatcher`). -- `JobSystemService` — background job execution; `TimerWheelService` + cron scheduling for timed work. -- `MainThreadDispatcherService` — marshal work back onto a main thread. -- `MetricsCollectionService` — aggregates `IMetricProvider` samples. -- `HealthCheckService` — aggregates `IHealthCheck`s into one report (`RegisterHealthChecksService`). -- AES-GCM-protected secret store. (File/object storage moved to `SquidStd.Storage`, opt-in via `AddFileStorage`.) - ## Usage ```csharp @@ -45,7 +33,7 @@ var container = new Container(); container.RegisterCoreServices("squidstd", Directory.GetCurrentDirectory()); ``` -## Command dispatch +### Command dispatch ```csharp // Register a dispatcher for your context type and handlers. @@ -85,6 +73,10 @@ await seeded.DispatchAsync(command, connection); // factory maps connection -> | `MainThreadDispatcherService` | Main-thread work dispatch. | | `MetricsCollectionService` | Metric sample aggregation. | +## Related + +- Tutorial: [Events, jobs & scheduling](https://tgiachi.github.io/squid-std/tutorials/events-jobs-scheduling.html) + ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). From a961341071c621e71694e15f0988f23dc7947628 Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 21:23:50 +0200 Subject: [PATCH 37/77] docs: normalise networking & actors package READMEs --- src/SquidStd.Actors/README.md | 18 +++++++++--------- src/SquidStd.Network/README.md | 12 ++++-------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/src/SquidStd.Actors/README.md b/src/SquidStd.Actors/README.md index d71004a4..3c5b15e6 100644 --- a/src/SquidStd.Actors/README.md +++ b/src/SquidStd.Actors/README.md @@ -53,6 +53,15 @@ using SquidStd.Actors.Extensions; using var sub = session.SubscribeToEventBus(eventBus, (UserJoinedEvent e) => new SendText($":{e.Nick} JOIN")); ``` +## Key types + +| Type | Purpose | +|---------------------------|----------------------------------------------------| +| `Actor` | Mailbox base class (`TellAsync` / `AskAsync`). | +| `ActorRequest` | Base record for request/response messages. | +| `IActorRequest` | Request contract (implement directly if not using the base). | +| `ActorOptions` | Capacity / overflow / error configuration. | + ## Options `ActorOptions` controls the mailbox: @@ -70,15 +79,6 @@ using var sub = session.SubscribeToEventBus(eventBus, (UserJoinedEvent e) => new `DisposeAsync` completes the mailbox, drains in-flight work, and faults any still-pending requests. -## Key types - -| Type | Purpose | -|---------------------------|----------------------------------------------------| -| `Actor` | Mailbox base class (`TellAsync` / `AskAsync`). | -| `ActorRequest` | Base record for request/response messages. | -| `IActorRequest` | Request contract (implement directly if not using the base). | -| `ActorOptions` | Capacity / overflow / error configuration. | - ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). diff --git a/src/SquidStd.Network/README.md b/src/SquidStd.Network/README.md index 6a2cb3c6..122c178d 100644 --- a/src/SquidStd.Network/README.md +++ b/src/SquidStd.Network/README.md @@ -21,14 +21,6 @@ designed for low-allocation, high-throughput byte processing. dotnet add package SquidStd.Network ``` -## Features - -- TCP server/client (`SquidTcpServer`, `SquidStdTcpClient`) with optional TLS. -- UDP server/client (`SquidStdUdpServer`, `SquidStdUdpClient`). -- Session management (`ISessionManager`) with typed per-connection state. -- Composable framing (`INetFramer`) and middleware pipeline (`INetMiddleware`). -- Zero-copy binary I/O via `SpanReader` / `SpanWriter` and a reusable `CircularBuffer`. - ## Usage ```csharp @@ -52,6 +44,10 @@ await server.StopAsync(CancellationToken.None); | `INetMiddleware` | Pipeline stage over inbound/outbound data. | | `SpanReader` / `SpanWriter` | Allocation-free binary read/write. | +## Related + +- Tutorial: [Networking](https://tgiachi.github.io/squid-std/tutorials/networking.html) + ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). From ea2e5ab15b6df2254a4d0426700566256d7eb460 Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 21:26:30 +0200 Subject: [PATCH 38/77] docs: normalise messaging package READMEs --- src/SquidStd.Messaging.Abstractions/README.md | 13 +++------ src/SquidStd.Messaging.RabbitMq/README.md | 11 +++---- src/SquidStd.Messaging.Sqs/README.md | 29 +++++++++++++++++++ src/SquidStd.Messaging/README.md | 12 +++----- 4 files changed, 41 insertions(+), 24 deletions(-) diff --git a/src/SquidStd.Messaging.Abstractions/README.md b/src/SquidStd.Messaging.Abstractions/README.md index 6995e8d6..98adf634 100644 --- a/src/SquidStd.Messaging.Abstractions/README.md +++ b/src/SquidStd.Messaging.Abstractions/README.md @@ -22,15 +22,6 @@ implementation (in-memory or RabbitMQ) from a companion package. dotnet add package SquidStd.Messaging.Abstractions ``` -## Features - -- `IMessageQueue` — typed `PublishAsync` / `Subscribe` facade over named queues. -- `IQueueProvider` — the raw transport contract implemented per backend. -- `IMessageSerializer` (+ default `JsonMessageSerializer`) — payload (de)serialization. -- `IQueueMessageListener` / `IQueueMessageListenerAsync` — sync/async subscribers. -- `IMessagingMetrics` (+ `MessagingMetricsProvider`, `NoOpMessagingMetrics`) — delivery metrics. -- `MessagingOptions` and `MessagingConnectionString` — configuration and connection parsing. - ## Usage ```csharp @@ -59,6 +50,10 @@ public async Task PublishAsync(IMessageQueue queue) | `IMessagingMetrics` | Delivery metrics sink. | | `MessagingOptions` | Delivery attempts, retry delay, and dead-letter-queue suffix. | +## Related + +- Tutorial: [Messaging](https://tgiachi.github.io/squid-std/tutorials/messaging.html) + ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). diff --git a/src/SquidStd.Messaging.RabbitMq/README.md b/src/SquidStd.Messaging.RabbitMq/README.md index 8c2fe3f3..19feb08f 100644 --- a/src/SquidStd.Messaging.RabbitMq/README.md +++ b/src/SquidStd.Messaging.RabbitMq/README.md @@ -21,13 +21,6 @@ so the same `IMessageQueue` API publishes to and consumes from a real broker. Re dotnet add package SquidStd.Messaging.RabbitMq ``` -## Features - -- One-line registration: `container.AddRabbitMqMessaging(connectionString)` or with `RabbitMqOptions`. -- Broker-backed `IQueueProvider` reusing the shared `IMessageQueue` facade and serializer. -- Connection via a `rabbitmq://` connection string or explicit `RabbitMqOptions` (host/port/vhost/credentials). -- Configurable consumer prefetch count (`?prefetch=` on the connection string, or `RabbitMqOptions.PrefetchCount`). - ## Usage ```csharp @@ -50,6 +43,10 @@ await queue.PublishAsync("orders", new { Id = 1 }); | `RabbitMqQueueProvider` | RabbitMQ-backed `IQueueProvider`. | | `RabbitMqOptions` | Connection + prefetch configuration. | +## Related + +- Tutorial: [Messaging](https://tgiachi.github.io/squid-std/tutorials/messaging.html) + ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). diff --git a/src/SquidStd.Messaging.Sqs/README.md b/src/SquidStd.Messaging.Sqs/README.md index 1cc0ae7e..49be76c5 100644 --- a/src/SquidStd.Messaging.Sqs/README.md +++ b/src/SquidStd.Messaging.Sqs/README.md @@ -1,5 +1,16 @@ +

+ SquidStd +

+

SquidStd.Messaging.Sqs

+

+ NuGet + Downloads + docs + license +

+ AWS SQS/SNS transport for SquidStd.Messaging. Implements `IQueueProvider` over SQS (with a redrive policy to a dead-letter queue) and `ITopicProvider` via SNS+SQS fan-out, behind the same `IMessageQueue` / `IMessageTopic` API as the other providers. Registered with a single @@ -29,3 +40,21 @@ container.AddSqsMessaging(new SqsOptions { Aws = new AwsConfigEntry { Region = " - Topics map to SNS; each subscriber gets a dedicated SQS queue subscribed with raw message delivery. - Names are sanitized to the SQS/SNS alphabet (the default `.dlq` suffix becomes `-dlq`). - Payloads travel base64-encoded in the message body. + +## Key types + +| Type | Purpose | +|-------------------------------------|------------------------------------------| +| `SqsMessagingRegistrationExtensions` | `AddSqsMessaging(...)` registration. | +| `SqsQueueProvider` | SQS-backed `IQueueProvider`. | +| `SqsTopicProvider` | SNS+SQS-backed `ITopicProvider`. | +| `SqsOptions` | AWS connection + queue/topic configuration. | + +## Related + +- Tutorial: [Messaging](https://tgiachi.github.io/squid-std/tutorials/messaging.html) + +## License + +MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). + diff --git a/src/SquidStd.Messaging/README.md b/src/SquidStd.Messaging/README.md index dec66618..8f1841a2 100644 --- a/src/SquidStd.Messaging/README.md +++ b/src/SquidStd.Messaging/README.md @@ -21,14 +21,6 @@ with a single `AddInMemoryMessaging()` call. Ideal for single-process apps, test dotnet add package SquidStd.Messaging ``` -## Features - -- One-line registration: `container.AddInMemoryMessaging()` (facade, provider, serializer, metrics). -- Channel-based per-queue buffering with a dedicated consumer loop. -- Round-robin delivery across multiple subscribers of the same queue. -- Retry with configurable max attempts and dead-letter queues (`MessagingOptions`). -- Built-in delivery metrics via `MessagingMetricsProvider`. - ## Usage ```csharp @@ -50,6 +42,10 @@ await queue.PublishAsync("orders", new { Id = 1 }); | `MessagingRegistrationExtensions` | `AddInMemoryMessaging(...)` registration. | | `InMemoryQueueProvider` | Channel-based in-memory `IQueueProvider`. | +## Related + +- Tutorial: [Messaging](https://tgiachi.github.io/squid-std/tutorials/messaging.html) + ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). From 39bce517950198cc4fd484eb898c53082eac298d Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 21:28:39 +0200 Subject: [PATCH 39/77] docs: normalise persistence & database package READMEs --- src/SquidStd.Database.Abstractions/README.md | 13 ++++--------- src/SquidStd.Database/README.md | 13 ++++--------- src/SquidStd.Persistence/README.md | 16 ---------------- 3 files changed, 8 insertions(+), 34 deletions(-) diff --git a/src/SquidStd.Database.Abstractions/README.md b/src/SquidStd.Database.Abstractions/README.md index 0526ab16..abd63625 100644 --- a/src/SquidStd.Database.Abstractions/README.md +++ b/src/SquidStd.Database.Abstractions/README.md @@ -21,15 +21,6 @@ operations, paging, and composable queries — without binding to any specific O dotnet add package SquidStd.Database.Abstractions ``` -## Features - -- `IDataAccess` — `InsertAsync`, `GetByIdAsync`, `UpdateAsync`, `DeleteAsync`, `CountAsync`, - `ExistsAsync`, bulk insert/update/delete, `QueryAsync`, and `GetPagedAsync`. -- `BaseEntity` — `Guid Id` plus `DateTimeOffset Created` / `Updated` (UTC), set by the data layer. -- `PagedResultData` — items + `Page`, `PageSize`, `TotalCount`, `TotalPages`, `HasNext`, `HasPrevious`. -- `DatabaseConfig` — URI connection string + `AutoMigrate` flag. -- `DatabaseProviderType` — `Sqlite`, `Postgres`, `SqlServer`, `MySql`. - ## Usage ```csharp @@ -59,6 +50,10 @@ public async Task ExampleAsync(IDataAccess users) | `DatabaseConfig` | Connection string + auto-migrate config section. | | `DatabaseProviderType` | Supported provider enum. | +## Related + +- Tutorial: [Database](https://tgiachi.github.io/squid-std/tutorials/database.html) + ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). diff --git a/src/SquidStd.Database/README.md b/src/SquidStd.Database/README.md index 87db6627..f9664144 100644 --- a/src/SquidStd.Database/README.md +++ b/src/SquidStd.Database/README.md @@ -22,15 +22,6 @@ on startup. dotnet add package SquidStd.Database ``` -## Features - -- One-line registration: `container.RegisterDatabase()` (config section + service + open-generic `IDataAccess<>`). -- Providers via URI scheme: `sqlite://`, `postgres://`, `sqlserver://`, `mysql://`. -- `FreeSqlDataAccess` — CRUD, bulk insert/update/delete, `QueryAsync`, `GetPagedAsync`; writes - run inside a unit of work and roll back on error. Sets `Id`/`Created`/`Updated` automatically. -- Optional `AutoMigrate` (FreeSql `AutoSyncStructure`) to create/update tables on startup. -- ZLinq in-memory helpers for zero-allocation post-processing of materialized results. - ## Usage ```csharp @@ -57,6 +48,10 @@ var page = await users.GetPagedAsync(page: 1, pageSize: 20, orderBy: u => u.Name | `ConnectionStringParser` | URI → provider + native connection string. | | `ZLinqResultExtensions` | Zero-alloc in-memory result helpers. | +## Related + +- Tutorial: [Database](https://tgiachi.github.io/squid-std/tutorials/database.html) + ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). diff --git a/src/SquidStd.Persistence/README.md b/src/SquidStd.Persistence/README.md index d6afcd5b..9a1ed633 100644 --- a/src/SquidStd.Persistence/README.md +++ b/src/SquidStd.Persistence/README.md @@ -23,22 +23,6 @@ dotnet add package SquidStd.Persistence dotnet add package SquidStd.Persistence.MessagePack # recommended binary serializer ``` -## Features - -- **Snapshot + journal**: in-memory state, WAL journal of every upsert/remove, periodic full snapshot + trim. -- **Crash-safe**: journal records are length+FNV-1a-checksum framed — a torn/corrupt trailing record is - detected on read and the tail is discarded. Snapshots are written atomically (temp + rename). -- **Serializer-agnostic**: per-entity payloads go through `IDataSerializer`/`IDataDeserializer`; the journal - and snapshot envelopes use a fixed binary layout. Pair with `SquidStd.Persistence.MessagePack` for a - compact binary default, or use the JSON serializer from `SquidStd.Core`. -- **Detached reads**: `GetByIdAsync`/`GetAllAsync`/`Query()` return deep clones, so callers never mutate - stored instances. -- **Write-ordered journaling**: writes serialize end-to-end (apply + append) so journal order always - matches sequence order — replay is deterministic. -- **Lifecycle service**: `PersistenceService` is an `ISquidStdService` that loads + replays at start, - autosaves on a timer, and snapshots on stop. Optional `IEventBus` integration raises - `SnapshotSaveStartedEvent`/`SnapshotSaveCompletedEvent`. - ## Usage ```csharp From e3c4bb560c1449d17e093aa5a49360d19dba3c0f Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 21:30:59 +0200 Subject: [PATCH 40/77] docs: normalise caching package READMEs --- src/SquidStd.Caching.Abstractions/README.md | 12 ++++-------- src/SquidStd.Caching.Redis/README.md | 12 ++++-------- src/SquidStd.Caching/README.md | 12 ++++-------- 3 files changed, 12 insertions(+), 24 deletions(-) diff --git a/src/SquidStd.Caching.Abstractions/README.md b/src/SquidStd.Caching.Abstractions/README.md index 93d452fe..c9780811 100644 --- a/src/SquidStd.Caching.Abstractions/README.md +++ b/src/SquidStd.Caching.Abstractions/README.md @@ -22,14 +22,6 @@ backend implementation (in-memory or Redis) from a companion package. dotnet add package SquidStd.Caching.Abstractions ``` -## Features - -- `ICacheService` — typed `GetAsync` / `SetAsync` / `RemoveAsync` / `ExistsAsync` / `GetOrSetAsync` facade. -- `ICacheProvider` — the raw byte-level backend contract implemented per provider. -- `CacheService` — shared facade that serializes values, applies the key prefix and default TTL, and implements cache-aside. -- `ICacheMetrics` (+ `CacheMetricsProvider`, `NoOpCacheMetrics`) — hit/miss/set/remove metrics. -- `CacheOptions` and `CacheConnectionString` — configuration and connection parsing. - ## Usage ```csharp @@ -51,6 +43,10 @@ public Task GetOrComputeAsync(ICacheService cache) | `CacheOptions` | Default TTL and key prefix. | | `CacheConnectionString` | `scheme://host[?params]` parsing into `CacheOptions`. | +## Related + +- Tutorial: [Caching](https://tgiachi.github.io/squid-std/tutorials/caching.html) + ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). diff --git a/src/SquidStd.Caching.Redis/README.md b/src/SquidStd.Caching.Redis/README.md index ebaa1742..a84ef265 100644 --- a/src/SquidStd.Caching.Redis/README.md +++ b/src/SquidStd.Caching.Redis/README.md @@ -21,14 +21,6 @@ Registered with a single `AddRedisCache(...)` call. dotnet add package SquidStd.Caching.Redis ``` -## Features - -- One-line registration: `container.AddRedisCache(connectionString)` or with `RedisCacheOptions`. -- Redis-backed `ICacheProvider` reusing the shared `ICacheService` facade and serializer. -- Native TTL via Redis key expiry (`SET ... EX`). -- Connection via a `redis://` connection string or an explicit StackExchange.Redis configuration string. -- Built-in hit/miss metrics via `CacheMetricsProvider`. - ## Usage ```csharp @@ -52,6 +44,10 @@ var user = await cache.GetAsync("user:1"); | `RedisCacheProvider` | StackExchange.Redis-backed `ICacheProvider`. | | `RedisCacheOptions` | StackExchange.Redis connection configuration. | +## Related + +- Tutorial: [Caching](https://tgiachi.github.io/squid-std/tutorials/caching.html) + ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). diff --git a/src/SquidStd.Caching/README.md b/src/SquidStd.Caching/README.md index 9e3f6622..0625be86 100644 --- a/src/SquidStd.Caching/README.md +++ b/src/SquidStd.Caching/README.md @@ -21,14 +21,6 @@ single `AddInMemoryCache()` call. Ideal for single-process apps, tests, and loca dotnet add package SquidStd.Caching ``` -## Features - -- One-line registration: `container.AddInMemoryCache()` (provider, facade, serializer, metrics). -- `IMemoryCache`-backed storage with absolute per-entry TTL and built-in eviction. -- Reuses the shared `CacheService` facade (key prefix, default TTL, cache-aside). -- Built-in hit/miss metrics via `CacheMetricsProvider`. -- Configure via `CacheOptions` or a `memory://` connection string (`?defaultTtlSeconds=`, `?keyPrefix=`). - ## Usage ```csharp @@ -51,6 +43,10 @@ var user = await cache.GetAsync("user:1"); | `CacheRegistrationExtensions` | `AddInMemoryCache(...)` registration. | | `InMemoryCacheProvider` | `IMemoryCache`-backed `ICacheProvider`. | +## Related + +- Tutorial: [Caching](https://tgiachi.github.io/squid-std/tutorials/caching.html) + ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). From 523e4ead24452d611f0b4c01dff3dd4b2f62c88c Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 21:38:48 +0200 Subject: [PATCH 41/77] docs: normalise storage & vfs package READMEs --- src/SquidStd.Storage.Abstractions/README.md | 38 ++++++------------- src/SquidStd.Storage.S3/README.md | 38 +++++++------------ src/SquidStd.Storage/README.md | 41 ++++++++------------- src/SquidStd.Vfs.Abstractions/README.md | 4 -- src/SquidStd.Vfs/README.md | 25 +++++++++---- 5 files changed, 57 insertions(+), 89 deletions(-) diff --git a/src/SquidStd.Storage.Abstractions/README.md b/src/SquidStd.Storage.Abstractions/README.md index cacca83c..24268d7f 100644 --- a/src/SquidStd.Storage.Abstractions/README.md +++ b/src/SquidStd.Storage.Abstractions/README.md @@ -1,19 +1,8 @@ -

- SquidStd -

-

SquidStd.Storage.Abstractions

-

- NuGet - Downloads - docs - license -

- -Backend-agnostic storage contracts for SquidStd. It defines the binary blob store (`IStorageService`), -the typed object store (`IObjectStorageService`), key enumeration (`ListKeysAsync`), and `StorageConfig`. -Pick a backend implementation (local file or S3/MinIO) from a companion package. +Backend-agnostic storage contracts for SquidStd: a binary blob store (`IStorageService`), a typed object +store (`IObjectStorageService`), key enumeration (`ListKeysAsync`), and `StorageConfig`. Pick a backend +implementation (local file or S3/MinIO) from a companion package. ## Install @@ -21,13 +10,6 @@ Pick a backend implementation (local file or S3/MinIO) from a companion package. dotnet add package SquidStd.Storage.Abstractions ``` -## Features - -- `IStorageService` — binary blobs by key: `SaveAsync` / `LoadAsync` / `DeleteAsync` / `ExistsAsync` / `ListKeysAsync`. -- `IObjectStorageService` — typed objects by key (serialized by the provider) with the same surface plus `ListKeysAsync`. -- `ListKeysAsync(prefix?)` — streams stored keys as `IAsyncEnumerable`, optionally filtered by prefix. -- `StorageConfig` — root-directory configuration for file-backed storage. - ## Usage ```csharp @@ -45,11 +27,15 @@ public async Task DumpKeysAsync(IStorageService storage) ## Key types -| Type | Purpose | -|-------------------------|---------------------------------------------------| -| `IStorageService` | Binary blob store (save/load/delete/exists/list). | -| `IObjectStorageService` | Typed object store over a blob backend. | -| `StorageConfig` | Root directory for file storage. | +| Type | Purpose | +|------|---------| +| `IStorageService` | Binary blob store: `SaveAsync` / `LoadAsync` / `DeleteAsync` / `ExistsAsync` / `ListKeysAsync`. | +| `IObjectStorageService` | Typed object store over a blob backend (serialized by the provider). | +| `StorageConfig` | Root directory for file storage. | + +## Related + +- Tutorial: [Object storage](https://tgiachi.github.io/squid-std/tutorials/storage.html) ## License diff --git a/src/SquidStd.Storage.S3/README.md b/src/SquidStd.Storage.S3/README.md index a8648a44..f3d74062 100644 --- a/src/SquidStd.Storage.S3/README.md +++ b/src/SquidStd.Storage.S3/README.md @@ -1,19 +1,8 @@ -

- SquidStd -

-

SquidStd.Storage.S3

-

- NuGet - Downloads - docs - license -

- S3-compatible storage for SquidStd, backed by the MinIO .NET SDK. Implements `IStorageService` against -AWS S3, MinIO, or any S3-compatible endpoint, so the same storage API reads and writes object storage. -The bucket is created lazily on first use. Registered with a single `AddS3Storage(...)` call. +AWS S3, MinIO, or any S3-compatible endpoint, so the same storage API reads and writes object storage. The +bucket is created lazily on first use. Registered with a single `AddS3Storage(...)` call. ## Install @@ -21,13 +10,6 @@ The bucket is created lazily on first use. Registered with a single `AddS3Storag dotnet add package SquidStd.Storage.S3 ``` -## Features - -- One-line registration: `container.AddS3Storage(options)`. -- `IStorageService` over `IMinioClient` (`SaveAsync` / `LoadAsync` / `DeleteAsync` / `ExistsAsync` / `ListKeysAsync`). -- Lazy bucket creation; `ListKeysAsync(prefix?)` streams object keys via the S3 list API. -- Works with AWS S3 and MinIO via `S3StorageOptions` (endpoint, credentials, bucket, TLS, region). - ## Usage ```csharp @@ -49,13 +31,19 @@ var storage = container.Resolve(); await storage.SaveAsync("reports/2026.json", "{}"u8.ToArray()); ``` +`ListKeysAsync(prefix?)` streams object keys via the S3 list API. + ## Key types -| Type | Purpose | -|-----------------------------------|---------------------------------------------| -| `S3StorageRegistrationExtensions` | `AddS3Storage(...)` registration. | -| `S3StorageService` | MinIO-backed `IStorageService`. | -| `S3StorageOptions` | Endpoint, credentials, bucket, TLS, region. | +| Type | Purpose | +|------|---------| +| `S3StorageRegistrationExtensions` | `AddS3Storage(...)` registration. | +| `S3StorageService` | MinIO-backed `IStorageService` with lazy bucket creation. | +| `S3StorageOptions` | Endpoint, credentials, bucket, TLS, region. | + +## Related + +- Tutorial: [Object storage](https://tgiachi.github.io/squid-std/tutorials/storage.html) ## License diff --git a/src/SquidStd.Storage/README.md b/src/SquidStd.Storage/README.md index 3d509a2e..c3fdf955 100644 --- a/src/SquidStd.Storage/README.md +++ b/src/SquidStd.Storage/README.md @@ -1,19 +1,8 @@ -

- SquidStd -

-

SquidStd.Storage

-

- NuGet - Downloads - docs - license -

- -Local file storage for SquidStd. Provides a filesystem-backed `IStorageService` (atomic writes, -path-safe keys) and a YAML-backed `IObjectStorageService` that layers typed objects on top of it — -registered with a single `AddFileStorage()` call. Storage is opt-in: it is not registered by `RegisterCoreServices`. +Local file storage for SquidStd. Provides a filesystem-backed `IStorageService` (atomic writes, path-safe +keys) and a YAML-backed `IObjectStorageService` that layers typed objects on top of it — registered with a +single `AddFileStorage()` call. Storage is opt-in: it is not registered by `RegisterCoreServices`. ## Install @@ -21,13 +10,6 @@ registered with a single `AddFileStorage()` call. Storage is opt-in: it is not r dotnet add package SquidStd.Storage ``` -## Features - -- One-line registration: `container.AddFileStorage()` (file `IStorageService` + YAML `IObjectStorageService`). -- Atomic saves (temp file + move) and keys constrained to the storage root. -- `YamlObjectStorageService` decorates `IStorageService`, serializing typed objects to YAML. -- `ListKeysAsync(prefix?)` enumerates stored keys (`/`-separated), excluding in-flight temp files. - ## Usage ```csharp @@ -43,13 +25,20 @@ var storage = container.Resolve(); await storage.SaveAsync("profiles/main.bin", new byte[] { 1, 2, 3 }); ``` +Saves are atomic (temp file + move) and keys are constrained to the storage root; `ListKeysAsync(prefix?)` +enumerates stored keys (`/`-separated), excluding in-flight temp files. + ## Key types -| Type | Purpose | -|---------------------------------|--------------------------------------------------------| -| `StorageRegistrationExtensions` | `AddFileStorage(...)` registration. | -| `FileStorageService` | Filesystem-backed `IStorageService`. | -| `YamlObjectStorageService` | YAML-backed `IObjectStorageService` over a blob store. | +| Type | Purpose | +|------|---------| +| `StorageRegistrationExtensions` | `AddFileStorage(...)` registration (file `IStorageService` + YAML `IObjectStorageService`). | +| `FileStorageService` | Filesystem-backed `IStorageService`. | +| `YamlObjectStorageService` | YAML-backed `IObjectStorageService` over a blob store. | + +## Related + +- Tutorial: [Object storage](https://tgiachi.github.io/squid-std/tutorials/storage.html) ## License diff --git a/src/SquidStd.Vfs.Abstractions/README.md b/src/SquidStd.Vfs.Abstractions/README.md index a68e0c3b..e0b459a8 100644 --- a/src/SquidStd.Vfs.Abstractions/README.md +++ b/src/SquidStd.Vfs.Abstractions/README.md @@ -17,10 +17,6 @@ dotnet add package SquidStd.Vfs.Abstractions | `VfsPath` | Static helper that normalizes logical paths to forward-slash, root-relative form and rejects `.`/`..` traversal segments. | | `VfsEntry` | Record describing a listed entry: its logical path, byte size and last-modified UTC timestamp. | -## Related - -- Tutorial: [Virtual filesystem](https://tgiachi.github.io/squid-std/tutorials/vfs.html) - ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). diff --git a/src/SquidStd.Vfs/README.md b/src/SquidStd.Vfs/README.md index b9b1f462..9e6babb8 100644 --- a/src/SquidStd.Vfs/README.md +++ b/src/SquidStd.Vfs/README.md @@ -1,6 +1,6 @@

SquidStd.Vfs

-A path-based **virtual filesystem** abstraction for SquidStd with interchangeable backends. One interface +A path-based virtual filesystem abstraction for SquidStd with interchangeable backends. One interface (`IVirtualFileSystem`) is implemented by real directories, a single zip archive, and an in-memory store — and decorated by an encrypted vault in `SquidStd.Crypto`. `VfsDirectories` is a VFS-backed analogue of `DirectoriesConfig`, so named directory layouts work over any backend (a zip or an encrypted container can @@ -38,7 +38,7 @@ await foreach (var entry in fs.ListAsync("docs")) await fs.DeleteAsync("docs/cv.pdf"); ``` -### Named directories over any backend +Named directories over any backend: ```csharp var dirs = new VfsDirectories(fs, ["data", "logs"]); @@ -46,11 +46,20 @@ var target = dirs.Combine("data", "report.csv"); // "data/report.csv" await fs.WriteAllBytesAsync(target, csvBytes); ``` -## Providers - -- **`PhysicalFileSystem(root)`** — maps logical paths onto a real directory tree. -- **`ZipFileSystem(path)`** — a single `.zip` archive opened in update mode; `IAsyncDisposable`. -- **`InMemoryFileSystem()`** — ephemeral, in-process; handy for tests and as a decorator target. - Logical paths are normalized (forward slashes, root-relative) and reject `..` traversal via `SquidStd.Vfs.Abstractions.VfsPath`. + +## Key types + +| Type | Purpose | +|------|---------| +| `IVirtualFileSystem` | Path-based filesystem over a pluggable backend. | +| `PhysicalFileSystem` | Maps logical paths onto a real directory tree. | +| `ZipFileSystem` | A single `.zip` archive opened in update mode; `IAsyncDisposable`. | +| `InMemoryFileSystem` | Ephemeral, in-process; handy for tests and as a decorator target. | +| `VfsDirectories` | Named directory layout (`DirectoriesConfig` analogue) over any backend. | +| `RegisterVfsExtensions` | `RegisterVfs(...)` registration. | + +## License + +MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). From 91f7b0f3d634c9491818ca862535181a51a4dbd0 Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 21:49:30 +0200 Subject: [PATCH 42/77] docs: normalise security package READMEs --- src/SquidStd.Crypto/README.md | 15 +++++++++++++++ src/SquidStd.Secrets.Aws/README.md | 9 +++++++++ 2 files changed, 24 insertions(+) diff --git a/src/SquidStd.Crypto/README.md b/src/SquidStd.Crypto/README.md index abec6e15..3859ffbf 100644 --- a/src/SquidStd.Crypto/README.md +++ b/src/SquidStd.Crypto/README.md @@ -50,6 +50,17 @@ await keyring.SaveAsync(container.Resolve()); await keyring.LoadAsync(container.Resolve()); ``` +## Key types + +| Type | Purpose | +|------|---------| +| `IPgpService` | Key generation, encrypt/decrypt, sign/verify, and combined encrypt+sign / decrypt+verify. | +| `IPgpKeyring` | Stateful, indexed keyring: import keys and save/load via an `IPgpKeyStore`. | +| `IPgpKeyStore` | Pluggable keyring persistence backend. | +| `FilePgpKeyStore` | One armored `.asc` per key (gpg-interoperable). | +| `AesGcmPgpKeyStore` | The whole keyring serialized to a single file, encrypted at rest via `ISecretProtector`. | +| `CryptoFileSystem` | `ILockableFileSystem` that encrypts content and names over any `IVirtualFileSystem`. | + ## Key stores - **`FilePgpKeyStore(directory)`** — one armored `.asc` per key (public, plus secret when held). gpg-interoperable. @@ -110,3 +121,7 @@ var folderVault = new CryptoFileSystem(new PhysicalFileSystem("/secure/dir")); index, prune orphaned blobs, and zero the key with `CryptographicOperations.ZeroMemory`. - A wrong passphrase fails the index authentication tag → `CryptographicException`; operations on a locked vault throw `InvalidOperationException`. + +## License + +MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). diff --git a/src/SquidStd.Secrets.Aws/README.md b/src/SquidStd.Secrets.Aws/README.md index 01ba2da8..c2df2569 100644 --- a/src/SquidStd.Secrets.Aws/README.md +++ b/src/SquidStd.Secrets.Aws/README.md @@ -44,6 +44,15 @@ string? value = await store.GetAsync("db/main"); await foreach (var name in store.ListNamesAsync("db/")) { /* ... */ } ``` +## Key types + +| Type | Purpose | +|------|---------| +| `KmsSecretProtector` | `ISecretProtector` using AWS KMS data keys (envelope encryption). | +| `AwsSecretsManagerStore` | `ISecretStore` backed by AWS Secrets Manager. | +| `KmsSecretProtectorOptions` | KMS key id/ARN/alias, region and credentials. | +| `AwsSecretsManagerOptions` | Optional name prefix, region and credentials. | + ## Notes - **Envelope encryption** — `KmsSecretProtector` calls `GenerateDataKey` per `Protect`, encrypts the payload From c7217923b77e036354f7bfb4eac51d1d67a123f7 Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 21:50:12 +0200 Subject: [PATCH 43/77] docs: normalise search & mail package READMEs --- src/SquidStd.Mail.Abstractions/README.md | 15 ++++--------- src/SquidStd.Mail.MailKit/README.md | 24 +++++++++++--------- src/SquidStd.Mail.Queue/README.md | 25 ++++++++++++--------- src/SquidStd.Search.Abstractions/README.md | 15 ++++--------- src/SquidStd.Search.Elasticsearch/README.md | 24 +++++++++++--------- 5 files changed, 48 insertions(+), 55 deletions(-) diff --git a/src/SquidStd.Mail.Abstractions/README.md b/src/SquidStd.Mail.Abstractions/README.md index f211c894..54f7867f 100644 --- a/src/SquidStd.Mail.Abstractions/README.md +++ b/src/SquidStd.Mail.Abstractions/README.md @@ -1,16 +1,5 @@ -

- SquidStd -

-

SquidStd.Mail.Abstractions

-

- NuGet - Downloads - docs - license -

- Mail contracts for SquidStd: the `MailMessage` model, the `MailReceivedEvent`, the `IMailReader` interface, and the plain `MailOptions`. @@ -33,6 +22,10 @@ dotnet add package SquidStd.Mail.Abstractions | `SmtpOptions` | SMTP host/port/SSL/credentials and a default sender. | | `MailSentEvent` / `MailSendFailedEvent` | Published on the event bus on send success/failure. | +## Related + +- Tutorial: [Email](https://tgiachi.github.io/squid-std/tutorials/email.html) + ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). diff --git a/src/SquidStd.Mail.MailKit/README.md b/src/SquidStd.Mail.MailKit/README.md index 8dc716ef..0465aa6c 100644 --- a/src/SquidStd.Mail.MailKit/README.md +++ b/src/SquidStd.Mail.MailKit/README.md @@ -1,16 +1,5 @@ -

- SquidStd -

-

SquidStd.Mail.MailKit

-

- NuGet - Downloads - docs - license -

- MailKit-backed IMAP/POP3 provider for SquidStd.Mail. Polls a mailbox on the timer wheel and publishes a `MailReceivedEvent` on the `IEventBus` for each new message. @@ -74,6 +63,19 @@ await sender.SendAsync(new OutgoingMailMessage `MailSentEvent` / `MailSendFailedEvent` are published on the `IEventBus`; failures throw `MailSendException`. +## Key types + +| Type | Purpose | +|------|---------| +| `MailRegistrationExtensions` | `AddMail(...)` registration (IMAP/POP3 polling). | +| `MailSenderRegistrationExtensions` | `AddMailSender(...)` registration (SMTP). | +| `ImapMailReader` / `Pop3MailReader` | `IMailReader` implementations. | +| `MailKitMailSender` | `IMailSender` implementation over SMTP. | + +## Related + +- Tutorial: [Email](https://tgiachi.github.io/squid-std/tutorials/email.html) + ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). diff --git a/src/SquidStd.Mail.Queue/README.md b/src/SquidStd.Mail.Queue/README.md index 8cc95d04..4d118cdd 100644 --- a/src/SquidStd.Mail.Queue/README.md +++ b/src/SquidStd.Mail.Queue/README.md @@ -1,16 +1,5 @@ -

- SquidStd -

-

SquidStd.Mail.Queue

-

- NuGet - Downloads - docs - license -

- Outbound mail send queue for SquidStd. Enqueue an `OutgoingMailMessage`; a background consumer sends it via `IMailSender`. Retry, backoff, and dead-lettering come from the SquidStd messaging queue. @@ -46,6 +35,20 @@ await queue.EnqueueAsync(new OutgoingMailMessage Retry/backoff/dead-letter are configured via `MessagingOptions` (`MaxDeliveryAttempts`, `RetryDelay`, `DeadLetterQueueSuffix`). With RabbitMQ the queue is durable across restarts. +## Key types + +| Type | Purpose | +|------|---------| +| `IMailQueue` | Enqueue an `OutgoingMailMessage` for background delivery. | +| `MailQueue` | `IMailQueue` implementation over the SquidStd messaging queue. | +| `MailSendConsumerService` | Background consumer that sends queued messages via `IMailSender`. | +| `MailQueueRegistrationExtensions` | `AddMailQueue(...)` registration. | +| `MailQueueOptions` | Queue name and send options. | + +## Related + +- Tutorial: [Email](https://tgiachi.github.io/squid-std/tutorials/email.html) + ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). diff --git a/src/SquidStd.Search.Abstractions/README.md b/src/SquidStd.Search.Abstractions/README.md index 250d1877..031b4f70 100644 --- a/src/SquidStd.Search.Abstractions/README.md +++ b/src/SquidStd.Search.Abstractions/README.md @@ -1,16 +1,5 @@ -

- SquidStd -

-

SquidStd.Search.Abstractions

-

- NuGet - Downloads - docs - license -

- Search/indexing contracts for SquidStd: tag entities with `IIndexableEntity` + `[SearchIndex]` (with `${VAR}` / `${VAR:-default}` environment expansion) and query them via `ISearchService`. @@ -29,6 +18,10 @@ dotnet add package SquidStd.Search.Abstractions | `ISearchService` | Index / delete / ensure-index / `Query()`. | | `SearchIndexNameResolver` | Resolves the index name from the attribute/type + environment. | +## Related + +- Tutorial: [Search](https://tgiachi.github.io/squid-std/tutorials/search.html) + ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). diff --git a/src/SquidStd.Search.Elasticsearch/README.md b/src/SquidStd.Search.Elasticsearch/README.md index 5f75cf97..2b80206a 100644 --- a/src/SquidStd.Search.Elasticsearch/README.md +++ b/src/SquidStd.Search.Elasticsearch/README.md @@ -1,16 +1,5 @@ -

- SquidStd -

-

SquidStd.Search.Elasticsearch

-

- NuGet - Downloads - docs - license -

- Elasticsearch provider for SquidStd.Search. Indexes `IIndexableEntity` documents and exposes a constrained LINQ `IQueryable` translated to the Elasticsearch query DSL; the native `ElasticsearchClient` is registered for advanced queries. @@ -46,6 +35,19 @@ Supported LINQ: `Where` (`==`, `!=`, `<`, `>`, `<=`, `>=`, `&&`, `||`, `!`, `str bool members), `OrderBy`/`ThenBy`(`Descending`), `Skip`/`Take`, and `.Match(field, text)` / `.FullText(text)`. Anything else throws `NotSupportedException` — drop down to the native `ElasticsearchClient`. +## Key types + +| Type | Purpose | +|------|---------| +| `SearchRegistrationExtensions` | `AddElasticsearch(...)` registration. | +| `ElasticSearchService` | `ISearchService` backed by the Elasticsearch client. | +| `ElasticsearchOptions` | Connection options (URI, credentials). | +| `ElasticQueryable` | Constrained `IQueryable` translated to the Elasticsearch query DSL. | + +## Related + +- Tutorial: [Search](https://tgiachi.github.io/squid-std/tutorials/search.html) + ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). From 01036af2904283a962b319179df27aa84e2bcfa6 Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 21:50:35 +0200 Subject: [PATCH 44/77] docs: normalise workers package READMEs --- src/SquidStd.Workers.Abstractions/README.md | 15 ++++----------- src/SquidStd.Workers.Manager/README.md | 15 ++++----------- src/SquidStd.Workers/README.md | 15 ++++----------- 3 files changed, 12 insertions(+), 33 deletions(-) diff --git a/src/SquidStd.Workers.Abstractions/README.md b/src/SquidStd.Workers.Abstractions/README.md index d9e579d9..7138606e 100644 --- a/src/SquidStd.Workers.Abstractions/README.md +++ b/src/SquidStd.Workers.Abstractions/README.md @@ -1,16 +1,5 @@ -

- SquidStd -

-

SquidStd.Workers.Abstractions

-

- NuGet - Downloads - docs - license -

- Shared, dependency-free contracts between the SquidStd worker runtime and manager: the job descriptor, the worker heartbeat, the manager-side worker view, the worker status enum, and the conventional channel names. @@ -30,6 +19,10 @@ dotnet add package SquidStd.Workers.Abstractions | `WorkerStatusType` | `Idle` / `Busy` / `Offline`. | | `WorkerChannels` | Default jobs-queue and heartbeat-topic names. | +## Related + +- Tutorial: [Worker system](https://tgiachi.github.io/squid-std/tutorials/worker-system.html) + ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). diff --git a/src/SquidStd.Workers.Manager/README.md b/src/SquidStd.Workers.Manager/README.md index 3265e8e8..7a164562 100644 --- a/src/SquidStd.Workers.Manager/README.md +++ b/src/SquidStd.Workers.Manager/README.md @@ -1,16 +1,5 @@ -

- SquidStd -

-

SquidStd.Workers.Manager

-

- NuGet - Downloads - docs - license -

- The SquidStd worker manager. Enqueues jobs, collects worker heartbeats into an in-memory registry, marks workers `Offline` on a periodic sweep, publishes status-transition events, and exposes an opt-in ASP.NET minimal-API surface. @@ -45,6 +34,10 @@ app.MapWorkerManagerEndpoints(); // GET /workers, GET /workers/{id}, POST /jobs | `WorkerManagerRegistrationExtensions` | `AddWorkerManager()`. | | `WorkerManagerEndpointsExtensions` | `MapWorkerManagerEndpoints()`. | +## Related + +- Tutorial: [Worker system](https://tgiachi.github.io/squid-std/tutorials/worker-system.html) + ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). diff --git a/src/SquidStd.Workers/README.md b/src/SquidStd.Workers/README.md index 5f31fe8c..0f7dc948 100644 --- a/src/SquidStd.Workers/README.md +++ b/src/SquidStd.Workers/README.md @@ -1,16 +1,5 @@ -

- SquidStd -

-

SquidStd.Workers

-

- NuGet - Downloads - docs - license -

- The SquidStd worker runtime. Consumes `JobRequest`s from the jobs queue, dispatches each to a named `IJobHandler` (up to `MaxConcurrency` in parallel), and publishes a `WorkerHeartbeat` on the heartbeat topic every few seconds. @@ -57,6 +46,10 @@ public sealed class ResizeImageHandler : IJobHandler | `RegisterJobHandlerAttribute` | Marks handlers for generated registration. | | `WorkersRegistrationExtensions` | `AddWorkers()` and `AddJobHandler()`. | +## Related + +- Tutorial: [Worker system](https://tgiachi.github.io/squid-std/tutorials/worker-system.html) + ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). From 02d61f0dda2899e843411787b733f34bdbd84379 Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 21:51:21 +0200 Subject: [PATCH 45/77] docs: normalise scripting, templating & telemetry package READMEs --- src/SquidStd.Aws.Abstractions/README.md | 10 +++++++ src/SquidStd.Scripting.Lua/README.md | 29 +++++------------- src/SquidStd.Telemetry.Abstractions/README.md | 14 ++++++++- .../README.md | 13 ++++++++ src/SquidStd.Templates/README.md | 15 +++------- src/SquidStd.Templating/README.md | 30 +++++-------------- 6 files changed, 55 insertions(+), 56 deletions(-) diff --git a/src/SquidStd.Aws.Abstractions/README.md b/src/SquidStd.Aws.Abstractions/README.md index 311af5db..20cc1518 100644 --- a/src/SquidStd.Aws.Abstractions/README.md +++ b/src/SquidStd.Aws.Abstractions/README.md @@ -26,3 +26,13 @@ var aws = new AwsConfigEntry When `AccessKey`/`SecretKey` are null, consumers fall back to the AWS default credential chain (environment variables, shared profile, IAM role). When `ServiceUrl` is set, consumers point their client at that endpoint instead of the regional AWS endpoint. + +## Key types + +| Type | Purpose | +|------|---------| +| `AwsConfigEntry` | Region, optional credentials and an optional endpoint override (e.g. LocalStack). | + +## License + +MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). diff --git a/src/SquidStd.Scripting.Lua/README.md b/src/SquidStd.Scripting.Lua/README.md index fcf0f1e4..5d41802f 100644 --- a/src/SquidStd.Scripting.Lua/README.md +++ b/src/SquidStd.Scripting.Lua/README.md @@ -1,19 +1,9 @@ -

- SquidStd -

-

SquidStd.Scripting.Lua

-

- NuGet - Downloads - docs - license -

- Lua scripting for SquidStd. Hosts a Lua engine (`IScriptEngineService`) that exposes .NET methods to -scripts through attribute-decorated modules, bridges events, generates `.luarc` typings/docs, and -supports init scripts, constants, and callbacks. +scripts through attribute-decorated modules, bridges events to the SquidStd event bus, ships built-in +modules (logging, events, random), generates `.luarc` typings/docs, and supports init scripts, constants, +and callbacks. ## Install @@ -21,15 +11,6 @@ supports init scripts, constants, and callbacks. dotnet add package SquidStd.Scripting.Lua ``` -## Features - -- `IScriptEngineService` — load and run Lua scripts; register modules, constants, callbacks, init scripts. -- Attribute-based modules: mark a class `[ScriptModule]` and methods `[ScriptFunction]` to expose them. -- `RegisterScriptModuleAttribute` — opt a `[ScriptModule]` class into generated registration. -- `container.RegisterScriptModule()` / `RegisterLuaUserData()` registration extensions. -- Event bridging to the SquidStd event bus (`ILuaEventBridge`). -- Built-in modules (logging, events, random) and `.luarc` documentation generation. - ## Usage ```csharp @@ -62,6 +43,10 @@ container.RegisterGeneratedScriptModules(); | `RegisterScriptModuleAttribute` | Marks script modules for generated registration. | | `AddScriptModuleExtension` | `RegisterScriptModule()` / `RegisterLuaUserData()`. | +## Related + +- Tutorial: [Scripting with Lua](https://tgiachi.github.io/squid-std/tutorials/scripting-lua.html) + ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). diff --git a/src/SquidStd.Telemetry.Abstractions/README.md b/src/SquidStd.Telemetry.Abstractions/README.md index 42c510f5..5f1405dd 100644 --- a/src/SquidStd.Telemetry.Abstractions/README.md +++ b/src/SquidStd.Telemetry.Abstractions/README.md @@ -11,7 +11,7 @@ dependency: custom spans use the BCL `System.Diagnostics.ActivitySource`. dotnet add package SquidStd.Telemetry.Abstractions ``` -## Custom spans +## Usage ```csharp using SquidStd.Telemetry.Abstractions; @@ -22,3 +22,15 @@ activity?.SetTag("order.id", orderId); Subsystems create their own `new ActivitySource("SquidStd.Something")`; the OpenTelemetry provider captures every `SquidStd.*` source automatically. + +## Key types + +| Type | Purpose | +|------|---------| +| `TelemetryOptions` | Service name, OTLP endpoint/protocol, sampling and exporter configuration. | +| `OtlpProtocolType` | OTLP transport: gRPC or HTTP. | +| `SquidStdActivity` | Shared `ActivitySource` and the `SquidStd.*` source naming convention. | + +## License + +MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). diff --git a/src/SquidStd.Telemetry.OpenTelemetry/README.md b/src/SquidStd.Telemetry.OpenTelemetry/README.md index 8893093b..82c9c48d 100644 --- a/src/SquidStd.Telemetry.OpenTelemetry/README.md +++ b/src/SquidStd.Telemetry.OpenTelemetry/README.md @@ -35,3 +35,16 @@ builder.Services.AddSquidStdTelemetry(new TelemetryOptions { ServiceName = "orde ActivitySource; ParentBased ratio sampler. - Metrics: runtime instrumentation + the SquidStd metrics snapshot bridged to OTel instruments. - Exporters: OTLP (gRPC/HTTP) plus an optional console exporter for development. + +## Key types + +| Type | Purpose | +|------|---------| +| `OpenTelemetryContainerExtensions` | `AddSquidStdTelemetry(...)` for the DryIoc/worker host. | +| `OpenTelemetryServiceCollectionExtensions` | `AddSquidStdTelemetry(...)` for the ASP.NET Core host. | +| `TelemetryService` | Configures tracing/metrics providers and exporters from `TelemetryOptions`. | +| `MetricsSnapshotBridge` | Exports the existing SquidStd metrics snapshot to OTel instruments. | + +## License + +MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). diff --git a/src/SquidStd.Templates/README.md b/src/SquidStd.Templates/README.md index fce6efba..07904d96 100644 --- a/src/SquidStd.Templates/README.md +++ b/src/SquidStd.Templates/README.md @@ -1,16 +1,5 @@ -

- SquidStd -

-

SquidStd.Templates

-

- NuGet - Downloads - docs - license -

- `dotnet new` templates for scaffolding SquidStd projects. The generated projects reference the SquidStd packages at the same version as this template pack. @@ -39,6 +28,10 @@ dotnet new squidstd-manager -n Acme.Manager `--messaging` (`rabbitmq` default | `inmemory`) is available on the worker and manager templates. +## Related + +- Tutorial: [Scaffolding projects](https://tgiachi.github.io/squid-std/tutorials/scaffolding.html) + ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). diff --git a/src/SquidStd.Templating/README.md b/src/SquidStd.Templating/README.md index aebc6a33..0955e1d8 100644 --- a/src/SquidStd.Templating/README.md +++ b/src/SquidStd.Templating/README.md @@ -1,19 +1,10 @@ -

- SquidStd -

-

SquidStd.Templating

-

- NuGet - Downloads - docs - license -

- Templating for SquidStd, backed by [Scriban](https://github.com/scriban/scriban). `ITemplateRenderer` -renders ad-hoc template strings and named templates (compiled and cached), and auto-loads -`*.tmpl` files from the `templates/` directory at startup. +renders ad-hoc template strings and named templates (compiled and cached), and auto-loads every +`templates/**/*.tmpl` file at startup by relative path without extension (`emails/welcome.tmpl` → +`emails/welcome`). Scriban default member naming applies (`snake_case`): a `UserName` property is +`{{ user.user_name }}`. ## Install @@ -21,15 +12,6 @@ renders ad-hoc template strings and named templates (compiled and cached), and a dotnet add package SquidStd.Templating ``` -## Features - -- `ITemplateRenderer.RenderAsync(template, model)` — render an ad-hoc template string. -- `Register(name, template)` + `RenderByNameAsync(name, model)` — compiled, cached named templates. -- Startup auto-load: every `templates/**/*.tmpl` is registered by relative path without extension (`emails/welcome.tmpl` → - `emails/welcome`). -- Scriban default member naming (`snake_case`): a `UserName` property is `{{ user.user_name }}`. -- Parse/render failures surface as `TemplateException`; unknown names as `InvalidOperationException`. - ## Usage ```csharp @@ -58,6 +40,10 @@ var welcome = await renderer.RenderByNameAsync("welcome", new { User = new { Nam | `TemplateException` | Raised on parse/render failures. | | `TemplatingRegistrationExtensions` | `AddTemplating(...)` registration. | +## Related + +- Tutorial: [Templating](https://tgiachi.github.io/squid-std/tutorials/templating.html) + ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). From f1befecca48ce46ceb0b79321dac69d90d3e7df7 Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 21:21:44 +0200 Subject: [PATCH 46/77] docs: normalise core & hosting package READMEs --- src/SquidStd.Abstractions/README.md | 15 +++------- src/SquidStd.AspNetCore/README.md | 13 +++------ src/SquidStd.Core/README.md | 19 +++---------- src/SquidStd.Generators/README.md | 33 +++++++++++++++++----- src/SquidStd.Plugin.Abstractions/README.md | 10 +++---- src/SquidStd.Services.Core/README.md | 18 ++++-------- 6 files changed, 47 insertions(+), 61 deletions(-) diff --git a/src/SquidStd.Abstractions/README.md b/src/SquidStd.Abstractions/README.md index 3a83a2d9..d4829145 100644 --- a/src/SquidStd.Abstractions/README.md +++ b/src/SquidStd.Abstractions/README.md @@ -21,17 +21,6 @@ discoverable way (tracked through ordered registration lists). dotnet add package SquidStd.Abstractions ``` -## Features - -- `ISquidStdService` — a `StartAsync`/`StopAsync` lifecycle contract for managed services. -- `RegisterEventListenerAttribute` — mark `IEventListener` classes for generated registration. -- `RegisterStdServiceAttribute` — mark service implementations for generated lifecycle registration. -- `RegisterConfigSectionAttribute` — mark config models for generated config-section registration. -- `RegisterStdService()` — register a singleton service and record it in the - ordered service list (with optional priority). -- `RegisterConfigSection(sectionName)` — register a YAML config section for the config manager. -- `AddToRegisterTypedList(...)` — maintain ordered registration lists in the container. - ## Usage ```csharp @@ -59,6 +48,10 @@ container.RegisterConfigSection("my"); | `ServiceRegistrationData` | Ordered service registration record. | | `ConfigRegistrationData` | Config section registration record. | +## Related + +- Tutorial: [Source generators: registration](https://tgiachi.github.io/squid-std/tutorials/source-generators-registration.html) + ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). diff --git a/src/SquidStd.AspNetCore/README.md b/src/SquidStd.AspNetCore/README.md index 2c621eb4..0d12ec48 100644 --- a/src/SquidStd.AspNetCore/README.md +++ b/src/SquidStd.AspNetCore/README.md @@ -21,15 +21,6 @@ DryIoc container into the web host and registers a hosted service that starts an dotnet add package SquidStd.AspNetCore ``` -## Features - -- `WebApplicationBuilder.UseSquidStd(...)` — plug the SquidStd container and services into a web app. -- Configures DryIoc as the host's service-provider factory. -- Registers `SquidStdHostedService` to start/stop SquidStd services with the host. -- Optional `SquidStdOptions` configuration callback. -- `WebApplicationBuilder.AddSquidStdHealthChecks()` — bridges every SquidStd `IHealthCheck` into the standard ASP.NET Core - health-check system (one entry per check), exposed via the standard `app.MapHealthChecks(...)`. - ## Usage ```csharp @@ -71,6 +62,10 @@ Each registered `IHealthCheck` appears as its own entry in the report. Check nam | `SquidStdHostedService` | Hosted service bridging SquidStd service lifecycle to the host. | | `SquidStdHealthChecksExtensions` | `AddSquidStdHealthChecks(...)` — bridge to ASP.NET Core health checks. | +## Related + +- Tutorial: [Build an ASP.NET Core app](https://tgiachi.github.io/squid-std/tutorials/aspnetcore-app.html) + ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). diff --git a/src/SquidStd.Core/README.md b/src/SquidStd.Core/README.md index 711bedae..b96c1c09 100644 --- a/src/SquidStd.Core/README.md +++ b/src/SquidStd.Core/README.md @@ -22,21 +22,6 @@ SquidStd packages build on. dotnet add package SquidStd.Core ``` -## Features - -- Configuration contracts: `IConfigEntry` (a YAML section) and `IConfigManagerService`. -- In-process messaging: `IEventBus` with `ISyncEventListener` / `IAsyncEventListener` over `IEvent`. -- Command dispatch: `ICommandDispatcher` with `ICommandHandler`, fan-out, fault isolation, and a `CommandDispatchResult`; `ICommandContextFactory` builds the context from a seed for `ISeededCommandDispatcher`. -- Background work & timing: `IJobSystem`, `ITimerService`, `IMainThreadDispatcher`. -- Metrics & secrets: `IMetricProvider` and secret-protection contracts. -- Serialization: `IDataSerializer` / `IDataDeserializer` (default `JsonDataSerializer`), plus `YamlUtils` / `JsonUtils`. -- File watching: `IFileWatcherService` / `FileWatcherService` — recursive, debounced watchers that publish `FileChangedEvent` on the event bus. -- Object pooling: `ObjectPool` — thread-safe, non-blocking, factory-based reuse with optional reset. -- Cryptography: `CryptoUtils` (AES-GCM authenticated encrypt/decrypt + key generation), `EncryptString`/`DecryptString` string helpers, base64 extensions, and `SslUtils` for loading PEM/PFX TLS certificates. -- Randomness: `BuiltInRng` (seedable ambient RNG), `RandomUtils` (dice, coin flips), and collection `Shuffle`/`RandomElement`/`RandomSample` extensions. -- Utilities: a Serilog `EventSink`, and string/env/directory extensions. -- Shared domain enums under `Types` (e.g. `LogLevelType`, `PlatformType`, `FileChangeKind`). - ## Usage ```csharp @@ -88,6 +73,10 @@ pool.Return(builder); | `ObjectPool` | Thread-safe, non-blocking object pool. | | `ICommandDispatcher` | Typed protocol command dispatch with context. | +## Related + +- Tutorial: [Events, jobs & scheduling](https://tgiachi.github.io/squid-std/tutorials/events-jobs-scheduling.html) + ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). diff --git a/src/SquidStd.Generators/README.md b/src/SquidStd.Generators/README.md index 07567ef1..dc4de0d9 100644 --- a/src/SquidStd.Generators/README.md +++ b/src/SquidStd.Generators/README.md @@ -1,4 +1,15 @@ -# SquidStd.Generators +

+ SquidStd +

+ +

SquidStd.Generators

+ +

+ NuGet + Downloads + docs + license +

Roslyn source generators for SquidStd compile-time registration helpers. @@ -8,7 +19,7 @@ Roslyn source generators for SquidStd compile-time registration helpers. dotnet add package SquidStd.Generators ``` -## Event listeners +## Usage The event listener generator discovers concrete `IEventListener` implementations marked with `[RegisterEventListener]` and generates a DryIoc registration extension: @@ -26,17 +37,25 @@ public sealed class PingListener : IEventListener container.RegisterGeneratedEventListeners(); ``` -The generated method reuses the normal `RegisterEventListener()` runtime path, so listener activation stays compatible with `SquidStd.Services.Core`. - -## Other generated registrations +The generated method reuses the normal `RegisterEventListener()` runtime path, so listener activation stays compatible with `SquidStd.Services.Core`. Each registration family has its own marker attribute and generated extension method, all calling the same runtime APIs as manual registration (`RegisterStdService`, `RegisterConfigSection`, `AddJobHandler`, `RegisterScriptModule`). -Each registration family has its own marker attribute and generated extension method: +## Key types | Marker attribute | Generated method | |------------------|------------------| +| `[RegisterEventListener]` | `RegisterGeneratedEventListeners()` | | `[RegisterStdService(typeof(IMyService), Priority = 10)]` | `RegisterGeneratedStdServices()` | | `[RegisterConfigSection("workers", Priority = -50)]` | `RegisterGeneratedConfigSections()` | | `[RegisterJobHandler]` | `RegisterGeneratedJobHandlers()` | | `[RegisterScriptModule]` with `[ScriptModule("name")]` | `RegisterGeneratedScriptModules()` | -The generated methods call the same runtime APIs as manual registration: `RegisterStdService`, `RegisterConfigSection`, `AddJobHandler`, and `RegisterScriptModule`. +## Related + +- Tutorial: [Source generators: event listeners](https://tgiachi.github.io/squid-std/tutorials/source-generators-event-listeners.html) +- Tutorial: [Source generators: registration](https://tgiachi.github.io/squid-std/tutorials/source-generators-registration.html) + +## License + +MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). + + diff --git a/src/SquidStd.Plugin.Abstractions/README.md b/src/SquidStd.Plugin.Abstractions/README.md index 57af815d..0952933b 100644 --- a/src/SquidStd.Plugin.Abstractions/README.md +++ b/src/SquidStd.Plugin.Abstractions/README.md @@ -21,12 +21,6 @@ with shared boot data. dotnet add package SquidStd.Plugin.Abstractions ``` -## Features - -- `ISquidStdPlugin` — the plugin entry point: `Metadata` + `Configure(IContainer, PluginContext)`. -- `PluginMetadata` — id, name, `Version`, author, optional description, and dependency declarations. -- `PluginContext` — a typed bag of boot data shared with the plugin (`GetData(key)`). - ## Usage ```csharp @@ -59,6 +53,10 @@ public sealed class MyPlugin : ISquidStdPlugin | `PluginMetadata` | Plugin identity and dependency declarations. | | `PluginContext` | Shared boot data passed to the plugin. | +## Related + +- Tutorial: [Plugins](https://tgiachi.github.io/squid-std/tutorials/plugins.html) + ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). diff --git a/src/SquidStd.Services.Core/README.md b/src/SquidStd.Services.Core/README.md index d541f658..e009f7fb 100644 --- a/src/SquidStd.Services.Core/README.md +++ b/src/SquidStd.Services.Core/README.md @@ -21,18 +21,6 @@ main-thread dispatcher, metrics collection, storage, and secrets services. dotnet add package SquidStd.Services.Core ``` -## Features - -- One-line bootstrap: `container.RegisterCoreServices()` registers the full default service set. -- `ConfigManagerService` — loads/saves YAML config sections and substitutes `$ENV_VAR` tokens. -- `EventBusService` — in-process publish/subscribe over `IEvent`. -- `CommandDispatcher` — typed protocol command dispatch with fan-out, fault isolation, and a `CommandDispatchResult` (`RegisterCommandDispatcher` / `RegisterCommandHandler` / `RegisterSeededCommandDispatcher`). -- `JobSystemService` — background job execution; `TimerWheelService` + cron scheduling for timed work. -- `MainThreadDispatcherService` — marshal work back onto a main thread. -- `MetricsCollectionService` — aggregates `IMetricProvider` samples. -- `HealthCheckService` — aggregates `IHealthCheck`s into one report (`RegisterHealthChecksService`). -- AES-GCM-protected secret store. (File/object storage moved to `SquidStd.Storage`, opt-in via `AddFileStorage`.) - ## Usage ```csharp @@ -45,7 +33,7 @@ var container = new Container(); container.RegisterCoreServices("squidstd", Directory.GetCurrentDirectory()); ``` -## Command dispatch +### Command dispatch ```csharp // Register a dispatcher for your context type and handlers. @@ -85,6 +73,10 @@ await seeded.DispatchAsync(command, connection); // factory maps connection -> | `MainThreadDispatcherService` | Main-thread work dispatch. | | `MetricsCollectionService` | Metric sample aggregation. | +## Related + +- Tutorial: [Events, jobs & scheduling](https://tgiachi.github.io/squid-std/tutorials/events-jobs-scheduling.html) + ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). From fefba859f28b4cb0add028bf933410674b846db4 Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 21:23:50 +0200 Subject: [PATCH 47/77] docs: normalise networking & actors package READMEs --- src/SquidStd.Actors/README.md | 18 +++++++++--------- src/SquidStd.Network/README.md | 12 ++++-------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/src/SquidStd.Actors/README.md b/src/SquidStd.Actors/README.md index d71004a4..3c5b15e6 100644 --- a/src/SquidStd.Actors/README.md +++ b/src/SquidStd.Actors/README.md @@ -53,6 +53,15 @@ using SquidStd.Actors.Extensions; using var sub = session.SubscribeToEventBus(eventBus, (UserJoinedEvent e) => new SendText($":{e.Nick} JOIN")); ``` +## Key types + +| Type | Purpose | +|---------------------------|----------------------------------------------------| +| `Actor` | Mailbox base class (`TellAsync` / `AskAsync`). | +| `ActorRequest` | Base record for request/response messages. | +| `IActorRequest` | Request contract (implement directly if not using the base). | +| `ActorOptions` | Capacity / overflow / error configuration. | + ## Options `ActorOptions` controls the mailbox: @@ -70,15 +79,6 @@ using var sub = session.SubscribeToEventBus(eventBus, (UserJoinedEvent e) => new `DisposeAsync` completes the mailbox, drains in-flight work, and faults any still-pending requests. -## Key types - -| Type | Purpose | -|---------------------------|----------------------------------------------------| -| `Actor` | Mailbox base class (`TellAsync` / `AskAsync`). | -| `ActorRequest` | Base record for request/response messages. | -| `IActorRequest` | Request contract (implement directly if not using the base). | -| `ActorOptions` | Capacity / overflow / error configuration. | - ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). diff --git a/src/SquidStd.Network/README.md b/src/SquidStd.Network/README.md index 6a2cb3c6..122c178d 100644 --- a/src/SquidStd.Network/README.md +++ b/src/SquidStd.Network/README.md @@ -21,14 +21,6 @@ designed for low-allocation, high-throughput byte processing. dotnet add package SquidStd.Network ``` -## Features - -- TCP server/client (`SquidTcpServer`, `SquidStdTcpClient`) with optional TLS. -- UDP server/client (`SquidStdUdpServer`, `SquidStdUdpClient`). -- Session management (`ISessionManager`) with typed per-connection state. -- Composable framing (`INetFramer`) and middleware pipeline (`INetMiddleware`). -- Zero-copy binary I/O via `SpanReader` / `SpanWriter` and a reusable `CircularBuffer`. - ## Usage ```csharp @@ -52,6 +44,10 @@ await server.StopAsync(CancellationToken.None); | `INetMiddleware` | Pipeline stage over inbound/outbound data. | | `SpanReader` / `SpanWriter` | Allocation-free binary read/write. | +## Related + +- Tutorial: [Networking](https://tgiachi.github.io/squid-std/tutorials/networking.html) + ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). From 3e458ce1a98663fd91182b77b80f165e5516318f Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 21:26:30 +0200 Subject: [PATCH 48/77] docs: normalise messaging package READMEs --- src/SquidStd.Messaging.Abstractions/README.md | 13 +++------ src/SquidStd.Messaging.RabbitMq/README.md | 11 +++---- src/SquidStd.Messaging.Sqs/README.md | 29 +++++++++++++++++++ src/SquidStd.Messaging/README.md | 12 +++----- 4 files changed, 41 insertions(+), 24 deletions(-) diff --git a/src/SquidStd.Messaging.Abstractions/README.md b/src/SquidStd.Messaging.Abstractions/README.md index 6995e8d6..98adf634 100644 --- a/src/SquidStd.Messaging.Abstractions/README.md +++ b/src/SquidStd.Messaging.Abstractions/README.md @@ -22,15 +22,6 @@ implementation (in-memory or RabbitMQ) from a companion package. dotnet add package SquidStd.Messaging.Abstractions ``` -## Features - -- `IMessageQueue` — typed `PublishAsync` / `Subscribe` facade over named queues. -- `IQueueProvider` — the raw transport contract implemented per backend. -- `IMessageSerializer` (+ default `JsonMessageSerializer`) — payload (de)serialization. -- `IQueueMessageListener` / `IQueueMessageListenerAsync` — sync/async subscribers. -- `IMessagingMetrics` (+ `MessagingMetricsProvider`, `NoOpMessagingMetrics`) — delivery metrics. -- `MessagingOptions` and `MessagingConnectionString` — configuration and connection parsing. - ## Usage ```csharp @@ -59,6 +50,10 @@ public async Task PublishAsync(IMessageQueue queue) | `IMessagingMetrics` | Delivery metrics sink. | | `MessagingOptions` | Delivery attempts, retry delay, and dead-letter-queue suffix. | +## Related + +- Tutorial: [Messaging](https://tgiachi.github.io/squid-std/tutorials/messaging.html) + ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). diff --git a/src/SquidStd.Messaging.RabbitMq/README.md b/src/SquidStd.Messaging.RabbitMq/README.md index 8c2fe3f3..19feb08f 100644 --- a/src/SquidStd.Messaging.RabbitMq/README.md +++ b/src/SquidStd.Messaging.RabbitMq/README.md @@ -21,13 +21,6 @@ so the same `IMessageQueue` API publishes to and consumes from a real broker. Re dotnet add package SquidStd.Messaging.RabbitMq ``` -## Features - -- One-line registration: `container.AddRabbitMqMessaging(connectionString)` or with `RabbitMqOptions`. -- Broker-backed `IQueueProvider` reusing the shared `IMessageQueue` facade and serializer. -- Connection via a `rabbitmq://` connection string or explicit `RabbitMqOptions` (host/port/vhost/credentials). -- Configurable consumer prefetch count (`?prefetch=` on the connection string, or `RabbitMqOptions.PrefetchCount`). - ## Usage ```csharp @@ -50,6 +43,10 @@ await queue.PublishAsync("orders", new { Id = 1 }); | `RabbitMqQueueProvider` | RabbitMQ-backed `IQueueProvider`. | | `RabbitMqOptions` | Connection + prefetch configuration. | +## Related + +- Tutorial: [Messaging](https://tgiachi.github.io/squid-std/tutorials/messaging.html) + ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). diff --git a/src/SquidStd.Messaging.Sqs/README.md b/src/SquidStd.Messaging.Sqs/README.md index 1cc0ae7e..49be76c5 100644 --- a/src/SquidStd.Messaging.Sqs/README.md +++ b/src/SquidStd.Messaging.Sqs/README.md @@ -1,5 +1,16 @@ +

+ SquidStd +

+

SquidStd.Messaging.Sqs

+

+ NuGet + Downloads + docs + license +

+ AWS SQS/SNS transport for SquidStd.Messaging. Implements `IQueueProvider` over SQS (with a redrive policy to a dead-letter queue) and `ITopicProvider` via SNS+SQS fan-out, behind the same `IMessageQueue` / `IMessageTopic` API as the other providers. Registered with a single @@ -29,3 +40,21 @@ container.AddSqsMessaging(new SqsOptions { Aws = new AwsConfigEntry { Region = " - Topics map to SNS; each subscriber gets a dedicated SQS queue subscribed with raw message delivery. - Names are sanitized to the SQS/SNS alphabet (the default `.dlq` suffix becomes `-dlq`). - Payloads travel base64-encoded in the message body. + +## Key types + +| Type | Purpose | +|-------------------------------------|------------------------------------------| +| `SqsMessagingRegistrationExtensions` | `AddSqsMessaging(...)` registration. | +| `SqsQueueProvider` | SQS-backed `IQueueProvider`. | +| `SqsTopicProvider` | SNS+SQS-backed `ITopicProvider`. | +| `SqsOptions` | AWS connection + queue/topic configuration. | + +## Related + +- Tutorial: [Messaging](https://tgiachi.github.io/squid-std/tutorials/messaging.html) + +## License + +MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). + diff --git a/src/SquidStd.Messaging/README.md b/src/SquidStd.Messaging/README.md index dec66618..8f1841a2 100644 --- a/src/SquidStd.Messaging/README.md +++ b/src/SquidStd.Messaging/README.md @@ -21,14 +21,6 @@ with a single `AddInMemoryMessaging()` call. Ideal for single-process apps, test dotnet add package SquidStd.Messaging ``` -## Features - -- One-line registration: `container.AddInMemoryMessaging()` (facade, provider, serializer, metrics). -- Channel-based per-queue buffering with a dedicated consumer loop. -- Round-robin delivery across multiple subscribers of the same queue. -- Retry with configurable max attempts and dead-letter queues (`MessagingOptions`). -- Built-in delivery metrics via `MessagingMetricsProvider`. - ## Usage ```csharp @@ -50,6 +42,10 @@ await queue.PublishAsync("orders", new { Id = 1 }); | `MessagingRegistrationExtensions` | `AddInMemoryMessaging(...)` registration. | | `InMemoryQueueProvider` | Channel-based in-memory `IQueueProvider`. | +## Related + +- Tutorial: [Messaging](https://tgiachi.github.io/squid-std/tutorials/messaging.html) + ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). From bbf6d1b60fb81389e8086dfd2f6b2e1937c89ce7 Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 21:28:39 +0200 Subject: [PATCH 49/77] docs: normalise persistence & database package READMEs --- src/SquidStd.Database.Abstractions/README.md | 13 ++++--------- src/SquidStd.Database/README.md | 13 ++++--------- src/SquidStd.Persistence/README.md | 16 ---------------- 3 files changed, 8 insertions(+), 34 deletions(-) diff --git a/src/SquidStd.Database.Abstractions/README.md b/src/SquidStd.Database.Abstractions/README.md index 0526ab16..abd63625 100644 --- a/src/SquidStd.Database.Abstractions/README.md +++ b/src/SquidStd.Database.Abstractions/README.md @@ -21,15 +21,6 @@ operations, paging, and composable queries — without binding to any specific O dotnet add package SquidStd.Database.Abstractions ``` -## Features - -- `IDataAccess` — `InsertAsync`, `GetByIdAsync`, `UpdateAsync`, `DeleteAsync`, `CountAsync`, - `ExistsAsync`, bulk insert/update/delete, `QueryAsync`, and `GetPagedAsync`. -- `BaseEntity` — `Guid Id` plus `DateTimeOffset Created` / `Updated` (UTC), set by the data layer. -- `PagedResultData` — items + `Page`, `PageSize`, `TotalCount`, `TotalPages`, `HasNext`, `HasPrevious`. -- `DatabaseConfig` — URI connection string + `AutoMigrate` flag. -- `DatabaseProviderType` — `Sqlite`, `Postgres`, `SqlServer`, `MySql`. - ## Usage ```csharp @@ -59,6 +50,10 @@ public async Task ExampleAsync(IDataAccess users) | `DatabaseConfig` | Connection string + auto-migrate config section. | | `DatabaseProviderType` | Supported provider enum. | +## Related + +- Tutorial: [Database](https://tgiachi.github.io/squid-std/tutorials/database.html) + ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). diff --git a/src/SquidStd.Database/README.md b/src/SquidStd.Database/README.md index 87db6627..f9664144 100644 --- a/src/SquidStd.Database/README.md +++ b/src/SquidStd.Database/README.md @@ -22,15 +22,6 @@ on startup. dotnet add package SquidStd.Database ``` -## Features - -- One-line registration: `container.RegisterDatabase()` (config section + service + open-generic `IDataAccess<>`). -- Providers via URI scheme: `sqlite://`, `postgres://`, `sqlserver://`, `mysql://`. -- `FreeSqlDataAccess` — CRUD, bulk insert/update/delete, `QueryAsync`, `GetPagedAsync`; writes - run inside a unit of work and roll back on error. Sets `Id`/`Created`/`Updated` automatically. -- Optional `AutoMigrate` (FreeSql `AutoSyncStructure`) to create/update tables on startup. -- ZLinq in-memory helpers for zero-allocation post-processing of materialized results. - ## Usage ```csharp @@ -57,6 +48,10 @@ var page = await users.GetPagedAsync(page: 1, pageSize: 20, orderBy: u => u.Name | `ConnectionStringParser` | URI → provider + native connection string. | | `ZLinqResultExtensions` | Zero-alloc in-memory result helpers. | +## Related + +- Tutorial: [Database](https://tgiachi.github.io/squid-std/tutorials/database.html) + ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). diff --git a/src/SquidStd.Persistence/README.md b/src/SquidStd.Persistence/README.md index d6afcd5b..9a1ed633 100644 --- a/src/SquidStd.Persistence/README.md +++ b/src/SquidStd.Persistence/README.md @@ -23,22 +23,6 @@ dotnet add package SquidStd.Persistence dotnet add package SquidStd.Persistence.MessagePack # recommended binary serializer ``` -## Features - -- **Snapshot + journal**: in-memory state, WAL journal of every upsert/remove, periodic full snapshot + trim. -- **Crash-safe**: journal records are length+FNV-1a-checksum framed — a torn/corrupt trailing record is - detected on read and the tail is discarded. Snapshots are written atomically (temp + rename). -- **Serializer-agnostic**: per-entity payloads go through `IDataSerializer`/`IDataDeserializer`; the journal - and snapshot envelopes use a fixed binary layout. Pair with `SquidStd.Persistence.MessagePack` for a - compact binary default, or use the JSON serializer from `SquidStd.Core`. -- **Detached reads**: `GetByIdAsync`/`GetAllAsync`/`Query()` return deep clones, so callers never mutate - stored instances. -- **Write-ordered journaling**: writes serialize end-to-end (apply + append) so journal order always - matches sequence order — replay is deterministic. -- **Lifecycle service**: `PersistenceService` is an `ISquidStdService` that loads + replays at start, - autosaves on a timer, and snapshots on stop. Optional `IEventBus` integration raises - `SnapshotSaveStartedEvent`/`SnapshotSaveCompletedEvent`. - ## Usage ```csharp From 943507bc41f2b11a15fd08a780e5b3ef6d62cdff Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 21:30:59 +0200 Subject: [PATCH 50/77] docs: normalise caching package READMEs --- src/SquidStd.Caching.Abstractions/README.md | 12 ++++-------- src/SquidStd.Caching.Redis/README.md | 12 ++++-------- src/SquidStd.Caching/README.md | 12 ++++-------- 3 files changed, 12 insertions(+), 24 deletions(-) diff --git a/src/SquidStd.Caching.Abstractions/README.md b/src/SquidStd.Caching.Abstractions/README.md index 93d452fe..c9780811 100644 --- a/src/SquidStd.Caching.Abstractions/README.md +++ b/src/SquidStd.Caching.Abstractions/README.md @@ -22,14 +22,6 @@ backend implementation (in-memory or Redis) from a companion package. dotnet add package SquidStd.Caching.Abstractions ``` -## Features - -- `ICacheService` — typed `GetAsync` / `SetAsync` / `RemoveAsync` / `ExistsAsync` / `GetOrSetAsync` facade. -- `ICacheProvider` — the raw byte-level backend contract implemented per provider. -- `CacheService` — shared facade that serializes values, applies the key prefix and default TTL, and implements cache-aside. -- `ICacheMetrics` (+ `CacheMetricsProvider`, `NoOpCacheMetrics`) — hit/miss/set/remove metrics. -- `CacheOptions` and `CacheConnectionString` — configuration and connection parsing. - ## Usage ```csharp @@ -51,6 +43,10 @@ public Task GetOrComputeAsync(ICacheService cache) | `CacheOptions` | Default TTL and key prefix. | | `CacheConnectionString` | `scheme://host[?params]` parsing into `CacheOptions`. | +## Related + +- Tutorial: [Caching](https://tgiachi.github.io/squid-std/tutorials/caching.html) + ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). diff --git a/src/SquidStd.Caching.Redis/README.md b/src/SquidStd.Caching.Redis/README.md index ebaa1742..a84ef265 100644 --- a/src/SquidStd.Caching.Redis/README.md +++ b/src/SquidStd.Caching.Redis/README.md @@ -21,14 +21,6 @@ Registered with a single `AddRedisCache(...)` call. dotnet add package SquidStd.Caching.Redis ``` -## Features - -- One-line registration: `container.AddRedisCache(connectionString)` or with `RedisCacheOptions`. -- Redis-backed `ICacheProvider` reusing the shared `ICacheService` facade and serializer. -- Native TTL via Redis key expiry (`SET ... EX`). -- Connection via a `redis://` connection string or an explicit StackExchange.Redis configuration string. -- Built-in hit/miss metrics via `CacheMetricsProvider`. - ## Usage ```csharp @@ -52,6 +44,10 @@ var user = await cache.GetAsync("user:1"); | `RedisCacheProvider` | StackExchange.Redis-backed `ICacheProvider`. | | `RedisCacheOptions` | StackExchange.Redis connection configuration. | +## Related + +- Tutorial: [Caching](https://tgiachi.github.io/squid-std/tutorials/caching.html) + ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). diff --git a/src/SquidStd.Caching/README.md b/src/SquidStd.Caching/README.md index 9e3f6622..0625be86 100644 --- a/src/SquidStd.Caching/README.md +++ b/src/SquidStd.Caching/README.md @@ -21,14 +21,6 @@ single `AddInMemoryCache()` call. Ideal for single-process apps, tests, and loca dotnet add package SquidStd.Caching ``` -## Features - -- One-line registration: `container.AddInMemoryCache()` (provider, facade, serializer, metrics). -- `IMemoryCache`-backed storage with absolute per-entry TTL and built-in eviction. -- Reuses the shared `CacheService` facade (key prefix, default TTL, cache-aside). -- Built-in hit/miss metrics via `CacheMetricsProvider`. -- Configure via `CacheOptions` or a `memory://` connection string (`?defaultTtlSeconds=`, `?keyPrefix=`). - ## Usage ```csharp @@ -51,6 +43,10 @@ var user = await cache.GetAsync("user:1"); | `CacheRegistrationExtensions` | `AddInMemoryCache(...)` registration. | | `InMemoryCacheProvider` | `IMemoryCache`-backed `ICacheProvider`. | +## Related + +- Tutorial: [Caching](https://tgiachi.github.io/squid-std/tutorials/caching.html) + ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). From eea8bf948983906f41171d419e9c0a2a671b00c4 Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 21:59:11 +0200 Subject: [PATCH 51/77] docs: harmonise package README headers to the bare-h1 template Strip the leading logo block and NuGet badge block from the core/hosting, messaging, persistence, database and caching READMEs so every package README opens with a centered

+ one-line summary, matching the rest of the set. --- src/SquidStd.Abstractions/README.md | 11 ----------- src/SquidStd.AspNetCore/README.md | 11 ----------- src/SquidStd.Caching.Abstractions/README.md | 11 ----------- src/SquidStd.Caching.Redis/README.md | 11 ----------- src/SquidStd.Caching/README.md | 11 ----------- src/SquidStd.Core/README.md | 11 ----------- src/SquidStd.Database.Abstractions/README.md | 11 ----------- src/SquidStd.Database/README.md | 11 ----------- src/SquidStd.Generators/README.md | 11 ----------- src/SquidStd.Messaging.Abstractions/README.md | 11 ----------- src/SquidStd.Messaging.RabbitMq/README.md | 11 ----------- src/SquidStd.Messaging.Sqs/README.md | 11 ----------- src/SquidStd.Messaging/README.md | 11 ----------- src/SquidStd.Network/README.md | 11 ----------- src/SquidStd.Persistence.Abstractions/README.md | 10 ---------- src/SquidStd.Persistence.MessagePack/README.md | 10 ---------- src/SquidStd.Persistence/README.md | 10 ---------- src/SquidStd.Plugin.Abstractions/README.md | 11 ----------- src/SquidStd.Services.Core/README.md | 11 ----------- 19 files changed, 206 deletions(-) diff --git a/src/SquidStd.Abstractions/README.md b/src/SquidStd.Abstractions/README.md index d4829145..00daacf7 100644 --- a/src/SquidStd.Abstractions/README.md +++ b/src/SquidStd.Abstractions/README.md @@ -1,16 +1,5 @@ -

- SquidStd -

-

SquidStd.Abstractions

-

- NuGet - Downloads - docs - license -

- DryIoc-based dependency-injection plumbing for SquidStd. It defines the `ISquidStdService` lifecycle contract and the container extensions used to register services and configuration sections in a uniform, discoverable way (tracked through ordered registration lists). diff --git a/src/SquidStd.AspNetCore/README.md b/src/SquidStd.AspNetCore/README.md index 0d12ec48..bf097116 100644 --- a/src/SquidStd.AspNetCore/README.md +++ b/src/SquidStd.AspNetCore/README.md @@ -1,16 +1,5 @@ -

- SquidStd -

-

SquidStd.AspNetCore

-

- NuGet - Downloads - docs - license -

- ASP.NET Core integration for SquidStd. A single `builder.UseSquidStd(...)` call wires the SquidStd DryIoc container into the web host and registers a hosted service that starts and stops every `ISquidStdService` alongside the application lifecycle. diff --git a/src/SquidStd.Caching.Abstractions/README.md b/src/SquidStd.Caching.Abstractions/README.md index c9780811..81fa611d 100644 --- a/src/SquidStd.Caching.Abstractions/README.md +++ b/src/SquidStd.Caching.Abstractions/README.md @@ -1,16 +1,5 @@ -

- SquidStd -

-

SquidStd.Caching.Abstractions

-

- NuGet - Downloads - docs - license -

- Backend-agnostic caching contracts for SquidStd. It defines the typed cache-aside facade (`ICacheService`), the low-level byte provider/metrics contracts, and the shared `CacheService` facade that applies key-prefixing, default TTL and cache-aside once over any provider. Pick a diff --git a/src/SquidStd.Caching.Redis/README.md b/src/SquidStd.Caching.Redis/README.md index a84ef265..6359de0d 100644 --- a/src/SquidStd.Caching.Redis/README.md +++ b/src/SquidStd.Caching.Redis/README.md @@ -1,16 +1,5 @@ -

- SquidStd -

-

SquidStd.Caching.Redis

-

- NuGet - Downloads - docs - license -

- Redis backend for SquidStd.Caching. Implements `ICacheProvider` on top of StackExchange.Redis, so the same `ICacheService` API reads and writes a real Redis server with native key expiry. Registered with a single `AddRedisCache(...)` call. diff --git a/src/SquidStd.Caching/README.md b/src/SquidStd.Caching/README.md index 0625be86..fa3ece9b 100644 --- a/src/SquidStd.Caching/README.md +++ b/src/SquidStd.Caching/README.md @@ -1,16 +1,5 @@ -

- SquidStd -

-

SquidStd.Caching

-

- NuGet - Downloads - docs - license -

- In-memory backend for SquidStd.Caching. Provides an `IMemoryCache`-backed `ICacheProvider` with absolute TTL and eviction, wired to the shared typed `ICacheService` facade — registered with a single `AddInMemoryCache()` call. Ideal for single-process apps, tests, and local dev. diff --git a/src/SquidStd.Core/README.md b/src/SquidStd.Core/README.md index b96c1c09..356350ad 100644 --- a/src/SquidStd.Core/README.md +++ b/src/SquidStd.Core/README.md @@ -1,16 +1,5 @@ -

- SquidStd -

-

SquidStd.Core

-

- NuGet - Downloads - docs - license -

- Foundational contracts and utilities for the SquidStd stack. It defines the core service interfaces (configuration, event bus, jobs, timing, metrics, storage) and ships dependency-free helpers — YAML/JSON serialization, a Serilog event sink, and string/environment/directory extensions — that the other diff --git a/src/SquidStd.Database.Abstractions/README.md b/src/SquidStd.Database.Abstractions/README.md index abd63625..bf5a5ca6 100644 --- a/src/SquidStd.Database.Abstractions/README.md +++ b/src/SquidStd.Database.Abstractions/README.md @@ -1,16 +1,5 @@ -

- SquidStd -

-

SquidStd.Database.Abstractions

-

- NuGet - Downloads - docs - license -

- Provider-agnostic data-access contracts for SquidStd. Entities derive from `BaseEntity` (a `Guid` id plus UTC timestamps) and are accessed through the generic `IDataAccess` — full CRUD, bulk operations, paging, and composable queries — without binding to any specific ORM. diff --git a/src/SquidStd.Database/README.md b/src/SquidStd.Database/README.md index f9664144..243d0af7 100644 --- a/src/SquidStd.Database/README.md +++ b/src/SquidStd.Database/README.md @@ -1,16 +1,5 @@ -

- SquidStd -

-

SquidStd.Database

-

- NuGet - Downloads - docs - license -

- FreeSql-backed implementation of the SquidStd.Database.Abstractions contracts. It owns a singleton `IFreeSql`, exposes a generic `FreeSqlDataAccess` with transactional writes (rollback on failure), bulk operations and paging, parses URI-style connection strings, and can auto-sync the schema diff --git a/src/SquidStd.Generators/README.md b/src/SquidStd.Generators/README.md index dc4de0d9..1b711b57 100644 --- a/src/SquidStd.Generators/README.md +++ b/src/SquidStd.Generators/README.md @@ -1,16 +1,5 @@ -

- SquidStd -

-

SquidStd.Generators

-

- NuGet - Downloads - docs - license -

- Roslyn source generators for SquidStd compile-time registration helpers. ## Install diff --git a/src/SquidStd.Messaging.Abstractions/README.md b/src/SquidStd.Messaging.Abstractions/README.md index 98adf634..67656222 100644 --- a/src/SquidStd.Messaging.Abstractions/README.md +++ b/src/SquidStd.Messaging.Abstractions/README.md @@ -1,16 +1,5 @@ -

- SquidStd -

-

SquidStd.Messaging.Abstractions

-

- NuGet - Downloads - docs - license -

- Transport-agnostic messaging contracts for SquidStd. It defines the typed queue facade (`IMessageQueue`), the low-level provider/serializer/metrics contracts, the message-listener interfaces, and the shared `MessageQueue` facade plus default serializer and metrics. Pick a transport diff --git a/src/SquidStd.Messaging.RabbitMq/README.md b/src/SquidStd.Messaging.RabbitMq/README.md index 19feb08f..5617a9cd 100644 --- a/src/SquidStd.Messaging.RabbitMq/README.md +++ b/src/SquidStd.Messaging.RabbitMq/README.md @@ -1,16 +1,5 @@ -

- SquidStd -

-

SquidStd.Messaging.RabbitMq

-

- NuGet - Downloads - docs - license -

- RabbitMQ transport for SquidStd.Messaging. Implements `IQueueProvider` on top of the RabbitMQ client, so the same `IMessageQueue` API publishes to and consumes from a real broker. Registered with a single `AddRabbitMqMessaging(...)` call. diff --git a/src/SquidStd.Messaging.Sqs/README.md b/src/SquidStd.Messaging.Sqs/README.md index 49be76c5..2b4017a6 100644 --- a/src/SquidStd.Messaging.Sqs/README.md +++ b/src/SquidStd.Messaging.Sqs/README.md @@ -1,16 +1,5 @@ -

- SquidStd -

-

SquidStd.Messaging.Sqs

-

- NuGet - Downloads - docs - license -

- AWS SQS/SNS transport for SquidStd.Messaging. Implements `IQueueProvider` over SQS (with a redrive policy to a dead-letter queue) and `ITopicProvider` via SNS+SQS fan-out, behind the same `IMessageQueue` / `IMessageTopic` API as the other providers. Registered with a single diff --git a/src/SquidStd.Messaging/README.md b/src/SquidStd.Messaging/README.md index 8f1841a2..98a37407 100644 --- a/src/SquidStd.Messaging/README.md +++ b/src/SquidStd.Messaging/README.md @@ -1,16 +1,5 @@ -

- SquidStd -

-

SquidStd.Messaging

-

- NuGet - Downloads - docs - license -

- In-memory transport for SquidStd.Messaging. Provides a channel-backed `IQueueProvider` with per-queue buffering, round-robin delivery to subscribers, retry/dead-letter handling, and metrics — registered with a single `AddInMemoryMessaging()` call. Ideal for single-process apps, tests, and local dev. diff --git a/src/SquidStd.Network/README.md b/src/SquidStd.Network/README.md index 122c178d..a8315ecc 100644 --- a/src/SquidStd.Network/README.md +++ b/src/SquidStd.Network/README.md @@ -1,16 +1,5 @@ -

- SquidStd -

-

SquidStd.Network

-

- NuGet - Downloads - docs - license -

- Networking primitives for SquidStd: TCP and UDP servers and clients with per-connection sessions, a pluggable framing + middleware pipeline, span-based binary readers/writers, and a circular buffer — designed for low-allocation, high-throughput byte processing. diff --git a/src/SquidStd.Persistence.Abstractions/README.md b/src/SquidStd.Persistence.Abstractions/README.md index b0cbb6b9..1276938d 100644 --- a/src/SquidStd.Persistence.Abstractions/README.md +++ b/src/SquidStd.Persistence.Abstractions/README.md @@ -1,15 +1,5 @@ -

- SquidStd -

-

SquidStd.Persistence.Abstractions

-

- NuGet - Downloads - license -

- Contracts and DTOs for SquidStd's binary persistence engine. Depend on this package to expose or consume persistence types without pulling in the engine implementation. diff --git a/src/SquidStd.Persistence.MessagePack/README.md b/src/SquidStd.Persistence.MessagePack/README.md index 4041619c..643f25f6 100644 --- a/src/SquidStd.Persistence.MessagePack/README.md +++ b/src/SquidStd.Persistence.MessagePack/README.md @@ -1,15 +1,5 @@ -

- SquidStd -

-

SquidStd.Persistence.MessagePack

-

- NuGet - Downloads - license -

- MessagePack-backed binary `IDataSerializer`/`IDataDeserializer` for `SquidStd.Persistence`. The recommended compact binary default for entity payloads. Uses the contractless resolver, so plain POCOs need no attributes. diff --git a/src/SquidStd.Persistence/README.md b/src/SquidStd.Persistence/README.md index 9a1ed633..e0842c16 100644 --- a/src/SquidStd.Persistence/README.md +++ b/src/SquidStd.Persistence/README.md @@ -1,15 +1,5 @@ -

- SquidStd -

-

SquidStd.Persistence

-

- NuGet - Downloads - license -

- Embeddable in-memory entity store with durable **binary snapshot + journal (write-ahead log)** persistence. Full state lives in memory (synchronous reads), every mutation is appended to a length+checksum-framed binary journal, and a periodic snapshot captures all state and trims the journal. On startup the engine diff --git a/src/SquidStd.Plugin.Abstractions/README.md b/src/SquidStd.Plugin.Abstractions/README.md index 0952933b..e10fc291 100644 --- a/src/SquidStd.Plugin.Abstractions/README.md +++ b/src/SquidStd.Plugin.Abstractions/README.md @@ -1,16 +1,5 @@ -

- SquidStd -

-

SquidStd.Plugin.Abstractions

-

- NuGet - Downloads - docs - license -

- Contracts for building SquidStd plugins. A plugin declares its identity through `PluginMetadata` and registers its services into the host's DryIoc container via `Configure`, receiving a `PluginContext` with shared boot data. diff --git a/src/SquidStd.Services.Core/README.md b/src/SquidStd.Services.Core/README.md index e009f7fb..ac26180b 100644 --- a/src/SquidStd.Services.Core/README.md +++ b/src/SquidStd.Services.Core/README.md @@ -1,16 +1,5 @@ -

- SquidStd -

-

SquidStd.Services.Core

-

- NuGet - Downloads - docs - license -

- Concrete implementations of the SquidStd.Core contracts, wired for DryIoc. A single `RegisterCoreServices()` call brings up the configuration manager, event bus, job system, timer wheel, main-thread dispatcher, metrics collection, storage, and secrets services. From 431dac2a6532ffe128d67382b96142097c42a103 Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 22:04:33 +0200 Subject: [PATCH 52/77] docs: remove stray tool-output tags from Messaging.Sqs and Generators READMEs --- src/SquidStd.Generators/README.md | 2 -- src/SquidStd.Messaging.Sqs/README.md | 1 - 2 files changed, 3 deletions(-) diff --git a/src/SquidStd.Generators/README.md b/src/SquidStd.Generators/README.md index 1b711b57..405eb948 100644 --- a/src/SquidStd.Generators/README.md +++ b/src/SquidStd.Generators/README.md @@ -46,5 +46,3 @@ The generated method reuses the normal `RegisterEventListener( ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). - - diff --git a/src/SquidStd.Messaging.Sqs/README.md b/src/SquidStd.Messaging.Sqs/README.md index 2b4017a6..6aad0023 100644 --- a/src/SquidStd.Messaging.Sqs/README.md +++ b/src/SquidStd.Messaging.Sqs/README.md @@ -46,4 +46,3 @@ container.AddSqsMessaging(new SqsOptions { Aws = new AwsConfigEntry { Region = " ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). - From 2b78a40129aee061f11a4fd3b77201fb38f3aeed Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 22:11:32 +0200 Subject: [PATCH 53/77] docs: add Concepts explanation pages with Mermaid diagrams --- docs/articles/concepts/abstractions-first.md | 21 ++++++++ docs/articles/concepts/architecture.md | 34 +++++++++++++ docs/articles/concepts/bootstrap-lifecycle.md | 51 +++++++++++++++++++ .../articles/concepts/dependency-injection.md | 32 ++++++++++++ docs/articles/concepts/messaging-models.md | 27 ++++++++++ 5 files changed, 165 insertions(+) create mode 100644 docs/articles/concepts/abstractions-first.md create mode 100644 docs/articles/concepts/architecture.md create mode 100644 docs/articles/concepts/bootstrap-lifecycle.md create mode 100644 docs/articles/concepts/dependency-injection.md create mode 100644 docs/articles/concepts/messaging-models.md diff --git a/docs/articles/concepts/abstractions-first.md b/docs/articles/concepts/abstractions-first.md new file mode 100644 index 00000000..30e42aa4 --- /dev/null +++ b/docs/articles/concepts/abstractions-first.md @@ -0,0 +1,21 @@ +# Abstractions first + +SquidStd is built abstractions-first: code depends on contracts, and concrete backends are chosen at composition time. This is what makes a SquidStd application easy to test and easy to retarget. + +## Contract package plus provider packages + +Each capability is a contract package paired with one or more provider packages. The contract package — `*.Abstractions` — defines the interfaces and DTOs. Provider packages implement them. Your application references the abstraction and depends on the provider only in the host project. See the [architecture](architecture.md) overview for how this shapes the package graph. + +## In-memory for tests, external for prod + +Most capabilities ship an in-memory provider for tests and an external-backend provider for production: + +- **Messaging** — in-memory, or `SquidStd.Messaging.RabbitMq` / `SquidStd.Messaging.Sqs`. +- **Caching** — in-memory, or `SquidStd.Caching.Redis`. +- **Storage** — file-backed, or `SquidStd.Storage.S3` (S3 and MinIO). + +Tests run fast and deterministically against the in-memory provider; production swaps in the external backend. + +## Swapping providers without touching call sites + +Because call sites depend only on the contract, switching backends is a registration change in the host — `container.AddInMemoryMessaging()` versus `container.AddRabbitMqMessaging(...)` — with no edits to the code that publishes or consumes messages. See [dependency injection](dependency-injection.md) for the registration pattern that makes the swap a one-line change. diff --git a/docs/articles/concepts/architecture.md b/docs/articles/concepts/architecture.md new file mode 100644 index 00000000..ddd86bd0 --- /dev/null +++ b/docs/articles/concepts/architecture.md @@ -0,0 +1,34 @@ +# Architecture + +SquidStd is a modular .NET toolkit. Rather than a single monolithic library, it ships one package per capability, so an application depends only on the pieces it actually uses. + +## Modular by design + +Every capability is split in two: an `*.Abstractions` package that holds the contracts (interfaces and DTOs) and one or more provider packages that implement them. For example `SquidStd.Messaging.Abstractions` defines the messaging contracts, while `SquidStd.Messaging`, `SquidStd.Messaging.RabbitMq`, and `SquidStd.Messaging.Sqs` provide implementations. This keeps call sites coupled to contracts, not to a concrete backend. See [abstractions first](abstractions-first.md) for why this matters. + +## Layers + +The dependency flow runs in one direction: + +- **Core** (`SquidStd.Core`) — primitives, options, and the building blocks everything else sits on. +- **Abstractions** — per-capability contract packages. +- **Providers** — concrete implementations of those contracts. +- **Host** — `SquidStdBootstrap` composes services and runs them. + +Higher layers depend on lower ones, never the reverse. + +## Package graph + +```mermaid +graph TD + Core[SquidStd.Core] --> Services[SquidStd.Services.Core] + Abstr[*.Abstractions contracts] --> Providers[Provider packages] + Services --> Host[SquidStdBootstrap host] + Providers --> Host + Host --> App[Your application] +``` + +## Next + +- [Bootstrap lifecycle](bootstrap-lifecycle.md) — how the host starts and stops services. +- [Abstractions first](abstractions-first.md) — the contract-plus-provider pattern in depth. diff --git a/docs/articles/concepts/bootstrap-lifecycle.md b/docs/articles/concepts/bootstrap-lifecycle.md new file mode 100644 index 00000000..247b8da7 --- /dev/null +++ b/docs/articles/concepts/bootstrap-lifecycle.md @@ -0,0 +1,51 @@ +# Bootstrap lifecycle + +`SquidStdBootstrap` is the entry point that wires up dependency injection and drives the lifecycle of every registered service. The flow is always the same: create, configure, start, stop. + +## Create + +Begin by creating the bootstrap from `SquidStdOptions`: + +```csharp +var bootstrap = SquidStdBootstrap.Create(new SquidStdOptions +{ + ConfigName = "squidstd", + RootDirectory = AppContext.BaseDirectory +}); +``` + +`ConfigName` selects the configuration file and `RootDirectory` anchors relative paths. + +## ConfigureServices + +Register your services into the DryIoc container: + +```csharp +bootstrap.ConfigureServices(container => +{ + container.AddSomething(); +}); +``` + +See [dependency injection](dependency-injection.md) for the container and the `AddXxx` / `RegisterXxx` pattern. + +## Start and stop over ISquidStdService + +Services implementing `ISquidStdService` participate in the lifecycle. On `StartAsync` they are started in registration order; on `StopAsync` they are stopped in reverse order, so dependencies remain available while their dependents shut down. + +```mermaid +sequenceDiagram + participant App + participant Bootstrap as SquidStdBootstrap + participant Svc as ISquidStdService(s) + App->>Bootstrap: Create(options) + App->>Bootstrap: ConfigureServices(container) + App->>Bootstrap: StartAsync() + Bootstrap->>Svc: StartAsync() (in order) + App->>Bootstrap: StopAsync() + Bootstrap->>Svc: StopAsync() (reverse order) +``` + +## RunAsync for long-running hosts + +For long-running hosts, call `RunAsync`. It starts every service and then blocks until cancellation, stopping services cleanly on shutdown. Resolve dependencies anywhere with `bootstrap.Resolve()`. See the [architecture](architecture.md) overview for how the host fits the layers. diff --git a/docs/articles/concepts/dependency-injection.md b/docs/articles/concepts/dependency-injection.md new file mode 100644 index 00000000..6133434a --- /dev/null +++ b/docs/articles/concepts/dependency-injection.md @@ -0,0 +1,32 @@ +# Dependency injection + +SquidStd resolves every service through dependency injection. Understanding the container and its registration pattern is the key to wiring an application together. + +## DryIoc container + +The DI container is [DryIoc](https://github.com/dadhi/DryIoc), exposed as `IContainer`. It is fast, supports rich lifetimes, and validates required services at resolution time. Because the container owns validation, constructor dependencies that arrive from DI do not need manual null guards. + +## The AddXxx / RegisterXxx extension pattern + +Modules register themselves through C# 14 `extension(IContainer)` members named `AddXxx(...)` or `RegisterXxx(...)`. Each capability ships its own registration entry point, so wiring a module is a single call: + +```csharp +container.AddInMemoryMessaging(); +container.RegisterCoreServices(); +``` + +This keeps registration discoverable and colocated with the package that owns it. + +## Resolving through the bootstrap + +Once configured, resolve services through the bootstrap: + +```csharp +var bus = bootstrap.Resolve(); +``` + +In most code you let constructor injection do the work and never call `Resolve` directly. See [bootstrap lifecycle](bootstrap-lifecycle.md) for where `ConfigureServices` runs. + +## Singletons and lifecycle + +Most infrastructure services are registered as singletons and live for the life of the host. Services implementing `ISquidStdService` are additionally started and stopped by the bootstrap, so a singleton can hold long-lived resources that are released on shutdown. diff --git a/docs/articles/concepts/messaging-models.md b/docs/articles/concepts/messaging-models.md new file mode 100644 index 00000000..30b98907 --- /dev/null +++ b/docs/articles/concepts/messaging-models.md @@ -0,0 +1,27 @@ +# Messaging models + +SquidStd offers three in-process messaging models. Each fits a different shape of communication, and they can be mixed freely within one application. + +## Event Bus + +The Event Bus (`IEventBus`) is a stateless broadcast. Publishers call `PublishAsync` and any number of subscribers registered with `Subscribe` receive the event. Senders do not know who listens, and there is no return value. Use it for fan-out notifications where many parts of the system react to something that happened. + +## Command Dispatcher + +The Command Dispatcher is a stateless request routed to a single handler that returns a result. Unlike the Event Bus there is exactly one handler per command, and the caller awaits its result. Use it for request/response work — validating input, performing an operation, returning an outcome — where ownership of the action is unambiguous. + +## Actors + +Actors (`Actor`) provide an ordered, stateful, per-entity mailbox. Each actor processes its messages one at a time in order, so state inside an actor needs no locks. Send fire-and-forget messages with `TellAsync` or request a reply with `AskAsync`. Use actors when you need serialized access to per-entity state, such as a single account, device, or session. + +## Choosing + +```mermaid +flowchart TD + Q{Need ordered, per-entity state?} -->|yes| A[Actors] + Q -->|no| R{Request → single handler with a result?} + R -->|yes| C[Command Dispatcher] + R -->|no| E[Event Bus broadcast] +``` + +All three are contract-first; see [abstractions first](abstractions-first.md) for swapping in external transports. From d01cf9a2586c189da97f96453d1b63a29e037629 Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 22:11:33 +0200 Subject: [PATCH 54/77] docs: add Concepts section to articles TOC --- docs/articles/toc.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/articles/toc.yml b/docs/articles/toc.yml index c80a43c9..e72c1219 100644 --- a/docs/articles/toc.yml +++ b/docs/articles/toc.yml @@ -1,3 +1,15 @@ +- name: Concepts + items: + - name: Architecture + href: concepts/architecture.md + - name: Bootstrap lifecycle + href: concepts/bootstrap-lifecycle.md + - name: Dependency injection + href: concepts/dependency-injection.md + - name: Abstractions first + href: concepts/abstractions-first.md + - name: Messaging models + href: concepts/messaging-models.md - name: Getting Started href: getting-started.md - name: SquidStd.Core From 645d270f0b958ae554d2238f369dbfd48bb51394 Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 22:19:46 +0200 Subject: [PATCH 55/77] docs: add how-to and provider decision guides --- docs/articles/guides/choosing-caching.md | 19 ++++++++ docs/articles/guides/choosing-messaging.md | 19 ++++++++ .../guides/choosing-search-database.md | 23 ++++++++++ docs/articles/guides/choosing-storage.md | 23 ++++++++++ docs/articles/guides/configuration.md | 45 +++++++++++++++++++ docs/articles/guides/observability.md | 35 +++++++++++++++ docs/articles/guides/security.md | 43 ++++++++++++++++++ docs/articles/guides/testing.md | 41 +++++++++++++++++ 8 files changed, 248 insertions(+) create mode 100644 docs/articles/guides/choosing-caching.md create mode 100644 docs/articles/guides/choosing-messaging.md create mode 100644 docs/articles/guides/choosing-search-database.md create mode 100644 docs/articles/guides/choosing-storage.md create mode 100644 docs/articles/guides/configuration.md create mode 100644 docs/articles/guides/observability.md create mode 100644 docs/articles/guides/security.md create mode 100644 docs/articles/guides/testing.md diff --git a/docs/articles/guides/choosing-caching.md b/docs/articles/guides/choosing-caching.md new file mode 100644 index 00000000..2c7769f7 --- /dev/null +++ b/docs/articles/guides/choosing-caching.md @@ -0,0 +1,19 @@ +# Choosing a cache + +The caching abstraction (`ICacheService`) has two backends. Both expose the same +API, so you can develop against in-memory and promote to Redis without code changes. + +| Backend | Package · entrypoint | Use case | Scope | Survives restart | Ops cost | +|---|---|---|---|---|---| +| In-memory | `SquidStd.Caching` · `AddInMemoryCache` | Tests, single instance | Per-process | No | None | +| Redis | `SquidStd.Caching.Redis` · `AddRedisCache` | Multiple instances, shared cache | Distributed | Yes (with persistence) | Run/host Redis | + +```csharp +bootstrap.ConfigureServices(container => container.AddRedisCache("localhost:6379")); +``` + +## Recommendation + +Use `AddInMemoryCache` for tests and single-instance deployments; switch to +`AddRedisCache` as soon as you run more than one instance or need the cache to +outlive the process. diff --git a/docs/articles/guides/choosing-messaging.md b/docs/articles/guides/choosing-messaging.md new file mode 100644 index 00000000..3af5c59f --- /dev/null +++ b/docs/articles/guides/choosing-messaging.md @@ -0,0 +1,19 @@ +# Choosing a messaging backend + +SquidStd exposes one messaging abstraction with three interchangeable backends. +Pick by where you run and what delivery guarantees you need. + +| Backend | Package · entrypoint | Use case | Ordering | Durability | Ops cost | +|---|---|---|---|---|---| +| In-memory | `SquidStd.Messaging` · `AddInMemoryMessaging` | Tests, single-process apps | Per-process | None (lost on restart) | None | +| RabbitMQ | `SquidStd.Messaging.RabbitMq` · `AddRabbitMqMessaging` | Self-hosted services, work queues | Per-queue | Durable queues | Run a broker | +| SQS/SNS | `SquidStd.Messaging.Sqs` · `AddSqsMessaging` | AWS-native, serverless | FIFO queues only | Managed, durable | Managed (AWS) | + +```csharp +bootstrap.ConfigureServices(container => container.AddRabbitMqMessaging("amqp://localhost")); +``` + +## Recommendation + +Use `AddInMemoryMessaging` for tests and single-process apps, `AddRabbitMqMessaging` +when you self-host, and `AddSqsMessaging` when you are already on AWS. diff --git a/docs/articles/guides/choosing-search-database.md b/docs/articles/guides/choosing-search-database.md new file mode 100644 index 00000000..fa5c3292 --- /dev/null +++ b/docs/articles/guides/choosing-search-database.md @@ -0,0 +1,23 @@ +# Choosing search & database + +Full-text search and relational persistence are different tools. Use the search +provider for queries over text and documents, and the database module (backed by +FreeSql) for structured, relational data. + +| Need | Module · entrypoint | Provider(s) | Use case | +|---|---|---|---| +| Full-text / document search | `SquidStd.Search.Elasticsearch` · `AddElasticsearch` | Elasticsearch | Relevance ranking, faceting, log/document search | +| Relational data | `SquidStd.Database` · `RegisterDatabase` | FreeSql: Sqlite, PostgreSQL, MySql, SqlServer | Transactional records, joins, constraints | + +The database provider is selected via `DatabaseProviderType` (`Sqlite`, +`PostgreSQL`, `MySql`, `SqlServer`) in the `database` config section. + +```csharp +bootstrap.ConfigureServices(container => container.RegisterDatabase()); +``` + +## Recommendation + +Reach for `AddElasticsearch` when the core requirement is searching text or +documents by relevance; use `RegisterDatabase` with the FreeSql provider that +matches your engine for everything relational. They compose — many apps use both. diff --git a/docs/articles/guides/choosing-storage.md b/docs/articles/guides/choosing-storage.md new file mode 100644 index 00000000..594f918e --- /dev/null +++ b/docs/articles/guides/choosing-storage.md @@ -0,0 +1,23 @@ +# Choosing a storage backend + +The storage abstraction (`IStorageService`) stores blobs by key. There are two +providers; the S3 provider also targets any S3-compatible server such as MinIO. + +| Backend | Package · entrypoint | Use case | Shared across hosts | Ops cost | +|---|---|---|---|---| +| Local file | `SquidStd.Storage` · `AddFileStorage` | Dev, single host, simple persistence | No | None | +| S3 | `SquidStd.Storage.S3` · `AddS3Storage` | AWS-native object storage | Yes | Managed (AWS) | +| MinIO | `SquidStd.Storage.S3` · `AddS3Storage` (set `ServiceUrl`) | Self-hosted S3-compatible storage | Yes | Run MinIO | + +```csharp +bootstrap.ConfigureServices(container => container.AddS3Storage(new S3StorageOptions +{ + // ServiceUrl points at AWS S3 or a self-hosted MinIO endpoint +})); +``` + +## Recommendation + +Use `AddFileStorage` for development and single-host apps. Use `AddS3Storage` +against AWS S3 in the cloud, or against a MinIO endpoint (via `ServiceUrl`) when +you need S3 semantics on self-hosted infrastructure. diff --git a/docs/articles/guides/configuration.md b/docs/articles/guides/configuration.md new file mode 100644 index 00000000..092468f5 --- /dev/null +++ b/docs/articles/guides/configuration.md @@ -0,0 +1,45 @@ +# Configuration + +SquidStd loads configuration from a single YAML file. Each module registers a +strongly-typed section; the config manager deserializes it, expands environment +variables, and publishes the populated object into the container so you can +resolve it like any other service. + +## Steps + +1. **Point the bootstrap at your config file.** `SquidStdOptions.ConfigName` is the + logical file name (default `squidstd`, so `squidstd.yaml`) and + `RootDirectory` is the directory it is searched in. +2. **Register a section.** Call `RegisterConfigSection` with the section + name (the top-level YAML key). Provide a `createDefault` factory so the file + is generated with sensible defaults on first run. +3. **Use environment variables.** Any `string` property whose value contains a + `$VAR` token is expanded from the environment when the section is loaded. + Unknown tokens are left untouched. +4. **Read values.** Resolve the config type from the container wherever you need it. + +```csharp +public sealed class MyServiceConfig +{ + public string Endpoint { get; set; } = string.Empty; // e.g. "https://$API_HOST" + public int Retries { get; set; } = 3; +} + +var bootstrap = SquidStdBootstrap.Create( + new SquidStdOptions { ConfigName = "squidstd", RootDirectory = AppContext.BaseDirectory }); + +bootstrap.ConfigureServices(container => + container.RegisterConfigSection("myService", static () => new MyServiceConfig())); + +await bootstrap.StartAsync(); + +var config = container.Resolve(); +``` + +The corresponding `squidstd.yaml`: + +```yaml +myService: + endpoint: "https://$API_HOST" + retries: 5 +``` diff --git a/docs/articles/guides/observability.md b/docs/articles/guides/observability.md new file mode 100644 index 00000000..77370620 --- /dev/null +++ b/docs/articles/guides/observability.md @@ -0,0 +1,35 @@ +# Observability + +`SquidStd.Telemetry.OpenTelemetry` wires OpenTelemetry tracing and metrics into +the bootstrap as a managed service. Spans flow from SquidStd's own +`ActivitySource` instances and from anything else in your process. + +## Steps + +1. **Add the package** `SquidStd.Telemetry.OpenTelemetry`. +2. **Register telemetry** with `AddSquidStdTelemetry`, passing a `TelemetryOptions` + with your `ServiceName`. +3. **Choose an exporter.** Set `OtlpEndpoint` / `OtlpProtocol` to ship to a + collector (the default endpoint is `http://localhost:4317`, gRPC). For local + debugging set `EnableConsoleExporter = true` to also print spans to stdout. +4. **Tune sampling** with `TracingSampleRatio` (0..1) and toggle pipelines with + `EnableTracing` / `EnableMetrics`. + +```csharp +bootstrap.ConfigureServices(container => + container.AddSquidStdTelemetry(new TelemetryOptions + { + ServiceName = "orders-worker", + OtlpEndpoint = "http://otel-collector:4317", + OtlpProtocol = OtlpProtocolType.Grpc, + EnableConsoleExporter = false, + TracingSampleRatio = 1.0 + })); +``` + +## ActivitySource convention + +SquidStd modules name their `ActivitySource` instances with the `SquidStd.*` +prefix (for example `SquidStd.Messaging`). Subscribe to that prefix in your +collector or tracer-provider configuration to capture all framework spans, and +follow the same convention for your own application sources. diff --git a/docs/articles/guides/security.md b/docs/articles/guides/security.md new file mode 100644 index 00000000..df2373f5 --- /dev/null +++ b/docs/articles/guides/security.md @@ -0,0 +1,43 @@ +# Security + +SquidStd ships focused primitives for the three common secret-handling needs: +hashing credentials, protecting payloads with a key, and storing named secrets. + +## Steps + +1. **Hash passwords** with `HashUtils` (PBKDF2-SHA256). Never store plaintext. + + ```csharp + var stored = HashUtils.HashPassword(password); + var ok = HashUtils.VerifyPassword(password, stored); + ``` + +2. **Protect payloads** through `ISecretProtector`. The default + (`SquidStd.Services.Core`) is AES-GCM; for managed keys use + `RegisterKmsSecretProtector` from `SquidStd.Secrets.Aws`. + + ```csharp + byte[] protectedData = protector.Protect(plaintext); + byte[] clear = protector.Unprotect(protectedData); + ``` + +3. **Store named secrets** behind `ISecretStore`. The default is file-backed; use + `RegisterAwsSecretsManagerStore` (`SquidStd.Secrets.Aws`) for AWS Secrets Manager. + + ```csharp + await store.SetAsync("db/password", value); + var secret = await store.GetAsync("db/password"); + ``` + +4. **Use a PGP keyring** for armored encryption/signing by registering a key + store with `RegisterPgp` from `SquidStd.Crypto`. + + ```csharp + container.RegisterPgp(resolver => myPgpKeyStore); + ``` + +## Recommendation + +Use `HashUtils` for credentials, `ISecretProtector` (AES-GCM locally, KMS in the +cloud) for encrypting blobs, and `ISecretStore` for named secrets — backed by +files in development and AWS Secrets Manager in production. diff --git a/docs/articles/guides/testing.md b/docs/articles/guides/testing.md new file mode 100644 index 00000000..ab1d29b6 --- /dev/null +++ b/docs/articles/guides/testing.md @@ -0,0 +1,41 @@ +# Testing + +For fast, isolated tests, bootstrap SquidStd with in-memory providers instead of +real infrastructure. The in-memory backends implement the same abstractions as +the production providers, so the code under test never changes between unit and +integration runs. + +## Steps + +1. **Build a fixture** that creates a `SquidStdBootstrap` and starts it once per + test class. +2. **Opt into in-memory providers** in `ConfigureServices`: `AddInMemoryCache`, + `AddInMemoryMessaging`, and an in-memory VFS via `RegisterVfs` with + `InMemoryFileSystem`. +3. **Resolve abstractions** (`ICacheService`, the event bus, `IVirtualFileSystem`, …) + from the container in your tests. +4. **Swap to real backends** in a separate integration suite by replacing the + `Add…` calls with `AddRedisCache`, `AddRabbitMqMessaging`, `AddS3Storage`, etc. + +```csharp +public sealed class TestHostFixture : IAsyncDisposable +{ + public ISquidStdBootstrap Bootstrap { get; } + + public TestHostFixture() + { + Bootstrap = SquidStdBootstrap.Create( + new SquidStdOptions { ConfigName = "squidstd", RootDirectory = AppContext.BaseDirectory }); + + Bootstrap.ConfigureServices(container => container + .AddInMemoryCache() + .AddInMemoryMessaging() + .RegisterVfs(_ => new InMemoryFileSystem())); + } + + public async ValueTask DisposeAsync() => await Bootstrap.StopAsync(); +} +``` + +Start the bootstrap (`await Bootstrap.StartAsync()`) before your first test and +resolve services from its container. From d7e255f17cbca863a65386975e077cd7a9de1dd3 Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 22:19:46 +0200 Subject: [PATCH 56/77] docs: add Guides section to articles TOC --- docs/articles/toc.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/articles/toc.yml b/docs/articles/toc.yml index e72c1219..cc2d4e4e 100644 --- a/docs/articles/toc.yml +++ b/docs/articles/toc.yml @@ -10,6 +10,24 @@ href: concepts/abstractions-first.md - name: Messaging models href: concepts/messaging-models.md +- name: Guides + items: + - name: Configuration + href: guides/configuration.md + - name: Observability + href: guides/observability.md + - name: Testing + href: guides/testing.md + - name: Security + href: guides/security.md + - name: Choosing a messaging backend + href: guides/choosing-messaging.md + - name: Choosing a cache + href: guides/choosing-caching.md + - name: Choosing a storage backend + href: guides/choosing-storage.md + - name: Choosing search & database + href: guides/choosing-search-database.md - name: Getting Started href: getting-started.md - name: SquidStd.Core From a8e7c6984da2549cb5fdac8cb4d34e949e47c950 Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 22:24:52 +0200 Subject: [PATCH 57/77] build: add Crypto sample project --- SquidStd.slnx | 4 ++ samples/SquidStd.Samples.Crypto/Program.cs | 61 +++++++++++++++++++ .../SquidStd.Samples.Crypto.csproj | 16 +++++ 3 files changed, 81 insertions(+) create mode 100644 samples/SquidStd.Samples.Crypto/Program.cs create mode 100644 samples/SquidStd.Samples.Crypto/SquidStd.Samples.Crypto.csproj diff --git a/SquidStd.slnx b/SquidStd.slnx index 3b463679..bbcd9f9d 100644 --- a/SquidStd.slnx +++ b/SquidStd.slnx @@ -59,6 +59,10 @@ + + + + diff --git a/samples/SquidStd.Samples.Crypto/Program.cs b/samples/SquidStd.Samples.Crypto/Program.cs new file mode 100644 index 00000000..9512974c --- /dev/null +++ b/samples/SquidStd.Samples.Crypto/Program.cs @@ -0,0 +1,61 @@ +using System.Text; +using SquidStd.Core.Data.Bootstrap; +using SquidStd.Crypto.Pgp.Extensions; +using SquidStd.Crypto.Pgp.Interfaces; +using SquidStd.Crypto.Pgp.Services; +using SquidStd.Services.Core.Services.Bootstrap; + +var bootstrap = SquidStdBootstrap.Create( + new SquidStdOptions + { + ConfigName = "squidstd", + RootDirectory = AppContext.BaseDirectory + } +); + +var keyStoreDirectory = Path.Combine(AppContext.BaseDirectory, "pgp-keys"); + +#region step-1 + +// Register the PGP keyring, service, and a file-backed key store (armored .asc files). +bootstrap.ConfigureServices(container => container.RegisterPgp(_ => new FilePgpKeyStore(keyStoreDirectory))); + +#endregion + +await bootstrap.StartAsync(); + +#region step-2 + +const string identity = "alice@example.com"; +const string passphrase = "correct horse battery staple"; + +var pgp = bootstrap.Resolve(); +var keyring = bootstrap.Resolve(); + +// Generate a key pair and import it so the service can resolve it by identity. +var key = pgp.GenerateKey(identity, passphrase); +keyring.Import(key.PrivateArmored!); + +Console.WriteLine($"Generated key {key.KeyId} for {key.Identity} (has secret: {key.HasSecret})"); + +#endregion + +#region step-3 + +// Encrypt + sign for the recipient, then decrypt + verify the round-trip. +var armored = await pgp.EncryptAndSignForAsync( + identity, + Encoding.UTF8.GetBytes("attack at dawn"), + identity, + passphrase +); + +var result = await pgp.DecryptAndVerifyAsync(armored, passphrase); + +Console.WriteLine( + $"Decrypted: '{Encoding.UTF8.GetString(result.Data)}' (signed: {result.IsSigned}, valid: {result.IsValid})" +); + +#endregion + +await bootstrap.StopAsync(); diff --git a/samples/SquidStd.Samples.Crypto/SquidStd.Samples.Crypto.csproj b/samples/SquidStd.Samples.Crypto/SquidStd.Samples.Crypto.csproj new file mode 100644 index 00000000..11ec9605 --- /dev/null +++ b/samples/SquidStd.Samples.Crypto/SquidStd.Samples.Crypto.csproj @@ -0,0 +1,16 @@ + + + + Exe + net10.0 + enable + enable + false + + + + + + + + From feba49786f90b2a5d81b56501a8891652e81f00d Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 22:24:52 +0200 Subject: [PATCH 58/77] build: add Vfs sample project --- samples/SquidStd.Samples.Vfs/Program.cs | 61 +++++++++++++++++++ .../SquidStd.Samples.Vfs.csproj | 17 ++++++ 2 files changed, 78 insertions(+) create mode 100644 samples/SquidStd.Samples.Vfs/Program.cs create mode 100644 samples/SquidStd.Samples.Vfs/SquidStd.Samples.Vfs.csproj diff --git a/samples/SquidStd.Samples.Vfs/Program.cs b/samples/SquidStd.Samples.Vfs/Program.cs new file mode 100644 index 00000000..5913d13c --- /dev/null +++ b/samples/SquidStd.Samples.Vfs/Program.cs @@ -0,0 +1,61 @@ +using System.Text; +using SquidStd.Core.Data.Bootstrap; +using SquidStd.Crypto.Vfs.Extensions; +using SquidStd.Services.Core.Services.Bootstrap; +using SquidStd.Vfs.Abstractions.Interfaces; +using SquidStd.Vfs.Extensions; +using SquidStd.Vfs.Services; + +var bootstrap = SquidStdBootstrap.Create( + new SquidStdOptions + { + ConfigName = "squidstd", + RootDirectory = AppContext.BaseDirectory + } +); + +var vaultPath = Path.Combine(AppContext.BaseDirectory, "secret.vault"); + +#region step-1 + +// Register a plain in-memory VFS and an encrypted single-file vault. +bootstrap.ConfigureServices(container => +{ + container.RegisterVfs(_ => new InMemoryFileSystem()); + container.RegisterCryptoVault(vaultPath); + + return container; +}); + +#endregion + +await bootstrap.StartAsync(); + +#region step-2 + +// Write and read a file through the virtual filesystem. +var vfs = bootstrap.Resolve(); + +await vfs.WriteAllBytesAsync("notes/hello.txt", Encoding.UTF8.GetBytes("plain content")); +var bytes = await vfs.ReadAllBytesAsync("notes/hello.txt"); + +Console.WriteLine($"VFS read: {Encoding.UTF8.GetString(bytes!)}"); + +#endregion + +#region step-3 + +// Unlock the encrypted vault, write a secret, then lock it again. +var vault = bootstrap.Resolve(); + +vault.Unlock("vault passphrase"); +await vault.WriteAllBytesAsync("secret.txt", Encoding.UTF8.GetBytes("top secret")); +var secret = await vault.ReadAllBytesAsync("secret.txt"); +Console.WriteLine($"Vault read: {Encoding.UTF8.GetString(secret!)}"); +vault.Lock(); + +Console.WriteLine($"Vault unlocked: {vault.IsUnlocked}"); + +#endregion + +await bootstrap.StopAsync(); diff --git a/samples/SquidStd.Samples.Vfs/SquidStd.Samples.Vfs.csproj b/samples/SquidStd.Samples.Vfs/SquidStd.Samples.Vfs.csproj new file mode 100644 index 00000000..14019d32 --- /dev/null +++ b/samples/SquidStd.Samples.Vfs/SquidStd.Samples.Vfs.csproj @@ -0,0 +1,17 @@ + + + + Exe + net10.0 + enable + enable + false + + + + + + + + + From 73d478361dabcf4f32429e9421ee823e93897d28 Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 22:24:53 +0200 Subject: [PATCH 59/77] build: add Secrets sample project --- samples/SquidStd.Samples.Secrets/Program.cs | 84 +++++++++++++++++++ .../SquidStd.Samples.Secrets.csproj | 16 ++++ 2 files changed, 100 insertions(+) create mode 100644 samples/SquidStd.Samples.Secrets/Program.cs create mode 100644 samples/SquidStd.Samples.Secrets/SquidStd.Samples.Secrets.csproj diff --git a/samples/SquidStd.Samples.Secrets/Program.cs b/samples/SquidStd.Samples.Secrets/Program.cs new file mode 100644 index 00000000..7f98cb9e --- /dev/null +++ b/samples/SquidStd.Samples.Secrets/Program.cs @@ -0,0 +1,84 @@ +using System.Text; +using SquidStd.Aws.Abstractions.Data.Config; +using SquidStd.Core.Data.Bootstrap; +using SquidStd.Core.Interfaces.Secrets; +using SquidStd.Secrets.Aws.Extensions; +using SquidStd.Services.Core.Services.Bootstrap; + +var bootstrap = SquidStdBootstrap.Create( + new SquidStdOptions + { + ConfigName = "squidstd", + RootDirectory = AppContext.BaseDirectory + } +); + +#region step-1 + +// Wire the KMS-backed protector and the Secrets Manager store against a LocalStack endpoint. +bootstrap.ConfigureServices(container => +{ + container.RegisterKmsSecretProtector(options => + { + options.KeyId = "alias/app"; + options.Aws = new AwsConfigEntry + { + Region = "us-east-1", + ServiceUrl = "http://localhost:4566" + }; + }); + + container.RegisterAwsSecretsManagerStore(options => + { + options.NamePrefix = "myapp/"; + options.Aws = new AwsConfigEntry + { + Region = "us-east-1", + ServiceUrl = "http://localhost:4566" + }; + }); + + return container; +}); + +#endregion + +await bootstrap.StartAsync(); + +var protector = bootstrap.Resolve(); +var store = bootstrap.Resolve(); + +// The calls below need a live KMS + Secrets Manager endpoint (e.g. LocalStack), so they +// only run when explicitly enabled. The wiring above compiles and resolves without AWS. +if (Environment.GetEnvironmentVariable("SQUIDSTD_RUN_AWS") is null) +{ + Console.WriteLine("Set SQUIDSTD_RUN_AWS=1 with LocalStack running to exercise the live calls."); + await bootstrap.StopAsync(); + return; +} + +#region step-2 + +// Envelope-encrypt a value with a KMS data key, then decrypt it back. +var protectedBytes = protector.Protect(Encoding.UTF8.GetBytes("api-token")); +var plaintext = protector.Unprotect(protectedBytes); + +Console.WriteLine($"Unprotected: {Encoding.UTF8.GetString(plaintext)}"); + +#endregion + +#region step-3 + +// Store, fetch, and list secrets through the Secrets Manager store. +await store.SetAsync("db-password", "s3cr3t"); +var password = await store.GetAsync("db-password"); +Console.WriteLine($"db-password: {password}"); + +await foreach (var name in store.ListNamesAsync()) +{ + Console.WriteLine($"secret: {name}"); +} + +#endregion + +await bootstrap.StopAsync(); diff --git a/samples/SquidStd.Samples.Secrets/SquidStd.Samples.Secrets.csproj b/samples/SquidStd.Samples.Secrets/SquidStd.Samples.Secrets.csproj new file mode 100644 index 00000000..3c6463e0 --- /dev/null +++ b/samples/SquidStd.Samples.Secrets/SquidStd.Samples.Secrets.csproj @@ -0,0 +1,16 @@ + + + + Exe + net10.0 + enable + enable + false + + + + + + + + From f2fa0aa4b36ab16b1608935b8061aba59475a5a3 Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 22:24:53 +0200 Subject: [PATCH 60/77] build: add Actors sample project --- samples/SquidStd.Samples.Actors/Program.cs | 53 +++++++++++++++++++ .../SquidStd.Samples.Actors.csproj | 15 ++++++ 2 files changed, 68 insertions(+) create mode 100644 samples/SquidStd.Samples.Actors/Program.cs create mode 100644 samples/SquidStd.Samples.Actors/SquidStd.Samples.Actors.csproj diff --git a/samples/SquidStd.Samples.Actors/Program.cs b/samples/SquidStd.Samples.Actors/Program.cs new file mode 100644 index 00000000..01f2f0bc --- /dev/null +++ b/samples/SquidStd.Samples.Actors/Program.cs @@ -0,0 +1,53 @@ +using SquidStd.Actors; +using SquidStd.Actors.Interfaces; + +await using var counter = new CounterActor(); + +#region step-2 + +// Fire-and-forget messages: TellAsync enqueues without awaiting a reply. +await counter.TellAsync(new Increment(5)); +await counter.TellAsync(new Increment(3)); + +#endregion + +#region step-3 + +// Request/response: AskAsync enqueues a request and awaits its typed reply. +var total = await counter.AskAsync(new GetTotal()); + +Console.WriteLine($"Total: {total}"); + +#endregion + +#region step-1 + +// The message contract: a marker interface, a fire-and-forget message, and an ask request. +internal interface ICounterMessage; + +internal sealed record Increment(int By) : ICounterMessage; + +internal sealed record GetTotal : ActorRequest, ICounterMessage; + +// A single-consumer actor: state is mutated without locks inside ReceiveAsync. +internal sealed class CounterActor : Actor +{ + private int _total; + + protected override ValueTask ReceiveAsync(ICounterMessage message, CancellationToken cancellationToken) + { + switch (message) + { + case Increment increment: + _total += increment.By; + break; + case GetTotal request: + request.Reply(_total); + break; + } + + return ValueTask.CompletedTask; + } +} + +#endregion diff --git a/samples/SquidStd.Samples.Actors/SquidStd.Samples.Actors.csproj b/samples/SquidStd.Samples.Actors/SquidStd.Samples.Actors.csproj new file mode 100644 index 00000000..dd253c83 --- /dev/null +++ b/samples/SquidStd.Samples.Actors/SquidStd.Samples.Actors.csproj @@ -0,0 +1,15 @@ + + + + Exe + net10.0 + enable + enable + false + + + + + + + From a222049e61a2e7ff5f3321335b1a914c0eb026fe Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 22:37:32 +0200 Subject: [PATCH 61/77] docs(samples): show real keyring persistence in Crypto sample Demonstrate IPgpKeyring.SaveAsync to the file-backed store and a fresh PgpKeyring.LoadAsync round-trip, instead of registering a key store that was never used. Clarify the PrivateArmored null-forgiving operator. --- samples/SquidStd.Samples.Crypto/Program.cs | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/samples/SquidStd.Samples.Crypto/Program.cs b/samples/SquidStd.Samples.Crypto/Program.cs index 9512974c..8fdd0532 100644 --- a/samples/SquidStd.Samples.Crypto/Program.cs +++ b/samples/SquidStd.Samples.Crypto/Program.cs @@ -31,12 +31,18 @@ var pgp = bootstrap.Resolve(); var keyring = bootstrap.Resolve(); +var keyStore = bootstrap.Resolve(); -// Generate a key pair and import it so the service can resolve it by identity. +// Generate a key pair and import it so the service can resolve it by identity. A freshly +// generated key always carries secret material, so PrivateArmored is non-null here. var key = pgp.GenerateKey(identity, passphrase); keyring.Import(key.PrivateArmored!); -Console.WriteLine($"Generated key {key.KeyId} for {key.Identity} (has secret: {key.HasSecret})"); +// Persist the keyring to the file-backed store: this creates the directory and writes the +// armored .asc files to disk. +await keyring.SaveAsync(keyStore); + +Console.WriteLine($"Generated key {key.KeyId} for {key.Identity}; saved to {keyStoreDirectory}"); #endregion @@ -58,4 +64,16 @@ #endregion +#region step-4 + +// Persistence round-trip: a brand-new keyring loads the same keys back from disk. +var reloaded = new PgpKeyring(); +await reloaded.LoadAsync(keyStore); + +Console.WriteLine( + $"Reloaded {reloaded.Keys.Count} key(s) from disk; contains '{identity}': {reloaded.Contains(identity)}" +); + +#endregion + await bootstrap.StopAsync(); From 2c95b77cb3a6d3bf4906219e324f1e936ac78e05 Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 22:37:33 +0200 Subject: [PATCH 62/77] docs(samples): make Vfs vault step an honest working round-trip Drive CryptoFileSystem over an in-memory backend through the full unlock/write/lock/reopen lifecycle so the encrypted round-trip is real, and document that the RegisterCryptoVault disk backend (ZipFileSystem) cannot currently be locked due to a ZipArchive Update-mode limitation. --- samples/SquidStd.Samples.Vfs/Program.cs | 44 ++++++++++++++----------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/samples/SquidStd.Samples.Vfs/Program.cs b/samples/SquidStd.Samples.Vfs/Program.cs index 5913d13c..4ecfa3e3 100644 --- a/samples/SquidStd.Samples.Vfs/Program.cs +++ b/samples/SquidStd.Samples.Vfs/Program.cs @@ -1,6 +1,6 @@ using System.Text; using SquidStd.Core.Data.Bootstrap; -using SquidStd.Crypto.Vfs.Extensions; +using SquidStd.Crypto.Vfs.Services; using SquidStd.Services.Core.Services.Bootstrap; using SquidStd.Vfs.Abstractions.Interfaces; using SquidStd.Vfs.Extensions; @@ -14,18 +14,10 @@ } ); -var vaultPath = Path.Combine(AppContext.BaseDirectory, "secret.vault"); - #region step-1 -// Register a plain in-memory VFS and an encrypted single-file vault. -bootstrap.ConfigureServices(container => -{ - container.RegisterVfs(_ => new InMemoryFileSystem()); - container.RegisterCryptoVault(vaultPath); - - return container; -}); +// Register a plain in-memory virtual filesystem. +bootstrap.ConfigureServices(container => container.RegisterVfs(_ => new InMemoryFileSystem())); #endregion @@ -39,22 +31,36 @@ await vfs.WriteAllBytesAsync("notes/hello.txt", Encoding.UTF8.GetBytes("plain content")); var bytes = await vfs.ReadAllBytesAsync("notes/hello.txt"); +// The file was just written, so it is present (the API returns null only when absent). Console.WriteLine($"VFS read: {Encoding.UTF8.GetString(bytes!)}"); #endregion #region step-3 -// Unlock the encrypted vault, write a secret, then lock it again. -var vault = bootstrap.Resolve(); +// Encrypted vault lifecycle: unlock -> write -> lock, then re-open and read. +// RegisterCryptoVault wires a DI vault over a single on-disk zip file, but that ZipFileSystem +// backend currently cannot be locked/persisted (its List reads ZipArchiveEntry.Length, which +// .NET marks unavailable in ZipArchiveMode.Update), so this sample drives CryptoFileSystem over +// an in-memory backend to demonstrate the full lifecycle without that limitation. +var backend = new InMemoryFileSystem(); -vault.Unlock("vault passphrase"); -await vault.WriteAllBytesAsync("secret.txt", Encoding.UTF8.GetBytes("top secret")); -var secret = await vault.ReadAllBytesAsync("secret.txt"); -Console.WriteLine($"Vault read: {Encoding.UTF8.GetString(secret!)}"); -vault.Lock(); +using (var vault = new CryptoFileSystem(backend)) +{ + vault.Unlock("vault passphrase"); + await vault.WriteAllBytesAsync("secret.txt", Encoding.UTF8.GetBytes("top secret")); + vault.Lock(); // zeroes the key and flushes the encrypted index into the backend +} + +// Re-open the same encrypted backend with the passphrase to prove the data round-trips at rest. +using (var reopened = new CryptoFileSystem(backend)) +{ + reopened.Unlock("vault passphrase"); -Console.WriteLine($"Vault unlocked: {vault.IsUnlocked}"); + // The secret was written above, so it is present. + var secret = await reopened.ReadAllBytesAsync("secret.txt"); + Console.WriteLine($"Vault read after reopen: {Encoding.UTF8.GetString(secret!)}"); +} #endregion From dc5b958e5a3cd2eb6d26545e4263fbe989f0d29c Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 22:40:31 +0200 Subject: [PATCH 63/77] build: add step regions to Commands sample --- samples/SquidStd.Samples.Commands/Program.cs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/samples/SquidStd.Samples.Commands/Program.cs b/samples/SquidStd.Samples.Commands/Program.cs index 56e23f85..fa133520 100644 --- a/samples/SquidStd.Samples.Commands/Program.cs +++ b/samples/SquidStd.Samples.Commands/Program.cs @@ -4,6 +4,9 @@ using SquidStd.Core.Interfaces.Commands; using SquidStd.Services.Core.Extensions; +#region step-1 + +// Register the dispatcher and the handlers. EchoCommand has two handlers — both run on dispatch. var container = new Container(); container.RegisterCommandDispatcher(); container.RegisterCommandHandler(); @@ -19,6 +22,10 @@ registration.Subscribe(dispatcher, container); } +#endregion + +#region step-2 + // The context (here a Session) is passed explicitly at dispatch time — in a server this is the // session the message arrived on. See RegisterSeededCommandDispatcher for building it from a seed. var session = new Session(); @@ -26,10 +33,17 @@ await Dispatch(dispatcher, new PingCommand(), session); await Dispatch(dispatcher, new EchoCommand("hello world"), session); -// Unknown command path: nothing registered for UnknownCommand. +#endregion + +#region step-3 + +// The result reports whether any handler matched and how many ran — here nothing is registered +// for UnknownCommand, so Matched is false and HandlerCount is 0. var unknown = await dispatcher.DispatchAsync(new UnknownCommand(), session); Console.WriteLine($"UnknownCommand -> matched={unknown.Matched} handlers={unknown.HandlerCount}"); +#endregion + return; static async Task Dispatch(ICommandDispatcher dispatcher, TCommand command, Session context) From 6fee64fe53665581547433a51a68290e82c72a20 Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 22:44:00 +0200 Subject: [PATCH 64/77] docs: add crypto, vfs, secrets, actors, persistence and command-dispatcher tutorials --- docs/tutorials/actors.md | 50 +++++++++++++++++++++++++ docs/tutorials/command-dispatcher.md | 49 +++++++++++++++++++++++++ docs/tutorials/crypto.md | 55 ++++++++++++++++++++++++++++ docs/tutorials/persistence.md | 50 +++++++++++++++++++++++++ docs/tutorials/secrets.md | 55 ++++++++++++++++++++++++++++ docs/tutorials/vfs.md | 53 +++++++++++++++++++++++++++ 6 files changed, 312 insertions(+) create mode 100644 docs/tutorials/actors.md create mode 100644 docs/tutorials/command-dispatcher.md create mode 100644 docs/tutorials/crypto.md create mode 100644 docs/tutorials/persistence.md create mode 100644 docs/tutorials/secrets.md create mode 100644 docs/tutorials/vfs.md diff --git a/docs/tutorials/actors.md b/docs/tutorials/actors.md new file mode 100644 index 00000000..8097cd62 --- /dev/null +++ b/docs/tutorials/actors.md @@ -0,0 +1,50 @@ +# Actors + +Build a single-consumer actor that mutates state without locks, then talk to it with +fire-and-forget and request/response messages. + +## What you'll build + +A `CounterActor` (`SquidStd.Actors`) that processes messages one at a time inside `ReceiveAsync`, +driven by `TellAsync` (fire-and-forget) and `AskAsync` (request/response). + +## Prerequisites + +- .NET 10 SDK +- `dotnet add package SquidStd.Actors` + +## Steps + +### 1. Define the message contract and the actor + +A marker interface groups the messages; `Increment` is fire-and-forget, `GetTotal` derives from +`ActorRequest` to carry a reply. The actor mutates `_total` without locks because messages +are processed one at a time. + +[!code-csharp[](../../samples/SquidStd.Samples.Actors/Program.cs#step-1)] + +### 2. Send fire-and-forget messages + +`TellAsync` enqueues a message and returns without waiting for it to be handled. + +[!code-csharp[](../../samples/SquidStd.Samples.Actors/Program.cs#step-2)] + +### 3. Ask for a reply + +`AskAsync` enqueues a request and awaits the typed reply the actor sends with +`request.Reply(...)`. + +[!code-csharp[](../../samples/SquidStd.Samples.Actors/Program.cs#step-3)] + +## Run it + +```bash +dotnet run --project samples/SquidStd.Samples.Actors +``` + +Prints `Total: 8`. + +## Next steps + +- [Messaging models](../articles/concepts/messaging-models.md) +- [SquidStd.Actors reference](../articles/actors.md) diff --git a/docs/tutorials/command-dispatcher.md b/docs/tutorials/command-dispatcher.md new file mode 100644 index 00000000..2c90c77a --- /dev/null +++ b/docs/tutorials/command-dispatcher.md @@ -0,0 +1,49 @@ +# Command dispatcher + +Register typed command handlers, dispatch commands against a context, and read back what matched. + +## What you'll build + +A `Container` with a `RegisterCommandDispatcher` and several handlers +(`SquidStd.Services.Core`), dispatching commands that carry a `Session` context. One command type +has two handlers — both run on dispatch. + +## Prerequisites + +- .NET 10 SDK +- `dotnet add package SquidStd.Services.Core` + +## Steps + +### 1. Register the dispatcher and handlers + +Register the dispatcher for the context type, then each handler. `EchoCommand` has two handlers +(`EchoHandler` and `AuditHandler`); both will run. The subscription loop is what +`CommandDispatcherActivator` does at runtime — inlined here to keep the sample +self-contained. + +[!code-csharp[](../../samples/SquidStd.Samples.Commands/Program.cs#step-1)] + +### 2. Dispatch commands with a context + +The context (here a `Session`) is passed explicitly at dispatch time — in a server this is the +session the message arrived on. + +[!code-csharp[](../../samples/SquidStd.Samples.Commands/Program.cs#step-2)] + +### 3. Read the dispatch result + +`DispatchAsync` returns a result reporting whether any handler matched and how many ran. Nothing +is registered for `UnknownCommand`, so `Matched` is `false` and `HandlerCount` is `0`. + +[!code-csharp[](../../samples/SquidStd.Samples.Commands/Program.cs#step-3)] + +## Run it + +```bash +dotnet run --project samples/SquidStd.Samples.Commands +``` + +## Next steps + +- [Messaging models](../articles/concepts/messaging-models.md) diff --git a/docs/tutorials/crypto.md b/docs/tutorials/crypto.md new file mode 100644 index 00000000..95070492 --- /dev/null +++ b/docs/tutorials/crypto.md @@ -0,0 +1,55 @@ +# Crypto (PGP) + +Generate a PGP key pair, persist it to disk, encrypt and sign a message, then reload the keyring from disk. + +## What you'll build + +A host that registers the PGP keyring, service, and a file-backed key store +(`SquidStd.Crypto.Pgp`), generates a key, performs an encrypt-and-sign round-trip, and proves the +keys survive a restart by reloading them from the armored `.asc` files on disk. + +## Prerequisites + +- .NET 10 SDK +- `dotnet add package SquidStd.Crypto` + +## Steps + +### 1. Register the PGP services and a file-backed key store + +`RegisterPgp` wires the keyring and `IPgpService`; the factory chooses the key store — here a +`FilePgpKeyStore` that reads and writes armored `.asc` files in a directory. + +[!code-csharp[](../../samples/SquidStd.Samples.Crypto/Program.cs#step-1)] + +### 2. Generate a key and save it to disk + +Generate a key pair, import it into the keyring so the service can resolve it by identity, then +persist the keyring to the file-backed store. + +[!code-csharp[](../../samples/SquidStd.Samples.Crypto/Program.cs#step-2)] + +### 3. Encrypt, sign, decrypt and verify + +`EncryptAndSignForAsync` produces armored ciphertext for a recipient; `DecryptAndVerifyAsync` +returns the plaintext together with the signature status. + +[!code-csharp[](../../samples/SquidStd.Samples.Crypto/Program.cs#step-3)] + +### 4. Reload the keyring from disk + +A brand-new `PgpKeyring` loads the same keys back from the store, proving the on-disk material +survives a restart. + +[!code-csharp[](../../samples/SquidStd.Samples.Crypto/Program.cs#step-4)] + +## Run it + +```bash +dotnet run --project samples/SquidStd.Samples.Crypto +``` + +## Next steps + +- [Security guide](../articles/guides/security.md) +- [SquidStd.Crypto reference](../articles/crypto.md) diff --git a/docs/tutorials/persistence.md b/docs/tutorials/persistence.md new file mode 100644 index 00000000..25a28a2f --- /dev/null +++ b/docs/tutorials/persistence.md @@ -0,0 +1,50 @@ +# Persistence + +Keep entities in an in-memory store backed by a durable binary snapshot plus a journal, so state +survives a restart. + +## What you'll build + +A standalone demo of `SquidStd.Persistence`: a `Player` store that loads existing state on +startup, appends every change to a journal, and captures a snapshot. Run it twice — the second run +reloads what the first saved. + +## Prerequisites + +- .NET 10 SDK +- `dotnet add package SquidStd.Persistence` (and `SquidStd.Persistence.MessagePack` for the + binary serializer) + +## Steps + +### 1. Initialize and load existing state + +`InitializeAsync` replays the snapshot and journal from the save directory; `GetStore` +returns the typed store for an entity registered in the `PersistenceEntityRegistry`. + +[!code-csharp[](../../samples/SquidStd.Samples.Persistence/Program.cs#step-1)] + +### 2. Mutate the store + +Every `UpsertAsync` / `RemoveAsync` is appended to the journal, so the change is durable before +the next snapshot. + +[!code-csharp[](../../samples/SquidStd.Samples.Persistence/Program.cs#step-2)] + +### 3. Snapshot and trim the journal + +`SaveSnapshotAsync` captures the full state and trims the journal. Re-run the sample to see the +state reload. + +[!code-csharp[](../../samples/SquidStd.Samples.Persistence/Program.cs#step-3)] + +## Run it + +```bash +dotnet run --project samples/SquidStd.Samples.Persistence +dotnet run --project samples/SquidStd.Samples.Persistence # reloads the saved state +``` + +## Next steps + +- [SquidStd.Persistence reference](../articles/persistence.md) diff --git a/docs/tutorials/secrets.md b/docs/tutorials/secrets.md new file mode 100644 index 00000000..a98460de --- /dev/null +++ b/docs/tutorials/secrets.md @@ -0,0 +1,55 @@ +# Secrets (KMS / Secrets Manager) + +Wire an AWS KMS-backed secret protector and a Secrets Manager store, then envelope-encrypt and +store values. + +## What you'll build + +A host that registers `ISecretProtector` (KMS envelope encryption) and `ISecretStore` +(AWS Secrets Manager) from `SquidStd.Secrets.Aws`, then exercises protect/unprotect and +set/get/list. + +## Prerequisites + +- .NET 10 SDK +- `dotnet add package SquidStd.Secrets.Aws` +- The live calls (steps 2–3) need a KMS + Secrets Manager endpoint. The sample is + compile-and-wire focused: it resolves the services without AWS and only runs the live calls + when `SQUIDSTD_RUN_AWS=1` is set with [LocalStack](https://localstack.cloud) (or real AWS) + reachable at the configured endpoint. + +## Steps + +### 1. Register the KMS protector and Secrets Manager store + +`RegisterKmsSecretProtector` and `RegisterAwsSecretsManagerStore` take the KMS key alias, the +secret name prefix, and the AWS endpoint — pointed here at a LocalStack `ServiceUrl`. + +[!code-csharp[](../../samples/SquidStd.Samples.Secrets/Program.cs#step-1)] + +### 2. Envelope-encrypt a value + +`Protect` requests a KMS data key, encrypts the payload with it, and wraps the encrypted data +key alongside the ciphertext; `Unprotect` reverses it. + +[!code-csharp[](../../samples/SquidStd.Samples.Secrets/Program.cs#step-2)] + +### 3. Store, fetch and list secrets + +`ISecretStore` reads and writes named secrets through Secrets Manager; `ListNamesAsync` streams +the names under the configured prefix. + +[!code-csharp[](../../samples/SquidStd.Samples.Secrets/Program.cs#step-3)] + +## Run it + +```bash +dotnet run --project samples/SquidStd.Samples.Secrets +# with the live calls: +SQUIDSTD_RUN_AWS=1 dotnet run --project samples/SquidStd.Samples.Secrets +``` + +## Next steps + +- [Security guide](../articles/guides/security.md) +- [SquidStd.Secrets.Aws reference](../articles/secrets-aws.md) diff --git a/docs/tutorials/vfs.md b/docs/tutorials/vfs.md new file mode 100644 index 00000000..4646c073 --- /dev/null +++ b/docs/tutorials/vfs.md @@ -0,0 +1,53 @@ +# Virtual filesystem + +Read and write files through an abstract virtual filesystem, then layer an encrypted vault on top. + +## What you'll build + +A host that resolves `IVirtualFileSystem` (`SquidStd.Vfs`) backed by an in-memory store, plus an +encrypted-vault round-trip using `CryptoFileSystem` (`SquidStd.Crypto.Vfs`). + +## Prerequisites + +- .NET 10 SDK +- `dotnet add package SquidStd.Vfs` (and `SquidStd.Crypto.Vfs` for the encrypted vault) + +## Steps + +### 1. Register a virtual filesystem + +`RegisterVfs` wires `IVirtualFileSystem`; the factory chooses the backend — here a plain +in-memory filesystem. + +[!code-csharp[](../../samples/SquidStd.Samples.Vfs/Program.cs#step-1)] + +### 2. Write and read a file + +The same `IVirtualFileSystem` API works regardless of the backend. `ReadAllBytesAsync` returns +`null` only when the path is absent. + +[!code-csharp[](../../samples/SquidStd.Samples.Vfs/Program.cs#step-2)] + +### 3. Encrypted vault round-trip + +`CryptoFileSystem` encrypts every entry over a backend filesystem. The lifecycle is +unlock → write → lock; locking zeroes the key and flushes the encrypted index into the backend, +so re-opening the same backend with the passphrase decrypts the data at rest. + +This sample drives `CryptoFileSystem` over an **in-memory** backend to demonstrate the full +lifecycle. The DI helper `RegisterCryptoVault` wires a vault over a single on-disk zip file, but +that `ZipFileSystem` backend currently cannot be locked/persisted (a known limitation — its +`List` reads `ZipArchiveEntry.Length`, which .NET marks unavailable in `ZipArchiveMode.Update`). +On-disk vault persistence is therefore not yet supported. + +[!code-csharp[](../../samples/SquidStd.Samples.Vfs/Program.cs#step-3)] + +## Run it + +```bash +dotnet run --project samples/SquidStd.Samples.Vfs +``` + +## Next steps + +- [SquidStd.Vfs reference](../articles/vfs.md) From f767a4216fd451f1e8d27ebac886056a9ece3966 Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 22:44:00 +0200 Subject: [PATCH 65/77] docs: add new tutorials to TOC --- docs/tutorials/toc.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/tutorials/toc.yml b/docs/tutorials/toc.yml index 5696494d..2d88e934 100644 --- a/docs/tutorials/toc.yml +++ b/docs/tutorials/toc.yml @@ -34,3 +34,15 @@ href: plugins.md - name: Scaffolding with dotnet new href: scaffolding.md +- name: Actors + href: actors.md +- name: Command dispatcher + href: command-dispatcher.md +- name: Persistence + href: persistence.md +- name: Virtual filesystem + href: vfs.md +- name: Crypto (PGP) + href: crypto.md +- name: Secrets (KMS / Secrets Manager) + href: secrets.md From e357df76e9f5f39d77ddf2b9175ec62d92b509cf Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 22:47:24 +0200 Subject: [PATCH 66/77] docs: link tutorials and concepts from the new-module READMEs --- src/SquidStd.Actors/README.md | 5 +++++ src/SquidStd.Crypto/README.md | 4 ++++ src/SquidStd.Persistence/README.md | 4 ++++ src/SquidStd.Secrets.Aws/README.md | 4 ++++ src/SquidStd.Vfs.Abstractions/README.md | 4 ++++ src/SquidStd.Vfs/README.md | 4 ++++ 6 files changed, 25 insertions(+) diff --git a/src/SquidStd.Actors/README.md b/src/SquidStd.Actors/README.md index 3c5b15e6..aa369882 100644 --- a/src/SquidStd.Actors/README.md +++ b/src/SquidStd.Actors/README.md @@ -79,6 +79,11 @@ using var sub = session.SubscribeToEventBus(eventBus, (UserJoinedEvent e) => new `DisposeAsync` completes the mailbox, drains in-flight work, and faults any still-pending requests. +## Related + +- Tutorial: [Actors](https://tgiachi.github.io/squid-std/tutorials/actors.html) +- Concept: [Messaging models](https://tgiachi.github.io/squid-std/articles/concepts/messaging-models.html) + ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). diff --git a/src/SquidStd.Crypto/README.md b/src/SquidStd.Crypto/README.md index 3859ffbf..27216f17 100644 --- a/src/SquidStd.Crypto/README.md +++ b/src/SquidStd.Crypto/README.md @@ -122,6 +122,10 @@ var folderVault = new CryptoFileSystem(new PhysicalFileSystem("/secure/dir")); - A wrong passphrase fails the index authentication tag → `CryptographicException`; operations on a locked vault throw `InvalidOperationException`. +## Related + +- Tutorial: [Crypto (PGP)](https://tgiachi.github.io/squid-std/tutorials/crypto.html) + ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). diff --git a/src/SquidStd.Persistence/README.md b/src/SquidStd.Persistence/README.md index e0842c16..05194669 100644 --- a/src/SquidStd.Persistence/README.md +++ b/src/SquidStd.Persistence/README.md @@ -66,6 +66,10 @@ container.ApplyPersistedEntityRegistrations(); // builds descriptors into IPer | `SnapshotService` | Atomic per-type binary snapshot files with payload checksum. | | `RegisterPersistedEntity()` | DI helper recording an entity for descriptor construction. | +## Related + +- Tutorial: [Persistence](https://tgiachi.github.io/squid-std/tutorials/persistence.html) + ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). diff --git a/src/SquidStd.Secrets.Aws/README.md b/src/SquidStd.Secrets.Aws/README.md index c2df2569..bbc215d0 100644 --- a/src/SquidStd.Secrets.Aws/README.md +++ b/src/SquidStd.Secrets.Aws/README.md @@ -68,6 +68,10 @@ await foreach (var name in store.ListNamesAsync("db/")) { /* ... */ } - **Tested against LocalStack** — the KMS and Secrets Manager adapters are covered by integration tests running on a `localstack/localstack` container. +## Related + +- Tutorial: [Secrets (KMS / Secrets Manager)](https://tgiachi.github.io/squid-std/tutorials/secrets.html) + ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). diff --git a/src/SquidStd.Vfs.Abstractions/README.md b/src/SquidStd.Vfs.Abstractions/README.md index e0b459a8..a68e0c3b 100644 --- a/src/SquidStd.Vfs.Abstractions/README.md +++ b/src/SquidStd.Vfs.Abstractions/README.md @@ -17,6 +17,10 @@ dotnet add package SquidStd.Vfs.Abstractions | `VfsPath` | Static helper that normalizes logical paths to forward-slash, root-relative form and rejects `.`/`..` traversal segments. | | `VfsEntry` | Record describing a listed entry: its logical path, byte size and last-modified UTC timestamp. | +## Related + +- Tutorial: [Virtual filesystem](https://tgiachi.github.io/squid-std/tutorials/vfs.html) + ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). diff --git a/src/SquidStd.Vfs/README.md b/src/SquidStd.Vfs/README.md index 9e6babb8..01137a3e 100644 --- a/src/SquidStd.Vfs/README.md +++ b/src/SquidStd.Vfs/README.md @@ -60,6 +60,10 @@ Logical paths are normalized (forward slashes, root-relative) and reject `..` tr | `VfsDirectories` | Named directory layout (`DirectoriesConfig` analogue) over any backend. | | `RegisterVfsExtensions` | `RegisterVfs(...)` registration. | +## Related + +- Tutorial: [Virtual filesystem](https://tgiachi.github.io/squid-std/tutorials/vfs.html) + ## License MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). From b0c7c0620442bc9f84903821485ae2f381dfb275 Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 28 Jun 2026 23:15:52 +0200 Subject: [PATCH 67/77] fix(vfs): make on-disk crypto vault persist and stop ListAsync crashing - ZipFileSystem.ListAsync no longer reads ZipArchiveEntry.Length (unavailable in ZipArchiveMode.Update for entries written this session); track uncompressed sizes in a dictionary on write/delete and fall back safely so listing never throws - CryptoFileSystem.Dispose now disposes its inner filesystem after Lock(), so a ZipFileSystem backend flushes its archive to disk and the vault persists - this fixes RegisterCryptoVault throwing InvalidOperationException on Lock()/Dispose() via PruneOrphans(), which made on-disk encrypted vaults unusable - add ZipFileSystem.ListAsync same-session test and disk-backed CryptoFileSystem lock + cross-instance persistence tests (previously untested: tests used InMemoryFileSystem) - update the Vfs sample and tutorial to demonstrate real on-disk vault persistence --- docs/tutorials/vfs.md | 16 ++-- samples/SquidStd.Samples.Vfs/Program.cs | 23 +++--- .../Vfs/Services/CryptoFileSystem.cs | 12 +++ src/SquidStd.Vfs/Services/ZipFileSystem.cs | 30 +++++++- .../Crypto/Vfs/CryptoFileSystemDiskTests.cs | 73 +++++++++++++++++++ .../SquidStd.Tests/Vfs/ZipFileSystemTests.cs | 35 +++++++++ 6 files changed, 166 insertions(+), 23 deletions(-) create mode 100644 tests/SquidStd.Tests/Crypto/Vfs/CryptoFileSystemDiskTests.cs diff --git a/docs/tutorials/vfs.md b/docs/tutorials/vfs.md index 4646c073..92082caa 100644 --- a/docs/tutorials/vfs.md +++ b/docs/tutorials/vfs.md @@ -31,14 +31,14 @@ The same `IVirtualFileSystem` API works regardless of the backend. `ReadAllBytes ### 3. Encrypted vault round-trip `CryptoFileSystem` encrypts every entry over a backend filesystem. The lifecycle is -unlock → write → lock; locking zeroes the key and flushes the encrypted index into the backend, -so re-opening the same backend with the passphrase decrypts the data at rest. - -This sample drives `CryptoFileSystem` over an **in-memory** backend to demonstrate the full -lifecycle. The DI helper `RegisterCryptoVault` wires a vault over a single on-disk zip file, but -that `ZipFileSystem` backend currently cannot be locked/persisted (a known limitation — its -`List` reads `ZipArchiveEntry.Length`, which .NET marks unavailable in `ZipArchiveMode.Update`). -On-disk vault persistence is therefore not yet supported. +unlock → write → dispose; disposing locks the vault (zeroing the key and flushing the encrypted +index) and disposes the backend, so a `ZipFileSystem` backend flushes its archive to disk. +Re-opening a fresh instance over the same file with the passphrase decrypts the data at rest. + +This sample backs the vault with a single on-disk zip file via `ZipFileSystem`, writes a secret, +disposes the vault, then re-opens a brand-new instance over the same file to prove on-disk +persistence. The DI helper `RegisterCryptoVault` wires exactly this — a vault over a single-file +zip — as a lockable singleton. [!code-csharp[](../../samples/SquidStd.Samples.Vfs/Program.cs#step-3)] diff --git a/samples/SquidStd.Samples.Vfs/Program.cs b/samples/SquidStd.Samples.Vfs/Program.cs index 4ecfa3e3..1a577c35 100644 --- a/samples/SquidStd.Samples.Vfs/Program.cs +++ b/samples/SquidStd.Samples.Vfs/Program.cs @@ -38,30 +38,27 @@ #region step-3 -// Encrypted vault lifecycle: unlock -> write -> lock, then re-open and read. -// RegisterCryptoVault wires a DI vault over a single on-disk zip file, but that ZipFileSystem -// backend currently cannot be locked/persisted (its List reads ZipArchiveEntry.Length, which -// .NET marks unavailable in ZipArchiveMode.Update), so this sample drives CryptoFileSystem over -// an in-memory backend to demonstrate the full lifecycle without that limitation. -var backend = new InMemoryFileSystem(); - -using (var vault = new CryptoFileSystem(backend)) +// Encrypted vault on a single on-disk zip file: unlock -> write -> dispose (flushes to disk), +// then re-open a brand-new instance over the same file to prove the data round-trips at rest. +var vaultPath = Path.Combine(Path.GetTempPath(), "squidstd-sample.vault"); + +using (var vault = new CryptoFileSystem(new ZipFileSystem(vaultPath))) { vault.Unlock("vault passphrase"); await vault.WriteAllBytesAsync("secret.txt", Encoding.UTF8.GetBytes("top secret")); - vault.Lock(); // zeroes the key and flushes the encrypted index into the backend -} +} // Dispose -> Lock (zeroes the key, flushes the encrypted index) -> flushes the zip to disk -// Re-open the same encrypted backend with the passphrase to prove the data round-trips at rest. -using (var reopened = new CryptoFileSystem(backend)) +// Re-open the same encrypted file with the passphrase; only the right passphrase decrypts it. +using (var reopened = new CryptoFileSystem(new ZipFileSystem(vaultPath))) { reopened.Unlock("vault passphrase"); - // The secret was written above, so it is present. var secret = await reopened.ReadAllBytesAsync("secret.txt"); Console.WriteLine($"Vault read after reopen: {Encoding.UTF8.GetString(secret!)}"); } +File.Delete(vaultPath); + #endregion await bootstrap.StopAsync(); diff --git a/src/SquidStd.Crypto/Vfs/Services/CryptoFileSystem.cs b/src/SquidStd.Crypto/Vfs/Services/CryptoFileSystem.cs index 044b6b00..078fca26 100644 --- a/src/SquidStd.Crypto/Vfs/Services/CryptoFileSystem.cs +++ b/src/SquidStd.Crypto/Vfs/Services/CryptoFileSystem.cs @@ -232,6 +232,18 @@ private void EnsureUnlocked() public void Dispose() { Lock(); + + // The vault owns its inner filesystem; disposing it flushes backends such as ZipFileSystem + // (which only writes its archive to disk on dispose) so the vault persists. + switch (_inner) + { + case IDisposable disposable: + disposable.Dispose(); + break; + case IAsyncDisposable asyncDisposable: + asyncDisposable.DisposeAsync().AsTask().GetAwaiter().GetResult(); + break; + } } private sealed class VaultWriteStream : MemoryStream diff --git a/src/SquidStd.Vfs/Services/ZipFileSystem.cs b/src/SquidStd.Vfs/Services/ZipFileSystem.cs index 742738d0..a080fa81 100644 --- a/src/SquidStd.Vfs/Services/ZipFileSystem.cs +++ b/src/SquidStd.Vfs/Services/ZipFileSystem.cs @@ -11,6 +11,7 @@ public sealed class ZipFileSystem : IVirtualFileSystem, IAsyncDisposable, IDispo { private readonly ZipArchive _archive; private readonly FileStream _file; + private readonly Dictionary _writtenLengths = new(StringComparer.Ordinal); public ZipFileSystem(string path) { @@ -50,6 +51,10 @@ public async ValueTask WriteAllBytesAsync( await using var stream = entry.Open(); await stream.WriteAsync(data, cancellationToken).ConfigureAwait(false); + + // ZipArchiveEntry.Length is unavailable in update mode once an entry has been opened for + // writing, so track the uncompressed size ourselves for ListAsync. + _writtenLengths[name] = data.Length; } public async Task OpenReadAsync(string path, CancellationToken cancellationToken = default) @@ -67,7 +72,8 @@ public Task OpenWriteAsync(string path, CancellationToken cancellationTo public ValueTask DeleteAsync(string path, CancellationToken cancellationToken = default) { - var entry = _archive.GetEntry(VfsPath.Normalize(path)); + var name = VfsPath.Normalize(path); + var entry = _archive.GetEntry(name); if (entry is null) { @@ -75,6 +81,7 @@ public ValueTask DeleteAsync(string path, CancellationToken cancellationTo } entry.Delete(); + _writtenLengths.Remove(name); return ValueTask.FromResult(true); } @@ -92,12 +99,31 @@ public async IAsyncEnumerable ListAsync( continue; } - yield return new VfsEntry(entry.FullName, entry.Length, entry.LastWriteTime); + yield return new VfsEntry(entry.FullName, EntrySize(entry), entry.LastWriteTime); await Task.CompletedTask; } } + private long EntrySize(ZipArchiveEntry entry) + { + if (_writtenLengths.TryGetValue(entry.FullName, out var length)) + { + return length; + } + + try + { + return entry.Length; + } + catch (InvalidOperationException) + { + // Unavailable in update mode for an entry opened for writing this session that we did + // not track (should not happen, but never let listing throw). + return 0; + } + } + public async ValueTask DisposeAsync() { _archive.Dispose(); diff --git a/tests/SquidStd.Tests/Crypto/Vfs/CryptoFileSystemDiskTests.cs b/tests/SquidStd.Tests/Crypto/Vfs/CryptoFileSystemDiskTests.cs new file mode 100644 index 00000000..b92cc5b5 --- /dev/null +++ b/tests/SquidStd.Tests/Crypto/Vfs/CryptoFileSystemDiskTests.cs @@ -0,0 +1,73 @@ +using System.Text; +using SquidStd.Crypto.Vfs.Data; +using SquidStd.Crypto.Vfs.Services; +using SquidStd.Vfs.Services; + +namespace SquidStd.Tests.Crypto.Vfs; + +public class CryptoFileSystemDiskTests +{ + private static CryptoVaultOptions FastOptions() + { + return new CryptoVaultOptions { Argon2MemoryKib = 8192, Argon2Iterations = 1 }; + } + + [Fact] + public async Task Lock_AfterWrites_OverZipBackend_DoesNotThrow() + { + var path = Path.Combine(Path.GetTempPath(), "squidstd-vault-" + Guid.NewGuid().ToString("N") + ".vault"); + + try + { + var vault = new CryptoFileSystem(new ZipFileSystem(path), FastOptions()); + vault.Unlock("pw"); + await vault.WriteAllBytesAsync("secret.txt", Encoding.UTF8.GetBytes("top secret")); + + // PruneOrphans() lists the zip backend, which previously threw because ZipArchiveEntry.Length + // is unavailable in update mode after a same-session write. + vault.Lock(); + + Assert.False(vault.IsUnlocked); + } + finally + { + if (File.Exists(path)) + { + File.Delete(path); + } + } + } + + [Fact] + public async Task Vault_PersistsToDisk_AcrossInstances() + { + var path = Path.Combine(Path.GetTempPath(), "squidstd-vault-" + Guid.NewGuid().ToString("N") + ".vault"); + + try + { + using (var vault = new CryptoFileSystem(new ZipFileSystem(path), FastOptions())) + { + vault.Unlock("correct horse"); + await vault.WriteAllBytesAsync("notes/plan.txt", Encoding.UTF8.GetBytes("attack at dawn")); + } // Dispose -> Lock -> flush the inner zip to disk + + Assert.True(new FileInfo(path).Length > 0, "the vault file must be written to disk"); + + using (var reopened = new CryptoFileSystem(new ZipFileSystem(path), FastOptions())) + { + reopened.Unlock("correct horse"); + var data = await reopened.ReadAllBytesAsync("notes/plan.txt"); + + Assert.NotNull(data); + Assert.Equal("attack at dawn", Encoding.UTF8.GetString(data!)); + } + } + finally + { + if (File.Exists(path)) + { + File.Delete(path); + } + } + } +} diff --git a/tests/SquidStd.Tests/Vfs/ZipFileSystemTests.cs b/tests/SquidStd.Tests/Vfs/ZipFileSystemTests.cs index 290a3983..f6e42f63 100644 --- a/tests/SquidStd.Tests/Vfs/ZipFileSystemTests.cs +++ b/tests/SquidStd.Tests/Vfs/ZipFileSystemTests.cs @@ -39,4 +39,39 @@ public async Task Write_Reopen_Read_Delete_RoundTrips() } } } + + [Fact] + public async Task ListAsync_AfterWritingInSameSession_DoesNotThrow_AndReportsSizes() + { + var path = Path.Combine(Path.GetTempPath(), "squidstd-vfs-" + Guid.NewGuid().ToString("N") + ".zip"); + + try + { + await using var fs = new ZipFileSystem(path); + await fs.WriteAllBytesAsync("a.txt", Encoding.UTF8.GetBytes("hello")); + await fs.WriteAllBytesAsync("b.txt", Encoding.UTF8.GetBytes("hi")); + + var entries = new List(); + long totalSize = 0; + + // In ZipArchiveMode.Update, ZipArchiveEntry.Length throws for entries written this + // session; ListAsync must not surface that. + await foreach (var entry in fs.ListAsync()) + { + entries.Add(entry.Path); + totalSize += entry.Size; + } + + Assert.Contains("a.txt", entries); + Assert.Contains("b.txt", entries); + Assert.Equal(7, totalSize); // 5 + 2 uncompressed bytes + } + finally + { + if (File.Exists(path)) + { + File.Delete(path); + } + } + } } From 11aaf7ae979895761da1ece415583de4805cc904 Mon Sep 17 00:00:00 2001 From: Tom Date: Mon, 29 Jun 2026 15:30:20 +0200 Subject: [PATCH 68/77] fix(vfs,crypto,actors): harden vault decoding, isolate reads, and make actor dispose race-safe - crypto: bound the unauthenticated per-record length prefix in EntryCipher.DecryptAsync against the configured chunk size before allocating, so a corrupt or tampered vault cannot trigger OOM or a negative-size crash - crypto: reject VaultBlob payloads shorter than nonce+tag instead of producing an opaque negative-length slice on a truncated index - crypto: make VaultIndex thread-safe (lock around Set/TryGet/Remove/Serialize, Entries returns a point-in-time snapshot) so concurrent vault operations cannot corrupt the map - vfs: InMemoryFileSystem.ReadAllBytesAsync returns a copy so callers mutating what they read no longer corrupt the stored bytes - vfs: PhysicalFileSystem.Resolve validates the resolved path stays within the root, rejecting segments that escape it (e.g. drive-rooted segments on Windows) - vfs: delete the dead duplicate SquidStd.Vfs/Internal/VfsPath.cs (unreferenced) - actors: claim the dispose guard atomically (Interlocked) and read it via Volatile so concurrent double-dispose cannot both run teardown --- src/SquidStd.Actors/Actor.cs | 7 ++-- .../Vfs/Internal/EntryCipher.cs | 7 ++++ src/SquidStd.Crypto/Vfs/Internal/VaultBlob.cs | 7 ++++ .../Vfs/Internal/VaultIndex.cs | 33 ++++++++++++++++--- src/SquidStd.Vfs/Internal/VfsPath.cs | 23 ------------- .../Services/InMemoryFileSystem.cs | 5 ++- .../Services/PhysicalFileSystem.cs | 11 ++++++- .../Actors/ActorLifecycleTests.cs | 13 ++++++++ .../Crypto/Vfs/EntryCipherTests.cs | 16 +++++++++ .../Crypto/Vfs/VaultBlobTests.cs | 31 +++++++++++++++++ .../Vfs/InMemoryFileSystemTests.cs | 13 ++++++++ 11 files changed, 132 insertions(+), 34 deletions(-) delete mode 100644 src/SquidStd.Vfs/Internal/VfsPath.cs create mode 100644 tests/SquidStd.Tests/Crypto/Vfs/VaultBlobTests.cs diff --git a/src/SquidStd.Actors/Actor.cs b/src/SquidStd.Actors/Actor.cs index 59f065cf..8ba5072d 100644 --- a/src/SquidStd.Actors/Actor.cs +++ b/src/SquidStd.Actors/Actor.cs @@ -21,7 +21,7 @@ public abstract class Actor : IAsyncDisposable private readonly CancellationTokenSource _shutdown; private readonly ConcurrentDictionary _outstanding; private readonly ILogger _logger; - private bool _disposed; + private int _disposed; /// Number of messages waiting in the mailbox. public int PendingCount @@ -163,7 +163,7 @@ private async Task ProcessAsync(TMessage message) private void ThrowIfDisposed() { - if (_disposed) + if (Volatile.Read(ref _disposed) != 0) { throw new ObjectDisposedException(GetType().Name); } @@ -172,12 +172,11 @@ private void ThrowIfDisposed() /// Completes the mailbox, drains in-flight work, and faults any still-pending requests. public async ValueTask DisposeAsync() { - if (_disposed) + if (Interlocked.Exchange(ref _disposed, 1) != 0) { return; } - _disposed = true; _shutdown.Cancel(); _mailbox.Complete(); diff --git a/src/SquidStd.Crypto/Vfs/Internal/EntryCipher.cs b/src/SquidStd.Crypto/Vfs/Internal/EntryCipher.cs index f6005edd..03ce8db0 100644 --- a/src/SquidStd.Crypto/Vfs/Internal/EntryCipher.cs +++ b/src/SquidStd.Crypto/Vfs/Internal/EntryCipher.cs @@ -43,6 +43,13 @@ public async Task DecryptAsync(Stream input, Stream output, CancellationToken ca var length = BinaryPrimitives.ReadInt32BigEndian(header); var nonce = header.AsMemory(4, NonceSize); + // The length prefix is read from the (unauthenticated) record header. Bound it before + // allocating so a corrupt or tampered vault cannot cause OOM or a negative-size crash. + if (length < 0 || length > _chunkSize) + { + throw new InvalidDataException("Encrypted entry record length is out of range."); + } + var cipher = new byte[length]; await ReadExactAsync(input, cipher, cancellationToken).ConfigureAwait(false); var tag = new byte[TagSize]; diff --git a/src/SquidStd.Crypto/Vfs/Internal/VaultBlob.cs b/src/SquidStd.Crypto/Vfs/Internal/VaultBlob.cs index 515f90d4..708a43bc 100644 --- a/src/SquidStd.Crypto/Vfs/Internal/VaultBlob.cs +++ b/src/SquidStd.Crypto/Vfs/Internal/VaultBlob.cs @@ -27,6 +27,13 @@ public static byte[] Encrypt(byte[] key, byte[] plaintext) public static byte[] Decrypt(byte[] key, byte[] blob) { + // A blob must hold at least a nonce and a tag; a shorter buffer (corrupt or truncated index) + // would otherwise produce a negative-length slice and throw an opaque ArgumentOutOfRange. + if (blob.Length < NonceSize + TagSize) + { + throw new InvalidDataException("Encrypted blob is too short to contain a nonce and tag."); + } + var nonce = blob.AsSpan(0, NonceSize); var tag = blob.AsSpan(blob.Length - TagSize, TagSize); var cipher = blob.AsSpan(NonceSize, blob.Length - NonceSize - TagSize); diff --git a/src/SquidStd.Crypto/Vfs/Internal/VaultIndex.cs b/src/SquidStd.Crypto/Vfs/Internal/VaultIndex.cs index d11447d1..fcd2c56e 100644 --- a/src/SquidStd.Crypto/Vfs/Internal/VaultIndex.cs +++ b/src/SquidStd.Crypto/Vfs/Internal/VaultIndex.cs @@ -9,8 +9,19 @@ internal sealed record VaultIndexEntry(string BlobId, long Size, DateTimeOffset internal sealed class VaultIndex { private readonly Dictionary _entries; + private readonly Lock _gate = new(); - public IReadOnlyDictionary Entries => _entries; + /// A point-in-time snapshot, safe to enumerate while other threads mutate the index. + public IReadOnlyDictionary Entries + { + get + { + lock (_gate) + { + return new Dictionary(_entries, StringComparer.Ordinal); + } + } + } public VaultIndex() { @@ -24,22 +35,34 @@ private VaultIndex(Dictionary entries) public void Set(string path, VaultIndexEntry entry) { - _entries[path] = entry; + lock (_gate) + { + _entries[path] = entry; + } } public bool TryGet(string path, out VaultIndexEntry? entry) { - return _entries.TryGetValue(path, out entry); + lock (_gate) + { + return _entries.TryGetValue(path, out entry); + } } public bool Remove(string path, out VaultIndexEntry? entry) { - return _entries.Remove(path, out entry); + lock (_gate) + { + return _entries.Remove(path, out entry); + } } public byte[] Serialize() { - return JsonSerializer.SerializeToUtf8Bytes(_entries); + lock (_gate) + { + return JsonSerializer.SerializeToUtf8Bytes(_entries); + } } public static VaultIndex Parse(byte[] data) diff --git a/src/SquidStd.Vfs/Internal/VfsPath.cs b/src/SquidStd.Vfs/Internal/VfsPath.cs deleted file mode 100644 index a9803ad6..00000000 --- a/src/SquidStd.Vfs/Internal/VfsPath.cs +++ /dev/null @@ -1,23 +0,0 @@ -namespace SquidStd.Vfs.Internal; - -/// Normalizes logical VFS paths to forward-slash, root-relative form and rejects traversal. -internal static class VfsPath -{ - public static string Normalize(string path) - { - ArgumentException.ThrowIfNullOrWhiteSpace(path); - - var segments = path.Replace('\\', '/') - .Split('/', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); - - foreach (var segment in segments) - { - if (segment is "." or "..") - { - throw new ArgumentException($"Path '{path}' must not contain relative segments.", nameof(path)); - } - } - - return string.Join('/', segments); - } -} diff --git a/src/SquidStd.Vfs/Services/InMemoryFileSystem.cs b/src/SquidStd.Vfs/Services/InMemoryFileSystem.cs index 31030dee..964002f3 100644 --- a/src/SquidStd.Vfs/Services/InMemoryFileSystem.cs +++ b/src/SquidStd.Vfs/Services/InMemoryFileSystem.cs @@ -20,7 +20,10 @@ public ValueTask ExistsAsync(string path, CancellationToken cancellationTo public ValueTask ReadAllBytesAsync(string path, CancellationToken cancellationToken = default) { - return ValueTask.FromResult(_files.TryGetValue(VfsPath.Normalize(path), out var entry) ? entry.Data : null); + // Return a copy: the stored array must not be aliased to callers, who may mutate what they read. + return ValueTask.FromResult( + _files.TryGetValue(VfsPath.Normalize(path), out var entry) ? (byte[])entry.Data.Clone() : null + ); } public ValueTask WriteAllBytesAsync( diff --git a/src/SquidStd.Vfs/Services/PhysicalFileSystem.cs b/src/SquidStd.Vfs/Services/PhysicalFileSystem.cs index 0231595d..4de9584c 100644 --- a/src/SquidStd.Vfs/Services/PhysicalFileSystem.cs +++ b/src/SquidStd.Vfs/Services/PhysicalFileSystem.cs @@ -105,6 +105,15 @@ public async IAsyncEnumerable ListAsync( private string Resolve(string path) { - return Path.Combine(_root, VfsPath.Normalize(path)); + var full = Path.GetFullPath(Path.Combine(_root, VfsPath.Normalize(path))); + + // Defend against segments that survive normalization yet escape the root (e.g. a Windows + // drive-rooted segment such as "C:/x", which Path.Combine would resolve outside _root). + if (full != _root && !full.StartsWith(_root + Path.DirectorySeparatorChar, StringComparison.Ordinal)) + { + throw new ArgumentException($"Path '{path}' escapes the filesystem root.", nameof(path)); + } + + return full; } } diff --git a/tests/SquidStd.Tests/Actors/ActorLifecycleTests.cs b/tests/SquidStd.Tests/Actors/ActorLifecycleTests.cs index 4b2617b0..03006962 100644 --- a/tests/SquidStd.Tests/Actors/ActorLifecycleTests.cs +++ b/tests/SquidStd.Tests/Actors/ActorLifecycleTests.cs @@ -27,4 +27,17 @@ public async Task TellAsync_AfterDispose_Throws() await Assert.ThrowsAsync(async () => await actor.TellAsync(new Append("x")) ); } + + [Fact] + public async Task DisposeAsync_CalledConcurrentlyTwice_IsIdempotent() + { + var actor = new ProbeActor(); + + // The dispose guard is claimed atomically, so concurrent disposers must not both run teardown. + var first = actor.DisposeAsync(); + var second = actor.DisposeAsync(); + + await first; + await second; + } } diff --git a/tests/SquidStd.Tests/Crypto/Vfs/EntryCipherTests.cs b/tests/SquidStd.Tests/Crypto/Vfs/EntryCipherTests.cs index fb67926a..4a10faa1 100644 --- a/tests/SquidStd.Tests/Crypto/Vfs/EntryCipherTests.cs +++ b/tests/SquidStd.Tests/Crypto/Vfs/EntryCipherTests.cs @@ -1,3 +1,4 @@ +using System.Buffers.Binary; using System.Security.Cryptography; using System.Text; using SquidStd.Crypto.Vfs.Internal; @@ -36,4 +37,19 @@ public async Task Decrypt_WrongKey_Throws() await Assert.ThrowsAsync(() => wrong.DecryptAsync(encrypted, new MemoryStream()) ); } + + [Fact] + public async Task Decrypt_RecordLengthExceedingChunkSize_Throws_WithoutAllocating() + { + const int chunkSize = 65536; + + // A record header whose unauthenticated length prefix claims a chunk larger than the cipher's + // configured chunk size must be rejected before allocation, not used to size a buffer. + var header = new byte[4 + 12]; + BinaryPrimitives.WriteInt32BigEndian(header, chunkSize + 1); + + using var cipher = new EntryCipher(RandomNumberGenerator.GetBytes(32), chunkSize); + await Assert.ThrowsAsync(() => cipher.DecryptAsync(new MemoryStream(header), new MemoryStream()) + ); + } } diff --git a/tests/SquidStd.Tests/Crypto/Vfs/VaultBlobTests.cs b/tests/SquidStd.Tests/Crypto/Vfs/VaultBlobTests.cs new file mode 100644 index 00000000..1262176b --- /dev/null +++ b/tests/SquidStd.Tests/Crypto/Vfs/VaultBlobTests.cs @@ -0,0 +1,31 @@ +using System.Security.Cryptography; +using System.Text; +using SquidStd.Crypto.Vfs.Internal; + +namespace SquidStd.Tests.Crypto.Vfs; + +public class VaultBlobTests +{ + [Fact] + public void EncryptThenDecrypt_RoundTrips() + { + var key = RandomNumberGenerator.GetBytes(32); + var plaintext = Encoding.UTF8.GetBytes("vault index payload"); + + var blob = VaultBlob.Encrypt(key, plaintext); + + Assert.Equal(plaintext, VaultBlob.Decrypt(key, blob)); + } + + [Fact] + public void Decrypt_BlobShorterThanNonceAndTag_Throws() + { + var key = RandomNumberGenerator.GetBytes(32); + + // A corrupt or truncated blob with fewer than nonce+tag bytes must fail with a clear data + // error instead of an opaque negative-length slice exception. + var truncated = new byte[12 + 16 - 1]; + + Assert.Throws(() => VaultBlob.Decrypt(key, truncated)); + } +} diff --git a/tests/SquidStd.Tests/Vfs/InMemoryFileSystemTests.cs b/tests/SquidStd.Tests/Vfs/InMemoryFileSystemTests.cs index 55cf8c16..8a2847b0 100644 --- a/tests/SquidStd.Tests/Vfs/InMemoryFileSystemTests.cs +++ b/tests/SquidStd.Tests/Vfs/InMemoryFileSystemTests.cs @@ -25,4 +25,17 @@ public async Task Write_Read_List_Delete_RoundTrips() Assert.True(await fs.DeleteAsync("a/b.txt")); Assert.Null(await fs.ReadAllBytesAsync("a/b.txt")); } + + [Fact] + public async Task ReadAllBytesAsync_ReturnsCopy_MutatingResultDoesNotCorruptStore() + { + var fs = new InMemoryFileSystem(); + await fs.WriteAllBytesAsync("note.txt", Encoding.UTF8.GetBytes("hi")); + + var first = (await fs.ReadAllBytesAsync("note.txt"))!; + first[0] = (byte)'X'; // a caller mutating what it read must not alter the stored bytes + + var second = (await fs.ReadAllBytesAsync("note.txt"))!; + Assert.Equal("hi", Encoding.UTF8.GetString(second)); + } } From 143b486cd92f86e9caba1032af00601391e861ad Mon Sep 17 00:00:00 2001 From: Tom Date: Mon, 29 Jun 2026 15:39:25 +0200 Subject: [PATCH 69/77] fix(persistence,network,rabbitmq): per-type replay, harden journal/UDP, retry failed declares MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - persistence: replay each journal entry against its own type's snapshot watermark instead of a single global maximum, so a partial snapshot save (one bucket persisted, another not) no longer silently drops the lagging type's journaled writes on the next startup - persistence: discard journal records whose framed length is smaller than the fixed entry header as a truncated tail, instead of letting the decoder crash startup - network: clear the UDP server's endpoint→listener routes on Stop so a later Start cannot send through a stale, disposed socket - network: mark a UDP session's connection closed when it is removed by the idle sweep, so holders observe IsConnected == false (CloseAsync stays idempotent for the explicit-close path) - rabbitmq: only keep a queue/exchange marked as declared when the declare succeeds; undo the optimistic mark on failure so a later call retries instead of assuming the topology exists --- .../Services/RabbitMqQueueProvider.cs | 72 +++++++++------ .../Services/RabbitMqTopicProvider.cs | 28 ++++-- .../Server/SquidStdUdpServer.cs | 4 + .../Sessions/UdpSessionManager.cs | 5 ++ .../Internal/JournalRecordCodec.cs | 2 +- .../Services/BinaryJournalService.cs | 4 +- .../Services/PersistenceService.cs | 8 +- .../Persistence/BinaryJournalServiceTests.cs | 27 ++++++ .../Persistence/PersistenceServiceTests.cs | 88 +++++++++++++++++++ 9 files changed, 199 insertions(+), 39 deletions(-) diff --git a/src/SquidStd.Messaging.RabbitMq/Services/RabbitMqQueueProvider.cs b/src/SquidStd.Messaging.RabbitMq/Services/RabbitMqQueueProvider.cs index b108174c..1478923e 100644 --- a/src/SquidStd.Messaging.RabbitMq/Services/RabbitMqQueueProvider.cs +++ b/src/SquidStd.Messaging.RabbitMq/Services/RabbitMqQueueProvider.cs @@ -148,11 +148,27 @@ private async Task EnsureTopologyAsync(IChannel channel, string queueName, Cance } } - // Dead-letter queues are terminal: declare them plainly, no DLX. - if (queueName.EndsWith(_messagingOptions.DeadLetterQueueSuffix, StringComparison.Ordinal)) + try { + // Dead-letter queues are terminal: declare them plainly, no DLX. + if (queueName.EndsWith(_messagingOptions.DeadLetterQueueSuffix, StringComparison.Ordinal)) + { + await channel.QueueDeclareAsync( + queueName, + true, + false, + false, + new Dictionary { ["x-queue-type"] = "quorum" }, + cancellationToken: cancellationToken + ); + + return; + } + + var deadLetterQueue = queueName + _messagingOptions.DeadLetterQueueSuffix; + await channel.QueueDeclareAsync( - queueName, + deadLetterQueue, true, false, false, @@ -160,34 +176,32 @@ await channel.QueueDeclareAsync( cancellationToken: cancellationToken ); - return; + await channel.QueueDeclareAsync( + queueName, + true, + false, + false, + new Dictionary + { + ["x-queue-type"] = "quorum", + ["x-delivery-limit"] = _messagingOptions.MaxDeliveryAttempts, + ["x-dead-letter-exchange"] = string.Empty, + ["x-dead-letter-routing-key"] = deadLetterQueue + }, + cancellationToken: cancellationToken + ); } - - var deadLetterQueue = queueName + _messagingOptions.DeadLetterQueueSuffix; - - await channel.QueueDeclareAsync( - deadLetterQueue, - true, - false, - false, - new Dictionary { ["x-queue-type"] = "quorum" }, - cancellationToken: cancellationToken - ); - - await channel.QueueDeclareAsync( - queueName, - true, - false, - false, - new Dictionary + catch + { + // A declare failed, so the topology is not actually established: undo the optimistic + // mark so a later call retries instead of assuming the queues exist. + lock (_topologySync) { - ["x-queue-type"] = "quorum", - ["x-delivery-limit"] = _messagingOptions.MaxDeliveryAttempts, - ["x-dead-letter-exchange"] = string.Empty, - ["x-dead-letter-routing-key"] = deadLetterQueue - }, - cancellationToken: cancellationToken - ); + _declared.Remove(queueName); + } + + throw; + } } private sealed class Subscription : IDisposable diff --git a/src/SquidStd.Messaging.RabbitMq/Services/RabbitMqTopicProvider.cs b/src/SquidStd.Messaging.RabbitMq/Services/RabbitMqTopicProvider.cs index 5d2d58b5..58bb3361 100644 --- a/src/SquidStd.Messaging.RabbitMq/Services/RabbitMqTopicProvider.cs +++ b/src/SquidStd.Messaging.RabbitMq/Services/RabbitMqTopicProvider.cs @@ -129,13 +129,27 @@ private async Task EnsureExchangeAsync(IChannel channel, string topic, Cancellat } } - await channel.ExchangeDeclareAsync( - topic, - ExchangeType.Fanout, - false, - false, - cancellationToken: cancellationToken - ); + try + { + await channel.ExchangeDeclareAsync( + topic, + ExchangeType.Fanout, + false, + false, + cancellationToken: cancellationToken + ); + } + catch + { + // The declare failed, so the exchange is not actually established: undo the optimistic + // mark so a later call retries instead of assuming the topology exists. + lock (_exchangeSync) + { + _declared.Remove(topic); + } + + throw; + } } private sealed class Subscription : IDisposable diff --git a/src/SquidStd.Network/Server/SquidStdUdpServer.cs b/src/SquidStd.Network/Server/SquidStdUdpServer.cs index 07a23b31..e62b9424 100644 --- a/src/SquidStd.Network/Server/SquidStdUdpServer.cs +++ b/src/SquidStd.Network/Server/SquidStdUdpServer.cs @@ -159,6 +159,10 @@ public async Task StopAsync(CancellationToken cancellationToken) loops = [.. _receiveLoops]; _listeners.Clear(); _receiveLoops.Clear(); + + // Drop the endpoint→listener routes: their UdpClients are about to be disposed, so a + // later Start must not reuse a stale, disposed socket from SendToAsync. + _endpointListeners.Clear(); } if (cts is not null) diff --git a/src/SquidStd.Network/Sessions/UdpSessionManager.cs b/src/SquidStd.Network/Sessions/UdpSessionManager.cs index 9ea76798..4842a55e 100644 --- a/src/SquidStd.Network/Sessions/UdpSessionManager.cs +++ b/src/SquidStd.Network/Sessions/UdpSessionManager.cs @@ -243,6 +243,11 @@ private void RemoveSession(IPEndPoint endPoint) if (_byEndpoint.TryRemove(endPoint, out var entry)) { _byId.TryRemove(entry.Session.SessionId, out _); + + // Mark the connection closed so holders observe IsConnected == false after a sweep + // removal. CloseAsync is idempotent, so the explicit-close path (which routes here via the + // close callback) is unaffected. + _ = entry.Session.Connection.CloseAsync(); RaiseSessionRemoved(entry.Session); } } diff --git a/src/SquidStd.Persistence/Internal/JournalRecordCodec.cs b/src/SquidStd.Persistence/Internal/JournalRecordCodec.cs index ca7a5b9c..43686e84 100644 --- a/src/SquidStd.Persistence/Internal/JournalRecordCodec.cs +++ b/src/SquidStd.Persistence/Internal/JournalRecordCodec.cs @@ -10,7 +10,7 @@ namespace SquidStd.Persistence.Internal; /// internal static class JournalRecordCodec { - private const int FixedHeader = 8 + 8 + 2 + 1 + 4; + internal const int FixedHeader = 8 + 8 + 2 + 1 + 4; public static byte[] Encode(JournalEntry entry) { diff --git a/src/SquidStd.Persistence/Services/BinaryJournalService.cs b/src/SquidStd.Persistence/Services/BinaryJournalService.cs index 529408cb..0667fbd5 100644 --- a/src/SquidStd.Persistence/Services/BinaryJournalService.cs +++ b/src/SquidStd.Persistence/Services/BinaryJournalService.cs @@ -141,7 +141,9 @@ private List ParseAll(byte[] bytes) var length = BinaryPrimitives.ReadInt32LittleEndian(bytes.AsSpan(offset)); var checksum = BinaryPrimitives.ReadUInt32LittleEndian(bytes.AsSpan(offset + 4)); - if (length <= 0 || offset + FrameHeaderSize + length > bytes.Length) + // A record shorter than the fixed entry header cannot be decoded (Decode reads the header + // unconditionally); treat it as a truncated tail rather than letting it crash startup. + if (length < JournalRecordCodec.FixedHeader || offset + FrameHeaderSize + length > bytes.Length) { _logger.Warning("Journal {Path}: truncated record at offset {Offset}; discarding tail", _path, offset); diff --git a/src/SquidStd.Persistence/Services/PersistenceService.cs b/src/SquidStd.Persistence/Services/PersistenceService.cs index beeab293..be27424b 100644 --- a/src/SquidStd.Persistence/Services/PersistenceService.cs +++ b/src/SquidStd.Persistence/Services/PersistenceService.cs @@ -49,6 +49,7 @@ public async ValueTask InitializeAsync(CancellationToken cancellationToken = def _registry.Freeze(); var maxSequenceId = 0L; + var snapshotThresholds = new Dictionary(); foreach (var descriptor in _registry.GetRegisteredDescriptors()) { @@ -60,12 +61,17 @@ public async ValueTask InitializeAsync(CancellationToken cancellationToken = def } ((IInternalEntityApplier)descriptor).LoadBucket(_stateStore, loaded.Bucket); + snapshotThresholds[descriptor.TypeId] = loaded.LastSequenceId; maxSequenceId = Math.Max(maxSequenceId, loaded.LastSequenceId); } foreach (var entry in await _journalService.ReadAllAsync(cancellationToken)) { - if (entry.SequenceId <= maxSequenceId) + // Replay each entry against its own type's snapshot watermark, not a global maximum: a + // single global threshold would skip journal entries newer than a lagging type's snapshot + // (e.g. after a partial snapshot save where one bucket persisted and another did not), + // silently dropping that type's recent writes. + if (entry.SequenceId <= snapshotThresholds.GetValueOrDefault(entry.TypeId)) { continue; } diff --git a/tests/SquidStd.Tests/Persistence/BinaryJournalServiceTests.cs b/tests/SquidStd.Tests/Persistence/BinaryJournalServiceTests.cs index a037391f..e345f481 100644 --- a/tests/SquidStd.Tests/Persistence/BinaryJournalServiceTests.cs +++ b/tests/SquidStd.Tests/Persistence/BinaryJournalServiceTests.cs @@ -1,3 +1,5 @@ +using System.Buffers.Binary; +using SquidStd.Core.Utils; using SquidStd.Persistence.Abstractions.Data; using SquidStd.Persistence.Abstractions.Types; using SquidStd.Persistence.Services; @@ -76,6 +78,31 @@ public async Task ReadAll_DiscardsTailOnChecksumMismatch() Assert.Empty(await reopened.ReadAllAsync()); } + [Fact] + public async Task ReadAll_DiscardsRecordShorterThanFixedHeader_WithoutThrowing() + { + await using (var journal = new BinaryJournalService(JournalPath)) + { + await journal.AppendAsync(Entry(1)); + } + + // Append a well-framed but undersized record (valid length + checksum, but fewer bytes than + // the fixed entry header). The reader must discard it as a truncated tail, not crash decoding. + var bytes = (await File.ReadAllBytesAsync(JournalPath)).ToList(); + var shortRecord = new byte[] { 1, 2, 3, 4, 5 }; + var frameHeader = new byte[8]; + BinaryPrimitives.WriteInt32LittleEndian(frameHeader, shortRecord.Length); + BinaryPrimitives.WriteUInt32LittleEndian(frameHeader.AsSpan(4), ChecksumUtils.Compute(shortRecord)); + bytes.AddRange(frameHeader); + bytes.AddRange(shortRecord); + await File.WriteAllBytesAsync(JournalPath, bytes.ToArray()); + + await using var reopened = new BinaryJournalService(JournalPath); + var entries = (await reopened.ReadAllAsync()).ToArray(); + + Assert.Equal([1], entries.Select(e => e.SequenceId)); + } + [Fact] public async Task TrimThroughSequence_KeepsNewerOnly() { diff --git a/tests/SquidStd.Tests/Persistence/PersistenceServiceTests.cs b/tests/SquidStd.Tests/Persistence/PersistenceServiceTests.cs index a8a580b9..72f97b7e 100644 --- a/tests/SquidStd.Tests/Persistence/PersistenceServiceTests.cs +++ b/tests/SquidStd.Tests/Persistence/PersistenceServiceTests.cs @@ -1,5 +1,6 @@ using SquidStd.Core.Json; using SquidStd.Persistence.Abstractions.Data; +using SquidStd.Persistence.Abstractions.Interfaces.Persistence; using SquidStd.Persistence.Data; using SquidStd.Persistence.Services; @@ -83,6 +84,52 @@ public async Task SaveSnapshot_TrimsJournal() await journal.DisposeAsync(); } + [Fact] + public async Task PartialSnapshotSave_ReplaysLaggingTypeFromJournal() + { + var serializer = new JsonDataSerializer(); + var config = new PersistenceConfig { SaveDirectory = _dir, AutosaveInterval = TimeSpan.FromHours(1) }; + + // First run: write one entity of each type, then attempt a snapshot whose second bucket save + // fails. One type's snapshot persists at the global watermark; the other's stays only in the + // (untrimmed) journal — the exact state a partial snapshot save leaves behind. + var journal = new BinaryJournalService(Path.Combine(_dir, config.JournalFileName)); + var faulty = new FailOnSecondSaveSnapshotService(new SnapshotService(_dir, config.SnapshotFileSuffix)); + var service = new PersistenceService(BuildRegistry(serializer), journal, faulty, config, eventBus: null); + + await service.InitializeAsync(); + await service.GetStore().UpsertAsync(new Player { Id = 1, Name = "First" }); + await service.GetStore().UpsertAsync(new Item { Id = 1, Name = "Sword" }); + + await Assert.ThrowsAnyAsync(async () => await service.SaveSnapshotAsync()); + await journal.DisposeAsync(); + + // Reload with a healthy snapshot service: both entities must survive. A single global replay + // watermark would skip the journal entry for whichever type's snapshot did persist, losing it. + var reloadJournal = new BinaryJournalService(Path.Combine(_dir, config.JournalFileName)); + var reloaded = new PersistenceService( + BuildRegistry(serializer), reloadJournal, new SnapshotService(_dir, config.SnapshotFileSuffix), config, eventBus: null + ); + + await reloaded.InitializeAsync(); + var player = await reloaded.GetStore().GetByIdAsync(1); + var item = await reloaded.GetStore().GetByIdAsync(1); + await reloadJournal.DisposeAsync(); + + Assert.NotNull(player); + Assert.NotNull(item); + Assert.Equal("Sword", item.Name); + } + + private static PersistenceEntityRegistry BuildRegistry(JsonDataSerializer serializer) + { + var registry = new PersistenceEntityRegistry(); + registry.Register(new PersistenceEntityDescriptor(serializer, serializer, 1, "Player", 1, p => p.Id)); + registry.Register(new PersistenceEntityDescriptor(serializer, serializer, 2, "Item", 1, i => i.Id)); + + return registry; + } + public void Dispose() { if (Directory.Exists(_dir)) @@ -96,4 +143,45 @@ private sealed class Player public int Id { get; set; } public string Name { get; set; } = string.Empty; } + + private sealed class Item + { + public int Id { get; set; } + public string Name { get; set; } = string.Empty; + } + + /// Delegates to a real snapshot service but throws on the second bucket save, simulating a + /// snapshot run that persists one type's bucket and then fails before the next. + private sealed class FailOnSecondSaveSnapshotService : ISnapshotService + { + private readonly ISnapshotService _inner; + private int _saveCount; + + public FailOnSecondSaveSnapshotService(ISnapshotService inner) + { + _inner = inner; + } + + public ValueTask SaveBucketAsync( + EntitySnapshotBucket bucket, long lastSequenceId, CancellationToken cancellationToken = default + ) + { + if (++_saveCount >= 2) + { + throw new IOException("Simulated snapshot write failure."); + } + + return _inner.SaveBucketAsync(bucket, lastSequenceId, cancellationToken); + } + + public ValueTask LoadBucketAsync(string typeName, CancellationToken cancellationToken = default) + { + return _inner.LoadBucketAsync(typeName, cancellationToken); + } + + public ValueTask DeleteBucketAsync(string typeName, CancellationToken cancellationToken = default) + { + return _inner.DeleteBucketAsync(typeName, cancellationToken); + } + } } From 95969b8637ff50613a11d0fa33c42728144ffd03 Mon Sep 17 00:00:00 2001 From: Tom Date: Mon, 29 Jun 2026 16:15:46 +0200 Subject: [PATCH 70/77] fix(actors): drain queued messages on dispose instead of dropping them Previously DisposeAsync cancelled the shutdown token before completing the mailbox. Because the mailbox was bound to that token, cancelling first aborted the block and silently discarded any queued fire-and-forget messages. - decouple the mailbox from the shutdown token so completing it drains the queue rather than aborting - DisposeAsync now completes the mailbox and drains queued work (Tells run, Asks reply) within the new ActorOptions.ShutdownDrainTimeout (default 5s); on timeout it cancels in-flight handlers, grants a brief grace period, then faults any still-pending requests with ObjectDisposedException - add ActorOptions.ShutdownDrainTimeout - document the new shutdown contract in the Actors README --- src/SquidStd.Actors/Actor.cs | 46 +++++++++++++++---- src/SquidStd.Actors/Data/ActorOptions.cs | 7 +++ src/SquidStd.Actors/README.md | 5 +- .../Actors/ActorLifecycleTests.cs | 31 +++++++++++-- 4 files changed, 74 insertions(+), 15 deletions(-) diff --git a/src/SquidStd.Actors/Actor.cs b/src/SquidStd.Actors/Actor.cs index 8ba5072d..23cd542e 100644 --- a/src/SquidStd.Actors/Actor.cs +++ b/src/SquidStd.Actors/Actor.cs @@ -38,14 +38,16 @@ protected Actor(ActorOptions? options = null) _outstanding = new ConcurrentDictionary(); _logger = Log.ForContext(GetType()); + // The mailbox is deliberately NOT bound to _shutdown.Token: cancelling that token would abort + // the block and discard queued messages. Dispose instead Completes the block to drain the queue, + // and only cancels _shutdown (observed by handlers via ReceiveAsync) once the drain budget elapses. var blockOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 1, EnsureOrdered = true, BoundedCapacity = _options.OverflowPolicy == ActorOverflowPolicy.Unbounded ? DataflowBlockOptions.Unbounded - : _options.Capacity, - CancellationToken = _shutdown.Token + : _options.Capacity }; _mailbox = new ActionBlock(ProcessAsync, blockOptions); @@ -169,7 +171,31 @@ private void ThrowIfDisposed() } } - /// Completes the mailbox, drains in-flight work, and faults any still-pending requests. + private async Task TryDrainAsync(TimeSpan timeout) + { + try + { + await _mailbox.Completion.WaitAsync(timeout); + + return true; + } + catch (TimeoutException) + { + return false; + } + catch (Exception ex) + { + // The mailbox completed in a faulted/cancelled state; that still counts as drained. + _logger.Debug(ex, "Actor {ActorType} mailbox completed with fault during dispose", GetType().Name); + + return true; + } + } + + /// + /// Completes the mailbox and drains queued work within , + /// then cancels any handlers still running and faults requests that never completed. + /// public async ValueTask DisposeAsync() { if (Interlocked.Exchange(ref _disposed, 1) != 0) @@ -177,18 +203,18 @@ public async ValueTask DisposeAsync() return; } - _shutdown.Cancel(); _mailbox.Complete(); - try + if (!await TryDrainAsync(_options.ShutdownDrainTimeout)) { - await _mailbox.Completion; - } - catch (Exception ex) - { - _logger.Debug(ex, "Actor {ActorType} mailbox completed with fault during dispose", GetType().Name); + // A handler is still running past the drain budget: cancel cancellation-honoring handlers + // and give them one more grace period to unwind before abandoning the drain. + _shutdown.Cancel(); + await TryDrainAsync(_options.ShutdownDrainTimeout); } + _shutdown.Cancel(); // idempotent; ensures handler tokens are observed cancelled before teardown + foreach (var request in _outstanding.Keys) { request.Fail(new ObjectDisposedException(GetType().Name)); diff --git a/src/SquidStd.Actors/Data/ActorOptions.cs b/src/SquidStd.Actors/Data/ActorOptions.cs index 3bdf693e..fefb36da 100644 --- a/src/SquidStd.Actors/Data/ActorOptions.cs +++ b/src/SquidStd.Actors/Data/ActorOptions.cs @@ -15,4 +15,11 @@ public sealed class ActorOptions /// Behavior when a fire-and-forget handler throws. public ActorErrorPolicy ErrorPolicy { get; init; } = ActorErrorPolicy.Isolate; + + /// + /// How long drains queued messages + /// before cancelling in-flight handlers. Queued work runs to completion within this budget; + /// once it elapses, the actor cancels and faults any still-pending requests. + /// + public TimeSpan ShutdownDrainTimeout { get; init; } = TimeSpan.FromSeconds(5); } diff --git a/src/SquidStd.Actors/README.md b/src/SquidStd.Actors/README.md index aa369882..c88652e1 100644 --- a/src/SquidStd.Actors/README.md +++ b/src/SquidStd.Actors/README.md @@ -71,13 +71,16 @@ using var sub = session.SubscribeToEventBus(eventBus, (UserJoinedEvent e) => new | `Capacity` | bounded mailbox size | `1024` | | `OverflowPolicy` | `Wait` / `DropNewest` / `Unbounded` | `Wait` | | `ErrorPolicy` | `Isolate` / `StopOnError` | `Isolate`| +| `ShutdownDrainTimeout` | any `TimeSpan` | `5s` | - **Wait**: `TellAsync` awaits until capacity frees (back-pressure). - **DropNewest**: `TellAsync` returns `false` when full. - **Isolate**: a throwing handler is logged and skipped; the actor stays alive. `AskAsync` exceptions always propagate to the caller regardless of policy. -`DisposeAsync` completes the mailbox, drains in-flight work, and faults any still-pending requests. +`DisposeAsync` completes the mailbox and drains queued messages — every `Tell` runs and every `Ask` +replies — within `ShutdownDrainTimeout`. If a handler is still running when that budget elapses, the +actor cancels its handlers and faults any requests that never completed with `ObjectDisposedException`. ## Related diff --git a/tests/SquidStd.Tests/Actors/ActorLifecycleTests.cs b/tests/SquidStd.Tests/Actors/ActorLifecycleTests.cs index 03006962..677c72f4 100644 --- a/tests/SquidStd.Tests/Actors/ActorLifecycleTests.cs +++ b/tests/SquidStd.Tests/Actors/ActorLifecycleTests.cs @@ -1,3 +1,4 @@ +using SquidStd.Actors.Data; using SquidStd.Tests.Actors.Support; namespace SquidStd.Tests.Actors; @@ -5,17 +6,39 @@ namespace SquidStd.Tests.Actors; public class ActorLifecycleTests { [Fact] - public async Task DisposeAsync_FaultsPendingAskRequests() + public async Task DisposeAsync_DrainsQueuedMessages() { + var gate = new TaskCompletionSource(); var actor = new ProbeActor(); - await actor.TellAsync(new HoldUntilCancelled()); // in-flight, honors cancellation + await actor.TellAsync(new Hold(gate)); // in-flight, blocks until released (ignores cancellation) + await actor.TellAsync(new Append("a")); // queued behind the hold + await actor.TellAsync(new Append("b")); // queued behind the hold + var logTask = actor.AskAsync(new GetLog()); // queued last + await Task.Delay(50); // let the request enqueue before dispose completes the mailbox - var ask = actor.AskAsync(new GetLog()); // queued behind the hold + var disposeTask = actor.DisposeAsync(); + gate.SetResult(); // release the hold so the queue can drain + + var log = await logTask; + await disposeTask; + + Assert.Equal("a,b", log); // the queued Tells ran during dispose instead of being dropped + } + + [Fact] + public async Task DisposeAsync_DrainTimeout_FaultsOutstandingAsk() + { + var gate = new TaskCompletionSource(); + var actor = new ProbeActor(new ActorOptions { ShutdownDrainTimeout = TimeSpan.FromMilliseconds(50) }); + await actor.TellAsync(new Hold(gate)); // in-flight, never released and ignores cancellation + var ask = actor.AskAsync(new GetLog()); // queued behind, never reached await Task.Delay(50); - await actor.DisposeAsync(); // cancels the hold; the queued request never runs + await actor.DisposeAsync(); // drain exceeds the budget; the still-pending request is faulted await Assert.ThrowsAsync(() => ask); + + gate.SetResult(); // release the orphaned handler (Reply is idempotent) } [Fact] From a47a2b355441f96f644dadc5b9ea38c807f33b20 Mon Sep 17 00:00:00 2001 From: Tom Date: Mon, 29 Jun 2026 18:17:18 +0200 Subject: [PATCH 71/77] fix(persistence): write the journal before applying an entity mutation in memory Upsert/Remove now build the entry and append it durably before mutating the in-memory bucket and committing LastSequenceId. A failed append leaves memory and the sequence counter untouched, so they never diverge from what journal replay would rebuild. --- .../Services/EntityStore.cs | 31 +++++-- .../EntityStoreWalOrderingTests.cs | 93 +++++++++++++++++++ 2 files changed, 118 insertions(+), 6 deletions(-) create mode 100644 tests/SquidStd.Tests/Persistence/EntityStoreWalOrderingTests.cs diff --git a/src/SquidStd.Persistence/Services/EntityStore.cs b/src/SquidStd.Persistence/Services/EntityStore.cs index 3dd5311e..e2f51e18 100644 --- a/src/SquidStd.Persistence/Services/EntityStore.cs +++ b/src/SquidStd.Persistence/Services/EntityStore.cs @@ -69,15 +69,18 @@ public async ValueTask UpsertAsync(TEntity entity, CancellationToken cancellatio try { JournalEntry entry; + TEntity clone; + TKey key; + long next; lock (_stateStore.SyncRoot) { - var clone = _descriptor.Clone(entity); - var key = _descriptor.GetKey(clone); - Bucket()[key] = clone; + clone = _descriptor.Clone(entity); + key = _descriptor.GetKey(clone); + next = _stateStore.LastSequenceId + 1; // computed, not yet committed entry = new JournalEntry { - SequenceId = ++_stateStore.LastSequenceId, + SequenceId = next, TimestampUnixMilliseconds = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), TypeId = _descriptor.TypeId, Operation = JournalEntityOperationType.Upsert, @@ -85,7 +88,14 @@ public async ValueTask UpsertAsync(TEntity entity, CancellationToken cancellatio }; } + // Write-ahead: the journal must be durable before the in-memory state reflects the change. await _journalService.AppendAsync(entry, cancellationToken); + + lock (_stateStore.SyncRoot) + { + Bucket()[key] = clone; + _stateStore.LastSequenceId = next; + } } finally { @@ -100,17 +110,19 @@ public async ValueTask RemoveAsync(TKey id, CancellationToken cancellation try { JournalEntry entry; + long next; lock (_stateStore.SyncRoot) { - if (!Bucket().Remove(id)) + if (!Bucket().ContainsKey(id)) { return false; } + next = _stateStore.LastSequenceId + 1; entry = new JournalEntry { - SequenceId = ++_stateStore.LastSequenceId, + SequenceId = next, TimestampUnixMilliseconds = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), TypeId = _descriptor.TypeId, Operation = JournalEntityOperationType.Remove, @@ -118,8 +130,15 @@ public async ValueTask RemoveAsync(TKey id, CancellationToken cancellation }; } + // Write-ahead: append before applying so a failed append leaves the entity in place. await _journalService.AppendAsync(entry, cancellationToken); + lock (_stateStore.SyncRoot) + { + Bucket().Remove(id); + _stateStore.LastSequenceId = next; + } + return true; } finally diff --git a/tests/SquidStd.Tests/Persistence/EntityStoreWalOrderingTests.cs b/tests/SquidStd.Tests/Persistence/EntityStoreWalOrderingTests.cs new file mode 100644 index 00000000..4a600045 --- /dev/null +++ b/tests/SquidStd.Tests/Persistence/EntityStoreWalOrderingTests.cs @@ -0,0 +1,93 @@ +using SquidStd.Core.Json; +using SquidStd.Persistence.Abstractions.Data; +using SquidStd.Persistence.Abstractions.Interfaces.Persistence; +using SquidStd.Persistence.Data; +using SquidStd.Persistence.Internal; +using SquidStd.Persistence.Services; + +namespace SquidStd.Tests.Persistence; + +public sealed class EntityStoreWalOrderingTests +{ + [Fact] + public async Task Upsert_WhenJournalAppendThrows_LeavesInMemoryStateUnchanged() + { + var serializer = new JsonDataSerializer(); + IPersistenceEntityDescriptor descriptor = + new PersistenceEntityDescriptor(serializer, serializer, 1, "Player", 1, p => p.Id); + var stateStore = new PersistenceStateStore(); + var store = new EntityStore(stateStore, new ThrowingJournalService(), descriptor); + + await Assert.ThrowsAsync(async () => await store.UpsertAsync(new Player { Id = 1, Name = "Bob" })); + + // The failed durable append must NOT have applied the mutation in memory. + Assert.Null(await store.GetByIdAsync(1)); + Assert.Equal(0, await store.CountAsync()); + } + + [Fact] + public async Task Remove_WhenJournalAppendThrows_KeepsEntity() + { + var serializer = new JsonDataSerializer(); + IPersistenceEntityDescriptor descriptor = + new PersistenceEntityDescriptor(serializer, serializer, 1, "Player", 1, p => p.Id); + var stateStore = new PersistenceStateStore(); + var ok = new CountingJournalService(); + var store = new EntityStore(stateStore, ok, descriptor); + await store.UpsertAsync(new Player { Id = 1, Name = "Bob" }); + + ok.FailNextAppend = true; + await Assert.ThrowsAsync(async () => await store.RemoveAsync(1)); + + Assert.NotNull(await store.GetByIdAsync(1)); // remove not applied because the append failed + } + + private sealed class Player + { + public int Id { get; set; } + public string Name { get; set; } = string.Empty; + } + + private sealed class ThrowingJournalService : IJournalService + { + public ValueTask AppendAsync(JournalEntry entry, CancellationToken cancellationToken = default) + => throw new IOException("simulated append failure"); + + public ValueTask AppendBatchAsync(IReadOnlyList entries, CancellationToken cancellationToken = default) + => throw new IOException("simulated append failure"); + + public ValueTask> ReadAllAsync(CancellationToken cancellationToken = default) + => ValueTask.FromResult>([]); + + public ValueTask ResetAsync(CancellationToken cancellationToken = default) => ValueTask.CompletedTask; + + public ValueTask TrimThroughSequenceAsync(long inclusiveSequenceId, CancellationToken cancellationToken = default) + => ValueTask.CompletedTask; + } + + private sealed class CountingJournalService : IJournalService + { + public bool FailNextAppend { get; set; } + + public ValueTask AppendAsync(JournalEntry entry, CancellationToken cancellationToken = default) + { + if (FailNextAppend) + { + throw new IOException("simulated append failure"); + } + + return ValueTask.CompletedTask; + } + + public ValueTask AppendBatchAsync(IReadOnlyList entries, CancellationToken cancellationToken = default) + => ValueTask.CompletedTask; + + public ValueTask> ReadAllAsync(CancellationToken cancellationToken = default) + => ValueTask.FromResult>([]); + + public ValueTask ResetAsync(CancellationToken cancellationToken = default) => ValueTask.CompletedTask; + + public ValueTask TrimThroughSequenceAsync(long inclusiveSequenceId, CancellationToken cancellationToken = default) + => ValueTask.CompletedTask; + } +} From c35d91b3cf3d1329f8d06d7e8797aa71ad7d3e77 Mon Sep 17 00:00:00 2001 From: Tom Date: Mon, 29 Jun 2026 18:18:39 +0200 Subject: [PATCH 72/77] fix(persistence): serialize SaveSnapshotAsync so concurrent runs cannot interleave Wrap the whole snapshot operation (capture, save, delete-empty, trim) in a dedicated lock so autosave, an explicit call, and StopAsync can no longer interleave their save/trim phases and regress a snapshot below an already-trimmed journal sequence. --- .../Services/PersistenceService.cs | 90 +++++++++++-------- .../PersistenceSnapshotConcurrencyTests.cs | 71 +++++++++++++++ 2 files changed, 122 insertions(+), 39 deletions(-) create mode 100644 tests/SquidStd.Tests/Persistence/PersistenceSnapshotConcurrencyTests.cs diff --git a/src/SquidStd.Persistence/Services/PersistenceService.cs b/src/SquidStd.Persistence/Services/PersistenceService.cs index be27424b..520f001a 100644 --- a/src/SquidStd.Persistence/Services/PersistenceService.cs +++ b/src/SquidStd.Persistence/Services/PersistenceService.cs @@ -22,6 +22,7 @@ public sealed class PersistenceService : IPersistenceService, IAsyncDisposable private readonly IPersistenceEntityRegistry _registry; private readonly ISnapshotService _snapshotService; private readonly PersistenceStateStore _stateStore = new(); + private readonly SemaphoreSlim _snapshotLock = new(1, 1); private CancellationTokenSource? _autosaveCts; private Task? _autosaveLoop; @@ -109,61 +110,72 @@ public async ValueTask InitializeAsync(CancellationToken cancellationToken = def public async ValueTask SaveSnapshotAsync(CancellationToken cancellationToken = default) { - long capturedSequenceId; - List buckets = []; - List emptyTypeNames = []; - - await _stateStore.WriteLock.WaitAsync(cancellationToken); + // One snapshot at a time: autosave, an explicit call, and StopAsync must not interleave their + // capture/save/trim phases, or a lower-sequence snapshot could land after a higher-sequence trim. + await _snapshotLock.WaitAsync(cancellationToken); try { - capturedSequenceId = _stateStore.LastSequenceId; + long capturedSequenceId; + List buckets = []; + List emptyTypeNames = []; + + await _stateStore.WriteLock.WaitAsync(cancellationToken); - foreach (var descriptor in _registry.GetRegisteredDescriptors()) + try { - lock (_stateStore.SyncRoot) - { - var bucket = ((IInternalEntityApplier)descriptor).CaptureBucket(_stateStore); + capturedSequenceId = _stateStore.LastSequenceId; - if (bucket is null) - { - emptyTypeNames.Add(descriptor.TypeName); - } - else + foreach (var descriptor in _registry.GetRegisteredDescriptors()) + { + lock (_stateStore.SyncRoot) { - buckets.Add(bucket); + var bucket = ((IInternalEntityApplier)descriptor).CaptureBucket(_stateStore); + + if (bucket is null) + { + emptyTypeNames.Add(descriptor.TypeName); + } + else + { + buckets.Add(bucket); + } } } } - } - finally - { - _stateStore.WriteLock.Release(); - } + finally + { + _stateStore.WriteLock.Release(); + } - if (_eventBus is not null) - { - await _eventBus.PublishAsync(new SnapshotSaveStartedEvent(capturedSequenceId), cancellationToken); - } + if (_eventBus is not null) + { + await _eventBus.PublishAsync(new SnapshotSaveStartedEvent(capturedSequenceId), cancellationToken); + } - foreach (var bucket in buckets) - { - await _snapshotService.SaveBucketAsync(bucket, capturedSequenceId, cancellationToken); - } + foreach (var bucket in buckets) + { + await _snapshotService.SaveBucketAsync(bucket, capturedSequenceId, cancellationToken); + } - foreach (var typeName in emptyTypeNames) - { - await _snapshotService.DeleteBucketAsync(typeName, cancellationToken); - } + foreach (var typeName in emptyTypeNames) + { + await _snapshotService.DeleteBucketAsync(typeName, cancellationToken); + } - await _journalService.TrimThroughSequenceAsync(capturedSequenceId, cancellationToken); + await _journalService.TrimThroughSequenceAsync(capturedSequenceId, cancellationToken); - if (_eventBus is not null) + if (_eventBus is not null) + { + await _eventBus.PublishAsync( + new SnapshotSaveCompletedEvent(capturedSequenceId, buckets.Count), + cancellationToken + ); + } + } + finally { - await _eventBus.PublishAsync( - new SnapshotSaveCompletedEvent(capturedSequenceId, buckets.Count), - cancellationToken - ); + _snapshotLock.Release(); } } diff --git a/tests/SquidStd.Tests/Persistence/PersistenceSnapshotConcurrencyTests.cs b/tests/SquidStd.Tests/Persistence/PersistenceSnapshotConcurrencyTests.cs new file mode 100644 index 00000000..e0e90790 --- /dev/null +++ b/tests/SquidStd.Tests/Persistence/PersistenceSnapshotConcurrencyTests.cs @@ -0,0 +1,71 @@ +using SquidStd.Core.Json; +using SquidStd.Persistence.Abstractions.Data; +using SquidStd.Persistence.Abstractions.Interfaces.Persistence; +using SquidStd.Persistence.Data; +using SquidStd.Persistence.Services; + +namespace SquidStd.Tests.Persistence; + +public sealed class PersistenceSnapshotConcurrencyTests : IDisposable +{ + private readonly string _dir = Path.Combine(Path.GetTempPath(), "squidstd-snapshot-conc-" + Guid.NewGuid().ToString("N")); + + [Fact] + public async Task SaveSnapshotAsync_ConcurrentCalls_DoNotInterleave() + { + var serializer = new JsonDataSerializer(); + var registry = new PersistenceEntityRegistry(); + registry.Register(new PersistenceEntityDescriptor(serializer, serializer, 1, "Player", 1, p => p.Id)); + var config = new PersistenceConfig { SaveDirectory = _dir, AutosaveInterval = TimeSpan.FromHours(1) }; + var journal = new BinaryJournalService(Path.Combine(_dir, config.JournalFileName)); + var snapshot = new ConcurrencyProbeSnapshotService(); + var service = new PersistenceService(registry, journal, snapshot, config, eventBus: null); + + await service.InitializeAsync(); + await service.GetStore().UpsertAsync(new Player { Id = 1, Name = "Bob" }); + + await Task.WhenAll( + Enumerable.Range(0, 8).Select(async _ => await service.SaveSnapshotAsync()) + ); + + await journal.DisposeAsync(); + Assert.Equal(1, snapshot.MaxObservedConcurrency); // never two snapshot operations at once + } + + public void Dispose() + { + if (Directory.Exists(_dir)) + { + Directory.Delete(_dir, true); + } + } + + private sealed class Player + { + public int Id { get; set; } + public string Name { get; set; } = string.Empty; + } + + private sealed class ConcurrencyProbeSnapshotService : ISnapshotService + { + private int _active; + + public int MaxObservedConcurrency { get; private set; } + + public async ValueTask SaveBucketAsync( + EntitySnapshotBucket bucket, long lastSequenceId, CancellationToken cancellationToken = default + ) + { + var now = Interlocked.Increment(ref _active); + MaxObservedConcurrency = Math.Max(MaxObservedConcurrency, now); + await Task.Delay(20, cancellationToken); + Interlocked.Decrement(ref _active); + } + + public ValueTask LoadBucketAsync(string typeName, CancellationToken cancellationToken = default) + => ValueTask.FromResult(null); + + public ValueTask DeleteBucketAsync(string typeName, CancellationToken cancellationToken = default) + => ValueTask.CompletedTask; + } +} From 8adbf32508f9d1c39dbec44fd7c77d7913ec16fe Mon Sep 17 00:00:00 2001 From: Tom Date: Mon, 29 Jun 2026 18:20:55 +0200 Subject: [PATCH 73/77] fix(persistence): checksum the whole snapshot envelope, not just the payload New snapshots are Version 2 with a checksum covering Version, LastSequenceId, TypeId, TypeName, SchemaVersion and Payload, so a corrupted sequence field is now detected. Version 1 files keep loading under the legacy payload-only checksum. Adds a two-span FNV overload to ChecksumUtils. --- src/SquidStd.Core/Utils/ChecksumUtils.cs | 20 +++++++ .../Internal/SnapshotEnvelopeCodec.cs | 11 ++++ .../Services/SnapshotService.cs | 18 ++++-- .../Persistence/SnapshotChecksumScopeTests.cs | 58 +++++++++++++++++++ 4 files changed, 103 insertions(+), 4 deletions(-) create mode 100644 tests/SquidStd.Tests/Persistence/SnapshotChecksumScopeTests.cs diff --git a/src/SquidStd.Core/Utils/ChecksumUtils.cs b/src/SquidStd.Core/Utils/ChecksumUtils.cs index 8d1fdb15..c70c5d1b 100644 --- a/src/SquidStd.Core/Utils/ChecksumUtils.cs +++ b/src/SquidStd.Core/Utils/ChecksumUtils.cs @@ -21,4 +21,24 @@ public static uint Compute(ReadOnlySpan data) return hash; } + + /// Computes the FNV-1a 32-bit checksum over two spans, in order, as if concatenated. + public static uint Compute(ReadOnlySpan first, ReadOnlySpan second) + { + var hash = FnvOffsetBasis; + + for (var i = 0; i < first.Length; i++) + { + hash ^= first[i]; + hash *= FnvPrime; + } + + for (var i = 0; i < second.Length; i++) + { + hash ^= second[i]; + hash *= FnvPrime; + } + + return hash; + } } diff --git a/src/SquidStd.Persistence/Internal/SnapshotEnvelopeCodec.cs b/src/SquidStd.Persistence/Internal/SnapshotEnvelopeCodec.cs index ccd42719..b0931f92 100644 --- a/src/SquidStd.Persistence/Internal/SnapshotEnvelopeCodec.cs +++ b/src/SquidStd.Persistence/Internal/SnapshotEnvelopeCodec.cs @@ -1,5 +1,6 @@ using System.Buffers.Binary; using System.Text; +using SquidStd.Core.Utils; using SquidStd.Persistence.Abstractions.Data; namespace SquidStd.Persistence.Internal; @@ -11,6 +12,16 @@ namespace SquidStd.Persistence.Internal; /// internal static class SnapshotEnvelopeCodec { + /// Byte offset of the 4-byte checksum field: after Version (4) and LastSequenceId (8). + internal const int ChecksumOffset = 12; + + /// + /// FNV-1a checksum over the whole encoded envelope except its own 4 checksum bytes — covering Version, + /// LastSequenceId, TypeId, TypeName, SchemaVersion and Payload (Version >= 2 snapshots). + /// + public static uint ComputeFullChecksum(ReadOnlySpan encoded) + => ChecksumUtils.Compute(encoded[..ChecksumOffset], encoded[(ChecksumOffset + 4)..]); + public static byte[] Encode(SnapshotFileEnvelope envelope) { var nameBytes = Encoding.UTF8.GetBytes(envelope.Bucket.TypeName); diff --git a/src/SquidStd.Persistence/Services/SnapshotService.cs b/src/SquidStd.Persistence/Services/SnapshotService.cs index 16e026f9..f82e6229 100644 --- a/src/SquidStd.Persistence/Services/SnapshotService.cs +++ b/src/SquidStd.Persistence/Services/SnapshotService.cs @@ -1,3 +1,4 @@ +using System.Buffers.Binary; using Serilog; using SquidStd.Core.Utils; using SquidStd.Persistence.Abstractions.Data; @@ -70,8 +71,11 @@ public async ValueTask DeleteBucketAsync(string typeName, CancellationToken canc var bytes = await File.ReadAllBytesAsync(path, cancellationToken); var envelope = SnapshotEnvelopeCodec.Decode(bytes); - if (ChecksumUtils.Compute(envelope.Bucket.Payload) != envelope.Checksum || - !string.Equals(envelope.Bucket.TypeName, typeName, StringComparison.Ordinal)) + var checksumValid = envelope.Version >= 2 + ? SnapshotEnvelopeCodec.ComputeFullChecksum(bytes) == envelope.Checksum + : ChecksumUtils.Compute(envelope.Bucket.Payload) == envelope.Checksum; + + if (!checksumValid || !string.Equals(envelope.Bucket.TypeName, typeName, StringComparison.Ordinal)) { _logger.Error("Snapshot {Path}: checksum or type-name mismatch; treating as absent", path); @@ -104,9 +108,9 @@ public async ValueTask SaveBucketAsync( var envelope = new SnapshotFileEnvelope { - Version = 1, + Version = 2, LastSequenceId = lastSequenceId, - Checksum = ChecksumUtils.Compute(bucket.Payload), + Checksum = 0, Bucket = bucket }; @@ -114,6 +118,12 @@ public async ValueTask SaveBucketAsync( var tempPath = path + ".tmp"; var bytes = SnapshotEnvelopeCodec.Encode(envelope); + // Checksum covers everything except its own 4 bytes; patch it into the encoded buffer in place. + BinaryPrimitives.WriteUInt32LittleEndian( + bytes.AsSpan(SnapshotEnvelopeCodec.ChecksumOffset), + SnapshotEnvelopeCodec.ComputeFullChecksum(bytes) + ); + await _ioLock.WaitAsync(cancellationToken); try diff --git a/tests/SquidStd.Tests/Persistence/SnapshotChecksumScopeTests.cs b/tests/SquidStd.Tests/Persistence/SnapshotChecksumScopeTests.cs new file mode 100644 index 00000000..167e9986 --- /dev/null +++ b/tests/SquidStd.Tests/Persistence/SnapshotChecksumScopeTests.cs @@ -0,0 +1,58 @@ +using SquidStd.Core.Utils; +using SquidStd.Persistence.Abstractions.Data; +using SquidStd.Persistence.Internal; +using SquidStd.Persistence.Services; + +namespace SquidStd.Tests.Persistence; + +public sealed class SnapshotChecksumScopeTests : IDisposable +{ + private readonly string _dir = Path.Combine(Path.GetTempPath(), "squidstd-snap-csum-" + Guid.NewGuid().ToString("N")); + + private static EntitySnapshotBucket Bucket() + => new() { TypeId = 1, TypeName = "Player", SchemaVersion = 1, Payload = [1, 2, 3, 4] }; + + [Fact] + public async Task LoadBucketAsync_WhenLastSequenceIdCorrupted_RejectsSnapshot() + { + var service = new SnapshotService(_dir, ".snapshot.bin"); + await service.SaveBucketAsync(Bucket(), lastSequenceId: 42); + + var file = Directory.GetFiles(_dir, "*.snapshot.bin").Single(); + var bytes = await File.ReadAllBytesAsync(file); + bytes[4] ^= 0xFF; // flip a byte inside the LastSequenceId field (offset 4..11) + await File.WriteAllBytesAsync(file, bytes); + + Assert.Null(await service.LoadBucketAsync("Player")); + } + + [Fact] + public async Task LoadBucketAsync_LegacyVersion1File_StillLoads() + { + var service = new SnapshotService(_dir, ".snapshot.bin"); + await service.SaveBucketAsync(Bucket(), lastSequenceId: 7); // creates the file at the correct path + + // Overwrite it with a legacy Version-1 envelope (payload-only checksum), as written before this change. + var file = Directory.GetFiles(_dir, "*.snapshot.bin").Single(); + var legacy = SnapshotEnvelopeCodec.Encode(new SnapshotFileEnvelope + { + Version = 1, + LastSequenceId = 7, + Checksum = ChecksumUtils.Compute(Bucket().Payload), + Bucket = Bucket() + }); + await File.WriteAllBytesAsync(file, legacy); + + var loaded = await service.LoadBucketAsync("Player"); + Assert.NotNull(loaded); + Assert.Equal(7, loaded.LastSequenceId); + } + + public void Dispose() + { + if (Directory.Exists(_dir)) + { + Directory.Delete(_dir, true); + } + } +} From 3179da16a5dafbc8fdd9fd970e3741161dbf6ac9 Mon Sep 17 00:00:00 2001 From: Tom Date: Mon, 29 Jun 2026 18:21:53 +0200 Subject: [PATCH 74/77] feat(persistence): add DurabilityMode config option (default Buffered) --- .../Data/PersistenceConfig.cs | 3 +++ .../Types/Persistence/DurabilityMode.cs | 11 ++++++++++ .../Persistence/PersistenceConfigTests.cs | 20 +++++++++++++++++++ 3 files changed, 34 insertions(+) create mode 100644 src/SquidStd.Persistence.Abstractions/Types/Persistence/DurabilityMode.cs create mode 100644 tests/SquidStd.Tests/Persistence/PersistenceConfigTests.cs diff --git a/src/SquidStd.Persistence.Abstractions/Data/PersistenceConfig.cs b/src/SquidStd.Persistence.Abstractions/Data/PersistenceConfig.cs index bfa8eedb..61065e8a 100644 --- a/src/SquidStd.Persistence.Abstractions/Data/PersistenceConfig.cs +++ b/src/SquidStd.Persistence.Abstractions/Data/PersistenceConfig.cs @@ -1,3 +1,5 @@ +using SquidStd.Persistence.Abstractions.Types.Persistence; + namespace SquidStd.Persistence.Abstractions.Data; /// Configuration for the persistence service: autosave cadence and snapshot/journal file names. @@ -8,4 +10,5 @@ public sealed class PersistenceConfig public string JournalFileName { get; set; } = "world.journal.bin"; public bool EnableFileLock { get; set; } = true; public string? SaveDirectory { get; set; } + public DurabilityMode DurabilityMode { get; set; } = DurabilityMode.Buffered; } diff --git a/src/SquidStd.Persistence.Abstractions/Types/Persistence/DurabilityMode.cs b/src/SquidStd.Persistence.Abstractions/Types/Persistence/DurabilityMode.cs new file mode 100644 index 00000000..83780497 --- /dev/null +++ b/src/SquidStd.Persistence.Abstractions/Types/Persistence/DurabilityMode.cs @@ -0,0 +1,11 @@ +namespace SquidStd.Persistence.Abstractions.Types.Persistence; + +/// How aggressively persistence writes are flushed to physical storage. +public enum DurabilityMode +{ + /// Flush to the OS cache only (fast; survives process crash, not power loss). Default. + Buffered, + + /// fsync the journal append and the snapshot temp file before rename (survives power loss). + Durable +} diff --git a/tests/SquidStd.Tests/Persistence/PersistenceConfigTests.cs b/tests/SquidStd.Tests/Persistence/PersistenceConfigTests.cs new file mode 100644 index 00000000..eef85fa2 --- /dev/null +++ b/tests/SquidStd.Tests/Persistence/PersistenceConfigTests.cs @@ -0,0 +1,20 @@ +using SquidStd.Persistence.Abstractions.Data; +using SquidStd.Persistence.Abstractions.Types.Persistence; + +namespace SquidStd.Tests.Persistence; + +public class PersistenceConfigTests +{ + [Fact] + public void DurabilityMode_DefaultsToBuffered() + { + Assert.Equal(DurabilityMode.Buffered, new PersistenceConfig().DurabilityMode); + } + + [Fact] + public void DurabilityMode_CanBeSetToDurable() + { + var config = new PersistenceConfig { DurabilityMode = DurabilityMode.Durable }; + Assert.Equal(DurabilityMode.Durable, config.DurabilityMode); + } +} From 37f388745422d20a4a645e9cd9fa954f5a245523 Mon Sep 17 00:00:00 2001 From: Tom Date: Mon, 29 Jun 2026 18:24:18 +0200 Subject: [PATCH 75/77] feat(persistence): optional fsync durability for journal and snapshot writes In DurabilityMode.Durable the journal opens its append stream WriteThrough and flushes to disk after each write/batch, and the snapshot fsyncs its temp file before the atomic rename. Buffered (default) keeps the current OS-cache flush. Documents the option and its portability limit. --- src/SquidStd.Persistence/README.md | 9 ++++ .../Services/BinaryJournalService.cs | 30 +++++++++-- .../Services/SnapshotService.cs | 15 +++++- .../Persistence/DurableWriteTests.cs | 53 +++++++++++++++++++ 4 files changed, 102 insertions(+), 5 deletions(-) create mode 100644 tests/SquidStd.Tests/Persistence/DurableWriteTests.cs diff --git a/src/SquidStd.Persistence/README.md b/src/SquidStd.Persistence/README.md index 05194669..39dde4a9 100644 --- a/src/SquidStd.Persistence/README.md +++ b/src/SquidStd.Persistence/README.md @@ -66,6 +66,15 @@ container.ApplyPersistedEntityRegistrations(); // builds descriptors into IPer | `SnapshotService` | Atomic per-type binary snapshot files with payload checksum. | | `RegisterPersistedEntity()` | DI helper recording an entity for descriptor construction. | +### Durability + +`PersistenceConfig.DurabilityMode` selects how writes reach disk. `Buffered` (default) flushes to the OS +cache — fast, and safe across a process crash. `Durable` fsyncs each journal append and the snapshot temp +file before its atomic rename, so committed data survives power loss. Pass it through when constructing the +services: `new BinaryJournalService(path, config.DurabilityMode)` and +`new SnapshotService(dir, suffix, config.DurabilityMode)`. (.NET has no portable directory fsync, so the +guarantee is per-file content durability plus atomic rename.) + ## Related - Tutorial: [Persistence](https://tgiachi.github.io/squid-std/tutorials/persistence.html) diff --git a/src/SquidStd.Persistence/Services/BinaryJournalService.cs b/src/SquidStd.Persistence/Services/BinaryJournalService.cs index 0667fbd5..308dfbec 100644 --- a/src/SquidStd.Persistence/Services/BinaryJournalService.cs +++ b/src/SquidStd.Persistence/Services/BinaryJournalService.cs @@ -3,6 +3,7 @@ using SquidStd.Core.Utils; using SquidStd.Persistence.Abstractions.Data; using SquidStd.Persistence.Abstractions.Interfaces.Persistence; +using SquidStd.Persistence.Abstractions.Types.Persistence; using SquidStd.Persistence.Internal; using ILogger = Serilog.ILogger; @@ -19,12 +20,18 @@ public sealed class BinaryJournalService : IJournalService, IAsyncDisposable private readonly SemaphoreSlim _ioLock = new(1, 1); private readonly ILogger _logger = Log.ForContext(); private readonly string _path; + private readonly DurabilityMode _durability; - public BinaryJournalService(string journalFilePath, bool enableFileLock = true) + public BinaryJournalService( + string journalFilePath, + DurabilityMode durability = DurabilityMode.Buffered, + bool enableFileLock = true + ) { ArgumentException.ThrowIfNullOrWhiteSpace(journalFilePath); _path = Path.GetFullPath(journalFilePath); + _durability = durability; _ = enableFileLock; var directory = Path.GetDirectoryName(_path); @@ -43,8 +50,13 @@ public async ValueTask AppendAsync(JournalEntry entry, CancellationToken cancell try { - await using var stream = new FileStream(_path, FileMode.Append, FileAccess.Write, FileShare.Read); + await using var stream = OpenAppendStream(); await WriteRecordAsync(stream, entry, cancellationToken); + + if (_durability == DurabilityMode.Durable) + { + stream.Flush(flushToDisk: true); + } } finally { @@ -62,12 +74,17 @@ public async ValueTask AppendBatchAsync( try { - await using var stream = new FileStream(_path, FileMode.Append, FileAccess.Write, FileShare.Read); + await using var stream = OpenAppendStream(); for (var i = 0; i < entries.Count; i++) { await WriteRecordAsync(stream, entries[i], cancellationToken); } + + if (_durability == DurabilityMode.Durable) + { + stream.Flush(flushToDisk: true); + } } finally { @@ -131,6 +148,13 @@ public async ValueTask TrimThroughSequenceAsync(long inclusiveSequenceId, Cancel } } + private FileStream OpenAppendStream() + { + var options = _durability == DurabilityMode.Durable ? FileOptions.WriteThrough : FileOptions.None; + + return new FileStream(_path, FileMode.Append, FileAccess.Write, FileShare.Read, bufferSize: 4096, options); + } + private List ParseAll(byte[] bytes) { var entries = new List(); diff --git a/src/SquidStd.Persistence/Services/SnapshotService.cs b/src/SquidStd.Persistence/Services/SnapshotService.cs index f82e6229..f7f42d5b 100644 --- a/src/SquidStd.Persistence/Services/SnapshotService.cs +++ b/src/SquidStd.Persistence/Services/SnapshotService.cs @@ -3,6 +3,7 @@ using SquidStd.Core.Utils; using SquidStd.Persistence.Abstractions.Data; using SquidStd.Persistence.Abstractions.Interfaces.Persistence; +using SquidStd.Persistence.Abstractions.Types.Persistence; using SquidStd.Persistence.Internal; using ILogger = Serilog.ILogger; @@ -22,14 +23,16 @@ public sealed class SnapshotService : ISnapshotService, IDisposable private readonly SemaphoreSlim _ioLock = new(1, 1); private readonly ILogger _logger = Log.ForContext(); private readonly string _suffix; + private readonly DurabilityMode _durability; - public SnapshotService(string saveDirectory, string fileSuffix) + public SnapshotService(string saveDirectory, string fileSuffix, DurabilityMode durability = DurabilityMode.Buffered) { ArgumentException.ThrowIfNullOrWhiteSpace(saveDirectory); ArgumentException.ThrowIfNullOrWhiteSpace(fileSuffix); _directory = Path.GetFullPath(saveDirectory); _suffix = fileSuffix; + _durability = durability; Directory.CreateDirectory(_directory); } @@ -131,7 +134,15 @@ public async ValueTask SaveBucketAsync( await using (var stream = new FileStream(tempPath, FileMode.Create, FileAccess.Write, FileShare.None)) { await stream.WriteAsync(bytes, cancellationToken); - await stream.FlushAsync(cancellationToken); + + if (_durability == DurabilityMode.Durable) + { + stream.Flush(flushToDisk: true); // fsync the temp file before the atomic rename + } + else + { + await stream.FlushAsync(cancellationToken); + } } File.Move(tempPath, path, true); diff --git a/tests/SquidStd.Tests/Persistence/DurableWriteTests.cs b/tests/SquidStd.Tests/Persistence/DurableWriteTests.cs new file mode 100644 index 00000000..8860e170 --- /dev/null +++ b/tests/SquidStd.Tests/Persistence/DurableWriteTests.cs @@ -0,0 +1,53 @@ +using SquidStd.Persistence.Abstractions.Data; +using SquidStd.Persistence.Abstractions.Types; +using SquidStd.Persistence.Abstractions.Types.Persistence; +using SquidStd.Persistence.Services; + +namespace SquidStd.Tests.Persistence; + +public sealed class DurableWriteTests : IDisposable +{ + private readonly string _dir = Path.Combine(Path.GetTempPath(), "squidstd-durable-" + Guid.NewGuid().ToString("N")); + + [Fact] + public async Task DurableJournal_AppendThenReadAll_RoundTrips() + { + var path = Path.Combine(_dir, "world.journal.bin"); + await using var journal = new BinaryJournalService(path, DurabilityMode.Durable); + + await journal.AppendAsync(new JournalEntry + { + SequenceId = 1, + TimestampUnixMilliseconds = 1000, + TypeId = 1, + Operation = JournalEntityOperationType.Upsert, + Payload = [1, 2, 3] + }); + + var entries = (await journal.ReadAllAsync()).ToArray(); + Assert.Single(entries); + Assert.Equal(1, entries[0].SequenceId); + } + + [Fact] + public async Task DurableSnapshot_SaveThenLoad_RoundTrips() + { + var service = new SnapshotService(_dir, ".snapshot.bin", DurabilityMode.Durable); + var bucket = new EntitySnapshotBucket { TypeId = 1, TypeName = "Player", SchemaVersion = 1, Payload = [9, 9] }; + + await service.SaveBucketAsync(bucket, lastSequenceId: 5); + var loaded = await service.LoadBucketAsync("Player"); + + Assert.NotNull(loaded); + Assert.Equal(5, loaded.LastSequenceId); + Assert.Equal(new byte[] { 9, 9 }, loaded.Bucket.Payload); + } + + public void Dispose() + { + if (Directory.Exists(_dir)) + { + Directory.Delete(_dir, true); + } + } +} From 21d331cc85cf96cd132606a591715d421704ba79 Mon Sep 17 00:00:00 2001 From: Tom Date: Mon, 29 Jun 2026 18:27:56 +0200 Subject: [PATCH 76/77] fix(persistence): disambiguate snapshot files by TypeId and migrate legacy names Snapshot files are now named snake(typeName)_{typeId}{suffix}, so two type names that snake-case to the same string no longer collide. LoadBucketAsync renames a legacy (pre-TypeId) file to the new scheme on first access; DeleteBucketAsync removes both forms. ISnapshotService Load/Delete gain a typeId parameter. --- .../Persistence/ISnapshotService.cs | 7 ++- .../Services/PersistenceService.cs | 10 ++-- .../Services/SnapshotService.cs | 40 ++++++++++--- .../Persistence/DurableWriteTests.cs | 2 +- .../Persistence/PersistenceServiceTests.cs | 10 ++-- .../PersistenceSnapshotConcurrencyTests.cs | 4 +- .../Persistence/SnapshotChecksumScopeTests.cs | 4 +- .../Persistence/SnapshotFilenameTests.cs | 57 +++++++++++++++++++ .../Persistence/SnapshotServiceTests.cs | 10 ++-- 9 files changed, 116 insertions(+), 28 deletions(-) create mode 100644 tests/SquidStd.Tests/Persistence/SnapshotFilenameTests.cs diff --git a/src/SquidStd.Persistence.Abstractions/Interfaces/Persistence/ISnapshotService.cs b/src/SquidStd.Persistence.Abstractions/Interfaces/Persistence/ISnapshotService.cs index d25c1798..fc772362 100644 --- a/src/SquidStd.Persistence.Abstractions/Interfaces/Persistence/ISnapshotService.cs +++ b/src/SquidStd.Persistence.Abstractions/Interfaces/Persistence/ISnapshotService.cs @@ -5,8 +5,11 @@ namespace SquidStd.Persistence.Abstractions.Interfaces.Persistence; /// Reads and writes per-type entity snapshot files. public interface ISnapshotService { - ValueTask DeleteBucketAsync(string typeName, CancellationToken cancellationToken = default); - ValueTask LoadBucketAsync(string typeName, CancellationToken cancellationToken = default); + ValueTask DeleteBucketAsync(string typeName, ushort typeId, CancellationToken cancellationToken = default); + + ValueTask LoadBucketAsync( + string typeName, ushort typeId, CancellationToken cancellationToken = default + ); ValueTask SaveBucketAsync( EntitySnapshotBucket bucket, long lastSequenceId, CancellationToken cancellationToken = default diff --git a/src/SquidStd.Persistence/Services/PersistenceService.cs b/src/SquidStd.Persistence/Services/PersistenceService.cs index 520f001a..17d4e2c9 100644 --- a/src/SquidStd.Persistence/Services/PersistenceService.cs +++ b/src/SquidStd.Persistence/Services/PersistenceService.cs @@ -54,7 +54,7 @@ public async ValueTask InitializeAsync(CancellationToken cancellationToken = def foreach (var descriptor in _registry.GetRegisteredDescriptors()) { - var loaded = await _snapshotService.LoadBucketAsync(descriptor.TypeName, cancellationToken); + var loaded = await _snapshotService.LoadBucketAsync(descriptor.TypeName, descriptor.TypeId, cancellationToken); if (loaded is null) { @@ -118,7 +118,7 @@ public async ValueTask SaveSnapshotAsync(CancellationToken cancellationToken = d { long capturedSequenceId; List buckets = []; - List emptyTypeNames = []; + List<(string TypeName, ushort TypeId)> emptyTypes = []; await _stateStore.WriteLock.WaitAsync(cancellationToken); @@ -134,7 +134,7 @@ public async ValueTask SaveSnapshotAsync(CancellationToken cancellationToken = d if (bucket is null) { - emptyTypeNames.Add(descriptor.TypeName); + emptyTypes.Add((descriptor.TypeName, descriptor.TypeId)); } else { @@ -158,9 +158,9 @@ public async ValueTask SaveSnapshotAsync(CancellationToken cancellationToken = d await _snapshotService.SaveBucketAsync(bucket, capturedSequenceId, cancellationToken); } - foreach (var typeName in emptyTypeNames) + foreach (var (typeName, typeId) in emptyTypes) { - await _snapshotService.DeleteBucketAsync(typeName, cancellationToken); + await _snapshotService.DeleteBucketAsync(typeName, typeId, cancellationToken); } await _journalService.TrimThroughSequenceAsync(capturedSequenceId, cancellationToken); diff --git a/src/SquidStd.Persistence/Services/SnapshotService.cs b/src/SquidStd.Persistence/Services/SnapshotService.cs index f7f42d5b..2212bcb5 100644 --- a/src/SquidStd.Persistence/Services/SnapshotService.cs +++ b/src/SquidStd.Persistence/Services/SnapshotService.cs @@ -37,11 +37,12 @@ public SnapshotService(string saveDirectory, string fileSuffix, DurabilityMode d Directory.CreateDirectory(_directory); } - public async ValueTask DeleteBucketAsync(string typeName, CancellationToken cancellationToken = default) + public async ValueTask DeleteBucketAsync(string typeName, ushort typeId, CancellationToken cancellationToken = default) { ArgumentException.ThrowIfNullOrWhiteSpace(typeName); - var path = PathFor(typeName); + var path = PathFor(typeName, typeId); + var legacyPath = LegacyPathFor(typeName); await _ioLock.WaitAsync(cancellationToken); @@ -49,6 +50,8 @@ public async ValueTask DeleteBucketAsync(string typeName, CancellationToken canc { File.Delete(path); File.Delete(path + ".tmp"); + File.Delete(legacyPath); + File.Delete(legacyPath + ".tmp"); } finally { @@ -56,16 +59,29 @@ public async ValueTask DeleteBucketAsync(string typeName, CancellationToken canc } } - public async ValueTask LoadBucketAsync(string typeName, CancellationToken cancellationToken = default) + public async ValueTask LoadBucketAsync( + string typeName, ushort typeId, CancellationToken cancellationToken = default + ) { ArgumentException.ThrowIfNullOrWhiteSpace(typeName); - var path = PathFor(typeName); + var path = PathFor(typeName, typeId); await _ioLock.WaitAsync(cancellationToken); try { + // Migrate a legacy (pre-TypeId) snapshot file name to the new scheme on first access. + if (!File.Exists(path)) + { + var legacyPath = LegacyPathFor(typeName); + + if (File.Exists(legacyPath)) + { + File.Move(legacyPath, path); + } + } + if (!File.Exists(path)) { return null; @@ -117,7 +133,7 @@ public async ValueTask SaveBucketAsync( Bucket = bucket }; - var path = PathFor(bucket.TypeName); + var path = PathFor(bucket.TypeName, bucket.TypeId); var tempPath = path + ".tmp"; var bytes = SnapshotEnvelopeCodec.Encode(envelope); @@ -153,14 +169,24 @@ public async ValueTask SaveBucketAsync( } } - private string PathFor(string typeName) + private string PathFor(string typeName, ushort typeId) + { + return Path.Combine(_directory, SnakeName(typeName) + "_" + typeId + _suffix); + } + + private string LegacyPathFor(string typeName) + { + return Path.Combine(_directory, SnakeName(typeName) + _suffix); + } + + private string SnakeName(string typeName) { if (typeName.AsSpan().IndexOfAny(_invalidTypeNameChars) >= 0) { throw new InvalidOperationException($"Persisted type name '{typeName}' cannot be used as a snapshot file name."); } - return Path.Combine(_directory, StringUtils.ToSnakeCase(typeName) + _suffix); + return StringUtils.ToSnakeCase(typeName); } public void Dispose() diff --git a/tests/SquidStd.Tests/Persistence/DurableWriteTests.cs b/tests/SquidStd.Tests/Persistence/DurableWriteTests.cs index 8860e170..4a554745 100644 --- a/tests/SquidStd.Tests/Persistence/DurableWriteTests.cs +++ b/tests/SquidStd.Tests/Persistence/DurableWriteTests.cs @@ -36,7 +36,7 @@ public async Task DurableSnapshot_SaveThenLoad_RoundTrips() var bucket = new EntitySnapshotBucket { TypeId = 1, TypeName = "Player", SchemaVersion = 1, Payload = [9, 9] }; await service.SaveBucketAsync(bucket, lastSequenceId: 5); - var loaded = await service.LoadBucketAsync("Player"); + var loaded = await service.LoadBucketAsync("Player", 1); Assert.NotNull(loaded); Assert.Equal(5, loaded.LastSequenceId); diff --git a/tests/SquidStd.Tests/Persistence/PersistenceServiceTests.cs b/tests/SquidStd.Tests/Persistence/PersistenceServiceTests.cs index 72f97b7e..63843288 100644 --- a/tests/SquidStd.Tests/Persistence/PersistenceServiceTests.cs +++ b/tests/SquidStd.Tests/Persistence/PersistenceServiceTests.cs @@ -174,14 +174,16 @@ public ValueTask SaveBucketAsync( return _inner.SaveBucketAsync(bucket, lastSequenceId, cancellationToken); } - public ValueTask LoadBucketAsync(string typeName, CancellationToken cancellationToken = default) + public ValueTask LoadBucketAsync( + string typeName, ushort typeId, CancellationToken cancellationToken = default + ) { - return _inner.LoadBucketAsync(typeName, cancellationToken); + return _inner.LoadBucketAsync(typeName, typeId, cancellationToken); } - public ValueTask DeleteBucketAsync(string typeName, CancellationToken cancellationToken = default) + public ValueTask DeleteBucketAsync(string typeName, ushort typeId, CancellationToken cancellationToken = default) { - return _inner.DeleteBucketAsync(typeName, cancellationToken); + return _inner.DeleteBucketAsync(typeName, typeId, cancellationToken); } } } diff --git a/tests/SquidStd.Tests/Persistence/PersistenceSnapshotConcurrencyTests.cs b/tests/SquidStd.Tests/Persistence/PersistenceSnapshotConcurrencyTests.cs index e0e90790..aba0e03b 100644 --- a/tests/SquidStd.Tests/Persistence/PersistenceSnapshotConcurrencyTests.cs +++ b/tests/SquidStd.Tests/Persistence/PersistenceSnapshotConcurrencyTests.cs @@ -62,10 +62,10 @@ public async ValueTask SaveBucketAsync( Interlocked.Decrement(ref _active); } - public ValueTask LoadBucketAsync(string typeName, CancellationToken cancellationToken = default) + public ValueTask LoadBucketAsync(string typeName, ushort typeId, CancellationToken cancellationToken = default) => ValueTask.FromResult(null); - public ValueTask DeleteBucketAsync(string typeName, CancellationToken cancellationToken = default) + public ValueTask DeleteBucketAsync(string typeName, ushort typeId, CancellationToken cancellationToken = default) => ValueTask.CompletedTask; } } diff --git a/tests/SquidStd.Tests/Persistence/SnapshotChecksumScopeTests.cs b/tests/SquidStd.Tests/Persistence/SnapshotChecksumScopeTests.cs index 167e9986..6116259f 100644 --- a/tests/SquidStd.Tests/Persistence/SnapshotChecksumScopeTests.cs +++ b/tests/SquidStd.Tests/Persistence/SnapshotChecksumScopeTests.cs @@ -23,7 +23,7 @@ public async Task LoadBucketAsync_WhenLastSequenceIdCorrupted_RejectsSnapshot() bytes[4] ^= 0xFF; // flip a byte inside the LastSequenceId field (offset 4..11) await File.WriteAllBytesAsync(file, bytes); - Assert.Null(await service.LoadBucketAsync("Player")); + Assert.Null(await service.LoadBucketAsync("Player", 1)); } [Fact] @@ -43,7 +43,7 @@ public async Task LoadBucketAsync_LegacyVersion1File_StillLoads() }); await File.WriteAllBytesAsync(file, legacy); - var loaded = await service.LoadBucketAsync("Player"); + var loaded = await service.LoadBucketAsync("Player", 1); Assert.NotNull(loaded); Assert.Equal(7, loaded.LastSequenceId); } diff --git a/tests/SquidStd.Tests/Persistence/SnapshotFilenameTests.cs b/tests/SquidStd.Tests/Persistence/SnapshotFilenameTests.cs new file mode 100644 index 00000000..065db64e --- /dev/null +++ b/tests/SquidStd.Tests/Persistence/SnapshotFilenameTests.cs @@ -0,0 +1,57 @@ +using SquidStd.Core.Utils; +using SquidStd.Persistence.Abstractions.Data; +using SquidStd.Persistence.Services; + +namespace SquidStd.Tests.Persistence; + +public sealed class SnapshotFilenameTests : IDisposable +{ + private readonly string _dir = Path.Combine(Path.GetTempPath(), "squidstd-snap-name-" + Guid.NewGuid().ToString("N")); + + private static EntitySnapshotBucket Bucket(ushort typeId, byte payload) + => new() { TypeId = typeId, TypeName = "Player", SchemaVersion = 1, Payload = [payload] }; + + [Fact] + public async Task SameTypeName_DifferentTypeId_ProduceDistinctFiles() + { + var service = new SnapshotService(_dir, ".snapshot.bin"); + + await service.SaveBucketAsync(Bucket(1, 0xAA), lastSequenceId: 1); + await service.SaveBucketAsync(Bucket(2, 0xBB), lastSequenceId: 2); + + Assert.Equal(2, Directory.GetFiles(_dir, "*.snapshot.bin").Length); + + var first = await service.LoadBucketAsync("Player", 1); + var second = await service.LoadBucketAsync("Player", 2); + + Assert.Equal(new byte[] { 0xAA }, first!.Bucket.Payload); + Assert.Equal(new byte[] { 0xBB }, second!.Bucket.Payload); + } + + [Fact] + public async Task LoadBucketAsync_MigratesLegacyNamedFile() + { + var service = new SnapshotService(_dir, ".snapshot.bin"); + + // Write a file at the OLD path (no TypeId) by saving then renaming to the legacy name. + await service.SaveBucketAsync(Bucket(1, 0x7), lastSequenceId: 9); + var newPath = Directory.GetFiles(_dir, "*.snapshot.bin").Single(); + var legacyPath = Path.Combine(_dir, StringUtils.ToSnakeCase("Player") + ".snapshot.bin"); + File.Move(newPath, legacyPath); + + var loaded = await service.LoadBucketAsync("Player", 1); + + Assert.NotNull(loaded); + Assert.Equal(9, loaded.LastSequenceId); + Assert.False(File.Exists(legacyPath)); // legacy file was migrated away + Assert.True(File.Exists(Path.Combine(_dir, "player_1.snapshot.bin"))); // to the new TypeId path + } + + public void Dispose() + { + if (Directory.Exists(_dir)) + { + Directory.Delete(_dir, true); + } + } +} diff --git a/tests/SquidStd.Tests/Persistence/SnapshotServiceTests.cs b/tests/SquidStd.Tests/Persistence/SnapshotServiceTests.cs index 1e26ae81..ca91d5fd 100644 --- a/tests/SquidStd.Tests/Persistence/SnapshotServiceTests.cs +++ b/tests/SquidStd.Tests/Persistence/SnapshotServiceTests.cs @@ -19,7 +19,7 @@ public async Task SaveThenLoad_RoundTrips() var service = Create(); await service.SaveBucketAsync(Bucket(), 42); - var loaded = await service.LoadBucketAsync("Player"); + var loaded = await service.LoadBucketAsync("Player", 1); Assert.NotNull(loaded); Assert.Equal(42, loaded.LastSequenceId); @@ -29,7 +29,7 @@ public async Task SaveThenLoad_RoundTrips() [Fact] public async Task LoadBucket_MissingFile_ReturnsNull() - => Assert.Null(await Create().LoadBucketAsync("Absent")); + => Assert.Null(await Create().LoadBucketAsync("Absent", 1)); [Fact] public async Task LoadBucket_CorruptPayload_ReturnsNull() @@ -41,7 +41,7 @@ public async Task LoadBucket_CorruptPayload_ReturnsNull() bytes[^1] ^= 0xFF; // corrupt the payload so checksum mismatches await File.WriteAllBytesAsync(path, bytes); - Assert.Null(await service.LoadBucketAsync("Player")); + Assert.Null(await service.LoadBucketAsync("Player", 1)); } [Fact] @@ -50,9 +50,9 @@ public async Task DeleteBucket_RemovesFile() var service = Create(); await service.SaveBucketAsync(Bucket(), 1); - await service.DeleteBucketAsync("Player"); + await service.DeleteBucketAsync("Player", 1); - Assert.Null(await service.LoadBucketAsync("Player")); + Assert.Null(await service.LoadBucketAsync("Player", 1)); } public void Dispose() From d10ff9b4f9927f22b75bbb511859d350775666af Mon Sep 17 00:00:00 2001 From: Tom Date: Mon, 29 Jun 2026 18:31:05 +0200 Subject: [PATCH 77/77] fix(network): bound per-connection TCP frame size to prevent unbounded buffering Add a configurable maxFrameLength (default 1 MiB), propagated from SquidTcpServer to each accepted client. When the pending buffer exceeds the cap without completing a frame, or a framer reports an oversized frame, the connection is closed before the buffer grows further. --- .../Client/SquidStdTcpClient.cs | 29 +++- src/SquidStd.Network/Server/SquidTcpServer.cs | 8 +- .../Network/TcpMaxFrameLengthTests.cs | 127 ++++++++++++++++++ 3 files changed, 159 insertions(+), 5 deletions(-) create mode 100644 tests/SquidStd.Tests/Network/TcpMaxFrameLengthTests.cs diff --git a/src/SquidStd.Network/Client/SquidStdTcpClient.cs b/src/SquidStd.Network/Client/SquidStdTcpClient.cs index d5514f7c..5bf29931 100644 --- a/src/SquidStd.Network/Client/SquidStdTcpClient.cs +++ b/src/SquidStd.Network/Client/SquidStdTcpClient.cs @@ -20,9 +20,11 @@ public sealed class SquidStdTcpClient : INetworkConnection, IAsyncDisposable, ID { private const int DefaultReceiveBufferSize = 8192; private const int DefaultHistoryBufferCapacity = 65536; + private const int DefaultMaxFrameLength = 1024 * 1024; private static long _sessionIdSequence; private readonly INetFramer? _framer; + private readonly int _maxFrameLength; private readonly CancellationTokenSource _internalCancellationTokenSource = new(); private readonly ILogger _logger = Log.ForContext(); @@ -137,7 +139,8 @@ public SquidStdTcpClient( INetFramer? framer = null, ITransportCodec? codec = null, int receiveBufferSize = DefaultReceiveBufferSize, - int historyBufferCapacity = DefaultHistoryBufferCapacity + int historyBufferCapacity = DefaultHistoryBufferCapacity, + int maxFrameLength = DefaultMaxFrameLength ) : this( socket, new NetworkStream(socket, false), @@ -145,7 +148,8 @@ public SquidStdTcpClient( framer, codec, receiveBufferSize, - historyBufferCapacity + historyBufferCapacity, + maxFrameLength ) { } @@ -160,7 +164,8 @@ public SquidStdTcpClient( INetFramer? framer = null, ITransportCodec? codec = null, int receiveBufferSize = DefaultReceiveBufferSize, - int historyBufferCapacity = DefaultHistoryBufferCapacity + int historyBufferCapacity = DefaultHistoryBufferCapacity, + int maxFrameLength = DefaultMaxFrameLength ) { ArgumentNullException.ThrowIfNull(socket); @@ -173,6 +178,7 @@ public SquidStdTcpClient( _codec = codec; _receiveBuffer = new CircularBuffer(historyBufferCapacity); ReceiveBufferSize = receiveBufferSize; + _maxFrameLength = maxFrameLength; SessionId = Interlocked.Increment(ref _sessionIdSequence); } @@ -484,6 +490,16 @@ private void EmitFrames() if (!_framer.TryReadFrame(view, out var frameLength)) { + // Incomplete frame. If the buffer already exceeds the cap, the in-progress frame is + // oversized (or the peer is streaming junk that never frames): reject and let the + // receive loop close the connection before the buffer grows further. + if (_pendingLength > _maxFrameLength) + { + throw new InvalidDataException( + $"Incoming frame exceeds the maximum of {_maxFrameLength} bytes." + ); + } + break; } @@ -495,6 +511,13 @@ private void EmitFrames() break; } + if (frameLength > _maxFrameLength) + { + throw new InvalidDataException( + $"Incoming frame of {frameLength} bytes exceeds the maximum of {_maxFrameLength} bytes." + ); + } + // Fresh copy so handlers can safely retain the payload. var frame = new byte[frameLength]; view[..frameLength].CopyTo(frame); diff --git a/src/SquidStd.Network/Server/SquidTcpServer.cs b/src/SquidStd.Network/Server/SquidTcpServer.cs index 3281a064..c02ab369 100644 --- a/src/SquidStd.Network/Server/SquidTcpServer.cs +++ b/src/SquidStd.Network/Server/SquidTcpServer.cs @@ -30,6 +30,7 @@ public sealed class SquidTcpServer : INetworkServer, IAsyncDisposable, IDisposab private readonly ILogger _logger = Log.ForContext(); private readonly Lock _middlewareSync = new(); private readonly int _receiveBufferSize; + private readonly int _maxFrameLength; private readonly SquidStdTcpServerTlsOptions? _tlsOptions; private Task? _acceptLoopTask; private CancellationTokenSource? _listenerCancellationTokenSource; @@ -74,7 +75,8 @@ public SquidTcpServer( int receiveBufferSize = 8192, int historyBufferCapacity = 65536, SquidStdTcpServerTlsOptions? tlsOptions = null, - Func? connectionPipelineFactory = null + Func? connectionPipelineFactory = null, + int maxFrameLength = 1024 * 1024 ) { _endPoint = endPoint; @@ -83,6 +85,7 @@ public SquidTcpServer( _historyBufferCapacity = historyBufferCapacity; _tlsOptions = tlsOptions; _connectionPipelineFactory = connectionPipelineFactory; + _maxFrameLength = maxFrameLength; } /// @@ -210,7 +213,8 @@ private async Task AcceptLoopAsync() framer, codec, _receiveBufferSize, - _historyBufferCapacity + _historyBufferCapacity, + _maxFrameLength ); WireClientEvents(client); diff --git a/tests/SquidStd.Tests/Network/TcpMaxFrameLengthTests.cs b/tests/SquidStd.Tests/Network/TcpMaxFrameLengthTests.cs new file mode 100644 index 00000000..7ad61377 --- /dev/null +++ b/tests/SquidStd.Tests/Network/TcpMaxFrameLengthTests.cs @@ -0,0 +1,127 @@ +using System.Buffers.Binary; +using System.Net; +using System.Net.Sockets; +using SquidStd.Network.Client; +using SquidStd.Network.Interfaces.Framing; + +namespace SquidStd.Tests.Network; + +public sealed class TcpMaxFrameLengthTests +{ + // 4-byte big-endian length prefix framer. + private sealed class LengthPrefixFramer : INetFramer + { + public bool TryReadFrame(ReadOnlySpan buffer, out int frameLength) + { + frameLength = 0; + + if (buffer.Length < 4) + { + return false; + } + + var payloadLength = BinaryPrimitives.ReadInt32BigEndian(buffer); + var total = 4 + payloadLength; + + if (buffer.Length < total) + { + return false; + } + + frameLength = total; + + return true; + } + } + + [Fact] + public async Task OversizedDeclaredFrame_ClosesConnection() + { + var (server, client) = await ConnectedPairAsync(maxFrameLength: 1024); + + try + { + // Declare a 10 MiB payload (way over the 1 KiB cap) then dribble bytes; the receiver must + // close instead of growing its pending buffer toward 10 MiB. + var header = new byte[4]; + BinaryPrimitives.WriteInt32BigEndian(header, 10 * 1024 * 1024); + await server.SendAsync(header, CancellationToken.None); + await server.SendAsync(new byte[4096], CancellationToken.None); + + var closed = await WaitUntilAsync(() => !client.IsConnected, TimeSpan.FromSeconds(5)); + Assert.True(closed); + } + finally + { + await client.DisposeAsync(); + await server.DisposeAsync(); + } + } + + [Fact] + public async Task FrameAtTheLimit_IsDelivered() + { + var (server, client) = await ConnectedPairAsync(maxFrameLength: 1024); + var received = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + client.OnDataReceived += (_, e) => received.TrySetResult(e.Data.ToArray()); + + try + { + var payload = new byte[1000]; + var frame = new byte[4 + payload.Length]; + BinaryPrimitives.WriteInt32BigEndian(frame, payload.Length); + payload.CopyTo(frame, 4); + await server.SendAsync(frame, CancellationToken.None); + + var got = await received.Task.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(4 + payload.Length, got.Length); + Assert.True(client.IsConnected); + } + finally + { + await client.DisposeAsync(); + await server.DisposeAsync(); + } + } + + private static async Task<(SquidStdTcpClient Server, SquidStdTcpClient Client)> ConnectedPairAsync(int maxFrameLength) + { + var listener = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + listener.Bind(new IPEndPoint(IPAddress.Loopback, 0)); + listener.Listen(1); + var port = ((IPEndPoint)listener.LocalEndPoint!).Port; + + var clientSocket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + var connectTask = clientSocket.ConnectAsync(IPAddress.Loopback, port); + var serverSocket = await listener.AcceptAsync(); + await connectTask; + listener.Dispose(); + + // "server" side here is just the sending end; the receiving end (client) enforces the cap. + var server = new SquidStdTcpClient(serverSocket); + var client = new SquidStdTcpClient(clientSocket, middlewares: null, framer: new LengthPrefixFramer(), + codec: null, maxFrameLength: maxFrameLength); + + await server.StartAsync(CancellationToken.None); + await client.StartAsync(CancellationToken.None); + + return (server, client); + } + + private static async Task WaitUntilAsync(Func condition, TimeSpan timeout) + { + var deadline = DateTime.UtcNow + timeout; + + while (DateTime.UtcNow < deadline) + { + if (condition()) + { + return true; + } + + await Task.Delay(25); + } + + return condition(); + } +}