diff --git a/SquidStd.slnx b/SquidStd.slnx index b6978e2..1905f0d 100644 --- a/SquidStd.slnx +++ b/SquidStd.slnx @@ -40,6 +40,8 @@ + + diff --git a/src/SquidStd.Vfs.Abstractions/DeferredWriteStream.cs b/src/SquidStd.Vfs.Abstractions/DeferredWriteStream.cs new file mode 100644 index 0000000..8a8ee28 --- /dev/null +++ b/src/SquidStd.Vfs.Abstractions/DeferredWriteStream.cs @@ -0,0 +1,42 @@ +namespace SquidStd.Vfs.Abstractions; + +/// +/// A writable in-memory stream that flushes its accumulated bytes through a callback exactly once on +/// disposal. Backs OpenWriteAsync for filesystems that persist the whole payload at close time. +/// +public sealed class DeferredWriteStream : MemoryStream +{ + private readonly Func _onFlush; + private readonly CancellationToken _cancellationToken; + private bool _flushed; + + public DeferredWriteStream(Func onFlush, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(onFlush); + + _onFlush = onFlush; + _cancellationToken = cancellationToken; + } + + public override async ValueTask DisposeAsync() + { + if (!_flushed) + { + _flushed = true; + await _onFlush(ToArray(), _cancellationToken).ConfigureAwait(false); + } + + await base.DisposeAsync().ConfigureAwait(false); + } + + protected override void Dispose(bool disposing) + { + if (disposing && !_flushed) + { + _flushed = true; + _onFlush(ToArray(), _cancellationToken).AsTask().GetAwaiter().GetResult(); + } + + base.Dispose(disposing); + } +} diff --git a/src/SquidStd.Vfs.Database/Data/Entities/VfsFileEntity.cs b/src/SquidStd.Vfs.Database/Data/Entities/VfsFileEntity.cs new file mode 100644 index 0000000..fff35a9 --- /dev/null +++ b/src/SquidStd.Vfs.Database/Data/Entities/VfsFileEntity.cs @@ -0,0 +1,18 @@ +using FreeSql.DataAnnotations; +using SquidStd.Database.Abstractions.Data.Entities; + +namespace SquidStd.Vfs.Database.Data.Entities; + +/// A stored file keyed by its logical VFS path. +[Index("uq_vfs_file_path", "Path", true)] +public sealed class VfsFileEntity : BaseEntity +{ + /// The logical VFS path (unique). + public string Path { get; set; } = string.Empty; + + /// The file bytes. + public byte[] Content { get; set; } = []; + + /// The file size in bytes. + public long Size { get; set; } +} diff --git a/src/SquidStd.Vfs.Database/Extensions/RegisterDatabaseFileSystemExtensions.cs b/src/SquidStd.Vfs.Database/Extensions/RegisterDatabaseFileSystemExtensions.cs new file mode 100644 index 0000000..c1b1b18 --- /dev/null +++ b/src/SquidStd.Vfs.Database/Extensions/RegisterDatabaseFileSystemExtensions.cs @@ -0,0 +1,29 @@ +using DryIoc; +using SquidStd.Database.Abstractions.Interfaces.Data; +using SquidStd.Vfs.Abstractions.Interfaces; +using SquidStd.Vfs.Database.Data.Entities; +using SquidStd.Vfs.Database.Services; + +namespace SquidStd.Vfs.Database.Extensions; + +/// DryIoc registration helper for the database VFS backend. +public static class RegisterDatabaseFileSystemExtensions +{ + /// Container that receives the VFS registration. + extension(IContainer container) + { + /// + /// Registers an backed by the database. + /// Requires the SquidStd.Database module to be registered (it supplies IDataAccess<>). + /// + public IContainer RegisterDatabaseFileSystem() + { + container.RegisterDelegate( + r => new DatabaseFileSystem(r.Resolve>()), + Reuse.Singleton + ); + + return container; + } + } +} diff --git a/src/SquidStd.Vfs.Database/README.md b/src/SquidStd.Vfs.Database/README.md new file mode 100644 index 0000000..d50e938 --- /dev/null +++ b/src/SquidStd.Vfs.Database/README.md @@ -0,0 +1,55 @@ +

SquidStd.Vfs.Database

+ +An `IVirtualFileSystem` that stores files as rows in a relational database via SquidStd.Database / FreeSql. +Each file is a single `VfsFileEntity` row keyed by path; writes are upserts (last write wins, no concurrent +write safety guarantee). + +## Install + +```bash +dotnet add package SquidStd.Vfs.Database +``` + +## Usage + +Register `SquidStd.Database` first (it provides `IDataAccess<>`), then add the VFS backend: + +```csharp +using DryIoc; +using SquidStd.Vfs.Abstractions.Interfaces; +using SquidStd.Vfs.Database.Extensions; + +// SquidStd.Database must already be registered on the container. +container.RegisterDatabaseFileSystem(); + +var fs = container.Resolve(); + +await fs.WriteAllBytesAsync("config/settings.json", payload); +byte[]? data = await fs.ReadAllBytesAsync("config/settings.json"); + +await foreach (var entry in fs.ListAsync("config")) + Console.WriteLine($"{entry.Path} ({entry.Size} bytes)"); +``` + +The `VfsFileEntity` table is created automatically by FreeSql on first access (sync-structure mode). + +> **Single-writer assumption** — upsert operations are not serialised across concurrent writers. Use this +> backend in single-process scenarios or where last-write-wins is acceptable. + +## Key types + +| Type | Purpose | +|---|---| +| `RegisterDatabaseFileSystemExtensions` | `RegisterDatabaseFileSystem()` DryIoc registration. | +| `DatabaseFileSystem` | FreeSql-backed `IVirtualFileSystem`. | +| `VfsFileEntity` | ORM entity: `Path` (PK), `Content` (blob), `UpdatedAt`. | + +## Related + +- Tutorial: [Virtual filesystem](https://tgiachi.github.io/squid-std/tutorials/vfs.html) +- [`SquidStd.Vfs`](../SquidStd.Vfs/README.md) — core backends and decorators +- [`SquidStd.Database`](../SquidStd.Database/README.md) — required database module + +## License + +MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). diff --git a/src/SquidStd.Vfs.Database/Services/DatabaseFileSystem.cs b/src/SquidStd.Vfs.Database/Services/DatabaseFileSystem.cs new file mode 100644 index 0000000..02e9265 --- /dev/null +++ b/src/SquidStd.Vfs.Database/Services/DatabaseFileSystem.cs @@ -0,0 +1,120 @@ +using System.Runtime.CompilerServices; +using SquidStd.Database.Abstractions.Interfaces.Data; +using SquidStd.Vfs.Abstractions; +using SquidStd.Vfs.Abstractions.Data; +using SquidStd.Vfs.Abstractions.Interfaces; +using SquidStd.Vfs.Database.Data.Entities; + +namespace SquidStd.Vfs.Database.Services; + +/// +/// A implementation that stores files as rows in a relational +/// database via the generic abstraction (FreeSql). +/// +public sealed class DatabaseFileSystem : IVirtualFileSystem +{ + private readonly IDataAccess _data; + + /// + /// Initializes the database filesystem. + /// + /// The data access for rows. + public DatabaseFileSystem(IDataAccess dataAccess) + { + _data = dataAccess; + } + + /// + public async ValueTask ExistsAsync(string path, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrEmpty(path); + + return await _data.ExistsAsync(e => e.Path == path, cancellationToken).ConfigureAwait(false); + } + + /// + public async ValueTask ReadAllBytesAsync(string path, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrEmpty(path); + + var rows = await _data.QueryAsync(e => e.Path == path, cancellationToken).ConfigureAwait(false); + + return rows.Count > 0 ? rows[0].Content : null; + } + + /// + public async ValueTask WriteAllBytesAsync( + string path, + ReadOnlyMemory data, + CancellationToken cancellationToken = default + ) + { + ArgumentException.ThrowIfNullOrEmpty(path); + + var rows = await _data.QueryAsync(e => e.Path == path, cancellationToken).ConfigureAwait(false); + var existing = rows.Count > 0 ? rows[0] : null; + var bytes = data.ToArray(); + + if (existing is not null) + { + existing.Content = bytes; + existing.Size = bytes.Length; + await _data.UpdateAsync(existing, cancellationToken).ConfigureAwait(false); + } + else + { + await _data.InsertAsync( + new VfsFileEntity { Path = path, Content = bytes, Size = bytes.Length }, + cancellationToken + ).ConfigureAwait(false); + } + } + + /// + public async Task OpenReadAsync(string path, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrEmpty(path); + + var bytes = await ReadAllBytesAsync(path, cancellationToken).ConfigureAwait(false) + ?? throw new FileNotFoundException($"No file at '{path}'.", path); + + return new MemoryStream(bytes, writable: false); + } + + /// + public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrEmpty(path); + + return Task.FromResult( + new DeferredWriteStream( + (bytes, ct) => WriteAllBytesAsync(path, bytes, ct), + cancellationToken + ) + ); + } + + /// + public async ValueTask DeleteAsync(string path, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrEmpty(path); + + return await _data.BulkDeleteAsync(e => e.Path == path, cancellationToken).ConfigureAwait(false) > 0; + } + + /// + public async IAsyncEnumerable ListAsync( + string? prefix = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default + ) + { + var rows = string.IsNullOrEmpty(prefix) + ? await _data.QueryAsync(cancellationToken: cancellationToken).ConfigureAwait(false) + : await _data.QueryAsync(e => e.Path.StartsWith(prefix), cancellationToken).ConfigureAwait(false); + + foreach (var e in rows) + { + yield return new VfsEntry(e.Path, e.Size, e.Updated); + } + } +} diff --git a/src/SquidStd.Vfs.Database/SquidStd.Vfs.Database.csproj b/src/SquidStd.Vfs.Database/SquidStd.Vfs.Database.csproj new file mode 100644 index 0000000..b267a42 --- /dev/null +++ b/src/SquidStd.Vfs.Database/SquidStd.Vfs.Database.csproj @@ -0,0 +1,20 @@ + + + + true + net10.0 + enable + enable + + + + + + + + + + + + + diff --git a/src/SquidStd.Vfs.S3/Data/S3FileSystemOptions.cs b/src/SquidStd.Vfs.S3/Data/S3FileSystemOptions.cs new file mode 100644 index 0000000..48adbf0 --- /dev/null +++ b/src/SquidStd.Vfs.S3/Data/S3FileSystemOptions.cs @@ -0,0 +1,13 @@ +using SquidStd.Aws.Abstractions.Data.Config; + +namespace SquidStd.Vfs.S3.Data; + +/// Connection options for the S3-compatible VFS backend (AWS / MinIO / R2 / B2). +public sealed class S3FileSystemOptions +{ + /// AWS-style connection config; ServiceUrl is the full endpoint for S3-compatibles. + public AwsConfigEntry Aws { get; init; } = new(); + + /// Bucket name. + public string Bucket { get; init; } = string.Empty; +} diff --git a/src/SquidStd.Vfs.S3/Extensions/RegisterS3FileSystemExtensions.cs b/src/SquidStd.Vfs.S3/Extensions/RegisterS3FileSystemExtensions.cs new file mode 100644 index 0000000..548a3df --- /dev/null +++ b/src/SquidStd.Vfs.S3/Extensions/RegisterS3FileSystemExtensions.cs @@ -0,0 +1,26 @@ +using DryIoc; +using SquidStd.Vfs.Abstractions.Interfaces; +using SquidStd.Vfs.S3.Data; +using SquidStd.Vfs.S3.Services; + +namespace SquidStd.Vfs.S3.Extensions; + +/// DryIoc registration helper for the S3-compatible VFS backend. +public static class RegisterS3FileSystemExtensions +{ + /// Container that receives the VFS registration. + extension(IContainer container) + { + /// Registers an backed by S3-compatible storage. + public IContainer RegisterS3FileSystem(Action configure) + { + ArgumentNullException.ThrowIfNull(configure); + + var options = new S3FileSystemOptions(); + configure(options); + container.RegisterDelegate(_ => new S3FileSystem(options), Reuse.Singleton); + + return container; + } + } +} diff --git a/src/SquidStd.Vfs.S3/README.md b/src/SquidStd.Vfs.S3/README.md new file mode 100644 index 0000000..85ac32c --- /dev/null +++ b/src/SquidStd.Vfs.S3/README.md @@ -0,0 +1,55 @@ +

SquidStd.Vfs.S3

+ +An `IVirtualFileSystem` over S3-compatible object storage — AWS S3, MinIO, Cloudflare R2, Backblaze B2 — via +the MinIO .NET SDK. Each logical path maps directly to an object key inside a single bucket. The bucket is +created lazily on first use. + +## Install + +```bash +dotnet add package SquidStd.Vfs.S3 +``` + +## Usage + +```csharp +using DryIoc; +using SquidStd.Vfs.Abstractions.Interfaces; +using SquidStd.Vfs.S3.Extensions; + +container.RegisterS3FileSystem(o => +{ + o.Bucket = "app-data"; + o.Aws.ServiceUrl = "https://s3.amazonaws.com"; // or MinIO/R2/B2 endpoint + o.Aws.AccessKey = "..."; + o.Aws.SecretKey = "..."; +}); + +var fs = container.Resolve(); + +await fs.WriteAllBytesAsync("reports/2026.json", payload); +byte[]? data = await fs.ReadAllBytesAsync("reports/2026.json"); + +await foreach (var entry in fs.ListAsync("reports")) + Console.WriteLine($"{entry.Path} ({entry.Size} bytes)"); +``` + +For native AWS with the default credential chain, omit `AccessKey`/`SecretKey` and set only the region via +`o.Aws.Region`. + +## Key types + +| Type | Purpose | +|---|---| +| `RegisterS3FileSystemExtensions` | `RegisterS3FileSystem(...)` DryIoc registration. | +| `S3FileSystem` | MinIO-backed `IVirtualFileSystem` with lazy bucket creation. | +| `S3FileSystemOptions` | `Bucket` + `Aws` (`ServiceUrl`, `AccessKey`, `SecretKey`, `Region`). | + +## Related + +- Tutorial: [Virtual filesystem](https://tgiachi.github.io/squid-std/tutorials/vfs.html) +- [`SquidStd.Vfs`](../SquidStd.Vfs/README.md) — core backends and decorators + +## License + +MIT — part of [SquidStd](https://github.com/tgiachi/squid-std). diff --git a/src/SquidStd.Vfs.S3/Services/S3FileSystem.cs b/src/SquidStd.Vfs.S3/Services/S3FileSystem.cs new file mode 100644 index 0000000..b54f73a --- /dev/null +++ b/src/SquidStd.Vfs.S3/Services/S3FileSystem.cs @@ -0,0 +1,253 @@ +using System.Runtime.CompilerServices; +using Minio; +using Minio.DataModel.Args; +using Minio.Exceptions; +using SquidStd.Vfs.Abstractions; +using SquidStd.Vfs.Abstractions.Data; +using SquidStd.Vfs.Abstractions.Interfaces; +using SquidStd.Vfs.S3.Data; + +namespace SquidStd.Vfs.S3.Services; + +/// +/// S3-compatible backed by the MinIO client. +/// The bucket is created lazily on first use and the raw object bytes map directly to file contents. +/// +public sealed class S3FileSystem : IVirtualFileSystem, IDisposable +{ + private readonly string _bucket; + private readonly SemaphoreSlim _bucketLock = new(1, 1); + private readonly IMinioClient _client; + private bool _bucketReady; + private int _disposed; + + /// Initializes a new from . + /// Thrown when any required connection field is empty. + public S3FileSystem(S3FileSystemOptions options) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentException.ThrowIfNullOrWhiteSpace(options.Aws.ServiceUrl); + ArgumentException.ThrowIfNullOrWhiteSpace(options.Aws.AccessKey); + ArgumentException.ThrowIfNullOrWhiteSpace(options.Aws.SecretKey); + ArgumentException.ThrowIfNullOrWhiteSpace(options.Bucket); + + _client = CreateClient(options); + _bucket = options.Bucket; + } + + /// + public async ValueTask ExistsAsync(string path, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + + await EnsureBucketAsync(cancellationToken); + + try + { + await _client.StatObjectAsync( + new StatObjectArgs().WithBucket(_bucket).WithObject(path), + cancellationToken + ); + + return true; + } + catch (ObjectNotFoundException) + { + return false; + } + } + + /// + public async ValueTask ReadAllBytesAsync(string path, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + + await EnsureBucketAsync(cancellationToken); + + using var buffer = new MemoryStream(); + + try + { + await _client.GetObjectAsync( + new GetObjectArgs() + .WithBucket(_bucket) + .WithObject(path) + .WithCallbackStream(async (stream, ct) => await stream.CopyToAsync(buffer, ct)), + cancellationToken + ); + } + catch (ObjectNotFoundException) + { + return null; + } + + return buffer.ToArray(); + } + + /// + public async ValueTask WriteAllBytesAsync(string path, ReadOnlyMemory data, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + + await EnsureBucketAsync(cancellationToken); + + var bytes = data.ToArray(); + using var stream = new MemoryStream(bytes, false); + + await _client.PutObjectAsync( + new PutObjectArgs() + .WithBucket(_bucket) + .WithObject(path) + .WithStreamData(stream) + .WithObjectSize(bytes.LongLength), + cancellationToken + ); + } + + /// + public async Task OpenReadAsync(string path, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + + await EnsureBucketAsync(cancellationToken); + + var buffer = new MemoryStream(); + + try + { + await _client.GetObjectAsync( + new GetObjectArgs() + .WithBucket(_bucket) + .WithObject(path) + .WithCallbackStream(async (stream, ct) => await stream.CopyToAsync(buffer, ct)), + cancellationToken + ); + } + catch (ObjectNotFoundException) + { + await buffer.DisposeAsync(); + throw new FileNotFoundException($"Object not found in S3 bucket '{_bucket}': '{path}'.", path); + } + + buffer.Position = 0; + return buffer; + } + + /// + public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + + return Task.FromResult( + new DeferredWriteStream((bytes, ct) => WriteAllBytesAsync(path, bytes, ct), cancellationToken) + ); + } + + /// + public async ValueTask DeleteAsync(string path, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + + if (!await ExistsAsync(path, cancellationToken)) + { + return false; + } + + await _client.RemoveObjectAsync( + new RemoveObjectArgs().WithBucket(_bucket).WithObject(path), + cancellationToken + ); + + return true; + } + + /// + public async IAsyncEnumerable ListAsync( + string? prefix = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default + ) + { + await EnsureBucketAsync(cancellationToken); + + var args = new ListObjectsArgs().WithBucket(_bucket).WithRecursive(true); + + if (!string.IsNullOrEmpty(prefix)) + { + args = args.WithPrefix(prefix); + } + + await foreach (var item in _client.ListObjectsEnumAsync(args, cancellationToken)) + { + if (!item.IsDir) + { + yield return new VfsEntry( + item.Key, + (long)item.Size, + item.LastModifiedDateTime?.ToUniversalTime() ?? default + ); + } + } + } + + private static IMinioClient CreateClient(S3FileSystemOptions options) + { + var uri = new Uri(options.Aws.ServiceUrl!, UriKind.Absolute); + var endpoint = uri.IsDefaultPort ? uri.Host : $"{uri.Host}:{uri.Port}"; + + var minio = new MinioClient() + .WithEndpoint(endpoint) + .WithCredentials(options.Aws.AccessKey, options.Aws.SecretKey) + .WithSSL(string.Equals(uri.Scheme, "https", StringComparison.OrdinalIgnoreCase)); + + if (!string.IsNullOrWhiteSpace(options.Aws.Region)) + { + minio = minio.WithRegion(options.Aws.Region); + } + + return minio.Build(); + } + + private async ValueTask EnsureBucketAsync(CancellationToken cancellationToken) + { + await _bucketLock.WaitAsync(cancellationToken); + + try + { + if (_bucketReady) + { + return; + } + + var exists = await _client.BucketExistsAsync( + new BucketExistsArgs().WithBucket(_bucket), + cancellationToken + ); + + if (!exists) + { + await _client.MakeBucketAsync( + new MakeBucketArgs().WithBucket(_bucket), + cancellationToken + ); + } + + _bucketReady = true; + } + finally + { + _bucketLock.Release(); + } + } + + /// + public void Dispose() + { + if (Interlocked.Exchange(ref _disposed, 1) != 0) + { + return; + } + + _client.Dispose(); + _bucketLock.Dispose(); + } +} diff --git a/src/SquidStd.Vfs.S3/SquidStd.Vfs.S3.csproj b/src/SquidStd.Vfs.S3/SquidStd.Vfs.S3.csproj new file mode 100644 index 0000000..3074c76 --- /dev/null +++ b/src/SquidStd.Vfs.S3/SquidStd.Vfs.S3.csproj @@ -0,0 +1,20 @@ + + + + net10.0 + enable + enable + true + + + + + + + + + + + + + diff --git a/src/SquidStd.Vfs/README.md b/src/SquidStd.Vfs/README.md index 82c5e88..02089b5 100644 --- a/src/SquidStd.Vfs/README.md +++ b/src/SquidStd.Vfs/README.md @@ -60,6 +60,32 @@ Logical paths are normalized (forward slashes, root-relative) and reject `..` tr | `VfsDirectories` | Named directory layout (`DirectoriesConfig` analogue) over any backend. | | `RegisterVfsExtensions` | `RegisterVfs(...)` registration. | +## Decorators + +Decorators wrap any `IVirtualFileSystem` to add behaviour without touching the backend. Stack them in any order. + +| Decorator | Description | +|---|---| +| `ReadOnlyFileSystem(inner)` | Delegates all reads to `inner`; rejects every mutation with `InvalidOperationException`. | +| `ScopedFileSystem(inner, prefix)` | Roots `inner` at a path prefix (chroot-like). All paths are resolved relative to the scope; list results are returned relative to it too. | +| `OverlayFileSystem(base, overlay)` | Reads overlay-first then falls back to base. Writes and deletes go to the overlay only. List returns the union of both; overlay entries shadow base entries with the same path. | +| `CachingFileSystem(remote, cache)` | Read-through cache: reads prefer the remote and refresh the cache copy on success; on a transport failure they fall back to the (possibly stale) cache. Writes are write-through (remote then cache) and fail when the remote is unreachable. | + +Composition example — S3 with a local disk cache for resilience to an unstable connection: + +```csharp +// S3 with a local disk cache for resilience to an unstable connection. +var fs = new CachingFileSystem( + remote: s3FileSystem, + cache: new PhysicalFileSystem("/var/cache/app")); +``` + +Decorators are not registered via DI helpers; construct them explicitly and pass the result to `RegisterVfs(...)`: + +```csharp +container.RegisterVfs(_ => new ReadOnlyFileSystem(new PhysicalFileSystem("/var/lib/app/data"))); +``` + ## Related - Tutorial: [Virtual filesystem](https://tgiachi.github.io/squid-std/tutorials/vfs.html) diff --git a/src/SquidStd.Vfs/Services/CachingFileSystem.cs b/src/SquidStd.Vfs/Services/CachingFileSystem.cs new file mode 100644 index 0000000..91f5821 --- /dev/null +++ b/src/SquidStd.Vfs/Services/CachingFileSystem.cs @@ -0,0 +1,142 @@ +using System.Net.Http; +using System.Runtime.CompilerServices; +using SquidStd.Vfs.Abstractions; +using SquidStd.Vfs.Abstractions.Data; +using SquidStd.Vfs.Abstractions.Interfaces; + +namespace SquidStd.Vfs.Services; + +/// +/// Read-through cache over a remote filesystem. Reads prefer the remote and refresh the cache; on a +/// transport failure they fall back to the (possibly stale) cache. Writes and deletes are write-through +/// (remote then cache) and surface the remote's error when it is unreachable. +/// +public sealed class CachingFileSystem : IVirtualFileSystem +{ + private readonly IVirtualFileSystem _remote; + private readonly IVirtualFileSystem _cache; + + public CachingFileSystem(IVirtualFileSystem remote, IVirtualFileSystem cache) + { + ArgumentNullException.ThrowIfNull(remote); + ArgumentNullException.ThrowIfNull(cache); + + _remote = remote; + _cache = cache; + } + + public async ValueTask ExistsAsync(string path, CancellationToken cancellationToken = default) + { + try + { + return await _remote.ExistsAsync(path, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (IsTransient(ex)) + { + return await _cache.ExistsAsync(path, cancellationToken).ConfigureAwait(false); + } + } + + public async ValueTask ReadAllBytesAsync(string path, CancellationToken cancellationToken = default) + { + try + { + var data = await _remote.ReadAllBytesAsync(path, cancellationToken).ConfigureAwait(false); + + if (data is not null) + { + await _cache.WriteAllBytesAsync(path, data, cancellationToken).ConfigureAwait(false); + } + + return data; + } + catch (Exception ex) when (IsTransient(ex)) + { + return await _cache.ReadAllBytesAsync(path, cancellationToken).ConfigureAwait(false); + } + } + + public async ValueTask WriteAllBytesAsync(string path, ReadOnlyMemory data, CancellationToken cancellationToken = default) + { + await _remote.WriteAllBytesAsync(path, data, cancellationToken).ConfigureAwait(false); + await _cache.WriteAllBytesAsync(path, data, cancellationToken).ConfigureAwait(false); + } + + public async Task OpenReadAsync(string path, CancellationToken cancellationToken = default) + { + var data = await ReadAllBytesAsync(path, cancellationToken).ConfigureAwait(false) + ?? throw new FileNotFoundException($"No file at '{path}'.", path); + + return new MemoryStream(data, writable: false); + } + + public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) + => Task.FromResult(new DeferredWriteStream((bytes, ct) => WriteAllBytesAsync(path, bytes, ct), cancellationToken)); + + public async ValueTask DeleteAsync(string path, CancellationToken cancellationToken = default) + { + var removed = await _remote.DeleteAsync(path, cancellationToken).ConfigureAwait(false); + await _cache.DeleteAsync(path, cancellationToken).ConfigureAwait(false); + + return removed; + } + + public async IAsyncEnumerable ListAsync(string? prefix = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + IAsyncEnumerator? enumerator = null; + var fellBack = false; + + try + { + try + { + enumerator = _remote.ListAsync(prefix, cancellationToken).GetAsyncEnumerator(cancellationToken); + } + catch (Exception ex) when (IsTransient(ex)) + { + fellBack = true; + } + + while (!fellBack) + { + VfsEntry current; + + try + { + if (!await enumerator!.MoveNextAsync().ConfigureAwait(false)) + { + break; + } + + current = enumerator.Current; + } + catch (Exception ex) when (IsTransient(ex)) + { + fellBack = true; + + break; + } + + yield return current; + } + } + finally + { + if (enumerator is not null) + { + await enumerator.DisposeAsync().ConfigureAwait(false); + } + } + + if (fellBack) + { + await foreach (var entry in _cache.ListAsync(prefix, cancellationToken).ConfigureAwait(false)) + { + yield return entry; + } + } + } + + private static bool IsTransient(Exception ex) + => ex is HttpRequestException or IOException or TimeoutException; +} diff --git a/src/SquidStd.Vfs/Services/OverlayFileSystem.cs b/src/SquidStd.Vfs/Services/OverlayFileSystem.cs new file mode 100644 index 0000000..8b319c7 --- /dev/null +++ b/src/SquidStd.Vfs/Services/OverlayFileSystem.cs @@ -0,0 +1,63 @@ +using System.Runtime.CompilerServices; +using SquidStd.Vfs.Abstractions.Data; +using SquidStd.Vfs.Abstractions.Interfaces; + +namespace SquidStd.Vfs.Services; + +/// Reads from an overlay first then a base; writes and deletes affect only the overlay. +public sealed class OverlayFileSystem : IVirtualFileSystem +{ + private readonly IVirtualFileSystem _base; + private readonly IVirtualFileSystem _overlay; + + public OverlayFileSystem(IVirtualFileSystem baseFileSystem, IVirtualFileSystem overlay) + { + ArgumentNullException.ThrowIfNull(baseFileSystem); + ArgumentNullException.ThrowIfNull(overlay); + + _base = baseFileSystem; + _overlay = overlay; + } + + public async ValueTask ExistsAsync(string path, CancellationToken cancellationToken = default) + => await _overlay.ExistsAsync(path, cancellationToken).ConfigureAwait(false) + || await _base.ExistsAsync(path, cancellationToken).ConfigureAwait(false); + + public async ValueTask ReadAllBytesAsync(string path, CancellationToken cancellationToken = default) + => await _overlay.ReadAllBytesAsync(path, cancellationToken).ConfigureAwait(false) + ?? await _base.ReadAllBytesAsync(path, cancellationToken).ConfigureAwait(false); + + public ValueTask WriteAllBytesAsync(string path, ReadOnlyMemory data, CancellationToken cancellationToken = default) + => _overlay.WriteAllBytesAsync(path, data, cancellationToken); + + public async Task OpenReadAsync(string path, CancellationToken cancellationToken = default) + => await _overlay.ExistsAsync(path, cancellationToken).ConfigureAwait(false) + ? await _overlay.OpenReadAsync(path, cancellationToken).ConfigureAwait(false) + : await _base.OpenReadAsync(path, cancellationToken).ConfigureAwait(false); + + public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) + => _overlay.OpenWriteAsync(path, cancellationToken); + + public ValueTask DeleteAsync(string path, CancellationToken cancellationToken = default) + => _overlay.DeleteAsync(path, cancellationToken); + + public async IAsyncEnumerable ListAsync(string? prefix = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var seen = new HashSet(StringComparer.Ordinal); + + await foreach (var entry in _overlay.ListAsync(prefix, cancellationToken).ConfigureAwait(false)) + { + seen.Add(entry.Path); + + yield return entry; + } + + await foreach (var entry in _base.ListAsync(prefix, cancellationToken).ConfigureAwait(false)) + { + if (seen.Add(entry.Path)) + { + yield return entry; + } + } + } +} diff --git a/src/SquidStd.Vfs/Services/ReadOnlyFileSystem.cs b/src/SquidStd.Vfs/Services/ReadOnlyFileSystem.cs new file mode 100644 index 0000000..9dc6e1a --- /dev/null +++ b/src/SquidStd.Vfs/Services/ReadOnlyFileSystem.cs @@ -0,0 +1,40 @@ +using SquidStd.Vfs.Abstractions.Data; +using SquidStd.Vfs.Abstractions.Interfaces; + +namespace SquidStd.Vfs.Services; + +/// Wraps a filesystem and rejects every mutation. +public sealed class ReadOnlyFileSystem : IVirtualFileSystem +{ + private const string Message = "This filesystem is read-only."; + + private readonly IVirtualFileSystem _inner; + + public ReadOnlyFileSystem(IVirtualFileSystem inner) + { + ArgumentNullException.ThrowIfNull(inner); + + _inner = inner; + } + + public ValueTask ExistsAsync(string path, CancellationToken cancellationToken = default) + => _inner.ExistsAsync(path, cancellationToken); + + public ValueTask ReadAllBytesAsync(string path, CancellationToken cancellationToken = default) + => _inner.ReadAllBytesAsync(path, cancellationToken); + + public ValueTask WriteAllBytesAsync(string path, ReadOnlyMemory data, CancellationToken cancellationToken = default) + => throw new InvalidOperationException(Message); + + public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) + => _inner.OpenReadAsync(path, cancellationToken); + + public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) + => throw new InvalidOperationException(Message); + + public ValueTask DeleteAsync(string path, CancellationToken cancellationToken = default) + => throw new InvalidOperationException(Message); + + public IAsyncEnumerable ListAsync(string? prefix = null, CancellationToken cancellationToken = default) + => _inner.ListAsync(prefix, cancellationToken); +} diff --git a/src/SquidStd.Vfs/Services/ScopedFileSystem.cs b/src/SquidStd.Vfs/Services/ScopedFileSystem.cs new file mode 100644 index 0000000..03fbd0f --- /dev/null +++ b/src/SquidStd.Vfs/Services/ScopedFileSystem.cs @@ -0,0 +1,62 @@ +using System.Runtime.CompilerServices; +using SquidStd.Vfs.Abstractions; +using SquidStd.Vfs.Abstractions.Data; +using SquidStd.Vfs.Abstractions.Interfaces; + +namespace SquidStd.Vfs.Services; + +/// Roots an inner filesystem at a fixed path prefix (a chroot-like view). +public sealed class ScopedFileSystem : IVirtualFileSystem +{ + private readonly IVirtualFileSystem _inner; + private readonly string _prefix; + + public ScopedFileSystem(IVirtualFileSystem inner, string prefix) + { + ArgumentNullException.ThrowIfNull(inner); + ArgumentException.ThrowIfNullOrWhiteSpace(prefix); + + _inner = inner; + _prefix = VfsPath.Normalize(prefix); + } + + public ValueTask ExistsAsync(string path, CancellationToken cancellationToken = default) + => _inner.ExistsAsync(Scope(path), cancellationToken); + + public ValueTask ReadAllBytesAsync(string path, CancellationToken cancellationToken = default) + => _inner.ReadAllBytesAsync(Scope(path), cancellationToken); + + public ValueTask WriteAllBytesAsync(string path, ReadOnlyMemory data, CancellationToken cancellationToken = default) + => _inner.WriteAllBytesAsync(Scope(path), data, cancellationToken); + + public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) + => _inner.OpenReadAsync(Scope(path), cancellationToken); + + public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) + => _inner.OpenWriteAsync(Scope(path), cancellationToken); + + public ValueTask DeleteAsync(string path, CancellationToken cancellationToken = default) + => _inner.DeleteAsync(Scope(path), cancellationToken); + + public async IAsyncEnumerable ListAsync(string? prefix = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var scopedPrefix = string.IsNullOrEmpty(prefix) ? _prefix : Scope(prefix); + var boundary = _prefix + "/"; + + await foreach (var entry in _inner.ListAsync(scopedPrefix, cancellationToken).ConfigureAwait(false)) + { + if (!entry.Path.StartsWith(boundary, StringComparison.Ordinal)) + { + continue; // a sibling scope sharing a name prefix (e.g. "tenant10" vs "tenant1") — not ours + } + + yield return entry with { Path = Unscope(entry.Path) }; + } + } + + private string Scope(string path) + => VfsPath.Normalize(_prefix + "/" + path); + + private string Unscope(string path) + => path.StartsWith(_prefix + "/", StringComparison.Ordinal) ? path[(_prefix.Length + 1)..] : path; +} diff --git a/tests/SquidStd.Tests/SquidStd.Tests.csproj b/tests/SquidStd.Tests/SquidStd.Tests.csproj index 69fab08..4c955dd 100644 --- a/tests/SquidStd.Tests/SquidStd.Tests.csproj +++ b/tests/SquidStd.Tests/SquidStd.Tests.csproj @@ -73,6 +73,8 @@ + + diff --git a/tests/SquidStd.Tests/Vfs/CachingFileSystemTests.cs b/tests/SquidStd.Tests/Vfs/CachingFileSystemTests.cs new file mode 100644 index 0000000..71b6ad2 --- /dev/null +++ b/tests/SquidStd.Tests/Vfs/CachingFileSystemTests.cs @@ -0,0 +1,97 @@ +using System.Net.Http; +using SquidStd.Vfs.Abstractions.Data; +using SquidStd.Vfs.Abstractions.Interfaces; +using SquidStd.Vfs.Services; + +namespace SquidStd.Tests.Vfs; + +public class CachingFileSystemTests +{ + // An InMemoryFileSystem-backed remote with a switch that makes every op throw a transport error. + private sealed class FlakyRemote : IVirtualFileSystem + { + private readonly InMemoryFileSystem _inner = new(); + + public bool Offline { get; set; } + + private void Guard() + { + if (Offline) + { + throw new HttpRequestException("offline"); + } + } + + public ValueTask ExistsAsync(string path, CancellationToken ct = default) { Guard(); return _inner.ExistsAsync(path, ct); } + public ValueTask ReadAllBytesAsync(string path, CancellationToken ct = default) { Guard(); return _inner.ReadAllBytesAsync(path, ct); } + public ValueTask WriteAllBytesAsync(string path, ReadOnlyMemory data, CancellationToken ct = default) { Guard(); return _inner.WriteAllBytesAsync(path, data, ct); } + public Task OpenReadAsync(string path, CancellationToken ct = default) { Guard(); return _inner.OpenReadAsync(path, ct); } + public Task OpenWriteAsync(string path, CancellationToken ct = default) { Guard(); return _inner.OpenWriteAsync(path, ct); } + public ValueTask DeleteAsync(string path, CancellationToken ct = default) { Guard(); return _inner.DeleteAsync(path, ct); } + public IAsyncEnumerable ListAsync(string? prefix = null, CancellationToken ct = default) { Guard(); return _inner.ListAsync(prefix, ct); } + } + + [Fact] + public async Task Read_PopulatesCache_AndServesFromCacheWhenOffline() + { + var remote = new FlakyRemote(); + var cache = new InMemoryFileSystem(); + var fs = new CachingFileSystem(remote, cache); + + await remote.WriteAllBytesAsync("a.txt", new byte[] { 1 }); + + Assert.Equal(new byte[] { 1 }, await fs.ReadAllBytesAsync("a.txt")); // remote read populates cache + Assert.True(await cache.ExistsAsync("a.txt")); + + remote.Offline = true; + Assert.Equal(new byte[] { 1 }, await fs.ReadAllBytesAsync("a.txt")); // served from cache while offline + } + + [Fact] + public async Task Write_IsWriteThrough_AndFailsOffline() + { + var remote = new FlakyRemote(); + var cache = new InMemoryFileSystem(); + var fs = new CachingFileSystem(remote, cache); + + await fs.WriteAllBytesAsync("a.txt", new byte[] { 5 }); + Assert.True(await remote.ExistsAsync("a.txt")); + Assert.True(await cache.ExistsAsync("a.txt")); + + remote.Offline = true; + await Assert.ThrowsAsync(async () => await fs.WriteAllBytesAsync("b.txt", new byte[] { 6 })); + } + + [Fact] + public async Task List_FallsBackToCache_WhenOffline() + { + var remote = new FlakyRemote(); + var cache = new InMemoryFileSystem(); + var fs = new CachingFileSystem(remote, cache); + + await remote.WriteAllBytesAsync("a.txt", new byte[] { 1 }); + await fs.ReadAllBytesAsync("a.txt"); // populate the cache while online + + remote.Offline = true; // remote.ListAsync now throws eagerly (HttpRequestException) + + var paths = new List(); + await foreach (var e in fs.ListAsync()) + { + paths.Add(e.Path); + } + + Assert.Contains("a.txt", paths); // served from cache despite the remote being offline + } + + [Fact] + public async Task Read_MissingFile_DoesNotFallBackToCache() + { + var remote = new FlakyRemote(); + var cache = new InMemoryFileSystem(); + await cache.WriteAllBytesAsync("ghost.txt", new byte[] { 7 }); // stale cache entry + var fs = new CachingFileSystem(remote, cache); + + // Remote is online and has no such file => null, NOT the stale cache copy. + Assert.Null(await fs.ReadAllBytesAsync("ghost.txt")); + } +} diff --git a/tests/SquidStd.Tests/Vfs/Database/DatabaseFileSystemTests.cs b/tests/SquidStd.Tests/Vfs/Database/DatabaseFileSystemTests.cs new file mode 100644 index 0000000..993ca6c --- /dev/null +++ b/tests/SquidStd.Tests/Vfs/Database/DatabaseFileSystemTests.cs @@ -0,0 +1,130 @@ +using System.Text; +using SquidStd.Database.Abstractions.Data.Database; +using SquidStd.Database.Data; +using SquidStd.Database.Services; +using SquidStd.Vfs.Database.Data.Entities; +using SquidStd.Vfs.Database.Services; + +namespace SquidStd.Tests.Vfs.Database; + +public sealed class DatabaseFileSystemTests : IAsyncLifetime +{ + private string _dbPath = string.Empty; + private DatabaseService _service = null!; + private DatabaseFileSystem _fs = null!; + + public async Task InitializeAsync() + { + _dbPath = Path.Combine(Path.GetTempPath(), "squidstd-vfsdb-" + Guid.NewGuid().ToString("N") + ".db"); + _service = new( + new DatabaseConfig + { + ConnectionString = $"sqlite://{_dbPath}", + AutoMigrate = true + } + ); + await _service.StartAsync(); + _fs = new DatabaseFileSystem(NewAccess()); + } + + public async Task DisposeAsync() + { + await _service.StopAsync(); + + if (File.Exists(_dbPath)) + { + File.Delete(_dbPath); + } + } + + [Fact] + public async Task WriteRead_RoundTrip() + { + var data = Encoding.UTF8.GetBytes("hello database"); + await _fs.WriteAllBytesAsync("docs/hello.txt", data); + + var result = await _fs.ReadAllBytesAsync("docs/hello.txt"); + + Assert.NotNull(result); + Assert.Equal("hello database", Encoding.UTF8.GetString(result)); + } + + [Fact] + public async Task WriteAsync_SamePath_Twice_ExactlyOneRow_LatestContent() + { + var access = NewAccess(); + var fs = new DatabaseFileSystem(access); + + await fs.WriteAllBytesAsync("file.txt", Encoding.UTF8.GetBytes("first")); + await fs.WriteAllBytesAsync("file.txt", Encoding.UTF8.GetBytes("second")); + + // Verify only one row exists and it has the latest content. + var rows = await access.QueryAsync(e => e.Path == "file.txt"); + Assert.Single(rows); + Assert.Equal("second", Encoding.UTF8.GetString(rows[0].Content)); + } + + [Fact] + public async Task ExistsAsync_TrueForWritten_FalseForMissing() + { + await _fs.WriteAllBytesAsync("present.bin", new byte[] { 0x01, 0x02 }); + + Assert.True(await _fs.ExistsAsync("present.bin")); + Assert.False(await _fs.ExistsAsync("absent.bin")); + } + + [Fact] + public async Task DeleteAsync_ExistingFile_ReturnsTrue_SubsequentReturnsFalse() + { + await _fs.WriteAllBytesAsync("todelete.txt", Encoding.UTF8.GetBytes("bye")); + + Assert.True(await _fs.DeleteAsync("todelete.txt")); + Assert.False(await _fs.DeleteAsync("todelete.txt")); + Assert.Null(await _fs.ReadAllBytesAsync("todelete.txt")); + } + + [Fact] + public async Task DeleteAsync_MissingFile_ReturnsFalse() + { + Assert.False(await _fs.DeleteAsync("nope.txt")); + } + + [Fact] + public async Task ListAsync_WithPrefix_ReturnsMatchingPaths() + { + await _fs.WriteAllBytesAsync("img/a.png", new byte[] { 1 }); + await _fs.WriteAllBytesAsync("img/b.png", new byte[] { 2 }); + await _fs.WriteAllBytesAsync("doc/c.txt", new byte[] { 3 }); + + var paths = new List(); + + await foreach (var entry in _fs.ListAsync("img")) + { + paths.Add(entry.Path); + } + + Assert.Contains("img/a.png", paths); + Assert.Contains("img/b.png", paths); + Assert.DoesNotContain("doc/c.txt", paths); + } + + [Fact] + public async Task ListAsync_NoPrefix_ReturnsAllPaths() + { + await _fs.WriteAllBytesAsync("x/1.dat", new byte[] { 10 }); + await _fs.WriteAllBytesAsync("y/2.dat", new byte[] { 20 }); + + var paths = new List(); + + await foreach (var entry in _fs.ListAsync()) + { + paths.Add(entry.Path); + } + + Assert.Contains("x/1.dat", paths); + Assert.Contains("y/2.dat", paths); + } + + private FreeSqlDataAccess NewAccess() + => new(_service); +} diff --git a/tests/SquidStd.Tests/Vfs/DeferredWriteStreamTests.cs b/tests/SquidStd.Tests/Vfs/DeferredWriteStreamTests.cs new file mode 100644 index 0000000..11235a8 --- /dev/null +++ b/tests/SquidStd.Tests/Vfs/DeferredWriteStreamTests.cs @@ -0,0 +1,45 @@ +using SquidStd.Vfs.Abstractions; + +namespace SquidStd.Tests.Vfs; + +public class DeferredWriteStreamTests +{ + [Fact] + public async Task DisposeAsync_FlushesWrittenBytesOnce() + { + byte[]? flushed = null; + var flushes = 0; + + var stream = new DeferredWriteStream((bytes, _) => + { + flushed = bytes; + flushes++; + + return ValueTask.CompletedTask; + }); + + await stream.WriteAsync(new byte[] { 1, 2, 3 }); + await stream.DisposeAsync(); + + Assert.Equal(new byte[] { 1, 2, 3 }, flushed); + Assert.Equal(1, flushes); + } + + [Fact] + public void Dispose_Synchronous_AlsoFlushes() + { + byte[]? flushed = null; + + using (var stream = new DeferredWriteStream((bytes, _) => + { + flushed = bytes; + + return ValueTask.CompletedTask; + })) + { + stream.Write(new byte[] { 9 }, 0, 1); + } + + Assert.Equal(new byte[] { 9 }, flushed); + } +} diff --git a/tests/SquidStd.Tests/Vfs/OverlayFileSystemTests.cs b/tests/SquidStd.Tests/Vfs/OverlayFileSystemTests.cs new file mode 100644 index 0000000..b3bd38f --- /dev/null +++ b/tests/SquidStd.Tests/Vfs/OverlayFileSystemTests.cs @@ -0,0 +1,66 @@ +using SquidStd.Vfs.Services; + +namespace SquidStd.Tests.Vfs; + +public class OverlayFileSystemTests +{ + [Fact] + public async Task Read_PrefersOverlay_ThenBase() + { + var @base = new InMemoryFileSystem(); + var overlay = new InMemoryFileSystem(); + await @base.WriteAllBytesAsync("a.txt", new byte[] { 1 }); + await @base.WriteAllBytesAsync("b.txt", new byte[] { 2 }); + await overlay.WriteAllBytesAsync("a.txt", new byte[] { 99 }); + var fs = new OverlayFileSystem(@base, overlay); + + Assert.Equal(new byte[] { 99 }, await fs.ReadAllBytesAsync("a.txt")); // overlay shadows base + Assert.Equal(new byte[] { 2 }, await fs.ReadAllBytesAsync("b.txt")); // falls through to base + } + + [Fact] + public async Task Write_GoesToOverlay_BaseUntouched() + { + var @base = new InMemoryFileSystem(); + var overlay = new InMemoryFileSystem(); + var fs = new OverlayFileSystem(@base, overlay); + + await fs.WriteAllBytesAsync("c.txt", new byte[] { 7 }); + + Assert.True(await overlay.ExistsAsync("c.txt")); + Assert.False(await @base.ExistsAsync("c.txt")); + } + + [Fact] + public async Task Delete_BaseOnlyFile_ReturnsFalse() + { + var @base = new InMemoryFileSystem(); + var overlay = new InMemoryFileSystem(); + await @base.WriteAllBytesAsync("a.txt", new byte[] { 1 }); + var fs = new OverlayFileSystem(@base, overlay); + + Assert.False(await fs.DeleteAsync("a.txt")); // overlay has nothing to delete; base untouched + Assert.True(await @base.ExistsAsync("a.txt")); + } + + [Fact] + public async Task List_IsUnion_OverlayShadowsBase() + { + var @base = new InMemoryFileSystem(); + var overlay = new InMemoryFileSystem(); + await @base.WriteAllBytesAsync("a.txt", new byte[] { 1 }); + await @base.WriteAllBytesAsync("b.txt", new byte[] { 2 }); + await overlay.WriteAllBytesAsync("a.txt", new byte[] { 99 }); + var fs = new OverlayFileSystem(@base, overlay); + + var paths = new List(); + await foreach (var e in fs.ListAsync()) + { + paths.Add(e.Path); + } + + Assert.Equal(2, paths.Count); // a.txt once + b.txt + Assert.Contains("a.txt", paths); + Assert.Contains("b.txt", paths); + } +} diff --git a/tests/SquidStd.Tests/Vfs/ReadOnlyFileSystemTests.cs b/tests/SquidStd.Tests/Vfs/ReadOnlyFileSystemTests.cs new file mode 100644 index 0000000..5e3ce56 --- /dev/null +++ b/tests/SquidStd.Tests/Vfs/ReadOnlyFileSystemTests.cs @@ -0,0 +1,34 @@ +using SquidStd.Vfs.Abstractions.Interfaces; +using SquidStd.Vfs.Services; + +namespace SquidStd.Tests.Vfs; + +public class ReadOnlyFileSystemTests +{ + private static async Task SeededInnerAsync() + { + var inner = new InMemoryFileSystem(); + await inner.WriteAllBytesAsync("a.txt", new byte[] { 1 }); + + return inner; + } + + [Fact] + public async Task Reads_Delegate() + { + var fs = new ReadOnlyFileSystem(await SeededInnerAsync()); + + Assert.True(await fs.ExistsAsync("a.txt")); + Assert.Equal(new byte[] { 1 }, await fs.ReadAllBytesAsync("a.txt")); + } + + [Fact] + public async Task Writes_Throw() + { + var fs = new ReadOnlyFileSystem(await SeededInnerAsync()); + + await Assert.ThrowsAsync(async () => await fs.WriteAllBytesAsync("b.txt", new byte[] { 2 })); + await Assert.ThrowsAsync(async () => await fs.DeleteAsync("a.txt")); + await Assert.ThrowsAsync(async () => await fs.OpenWriteAsync("b.txt")); + } +} diff --git a/tests/SquidStd.Tests/Vfs/S3/S3FileSystemTests.cs b/tests/SquidStd.Tests/Vfs/S3/S3FileSystemTests.cs new file mode 100644 index 0000000..6695fef --- /dev/null +++ b/tests/SquidStd.Tests/Vfs/S3/S3FileSystemTests.cs @@ -0,0 +1,141 @@ +using System.Text; +using SquidStd.Tests.Storage.S3; +using SquidStd.Vfs.S3.Data; +using SquidStd.Vfs.S3.Services; + +namespace SquidStd.Tests.Vfs.S3; + +[Collection(MinioCollection.Name)] +public class S3FileSystemTests +{ + private readonly MinioContainerFixture _fixture; + + public S3FileSystemTests(MinioContainerFixture fixture) + { + _fixture = fixture; + } + + [Fact] + public void Ctor_MissingServiceUrl_Throws() + => Assert.ThrowsAny( + () => new S3FileSystem(new() { Aws = new() { AccessKey = "a", SecretKey = "b" }, Bucket = "c" }) + ); + + [Fact] + public async Task WriteAllBytesAsync_Then_ReadAllBytesAsync_RoundTrips() + { + using var fs = NewFileSystem(); + var path = Path(); + var data = Encoding.UTF8.GetBytes("hello s3 vfs"); + + await fs.WriteAllBytesAsync(path, data); + var read = await fs.ReadAllBytesAsync(path); + + Assert.NotNull(read); + Assert.Equal(data, read); + } + + [Fact] + public async Task OpenWriteAsync_Then_OpenReadAsync_RoundTrips() + { + using var fs = NewFileSystem(); + var path = Path(); + var data = Encoding.UTF8.GetBytes("deferred write stream"); + + await using (var write = await fs.OpenWriteAsync(path)) + { + await write.WriteAsync(data); + } + + await using var read = await fs.OpenReadAsync(path); + using var buffer = new MemoryStream(); + await read.CopyToAsync(buffer); + + Assert.Equal(data, buffer.ToArray()); + } + + [Fact] + public async Task ExistsAsync_ReturnsTrueAfterWrite_AndFalseForMissing() + { + using var fs = NewFileSystem(); + var path = Path(); + + Assert.False(await fs.ExistsAsync(path)); + + await fs.WriteAllBytesAsync(path, Encoding.UTF8.GetBytes("x")); + + Assert.True(await fs.ExistsAsync(path)); + } + + [Fact] + public async Task DeleteAsync_ExistingObject_ReturnsTrueAndRemoves() + { + using var fs = NewFileSystem(); + var path = Path(); + + await fs.WriteAllBytesAsync(path, Encoding.UTF8.GetBytes("delete me")); + + Assert.True(await fs.DeleteAsync(path)); + Assert.False(await fs.ExistsAsync(path)); + } + + [Fact] + public async Task DeleteAsync_MissingObject_ReturnsFalse() + { + using var fs = NewFileSystem(); + + Assert.False(await fs.DeleteAsync(Path())); + } + + [Fact] + public async Task ReadAllBytesAsync_MissingObject_ReturnsNull() + { + using var fs = NewFileSystem(); + + Assert.Null(await fs.ReadAllBytesAsync(Path())); + } + + [Fact] + public async Task OpenReadAsync_MissingObject_ThrowsFileNotFoundException() + { + using var fs = NewFileSystem(); + + await Assert.ThrowsAsync(() => fs.OpenReadAsync(Path())); + } + + [Fact] + public async Task ListAsync_Prefix_ReturnsWrittenKeys() + { + using var fs = NewFileSystem(); + var prefix = "list-" + Guid.NewGuid().ToString("N") + "/"; + + await fs.WriteAllBytesAsync(prefix + "a.txt", Encoding.UTF8.GetBytes("1")); + await fs.WriteAllBytesAsync(prefix + "b.txt", Encoding.UTF8.GetBytes("2")); + + var entries = new List(); + + await foreach (var entry in fs.ListAsync(prefix)) + { + entries.Add(entry.Path); + } + + Assert.Equal(2, entries.Count); + Assert.Contains(prefix + "a.txt", entries); + Assert.Contains(prefix + "b.txt", entries); + } + + private static string Path() + => "vfs-" + Guid.NewGuid().ToString("N"); + + private S3FileSystem NewFileSystem() + => new(new S3FileSystemOptions + { + Aws = new() + { + ServiceUrl = _fixture.ServiceUrl, + AccessKey = _fixture.AccessKey, + SecretKey = _fixture.SecretKey + }, + Bucket = "vfs-tests-" + Guid.NewGuid().ToString("N")[..8] + }); +} diff --git a/tests/SquidStd.Tests/Vfs/ScopedFileSystemTests.cs b/tests/SquidStd.Tests/Vfs/ScopedFileSystemTests.cs new file mode 100644 index 0000000..9b2d866 --- /dev/null +++ b/tests/SquidStd.Tests/Vfs/ScopedFileSystemTests.cs @@ -0,0 +1,53 @@ +using SquidStd.Vfs.Services; + +namespace SquidStd.Tests.Vfs; + +public class ScopedFileSystemTests +{ + [Fact] + public async Task Scopes_ReadWrite_UnderPrefix() + { + var inner = new InMemoryFileSystem(); + var fs = new ScopedFileSystem(inner, "tenant1"); + + await fs.WriteAllBytesAsync("docs/a.txt", new byte[] { 1 }); + + Assert.True(await inner.ExistsAsync("tenant1/docs/a.txt")); // stored under the prefix + Assert.Equal(new byte[] { 1 }, await fs.ReadAllBytesAsync("docs/a.txt")); + } + + [Fact] + public async Task List_StripsPrefix_FromReturnedPaths() + { + var inner = new InMemoryFileSystem(); + var fs = new ScopedFileSystem(inner, "tenant1"); + await fs.WriteAllBytesAsync("docs/a.txt", new byte[] { 1 }); + await inner.WriteAllBytesAsync("tenant2/secret.txt", new byte[] { 9 }); // outside the scope + + var paths = new List(); + await foreach (var e in fs.ListAsync()) + { + paths.Add(e.Path); + } + + Assert.Contains("docs/a.txt", paths); + Assert.DoesNotContain(paths, p => p.Contains("tenant2")); + } + + [Fact] + public async Task List_DoesNotLeak_SiblingScopeSharingNamePrefix() + { + var inner = new InMemoryFileSystem(); + var fs = new ScopedFileSystem(inner, "tenant1"); + await fs.WriteAllBytesAsync("a.txt", new byte[] { 1 }); // -> tenant1/a.txt + await inner.WriteAllBytesAsync("tenant10/b.txt", new byte[] { 9 }); // sibling scope, shares the "tenant1" prefix + + var paths = new List(); + await foreach (var e in fs.ListAsync()) + { + paths.Add(e.Path); + } + + Assert.Equal(new[] { "a.txt" }, paths); // only our scope, and stripped + } +}