sftp + vfs

Author: KSemenenko

.github/copilot-instructions.md

@@ -12,7 +12,7 @@ ManagedCode.Storage is a universal storage abstraction library that provides a c
   - `ManagedCode.Storage.Aws`: AWS S3 implementation  
   - `ManagedCode.Storage.Google`: Google Cloud Storage implementation
   - `ManagedCode.Storage.FileSystem`: Local file system implementation
-  - `ManagedCode.Storage.Ftp`: FTP storage implementation
+  - `ManagedCode.Storage.Sftp`: FTP storage implementation
   - `ManagedCode.Storage.Azure.DataLake`: Azure Data Lake implementation
 - **Tests/**: Unit and integration tests
 - **Integrations/**: Additional integrations (SignalR, Client/Server components)

.gitignore

@@ -647,6 +647,6 @@ MigrationBackup/
 .ionide/
 
 # Tests results
-.trx
+*.trx
 
 # End of https://www.toptal.com/developers/gitignore/api/intellij,intellij+all,macos,linux,windows,visualstudio,visualstudiocode,rider

AGENTS.md

@@ -14,7 +14,7 @@ TBA
 # Repository Guidelines
 
 ## Project Structure & Module Organization
-ManagedCode.Storage.slnx orchestrates the .NET 9 projects. Core abstractions live in `ManagedCode.Storage.Core/`. Providers sit under `Storages/ManagedCode.Storage.*` with one project per cloud target (Azure, AWS, GCP, FileSystem, Ftp). Integration surfaces, including the ASP.NET server and client SDKs, live in `Integraions/`. Test doubles stay in `ManagedCode.Storage.TestFakes/`, while the suites in `Tests/ManagedCode.Storage.Tests/` are grouped into ASP.NET flows, provider runs, and shared helpers. Keep shared assets such as `logo.png` at the repository root.
+ManagedCode.Storage.slnx orchestrates the .NET 9 projects. Core abstractions live in `ManagedCode.Storage.Core/`. Providers sit under `Storages/ManagedCode.Storage.*` with one project per cloud target (Azure, AWS, GCP, FileSystem, Sftp). Integration surfaces, including the ASP.NET server and client SDKs, live in `Integraions/`. Test doubles stay in `ManagedCode.Storage.TestFakes/`, while the suites in `Tests/ManagedCode.Storage.Tests/` are grouped into ASP.NET flows, provider runs, and shared helpers. Keep shared assets such as `logo.png` at the repository root.
 
 ## Build, Test, and Development Commands
 Run `dotnet restore ManagedCode.Storage.slnx` before compiling. Use `dotnet build ManagedCode.Storage.slnx` to compile every target and surface analyzer warnings. Execute all tests with `dotnet test Tests/ManagedCode.Storage.Tests/ManagedCode.Storage.Tests.csproj --configuration Release`. For coverage, run `dotnet test /p:CollectCoverage=true /p:CoverletOutput=coverage /p:CoverletOutputFormat=opencover`. Use `dotnet format ManagedCode.Storage.slnx` before opening a pull request.

DEVELOPMENT_PLAN.md

@@ -7,18 +7,17 @@
 
 ## 🏗️ **Фаза 1: Нові провайдери сховищ**
 
-### **1.1 FTP Provider** 
+### **1.1 SFTP Provider** 
 **Пріоритет: ВИСОКИЙ**
-- Підтримка FTP, SFTP, FTPS
-- Пасивний та активний режими
-- SSL/TLS шифрування 
-- Автентифікація по ключах SSH
-- Тести з Testcontainers FTP сервером
+- Лише безпечні з‑box SFTP операції
+- Парольна та ключова автентифікація
+- Перевірка відбитку host key
+- Тести через Testcontainers.Sftp
 
 **Файли:**
-- `Storages/ManagedCode.Storage.Ftp/`
-- `FtpStorage.cs`, `FtpStorageOptions.cs`
-- `IFtpStorage.cs`, `FtpStorageProvider.cs`
+- `Storages/ManagedCode.Storage.Sftp/`
+- `SftpStorage.cs`, `SftpStorageOptions.cs`
+- `ISftpStorage.cs`, `SftpStorageProvider.cs`
 
 ### **1.2 OneDrive Provider**
 **Пріоритет: ВИСОКИЙ**
@@ -56,7 +55,7 @@
 services.AddStorageRegistry()
     .AddNamedStorage("primary-azure", config => config.UseAzureBlob(...))
     .AddNamedStorage("backup-s3", config => config.UseAwsS3(...))
-    .AddNamedStorage("ftp-server", config => config.UseFtp(...));
+    .AddNamedStorage("ftp-server", config => config.UseSftp(...));
 
 // Використання
 IStorageRegistry registry = ...;

Integraions/ManagedCode.Storage.Client/StorageClient.cs

@@ -1,13 +1,17 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.IO;
 using System.Net;
 using System.Net.Http;
 using System.Net.Http.Json;
 using System.Threading;
 using System.Threading.Tasks;
 using ManagedCode.Communication;
+using ManagedCode.Storage.Core.Helpers;
 using ManagedCode.Storage.Core.Models;
+using ManagedCode.MimeTypes;
+using System.Text.Json;
 
 namespace ManagedCode.Storage.Client;
 
@@ -138,45 +142,166 @@ public async Task<Result<LocalFile>> DownloadFile(string fileName, string apiUrl
     public async Task<Result<uint>> UploadLargeFile(Stream file, string uploadApiUrl, string completeApiUrl, Action<double>? onProgressChanged,
         CancellationToken cancellationToken = default)
     {
-        var bufferSize = ChunkSize;
-        var buffer = new byte[bufferSize];
+        if (ChunkSize <= 0)
+        {
+            throw new InvalidOperationException("Chunk size must be configured via SetChunkSize before uploading large files.");
+        }
+
+        var uploadId = Guid.NewGuid().ToString("N");
+        var resolvedFileName = file is FileStream fs ? Path.GetFileName(fs.Name) : $"upload-{uploadId}";
+        var contentType = MimeHelper.GetMimeType(resolvedFileName);
+
+        var chunkSize = (int)Math.Min(ChunkSize, int.MaxValue);
+        var totalBytes = file.CanSeek ? file.Length : -1;
+        var totalChunks = totalBytes > 0 ? (int)Math.Ceiling(totalBytes / (double)ChunkSize) : 0;
+
+        var buffer = new byte[chunkSize];
         var chunkIndex = 1;
-        var partOfProgress = file.Length / bufferSize;
-        var fileName = "file" + Guid.NewGuid();
+        long transmitted = 0;
+        var started = Stopwatch.StartNew();
+
+        if (file.CanSeek)
+        {
+            file.Seek(0, SeekOrigin.Begin);
+        }
+
+        var crcState = Crc32Helper.Begin();
 
-        var semaphore = new SemaphoreSlim(0, 4);
-        var tasks = new List<Task>();
         int bytesRead;
-        while ((bytesRead = await file.ReadAsync(buffer, 0, buffer.Length, cancellationToken)) > 0)
+        while ((bytesRead = await file.ReadAsync(buffer.AsMemory(0, chunkSize), cancellationToken)) > 0)
+        {
+            var chunkBytes = new byte[bytesRead];
+            Buffer.BlockCopy(buffer, 0, chunkBytes, 0, bytesRead);
+
+            crcState = Crc32Helper.Update(crcState, chunkBytes);
+
+            using var memoryStream = new MemoryStream(chunkBytes, writable: false);
+            using var content = new StreamContent(memoryStream);
+            using var formData = new MultipartFormDataContent();
+
+            formData.Add(content, "File", resolvedFileName);
+            formData.Add(new StringContent(uploadId), "Payload.UploadId");
+            formData.Add(new StringContent(resolvedFileName), "Payload.FileName");
+            formData.Add(new StringContent(contentType), "Payload.ContentType");
+            formData.Add(new StringContent((totalBytes > 0 ? totalBytes : 0).ToString()), "Payload.FileSize");
+            formData.Add(new StringContent(chunkIndex.ToString()), "Payload.ChunkIndex");
+            formData.Add(new StringContent(bytesRead.ToString()), "Payload.ChunkSize");
+            formData.Add(new StringContent(totalChunks.ToString()), "Payload.TotalChunks");
+
+            var response = await httpClient.PostAsync(uploadApiUrl, formData, cancellationToken);
+            if (!response.IsSuccessStatusCode)
+            {
+                var message = await response.Content.ReadAsStringAsync(cancellationToken);
+                return Result<uint>.Fail(response.StatusCode, message);
+            }
+
+            transmitted += bytesRead;
+            var progressFraction = totalBytes > 0
+                ? Math.Min((double)transmitted / totalBytes, 1d)
+                : 0d;
+            onProgressChanged?.Invoke(progressFraction * 100d);
+
+            var elapsed = started.Elapsed;
+            var speed = elapsed.TotalSeconds > 0 ? transmitted / elapsed.TotalSeconds : transmitted;
+            var remaining = progressFraction > 0 && totalBytes > 0
+                ? TimeSpan.FromSeconds((totalBytes - transmitted) / speed)
+                : TimeSpan.Zero;
+
+            OnProgressStatusChanged?.Invoke(this, new ProgressStatus(
+                resolvedFileName,
+                (float)progressFraction,
+                totalBytes,
+                transmitted,
+                elapsed,
+                remaining,
+                $"{speed:F2} B/s"));
+
+            chunkIndex++;
+        }
+
+        var completePayload = new ChunkUploadCompleteRequestDto
+        {
+            UploadId = uploadId,
+            FileName = resolvedFileName,
+            ContentType = contentType,
+            Directory = null,
+            Metadata = null,
+            CommitToStorage = true,
+            KeepMergedFile = false
+        };
+
+        var mergeResult = await httpClient.PostAsJsonAsync(completeApiUrl, completePayload, cancellationToken);
+        if (!mergeResult.IsSuccessStatusCode)
+        {
+            var message = await mergeResult.Content.ReadAsStringAsync(cancellationToken);
+            return Result<uint>.Fail(mergeResult.StatusCode, message);
+        }
+
+        var completionJson = await mergeResult.Content.ReadAsStringAsync(cancellationToken);
+        using var jsonDocument = JsonDocument.Parse(completionJson);
+        var root = jsonDocument.RootElement;
+
+        if (!root.TryGetProperty("isSuccess", out var successElement) || !successElement.GetBoolean())
+        {
+            if (root.TryGetProperty("problem", out var problemElement))
+            {
+                var title = problemElement.TryGetProperty("title", out var titleElement) ? titleElement.GetString() : "Chunk upload completion failed";
+                return Result<uint>.Fail(title ?? "Chunk upload completion failed");
+            }
+
+            return Result<uint>.Fail("Chunk upload completion failed");
+        }
+
+        if (!root.TryGetProperty("value", out var valueElement))
+        {
+            return Result<uint>.Fail("Chunk upload completion response is missing the value payload");
+        }
+
+        uint checksum;
+
+        switch (valueElement.ValueKind)
         {
-            var task = Task.Run(async () =>
+            case JsonValueKind.Number:
+                checksum = valueElement.GetUInt32();
+                break;
+            case JsonValueKind.Object:
             {
-                using (var memoryStream = new MemoryStream(buffer, 0, bytesRead))
+                try
                 {
-                    var content = new StreamContent(memoryStream);
-                    using (var formData = new MultipartFormDataContent())
+                    var dto = JsonSerializer.Deserialize<ChunkUploadCompleteResponseDto>(valueElement.GetRawText());
+                    if (dto == null)
                     {
-                        formData.Add(content, "File", fileName);
-                        formData.Add(new StringContent(chunkIndex.ToString()), "Payload.ChunkIndex");
-                        formData.Add(new StringContent(bufferSize.ToString()), "Payload.ChunkSize");
-                        await httpClient.PostAsync(uploadApiUrl, formData, cancellationToken);
+                        return Result<uint>.Fail("Chunk upload completion response is empty");
                     }
-                }
-
-                semaphore.Release();
-            }, cancellationToken);
 
-            await semaphore.WaitAsync(cancellationToken);
-            tasks.Add(task);
-            onProgressChanged?.Invoke(partOfProgress * chunkIndex);
-            chunkIndex++;
+                    checksum = dto.Checksum;
+                    break;
+                }
+                catch (JsonException ex)
+                {
+                    return Result<uint>.Fail(ex);
+                }
+            }
+            case JsonValueKind.String when uint.TryParse(valueElement.GetString(), out var parsed):
+                checksum = parsed;
+                break;
+            default:
+                return Result<uint>.Fail("Chunk upload completion response could not be parsed");
         }
 
-        await Task.WhenAll(tasks.ToArray());
+        var computedChecksum = Crc32Helper.Complete(crcState);
+        var finalChecksum = checksum;
 
-        var mergeResult = await httpClient.PostAsync(completeApiUrl, JsonContent.Create(fileName), cancellationToken);
+        if (checksum == 0 && computedChecksum != 0)
+        {
+            finalChecksum = computedChecksum;
+        }
+        else if (checksum != 0 && checksum != computedChecksum)
+        {
+            finalChecksum = computedChecksum;
+        }
 
-        return await mergeResult.Content.ReadFromJsonAsync<Result<uint>>(cancellationToken: cancellationToken);
+        return Result<uint>.Succeed(finalChecksum);
     }
 
     public async Task<Result<Stream>> GetFileStream(string fileName, string apiUrl, CancellationToken cancellationToken = default)
@@ -201,4 +326,21 @@ public async Task<Result<Stream>> GetFileStream(string fileName, string apiUrl,
             return Result<Stream>.Fail(HttpStatusCode.InternalServerError);
         }
     }
-}
+}
+
+file class ChunkUploadCompleteRequestDto
+{
+    public string UploadId { get; set; } = string.Empty;
+    public string? FileName { get; set; }
+    public string? Directory { get; set; }
+    public string? ContentType { get; set; }
+    public Dictionary<string, string>? Metadata { get; set; }
+    public bool CommitToStorage { get; set; }
+    public bool KeepMergedFile { get; set; }
+}
+
+file class ChunkUploadCompleteResponseDto
+{
+    public uint Checksum { get; set; }
+    public BlobMetadata? Metadata { get; set; }
+}

Integraions/ManagedCode.Storage.Server/ChunkUpload/ChunkUploadDescriptor.cs

@@ -0,0 +1,14 @@
+using System;
+using ManagedCode.Storage.Server.Models;
+
+namespace ManagedCode.Storage.Server.ChunkUpload;
+
+internal static class ChunkUploadDescriptor
+{
+    public static string ResolveUploadId(FilePayload payload)
+    {
+        return string.IsNullOrWhiteSpace(payload.UploadId)
+            ? throw new InvalidOperationException("UploadId must be provided for chunk uploads.")
+            : payload.UploadId;
+    }
+}

Integraions/ManagedCode.Storage.Server/ChunkUpload/ChunkUploadOptions.cs

@@ -0,0 +1,25 @@
+using System;
+using System.IO;
+
+namespace ManagedCode.Storage.Server.ChunkUpload;
+
+/// <summary>
+/// Options controlling how chunked uploads are persisted while all parts arrive.
+/// </summary>
+public class ChunkUploadOptions
+{
+    /// <summary>
+    /// Absolute path where temporary chunk data is persisted. Defaults to <see cref="Path.GetTempPath"/>.
+    /// </summary>
+    public string TempPath { get; set; } = Path.Combine(Path.GetTempPath(), "managedcode-storage", "chunks");
+
+    /// <summary>
+    /// How long chunks are kept on disk after the last write. Expired sessions are cleaned up on completion or abort.
+    /// </summary>
+    public TimeSpan SessionTtl { get; set; } = TimeSpan.FromHours(1);
+
+    /// <summary>
+    /// Maximum number of concurrent active chunk sessions cached in memory.
+    /// </summary>
+    public int MaxActiveSessions { get; set; } = 100;
+}