Добавить перенос файлов при создании курса из шаблона (#610)

bygu4 · DedSec256 · web-flow · commit 8db881903ad0 · 2025-08-30T06:26:17.000+03:00
* add method for adding existing file reference to the file record repository

* add file transfer method to ContentService

* implement files transfer on course creation inside a transaction

* add ContentService client on startup

* debug and refactor: use List instead of Dictionary for serialization, move CourseUnitType to Common

* add models for handling file transfer

* optimize files transfer in FIleRecordRepository: get files info by course id. Refactor

* reset course loading flag on error

* handle possible references to the same file in FileRecordRepository

* change naming, rename TransferFiles models to CourseFilesTransfer

* handle possible undefined mapping keys in FileRecordRepository

* rename GetFilesTransferDTO to GetCourseFilesTransfer

* change GetCourseFilesTransfer signature

* docker fix

* refactoring

* revert

* fixes + local mode

---------

Co-authored-by: Alexey.Berezhnykh &lt;alexey.berezhnykh@jetbrains.com&gt;
diff --git a/HwProj.Common/HwProj.Models/ContentService/DTO/CourseFilesTransferDTO.cs b/HwProj.Common/HwProj.Models/ContentService/DTO/CourseFilesTransferDTO.cs
@@ -0,0 +1,11 @@
+using System.Collections.Generic;
+
+namespace HwProj.Models.ContentService.DTO
+{
+    public class CourseFilesTransferDto
+    {
+        public long SourceCourseId { get; set; }
+        public long TargetCourseId { get; set; }
+        public List<ScopeMappingDto> HomeworksMapping { get; set; } = new List<ScopeMappingDto>();
+    }
+}
diff --git a/HwProj.Common/HwProj.Models/ContentService/DTO/ScopeMappingPairDTO.cs b/HwProj.Common/HwProj.Models/ContentService/DTO/ScopeMappingPairDTO.cs
@@ -0,0 +1,8 @@
+namespace HwProj.Models.ContentService.DTO
+{
+    public class ScopeMappingDto
+    {
+        public long Source { get; set; }
+        public long Target { get; set; }
+    }
+}
diff --git a/HwProj.ContentService/HwProj.ContentService.API/Controllers/FilesController.cs b/HwProj.ContentService/HwProj.ContentService.API/Controllers/FilesController.cs
@@ -39,7 +39,7 @@ public async Task<IActionResult> Process([FromForm] ProcessFilesDTO processFiles
     {
         var userId = Request.GetUserIdFromHeader();
         var scope = processFilesDto.FilesScope.ToScope();
-        
+
         if (processFilesDto.DeletingFileIds.Count > 0)
             await _messageProducer.PushDeleteFilesMessages(scope, processFilesDto.DeletingFileIds, userId);
 
@@ -53,7 +53,7 @@ public async Task<IActionResult> Process([FromForm] ProcessFilesDTO processFiles
                 await _localFilesService.SaveFile(newFormFile.OpenReadStream(), localFilePath);
                 _logger.LogInformation("Файл {FileName} успешно сохранён в локальное хранилище по пути {localFilePath}",
                     newFormFile.FileName, localFilePath);
-                
+
                 var message = new UploadFileMessage(
                     Scope: processFilesDto.FilesScope.ToScope(),
                     LocalFilePath: localFilePath,
@@ -64,10 +64,10 @@ public async Task<IActionResult> Process([FromForm] ProcessFilesDTO processFiles
                 );
                 uploadFilesMessages.Add(message);
             }
-            
+
             await _messageProducer.PushUploadFilesMessages(uploadFilesMessages);
         }
-        
+
         return Ok();
     }
 
@@ -86,7 +86,7 @@ public async Task<IActionResult> GetDownloadLink([FromQuery] long fileId)
     {
         var externalKey = await _filesInfoService.GetFileExternalKeyAsync(fileId);
         if (externalKey is null) return Ok(Result<string>.Failed("Файл не найден"));
-        
+
         var downloadUrlResult = await _s3FilesService.GetDownloadUrl(externalKey);
         return Ok(downloadUrlResult);
     }
@@ -98,12 +98,20 @@ public async Task<IActionResult> GetFilesInfo(long courseId)
         var filesInfo = await _filesInfoService.GetFilesInfoAsync(courseId);
         return Ok(filesInfo);
     }
-    
+
     [HttpGet("info/course/{courseId}/uploaded")]
     [ProducesResponseType(typeof(FileInfoDTO[]), (int)HttpStatusCode.OK)]
     public async Task<IActionResult> GetUploadedFilesInfo(long courseId)
     {
         var filesInfo = await _filesInfoService.GetFilesInfoAsync(courseId, FileStatus.ReadyToUse);
         return Ok(filesInfo);
     }
-}
+
+    [HttpPost("transfer")]
+    [ProducesResponseType((int)HttpStatusCode.OK)]
+    public async Task<IActionResult> TransferFilesFromCourse(CourseFilesTransferDto filesTransferDto)
+    {
+        await _filesInfoService.TransferFilesFromCourse(filesTransferDto);
+        return Ok();
+    }
+}
diff --git a/HwProj.ContentService/HwProj.ContentService.API/Extensions/ConfigurationExtensions.cs b/HwProj.ContentService/HwProj.ContentService.API/Extensions/ConfigurationExtensions.cs
@@ -22,7 +22,7 @@ namespace HwProj.ContentService.API.Extensions;
 
 public static class ConfigurationExtensions
 {
-    public static IServiceCollection ConfigureWithAWS(this IServiceCollection services,
+    public static IServiceCollection ConfigureWithAWS(this IServiceCollection services, IHostEnvironment env,
         IConfigurationRoot configuration)
     {
         // Достаем конфигурацию удаленного хранилища
@@ -43,13 +43,22 @@ public static IServiceCollection ConfigureWithAWS(this IServiceCollection servic
         var connectionString = ConnectionString.GetConnectionString(configuration);
         services.AddDbContext<ContentContext>(options => options.UseSqlServer(connectionString));
         services.AddScoped<IFileRecordRepository, FileRecordRepository>();
-        
-        services.ConfigureExternalStorageClient(externalStorageSection);
+
+        if (env.IsDevelopment())
+        {
+            services.AddSingleton<IS3FilesService, LocalTestingS3FilesService>();
+        }
+        else
+        {
+            services.ConfigureExternalStorageClient(externalStorageSection);
+            services.AddSingleton<IS3FilesService, S3FilesService>();
+        }
+
         services.ConfigureChannelInfrastructure<IProcessFileMessage>();
         
         // Регистрируем как синглтоны, чтобы использовать в MessageConsumer
         services.AddSingleton<IFileKeyService, FileKeyService>();
-        services.AddSingleton<IS3FilesService, S3FilesService>();
+
         services.AddSingleton<ILocalFilesService, LocalFilesService>();
         
         services.AddScoped<IFilesInfoService, FilesInfoService>();
diff --git a/HwProj.ContentService/HwProj.ContentService.API/Extensions/MappingExtensions.cs b/HwProj.ContentService/HwProj.ContentService.API/Extensions/MappingExtensions.cs
@@ -1,11 +1,19 @@
 using HwProj.ContentService.API.Models;
+using HwProj.ContentService.API.Models.Database;
 using HwProj.ContentService.API.Models.Enums;
 using HwProj.Models.ContentService.DTO;
 
 namespace HwProj.ContentService.API.Extensions;
 
 public static class MappingExtensions
 {
+    public static Scope ToScope(this FileToCourseUnit fileToCourseUnit)
+        => new Scope(
+            CourseId: fileToCourseUnit.CourseId,
+            CourseUnitId: fileToCourseUnit.CourseUnitId,
+            CourseUnitType: fileToCourseUnit.CourseUnitType
+        );
+
     public static Scope ToScope(this ScopeDTO scopeDTO)
         => new Scope(
             CourseId: scopeDTO.CourseId,
@@ -20,4 +28,4 @@ public static ScopeDTO ToScopeDTO(this Scope scope)
             CourseUnitType = scope.CourseUnitType.ToString(),
             CourseUnitId = scope.CourseUnitId
         };
-}
+}
diff --git a/HwProj.ContentService/HwProj.ContentService.API/Program.cs b/HwProj.ContentService/HwProj.ContentService.API/Program.cs
@@ -2,7 +2,7 @@
 using HwProj.ContentService.API.Services.Interfaces;
 
 var builder = WebApplication.CreateBuilder(args);
-builder.Services.ConfigureWithAWS(builder.Configuration);
+builder.Services.ConfigureWithAWS(builder.Environment, builder.Configuration);
 
 // Увеличиваем размер принимаемых запросов до 200 МБ
 builder.WebHost.ConfigureKestrel(options => { options.Limits.MaxRequestBodySize = 200 * 1024 * 1024; });
@@ -15,7 +15,7 @@
 app.ConfigureWebApp();
 
 // При необходимости создаем пустой бакет в хранилище
-await app.CreateBucketIfNotExists();
+if (!app.Environment.IsDevelopment()) await app.CreateBucketIfNotExists();
 
 var lifetime = app.Services.GetRequiredService<IHostApplicationLifetime>();
 lifetime.ApplicationStarted.Register(async () =>
diff --git a/HwProj.ContentService/HwProj.ContentService.API/Repositories/FileRecordRepository.cs b/HwProj.ContentService/HwProj.ContentService.API/Repositories/FileRecordRepository.cs
@@ -4,6 +4,7 @@
 using HwProj.ContentService.API.Models.Enums;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.EntityFrameworkCore.Query;
+using HwProj.ContentService.API.Extensions;
 
 namespace HwProj.ContentService.API.Repositories;
 
@@ -65,7 +66,7 @@ public async Task<List<FileToCourseUnit>> GetByCourseIdAsync(long courseId)
             .Where(fc => fc.CourseId == courseId)
             .Include(fc => fc.FileRecord)
             .ToListAsync();
-    
+
     public async Task<List<FileToCourseUnit>> GetByCourseIdAndStatusAsync(long courseId, FileStatus filesStatus)
         => await _contentContext.FileToCourseUnits
             .AsNoTracking()
@@ -140,10 +141,28 @@ await _contentContext.FileRecords
         return fileRecord.ReferenceCount;
     }
 
+    /// Переносит файлы с помощью добавления записей в FileToCourseUnit согласно переданному отображению.
+    /// Увеличивает число ссылок на файлы на число добавленных записей, соответствующих файлу.
+    public async Task AddFileUnitsAsync(List<FileToCourseUnit> unitsToAdd)
+    {
+        await using var transaction = await _contentContext.Database.BeginTransactionAsync();
+
+        await _contentContext.FileToCourseUnits.AddRangeAsync(unitsToAdd);
+        foreach (var unit in unitsToAdd)
+        {
+            await UpdateAsync(
+                unit.FileRecordId,
+                setters => setters.SetProperty(x => x.ReferenceCount, x => x.ReferenceCount + 1));
+        }
+
+        await _contentContext.SaveChangesAsync();
+        await transaction.CommitAsync();
+    }
+
     public async Task UpdateAsync(long id,
         Expression<Func<SetPropertyCalls<FileRecord>, SetPropertyCalls<FileRecord>>> setPropertyCalls)
         => await _contentContext.FileRecords
             .AsNoTracking()
             .Where(fr => fr.Id == id)
             .ExecuteUpdateAsync(setPropertyCalls);
-}
+}
diff --git a/HwProj.ContentService/HwProj.ContentService.API/Repositories/IFileRecordRepository.cs b/HwProj.ContentService/HwProj.ContentService.API/Repositories/IFileRecordRepository.cs
@@ -19,5 +19,6 @@ public Task UpdateAsync(long id,
     public Task<List<long>> GetIdsByStatusAsync(FileStatus status);
     public Task DeleteWithCourseUnitInfoAsync(long fileRecordId);
     public Task DeleteWithCourseUnitInfoAsync(List<long> fileRecordIds);
+    public Task AddFileUnitsAsync(List<FileToCourseUnit> unitsToAdd);
     public Task<int> ReduceReferenceCountAsync(FileRecord fileRecord, Scope scope);
 }
diff --git a/HwProj.ContentService/HwProj.ContentService.API/Services/FilesInfoService.cs b/HwProj.ContentService/HwProj.ContentService.API/Services/FilesInfoService.cs
@@ -1,4 +1,6 @@
+using HwProj.ContentService.API.Extensions;
 using HwProj.ContentService.API.Models;
+using HwProj.ContentService.API.Models.Database;
 using HwProj.ContentService.API.Models.Enums;
 using HwProj.ContentService.API.Repositories;
 using HwProj.ContentService.API.Services.Interfaces;
@@ -62,4 +64,31 @@ public async Task<List<FileInfoDTO>> GetFilesInfoAsync(long courseId, FileStatus
             CourseUnitId = fcu.CourseUnitId
         }).ToList();
     }
-}
+
+    public async Task TransferFilesFromCourse(CourseFilesTransferDto filesTransfer)
+    {
+        var map = filesTransfer.HomeworksMapping.ToDictionary(
+            x => new Scope(filesTransfer.SourceCourseId, CourseUnitType.Homework, x.Source),
+            x => new Scope(filesTransfer.TargetCourseId, CourseUnitType.Homework, x.Target)
+        );
+
+        var sourceCourseUnits = await _fileRecordRepository.GetByCourseIdAsync(filesTransfer.SourceCourseId);
+        var unitsToAdd = sourceCourseUnits
+            .Select(unit => (unit.FileRecord, Scope: unit.ToScope()))
+            .Where(pair => map.ContainsKey(pair.Scope))
+            .Select(pair =>
+            {
+                var targetScope = map[pair.Scope];
+                return new FileToCourseUnit
+                {
+                    FileRecordId = pair.FileRecord.Id,
+                    CourseId = targetScope.CourseId,
+                    CourseUnitId = targetScope.CourseUnitId,
+                    CourseUnitType = targetScope.CourseUnitType
+                };
+            })
+            .ToList();
+
+        await _fileRecordRepository.AddFileUnitsAsync(unitsToAdd);
+    }
+}
diff --git a/HwProj.ContentService/HwProj.ContentService.API/Services/Interfaces/IFilesInfoService.cs b/HwProj.ContentService/HwProj.ContentService.API/Services/Interfaces/IFilesInfoService.cs
@@ -10,4 +10,5 @@ public interface IFilesInfoService
     public Task<string?> GetFileExternalKeyAsync(long fileId);
     public Task<List<FileInfoDTO>> GetFilesInfoAsync(long courseId);
     public Task<List<FileInfoDTO>> GetFilesInfoAsync(long courseId, FileStatus filesStatus);
-}
+    public Task TransferFilesFromCourse(CourseFilesTransferDto filesTransfer);
+}
diff --git a/HwProj.ContentService/HwProj.ContentService.API/Services/S3FilesService.cs b/HwProj.ContentService/HwProj.ContentService.API/Services/S3FilesService.cs
@@ -26,7 +26,7 @@ public S3FilesService(IAmazonS3 s3Client, IOptions<ExternalStorageConfiguration>
         _bucketName = externalStorageConfiguration.Value.DefaultBucketName
                       ?? throw new NullReferenceException("Не указано имя бакета для сохранения файлов");
     }
-    
+
     // Метод позволяет получить файлы вместе с дополнительной информацией из бакета.
     // filePathPattern должен содержать группы FileName, HomeworkId и CourseId,
     // чтобы эту дополнительную информацию из пути к файлу извлечь
@@ -45,7 +45,7 @@ public async Task<List<FileTransferDTO>> GetBucketFilesAsync(string bucketName,
                 BucketName = bucketName,
                 Key = key
             });
-            
+
             // Получаем сам файл
             var fileStream = await GetFileStream(bucketName, key);
 
@@ -61,11 +61,11 @@ public async Task<List<FileTransferDTO>> GetBucketFilesAsync(string bucketName,
                 CourseId: long.Parse(match.Groups["CourseId"].Value)
             );
         }).ToList();
-        
+
         var fileTransfers = await Task.WhenAll(tasks);
         return fileTransfers.ToList();
     }
-    
+
     // Если файл с таким ключем уже существует, в текущей реализации он будет перезаписываться
     public async Task<Result> UploadFileAsync(UploadFileToS3Dto uploadFileToS3Dto)
     {
@@ -132,7 +132,7 @@ public async Task<Result> DeleteFileAsync(string fileKey)
             return Result.Failed(e.Message);
         }
     }
-    
+
     private async Task<Stream> GetFileStream(string bucketName, string fileKey)
         => await _s3AmazonClient.GetObjectStreamAsync(bucketName, fileKey, null);
 
@@ -175,4 +175,19 @@ private async Task<MetadataCollection> GetFileMetadataAsync(string fileKey)
 
         return metadataResponse.Metadata;
     }
+}
+
+
+
+internal class LocalTestingS3FilesService : IS3FilesService
+{
+    public Task<Result> UploadFileAsync(UploadFileToS3Dto uploadFileToS3Dto) =>  Task.FromResult(Result.Success());
+
+    public Task<List<FileTransferDTO>> GetBucketFilesAsync(string bucketName, string filePathPattern) => Task.FromResult(new List<FileTransferDTO>());
+
+    public Task<Result<string>> GetDownloadUrl(string fileKey) => Task.FromResult(Result<string>.Success("download"));
+
+    public Task<bool> CheckFileExistence(string fileKey) =>  Task.FromResult(true);
+
+    public Task<Result> DeleteFileAsync(string fileKey) => Task.FromResult(Result.Success());
 }
diff --git a/HwProj.ContentService/HwProj.ContentService.Client/ContentServiceClient.cs b/HwProj.ContentService/HwProj.ContentService.Client/ContentServiceClient.cs
@@ -1,5 +1,4 @@
 using System;
-using System.Collections.Generic;
 using System.Net.Http;
 using System.Net.Http.Headers;
 using System.Text;
@@ -146,5 +145,27 @@ public async Task<Result<FileInfoDTO[]>> GetUploadedFilesInfo(long courseId)
                     "Пока не можем получить информацию о файлах. \nВсе ваши данные сохранены — попробуйте повторить позже");
             }
         }
+
+        public async Task<Result> TransferFilesFromCourse(CourseFilesTransferDto filesTransfer)
+        {
+            var url = _contentServiceUri + "api/Files/transfer";
+            using var httpRequest = new HttpRequestMessage(HttpMethod.Post, url)
+            {
+                Content = new StringContent(
+                    JsonConvert.SerializeObject(filesTransfer),
+                    Encoding.UTF8,
+                    "application/json")
+            };
+
+            try
+            {
+                await _httpClient.SendAsync(httpRequest);
+                return Result.Success();
+            }
+            catch (HttpRequestException e)
+            {
+                return Result.Failed("Не удалось перенести информацию о файлах — попробуйте повторить позже");
+            }
+        }
     }
-}
+}
diff --git a/HwProj.ContentService/HwProj.ContentService.Client/IContentServiceClient.cs b/HwProj.ContentService/HwProj.ContentService.Client/IContentServiceClient.cs
@@ -11,5 +11,6 @@ public interface IContentServiceClient
         Task<Result<string>> GetDownloadLinkAsync(long fileId);
         Task<Result<FileInfoDTO[]>> GetFilesInfo(long courseId);
         Task<Result<FileInfoDTO[]>> GetUploadedFilesInfo(long courseId);
+        Task<Result> TransferFilesFromCourse(CourseFilesTransferDto filesTransfer);
     }
-}
+}
diff --git a/HwProj.CoursesService/HwProj.CoursesService.API/Dockerfile b/HwProj.CoursesService/HwProj.CoursesService.API/Dockerfile
@@ -9,6 +9,7 @@ COPY ["HwProj.EventBus/HwProj.EventBus.Client/", "HwProj.EventBus/HwProj.EventBu
 COPY ["HwProj.Common/HwProj.Models/", "HwProj.Common/HwProj.Models/"]
 COPY ["HwProj.Common/HwProj.Repositories/", "HwProj.Common/HwProj.Repositories/"]
 COPY ["HwProj.AuthService/HwProj.AuthService.Client/", "HwProj.AuthService/HwProj.AuthService.Client/"]
+COPY ["HwProj.ContentService/HwProj.ContentService.Client/", "HwProj.ContentService/HwProj.ContentService.Client/"]
 COPY ["HwProj.Common/HwProj.HttpUtils/", "HwProj.Common/HwProj.HttpUtils/"]
 
 WORKDIR "/src/HwProj.CoursesService/HwProj.CoursesService.API"
diff --git a/HwProj.CoursesService/HwProj.CoursesService.API/HwProj.CoursesService.API.csproj b/HwProj.CoursesService/HwProj.CoursesService.API/HwProj.CoursesService.API.csproj
@@ -22,6 +22,7 @@
     <PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.10.13" />
   </ItemGroup>
   <ItemGroup>
+    <ProjectReference Include="..\..\HwProj.ContentService\HwProj.ContentService.Client\HwProj.ContentService.Client.csproj" />
     <ProjectReference Include="..\..\HwProj.AuthService\HwProj.AuthService.Client\HwProj.AuthService.Client.csproj" />
     <ProjectReference Include="..\..\HwProj.Common\HwProj.Repositories\HwProj.Repositories.csproj" />
     <ProjectReference Include="..\..\HwProj.Common\HwProj.Utils\HwProj.Utils.csproj" />
diff --git a/HwProj.CoursesService/HwProj.CoursesService.API/Services/CoursesService.cs b/HwProj.CoursesService/HwProj.CoursesService.API/Services/CoursesService.cs
diff --git a/HwProj.CoursesService/HwProj.CoursesService.API/Startup.cs b/HwProj.CoursesService/HwProj.CoursesService.API/Startup.cs
diff --git a/hwproj.front/src/components/Courses/CreateCourse.tsx b/hwproj.front/src/components/Courses/CreateCourse.tsx

Original file line number	Diff line number	Diff line change
`@@ -19,5 +19,6 @@ public Task UpdateAsync(long id,`
`19`	`19`	`public Task<List<long>> GetIdsByStatusAsync(FileStatus status);`
`20`	`20`	`public Task DeleteWithCourseUnitInfoAsync(long fileRecordId);`
`21`	`21`	`public Task DeleteWithCourseUnitInfoAsync(List<long> fileRecordIds);`
	`22`	`+ public Task AddFileUnitsAsync(List<FileToCourseUnit> unitsToAdd);`
`22`	`23`	`public Task<int> ReduceReferenceCountAsync(FileRecord fileRecord, Scope scope);`
`23`	`24`	`}`