Improve performances on large number of files (#80)

Author: paul-fresquet

src/ByteSync.Client/Business/Synchronizations/MultiUploadZip.cs

@@ -12,7 +12,6 @@ public MultiUploadZip(string key, SharedFileDefinition sharedFileDefinition)
         Key = key;
 
         SharedFileDefinition = sharedFileDefinition;
-        // ZipPath = zipPath;
 
         SharedFileDefinition.IsMultiFileZip = true;
         CreationDate = DateTime.Now;
@@ -21,8 +20,6 @@ public MultiUploadZip(string key, SharedFileDefinition sharedFileDefinition)
         MemoryStream = new MemoryStream();
 
         ZipArchive = new ZipArchive(MemoryStream, ZipArchiveMode.Create, true);
-        
-        // ZipArchive = ZipFile.Open(zipPath, ZipArchiveMode.Update);
 
         ActionGroupsIds = new List<string>();
 
@@ -35,8 +32,6 @@ public MultiUploadZip(string key, SharedFileDefinition sharedFileDefinition)
 
     public SharedFileDefinition SharedFileDefinition { get; }
 
-    // public string ZipPath { get; }
-    
     public MemoryStream MemoryStream { get; }
 
     public ZipArchive ZipArchive { get; }
@@ -54,7 +49,7 @@ public MultiUploadZip(string key, SharedFileDefinition sharedFileDefinition)
     public bool CanAdd(FileInfo fileInfo, string actionsGroupId)
     {
         return 
-            ActionGroupsIds.Count < 1500 &&
+            ActionGroupsIds.Count < 100 &&
             ActionsGroupIdsConcatenationLength + actionsGroupId.Length + 5 < 25000 &&
             Size + fileInfo.Length < 8 * SizeConstants.ONE_MEGA_BYTES;
     }

src/ByteSync.Client/DependencyInjection/Modules/GenericTypesModule.cs

@@ -12,7 +12,11 @@ public class GenericTypesModule : Module
     protected override void Load(ContainerBuilder builder)
     {
         builder.RegisterGeneric(typeof(SessionInvalidationCachePolicy<,>))
-            .As(typeof(ISessionInvalidationSourceCachePolicy<,>))
+            .As(typeof(ISessionInvalidationCachePolicy<,>))
+            .InstancePerDependency();
+        
+        builder.RegisterGeneric(typeof(PropertyIndexer<,>))
+            .As(typeof(IPropertyIndexer<,>))
             .InstancePerDependency();
 
         builder.RegisterType<ConfigurationReader<ApplicationSettings>>().As<IConfigurationReader<ApplicationSettings>>();

src/ByteSync.Client/Interfaces/Controls/Communications/Http/ISynchronizationApiClient.cs

@@ -7,7 +7,7 @@ namespace ByteSync.Interfaces.Controls.Communications.Http;
 
 public interface ISynchronizationApiClient
 {
-    public Task<Synchronization> StartSynchronization(SynchronizationStartRequest synchronizationStartRequest);
+    public Task StartSynchronization(SynchronizationStartRequest synchronizationStartRequest);
 
     public Task AssertLocalCopyIsDone(string sessionId, List<string> actionsGroupIds);
 

src/ByteSync.Client/Interfaces/Controls/Synchronizations/ISynchronizationLooper.cs

@@ -2,9 +2,7 @@
 
 namespace ByteSync.Interfaces.Controls.Synchronizations;
 
-public interface ISynchronizationLooper
+public interface ISynchronizationLooper : IAsyncDisposable
 {
-    // Task InitializeData();
-
     Task CloudSessionSynchronizationLoop();
 }

src/ByteSync.Client/Interfaces/Repositories/IPropertyIndexer.cs

@@ -0,0 +1,12 @@
+using DynamicData;
+
+namespace ByteSync.Interfaces.Repositories;
+
+public interface IPropertyIndexer<TObject, in TIndex> 
+    where TObject : notnull 
+    where TIndex : notnull
+{
+    void Initialize(SourceCache<TObject, string> sourceCache, Func<TObject, TIndex> indexSelector);
+
+    List<TObject> GetByIndex(TIndex index);
+}

…ISessionInvalidationSourceCachePolicy.cs …ories/ISessionInvalidationCachePolicy.cssrc/ByteSync.Client/Interfaces/Repositories/ISessionInvalidationSourceCachePolicy.cs renamed to src/ByteSync.Client/Interfaces/Repositories/ISessionInvalidationCachePolicy.cs src/ByteSync.Client/Interfaces/Repositories/ISessionInvalidationSourceCachePolicy.cs renamed to src/ByteSync.Client/Interfaces/Repositories/ISessionInvalidationCachePolicy.cs

@@ -2,7 +2,9 @@
 
 namespace ByteSync.Interfaces.Repositories;
 
-public interface ISessionInvalidationSourceCachePolicy<TObject, TKey> : IDisposable where TKey : notnull
+public interface ISessionInvalidationCachePolicy<TObject, TKey> : IDisposable 
+    where TKey : notnull 
+    where TObject : notnull
 {
     void Initialize(SourceCache<TObject, TKey> sourceCache, bool b, bool b1);
 }

src/ByteSync.Client/Repositories/AtomicActionRepository.cs

@@ -6,22 +6,23 @@ namespace ByteSync.Repositories;
 
 public class AtomicActionRepository : BaseSourceCacheRepository<AtomicAction, string>, IAtomicActionRepository
 {
-    private readonly ISessionInvalidationSourceCachePolicy<AtomicAction, string> _sessionInvalidationSourceCachePolicy;
+    private readonly ISessionInvalidationCachePolicy<AtomicAction, string> _sessionInvalidationCachePolicy;
+    private readonly IPropertyIndexer<AtomicAction, ComparisonItem> _propertyIndexer;
 
-    public AtomicActionRepository(ISessionInvalidationSourceCachePolicy<AtomicAction, string> sessionInvalidationSourceCachePolicy)
+    public AtomicActionRepository(ISessionInvalidationCachePolicy<AtomicAction, string> sessionInvalidationCachePolicy,
+        IPropertyIndexer<AtomicAction, ComparisonItem> propertyIndexer)
     {
-        _sessionInvalidationSourceCachePolicy = sessionInvalidationSourceCachePolicy;
-        _sessionInvalidationSourceCachePolicy.Initialize(SourceCache, true, true);
+        _sessionInvalidationCachePolicy = sessionInvalidationCachePolicy;
+        _sessionInvalidationCachePolicy.Initialize(SourceCache, true, true);
+
+        _propertyIndexer = propertyIndexer;
+        _propertyIndexer.Initialize(SourceCache, atomicAction => atomicAction.ComparisonItem!);
     }
-    
+
     protected override string KeySelector(AtomicAction atomicAction) => atomicAction.AtomicActionId;
 
     public List<AtomicAction> GetAtomicActions(ComparisonItem comparisonItem)
     {
-        var result = SourceCache.Items
-            .Where(atomicAction => Equals(atomicAction.ComparisonItem, comparisonItem))
-            .ToList();
-
-        return result;
+        return _propertyIndexer.GetByIndex(comparisonItem);
     }
-}
+}

src/ByteSync.Client/Repositories/BaseSourceCacheRepository.cs

@@ -4,7 +4,7 @@
 namespace ByteSync.Repositories;
 
 public abstract class BaseSourceCacheRepository<TObject, TKey> : IBaseSourceCacheRepository<TObject, TKey> 
-    where TKey : notnull 
+    where TKey : notnull where TObject : notnull
 {
     protected BaseSourceCacheRepository()
     {

src/ByteSync.Client/Repositories/InventoryFileRepository.cs

@@ -6,12 +6,12 @@ namespace ByteSync.Repositories;
 
 public class InventoryFileRepository : BaseSourceCacheRepository<InventoryFile, string>, IInventoryFileRepository
 {
-    private readonly ISessionInvalidationSourceCachePolicy<InventoryFile, string> _sessionInvalidationSourceCachePolicy;
+    private readonly ISessionInvalidationCachePolicy<InventoryFile, string> _sessionInvalidationCachePolicy;
 
-    public InventoryFileRepository(ISessionInvalidationSourceCachePolicy<InventoryFile, string> sessionInvalidationSourceCachePolicy)
+    public InventoryFileRepository(ISessionInvalidationCachePolicy<InventoryFile, string> sessionInvalidationCachePolicy)
     {
-        _sessionInvalidationSourceCachePolicy = sessionInvalidationSourceCachePolicy;
-        _sessionInvalidationSourceCachePolicy.Initialize(SourceCache, true, true);
+        _sessionInvalidationCachePolicy = sessionInvalidationCachePolicy;
+        _sessionInvalidationCachePolicy.Initialize(SourceCache, true, true);
     }
 
     protected override string KeySelector(InventoryFile inventoryFile) => inventoryFile.FullName;

src/ByteSync.Client/Repositories/PathItemRepository.cs

@@ -7,17 +7,17 @@ namespace ByteSync.Repositories;
 
 public class PathItemRepository : BaseSourceCacheRepository<PathItem, string>, IPathItemRepository
 {
-    private readonly ISessionInvalidationSourceCachePolicy<PathItem, string> _sessionInvalidationSourceCachePolicy;
+    private readonly ISessionInvalidationCachePolicy<PathItem, string> _sessionInvalidationCachePolicy;
 
-    public PathItemRepository(IEnvironmentService environmentService, ISessionInvalidationSourceCachePolicy<PathItem, string> sessionInvalidationSourceCachePolicy)
+    public PathItemRepository(IEnvironmentService environmentService, ISessionInvalidationCachePolicy<PathItem, string> sessionInvalidationCachePolicy)
     {
         CurrentMemberPathItems = SourceCache
             .Connect()
             .Filter(pathItem => Equals(pathItem.ClientInstanceId, environmentService.ClientInstanceId!))
             .AsObservableCache();
 
-        _sessionInvalidationSourceCachePolicy = sessionInvalidationSourceCachePolicy;
-        _sessionInvalidationSourceCachePolicy.Initialize(SourceCache, true, false);
+        _sessionInvalidationCachePolicy = sessionInvalidationCachePolicy;
+        _sessionInvalidationCachePolicy.Initialize(SourceCache, true, false);
     }
 
     protected override string KeySelector(PathItem pathItem) => pathItem.Key;