Improve performances on large number of files

src/ByteSync.Client/Business/Synchronizations/MultiUploadZip.cs

@@ -12,7 +12,6 @@ public MultiUploadZip(string key, SharedFileDefinition sharedFileDefinition)
         Key = key;
 
         SharedFileDefinition = sharedFileDefinition;
-        // ZipPath = zipPath;
 
         SharedFileDefinition.IsMultiFileZip = true;
         CreationDate = DateTime.Now;
@@ -21,8 +20,6 @@ public MultiUploadZip(string key, SharedFileDefinition sharedFileDefinition)
         MemoryStream = new MemoryStream();
 
         ZipArchive = new ZipArchive(MemoryStream, ZipArchiveMode.Create, true);
-
-        // ZipArchive = ZipFile.Open(zipPath, ZipArchiveMode.Update);
 
         ActionGroupsIds = new List<string>();
 
@@ -35,8 +32,6 @@ public MultiUploadZip(string key, SharedFileDefinition sharedFileDefinition)
 
     public SharedFileDefinition SharedFileDefinition { get; }
 
-    // public string ZipPath { get; }
-
     public MemoryStream MemoryStream { get; }
 
     public ZipArchive ZipArchive { get; }
@@ -54,7 +49,7 @@ public MultiUploadZip(string key, SharedFileDefinition sharedFileDefinition)
     public bool CanAdd(FileInfo fileInfo, string actionsGroupId)
     {
         return 
-            ActionGroupsIds.Count < 1500 &&
+            ActionGroupsIds.Count < 100 &&
             ActionsGroupIdsConcatenationLength + actionsGroupId.Length + 5 < 25000 &&
             Size + fileInfo.Length < 8 * SizeConstants.ONE_MEGA_BYTES;
     }

src/ByteSync.Client/DependencyInjection/Modules/GenericTypesModule.cs

@@ -12,7 +12,11 @@ public class GenericTypesModule : Module
     protected override void Load(ContainerBuilder builder)
     {
         builder.RegisterGeneric(typeof(SessionInvalidationCachePolicy<,>))
-            .As(typeof(ISessionInvalidationSourceCachePolicy<,>))
+            .As(typeof(ISessionInvalidationCachePolicy<,>))
+            .InstancePerDependency();
+
+        builder.RegisterGeneric(typeof(PropertyIndexer<,>))
+            .As(typeof(IPropertyIndexer<,>))
             .InstancePerDependency();
 
         builder.RegisterType<ConfigurationReader<ApplicationSettings>>().As<IConfigurationReader<ApplicationSettings>>();

src/ByteSync.Client/Interfaces/Controls/Communications/Http/ISynchronizationApiClient.cs

@@ -7,7 +7,7 @@ namespace ByteSync.Interfaces.Controls.Communications.Http;
 
 public interface ISynchronizationApiClient
 {
-    public Task<Synchronization> StartSynchronization(SynchronizationStartRequest synchronizationStartRequest);
+    public Task StartSynchronization(SynchronizationStartRequest synchronizationStartRequest);
 
     public Task AssertLocalCopyIsDone(string sessionId, List<string> actionsGroupIds);
 

src/ByteSync.Client/Interfaces/Controls/Synchronizations/ISynchronizationLooper.cs

@@ -2,9 +2,7 @@
 
 namespace ByteSync.Interfaces.Controls.Synchronizations;
 
-public interface ISynchronizationLooper
+public interface ISynchronizationLooper : IAsyncDisposable
 {
-    // Task InitializeData();
-
     Task CloudSessionSynchronizationLoop();
 }

src/ByteSync.Client/Interfaces/Repositories/IPropertyIndexer.cs

@@ -0,0 +1,12 @@
+using DynamicData;
+
+namespace ByteSync.Interfaces.Repositories;
+
+public interface IPropertyIndexer<TObject, in TIndex> 
+    where TObject : notnull 
+    where TIndex : notnull
+{
+    void Initialize(SourceCache<TObject, string> sourceCache, Func<TObject, TIndex> indexSelector);
+
+    List<TObject> GetByIndex(TIndex index);
+}

src/ByteSync.Client/Interfaces/Repositories/ISessionInvalidationSourceCachePolicy.cs → src/ByteSync.Client/Interfaces/Repositories/ISessionInvalidationCachePolicy.cs

@@ -2,7 +2,9 @@
 
 namespace ByteSync.Interfaces.Repositories;
 
-public interface ISessionInvalidationSourceCachePolicy<TObject, TKey> : IDisposable where TKey : notnull
+public interface ISessionInvalidationCachePolicy<TObject, TKey> : IDisposable 
+    where TKey : notnull 
+    where TObject : notnull
 {
     void Initialize(SourceCache<TObject, TKey> sourceCache, bool b, bool b1);
 }

src/ByteSync.Client/Repositories/AtomicActionRepository.cs

@@ -6,22 +6,23 @@ namespace ByteSync.Repositories;
 
 public class AtomicActionRepository : BaseSourceCacheRepository<AtomicAction, string>, IAtomicActionRepository
 {
-    private readonly ISessionInvalidationSourceCachePolicy<AtomicAction, string> _sessionInvalidationSourceCachePolicy;
+    private readonly ISessionInvalidationCachePolicy<AtomicAction, string> _sessionInvalidationCachePolicy;
+    private readonly IPropertyIndexer<AtomicAction, ComparisonItem> _propertyIndexer;
 
-    public AtomicActionRepository(ISessionInvalidationSourceCachePolicy<AtomicAction, string> sessionInvalidationSourceCachePolicy)
+    public AtomicActionRepository(ISessionInvalidationCachePolicy<AtomicAction, string> sessionInvalidationCachePolicy,
+        IPropertyIndexer<AtomicAction, ComparisonItem> propertyIndexer)
     {
-        _sessionInvalidationSourceCachePolicy = sessionInvalidationSourceCachePolicy;
-        _sessionInvalidationSourceCachePolicy.Initialize(SourceCache, true, true);
+        _sessionInvalidationCachePolicy = sessionInvalidationCachePolicy;
+        _sessionInvalidationCachePolicy.Initialize(SourceCache, true, true);
+
+        _propertyIndexer = propertyIndexer;
+        _propertyIndexer.Initialize(SourceCache, atomicAction => atomicAction.ComparisonItem!);
     }
-    
+
     protected override string KeySelector(AtomicAction atomicAction) => atomicAction.AtomicActionId;
 
     public List<AtomicAction> GetAtomicActions(ComparisonItem comparisonItem)
     {
-        var result = SourceCache.Items
-            .Where(atomicAction => Equals(atomicAction.ComparisonItem, comparisonItem))
-            .ToList();
-
-        return result;
+        return _propertyIndexer.GetByIndex(comparisonItem);
     }
-}
+}

src/ByteSync.Client/Repositories/BaseSourceCacheRepository.cs

@@ -4,7 +4,7 @@
 namespace ByteSync.Repositories;
 
 public abstract class BaseSourceCacheRepository<TObject, TKey> : IBaseSourceCacheRepository<TObject, TKey> 
-    where TKey : notnull 
+    where TKey : notnull where TObject : notnull
 {
     protected BaseSourceCacheRepository()
     {

src/ByteSync.Client/Repositories/InventoryFileRepository.cs

@@ -6,12 +6,12 @@ namespace ByteSync.Repositories;
 
 public class InventoryFileRepository : BaseSourceCacheRepository<InventoryFile, string>, IInventoryFileRepository
 {
-    private readonly ISessionInvalidationSourceCachePolicy<InventoryFile, string> _sessionInvalidationSourceCachePolicy;
+    private readonly ISessionInvalidationCachePolicy<InventoryFile, string> _sessionInvalidationCachePolicy;
 
-    public InventoryFileRepository(ISessionInvalidationSourceCachePolicy<InventoryFile, string> sessionInvalidationSourceCachePolicy)
+    public InventoryFileRepository(ISessionInvalidationCachePolicy<InventoryFile, string> sessionInvalidationCachePolicy)
     {
-        _sessionInvalidationSourceCachePolicy = sessionInvalidationSourceCachePolicy;
-        _sessionInvalidationSourceCachePolicy.Initialize(SourceCache, true, true);
+        _sessionInvalidationCachePolicy = sessionInvalidationCachePolicy;
+        _sessionInvalidationCachePolicy.Initialize(SourceCache, true, true);
     }
 
     protected override string KeySelector(InventoryFile inventoryFile) => inventoryFile.FullName;

src/ByteSync.Client/Repositories/PathItemRepository.cs

@@ -7,17 +7,17 @@ namespace ByteSync.Repositories;
 
 public class PathItemRepository : BaseSourceCacheRepository<PathItem, string>, IPathItemRepository
 {
-    private readonly ISessionInvalidationSourceCachePolicy<PathItem, string> _sessionInvalidationSourceCachePolicy;
+    private readonly ISessionInvalidationCachePolicy<PathItem, string> _sessionInvalidationCachePolicy;
 
-    public PathItemRepository(IEnvironmentService environmentService, ISessionInvalidationSourceCachePolicy<PathItem, string> sessionInvalidationSourceCachePolicy)
+    public PathItemRepository(IEnvironmentService environmentService, ISessionInvalidationCachePolicy<PathItem, string> sessionInvalidationCachePolicy)
     {
         CurrentMemberPathItems = SourceCache
             .Connect()
             .Filter(pathItem => Equals(pathItem.ClientInstanceId, environmentService.ClientInstanceId!))
             .AsObservableCache();
 
-        _sessionInvalidationSourceCachePolicy = sessionInvalidationSourceCachePolicy;
-        _sessionInvalidationSourceCachePolicy.Initialize(SourceCache, true, false);
+        _sessionInvalidationCachePolicy = sessionInvalidationCachePolicy;
+        _sessionInvalidationCachePolicy.Initialize(SourceCache, true, false);
     }
 
     protected override string KeySelector(PathItem pathItem) => pathItem.Key;

src/ByteSync.Client/Repositories/PropertyIndexer.cs

@@ -0,0 +1,80 @@
+using ByteSync.Interfaces.Repositories;
+using DynamicData;
+
+namespace ByteSync.Repositories;
+
+public class PropertyIndexer<TObject, TIndex> : IPropertyIndexer<TObject, TIndex> 
+    where TObject : notnull 
+    where TIndex : notnull
+{
+    private Func<TObject, TIndex> _indexSelector;
+    private readonly Dictionary<TIndex, List<TObject>> _cache = new();
+
+    public void Initialize(SourceCache<TObject, string> sourceCache, Func<TObject, TIndex> indexSelector)
+    {
+        _indexSelector = indexSelector;
+
+        // Initializing the cache with existing objects
+        foreach (var obj in sourceCache.Items)
+        {
+            Update(obj);
+        }
+
+        // Synchronization with SourceCache
+        sourceCache.Connect()
+            .Subscribe(changes =>
+            {
+                foreach (var change in changes)
+                {
+                    switch (change.Reason)
+                    {
+                        case ChangeReason.Add:
+                        case ChangeReason.Update:
+                            Update(change.Current);
+                            break;
+                        case ChangeReason.Remove:
+                            Remove(change.Current);
+                            break;
+                    }
+                }
+            });
+    }
+
+    public void Update(TObject obj)
+    {
+        var index = _indexSelector(obj);
+
+        if (!_cache.TryGetValue(index, out var objects))
+        {
+            objects = new List<TObject>();
+            _cache[index] = objects;
+        }
+
+        var existingObject = objects.FirstOrDefault(o => o.Equals(obj));
+        if (existingObject != null)
+        {
+            objects.Remove(existingObject);
+        }
+
+        objects.Add(obj);
+    }
+
+    public void Remove(TObject obj)
+    {
+        var index = _indexSelector(obj);
+
+        if (_cache.TryGetValue(index, out var objects))
+        {
+            objects.RemoveAll(o => o.Equals(obj));
+            if (objects.Count == 0)
+            {
+                _cache.Remove(index);
+            }
+        }
+    }
+
+    public List<TObject> GetByIndex(TIndex index)
+    {
+        return _cache.TryGetValue(index, out var objects) ? objects : new List<TObject>();
+    }
+}

src/ByteSync.Client/Repositories/SessionInvalidationCachePolicy.cs

@@ -6,7 +6,9 @@
 
 namespace ByteSync.Repositories;
 
-public class SessionInvalidationCachePolicy<TObject, TKey> : ISessionInvalidationSourceCachePolicy<TObject, TKey> where TKey : notnull
+public class SessionInvalidationCachePolicy<TObject, TKey> : ISessionInvalidationCachePolicy<TObject, TKey> 
+    where TKey : notnull 
+    where TObject : notnull
 {
     private readonly ISessionService _sessionService;
     private IDisposable? _sessionSubscription;

src/ByteSync.Client/Repositories/SessionMemberRepository.cs

@@ -14,10 +14,10 @@ public class SessionMemberRepository : BaseSourceCacheRepository<SessionMemberIn
     private readonly IConnectionService _connectionService;
     private readonly ReadOnlyObservableCollection<SessionMemberInfo> _sortedSessionMembersList;
     private readonly ReadOnlyObservableCollection<SessionMemberInfo> _sortedOtherSessionMembersList;
-    private readonly ISessionInvalidationSourceCachePolicy<SessionMemberInfo, string> _sessionInvalidationSourceCachePolicy;
+    private readonly ISessionInvalidationCachePolicy<SessionMemberInfo, string> _sessionInvalidationCachePolicy;
 
     public SessionMemberRepository(IConnectionService connectionService, 
-        ISessionInvalidationSourceCachePolicy<SessionMemberInfo, string> sessionInvalidationSourceCachePolicy)
+        ISessionInvalidationCachePolicy<SessionMemberInfo, string> sessionInvalidationCachePolicy)
     {
         _connectionService = connectionService;
 
@@ -34,8 +34,8 @@ public SessionMemberRepository(IConnectionService connectionService,
         IsCurrentUserFirstSessionMemberObservable
             .Subscribe(value => IsCurrentUserFirstSessionMemberCurrentValue = value);
 
-        _sessionInvalidationSourceCachePolicy = sessionInvalidationSourceCachePolicy;
-        _sessionInvalidationSourceCachePolicy.Initialize(SourceCache, true, false);
+        _sessionInvalidationCachePolicy = sessionInvalidationCachePolicy;
+        _sessionInvalidationCachePolicy.Initialize(SourceCache, true, false);
     }
 
     protected override string KeySelector(SessionMemberInfo sessionMemberInfo) => sessionMemberInfo.ClientInstanceId;

src/ByteSync.Client/Repositories/SharedActionsGroupRepository.cs

@@ -9,14 +9,14 @@ namespace ByteSync.Repositories;
 
 public class SharedActionsGroupRepository : BaseSourceCacheRepository<SharedActionsGroup, string>, ISharedActionsGroupRepository
 {
-    private readonly ISessionInvalidationSourceCachePolicy<SharedActionsGroup, string> _sessionInvalidationSourceCachePolicy;
+    private readonly ISessionInvalidationCachePolicy<SharedActionsGroup, string> _sessionInvalidationCachePolicy;
 
-    public SharedActionsGroupRepository(ISessionInvalidationSourceCachePolicy<SharedActionsGroup, string> sessionInvalidationSourceCachePolicy)
+    public SharedActionsGroupRepository(ISessionInvalidationCachePolicy<SharedActionsGroup, string> sessionInvalidationCachePolicy)
     {
         OrganizedSharedActionsGroups = new List<SharedActionsGroup>();
 
-        _sessionInvalidationSourceCachePolicy = sessionInvalidationSourceCachePolicy;
-        _sessionInvalidationSourceCachePolicy.Initialize(SourceCache, true, true);
+        _sessionInvalidationCachePolicy = sessionInvalidationCachePolicy;
+        _sessionInvalidationCachePolicy.Initialize(SourceCache, true, true);
     }
 
     protected override string KeySelector(SharedActionsGroup sharedAtomicAction) => sharedAtomicAction.ActionsGroupId;

src/ByteSync.Client/Repositories/SharedAtomicActionRepository.cs

@@ -6,12 +6,12 @@ namespace ByteSync.Repositories;
 
 public class SharedAtomicActionRepository : BaseSourceCacheRepository<SharedAtomicAction, string>, ISharedAtomicActionRepository
 {
-    private readonly ISessionInvalidationSourceCachePolicy<SharedAtomicAction, string> _sessionInvalidationSourceCachePolicy;
+    private readonly ISessionInvalidationCachePolicy<SharedAtomicAction, string> _sessionInvalidationCachePolicy;
 
-    public SharedAtomicActionRepository(ISessionInvalidationSourceCachePolicy<SharedAtomicAction, string> sessionInvalidationSourceCachePolicy)
+    public SharedAtomicActionRepository(ISessionInvalidationCachePolicy<SharedAtomicAction, string> sessionInvalidationCachePolicy)
     {
-        _sessionInvalidationSourceCachePolicy = sessionInvalidationSourceCachePolicy;
-        _sessionInvalidationSourceCachePolicy.Initialize(SourceCache, true, true);
+        _sessionInvalidationCachePolicy = sessionInvalidationCachePolicy;
+        _sessionInvalidationCachePolicy.Initialize(SourceCache, true, true);
     }
 
     protected override string KeySelector(SharedAtomicAction sharedAtomicAction) => sharedAtomicAction.AtomicActionId;

src/ByteSync.Client/Repositories/SynchronizationRuleRepository.cs

@@ -5,12 +5,12 @@ namespace ByteSync.Repositories;
 
 public class SynchronizationRuleRepository : BaseSourceCacheRepository<SynchronizationRule, string>, ISynchronizationRuleRepository
 {
-    private readonly ISessionInvalidationSourceCachePolicy<SynchronizationRule, string> _sessionInvalidationSourceCachePolicy;
+    private readonly ISessionInvalidationCachePolicy<SynchronizationRule, string> _sessionInvalidationCachePolicy;
 
-    public SynchronizationRuleRepository(ISessionInvalidationSourceCachePolicy<SynchronizationRule, string> sessionInvalidationSourceCachePolicy)
+    public SynchronizationRuleRepository(ISessionInvalidationCachePolicy<SynchronizationRule, string> sessionInvalidationCachePolicy)
     {
-        _sessionInvalidationSourceCachePolicy = sessionInvalidationSourceCachePolicy;
-        _sessionInvalidationSourceCachePolicy.Initialize(SourceCache, true, true);
+        _sessionInvalidationCachePolicy = sessionInvalidationCachePolicy;
+        _sessionInvalidationCachePolicy.Initialize(SourceCache, true, true);
     }
 
     protected override string KeySelector(SynchronizationRule synchronizationRule) => synchronizationRule.SynchronizationRuleId;