Implement more gentle depth-1 paginated PROPFIND approach for working set enumeration

Sources/NextcloudFileProviderKit/Enumeration/Enumerator.swift

@@ -29,12 +29,12 @@
     private let anchor = NSFileProviderSyncAnchor(Date().description.data(using: .utf8)!)
     private let pageItemCount: Int
     private var pageNum = 0
+    private var nextFoldersServerUrlsToEnumerateWorkingSet = [String]()
     static let logger = Logger(subsystem: Logger.subsystem, category: "enumerator")
     let account: Account
     let remoteInterface: RemoteInterface
-    var serverUrl: String = ""
     var isInvalidated = false
-    weak var listener: EnumerationListener?
+    private(set) var serverUrl: String = ""
 
     private static func isSystemIdentifier(_ identifier: NSFileProviderItemIdentifier) -> Bool {
         identifier == .rootContainer || identifier == .trashContainer || identifier == .workingSet
@@ -46,15 +46,13 @@
         remoteInterface: RemoteInterface,
         dbManager: FilesDatabaseManager,
         domain: NSFileProviderDomain? = nil,
-        listener: EnumerationListener? = nil,
         pageSize: Int = 100
     ) {
         self.enumeratedItemIdentifier = enumeratedItemIdentifier
         self.remoteInterface = remoteInterface
         self.account = account
         self.dbManager = dbManager
         self.domain = domain
-        self.listener = listener
         self.pageItemCount = pageSize
 
         if Self.isSystemIdentifier(enumeratedItemIdentifier) {
@@ -112,9 +110,6 @@
     public func enumerateItems(
         for observer: NSFileProviderEnumerationObserver, startingAt page: NSFileProviderPage
     ) {
-        let actionId = UUID()
-        listener?.enumerationActionStarted(actionId: actionId)
-
         Self.logger.debug(
             """
             Received enumerate items request for enumerator with user:
@@ -182,7 +177,6 @@
                     let error = trashReadError.fileProviderError(
                         handlingNoSuchItemErrorUsingItemIdentifier: self.enumeratedItemIdentifier
                     ) ?? NSFileProviderError(.cannotSynchronize)
-                    listener?.enumerationActionFailed(actionId: actionId, error: error)
                     observer.finishEnumeratingWithError(error)
                     return
                 }
@@ -195,7 +189,6 @@
                     numPage: 1,
                     trashItems: trashedItems
                 )
-                listener?.enumerationActionFinished(actionId: actionId)
             }
             return
         }
@@ -214,7 +207,6 @@
             let error = NSError.fileProviderErrorForNonExistentItem(
                 withIdentifier: self.enumeratedItemIdentifier
             )
-            listener?.enumerationActionFailed(actionId: actionId, error: error)
             observer.finishEnumeratingWithError(error)
             return
         }
@@ -228,24 +220,42 @@
         )
 
         Task {
+            var providedPage: NSFileProviderPage? = nil // Used for pagination token sent to server
+            var pageNum = pageNum // If we are paginating a new target server URL, reset to 0
             // Do not pass in the NSFileProviderPage default pages, these are not valid Nextcloud
             // pagination tokens
-            var providedPage: NSFileProviderPage? = nil
             if page != NSFileProviderPage.initialPageSortedByName as NSFileProviderPage &&
                page != NSFileProviderPage.initialPageSortedByDate as NSFileProviderPage
             {
-                providedPage = page
+                if let pageString = String(data: page.rawValue, encoding: .utf8),
+                   let urlDetector = try? NSDataDetector(
+                    types: NSTextCheckingResult.CheckingType.link.rawValue
+                   ),
+                   !urlDetector.matches(
+                    in: pageString, options: [], range: .init(location: 0, length: pageString.count)
+                   ).isEmpty
+                {
+                    Self.logger.info(
+                        "Setting enumerator server URL to \(self.serverUrl, privacy: .public)"
+                    )
+                    serverUrl = pageString
+                    pageNum = 0
+                } else {
+                    providedPage = page
+                }
             }
-            let depth: EnumerateDepth = enumeratedItemIdentifier == .workingSet ?
-                .targetAndAllChildren : .targetAndDirectChildren
-            let (metadatas, _, _, _, nextPage, readError) = await Self.readServerUrl(
+
+            let readResult = await Self.readServerUrl(
                 serverUrl,
                 pageSettings: (page: providedPage, index: pageNum, size: pageItemCount),
                 account: account,
                 remoteInterface: remoteInterface,
                 dbManager: dbManager,
-                depth: depth
+                depth: .targetAndDirectChildren
             )
+            let metadatas = readResult.metadatas
+            let readError = readResult.readError
+            var nextPage = readResult.nextPage
 
             guard readError == nil else {
                 Self.logger.error(
@@ -261,7 +271,6 @@
                 let error = readError?.fileProviderError(
                     handlingNoSuchItemErrorUsingItemIdentifier: self.enumeratedItemIdentifier
                 ) ?? NSFileProviderError(.cannotSynchronize)
-                listener?.enumerationActionFailed(actionId: actionId, error: error)
                 observer.finishEnumeratingWithError(error)
                 return
             }
@@ -270,42 +279,50 @@
                 Self.logger.error(
                     """
                     Finishing enumeration for: \(self.account.ncKitAccount, privacy: .public)
-                    with serverUrl: \(self.serverUrl, privacy: .public)
-                    with invalid metadatas.
+                        with serverUrl: \(self.serverUrl, privacy: .public)
+                        with invalid metadatas.
                     """
                 )
-                listener?.enumerationActionFailed(
-                    actionId: actionId, error: NSFileProviderError(.cannotSynchronize)
-                )
                 observer.finishEnumeratingWithError(NSFileProviderError(.cannotSynchronize))
                 return
             }
 
+            if enumeratedItemIdentifier == .workingSet {
+                for metadata in metadatas where metadata.directory {
+                    let metadataRemoteUrl = metadata.serverUrl + "/" + metadata.fileName
+                    nextFoldersServerUrlsToEnumerateWorkingSet.append(metadataRemoteUrl)
+                }
+
+                // If we have finished paged enumeration of the current serverUrl, move to next
+                // child to scan
+                if nextPage == nil && !nextFoldersServerUrlsToEnumerateWorkingSet.isEmpty {
+                    let nextServerUrl = nextFoldersServerUrlsToEnumerateWorkingSet.removeFirst()
+                    nextPage = EnumeratorPageResponse(nextServerUrl: nextServerUrl)
+                }
+            }
+
             Self.logger.info(
                 """
                 Finished reading page: \(self.pageNum, privacy: .public)
                     serverUrl: \(self.serverUrl, privacy: .public)
                     for user: \(self.account.ncKitAccount, privacy: .public).
+                    Next page token is: \(nextPage?.token ?? "", privacy: .public)
                     Processed \(metadatas.count) metadatas
                 """
             )
 
             completeEnumerationObserver(observer, nextPage: nextPage, itemMetadatas: metadatas)
-            listener?.enumerationActionFinished(actionId: actionId)
         }
     }
 
     public func enumerateChanges(
         for observer: NSFileProviderChangeObserver, from anchor: NSFileProviderSyncAnchor
     ) {
-        let actionId = UUID()
-        listener?.enumerationActionStarted(actionId: actionId)
-
         Self.logger.debug(
             """
-            Received enumerate changes request for enumerator for user:
-            \(self.account.ncKitAccount, privacy: .public)
-            with serverUrl: \(self.serverUrl, privacy: .public)
+            Received enumerate changes request for enumerator
+                for user: \(self.account.ncKitAccount, privacy: .public)
+                with serverUrl: \(self.serverUrl, privacy: .public)
             """
         )
         /*
@@ -320,7 +337,7 @@
 
         if enumeratedItemIdentifier == .workingSet {
             Self.logger.debug(
-                "Enumerating changes in working set for: \(self.account.ncKitAccount, privacy: .public)"
+                "Enumerating working set changes for \(self.account.ncKitAccount, privacy: .public)"
             )
 
             // Unlike when enumerating items we can't progressively enumerate items as we need to 
@@ -342,9 +359,6 @@
                         For user: \(self.account.ncKitAccount, privacy: .public)
                         """
                     )
-                    listener?.enumerationActionFailed(
-                        actionId: actionId, error: NSFileProviderError(.cannotSynchronize)
-                    )
                     observer.finishEnumeratingWithError(NSFileProviderError(.cannotSynchronize))
                     return
                 }
@@ -361,7 +375,6 @@
                     let fpError = error?.fileProviderError(
                         handlingNoSuchItemErrorUsingItemIdentifier: self.enumeratedItemIdentifier
                     ) ?? NSFileProviderError(.cannotSynchronize)
-                    listener?.enumerationActionFailed(actionId: actionId, error: fpError)
                     observer.finishEnumeratingWithError(fpError)
                     return
                 }
@@ -384,7 +397,6 @@
                     updatedMetadatas: updatedMetadatas,
                     deletedMetadatas: deletedMetadatas
                 )
-                listener?.enumerationActionFinished(actionId: actionId)
             }
             return
         } else if enumeratedItemIdentifier == .trashContainer {
@@ -431,7 +443,6 @@
                     let error = trashReadError.fileProviderError(
                         handlingNoSuchItemErrorUsingItemIdentifier: self.enumeratedItemIdentifier
                     ) ?? NSFileProviderError(.cannotSynchronize)
-                    listener?.enumerationActionFailed(actionId: actionId, error: error)
                     observer.finishEnumeratingWithError(error)
                     return
                 }
@@ -444,7 +455,6 @@
                     dbManager: dbManager,
                     trashItems: trashedItems
                 )
-                listener?.enumerationActionFinished(actionId: actionId)
             }
             return
         }
@@ -504,7 +514,6 @@
                             Could not delete metadata nor report deletion.
                             """
                         )
-                        listener?.enumerationActionFailed(actionId: actionId, error: error)
                         observer.finishEnumeratingWithError(error)
                         return
                     }
@@ -538,7 +547,6 @@
                         updatedMetadatas: nil,
                         deletedMetadatas: [itemMetadata]
                     )
-                    listener?.enumerationActionFinished(actionId: actionId)
                     return
                 } else if readError!.isNoChangesError {  // All is well, just no changed etags
                     Self.logger.info(
@@ -547,12 +555,10 @@
                         Finishing change enumeration.
                         """
                     )
-                    listener?.enumerationActionFinished(actionId: actionId)
                     observer.finishEnumeratingChanges(upTo: anchor, moreComing: false)
                     return
                 }
 
-                listener?.enumerationActionFailed(actionId: actionId, error: error)
                 observer.finishEnumeratingWithError(error)
                 return
             }
@@ -575,7 +581,6 @@
                 updatedMetadatas: updatedMetadatas,
                 deletedMetadatas: deletedMetadatas
             )
-            listener?.enumerationActionFinished(actionId: actionId)
         }
     }
 

Sources/NextcloudFileProviderKit/Enumeration/EnumeratorPageResponse.swift

@@ -20,7 +20,6 @@ struct EnumeratorPageResponse: Sendable {
 
         let normalisedHeaders =
             Dictionary(uniqueKeysWithValues: headers.map { ($0.key.lowercased(), $0.value) })
-        print(normalisedHeaders)
         guard Bool(normalisedHeaders["x-nc-paginate"]?.lowercased() ?? "false") == true,
               let responsePaginateToken = normalisedHeaders["x-nc-paginate-token"]
         else { return nil }
@@ -33,4 +32,10 @@ struct EnumeratorPageResponse: Sendable {
             total = nil
         }
     }
+
+    init(nextServerUrl: String) {
+        self.token = nextServerUrl
+        self.index = -1
+        self.total = nil
+    }
 }

Tests/Interface/MockRemoteInterface.swift

@@ -1083,6 +1083,7 @@ public class MockRemoteInterface: RemoteInterface {
             }
             let reachedEnd = firstItem + itemCount >= files.count
             let lastItem = min(firstItem + itemCount, files.count) - 1
+            assert(firstItem <= lastItem)
             let itemsPage = Array(files[firstItem...lastItem])
             let responseData = generateResponse(itemCount: files.count, finalPage: reachedEnd)
             return (account.ncKitAccount, itemsPage, responseData, .success)