Merge pull request #88 from claucambra/bugfix/pagination-pt4

Further pagination-related fixes

Author: claucambra

Sources/NextcloudFileProviderKit/Database/FilesDatabaseManager.swift

@@ -452,6 +452,7 @@ public final class FilesDatabaseManager: Sendable {
                         parentDirectoryUrl: \(metadata.serverUrl, privacy: .public)
                         account: \(metadata.account, privacy: .public)
                         content type: \(metadata.contentType, privacy: .public)
+                        is directory: \(metadata.directory, privacy: .public)
                         creation date: \(metadata.creationDate, privacy: .public)
                         date: \(metadata.date, privacy: .public)
                         lock: \(metadata.lock, privacy: .public)

Sources/NextcloudFileProviderKit/Enumeration/Enumerator+SyncEngine.swift

@@ -274,7 +274,10 @@ extension Enumerator {
         if pageIndex == 0 {
             guard let firstFile = files.first else { return (nil, .invalidResponseError) }
             // Do not ingest metadata for the root container
-            if !firstFile.fullUrlMatches(dbManager.account.davFilesUrl) {
+            if !firstFile.fullUrlMatches(dbManager.account.davFilesUrl),
+               !firstFile.fullUrlMatches(dbManager.account.davFilesUrl + "/."),
+               !(firstFile.fileName == "." && firstFile.serverUrl == "..")
+            {
                 var metadata = firstFile.toItemMetadata()
                 if metadata.directory,
                    let existingMetadata = dbManager.itemMetadata(ocId: metadata.ocId)
@@ -388,6 +391,9 @@ extension Enumerator {
                 at depth \(depth.rawValue, privacy: .public).
                 username: \(account.username, privacy: .public),
                 password is empty: \(account.password == "" ? "EMPTY" : "NOT EMPTY"),
+                pageToken: \(String(data: pageSettings?.page?.rawValue ?? Data(), encoding: .utf8) ?? "NIL", privacy: .public)
+                pageIndex: \(pageSettings?.index ?? -1, privacy: .public)
+                pageSize: \(pageSettings?.size ?? -1, privacy: .public)
                 serverUrl: \(account.serverUrl, privacy: .public)
             """
         )
@@ -452,7 +458,7 @@ extension Enumerator {
             Self.logger.error(
                 """
                 Received no items from readFileOrFolder of \(serverUrl, privacy: .public),
-                not much we can do...
+                    not much we can do...
                 """
             )
             return (nil, nil, nil, nil, nextPage, error)
@@ -463,7 +469,9 @@ extension Enumerator {
         let isFollowUpPaginatedRequest = (pageSettings?.page != nil && pageSettings?.index ?? 0 > 0)
         if !isFollowUpPaginatedRequest {
             guard receivedFile.directory ||
-                  receivedFile.fullUrlMatches(dbManager.account.davFilesUrl)
+                  serverUrl == dbManager.account.davFilesUrl ||
+                  receivedFile.fullUrlMatches(dbManager.account.davFilesUrl + "/.") ||
+                  (receivedFile.fileName == "." && receivedFile.serverUrl == "..")
             else {
                 Self.logger.debug(
                     """

Sources/NextcloudFileProviderKit/Enumeration/Enumerator.swift

@@ -214,6 +214,7 @@ public class Enumerator: NSObject, NSFileProviderEnumerator {
         Self.logger.debug(
             """
             Enumerating page: \(self.pageNum, privacy: .public)
+                with data: \(String(data: page.rawValue, encoding: .utf8) ?? "", privacy: .public)
                 for user: \(self.account.ncKitAccount, privacy: .public)
                 with serverUrl: \(self.serverUrl, privacy: .public)
             """
@@ -291,12 +292,23 @@ public class Enumerator: NSObject, NSFileProviderEnumerator {
                     let metadataRemoteUrl = metadata.serverUrl + "/" + metadata.fileName
                     nextFoldersServerUrlsToEnumerateWorkingSet.append(metadataRemoteUrl)
                 }
+                let nextPageValid = nextPage != nil
+                let nextPageToken = nextPage?.token ?? "" // For logging
+                Self.logger.debug(
+                    """
+                    Current folders awaiting paged enumeration:
+                        \(self.nextFoldersServerUrlsToEnumerateWorkingSet, privacy: .public)
+                        next page is valid: \(nextPageValid, privacy: .public)
+                        next page token: \(nextPageToken, privacy: .public)
+                    """
+                )
 
                 // If we have finished paged enumeration of the current serverUrl, move to next
                 // child to scan
-                if nextPage == nil && !nextFoldersServerUrlsToEnumerateWorkingSet.isEmpty {
+                if !nextPageValid && !nextFoldersServerUrlsToEnumerateWorkingSet.isEmpty {
                     let nextServerUrl = nextFoldersServerUrlsToEnumerateWorkingSet.removeFirst()
                     nextPage = EnumeratorPageResponse(nextServerUrl: nextServerUrl)
+                    Self.logger.info("Next page token set to be \(nextServerUrl, privacy: .public)")
                 }
             }
 
@@ -305,7 +317,7 @@ public class Enumerator: NSObject, NSFileProviderEnumerator {
                 Finished reading page: \(self.pageNum, privacy: .public)
                     serverUrl: \(self.serverUrl, privacy: .public)
                     for user: \(self.account.ncKitAccount, privacy: .public).
-                    Next page token is: \(nextPage?.token ?? "", privacy: .public)
+                    Next page token is: \(nextPage?.token ?? "NIL", privacy: .public)
                     Processed \(metadatas.count) metadatas
                 """
             )
@@ -609,6 +621,14 @@ public class Enumerator: NSObject, NSFileProviderEnumerator {
                 Task { @MainActor in
                     observer.didEnumerate(items)
                     Self.logger.info("Did enumerate \(items.count) items")
+                    let nextPageNil = nextPage == nil // For logging
+                    let nextPageToken = nextPage?.token ?? "" // For logging
+                    Self.logger.info(
+                        """
+                        Next page is nil: \(nextPageNil, privacy: .public)
+                            next page token: \(nextPageToken, privacy: .public)
+                        """
+                    )
                     if let nextPage, let nextPageData = nextPage.token.data(using: .utf8) {
                         self.pageNum = nextPage.index
                         observer.finishEnumerating(upTo: NSFileProviderPage(nextPageData))

Sources/NextcloudFileProviderKit/Enumeration/EnumeratorPageResponse.swift

@@ -7,22 +7,30 @@
 
 import Alamofire
 import Foundation
+import OSLog
 
 struct EnumeratorPageResponse: Sendable {
     let token: String   // Required by server to serve the next page of items
     let index: Int      // Needed to calculate the offset for the next paginated request
     let total: Int?     // Total item count, provided in the first non-offset paginated response
+    let logger = Logger(subsystem: Logger.subsystem, category: "enumeratorpageresponse")
 
     init?(nkResponseData: AFDataResponse<Data>?, index: Int) {
         guard let headers = nkResponseData?.response?.allHeaderFields as? [String: String] else {
+            logger.debug("Page response nil as could not get header fields from Alamofire response")
             return nil
         }
 
         let normalisedHeaders =
             Dictionary(uniqueKeysWithValues: headers.map { ($0.key.lowercased(), $0.value) })
         guard Bool(normalisedHeaders["x-nc-paginate"]?.lowercased() ?? "false") == true,
               let responsePaginateToken = normalisedHeaders["x-nc-paginate-token"]
-        else { return nil }
+        else {
+            logger.debug(
+                "Page response nil, page headers missing. \(normalisedHeaders, privacy: .public)"
+            )
+            return nil
+        }
 
         self.index = index
         token = responsePaginateToken
@@ -31,9 +39,19 @@ struct EnumeratorPageResponse: Sendable {
         } else {
             total = nil
         }
+        let totalString = total != nil ? String(total ?? -1) : "nil"
+        logger.debug(
+            """
+            Created enumerator page response with data from PROPFIND reply.
+                token: \(responsePaginateToken, privacy: .public)
+                index: \(index, privacy: .public)
+                total: \(totalString, privacy: .public)
+            """
+        )
     }
 
     init(nextServerUrl: String) {
+        logger.debug("Creating artificial page response with \(nextServerUrl, privacy: .public)")
         self.token = nextServerUrl
         self.index = -1
         self.total = nil

Tests/Interface/MockRemoteItem.swift

@@ -60,7 +60,7 @@ public class MockRemoteItem: Equatable {
         MockRemoteItem(
             identifier: NSFileProviderItemIdentifier.rootContainer.rawValue,
             versionIdentifier: "root",
-            name: "",
+            name: ".",
             remotePath: account.davFilesUrl,
             directory: true,
             account: account.ncKitAccount,