Implement mitigations for "parent item identifier not found" errors

Sources/NextcloudFileProviderKit/Database/FilesDatabaseManager.swift

@@ -24,12 +24,25 @@
 public let databaseFilename = "fileproviderextdatabase.realm"
 
 public final class FilesDatabaseManager: Sendable {
-    enum ErrorCode: Int {
+    public enum ErrorCode: Int {
         case metadataNotFound = -1000
+        case parentMetadataNotFound = -1001
+    }
+    public enum ErrorUserInfoKey: String {
+        case missingParentServerUrlAndFileName = "MissingParentServerUrlAndFileName"
+    }
+    static let errorDomain = "FilesDatabaseManager"
+    static func error(code: ErrorCode, userInfo: [String: String]) -> NSError {
+        NSError(domain: Self.errorDomain, code: code.rawValue, userInfo: userInfo)
+    }
+    static func parentMetadataNotFoundError(itemUrl: String) -> NSError {
+        error(
+            code: .parentMetadataNotFound,
+            userInfo: [ErrorUserInfoKey.missingParentServerUrlAndFileName.rawValue: itemUrl]
+        )
     }
 
     private static let schemaVersion = stable2_0SchemaVersion
-    static let errorDomain = "FilesDatabaseManager"
     static let logger = Logger(subsystem: Logger.subsystem, category: "filesdatabase")
     let account: Account
 
@@ -553,34 +566,50 @@
             return .trashContainer
         }
 
-        guard let itemParentDirectory = parentDirectoryMetadataForItem(metadata) else {
+        guard let parentDirectoryMetadata = parentDirectoryMetadataForItem(metadata) else {
             Self.logger.error(
                 """
-                Could not get item parent directory metadata for metadata.
+                Could not get item parent directory item metadata for metadata.
                     ocID: \(metadata.ocId, privacy: .public),
-                    etag: \(metadata.etag, privacy: .public),
+                    etag: \(metadata.etag, privacy: .public), 
                     fileName: \(metadata.fileName, privacy: .public),
                     serverUrl: \(metadata.serverUrl, privacy: .public),
                     account: \(metadata.account, privacy: .public),
                 """
             )
             return nil
         }
+        return NSFileProviderItemIdentifier(parentDirectoryMetadata.ocId)
+    }
 
-        if let parentDirectoryMetadata = itemMetadata(ocId: itemParentDirectory.ocId) {
-            return NSFileProviderItemIdentifier(parentDirectoryMetadata.ocId)
+    public func parentItemIdentifierWithRemoteFallback(
+        fromMetadata metadata: SendableItemMetadata,
+        remoteInterface: RemoteInterface,
+        account: Account
+    ) async -> NSFileProviderItemIdentifier? {
+        if let parentItemIdentifier = parentItemIdentifierFromMetadata(metadata) {
+            return parentItemIdentifier
         }
 
-        Self.logger.error(
-            """
-            Could not get item parent directory item metadata for metadata.
-                ocID: \(metadata.ocId, privacy: .public),
-                etag: \(metadata.etag, privacy: .public), 
-                fileName: \(metadata.fileName, privacy: .public),
-                serverUrl: \(metadata.serverUrl, privacy: .public),
-                account: \(metadata.account, privacy: .public),
-            """
+        let (metadatas, _, _, _, error) = await Enumerator.readServerUrl(
+            metadata.serverUrl,
+            account: account,
+            remoteInterface: remoteInterface,
+            dbManager: self,
+            depth: .target
         )
-        return nil
+        guard error == nil, let parentMetadata = metadatas?.first else {
+            Self.logger.error(
+                """
+                Could not retrieve parent item identifier remotely, received error.
+                    target metadata: \(metadata.ocId, privacy: .public)
+                    target filename: \(metadata.fileName, privacy: .public)
+                    received metadatas: \(metadatas?.count ?? 0, privacy: .public)
+                    error: \(error?.errorDescription ?? "NO ERROR", privacy: .public)
+                """
+            )
+            return nil
+        }
+        return NSFileProviderItemIdentifier(parentMetadata.ocId)
     }
 }

Sources/NextcloudFileProviderKit/Enumeration/Enumerator+Trash.swift

@@ -25,14 +25,18 @@
         }
 
         Task { [metadatas] in
-            let items = await metadatas.toFileProviderItems(
-                account: account, remoteInterface: remoteInterface, dbManager: dbManager
-            )
-
-            Task { @MainActor in
-                observer.didEnumerate(items)
-                Self.logger.info("Did enumerate \(items.count) trash items")
-                observer.finishEnumerating(upTo: fileProviderPageforNumPage(numPage))
+            do {
+                let items = try await metadatas.toFileProviderItems(
+                    account: account, remoteInterface: remoteInterface, dbManager: dbManager
+                )
+                Task { @MainActor in
+                    observer.didEnumerate(items)
+                    Self.logger.info("Did enumerate \(items.count) trash items")
+                    observer.finishEnumerating(upTo: fileProviderPageforNumPage(numPage))
+                }
+            } catch let error {
+                Self.logger.info("Unexpected error enumerating trash items, observing error.")
+                Task { @MainActor in observer.finishEnumeratingWithError(error) }
             }
         }
     }

Sources/NextcloudFileProviderKit/Enumeration/Enumerator.swift

@@ -612,28 +612,56 @@
         remoteInterface: RemoteInterface,
         dbManager: FilesDatabaseManager,
         numPage: Int,
-        itemMetadatas: [SendableItemMetadata]
+        itemMetadatas: [SendableItemMetadata],
+        handleInvalidParent: Bool = true
     ) {
         Task {
-            let items = await itemMetadatas.toFileProviderItems(
-                account: account, remoteInterface: remoteInterface, dbManager: dbManager
-            )
+            do {
+                let items = try await itemMetadatas.toFileProviderItems(
+                    account: account, remoteInterface: remoteInterface, dbManager: dbManager
+                )
 
-            Task { @MainActor in
-                observer.didEnumerate(items)
-                Self.logger.info("Did enumerate \(items.count) items")
-
-                // TODO: Handle paging properly
-                /*
-                 if items.count == maxItemsPerFileProviderPage {
-                 let nextPage = numPage + 1
-                 let providerPage = NSFileProviderPage("\(nextPage)".data(using: .utf8)!)
-                 observer.finishEnumerating(upTo: providerPage)
-                 } else {
-                 observer.finishEnumerating(upTo: nil)
-                 }
-                 */
-                observer.finishEnumerating(upTo: fileProviderPageforNumPage(numPage))
+                Task { @MainActor in
+                    observer.didEnumerate(items)
+                    Self.logger.info("Did enumerate \(items.count) items")
+
+                    // TODO: Handle paging properly
+                    /*
+                     if items.count == maxItemsPerFileProviderPage {
+                     let nextPage = numPage + 1
+                     let providerPage = NSFileProviderPage("\(nextPage)".data(using: .utf8)!)
+                     observer.finishEnumerating(upTo: providerPage)
+                     } else {
+                     observer.finishEnumerating(upTo: nil)
+                     }
+                     */
+                    observer.finishEnumerating(upTo: fileProviderPageforNumPage(numPage))
+                }
+            } catch let error as NSError { // This error can only mean a missing parent item identifier
+                guard handleInvalidParent else {
+                    Self.logger.info("Not handling invalid parent in enumeration")
+                    observer.finishEnumeratingWithError(error)
+                    return
+                }
+                do {
+                    let metadata = try await Self.attemptInvalidParentRecovery(
+                        error: error,
+                        account: account,
+                        remoteInterface: remoteInterface,
+                        dbManager: dbManager
+                    )
+                    Self.completeEnumerationObserver(
+                        observer,
+                        account: account,
+                        remoteInterface: remoteInterface,
+                        dbManager: dbManager,
+                        numPage: numPage,
+                        itemMetadatas: [metadata] + itemMetadatas,
+                        handleInvalidParent: false
+                    )
+                } catch let error {
+                    observer.finishEnumeratingWithError(error)
+                }
             }
         }
     }
@@ -647,7 +675,8 @@
         dbManager: FilesDatabaseManager,
         newMetadatas: [SendableItemMetadata]?,
         updatedMetadatas: [SendableItemMetadata]?,
-        deletedMetadatas: [SendableItemMetadata]?
+        deletedMetadatas: [SendableItemMetadata]?,
+        handleInvalidParent: Bool = true
     ) {
         guard newMetadatas != nil || updatedMetadatas != nil || deletedMetadatas != nil else {
             Self.logger.error(
@@ -687,23 +716,95 @@
         }
 
         Task { [allUpdatedMetadatas, allDeletedMetadatas] in
-            let updatedItems = await allUpdatedMetadatas.toFileProviderItems(
-                account: account, remoteInterface: remoteInterface, dbManager: dbManager
-            )
+            do {
+                let updatedItems = try await allUpdatedMetadatas.toFileProviderItems(
+                    account: account, remoteInterface: remoteInterface, dbManager: dbManager
+                )
 
-            Task { @MainActor in
-                if !updatedItems.isEmpty {
-                    observer.didUpdate(updatedItems)
-                }
+                Task { @MainActor in
+                    if !updatedItems.isEmpty {
+                        observer.didUpdate(updatedItems)
+                    }
 
-                Self.logger.info(
+                    Self.logger.info(
                     """
                     Processed \(updatedItems.count) new or updated metadatas.
-                    \(allDeletedMetadatas.count) deleted metadatas.
+                        \(allDeletedMetadatas.count) deleted metadatas.
                     """
-                )
-                observer.finishEnumeratingChanges(upTo: anchor, moreComing: false)
+                    )
+                    observer.finishEnumeratingChanges(upTo: anchor, moreComing: false)
+                }
+            } catch let error as NSError { // This error can only mean a missing parent item identifier
+                guard handleInvalidParent else {
+                    Self.logger.info("Not handling invalid parent in change enumeration")
+                    observer.finishEnumeratingWithError(error)
+                    return
+                }
+                do {
+                    let metadata = try await Self.attemptInvalidParentRecovery(
+                        error: error,
+                        account: account,
+                        remoteInterface: remoteInterface,
+                        dbManager: dbManager
+                    )
+                    var modifiedNewMetadatas = newMetadatas
+                    modifiedNewMetadatas?.append(metadata)
+                    Self.completeChangesObserver(
+                        observer,
+                        anchor: anchor,
+                        enumeratedItemIdentifier: enumeratedItemIdentifier,
+                        account: account,
+                        remoteInterface: remoteInterface,
+                        dbManager: dbManager,
+                        newMetadatas: modifiedNewMetadatas,
+                        updatedMetadatas: updatedMetadatas,
+                        deletedMetadatas: deletedMetadatas,
+                        handleInvalidParent: false
+                    )
+                } catch let error {
+                    observer.finishEnumeratingWithError(error)
+                }
             }
         }
     }
+
+    private static func attemptInvalidParentRecovery(
+        error: NSError,
+        account: Account,
+        remoteInterface: RemoteInterface,
+        dbManager: FilesDatabaseManager
+    ) async throws -> SendableItemMetadata {
+        Self.logger.info("Attempting recovery from invalid parent identifier.")
+        // Try to recover from errors involving missing metadata for a parent
+        let userInfoKey =
+            FilesDatabaseManager.ErrorUserInfoKey.missingParentServerUrlAndFileName.rawValue
+        guard let urlToEnumerate = (error as NSError).userInfo[userInfoKey] as? String else {
+            Self.logger.fault("No missing parent server url and filename in error user info.")
+            assert(false)
+            throw NSError()
+        }
+
+        Self.logger.info(
+            "Recovering from invalid parent identifier at \(urlToEnumerate, privacy: .public)"
+        )
+        let (metadatas, _, _, _, error) = await Enumerator.readServerUrl(
+            urlToEnumerate,
+            account: account,
+            remoteInterface: remoteInterface,
+            dbManager: dbManager,
+            depth: .target
+        )
+        guard error == nil || error == .success, let metadata = metadatas?.first else {
+            Self.logger.error(
+                """
+                Problem retrieving parent for metadata.
+                    Error: \(error?.errorDescription ?? "NONE", privacy: .public)
+                    Metadatas: \(metadatas?.count ?? -1, privacy: .public)
+                """
+            )
+            throw error?.fileProviderError ?? NSFileProviderError(.cannotSynchronize)
+        }
+        // Provide it to the caller method so it can ingest it into the database and fix future errs
+        return metadata
+    }
 }

Sources/NextcloudFileProviderKit/Extensions/Array+Extensions.swift

@@ -36,19 +36,19 @@ extension Array {
     }
 
     func concurrentChunkedCompactMap<T>(
-        into size: Int = defaultChunkSize, transform: @escaping (Element) -> T?
-    ) async -> [T] {
-        await withTaskGroup(of: [T].self) { group in
+        into size: Int = defaultChunkSize, transform: @escaping (Element) throws -> T?
+    ) async throws -> [T] {
+        try await withThrowingTaskGroup(of: [T].self) { group in
             var results = [T]()
             results.reserveCapacity(self.count)
 
             for chunk in chunked(into: size) {
                 group.addTask {
-                    return chunk.compactMap { transform($0) }
+                    return try chunk.compactMap { try transform($0) }
                 }
             }
 
-            for await chunkResult in group {
+            for try await chunkResult in group {
                 results += chunkResult
             }
 

Sources/NextcloudFileProviderKit/Extensions/NKError+Extensions.swift

@@ -83,13 +83,13 @@
         handlingCollisionAgainstItemInRemotePath problemRemotePath: String,
         dbManager: FilesDatabaseManager,
         remoteInterface: RemoteInterface
-    ) -> Error? {
+    ) async -> Error? {
         guard fileProviderError?.code == .filenameCollision else {
             return fileProviderError as Error?
         }
         guard let collidingItemMetadata = dbManager.itemMetadata(
             account: dbManager.account.ncKitAccount, locatedAtRemoteUrl: problemRemotePath
-        ), let collidingItem = Item.storedItem(
+        ), let collidingItem = await Item.storedItem(
             identifier: .init(collidingItemMetadata.ocId),
             account: dbManager.account,
             remoteInterface: remoteInterface,

Sources/NextcloudFileProviderKit/Item/Item+Create.swift

@@ -44,7 +44,7 @@
                     \(createError.errorDescription, privacy: .public)
                 """
             )
-            return (nil, createError.fileProviderError(
+            return (nil, await createError.fileProviderError(
                 handlingCollisionAgainstItemInRemotePath: remotePath,
                 dbManager: dbManager,
                 remoteInterface: remoteInterface
@@ -79,7 +79,7 @@
                     \(readError.errorDescription, privacy: .public)
                 """
             )
-            return (nil, readError.fileProviderError(
+            return (nil, await readError.fileProviderError(
                 handlingCollisionAgainstItemInRemotePath: remotePath,
                 dbManager: dbManager,
                 remoteInterface: remoteInterface
@@ -151,7 +151,7 @@
                     received ocId: \(ocId ?? "empty", privacy: .public)
                 """
             )
-            return (nil, error.fileProviderError(
+            return (nil, await error.fileProviderError(
                 handlingCollisionAgainstItemInRemotePath: remotePath,
                 dbManager: dbManager,
                 remoteInterface: remoteInterface

Sources/NextcloudFileProviderKit/Item/Item+Fetch.swift

@@ -282,8 +282,10 @@ public extension Item {
 
         dbManager.addItemMetadata(updatedMetadata)
 
-        guard let parentItemIdentifier = dbManager.parentItemIdentifierFromMetadata(
-            updatedMetadata
+        guard let parentItemIdentifier = await dbManager.parentItemIdentifierWithRemoteFallback(
+            fromMetadata: metadata,
+            remoteInterface: remoteInterface,
+            account: account
         ) else {
             Self.logger.error(
                 """