Merge branch 'main' of github.com:utopia-php/database into joins9

fogelito · fogelito · commit de44f6a679c3 · 2026-01-21T13:13:21.000+02:00
diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php
@@ -1572,6 +1572,7 @@ public function upsertDocuments(Document $collection, string $attribute, array $
             $operations = [];
             foreach ($changes as $change) {
                 $document = $change->getNew();
+                $oldDocument = $change->getOld();
                 $attributes = $document->getAttributes();
                 $attributes['_uid'] = $document->getId();
                 $attributes['_createdAt'] = $document['$createdAt'];
@@ -1597,6 +1598,9 @@ public function upsertDocuments(Document $collection, string $attribute, array $
 
                 unset($record['_id']); // Don't update _id
 
+                // Get fields to unset for schemaless mode
+                $unsetFields = $this->getUpsertAttributeRemovals($oldDocument, $document, $record);
+
                 if (!empty($attribute)) {
                     // Get the attribute value before removing it from $set
                     $attributeValue = $record[$attribute] ?? 0;
@@ -1605,17 +1609,28 @@ public function upsertDocuments(Document $collection, string $attribute, array $
                     // it is requierd to mimic the behaver of SQL on duplicate key update
                     unset($record[$attribute]);
 
+                    // Also remove from unset if it was there
+                    unset($unsetFields[$attribute]);
+
                     // Increment the specific attribute and update all other fields
                     $update = [
                         '$inc' => [$attribute => $attributeValue],
                         '$set' => $record
                     ];
+
+                    if (!empty($unsetFields)) {
+                        $update['$unset'] = $unsetFields;
+                    }
                 } else {
                     // Update all fields
                     $update = [
                         '$set' => $record
                     ];
 
+                    if (!empty($unsetFields)) {
+                        $update['$unset'] = $unsetFields;
+                    }
+
                     // Add UUID7 _id for new documents in upsert operations
                     if (empty($document->getSequence())) {
                         $update['$setOnInsert'] = [
@@ -1644,6 +1659,43 @@ public function upsertDocuments(Document $collection, string $attribute, array $
         return \array_map(fn ($change) => $change->getNew(), $changes);
     }
 
+    /**
+     * Get fields to unset for schemaless upsert operations
+     *
+     * @param Document $oldDocument
+     * @param Document $newDocument
+     * @param array<string, mixed> $record
+     * @return array<string, string>
+     */
+    private function getUpsertAttributeRemovals(Document $oldDocument, Document $newDocument, array $record): array
+    {
+        $unsetFields = [];
+
+        if ($this->getSupportForAttributes() || $oldDocument->isEmpty()) {
+            return $unsetFields;
+        }
+
+        $oldUserAttributes = $oldDocument->getAttributes();
+        $newUserAttributes = $newDocument->getAttributes();
+
+        $protectedFields = ['_uid', '_id', '_createdAt', '_updatedAt', '_permissions', '_tenant'];
+
+        foreach ($oldUserAttributes as $originalKey => $originalValue) {
+            if (in_array($originalKey, $protectedFields) || array_key_exists($originalKey, $newUserAttributes)) {
+                continue;
+            }
+
+            $transformed = $this->replaceChars('$', '_', [$originalKey => $originalValue]);
+            $dbKey = array_key_first($transformed);
+
+            if ($dbKey && !array_key_exists($dbKey, $record) && !in_array($dbKey, $protectedFields)) {
+                $unsetFields[$dbKey] = '';
+            }
+        }
+
+        return $unsetFields;
+    }
+
     /**
      * Get sequences for documents that were created
      *
diff --git a/src/Database/Database.php b/src/Database/Database.php
@@ -407,6 +407,8 @@ class Database
 
     protected bool $preserveDates = false;
 
+    protected bool $preserveSequence = false;
+
     protected int $maxQueryValues = 5000;
 
     protected bool $migrating = false;
@@ -1399,6 +1401,30 @@ public function withPreserveDates(callable $callback): mixed
         }
     }
 
+    public function getPreserveSequence(): bool
+    {
+        return $this->preserveSequence;
+    }
+
+    public function setPreserveSequence(bool $preserve): static
+    {
+        $this->preserveSequence = $preserve;
+
+        return $this;
+    }
+
+    public function withPreserveSequence(callable $callback): mixed
+    {
+        $previous = $this->preserveSequence;
+        $this->preserveSequence = true;
+
+        try {
+            return $callback();
+        } finally {
+            $this->preserveSequence = $previous;
+        }
+    }
+
     public function setMaxQueryValues(int $max): self
     {
         $this->maxQueryValues = $max;
@@ -6627,8 +6653,11 @@ public function upsertDocumentsWithIncrease(
             $document
                 ->setAttribute('$id', empty($document->getId()) ? ID::unique() : $document->getId())
                 ->setAttribute('$collection', $collection->getId())
-                ->setAttribute('$updatedAt', ($updatedAt === null || !$this->preserveDates) ? $time : $updatedAt)
-                ->removeAttribute('$sequence');
+                ->setAttribute('$updatedAt', ($updatedAt === null || !$this->preserveDates) ? $time : $updatedAt);
+
+            if (!$this->preserveSequence) {
+                $document->removeAttribute('$sequence');
+            }
 
             $createdAt = $document->getCreatedAt();
             if ($createdAt === null || !$this->preserveDates) {
diff --git a/src/Database/Mirror.php b/src/Database/Mirror.php
@@ -139,6 +139,15 @@ public function setPreserveDates(bool $preserve): static
         return $this;
     }
 
+    public function setPreserveSequence(bool $preserve): static
+    {
+        $this->delegate(__FUNCTION__, \func_get_args());
+
+        $this->preserveSequence = $preserve;
+
+        return $this;
+    }
+
     public function enableValidation(): static
     {
         $this->delegate(__FUNCTION__);
diff --git a/tests/e2e/Adapter/Scopes/DocumentTests.php b/tests/e2e/Adapter/Scopes/DocumentTests.php
@@ -1179,6 +1179,144 @@ public function testUpsertMixedPermissionDelta(): void
         ], $db->getDocument(__FUNCTION__, 'b')->getPermissions());
     }
 
+    public function testPreserveSequenceUpsert(): void
+    {
+        /** @var Database $database */
+        $database = $this->getDatabase();
+
+        if (!$database->getAdapter()->getSupportForUpserts()) {
+            $this->expectNotToPerformAssertions();
+            return;
+        }
+
+        $collectionName = 'preserve_sequence_upsert';
+
+        $database->createCollection($collectionName);
+
+        if ($database->getAdapter()->getSupportForAttributes()) {
+            $database->createAttribute($collectionName, 'name', Database::VAR_STRING, 128, true);
+        }
+
+        // Create initial documents
+        $doc1 = $database->createDocument($collectionName, new Document([
+            '$id' => 'doc1',
+            '$permissions' => [
+                Permission::read(Role::any()),
+                Permission::update(Role::any()),
+            ],
+            'name' => 'Alice',
+        ]));
+
+        $doc2 = $database->createDocument($collectionName, new Document([
+            '$id' => 'doc2',
+            '$permissions' => [
+                Permission::read(Role::any()),
+                Permission::update(Role::any()),
+            ],
+            'name' => 'Bob',
+        ]));
+
+        $originalSeq1 = $doc1->getSequence();
+        $originalSeq2 = $doc2->getSequence();
+
+        $this->assertNotEmpty($originalSeq1);
+        $this->assertNotEmpty($originalSeq2);
+
+        // Test: Without preserveSequence (default), $sequence should be ignored
+        $database->setPreserveSequence(false);
+
+        $database->upsertDocuments($collectionName, [
+            new Document([
+                '$id' => 'doc1',
+                '$sequence' => 999, // Try to set a different sequence
+                '$permissions' => [
+                    Permission::read(Role::any()),
+                    Permission::update(Role::any()),
+                ],
+                'name' => 'Alice Updated',
+            ]),
+        ]);
+
+        $doc1Updated = $database->getDocument($collectionName, 'doc1');
+        $this->assertEquals('Alice Updated', $doc1Updated->getAttribute('name'));
+        $this->assertEquals($originalSeq1, $doc1Updated->getSequence()); // Sequence unchanged
+
+        // Test: With preserveSequence=true, $sequence from document should be used
+        $database->setPreserveSequence(true);
+
+        $database->upsertDocuments($collectionName, [
+            new Document([
+                '$id' => 'doc2',
+                '$sequence' => $originalSeq2, // Keep original sequence
+                '$permissions' => [
+                    Permission::read(Role::any()),
+                    Permission::update(Role::any()),
+                ],
+                'name' => 'Bob Updated',
+            ]),
+        ]);
+
+        $doc2Updated = $database->getDocument($collectionName, 'doc2');
+        $this->assertEquals('Bob Updated', $doc2Updated->getAttribute('name'));
+        $this->assertEquals($originalSeq2, $doc2Updated->getSequence()); // Sequence preserved
+
+        // Test: withPreserveSequence helper
+        $database->setPreserveSequence(false);
+
+        $doc1 = $database->getDocument($collectionName, 'doc1');
+        $currentSeq1 = $doc1->getSequence();
+
+        $database->withPreserveSequence(function () use ($database, $collectionName, $currentSeq1) {
+            $database->upsertDocuments($collectionName, [
+                new Document([
+                    '$id' => 'doc1',
+                    '$sequence' => $currentSeq1,
+                    '$permissions' => [
+                        Permission::read(Role::any()),
+                        Permission::update(Role::any()),
+                    ],
+                    'name' => 'Alice Final',
+                ]),
+            ]);
+        });
+
+        $doc1Final = $database->getDocument($collectionName, 'doc1');
+        $this->assertEquals('Alice Final', $doc1Final->getAttribute('name'));
+        $this->assertEquals($currentSeq1, $doc1Final->getSequence());
+
+        // Verify flag was reset after withPreserveSequence
+        $this->assertFalse($database->getPreserveSequence());
+
+        // Test: With preserveSequence=true, invalid $sequence should throw error (SQL adapters only)
+        $database->setPreserveSequence(true);
+
+        try {
+            $database->upsertDocuments($collectionName, [
+                new Document([
+                    '$id' => 'doc1',
+                    '$sequence' => 'abc', // Invalid sequence value
+                    '$permissions' => [
+                        Permission::read(Role::any()),
+                        Permission::update(Role::any()),
+                    ],
+                    'name' => 'Alice Invalid',
+                ]),
+            ]);
+            // Schemaless adapters may not validate sequence type, so only fail for schemaful
+            if ($database->getAdapter()->getSupportForAttributes()) {
+                $this->fail('Expected StructureException for invalid sequence');
+            }
+        } catch (Throwable $e) {
+            if ($database->getAdapter()->getSupportForAttributes()) {
+                $this->assertInstanceOf(StructureException::class, $e);
+                $this->assertStringContainsString('sequence', $e->getMessage());
+            }
+        }
+
+        $database->setPreserveSequence(false);
+        $database->deleteCollection($collectionName);
+    }
+
     public function testRespectNulls(): Document
     {
         /** @var Database $database */
diff --git a/tests/e2e/Adapter/Scopes/SchemalessTests.php b/tests/e2e/Adapter/Scopes/SchemalessTests.php
