(test): Add array index schema and query tests, update expectations for NOT NULL arrays

abnegate · abnegate · commit 4f02d63f2d5f · 2026-02-19T19:39:47.000+13:00
diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php
@@ -49,6 +49,11 @@ abstract class Base extends TestCase
      */
     abstract protected function getDatabase(): Database;
 
+    /**
+     * @return \Utopia\Database\PDO|\PDO|null
+     */
+    abstract protected function getPDO(): mixed;
+
     /**
      * @param string $collection
      * @param string $column
diff --git a/tests/e2e/Adapter/MariaDBTest.php b/tests/e2e/Adapter/MariaDBTest.php
@@ -52,6 +52,11 @@ public function getDatabase(bool $fresh = false): Database
         return self::$database = $database;
     }
 
+    protected function getPDO(): mixed
+    {
+        return self::$pdo;
+    }
+
     protected function deleteColumn(string $collection, string $column): bool
     {
         $sqlTable = "`" . $this->getDatabase()->getDatabase() . "`.`" . $this->getDatabase()->getNamespace() . "_" . $collection . "`";
diff --git a/tests/e2e/Adapter/MirrorTest.php b/tests/e2e/Adapter/MirrorTest.php
@@ -313,6 +313,11 @@ public function testDeleteMirroredDocument(): void
         $this->assertTrue($database->getDestination()->getDocument('testDeleteMirroredDocument', $document->getId())->isEmpty());
     }
 
+    protected function getPDO(): mixed
+    {
+        return self::$sourcePdo;
+    }
+
     protected function deleteColumn(string $collection, string $column): bool
     {
         $sqlTable = "`" . self::$source->getDatabase() . "`.`" . self::$source->getNamespace() . "_" . $collection . "`";
diff --git a/tests/e2e/Adapter/MongoDBTest.php b/tests/e2e/Adapter/MongoDBTest.php
@@ -98,6 +98,11 @@ public function testKeywords(): void
         $this->assertTrue(true);
     }
 
+    protected function getPDO(): mixed
+    {
+        return null;
+    }
+
     protected function deleteColumn(string $collection, string $column): bool
     {
         return true;
diff --git a/tests/e2e/Adapter/MySQLTest.php b/tests/e2e/Adapter/MySQLTest.php
@@ -58,6 +58,11 @@ public function getDatabase(): Database
         return self::$database = $database;
     }
 
+    protected function getPDO(): mixed
+    {
+        return self::$pdo;
+    }
+
     protected function deleteColumn(string $collection, string $column): bool
     {
         $sqlTable = "`" . $this->getDatabase()->getDatabase() . "`.`" . $this->getDatabase()->getNamespace() . "_" . $collection . "`";
diff --git a/tests/e2e/Adapter/PoolTest.php b/tests/e2e/Adapter/PoolTest.php
@@ -79,6 +79,18 @@ public function getDatabase(): Database
         return self::$database = $database;
     }
 
+    protected function getPDO(): mixed
+    {
+        $pdo = null;
+        self::$pool->use(function (Adapter $adapter) use (&$pdo) {
+            $class = new ReflectionClass($adapter);
+            $property = $class->getProperty('pdo');
+            $property->setAccessible(true);
+            $pdo = $property->getValue($adapter);
+        });
+        return $pdo;
+    }
+
     protected function deleteColumn(string $collection, string $column): bool
     {
         $sqlTable = "`" . $this->getDatabase()->getDatabase() . "`.`" . $this->getDatabase()->getNamespace() . "_" . $collection . "`";
diff --git a/tests/e2e/Adapter/PostgresTest.php b/tests/e2e/Adapter/PostgresTest.php
@@ -51,6 +51,11 @@ public function getDatabase(): Database
         return self::$database = $database;
     }
 
+    protected function getPDO(): mixed
+    {
+        return self::$pdo;
+    }
+
     protected function deleteColumn(string $collection, string $column): bool
     {
         $sqlTable = '"' . $this->getDatabase()->getDatabase(). '"."' . $this->getDatabase()->getNamespace() . '_' . $collection . '"';
diff --git a/tests/e2e/Adapter/SQLiteTest.php b/tests/e2e/Adapter/SQLiteTest.php
@@ -55,6 +55,11 @@ public function getDatabase(): Database
         return self::$database = $database;
     }
 
+    protected function getPDO(): mixed
+    {
+        return self::$pdo;
+    }
+
     protected function deleteColumn(string $collection, string $column): bool
     {
         $sqlTable = "`" . $this->getDatabase()->getNamespace() . "_" . $collection . "`";
diff --git a/tests/e2e/Adapter/Scopes/AttributeTests.php b/tests/e2e/Adapter/Scopes/AttributeTests.php
@@ -1620,10 +1620,11 @@ public function testArrayAttribute(): void
                 $this->assertEquals('Invalid query: Cannot query contains on attribute "age" because it is not an array, string, or object.', $e->getMessage());
             }
 
+            // Array columns are NOT NULL with DEFAULT empty array, so isNull returns nothing
             $documents = $database->find($collection, [
                 Query::isNull('long_size')
             ]);
-            $this->assertCount(1, $documents);
+            $this->assertCount(0, $documents);
 
             $documents = $database->find($collection, [
                 Query::contains('tv_show', ['love'])
diff --git a/tests/e2e/Adapter/Scopes/CollectionTests.php b/tests/e2e/Adapter/Scopes/CollectionTests.php
@@ -569,10 +569,10 @@ public function testSchemaAttributes(): void
 
         $attribute = $attributes['string_list'];
         $this->assertEquals('string_list', $attribute['$id']);
-        $this->assertTrue(in_array($attribute['dataType'], ['json', 'longtext'])); // mysql vs maria
-        $this->assertTrue(in_array($attribute['columnType'], ['json', 'longtext']));
+        $this->assertTrue(in_array($attribute['dataType'], ['json', 'jsonb', 'longtext']));
+        $this->assertTrue(in_array($attribute['columnType'], ['json', 'jsonb', 'longtext']));
         $this->assertTrue(in_array($attribute['characterMaximumLength'], [null, '4294967295']));
-        $this->assertEquals('YES', $attribute['isNullable']);
+        $this->assertEquals('NO', $attribute['isNullable']);
 
         $attribute = $attributes['dob'];
         $this->assertEquals('dob', $attribute['$id']);
diff --git a/tests/e2e/Adapter/Scopes/IndexTests.php b/tests/e2e/Adapter/Scopes/IndexTests.php
@@ -1048,4 +1048,148 @@ public function testTTLIndexDuplicatePrevention(): void
         // Cleanup
         $database->deleteCollection($col);
     }
+
+    /**
+     * Verifies that array attributes backed by JSON columns are created as
+     * NOT NULL DEFAULT (empty array). This is required because:
+     *
+     * 1. MySQL bug #111037: the optimizer can skip functional indexes
+     *    (CAST ... AS ARRAY) on nullable columns in complex query plans.
+     * 2. Correctness: JSON_CONTAINS(NULL, ...) returns NULL, not FALSE,
+     *    so rows with NULL array columns are invisible to both positive
+     *    and negative array queries (containsAny / notContains).
+     *
+     * @link https://bugs.mysql.com/bug.php?id=111037
+     */
+    public function testArrayAttributeNotNullSchema(): void
+    {
+        $database = $this->getDatabase();
+        $adapter = $database->getAdapter();
+
+        if (!$adapter->getSupportForSchemaAttributes()) {
+            $this->expectNotToPerformAssertions();
+            return;
+        }
+
+        $collection = 'arrayNotNull';
+
+        $database->createCollection($collection, permissions: [
+            Permission::create(Role::any()),
+            Permission::read(Role::any()),
+        ]);
+
+        $database->createAttribute($collection, 'tags', Database::VAR_STRING, 255, false, array: true);
+
+        // Verify the physical column is NOT NULL via INFORMATION_SCHEMA
+        $pdo = $this->getPDO();
+
+        $ns = $database->getNamespace();
+        $db = $database->getDatabase();
+        $tableName = "{$ns}_{$collection}";
+
+        $stmt = $pdo->prepare(
+            'SELECT IS_NULLABLE, COLUMN_DEFAULT '
+            . 'FROM INFORMATION_SCHEMA.COLUMNS '
+            . 'WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? AND COLUMN_NAME = ?'
+        );
+        $stmt->execute([$db, $tableName, 'tags']);
+        $col = $stmt->fetch(\PDO::FETCH_ASSOC);
+        // PostgreSQL returns lowercase keys from INFORMATION_SCHEMA
+        $col = $col ? \array_change_key_case($col, CASE_UPPER) : false;
+
+        $this->assertNotFalse($col, 'Column "tags" should exist in INFORMATION_SCHEMA');
+        $this->assertEquals(
+            'NO',
+            $col['IS_NULLABLE'],
+            'Array (JSON) column must be NOT NULL so the optimizer can use multi-valued indexes '
+            . 'and JSON_CONTAINS/JSON_OVERLAPS never encounter NULL'
+        );
+
+        $database->deleteCollection($collection);
+    }
+
+    /**
+     * Verifies that the index on an array attribute is picked by the
+     * optimizer and that queries return correct results.
+     */
+    public function testArrayIndexUsedInQuery(): void
+    {
+        $database = $this->getDatabase();
+        $adapter = $database->getAdapter();
+
+        if (!$adapter->getSupportForCastIndexArray()) {
+            $this->expectNotToPerformAssertions();
+            return;
+        }
+
+        $collection = 'arrayIndexExplain';
+
+        $database->createCollection($collection, permissions: [
+            Permission::create(Role::any()),
+            Permission::read(Role::any()),
+        ]);
+
+        $database->createAttribute($collection, 'tags', Database::VAR_STRING, 255, false, array: true);
+        $database->createIndex($collection, 'idx_tags', Database::INDEX_KEY, ['tags'], [255]);
+
+        // Insert enough documents for the optimizer to prefer the index over a full table scan
+        $documents = [];
+        for ($i = 0; $i < 500; $i++) {
+            $documents[] = new Document([
+                '$id' => ID::unique(),
+                '$permissions' => [Permission::read(Role::any())],
+                'tags' => ['tag_' . ($i % 50), 'common'],
+            ]);
+        }
+
+        foreach (\array_chunk($documents, 100) as $chunk) {
+            $database->createDocuments($collection, $chunk);
+        }
+
+        $pdo = $this->getPDO();
+        $ns = $database->getNamespace();
+        $db = $database->getDatabase();
+
+        if ($adapter instanceof \Utopia\Database\Adapter\Postgres) {
+            $table = "\"{$db}\".\"{$ns}_{$collection}\"";
+
+            // Verify GIN index exists
+            $stmt = $pdo->prepare(
+                "SELECT indexdef FROM pg_indexes WHERE tablename = ? AND indexname LIKE ?"
+            );
+            $stmt->execute(["{$ns}_{$collection}", '%idx_tags%']);
+            $indexDef = $stmt->fetchColumn();
+            $this->assertNotFalse($indexDef, 'GIN index should exist for array column');
+            $this->assertStringContainsString('gin', \strtolower($indexDef), 'Index on array column should be a GIN index');
+        } else {
+            $table = "`{$db}`.`{$ns}_{$collection}`";
+
+            $pdo->prepare("ANALYZE TABLE {$table}")->execute();
+
+            $sql = "EXPLAIN SELECT * FROM {$table} WHERE JSON_CONTAINS(`tags`, '[\"tag_1\"]')";
+            $stmt = $pdo->prepare($sql);
+            $stmt->execute();
+            $explain = $stmt->fetchAll();
+
+            $usedKey = $explain[0]['key'] ?? null;
+            $this->assertNotNull(
+                $usedKey,
+                'MySQL should use an index for JSON_CONTAINS on a NOT NULL JSON column. '
+                . "EXPLAIN output:\n" . \print_r($explain, true)
+            );
+            $this->assertStringContainsString(
+                'idx_tags',
+                $usedKey,
+                "Expected the 'idx_tags' index to be used. EXPLAIN output:\n" . \print_r($explain, true)
+            );
+        }
+
+        // Verify the query also returns correct functional results
+        $results = $database->find($collection, [
+            Query::containsAny('tags', ['tag_1']),
+        ]);
+        $this->assertCount(10, $results);
+
+        $database->deleteCollection($collection);
+    }
 }
diff --git a/tests/e2e/Adapter/Scopes/RelationshipTests.php b/tests/e2e/Adapter/Scopes/RelationshipTests.php
@@ -157,7 +157,7 @@ public function testZoo(): void
             'price' => 1000,
             'dateOfBirth' => '1975-06-12',
             'isActive' => true,
-            'integers' => null,
+            'integers' => [],
             'email' => null,
             'enum' => null,
             'ip' => '255.0.0.1',
diff --git a/tests/e2e/Adapter/SharedTables/MariaDBTest.php b/tests/e2e/Adapter/SharedTables/MariaDBTest.php
@@ -67,6 +67,11 @@ public function getDatabase(bool $fresh = false): Database
         return self::$database = $database;
     }
 
+    protected function getPDO(): mixed
+    {
+        return self::$pdo;
+    }
+
     protected function deleteColumn(string $collection, string $column): bool
     {
         $sqlTable = "`" . $this->getDatabase()->getDatabase() . "`.`" . $this->getDatabase()->getNamespace() . "_" . $collection . "`";
diff --git a/tests/e2e/Adapter/SharedTables/MongoDBTest.php b/tests/e2e/Adapter/SharedTables/MongoDBTest.php
@@ -100,6 +100,11 @@ public function testKeywords(): void
         $this->assertTrue(true);
     }
 
+    protected function getPDO(): mixed
+    {
+        return null;
+    }
+
     protected function deleteColumn(string $collection, string $column): bool
     {
         return true;
diff --git a/tests/e2e/Adapter/SharedTables/MySQLTest.php b/tests/e2e/Adapter/SharedTables/MySQLTest.php
@@ -69,6 +69,11 @@ public function getDatabase(): Database
         return self::$database = $database;
     }
 
+    protected function getPDO(): mixed
+    {
+        return self::$pdo;
+    }
+
     protected function deleteColumn(string $collection, string $column): bool
     {
         $sqlTable = "`" . $this->getDatabase()->getDatabase() . "`.`" . $this->getDatabase()->getNamespace() . "_" . $collection . "`";
diff --git a/tests/e2e/Adapter/SharedTables/PostgresTest.php b/tests/e2e/Adapter/SharedTables/PostgresTest.php
@@ -64,6 +64,11 @@ public function getDatabase(): Database
         return self::$database = $database;
     }
 
+    protected function getPDO(): mixed
+    {
+        return self::$pdo;
+    }
+
     protected function deleteColumn(string $collection, string $column): bool
     {
         $sqlTable = '"' . $this->getDatabase()->getDatabase() . '"."' . $this->getDatabase()->getNamespace() . '_' . $collection . '"';
diff --git a/tests/e2e/Adapter/SharedTables/SQLiteTest.php b/tests/e2e/Adapter/SharedTables/SQLiteTest.php
@@ -70,6 +70,11 @@ public function getDatabase(): Database
         return self::$database = $database;
     }
 
+    protected function getPDO(): mixed
+    {
+        return self::$pdo;
+    }
+
     protected function deleteColumn(string $collection, string $column): bool
     {
         $sqlTable = "`" . $this->getDatabase()->getNamespace() . "_" . $collection . "`";

Original file line number	Diff line number	Diff line change
`@@ -52,6 +52,11 @@ public function getDatabase(bool $fresh = false): Database`
`52`	`52`	`return self::$database = $database;`
`53`	`53`	`}`
`54`	`54`
	`55`	`+ protected function getPDO(): mixed`
	`56`	`+ {`
	`57`	`+ return self::$pdo;`
	`58`	`+ }`
	`59`	`+`
`55`	`60`	`protected function deleteColumn(string $collection, string $column): bool`
`56`	`61`	`{`
`57`	`62`	$sqlTable = "`" . $this->getDatabase()->getDatabase() . "`.`" . $this->getDatabase()->getNamespace() . "_" . $collection . "`";
Original file line number	Diff line number	Diff line change
`@@ -313,6 +313,11 @@ public function testDeleteMirroredDocument(): void`
`313`	`313`	`$this->assertTrue($database->getDestination()->getDocument('testDeleteMirroredDocument', $document->getId())->isEmpty());`
`314`	`314`	`}`
`315`	`315`
	`316`	`+ protected function getPDO(): mixed`
	`317`	`+ {`
	`318`	`+ return self::$sourcePdo;`
	`319`	`+ }`
	`320`	`+`
`316`	`321`	`protected function deleteColumn(string $collection, string $column): bool`
`317`	`322`	`{`
`318`	`323`	$sqlTable = "`" . self::$source->getDatabase() . "`.`" . self::$source->getNamespace() . "_" . $collection . "`";
Original file line number	Diff line number	Diff line change
`@@ -98,6 +98,11 @@ public function testKeywords(): void`
`98`	`98`	`$this->assertTrue(true);`
`99`	`99`	`}`
`100`	`100`
	`101`	`+ protected function getPDO(): mixed`
	`102`	`+ {`
	`103`	`+ return null;`
	`104`	`+ }`
	`105`	`+`
`101`	`106`	`protected function deleteColumn(string $collection, string $column): bool`
`102`	`107`	`{`
`103`	`108`	`return true;`
Original file line number	Diff line number	Diff line change
`@@ -58,6 +58,11 @@ public function getDatabase(): Database`
`58`	`58`	`return self::$database = $database;`
`59`	`59`	`}`
`60`	`60`
	`61`	`+ protected function getPDO(): mixed`
	`62`	`+ {`
	`63`	`+ return self::$pdo;`
	`64`	`+ }`
	`65`	`+`
`61`	`66`	`protected function deleteColumn(string $collection, string $column): bool`
`62`	`67`	`{`
`63`	`68`	$sqlTable = "`" . $this->getDatabase()->getDatabase() . "`.`" . $this->getDatabase()->getNamespace() . "_" . $collection . "`";
Original file line number	Diff line number	Diff line change
`@@ -51,6 +51,11 @@ public function getDatabase(): Database`
`51`	`51`	`return self::$database = $database;`
`52`	`52`	`}`
`53`	`53`
	`54`	`+ protected function getPDO(): mixed`
	`55`	`+ {`
	`56`	`+ return self::$pdo;`
	`57`	`+ }`
	`58`	`+`
`54`	`59`	`protected function deleteColumn(string $collection, string $column): bool`
`55`	`60`	`{`
`56`	`61`	`$sqlTable = '"' . $this->getDatabase()->getDatabase(). '"."' . $this->getDatabase()->getNamespace() . '_' . $collection . '"';`
Original file line number	Diff line number	Diff line change
`@@ -55,6 +55,11 @@ public function getDatabase(): Database`
`55`	`55`	`return self::$database = $database;`
`56`	`56`	`}`
`57`	`57`
	`58`	`+ protected function getPDO(): mixed`
	`59`	`+ {`
	`60`	`+ return self::$pdo;`
	`61`	`+ }`
	`62`	`+`
`58`	`63`	`protected function deleteColumn(string $collection, string $column): bool`
`59`	`64`	`{`
`60`	`65`	$sqlTable = "`" . $this->getDatabase()->getNamespace() . "_" . $collection . "`";