Skip to content

Commit 27d36f4

Browse files
abnegateabnegate
committed
Merge remote-tracking branch 'origin/main' into chore-sync-3.x
# Conflicts: # src/Database/Adapter/Postgres.php
2 parents 6d90ebe + 414dbde commit 27d36f4

File tree

14 files changed

+190
-55
lines changed

14 files changed

+190
-55
lines changed

‎.github/workflows/tests.yml‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -57,8 +57,7 @@ jobs:
5757
- name: Load and Start Services
5858
run: |
5959
docker load --input /tmp/${{ env.IMAGE }}.tar
60-
docker compose up -d
61-
sleep 10
60+
docker compose up -d --wait
6261
6362
- name: Run Unit Tests
6463
run: docker compose exec tests vendor/bin/phpunit /usr/src/code/tests/unit
@@ -101,9 +100,7 @@ jobs:
101100
- name: Load and Start Services
102101
run: |
103102
docker load --input /tmp/${{ env.IMAGE }}.tar
104-
docker compose up -d
105-
sleep 10
103+
docker compose up -d --wait
106104
107105
- name: Run Tests
108106
run: docker compose exec -T tests vendor/bin/phpunit /usr/src/code/tests/e2e/Adapter/${{matrix.adapter}}Test.php --debug
109-

‎docker-compose.yml‎

Lines changed: 69 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,25 @@ services:
1919
- ./docker-compose.yml:/usr/src/code/docker-compose.yml
2020
environment:
2121
PHP_IDE_CONFIG: serverName=tests
22+
depends_on:
23+
postgres:
24+
condition: service_healthy
25+
postgres-mirror:
26+
condition: service_healthy
27+
mariadb:
28+
condition: service_healthy
29+
mariadb-mirror:
30+
condition: service_healthy
31+
mysql:
32+
condition: service_healthy
33+
mysql-mirror:
34+
condition: service_healthy
35+
redis:
36+
condition: service_healthy
37+
redis-mirror:
38+
condition: service_healthy
39+
mongo:
40+
condition: service_healthy
2241

2342
adminer:
2443
image: adminer
@@ -44,6 +63,12 @@ services:
4463
POSTGRES_USER: root
4564
POSTGRES_PASSWORD: password
4665
POSTGRES_DB: root
66+
healthcheck:
67+
test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER"]
68+
interval: 10s
69+
timeout: 5s
70+
retries: 5
71+
start_period: 5s
4772

4873
postgres-mirror:
4974
build:
@@ -60,6 +85,12 @@ services:
6085
POSTGRES_USER: root
6186
POSTGRES_PASSWORD: password
6287
POSTGRES_DB: root
88+
healthcheck:
89+
test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER"]
90+
interval: 10s
91+
timeout: 5s
92+
retries: 10
93+
start_period: 5s
6394

6495
mariadb:
6596
image: mariadb:10.11
@@ -71,6 +102,12 @@ services:
71102
- "8703:3306"
72103
environment:
73104
- MYSQL_ROOT_PASSWORD=password
105+
healthcheck:
106+
test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"]
107+
start_period: 10s
108+
interval: 10s
109+
timeout: 5s
110+
retries: 3
74111

75112
mariadb-mirror:
76113
image: mariadb:10.11
@@ -82,6 +119,12 @@ services:
82119
- "8704:3306"
83120
environment:
84121
- MYSQL_ROOT_PASSWORD=password
122+
healthcheck:
123+
test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"]
124+
start_period: 10s
125+
interval: 10s
126+
timeout: 5s
127+
retries: 3
85128

86129
mongo:
87130
image: mongo:8.0.14
@@ -110,10 +153,10 @@ services:
110153
\" 2>/dev/null || exit 1
111154
fi
112155
"
113-
interval: 10s
156+
interval: 5s
114157
timeout: 10s
115158
retries: 10
116-
start_period: 30s
159+
start_period: 10s
117160

118161
mongo-express:
119162
image: mongo-express
@@ -145,6 +188,12 @@ services:
145188
MYSQL_TCP_PORT: 3307
146189
cap_add:
147190
- SYS_NICE
191+
healthcheck:
192+
test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-u $$MYSQL_USER", "-p $$MYSQL_PASSWORD"]
193+
interval: 10s
194+
timeout: 5s
195+
retries: 5
196+
start_period: 30s
148197

149198
mysql-mirror:
150199
image: mysql:8.0.43
@@ -161,6 +210,12 @@ services:
161210
MYSQL_TCP_PORT: 3307
162211
cap_add:
163212
- SYS_NICE
213+
healthcheck:
214+
test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-u $$MYSQL_USER", "-p $$MYSQL_PASSWORD"]
215+
interval: 10s
216+
timeout: 5s
217+
retries: 5
218+
start_period: 30s
164219

165220
redis:
166221
image: redis:8.2.1-alpine3.22
@@ -169,6 +224,12 @@ services:
169224
- "8708:6379"
170225
networks:
171226
- database
227+
healthcheck:
228+
test: ["CMD", "redis-cli", "ping"]
229+
interval: 10s
230+
timeout: 5s
231+
retries: 5
232+
start_period: 5s
172233

173234
redis-mirror:
174235
image: redis:8.2.1-alpine3.22
@@ -177,6 +238,12 @@ services:
177238
- "8709:6379"
178239
networks:
179240
- database
241+
healthcheck:
242+
test: ["CMD", "redis-cli", "ping"]
243+
interval: 10s
244+
timeout: 5s
245+
retries: 5
246+
start_period: 5s
180247

181248
volumes:
182249
mongo-data:

‎src/Database/Adapter.php‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1464,4 +1464,11 @@ public function enableAlterLocks(bool $enable): self
14641464

14651465
return $this;
14661466
}
1467+
1468+
/**
1469+
* Handle non utf characters supported?
1470+
*
1471+
* @return bool
1472+
*/
1473+
abstract public function getSupportNonUtfCharacters(): bool;
14671474
}

‎src/Database/Adapter/MariaDB.php‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
use Utopia\Database\Database;
88
use Utopia\Database\Document;
99
use Utopia\Database\Exception as DatabaseException;
10+
use Utopia\Database\Exception\Character as CharacterException;
1011
use Utopia\Database\Exception\Duplicate as DuplicateException;
1112
use Utopia\Database\Exception\Limit as LimitException;
1213
use Utopia\Database\Exception\NotFound as NotFoundException;
@@ -1838,6 +1839,10 @@ public function getInternalIndexesKeys(): array
18381839

18391840
protected function processException(PDOException $e): \Exception
18401841
{
1842+
if ($e->getCode() === '22007' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1366) {
1843+
return new CharacterException('Invalid character', $e->getCode(), $e);
1844+
}
1845+
18411846
// Timeout
18421847
if ($e->getCode() === '70100' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1969) {
18431848
return new TimeoutException('Query timed out', $e->getCode(), $e);
@@ -2230,4 +2235,9 @@ public function getSupportForAlterLocks(): bool
22302235
{
22312236
return true;
22322237
}
2238+
2239+
public function getSupportNonUtfCharacters(): bool
2240+
{
2241+
return true;
2242+
}
22332243
}

‎src/Database/Adapter/Mongo.php‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2882,7 +2882,7 @@ public function getAttributeWidth(Document $collection): int
28822882
*/
28832883
public function getSupportForCasting(): bool
28842884
{
2885-
return true;
2885+
return false;
28862886
}
28872887

28882888
/**
@@ -3230,4 +3230,9 @@ public function getSupportForAlterLocks(): bool
32303230
{
32313231
return false;
32323232
}
3233+
3234+
public function getSupportNonUtfCharacters(): bool
3235+
{
3236+
return false;
3237+
}
32333238
}

‎src/Database/Adapter/MySQL.php‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
use PDOException;
66
use Utopia\Database\Database;
77
use Utopia\Database\Exception as DatabaseException;
8+
use Utopia\Database\Exception\Character as CharacterException;
89
use Utopia\Database\Exception\Dependency as DependencyException;
910
use Utopia\Database\Exception\Structure as StructureException;
1011
use Utopia\Database\Exception\Timeout as TimeoutException;
@@ -147,6 +148,10 @@ public function getSupportForCastIndexArray(): bool
147148

148149
protected function processException(PDOException $e): \Exception
149150
{
151+
if ($e->getCode() === 'HY000' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1366) {
152+
return new CharacterException('Invalid character', $e->getCode(), $e);
153+
}
154+
150155
// Timeout
151156
if ($e->getCode() === 'HY000' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 3024) {
152157
return new TimeoutException('Query timed out', $e->getCode(), $e);

‎src/Database/Adapter/Pool.php‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -642,4 +642,9 @@ public function getSupportForAlterLocks(): bool
642642
{
643643
return $this->delegate(__FUNCTION__, \func_get_args());
644644
}
645+
646+
public function getSupportNonUtfCharacters(): bool
647+
{
648+
return $this->delegate(__FUNCTION__, \func_get_args());
649+
}
645650
}

‎src/Database/Adapter/Postgres.php‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2752,6 +2752,11 @@ protected function bindOperatorParams(\PDOStatement|PDOStatementProxy $stmt, Ope
27522752
}
27532753
}
27542754

2755+
public function getSupportNonUtfCharacters(): bool
2756+
{
2757+
return false;
2758+
}
2759+
27552760
/**
27562761
* Ensure index key length stays within PostgreSQL's 63 character limit.
27572762
*

‎src/Database/Adapter/SQL.php‎

Lines changed: 3 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1497,7 +1497,7 @@ public function getKeywords(): array
14971497
*/
14981498
public function getSupportForCasting(): bool
14991499
{
1500-
return false;
1500+
return true;
15011501
}
15021502

15031503
public function getSupportForNumericCasting(): bool
@@ -2966,7 +2966,6 @@ protected function convertArrayToWKT(array $geometry): string
29662966
*/
29672967
public function find(Document $collection, array $queries = [], ?int $limit = 25, ?int $offset = null, array $orderAttributes = [], array $orderTypes = [], array $cursor = [], string $cursorDirection = Database::CURSOR_AFTER, string $forPermission = Database::PERMISSION_READ): array
29682968
{
2969-
$attributes = $collection->getAttribute('attributes', []);
29702969
$collection = $collection->getId();
29712970
$name = $this->filter($collection);
29722971
$roles = $this->authorization->getRoles();
@@ -3183,7 +3182,6 @@ public function find(Document $collection, array $queries = [], ?int $limit = 25
31833182
*/
31843183
public function count(Document $collection, array $queries = [], ?int $max = null): int
31853184
{
3186-
$attributes = $collection->getAttribute("attributes", []);
31873185
$collection = $collection->getId();
31883186
$name = $this->filter($collection);
31893187
$roles = $this->authorization->getRoles();
@@ -3199,13 +3197,9 @@ public function count(Document $collection, array $queries = [], ?int $max = nul
31993197

32003198
$queries = array_map(fn ($query) => clone $query, $queries);
32013199

3202-
// Extract vector queries (used for ORDER BY) and keep non-vector for WHERE
3203-
$vectorQueries = [];
32043200
$otherQueries = [];
32053201
foreach ($queries as $query) {
3206-
if (in_array($query->getMethod(), Query::VECTOR_TYPES)) {
3207-
$vectorQueries[] = $query;
3208-
} else {
3202+
if (!in_array($query->getMethod(), Query::VECTOR_TYPES)) {
32093203
$otherQueries[] = $query;
32103204
}
32113205
}
@@ -3228,22 +3222,11 @@ public function count(Document $collection, array $queries = [], ?int $max = nul
32283222
? 'WHERE ' . \implode(' AND ', $where)
32293223
: '';
32303224

3231-
// Add vector distance calculations to ORDER BY (similarity-aware LIMIT)
3232-
$vectorOrders = [];
3233-
foreach ($vectorQueries as $query) {
3234-
$vectorOrder = $this->getVectorDistanceOrder($query, $binds, $alias);
3235-
if ($vectorOrder) {
3236-
$vectorOrders[] = $vectorOrder;
3237-
}
3238-
}
3239-
$sqlOrder = !empty($vectorOrders) ? 'ORDER BY ' . implode(', ', $vectorOrders) : '';
3240-
32413225
$sql = "
32423226
SELECT COUNT(1) as sum FROM (
32433227
SELECT 1
32443228
FROM {$this->getSQLTable($name)} AS {$this->quote($alias)}
32453229
{$sqlWhere}
3246-
{$sqlOrder}
32473230
{$limit}
32483231
) table_count
32493232
";
@@ -3280,7 +3263,6 @@ public function count(Document $collection, array $queries = [], ?int $max = nul
32803263
*/
32813264
public function sum(Document $collection, string $attribute, array $queries = [], ?int $max = null): int|float
32823265
{
3283-
$collectionAttributes = $collection->getAttribute("attributes", []);
32843266
$collection = $collection->getId();
32853267
$name = $this->filter($collection);
32863268
$attribute = $this->filter($attribute);
@@ -3297,13 +3279,9 @@ public function sum(Document $collection, string $attribute, array $queries = []
32973279

32983280
$queries = array_map(fn ($query) => clone $query, $queries);
32993281

3300-
// Extract vector queries (used for ORDER BY) and keep non-vector for WHERE
3301-
$vectorQueries = [];
33023282
$otherQueries = [];
33033283
foreach ($queries as $query) {
3304-
if (in_array($query->getMethod(), Query::VECTOR_TYPES)) {
3305-
$vectorQueries[] = $query;
3306-
} else {
3284+
if (!in_array($query->getMethod(), Query::VECTOR_TYPES)) {
33073285
$otherQueries[] = $query;
33083286
}
33093287
}
@@ -3326,22 +3304,11 @@ public function sum(Document $collection, string $attribute, array $queries = []
33263304
? 'WHERE ' . \implode(' AND ', $where)
33273305
: '';
33283306

3329-
// Add vector distance calculations to ORDER BY (similarity-aware LIMIT)
3330-
$vectorOrders = [];
3331-
foreach ($vectorQueries as $query) {
3332-
$vectorOrder = $this->getVectorDistanceOrder($query, $binds, $alias);
3333-
if ($vectorOrder) {
3334-
$vectorOrders[] = $vectorOrder;
3335-
}
3336-
}
3337-
$sqlOrder = !empty($vectorOrders) ? 'ORDER BY ' . implode(', ', $vectorOrders) : '';
3338-
33393307
$sql = "
33403308
SELECT SUM({$this->quote($attribute)}) as sum FROM (
33413309
SELECT {$this->quote($attribute)}
33423310
FROM {$this->getSQLTable($name)} AS {$this->quote($alias)}
33433311
{$sqlWhere}
3344-
{$sqlOrder}
33453312
{$limit}
33463313
) table_count
33473314
";

‎src/Database/Adapter/SQLite.php‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1876,4 +1876,9 @@ public function getSupportForAlterLocks(): bool
18761876
{
18771877
return false;
18781878
}
1879+
1880+
public function getSupportNonUtfCharacters(): bool
1881+
{
1882+
return false;
1883+
}
18791884
}

0 commit comments

Comments
 (0)