Skip to content

Commit c2ad814

Browse files
abnegateabnegate
authored
Merge pull request #779 from utopia-php/mongo-ttl
2 parents 93025fd + 3fca7d7 commit c2ad814

File tree

14 files changed

+1814
-76
lines changed

14 files changed

+1814
-76
lines changed

‎composer.lock‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/Database/Adapter.php‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -688,10 +688,12 @@ abstract public function renameIndex(string $collection, string $old, string $ne
688688
* @param array<int> $lengths
689689
* @param array<string> $orders
690690
* @param array<string,string> $indexAttributeTypes
691+
* @param array<string, mixed> $collation
692+
* @param int $ttl
691693
*
692694
* @return bool
693695
*/
694-
abstract public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = []): bool;
696+
abstract public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = [], array $collation = [], int $ttl = 1): bool;
695697

696698
/**
697699
* Delete Index
@@ -1519,4 +1521,14 @@ public function getSupportForRegex(): bool
15191521
{
15201522
return $this->getSupportForPCRERegex() || $this->getSupportForPOSIXRegex();
15211523
}
1524+
1525+
/**
1526+
* Are ttl indexes supported?
1527+
*
1528+
* @return bool
1529+
*/
1530+
public function getSupportForTTLIndexes(): bool
1531+
{
1532+
return false;
1533+
}
15221534
}

‎src/Database/Adapter/MariaDB.php‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -716,7 +716,7 @@ public function renameIndex(string $collection, string $old, string $new): bool
716716
* @return bool
717717
* @throws DatabaseException
718718
*/
719-
public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = []): bool
719+
public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = [], array $collation = [], int $ttl = 1): bool
720720
{
721721
$metadataCollection = new Document(['$id' => Database::METADATA]);
722722
$collection = $this->getDocument($metadataCollection, $collection);
@@ -2283,4 +2283,9 @@ public function getSupportForPOSIXRegex(): bool
22832283
{
22842284
return false;
22852285
}
2286+
2287+
public function getSupportForTTLIndexes(): bool
2288+
{
2289+
return false;
2290+
}
22862291
}

‎src/Database/Adapter/Mongo.php‎

Lines changed: 187 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -488,7 +488,7 @@ public function createCollection(string $name, array $attributes = [], array $in
488488
$orders = $index->getAttribute('orders');
489489

490490
// If sharedTables, always add _tenant as the first key
491-
if ($this->sharedTables) {
491+
if ($this->shouldAddTenantToIndex($index)) {
492492
$key['_tenant'] = $this->getOrder(Database::ORDER_ASC);
493493
}
494494

@@ -508,6 +508,9 @@ public function createCollection(string $name, array $attributes = [], array $in
508508
$order = $this->getOrder($this->filter($orders[$j] ?? Database::ORDER_ASC));
509509
$unique = true;
510510
break;
511+
case Database::INDEX_TTL:
512+
$order = $this->getOrder($this->filter($orders[$j] ?? Database::ORDER_ASC));
513+
break;
511514
default:
512515
// index not supported
513516
return false;
@@ -526,6 +529,14 @@ public function createCollection(string $name, array $attributes = [], array $in
526529
$newIndexes[$i]['default_language'] = 'none';
527530
}
528531

532+
// Handle TTL indexes
533+
if ($index->getAttribute('type') === Database::INDEX_TTL) {
534+
$ttl = $index->getAttribute('ttl', 0);
535+
if ($ttl > 0) {
536+
$newIndexes[$i]['expireAfterSeconds'] = $ttl;
537+
}
538+
}
539+
529540
// Add partial filter for indexes to avoid indexing null values
530541
if (in_array($index->getAttribute('type'), [
531542
Database::INDEX_UNIQUE,
@@ -901,10 +912,11 @@ public function deleteRelationship(
901912
* @param array<string> $orders
902913
* @param array<string, string> $indexAttributeTypes
903914
* @param array<string, mixed> $collation
915+
* @param int $ttl
904916
* @return bool
905917
* @throws Exception
906918
*/
907-
public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = [], array $collation = []): bool
919+
public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = [], array $collation = [], int $ttl = 1): bool
908920
{
909921
$name = $this->getNamespace() . '_' . $this->filter($collection);
910922
$id = $this->filter($id);
@@ -913,7 +925,7 @@ public function createIndex(string $collection, string $id, string $type, array
913925
$indexes['name'] = $id;
914926

915927
// If sharedTables, always add _tenant as the first key
916-
if ($this->sharedTables) {
928+
if ($this->shouldAddTenantToIndex($type)) {
917929
$indexes['key']['_tenant'] = $this->getOrder(Database::ORDER_ASC);
918930
}
919931

@@ -933,6 +945,8 @@ public function createIndex(string $collection, string $id, string $type, array
933945
case Database::INDEX_UNIQUE:
934946
$indexes['unique'] = true;
935947
break;
948+
case Database::INDEX_TTL:
949+
break;
936950
default:
937951
return false;
938952
}
@@ -961,6 +975,11 @@ public function createIndex(string $collection, string $id, string $type, array
961975
$indexes['default_language'] = 'none';
962976
}
963977

978+
// Handle TTL indexes
979+
if ($type === Database::INDEX_TTL && $ttl > 0) {
980+
$indexes['expireAfterSeconds'] = $ttl;
981+
}
982+
964983
// Add partial filter for indexes to avoid indexing null values
965984
if (in_array($type, [Database::INDEX_UNIQUE, Database::INDEX_KEY])) {
966985
$partialFilter = [];
@@ -1073,7 +1092,7 @@ public function renameIndex(string $collection, string $old, string $new): bool
10731092

10741093
try {
10751094
$deletedindex = $this->deleteIndex($collection, $old);
1076-
$createdindex = $this->createIndex($collection, $new, $index['type'], $index['attributes'], $index['lengths'] ?? [], $index['orders'] ?? [], $indexAttributeTypes);
1095+
$createdindex = $this->createIndex($collection, $new, $index['type'], $index['attributes'], $index['lengths'] ?? [], $index['orders'] ?? [], $indexAttributeTypes, [], $index['ttl'] ?? 0);
10771096
} catch (\Exception $e) {
10781097
throw $this->processException($e);
10791098
}
@@ -1235,30 +1254,7 @@ public function castingAfter(Document $collection, Document $document): Document
12351254
$node = (int)$node;
12361255
break;
12371256
case Database::VAR_DATETIME:
1238-
if ($node instanceof UTCDateTime) {
1239-
// Handle UTCDateTime objects
1240-
$node = DateTime::format($node->toDateTime());
1241-
} elseif (is_array($node) && isset($node['$date'])) {
1242-
// Handle Extended JSON format from (array) cast
1243-
// Format: {"$date":{"$numberLong":"1760405478290"}}
1244-
if (is_array($node['$date']) && isset($node['$date']['$numberLong'])) {
1245-
$milliseconds = (int)$node['$date']['$numberLong'];
1246-
$seconds = intdiv($milliseconds, 1000);
1247-
$microseconds = ($milliseconds % 1000) * 1000;
1248-
$dateTime = \DateTime::createFromFormat('U.u', $seconds . '.' . str_pad((string)$microseconds, 6, '0'));
1249-
if ($dateTime) {
1250-
$dateTime->setTimezone(new \DateTimeZone('UTC'));
1251-
$node = DateTime::format($dateTime);
1252-
}
1253-
}
1254-
} elseif (is_string($node)) {
1255-
// Already a string, validate and pass through
1256-
try {
1257-
new \DateTime($node);
1258-
} catch (\Exception $e) {
1259-
// Invalid date string, skip
1260-
}
1261-
}
1257+
$node = $this->convertUTCDateToString($node);
12621258
break;
12631259
case Database::VAR_OBJECT:
12641260
// Convert stdClass objects to arrays for object attributes
@@ -1279,6 +1275,8 @@ public function castingAfter(Document $collection, Document $document): Document
12791275
// mongodb results out a stdclass for objects
12801276
if (is_object($value) && get_class($value) === stdClass::class) {
12811277
$document->setAttribute($key, $this->convertStdClassToArray($value));
1278+
} elseif ($value instanceof UTCDateTime) {
1279+
$document->setAttribute($key, $this->convertUTCDateToString($value));
12821280
}
12831281
}
12841282
}
@@ -1361,6 +1359,24 @@ public function castingBefore(Document $collection, Document $document): Documen
13611359
unset($node);
13621360
$document->setAttribute($key, ($array) ? $value : $value[0]);
13631361
}
1362+
$indexes = $collection->getAttribute('indexes');
1363+
$ttlIndexes = array_filter($indexes, fn ($index) => $index->getAttribute('type') === Database::INDEX_TTL);
1364+
1365+
if (!$this->getSupportForAttributes()) {
1366+
foreach ($document->getArrayCopy() as $key => $value) {
1367+
if (in_array($this->getInternalKeyForAttribute($key), Database::INTERNAL_ATTRIBUTE_KEYS)) {
1368+
continue;
1369+
}
1370+
if (is_string($value) && (in_array($key, $ttlIndexes) || $this->isExtendedISODatetime($value))) {
1371+
try {
1372+
$newValue = new UTCDateTime(new \DateTime($value));
1373+
$document->setAttribute($key, $newValue);
1374+
} catch (\Throwable $th) {
1375+
// skip -> a valid string
1376+
}
1377+
}
1378+
}
1379+
}
13641380

13651381
return $document;
13661382
}
@@ -2708,6 +2724,25 @@ protected function getOrder(string $order): int
27082724
};
27092725
}
27102726

2727+
/**
2728+
* Check if tenant should be added to index
2729+
*
2730+
* @param Document|string $indexOrType Index document or index type string
2731+
* @return bool
2732+
*/
2733+
protected function shouldAddTenantToIndex(Document|string $indexOrType): bool
2734+
{
2735+
if (!$this->sharedTables) {
2736+
return false;
2737+
}
2738+
2739+
$indexType = $indexOrType instanceof Document
2740+
? $indexOrType->getAttribute('type')
2741+
: $indexOrType;
2742+
2743+
return $indexType !== Database::INDEX_TTL;
2744+
}
2745+
27112746
/**
27122747
* @param array<string> $selections
27132748
* @param string $prefix
@@ -3467,4 +3502,128 @@ public function getSupportForTrigramIndex(): bool
34673502
{
34683503
return false;
34693504
}
3505+
3506+
public function getSupportForTTLIndexes(): bool
3507+
{
3508+
return true;
3509+
}
3510+
3511+
protected function isExtendedISODatetime(string $val): bool
3512+
{
3513+
/**
3514+
* Min:
3515+
* YYYY-MM-DDTHH:mm:ssZ (20)
3516+
* YYYY-MM-DDTHH:mm:ss+HH:MM (25)
3517+
*
3518+
* Max:
3519+
* YYYY-MM-DDTHH:mm:ss.fffffZ (26)
3520+
* YYYY-MM-DDTHH:mm:ss.fffff+HH:MM (31)
3521+
*/
3522+
3523+
$len = strlen($val);
3524+
3525+
// absolute minimum
3526+
if ($len < 20) {
3527+
return false;
3528+
}
3529+
3530+
// fixed datetime fingerprints
3531+
if (
3532+
!isset($val[19]) ||
3533+
$val[4] !== '-' ||
3534+
$val[7] !== '-' ||
3535+
$val[10] !== 'T' ||
3536+
$val[13] !== ':' ||
3537+
$val[16] !== ':'
3538+
) {
3539+
return false;
3540+
}
3541+
3542+
// timezone detection
3543+
$hasZ = ($val[$len - 1] === 'Z');
3544+
3545+
$hasOffset = (
3546+
$len >= 25 &&
3547+
($val[$len - 6] === '+' || $val[$len - 6] === '-') &&
3548+
$val[$len - 3] === ':'
3549+
);
3550+
3551+
if (!$hasZ && !$hasOffset) {
3552+
return false;
3553+
}
3554+
3555+
if ($hasOffset && $len > 31) {
3556+
return false;
3557+
}
3558+
3559+
if ($hasZ && $len > 26) {
3560+
return false;
3561+
}
3562+
3563+
$digitPositions = [
3564+
0,1,2,3,
3565+
5,6,
3566+
8,9,
3567+
11,12,
3568+
14,15,
3569+
17,18
3570+
];
3571+
3572+
$timeEnd = $hasZ ? $len - 1 : $len - 6;
3573+
3574+
// fractional seconds
3575+
if ($timeEnd > 19) {
3576+
if ($val[19] !== '.' || $timeEnd < 21) {
3577+
return false;
3578+
}
3579+
for ($i = 20; $i < $timeEnd; $i++) {
3580+
$digitPositions[] = $i;
3581+
}
3582+
}
3583+
3584+
// timezone offset numeric digits
3585+
if ($hasOffset) {
3586+
foreach ([$len - 5, $len - 4, $len - 2, $len - 1] as $i) {
3587+
$digitPositions[] = $i;
3588+
}
3589+
}
3590+
3591+
foreach ($digitPositions as $i) {
3592+
if (!ctype_digit($val[$i])) {
3593+
return false;
3594+
}
3595+
}
3596+
3597+
return true;
3598+
}
3599+
3600+
protected function convertUTCDateToString(mixed $node): mixed
3601+
{
3602+
if ($node instanceof UTCDateTime) {
3603+
// Handle UTCDateTime objects
3604+
$node = DateTime::format($node->toDateTime());
3605+
} elseif (is_array($node) && isset($node['$date'])) {
3606+
// Handle Extended JSON format from (array) cast
3607+
// Format: {"$date":{"$numberLong":"1760405478290"}}
3608+
if (is_array($node['$date']) && isset($node['$date']['$numberLong'])) {
3609+
$milliseconds = (int)$node['$date']['$numberLong'];
3610+
$seconds = intdiv($milliseconds, 1000);
3611+
$microseconds = ($milliseconds % 1000) * 1000;
3612+
$dateTime = \DateTime::createFromFormat('U.u', $seconds . '.' . str_pad((string)$microseconds, 6, '0'));
3613+
if ($dateTime) {
3614+
$dateTime->setTimezone(new \DateTimeZone('UTC'));
3615+
$node = DateTime::format($dateTime);
3616+
}
3617+
}
3618+
} elseif (is_string($node)) {
3619+
// Already a string, validate and pass through
3620+
try {
3621+
new \DateTime($node);
3622+
} catch (\Exception $e) {
3623+
// Invalid date string, skip
3624+
}
3625+
}
3626+
3627+
return $node;
3628+
}
34703629
}

‎src/Database/Adapter/MySQL.php‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -319,4 +319,9 @@ protected function getOperatorSQL(string $column, \Utopia\Database\Operator $ope
319319
// For all other operators, use parent implementation
320320
return parent::getOperatorSQL($column, $operator, $bindIndex);
321321
}
322+
323+
public function getSupportForTTLIndexes(): bool
324+
{
325+
return false;
326+
}
322327
}

‎src/Database/Adapter/Pool.php‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -187,7 +187,7 @@ public function renameIndex(string $collection, string $old, string $new): bool
187187
return $this->delegate(__FUNCTION__, \func_get_args());
188188
}
189189

190-
public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = []): bool
190+
public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = [], array $collation = [], int $ttl = 1): bool
191191
{
192192
return $this->delegate(__FUNCTION__, \func_get_args());
193193
}
@@ -642,4 +642,9 @@ public function getSupportNonUtfCharacters(): bool
642642
{
643643
return $this->delegate(__FUNCTION__, \func_get_args());
644644
}
645+
646+
public function getSupportForTTLIndexes(): bool
647+
{
648+
return $this->delegate(__FUNCTION__, \func_get_args());
649+
}
645650
}

0 commit comments

Comments
 (0)