Skip to content

Commit 7e5bc62

Browse files
lohanidamodarlohanidamodar
committed
feat: update purge method to accept queries for selective deletion of usage metrics
1 parent 2d0ab43 commit 7e5bc62

6 files changed

Lines changed: 96 additions & 63 deletions

File tree

‎src/Usage/Adapter.php‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -153,9 +153,12 @@ abstract public function sumByPeriodBatch(array $metrics, string $period, array
153153
abstract public function getByPeriodBatch(array $metrics, string $period, array $queries = []): array;
154154

155155
/**
156-
* Purge old usage metrics
156+
* Purge usage metrics matching the given queries.
157+
* When no queries are provided, all metrics are deleted.
158+
*
159+
* @param array<\Utopia\Usage\Query> $queries
157160
*/
158-
abstract public function purge(string $datetime): bool;
161+
abstract public function purge(array $queries = []): bool;
159162

160163
/**
161164
* Find metrics using Query objects.

‎src/Usage/Adapter/ClickHouse.php‎

Lines changed: 12 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2214,33 +2214,32 @@ public function getByPeriodBatch(array $metrics, string $period, array $queries
22142214
*
22152215
* @throws Exception
22162216
*/
2217-
public function purge(string $datetime): bool
2217+
public function purge(array $queries = []): bool
22182218
{
22192219
$this->setOperationContext('purge()');
22202220

22212221
$tableName = $this->getTableName();
22222222
$snapshotTableName = $this->getSnapshotTableName();
22232223
$escapedTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName);
22242224
$escapedSnapshotTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($snapshotTableName);
2225-
$tenantFilter = $this->getTenantFilter();
22262225

2227-
$params = ['datetime' => $datetime];
2228-
if ($this->sharedTables) {
2229-
$params['tenant'] = $this->tenant;
2226+
// Parse queries into WHERE clause
2227+
$parsed = $this->parseQueries($queries);
2228+
$whereData = $this->buildWhereClause($parsed['filters'], $parsed['params']);
2229+
$whereClause = $whereData['clause'];
2230+
$params = $whereData['params'];
2231+
2232+
// When no queries provided, delete everything (WHERE 1=1 for tenant filter support)
2233+
if (empty($whereClause)) {
2234+
$whereClause = ' WHERE 1=1';
22302235
}
22312236

22322237
// Purge from aggregated table
2233-
$sql = "
2234-
DELETE FROM {$escapedTable}
2235-
WHERE time < {datetime:DateTime64(3)}{$tenantFilter}
2236-
";
2238+
$sql = "DELETE FROM {$escapedTable}{$whereClause}";
22372239
$this->query($sql, $params);
22382240

22392241
// Purge from snapshot table
2240-
$sql = "
2241-
DELETE FROM {$escapedSnapshotTable}
2242-
WHERE time < {datetime:DateTime64(3)}{$tenantFilter}
2243-
";
2242+
$sql = "DELETE FROM {$escapedSnapshotTable}{$whereClause}";
22442243
$this->query($sql, $params);
22452244

22462245
return true;

‎src/Usage/Adapter/Database.php‎

Lines changed: 5 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -409,32 +409,16 @@ public function getByPeriodBatch(array $metrics, string $period, array $queries
409409
return $grouped;
410410
}
411411

412-
public function purge(string $datetime): bool
412+
public function purge(array $queries = []): bool
413413
{
414-
$this->db->getAuthorization()->skip(function () use ($datetime) {
415-
// Purge documents with time < datetime
416-
do {
417-
$documents = $this->db->find(
418-
collection: $this->collection,
419-
queries: [
420-
DatabaseQuery::lessThan('time', $datetime),
421-
DatabaseQuery::limit(100),
422-
]
423-
);
424-
425-
foreach ($documents as $document) {
426-
$this->db->deleteDocument($this->collection, $document->getId());
427-
}
428-
} while (! empty($documents));
414+
$this->db->getAuthorization()->skip(function () use ($queries) {
415+
$dbQueries = $this->convertQueriesToDatabase($queries);
416+
$dbQueries[] = DatabaseQuery::limit(100);
429417

430-
// Purge inf-period documents (time=null, not matched by time < datetime)
431418
do {
432419
$documents = $this->db->find(
433420
collection: $this->collection,
434-
queries: [
435-
DatabaseQuery::isNull('time'),
436-
DatabaseQuery::limit(100),
437-
]
421+
queries: $dbQueries,
438422
);
439423

440424
foreach ($documents as $document) {

‎src/Usage/Usage.php‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -204,13 +204,15 @@ public function getByPeriodBatch(array $metrics, string $period, array $queries
204204
}
205205

206206
/**
207-
* Purge usage metrics older than the specified datetime.
207+
* Purge usage metrics matching the given queries.
208+
* When no queries are provided, all metrics are deleted.
208209
*
210+
* @param array<\Utopia\Usage\Query> $queries
209211
* @throws \Exception
210212
*/
211-
public function purge(string $datetime): bool
213+
public function purge(array $queries = []): bool
212214
{
213-
return $this->adapter->purge($datetime);
215+
return $this->adapter->purge($queries);
214216
}
215217

216218
/**

‎tests/Usage/Adapter/ClickHouseTest.php‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
namespace Utopia\Tests\Adapter;
44

55
use PHPUnit\Framework\TestCase;
6-
use Utopia\Database\DateTime;
76
use Utopia\Tests\Usage\UsageBase;
87
use Utopia\Usage\Adapter\ClickHouse as ClickHouseAdapter;
98
use Utopia\Usage\Usage;
@@ -53,7 +52,7 @@ public function testMetricTenantOverridesAdapterTenantInBatch(): void
5352

5453
$usage = new Usage($adapter);
5554
$usage->setup();
56-
$usage->purge(DateTime::now());
55+
$usage->purge();
5756

5857
$metrics = [
5958
[
@@ -75,7 +74,7 @@ public function testMetricTenantOverridesAdapterTenantInBatch(): void
7574
$this->assertCount(1, $results);
7675
$this->assertEquals(2, $results[0]->getTenant());
7776

78-
$usage->purge(DateTime::now());
77+
$usage->purge();
7978
}
8079

8180
/**
@@ -313,10 +312,9 @@ public function testMetricsWithSpecialCharacters(): void
313312
public function testFind(): void
314313
{
315314
// Cleanup
316-
$this->usage->purge(DateTime::now());
315+
$this->usage->purge();
317316

318317
// Setup test data
319-
$now = DateTime::now();
320318
// metric A: value 10, time NOW
321319
$this->usage->incrementBatch([['metric' => 'metric-A', 'value' => 10, 'period' => '1h', 'tags' => ['category' => 'cat1']]]);
322320
// metric B: value 20, time NOW
@@ -820,7 +818,7 @@ public function testAsyncInsertConfiguration(): void
820818
// Verify it works with async inserts enabled
821819
$usage = new Usage($adapter);
822820
$usage->setup();
823-
$usage->purge(\Utopia\Database\DateTime::now());
821+
$usage->purge();
824822

825823
$this->assertTrue($usage->increment('async-test', 42));
826824

@@ -837,6 +835,6 @@ public function testAsyncInsertConfiguration(): void
837835
$stats = $adapter->getConnectionStats();
838836
$this->assertFalse($stats['async_inserts']);
839837

840-
$usage->purge(\Utopia\Database\DateTime::now());
838+
$usage->purge();
841839
}
842840
}

‎tests/Usage/UsageBase.php‎

Lines changed: 64 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ public function setUp(): void
2020

2121
public function tearDown(): void
2222
{
23-
$this->usage->purge(DateTime::now());
23+
$this->usage->purge();
2424
}
2525

2626
public function createUsageMetrics(): void
@@ -33,7 +33,7 @@ public function createUsageMetrics(): void
3333

3434
public function testIncrement(): void
3535
{
36-
$this->usage->purge(DateTime::now());
36+
$this->usage->purge();
3737

3838
// increment() should auto fan-out to all 3 periods
3939
$this->assertTrue($this->usage->increment('inc-metric', 10));
@@ -52,7 +52,7 @@ public function testIncrement(): void
5252
public function testIncrementBatch(): void
5353
{
5454
// First cleanup existing data
55-
$this->usage->purge(DateTime::now());
55+
$this->usage->purge();
5656

5757
$metrics = [
5858
[
@@ -123,7 +123,7 @@ public function testSumByPeriod(): void
123123
public function testIncrementingDefaultBehavior(): void
124124
{
125125
// Ensure clean state
126-
$this->usage->purge(\Utopia\Database\DateTime::now());
126+
$this->usage->purge();
127127

128128
// Increment the same metric twice
129129
$this->assertTrue($this->usage->increment('increment-test', 5));
@@ -211,15 +211,62 @@ public function testPurge(): void
211211
// Wait a bit
212212
sleep(2);
213213

214-
// Purge all metrics
215-
$status = $this->usage->purge(DateTime::now());
214+
// Purge all metrics (no queries = delete everything)
215+
$status = $this->usage->purge();
216216
$this->assertTrue($status);
217217

218218
// Verify metrics were purged
219219
$results = $this->usage->getByPeriod('purge-test', '1h');
220220
$this->assertEquals(0, count($results));
221221
}
222222

223+
public function testPurgeWithQueries(): void
224+
{
225+
$this->usage->purge();
226+
227+
$this->assertTrue($this->usage->increment('purge-keep', 10));
228+
$this->assertTrue($this->usage->increment('purge-remove', 20));
229+
230+
// Purge only the 'purge-remove' metric
231+
$status = $this->usage->purge([
232+
Query::equal('metric', ['purge-remove']),
233+
]);
234+
$this->assertTrue($status);
235+
236+
// 'purge-remove' should be gone
237+
$sum = $this->usage->sumByPeriod('purge-remove', '1h');
238+
$this->assertEquals(0, $sum);
239+
240+
// 'purge-keep' should still exist
241+
$sum = $this->usage->sumByPeriod('purge-keep', '1h');
242+
$this->assertEquals(10, $sum);
243+
}
244+
245+
public function testPurgeByPeriod(): void
246+
{
247+
$this->usage->purge();
248+
249+
// Insert into specific periods
250+
$this->assertTrue($this->usage->incrementBatch([
251+
['metric' => 'purge-period', 'value' => 10, 'period' => '1h', 'tags' => []],
252+
['metric' => 'purge-period', 'value' => 20, 'period' => '1d', 'tags' => []],
253+
]));
254+
255+
// Purge only 1h period
256+
$this->assertTrue($this->usage->purge([
257+
Query::equal('metric', ['purge-period']),
258+
Query::equal('period', ['1h']),
259+
]));
260+
261+
// 1h should be gone
262+
$sum1h = $this->usage->sumByPeriod('purge-period', '1h');
263+
$this->assertEquals(0, $sum1h);
264+
265+
// 1d should still exist
266+
$sum1d = $this->usage->sumByPeriod('purge-period', '1d');
267+
$this->assertEquals(20, $sum1d);
268+
}
269+
223270
public function testPeriodFormats(): void
224271
{
225272
$periods = Usage::PERIODS;
@@ -235,7 +282,7 @@ public function testPeriodFormats(): void
235282

236283
public function testSet(): void
237284
{
238-
$this->usage->purge(DateTime::now());
285+
$this->usage->purge();
239286

240287
// set() should auto fan-out to all 3 periods with replace semantics
241288
$this->assertTrue($this->usage->set('set-metric', 100));
@@ -253,7 +300,7 @@ public function testSet(): void
253300

254301
public function testCollectAndFlush(): void
255302
{
256-
$this->usage->purge(DateTime::now());
303+
$this->usage->purge();
257304

258305
// collect() accumulates in memory, nothing written yet
259306
$this->usage->collect('collect-metric', 10);
@@ -288,7 +335,7 @@ public function testCollectAndFlush(): void
288335

289336
public function testCollectMultipleMetrics(): void
290337
{
291-
$this->usage->purge(DateTime::now());
338+
$this->usage->purge();
292339

293340
$this->usage->collect('metric-a', 10);
294341
$this->usage->collect('metric-b', 20);
@@ -309,7 +356,7 @@ public function testCollectMultipleMetrics(): void
309356

310357
public function testCollectSetAndFlush(): void
311358
{
312-
$this->usage->purge(DateTime::now());
359+
$this->usage->purge();
313360

314361
// collectSet() uses last-write-wins semantics
315362
$this->usage->collectSet('set-collect', 100);
@@ -334,7 +381,7 @@ public function testCollectSetAndFlush(): void
334381

335382
public function testMixedCollectAndCollectSet(): void
336383
{
337-
$this->usage->purge(DateTime::now());
384+
$this->usage->purge();
338385

339386
// Mix both types in the same buffer
340387
$this->usage->collect('inc-mixed', 10);
@@ -417,7 +464,7 @@ public function testFlushThresholdConfiguration(): void
417464

418465
public function testSumByPeriodBatch(): void
419466
{
420-
$this->usage->purge(DateTime::now());
467+
$this->usage->purge();
421468

422469
// Insert known metrics
423470
$this->assertTrue($this->usage->increment('batch-sum-a', 10));
@@ -440,7 +487,7 @@ public function testSumByPeriodBatch(): void
440487

441488
public function testSumByPeriodBatchWithMissingMetric(): void
442489
{
443-
$this->usage->purge(DateTime::now());
490+
$this->usage->purge();
444491

445492
$this->assertTrue($this->usage->increment('batch-exists', 42));
446493

@@ -460,7 +507,7 @@ public function testSumByPeriodBatchEmpty(): void
460507

461508
public function testGetByPeriodBatch(): void
462509
{
463-
$this->usage->purge(DateTime::now());
510+
$this->usage->purge();
464511

465512
$this->assertTrue($this->usage->increment('batch-get-a', 10));
466513
$this->assertTrue($this->usage->increment('batch-get-b', 20));
@@ -482,7 +529,7 @@ public function testGetByPeriodBatch(): void
482529

483530
public function testGetByPeriodBatchWithMissingMetric(): void
484531
{
485-
$this->usage->purge(DateTime::now());
532+
$this->usage->purge();
486533

487534
$this->assertTrue($this->usage->increment('batch-get-exists', 99));
488535

@@ -501,7 +548,7 @@ public function testGetByPeriodBatchEmpty(): void
501548

502549
public function testSumByPeriodBatchConsistencyWithSumByPeriod(): void
503550
{
504-
$this->usage->purge(DateTime::now());
551+
$this->usage->purge();
505552

506553
$this->assertTrue($this->usage->increment('consistency-a', 15));
507554
$this->assertTrue($this->usage->increment('consistency-b', 25));
@@ -517,7 +564,7 @@ public function testSumByPeriodBatchConsistencyWithSumByPeriod(): void
517564

518565
public function testSumByPeriodBatchAcrossPeriods(): void
519566
{
520-
$this->usage->purge(DateTime::now());
567+
$this->usage->purge();
521568

522569
// increment() fans out to all periods
523570
$this->assertTrue($this->usage->increment('period-batch', 77));

0 commit comments

Comments
 (0)