Skip to content

Commit fa0e373

Browse files
committed
Chunk id-lookup queries to respect RELATION_QUERY_CHUNK_SIZE
Addresses CodeRabbit #3084994641 and #3084994657 (+ Greptile #3084466265). Three call sites were passing unchunked id arrays into Query::equal('$id', ...), which throws QueryException from DocumentsValidator once the array exceeds 5000 values. Mirror::createDocuments (skipDuplicates pre-filter): defensive — Mirror's public API accepts arbitrary-size input, and while current callers pass batches below the limit, the code shouldn't throw on a theoretical direct caller with >5000 input ids. Database::upsertDocumentsWithIncrease (both tenantPerDocument and single- tenant branches): real regression. The batched pre-read was introduced in ae929db to restore per-tenant grouping for StatsUsage. It replaced a per-doc getDocument() loop on main (which is safe from validator limits because each call queries one id). Passing the full input array to Query::equal now breaks at >5000 input docs — a path that previously worked slowly on main. All three sites use self::RELATION_QUERY_CHUNK_SIZE (5000) and Query::limit(PHP_INT_MAX), matching the existing pattern used by the five populateDocumentsRelationshipsBatch chunking sites. Accumulator shape is adapted to our map-only use case: we build the existence set / lookup map directly inside the chunk loop instead of carrying a flat list we don't need.
1 parent 934ec04 commit fa0e373

2 files changed

Lines changed: 30 additions & 24 deletions

File tree

src/Database/Database.php

Lines changed: 20 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -7159,14 +7159,16 @@ public function upsertDocumentsWithIncrease(
71597159
}
71607160
foreach ($idsByTenant as $tenant => $tenantIds) {
71617161
$tenantIds = \array_values(\array_unique($tenantIds));
7162-
$found = $this->authorization->skip(fn () => $this->withTenant($tenant, fn () => $this->silent(
7163-
fn () => $this->find($collection->getId(), [
7164-
Query::equal('$id', $tenantIds),
7165-
Query::limit(\count($tenantIds)),
7166-
])
7167-
)));
7168-
foreach ($found as $doc) {
7169-
$existingDocs[$tenant . ':' . $doc->getId()] = $doc;
7162+
foreach (\array_chunk($tenantIds, self::RELATION_QUERY_CHUNK_SIZE) as $chunk) {
7163+
$found = $this->authorization->skip(fn () => $this->withTenant($tenant, fn () => $this->silent(
7164+
fn () => $this->find($collection->getId(), [
7165+
Query::equal('$id', $chunk),
7166+
Query::limit(PHP_INT_MAX),
7167+
])
7168+
)));
7169+
foreach ($found as $doc) {
7170+
$existingDocs[$tenant . ':' . $doc->getId()] = $doc;
7171+
}
71707172
}
71717173
}
71727174
} else {
@@ -7176,14 +7178,16 @@ public function upsertDocumentsWithIncrease(
71767178
)));
71777179

71787180
if (!empty($docIds)) {
7179-
$existing = $this->authorization->skip(fn () => $this->silent(
7180-
fn () => $this->find($collection->getId(), [
7181-
Query::equal('$id', $docIds),
7182-
Query::limit(\count($docIds)),
7183-
])
7184-
));
7185-
foreach ($existing as $doc) {
7186-
$existingDocs[$this->tenantKey($doc)] = $doc;
7181+
foreach (\array_chunk($docIds, self::RELATION_QUERY_CHUNK_SIZE) as $chunk) {
7182+
$existing = $this->authorization->skip(fn () => $this->silent(
7183+
fn () => $this->find($collection->getId(), [
7184+
Query::equal('$id', $chunk),
7185+
Query::limit(PHP_INT_MAX),
7186+
])
7187+
));
7188+
foreach ($existing as $doc) {
7189+
$existingDocs[$this->tenantKey($doc)] = $doc;
7190+
}
71877191
}
71887192
}
71897193
}

src/Database/Mirror.php

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -612,14 +612,16 @@ public function createDocuments(
612612
));
613613

614614
if (!empty($ids)) {
615-
$existing = $this->source->silent(
616-
fn () => $this->source->find($collection, [
617-
Query::equal('$id', $ids),
618-
Query::limit(\count($ids)),
619-
])
620-
);
621-
foreach ($existing as $doc) {
622-
$existingIds[$doc->getId()] = true;
615+
foreach (\array_chunk(\array_unique($ids), self::RELATION_QUERY_CHUNK_SIZE) as $chunk) {
616+
$existing = $this->source->silent(
617+
fn () => $this->source->find($collection, [
618+
Query::equal('$id', $chunk),
619+
Query::limit(PHP_INT_MAX),
620+
])
621+
);
622+
foreach ($existing as $doc) {
623+
$existingIds[$doc->getId()] = true;
624+
}
623625
}
624626
}
625627
}

0 commit comments

Comments
 (0)