Skip to content

Commit a2203ae

Browse files
abnegateabnegate
authored
Revert "Optimize reports fetching"
1 parent b3c51e3 commit a2203ae

2 files changed

Lines changed: 42 additions & 136 deletions

File tree

‎src/Migration/Sources/Appwrite.php‎

Lines changed: 26 additions & 103 deletions
Original file line numberDiff line numberDiff line change
@@ -199,59 +199,18 @@ public function report(array $resources = []): array
199199
*/
200200
private function reportAuth(array $resources, array &$report): void
201201
{
202-
// check if we need to fetch teams!
203-
$needTeams = !empty(array_intersect(
204-
[Resource::TYPE_TEAM, Resource::TYPE_MEMBERSHIP],
205-
$resources
206-
));
207-
208-
$pageLimit = 25;
209-
$teams = ['total' => 0, 'teams' => []];
210-
211202
if (\in_array(Resource::TYPE_USER, $resources)) {
212-
$report[Resource::TYPE_USER] = $this->users->list(
213-
[Query::limit(1)]
214-
)['total'];
215-
}
216-
217-
if ($needTeams) {
218-
if (\in_array(Resource::TYPE_MEMBERSHIP, $resources)) {
219-
$allTeams = [];
220-
$lastTeam = null;
221-
222-
while (true) {
223-
$params = $lastTeam
224-
// TODO: should we use offset here?
225-
// this, realistically, shouldn't be too much ig
226-
? [Query::cursorAfter($lastTeam)]
227-
: [Query::limit($pageLimit)];
228-
229-
$teamList = $this->teams->list($params);
230-
231-
$totalTeams = $teamList['total'];
232-
$currentTeams = $teamList['teams'];
233-
234-
$allTeams = array_merge($allTeams, $currentTeams);
235-
$lastTeam = $currentTeams[count($currentTeams) - 1]['$id'] ?? null;
236-
237-
if (count($currentTeams) < $pageLimit) {
238-
break;
239-
}
240-
}
241-
$teams = ['total' => $totalTeams, 'teams' => $allTeams];
242-
} else {
243-
$teamList = $this->teams->list([Query::limit(1)]);
244-
$teams = ['total' => $teamList['total'], 'teams' => []];
245-
}
203+
$report[Resource::TYPE_USER] = $this->users->list()['total'];
246204
}
247205

248206
if (\in_array(Resource::TYPE_TEAM, $resources)) {
249-
$report[Resource::TYPE_TEAM] = $teams['total'];
207+
$report[Resource::TYPE_TEAM] = $this->teams->list()['total'];
250208
}
251209

252210
if (\in_array(Resource::TYPE_MEMBERSHIP, $resources)) {
253211
$report[Resource::TYPE_MEMBERSHIP] = 0;
254-
foreach ($teams['teams'] as $team) {
212+
$teams = $this->teams->list()['teams'];
213+
foreach ($teams as $team) {
255214
$report[Resource::TYPE_MEMBERSHIP] += $this->teams->listMemberships(
256215
$team['$id'],
257216
[Query::limit(1)]
@@ -277,14 +236,9 @@ private function reportDatabases(array $resources, array &$report): void
277236
private function reportStorage(array $resources, array &$report): void
278237
{
279238
if (\in_array(Resource::TYPE_BUCKET, $resources)) {
280-
// just fetch one bucket for the `total`
281-
$report[Resource::TYPE_BUCKET] = $this->storage->listBuckets([
282-
Query::limit(1)
283-
])['total'];
239+
$report[Resource::TYPE_BUCKET] = $this->storage->listBuckets()['total'];
284240
}
285241

286-
$pageLimit = 25;
287-
288242
if (\in_array(Resource::TYPE_FILE, $resources)) {
289243
$report[Resource::TYPE_FILE] = 0;
290244
$report['size'] = 0;
@@ -295,89 +249,58 @@ private function reportStorage(array $resources, array &$report): void
295249
$currentBuckets = $this->storage->listBuckets(
296250
$lastBucket
297251
? [Query::cursorAfter($lastBucket)]
298-
: [Query::limit($pageLimit)]
252+
: [Query::limit(20)]
299253
)['buckets'];
300254

301255
$buckets = array_merge($buckets, $currentBuckets);
302256
$lastBucket = $buckets[count($buckets) - 1]['$id'] ?? null;
303257

304-
if (count($currentBuckets) < $pageLimit) {
258+
if (count($currentBuckets) < 20) {
305259
break;
306260
}
307261
}
308262

309263
foreach ($buckets as $bucket) {
264+
$files = [];
310265
$lastFile = null;
266+
311267
while (true) {
312-
$files = $this->storage->listFiles(
268+
$currentFiles = $this->storage->listFiles(
313269
$bucket['$id'],
314270
$lastFile
315271
? [Query::cursorAfter($lastFile)]
316-
: [Query::limit($pageLimit)]
272+
: [Query::limit(20)]
317273
)['files'];
318274

319-
$report[Resource::TYPE_FILE] += count($files);
320-
foreach ($files as $file) {
321-
// already includes the `sizeOriginal`
322-
$report['size'] += $file['sizeOriginal'] ?? 0;
323-
}
324-
275+
$files = array_merge($files, $currentFiles);
325276
$lastFile = $files[count($files) - 1]['$id'] ?? null;
326277

327-
if (count($files) < $pageLimit) {
278+
if (count($currentFiles) < 20) {
328279
break;
329280
}
330281
}
331-
}
332282

283+
$report[Resource::TYPE_FILE] += count($files);
284+
foreach ($files as $file) {
285+
$report['size'] += $this->storage->getFile(
286+
$bucket['$id'],
287+
$file['$id']
288+
)['sizeOriginal'];
289+
}
290+
}
333291
$report['size'] = $report['size'] / 1000 / 1000; // MB
334292
}
335293
}
336294

337295
private function reportFunctions(array $resources, array &$report): void
338296
{
339-
$pageLimit = 25;
340-
$needVarsOrDeployments = (
341-
\in_array(Resource::TYPE_DEPLOYMENT, $resources) ||
342-
\in_array(Resource::TYPE_ENVIRONMENT_VARIABLE, $resources)
343-
);
344-
345-
$functions = [];
346-
$totalFunctions = 0;
347-
348-
if (!$needVarsOrDeployments && \in_array(Resource::TYPE_FUNCTION, $resources)) {
349-
// Only function count needed, short-circuit
350-
$funcList = $this->functions->list([Query::limit(1)]);
351-
$report[Resource::TYPE_FUNCTION] = $funcList['total'];
352-
return;
353-
}
354-
355-
if ($needVarsOrDeployments) {
356-
$lastFunction = null;
357-
while (true) {
358-
$params = $lastFunction
359-
? [Query::cursorAfter($lastFunction)]
360-
: [Query::limit($pageLimit)];
361-
362-
$funcList = $this->functions->list($params);
363-
364-
$totalFunctions = $funcList['total'];
365-
$currentFunctions = $funcList['functions'];
366-
$functions = array_merge($functions, $currentFunctions);
367-
368-
$lastFunction = $currentFunctions[count($currentFunctions) - 1]['$id'] ?? null;
369-
if (count($currentFunctions) < $pageLimit) {
370-
break;
371-
}
372-
}
373-
}
374-
375297
if (\in_array(Resource::TYPE_FUNCTION, $resources)) {
376-
$report[Resource::TYPE_FUNCTION] = $totalFunctions;
298+
$report[Resource::TYPE_FUNCTION] = $this->functions->list()['total'];
377299
}
378300

379301
if (\in_array(Resource::TYPE_DEPLOYMENT, $resources)) {
380302
$report[Resource::TYPE_DEPLOYMENT] = 0;
303+
$functions = $this->functions->list()['functions'];
381304
foreach ($functions as $function) {
382305
if (!empty($function['deploymentId'])) {
383306
$report[Resource::TYPE_DEPLOYMENT] += 1;
@@ -387,9 +310,9 @@ private function reportFunctions(array $resources, array &$report): void
387310

388311
if (\in_array(Resource::TYPE_ENVIRONMENT_VARIABLE, $resources)) {
389312
$report[Resource::TYPE_ENVIRONMENT_VARIABLE] = 0;
313+
$functions = $this->functions->list()['functions'];
390314
foreach ($functions as $function) {
391-
// function model contains `vars`, we don't need to fetch the list again.
392-
$report[Resource::TYPE_ENVIRONMENT_VARIABLE] += count($function['vars'] ?? []);
315+
$report[Resource::TYPE_ENVIRONMENT_VARIABLE] += $this->functions->listVariables($function['$id'])['total'];
393316
}
394317
}
395318
}
@@ -1374,7 +1297,7 @@ private function exportFunctions(int $batchSize): void
13741297
$function['events'],
13751298
$function['schedule'],
13761299
$function['timeout'],
1377-
$function['deploymentId'] ?? '',
1300+
$function['deploymentId'],
13781301
$function['entrypoint']
13791302
);
13801303

‎src/Migration/Sources/Appwrite/Reader/API.php‎

Lines changed: 16 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -61,56 +61,39 @@ public function report(array $resources, array &$report): mixed
6161
foreach ($databases as $database) {
6262
$databaseId = $database['$id'];
6363

64-
$tables = [];
65-
$pageLimit = 25;
66-
$lastTable = null;
67-
68-
while (true) {
69-
/* $currentTables = $this->tables->list(...); */
70-
$currentTables = $this->database->listCollections(
71-
$databaseId,
72-
$lastTable
73-
? [Query::cursorAfter($lastTable)]
74-
: [Query::limit($pageLimit)]
75-
)['collections']; /* ['tables'] */
76-
77-
$tables = array_merge($tables, $currentTables);
78-
$lastTable = $tables[count($tables) - 1]['$id'] ?? null;
79-
80-
if (count($currentTables) < $pageLimit) {
81-
break;
82-
}
83-
}
64+
/* $tablesResponse = $this->tables->list(...); */
65+
$tablesResponse = $this->database->listCollections($databaseId);
66+
$tables = $tablesResponse['collections'];
8467

8568
if (Resource::isSupported(Resource::TYPE_TABLE, $resources)) {
86-
$report[Resource::TYPE_TABLE] += count($tables);
69+
$report[Resource::TYPE_TABLE] += $tablesResponse['total'];
8770
}
8871

8972
if (Resource::isSupported([Resource::TYPE_ROW, Resource::TYPE_COLUMN, Resource::TYPE_INDEX], $resources)) {
9073
foreach ($tables as $table) {
9174
$tableId = $table['$id'];
9275

93-
if (Resource::isSupported(Resource::TYPE_COLUMN, $resources)) {
94-
// a table already returns a list of attributes
95-
$report[Resource::TYPE_COLUMN] += count($table['columns'] ?? $table['attributes'] ?? []);
96-
}
97-
98-
if (in_array(Resource::TYPE_INDEX, $resources)) {
99-
// a table already returns a list of indexes
100-
$report[Resource::TYPE_INDEX] += count($table['indexes'] ?? []);
101-
}
102-
103-
// this one's a bit heavy if the number of tables are high!
10476
if (Resource::isSupported(Resource::TYPE_ROW, $resources)) {
10577
/* $rowsResponse = $this->tables->listRows(...) */
10678
$rowsResponse = $this->database->listDocuments(
10779
$databaseId,
10880
$tableId,
10981
[Query::limit(1)]
11082
);
111-
11283
$report[Resource::TYPE_ROW] += $rowsResponse['total'];
11384
}
85+
86+
if (Resource::isSupported(Resource::TYPE_COLUMN, $resources)) {
87+
/* $columnsResponse = $this->tables->listColumns(...); */
88+
$columnsResponse = $this->database->listAttributes($databaseId, $tableId);
89+
$report[Resource::TYPE_COLUMN] += $columnsResponse['total'];
90+
}
91+
92+
if (in_array(Resource::TYPE_INDEX, $resources)) {
93+
/* $indexesResponse = $this->tables->listIndexes(...); */
94+
$indexesResponse = $this->database->listIndexes($databaseId, $tableId);
95+
$report[Resource::TYPE_INDEX] += $indexesResponse['total'];
96+
}
11497
}
11598
}
11699
}

0 commit comments

Comments
 (0)