diff --git a/app-config.yaml b/app-config.yaml index 1da8e2dfa..489afa20c 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -85,8 +85,15 @@ backend: # This is for local development only, it is not recommended to use this in production # The production database configuration is stored in app-config.production.yaml database: - client: better-sqlite3 - connection: ':memory:' + # client: better-sqlite3 + # connection: ':memory:' + # Use postgres instead of sqlite3 for testing + client: pg + connection: + host: localhost + port: 5432 + user: backstage + password: password # workingDirectory: /tmp # Use this to configure a working directory for the scaffolder, defaults to the OS temp-dir integrations: diff --git a/docker-compose.yaml b/docker-compose.yaml index 87303b2e6..4e48e8217 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -9,7 +9,7 @@ services: POSTGRES_USER: backstage POSTGRES_PASSWORD: password volumes: - - postgres:/var/lib/postgresql/data + - postgres:/var/lib/postgresql ports: - '127.0.0.1:5432:5432' diff --git a/plugins/renovate-backend/migrations/20251224_dependency_indexes.js b/plugins/renovate-backend/migrations/20251224_dependency_indexes.js new file mode 100644 index 000000000..ae523c509 --- /dev/null +++ b/plugins/renovate-backend/migrations/20251224_dependency_indexes.js @@ -0,0 +1,29 @@ +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + await knex.schema.alterTable('dependencies', table => { + table.index(['run_id'], 'runIdIndex'); + table.index(['extractionTimestamp'], 'extractionTimestampIndex'); + table.index( + ['host', 'repository', 'extractionTimestamp'], + 'latestLookupIndex', + ); + }); +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + await knex.schema.alterTable('dependencies', table => { + table.dropIndex(['run_id'], 'runIdIndex'); + table.dropIndex(['extractionTimestamp'], 'extractionTimestampIndex'); + table.dropIndex( + ['host', 'repository', 'extractionTimestamp'], + 'latestLookupIndex', + ); + }); +}; diff --git a/plugins/renovate-backend/src/service/databaseHandler.ts b/plugins/renovate-backend/src/service/databaseHandler.ts index 9efa82519..f8095cc15 100644 --- a/plugins/renovate-backend/src/service/databaseHandler.ts +++ b/plugins/renovate-backend/src/service/databaseHandler.ts @@ -185,7 +185,10 @@ export class DatabaseHandler { } } } - await this.client('dependencies').insert(dependencies); + + // SQLite has a limit on variables per query (typically 999-32766) + // With 16 columns per row, batch size of 50 is safe for all SQLite versions + await this.client.batchInsert('dependencies', dependencies, 50); } async getDependencies( @@ -194,15 +197,31 @@ export class DatabaseHandler { ): Promise> { const page = pagination?.page ?? 0; const pageSize = pagination?.pageSize ?? 500; - const builder = this.client('dependencies').select(); - - this.applyDependencyFilters(builder, filters); + const offset = page * pageSize; - const total = await this.getDependenciesCount(filters); + // Build subquery with filters + const filteredQuery = this.client('dependencies').select('*'); + this.applyDependencyFilters(filteredQuery, filters); + + // Use window function for total count in single query + const results = await this.client + .select('*') + .select(this.client.raw('COUNT(*) OVER() as total_count')) + .from< + DependencyRow & { total_count: string | number } + >(filteredQuery.as('filtered')) + .offset(offset) + .limit(pageSize); + + const total = results.length > 0 ? Number(results[0].total_count) : 0; + + // Remove total_count from results - properly typed + const cleanResults: DependencyRow[] = results.map( + ({ total_count, ...row }) => row, + ); - const offset = page * pageSize; return { - result: await builder.offset(offset).limit(pageSize), + result: cleanResults, total, page, pageSize, @@ -246,25 +265,22 @@ export class DatabaseHandler { } if (filters.latestOnly) { - const runIDs = this.client('dependencies') - .select('d.run_id') - .from('dependencies as d') - .join( - this.client('dependencies') - .select('host', 'repository') - .max('extractionTimestamp as max_timestamp') - .groupBy('host', 'repository') - .as('max_d'), - // eslint-disable-next-line func-names - function () { - this.on('d.host', '=', 'max_d.host') - .andOn('d.repository', '=', 'max_d.repository') - .andOn('d.extractionTimestamp', '=', 'max_d.max_timestamp'); - }, + const rankedSubquery = this.client + .select('run_id') + .select( + this.client.raw( + 'ROW_NUMBER() OVER (PARTITION BY host, repository ORDER BY "extractionTimestamp" DESC) as rn', + ), ) - .distinct(); + .from('dependencies') + .as('ranked'); - builder.whereIn('run_id', runIDs); + const latestRunIds = this.client + .select('run_id') + .from(rankedSubquery) + .where('rn', 1); + + builder.whereIn('run_id', latestRunIds); } } @@ -276,25 +292,44 @@ export class DatabaseHandler { async getDependenciesValues( filters?: DependencyValueFilters, ): Promise { - const baseBuilder = this.client( - 'dependencies', - ); - const limitedValuesBuilder = baseBuilder.clone(); - - const allValuesKeys: DependencyValueFiltersKey[] = []; - const limitedValuesKeys: DependencyValueFiltersKey[] = []; - for (const filterKey of DependencyValueFiltersKeys) { - const suppliedFilter = filters?.[filterKey]; - // if no filter is supplied, return all values - if (suppliedFilter) { - limitedValuesBuilder.whereIn(filterKey, suppliedFilter); - limitedValuesKeys.push(filterKey); - continue; + // Build base builder with filters applied + const baseBuilder = this.client('dependencies'); + + // Apply filters to create the filtered dataset + const filteredKeys: DependencyValueFiltersKey[] = []; + if (filters) { + for (const [key, value] of Object.entries(filters)) { + if ( + value && + DependencyValueFiltersKeys.includes(key as DependencyValueFiltersKey) + ) { + baseBuilder.whereIn(key, value); + filteredKeys.push(key as DependencyValueFiltersKey); + } } - - allValuesKeys.push(filterKey); } + // Use UNION ALL to get all distinct values in one roundtrip + // For each column, we need the values from either filtered or unfiltered dataset + const queries = DependencyValueFiltersKeys.map(key => { + const builder = filteredKeys.includes(key) + ? this.client('dependencies') // unfiltered for this column + : baseBuilder.clone(); // filtered for other columns + + return builder + .select(this.client.raw('? as column_name', [key])) + .select(this.client.raw(`?? as value`, [key])) + .distinct(); + }); + + // Combine all queries + const combinedQuery = queries.reduce((acc, q, i) => + i === 0 ? q : acc.unionAll(q), + ); + + const rows = await combinedQuery; + + // Group results by column name const result: DependencyValues = { datasource: [], manager: [], @@ -305,27 +340,12 @@ export class DatabaseHandler { repository: [], }; - const allValues = allValuesKeys.map( - async (filterKey: DependencyValueFiltersKey) => { - // get all unique values for the column and do not return as object but list - const values = await limitedValuesBuilder - .clone() - .select(filterKey) - .distinct() - .pluck(filterKey); - result[filterKey] = values.filter(is.string); - }, - ); - const limitedValues = limitedValuesKeys.map(async filterKey => { - const values = await baseBuilder - .clone() - .select(filterKey) - .distinct() - .pluck(filterKey); - - result[filterKey] = values.filter(is.string); - }); - await Promise.all([...allValues, ...limitedValues]); + for (const row of rows) { + if (row.value && is.string(row.value)) { + result[row.column_name as DependencyValueFiltersKey].push(row.value); + } + } + return result; } @@ -344,21 +364,21 @@ export class DatabaseHandler { ): Promise { const offset = getOffset(options); - const dependencies = this.client('dependencies') + // Get ordered run_ids - must include extractionTimestamp for ORDER BY with DISTINCT + const orderedRuns = this.client('dependencies') .select('run_id', 'extractionTimestamp') - .distinct('host', 'repository') .where('host', host) .andWhere('repository', repository) - .as('runs'); - - const toBeDeletedIDs = this.client(dependencies) - .select('run_id') .orderBy('extractionTimestamp', 'DESC') - .offset(offset); + .offset(offset) + .distinct() + .as('ordered'); + + const toBeDeletedIDs = this.client.select('run_id').from(orderedRuns); return this.client('dependencies') .delete() - .whereIn('run_id', [toBeDeletedIDs]); + .whereIn('run_id', toBeDeletedIDs); } }