diff --git a/lib/database/adapters/RunAdapter.js b/lib/database/adapters/RunAdapter.js index 2cd8e84c14..0211f7bbef 100644 --- a/lib/database/adapters/RunAdapter.js +++ b/lib/database/adapters/RunAdapter.js @@ -75,6 +75,11 @@ class RunAdapter { */ this.userAdapter = null; + /** + * @type {QcFlagAdapter|null} + */ + this.qcFlagAdapter = null; + this.toEntity = this.toEntity.bind(this); this.toDatabase = this.toDatabase.bind(this); this.toMinifiedEntity = this.toMinifiedEntity.bind(this); @@ -167,6 +172,7 @@ class RunAdapter { phaseShiftAtEndBeam2, userStart, userStop, + qcFlags, } = databaseObject; /** @@ -285,6 +291,13 @@ class RunAdapter { ? lhcFill.collidingBunchesCount ?? extractNumberOfCollidingLhcBunchCrossings(lhcFill.fillingSchemeName) : null; + const adaptedQcFlags = qcFlags ? qcFlags.map(this.qcFlagAdapter.toEntity) : []; + entityObject.qcFlags = adaptedQcFlags.reduce((acc, qcFlag) => { + acc[qcFlag.dplDetectorId] = acc[qcFlag.dplDetectorId] ?? []; + acc[qcFlag.dplDetectorId].push(qcFlag); + return acc; + }, {}); + return entityObject; } diff --git a/lib/database/adapters/index.js b/lib/database/adapters/index.js index 6566fd1b95..5ff6404b3d 100644 --- a/lib/database/adapters/index.js +++ b/lib/database/adapters/index.js @@ -139,6 +139,7 @@ runAdapter.logAdapter = logAdapter; runAdapter.runTypeAdapter = runTypeAdapter; runAdapter.tagAdapter = tagAdapter; runAdapter.userAdapter = userAdapter; +runAdapter.qcFlagAdapter = qcFlagAdapter; simulationPassQcFlagAdapter.simulationPassAdapter = simulationPassAdapter; simulationPassQcFlagAdapter.qcFlagAdapter = qcFlagAdapter; diff --git a/lib/database/seeders/20240404100811-qc-flags.js b/lib/database/seeders/20240404100811-qc-flags.js index 4ff17be617..b4933d7aa0 100644 --- a/lib/database/seeders/20240404100811-qc-flags.js +++ b/lib/database/seeders/20240404100811-qc-flags.js @@ -298,6 +298,12 @@ module.exports = { from: '2019-08-08 13:46:40', to: '2019-08-09 07:50:00', }, + { + id: 6, + flag_id: 6, + from: '2019-08-09 08:50:00', + to: null, + }, { id: 7, flag_id: 7, diff --git a/lib/domain/dtos/GetAllRunsDto.js b/lib/domain/dtos/GetAllRunsDto.js index 1c886740cf..4f2eab197f 100644 --- a/lib/domain/dtos/GetAllRunsDto.js +++ b/lib/domain/dtos/GetAllRunsDto.js @@ -15,11 +15,24 @@ const Joi = require('joi'); const PaginationDto = require('./PaginationDto'); const { RunFilterDto } = require('./filters/RunFilterDto.js'); const { DtoFactory } = require('./DtoFactory'); +const { singleRunsCollectionCustomCheck } = require('./utils.js'); const QueryDto = Joi.object({ filter: RunFilterDto, page: PaginationDto, sort: DtoFactory.order(['id', 'runNumber', 'text']), + include: Joi.object({ effectiveQcFlags: Joi.boolean().custom((effectiveQcFlags, helpers) => { + const [, { filter: { dataPassIds, simulationPassIds, lhcPeriodIds } = {} }] = helpers.state.ancestors; + + singleRunsCollectionCustomCheck( + { dataPassIds, simulationPassIds, lhcPeriodIds }, + helpers, + 'Including effectiveQcFlags is allowed only when ' + + 'the dataPassIds, simulationPassIds and lhcPeriodIds filters collectively contain exactly one ID', + ); + + return effectiveQcFlags; + }) }), token: Joi.string(), }); diff --git a/lib/domain/dtos/filters/RunFilterDto.js b/lib/domain/dtos/filters/RunFilterDto.js index 2f9a2d9ba5..0c8c032b63 100644 --- a/lib/domain/dtos/filters/RunFilterDto.js +++ b/lib/domain/dtos/filters/RunFilterDto.js @@ -17,6 +17,7 @@ const { RUN_QUALITIES } = require('../../enums/RunQualities.js'); const { IntegerComparisonDto, FloatComparisonDto } = require('./NumericalComparisonDto.js'); const { RUN_CALIBRATION_STATUS } = require('../../enums/RunCalibrationStatus.js'); const { RUN_DEFINITIONS } = require('../../enums/RunDefinition.js'); +const { singleRunsCollectionCustomCheck } = require('../utils.js'); const DetectorsFilterDto = Joi.object({ operator: Joi.string().valid('or', 'and', 'none').required(), @@ -128,12 +129,13 @@ exports.RunFilterDto = Joi.object({ }) .custom((detectorsQcObj, helpers) => { const [{ dataPassIds, simulationPassIds, lhcPeriodIds }] = helpers.state.ancestors; - const runsCollectionFilters = [dataPassIds, simulationPassIds, lhcPeriodIds].filter(({ length } = {}) => length >= 1); - if (runsCollectionFilters.length !== 1 || runsCollectionFilters[0].length !== 1) { - return helpers.message('Filtering by detector not-bad fraction is allowed only with exactly one of: ' + - 'dataPassIds, simulationPassIds, lhcPeriodIds with exactly one ID.'); - } + singleRunsCollectionCustomCheck( + { dataPassIds, simulationPassIds, lhcPeriodIds }, + helpers, + 'Filtering by detector not-bad fraction is allowed only when ' + + 'the dataPassIds, simulationPassIds and lhcPeriodIds filters collectively contain exactly one ID', + ); return detectorsQcObj; }), diff --git a/lib/domain/dtos/utils.js b/lib/domain/dtos/utils.js new file mode 100644 index 0000000000..34be3ee2ca --- /dev/null +++ b/lib/domain/dtos/utils.js @@ -0,0 +1,33 @@ +/** + * @license + * Copyright CERN and copyright holders of ALICE O2. This software is + * distributed under the terms of the GNU General Public License v3 (GPL + * Version 3), copied verbatim in the file "COPYING". + * + * See http://alice-o2.web.cern.ch/license for full licensing information. + * + * In applying this license CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization + * or submit itself to any jurisdiction. + */ + +/** + * Used in Joi schemas to assert that the dataPassIds, simulationPassIds and lhcPeriodIds filters collectively contain exactly one ID + * + * @param {object} collections runs ids collections + * @param {number[]} collections.dataPassIds data pass ids + * @param {number[]} collections.simulationPassIds simulation pass ids + * @param {number[]} collections.lhcPeriodIds LHC periods ids + * @param {Joi.helpers} helpers joi helpers + * @param {string} message to be send in case of filters refer to more than one runs collection + * @returns {void} + */ +const singleRunsCollectionCustomCheck = ({ dataPassIds, simulationPassIds, lhcPeriodIds }, helpers, message) => { + const runsCollectionFilters = [dataPassIds, simulationPassIds, lhcPeriodIds].filter(({ length } = {}) => length >= 1); + + if (runsCollectionFilters.length !== 1 || runsCollectionFilters[0].length !== 1) { + return helpers.message(message); + } +}; + +module.exports.singleRunsCollectionCustomCheck = singleRunsCollectionCustomCheck; diff --git a/lib/usecases/run/GetAllRunsUseCase.js b/lib/usecases/run/GetAllRunsUseCase.js index ea8ddf03f1..1af0e385f2 100644 --- a/lib/usecases/run/GetAllRunsUseCase.js +++ b/lib/usecases/run/GetAllRunsUseCase.js @@ -36,7 +36,7 @@ class GetAllRunsUseCase { async execute(dto = {}) { const filteringQueryBuilder = new QueryBuilder(); const { query = {} } = dto; - const { filter, page = {}, sort = { runNumber: 'desc' } } = query; + const { filter, page = {}, sort = { runNumber: 'desc' }, include: { effectiveQcFlags = false } = {} } = query; const SEARCH_ITEMS_SEPARATOR = ','; @@ -343,6 +343,19 @@ class GetAllRunsUseCase { }); } + if (lhcPeriodIds) { + const runNumbers = (await RunRepository.findAll({ + attributes: ['runNumber'], + raw: true, + include: { + association: 'lhcPeriod', + attributes: [], + where: { id: { [Op.in]: lhcPeriodIds } }, + }, + })).map(({ runNumber }) => runNumber); + filteringQueryBuilder.where('runNumber').oneOf(...runNumbers); + } + if (dataPassIds) { const runNumbers = (await RunRepository.findAll({ attributes: ['runNumber'], @@ -445,7 +458,7 @@ class GetAllRunsUseCase { attributes: ['runNumber'], raw: true, include: { - association: 'lhcPeriods', + association: 'lhcPeriod', attributes: [], where: { id: { [Op.in]: lhcPeriodIds } }, }, @@ -516,6 +529,60 @@ class GetAllRunsUseCase { fetchQueryBuilder.orderBy(property, sort[property]); } + if (effectiveQcFlags) { + const [dataPassId] = filter?.dataPassIds ?? []; + const [simulationPassId] = filter?.simulationPassIds ?? []; + const [lhcPeriodId] = filter?.lhcPeriodIds ?? []; + + if (Boolean(dataPassId) + Boolean(simulationPassId) + Boolean(lhcPeriodId) !== 1) { + throw new BadParameterError('If including QC flags, one and exactly one' + + 'of `dataPassId`, `simulationPassId` or `lhcPeriodId` is required'); + } + + const commondQcFlagsAssociationDef = { + association: 'qcFlags', + required: false, + where: { deleted: false }, + }; + + if (dataPassId) { + fetchQueryBuilder.include({ + ...commondQcFlagsAssociationDef, + include: [ + { association: 'dataPasses', required: true, where: { id: dataPassId } }, + { association: 'effectivePeriods', required: true }, + ], + }); + } else if (simulationPassId) { + fetchQueryBuilder.include({ + ...commondQcFlagsAssociationDef, + include: [ + { association: 'simulationPasses', required: true, where: { id: simulationPassId } }, + { association: 'effectivePeriods', required: true }, + ], + }); + } else { + fetchQueryBuilder.include({ + ...commondQcFlagsAssociationDef, + include: [ + { association: 'simulationPasses', required: false }, + { association: 'dataPasses', required: false }, + { association: 'effectivePeriods', required: true }, + ], + where: { + deleted: false, + [Op.or]: [ + { '$qcFlags.id$': null }, + { + '$qcFlags.dataPasses.id$': null, + '$qcFlags.simulationPasses.id$': null, + }, + ], + }, + }); + } + } + const runs = (await RunRepository.findAll(fetchQueryBuilder)).map(runAdapter.toEntity); return { count, runs }; diff --git a/test/api/qcFlags.test.js b/test/api/qcFlags.test.js index f511365c58..ad43a30cdc 100644 --- a/test/api/qcFlags.test.js +++ b/test/api/qcFlags.test.js @@ -150,9 +150,9 @@ module.exports = () => { expect(data).to.be.eql({ 106: { 1: { - missingVerificationsCount: 1, + missingVerificationsCount: 2, mcReproducible: false, - badEffectiveRunCoverage: 0.7222222, + badEffectiveRunCoverage: 0.9288889, explicitlyNotBadEffectiveRunCoverage: 0, }, }, diff --git a/test/api/runs.test.js b/test/api/runs.test.js index 45973e6e8b..1e12b1a202 100644 --- a/test/api/runs.test.js +++ b/test/api/runs.test.js @@ -21,6 +21,7 @@ const { RunDetectorQualities } = require('../../lib/domain/enums/RunDetectorQual const { RunCalibrationStatus } = require('../../lib/domain/enums/RunCalibrationStatus.js'); const { updateRun } = require('../../lib/server/services/run/updateRun.js'); const { RunDefinition } = require('../../lib/domain/enums/RunDefinition.js'); +const { qcFlagService } = require('../../lib/server/services/qualityControlFlag/QcFlagService.js'); module.exports = () => { before(resetDatabaseContent); @@ -606,6 +607,39 @@ module.exports = () => { expect(runs.every(({ aliceDipoleCurrent, aliceDipolePolarity }) => Math.round(aliceDipoleCurrent * (aliceDipolePolarity === 'NEGATIVE' ? -1 : 1) / 1000) === 0)).to.be.true; }); + + it('should successfully handle query including QC flags', async () => { + { // Data Passes + const response = await request(server).get(`/api/runs?filter[dataPassIds][]=1&include[effectiveQcFlags]=true`) + expect(response.status).to.equal(200); + const { data: runs } = response.body; + + expect(runs).to.have.lengthOf(3); + expect(runs.find(({ runNumber }) => runNumber === 107).qcFlags['1'].map(({ id }) => id)).to.have.all.members([202, 201]); + expect(runs.find(({ runNumber }) => runNumber === 107).qcFlags['2'].map(({ id }) => id)).to.have.all.members([203]); + expect(runs.find(({ runNumber }) => runNumber === 106).qcFlags['1'].map(({ id }) => id)).to.have.all.members([3, 2, 1]); + } + + { // Simulation Passes + const response = await request(server).get(`/api/runs?filter[simulationPassIds][]=1&include[effectiveQcFlags]=true`) + expect(response.status).to.equal(200); + const { data: runs } = response.body; + expect(runs).to.have.lengthOf(3); + + console.log('TOBEC', await qcFlagService.getAllPerSimulationPassAndRunAndDetector({ simulationPassId: 1, runNumber: 106, detectorId: 1})); + + expect(runs.find(({ runNumber }) => runNumber === 106).qcFlags['1'].map(({ id }) => id)).to.have.all.members([6, 5]); + } + + { // Synchronous flags + const response = await request(server).get(`/api/runs?filter[lhcPeriodIds][]=1&include[effectiveQcFlags]=true`) + expect(response.status).to.equal(200); + const { data: runs } = response.body; + expect(runs).to.have.lengthOf(4); + expect(runs.find(({ runNumber }) => runNumber === 56).qcFlags['7'].map(({ id }) => id)).to.have.all.members([101, 100]); + expect(runs.find(({ runNumber }) => runNumber === 56).qcFlags['4'].map(({ id }) => id)).to.have.all.members([102]); + } + }); }); describe('GET /api/runs/reasonTypes', () => { diff --git a/test/lib/server/services/qualityControlFlag/QcFlagService.test.js b/test/lib/server/services/qualityControlFlag/QcFlagService.test.js index 16e05d3c2c..0636cb833a 100644 --- a/test/lib/server/services/qualityControlFlag/QcFlagService.test.js +++ b/test/lib/server/services/qualityControlFlag/QcFlagService.test.js @@ -290,9 +290,9 @@ module.exports = () => { expect(await qcFlagSummaryService.getSummary({ simulationPassId: 1 })).to.be.eql({ 106: { 1: { - missingVerificationsCount: 1, + missingVerificationsCount: 2, mcReproducible: false, - badEffectiveRunCoverage: 0.7222222, + badEffectiveRunCoverage: 0.9288889, explicitlyNotBadEffectiveRunCoverage: 0, }, }, diff --git a/test/lib/usecases/run/GetAllRunsUseCase.test.js b/test/lib/usecases/run/GetAllRunsUseCase.test.js index d882dc4a94..d8c1f788cb 100644 --- a/test/lib/usecases/run/GetAllRunsUseCase.test.js +++ b/test/lib/usecases/run/GetAllRunsUseCase.test.js @@ -17,6 +17,8 @@ const chai = require('chai'); const { RunQualities } = require('../../../../lib/domain/enums/RunQualities.js'); const { RunCalibrationStatus } = require('../../../../lib/domain/enums/RunCalibrationStatus.js'); const { RunDefinition } = require('../../../../lib/domain/enums/RunDefinition.js'); +const assert = require('assert'); +const { BadParameterError } = require('../../../../lib/server/errors/BadParameterError.js'); const { expect } = chai; @@ -795,7 +797,7 @@ module.exports = () => { expect(runs).to.have.lengthOf(0); } }); - + it('should successfully filter by detectors notBadFraction', async () => { const dataPassIds = [1]; { @@ -849,4 +851,57 @@ module.exports = () => { expect(runs.map(({ runNumber }) => runNumber)).to.have.all.members([107]); } }); + + it('should successfully handle query including QC flags', async () => { + { + await assert.rejects(() => new GetAllRunsUseCase().execute({ + query: { + include: { effectiveQcFlags: true }, + }, + }), new BadParameterError('If including QC flags, one and exactly one' + + 'of `dataPassId`, `simulationPassId` or `lhcPeriodId` is required')); + } + + { // Data Passes + const { runs } = await new GetAllRunsUseCase().execute({ + query: { + filter: { + dataPassIds: [1], + }, + include: { effectiveQcFlags: true }, + }, + }); + expect(runs).to.have.lengthOf(3); + expect(runs.find(({ runNumber }) => runNumber === 107).qcFlags['1'].map(({ id }) => id)).to.have.all.members([202, 201]); + expect(runs.find(({ runNumber }) => runNumber === 107).qcFlags['2'].map(({ id }) => id)).to.have.all.members([203]); + expect(runs.find(({ runNumber }) => runNumber === 106).qcFlags['1'].map(({ id }) => id)).to.have.all.members([3, 2, 1]); + } + + { // Simulation Passes + const { runs } = await new GetAllRunsUseCase().execute({ + query: { + filter: { + simulationPassIds: [1], + }, + include: { effectiveQcFlags: true }, + }, + }); + expect(runs).to.have.lengthOf(3); + expect(runs.find(({ runNumber }) => runNumber === 106).qcFlags['1'].map(({ id }) => id)).to.have.all.members([6, 5]); + } + + { // Synchronous flags + const { runs } = await new GetAllRunsUseCase().execute({ + query: { + filter: { + lhcPeriodIds: [1], + }, + include: { effectiveQcFlags: true }, + }, + }); + expect(runs).to.have.lengthOf(4); + expect(runs.find(({ runNumber }) => runNumber === 56).qcFlags['7'].map(({ id }) => id)).to.have.all.members([101, 100]); + expect(runs.find(({ runNumber }) => runNumber === 56).qcFlags['4'].map(({ id }) => id)).to.have.all.members([102]); + } + }); };