[O2B-1462] Extend runs API to include QC flags

lib/database/adapters/RunAdapter.js

@@ -75,6 +75,11 @@ class RunAdapter {
          */
         this.userAdapter = null;
 
+        /**
+         * @type {QcFlagAdapter|null}
+         */
+        this.qcFlagAdapter = null;
+
         this.toEntity = this.toEntity.bind(this);
         this.toDatabase = this.toDatabase.bind(this);
         this.toMinifiedEntity = this.toMinifiedEntity.bind(this);
@@ -167,6 +172,7 @@ class RunAdapter {
             phaseShiftAtEndBeam2,
             userStart,
             userStop,
+            qcFlags,
         } = databaseObject;
 
         /**
@@ -285,6 +291,13 @@ class RunAdapter {
             ? lhcFill.collidingBunchesCount ?? extractNumberOfCollidingLhcBunchCrossings(lhcFill.fillingSchemeName)
             : null;
 
+        const adaptedQcFlags = qcFlags ? qcFlags.map(this.qcFlagAdapter.toEntity) : [];
+        entityObject.qcFlags = adaptedQcFlags.reduce((acc, qcFlag) => {
+            acc[qcFlag.dplDetectorId] = acc[qcFlag.dplDetectorId] ?? [];
+            acc[qcFlag.dplDetectorId].push(qcFlag);
+            return acc;
+        }, {});
+
         return entityObject;
     }
 

lib/database/adapters/index.js

@@ -139,6 +139,7 @@ runAdapter.logAdapter = logAdapter;
 runAdapter.runTypeAdapter = runTypeAdapter;
 runAdapter.tagAdapter = tagAdapter;
 runAdapter.userAdapter = userAdapter;
+runAdapter.qcFlagAdapter = qcFlagAdapter;
 
 simulationPassQcFlagAdapter.simulationPassAdapter = simulationPassAdapter;
 simulationPassQcFlagAdapter.qcFlagAdapter = qcFlagAdapter;

lib/database/seeders/20240404100811-qc-flags.js

@@ -298,6 +298,12 @@ module.exports = {
                         from: '2019-08-08 13:46:40',
                         to: '2019-08-09 07:50:00',
                     },
+                    {
+                        id: 6,
+                        flag_id: 6,
+                        from: '2019-08-09 08:50:00',
+                        to: null,
+                    },
                     {
                         id: 7,
                         flag_id: 7,

lib/domain/dtos/GetAllRunsDto.js

@@ -15,11 +15,24 @@ const Joi = require('joi');
 const PaginationDto = require('./PaginationDto');
 const { RunFilterDto } = require('./filters/RunFilterDto.js');
 const { DtoFactory } = require('./DtoFactory');
+const { singleRunsCollectionCustomCheck } = require('./utils.js');
 
 const QueryDto = Joi.object({
     filter: RunFilterDto,
     page: PaginationDto,
     sort: DtoFactory.order(['id', 'runNumber', 'text']),
+    include: Joi.object({ effectiveQcFlags: Joi.boolean().custom((effectiveQcFlags, helpers) => {
+        const [, { filter: { dataPassIds, simulationPassIds, lhcPeriodIds } = {} }] = helpers.state.ancestors;
+
+        singleRunsCollectionCustomCheck(
+            { dataPassIds, simulationPassIds, lhcPeriodIds },
+            helpers,
+            'Including effectiveQcFlags is allowed only when ' +
+            'the dataPassIds, simulationPassIds and lhcPeriodIds filters collectively contain exactly one ID',
+        );
+
+        return effectiveQcFlags;
+    }) }),
     token: Joi.string(),
 });
 

lib/domain/dtos/filters/RunFilterDto.js

@@ -17,6 +17,7 @@ const { RUN_QUALITIES } = require('../../enums/RunQualities.js');
 const { IntegerComparisonDto, FloatComparisonDto } = require('./NumericalComparisonDto.js');
 const { RUN_CALIBRATION_STATUS } = require('../../enums/RunCalibrationStatus.js');
 const { RUN_DEFINITIONS } = require('../../enums/RunDefinition.js');
+const { singleRunsCollectionCustomCheck } = require('../utils.js');
 
 const DetectorsFilterDto = Joi.object({
     operator: Joi.string().valid('or', 'and', 'none').required(),
@@ -128,12 +129,13 @@ exports.RunFilterDto = Joi.object({
         })
         .custom((detectorsQcObj, helpers) => {
             const [{ dataPassIds, simulationPassIds, lhcPeriodIds }] = helpers.state.ancestors;
-            const runsCollectionFilters = [dataPassIds, simulationPassIds, lhcPeriodIds].filter(({ length } = {}) => length >= 1);
 
-            if (runsCollectionFilters.length !== 1 || runsCollectionFilters[0].length !== 1) {
-                return helpers.message('Filtering by detector not-bad fraction is allowed only with exactly one of: ' +
-                    'dataPassIds, simulationPassIds, lhcPeriodIds with exactly one ID.');
-            }
+            singleRunsCollectionCustomCheck(
+                { dataPassIds, simulationPassIds, lhcPeriodIds },
+                helpers,
+                'Filtering by detector not-bad fraction is allowed only when ' +
+                'the dataPassIds, simulationPassIds and lhcPeriodIds filters collectively contain exactly one ID',
+            );
 
             return detectorsQcObj;
         }),

lib/domain/dtos/utils.js

@@ -0,0 +1,33 @@
+/**
+ * @license
+ * Copyright CERN and copyright holders of ALICE O2. This software is
+ * distributed under the terms of the GNU General Public License v3 (GPL
+ * Version 3), copied verbatim in the file "COPYING".
+ *
+ * See http://alice-o2.web.cern.ch/license for full licensing information.
+ *
+ * In applying this license CERN does not waive the privileges and immunities
+ * granted to it by virtue of its status as an Intergovernmental Organization
+ * or submit itself to any jurisdiction.
+ */
+
+/**
+ * Used in Joi schemas to assert that the dataPassIds, simulationPassIds and lhcPeriodIds filters collectively contain exactly one ID
+ *
+ * @param {object} collections runs ids collections
+ * @param {number[]} collections.dataPassIds data pass ids
+ * @param {number[]} collections.simulationPassIds simulation pass ids
+ * @param {number[]} collections.lhcPeriodIds LHC periods ids
+ * @param {Joi.helpers} helpers joi helpers
+ * @param {string} message to be send in case of filters refer to more than one runs collection
+ * @returns {void}
+ */
+const singleRunsCollectionCustomCheck = ({ dataPassIds, simulationPassIds, lhcPeriodIds }, helpers, message) => {
+    const runsCollectionFilters = [dataPassIds, simulationPassIds, lhcPeriodIds].filter(({ length } = {}) => length >= 1);
+
+    if (runsCollectionFilters.length !== 1 || runsCollectionFilters[0].length !== 1) {
+        return helpers.message(message);
+    }
+};
+
+module.exports.singleRunsCollectionCustomCheck = singleRunsCollectionCustomCheck;

lib/usecases/run/GetAllRunsUseCase.js

@@ -36,7 +36,7 @@ class GetAllRunsUseCase {
     async execute(dto = {}) {
         const filteringQueryBuilder = new QueryBuilder();
         const { query = {} } = dto;
-        const { filter, page = {}, sort = { runNumber: 'desc' } } = query;
+        const { filter, page = {}, sort = { runNumber: 'desc' }, include: { effectiveQcFlags = false } = {} } = query;
 
         const SEARCH_ITEMS_SEPARATOR = ',';
 
@@ -343,6 +343,19 @@ class GetAllRunsUseCase {
                 });
             }
 
+            if (lhcPeriodIds) {
+                const runNumbers = (await RunRepository.findAll({
+                    attributes: ['runNumber'],
+                    raw: true,
+                    include: {
+                        association: 'lhcPeriod',
+                        attributes: [],
+                        where: { id: { [Op.in]: lhcPeriodIds } },
+                    },
+                })).map(({ runNumber }) => runNumber);
+                filteringQueryBuilder.where('runNumber').oneOf(...runNumbers);
+            }
+
             if (dataPassIds) {
                 const runNumbers = (await RunRepository.findAll({
                     attributes: ['runNumber'],
@@ -445,7 +458,7 @@ class GetAllRunsUseCase {
                     attributes: ['runNumber'],
                     raw: true,
                     include: {
-                        association: 'lhcPeriods',
+                        association: 'lhcPeriod',
                         attributes: [],
                         where: { id: { [Op.in]: lhcPeriodIds } },
                     },
@@ -516,6 +529,60 @@ class GetAllRunsUseCase {
             fetchQueryBuilder.orderBy(property, sort[property]);
         }
 
+        if (effectiveQcFlags) {
+            const [dataPassId] = filter?.dataPassIds ?? [];
+            const [simulationPassId] = filter?.simulationPassIds ?? [];
+            const [lhcPeriodId] = filter?.lhcPeriodIds ?? [];
+
+            if (Boolean(dataPassId) + Boolean(simulationPassId) + Boolean(lhcPeriodId) !== 1) {
+                throw new BadParameterError('If including QC flags, one and exactly one'
+                    + 'of `dataPassId`, `simulationPassId` or `lhcPeriodId` is required');
+            }
+
+            const commondQcFlagsAssociationDef = {
+                association: 'qcFlags',
+                required: false,
+                where: { deleted: false },
+            };
+
+            if (dataPassId) {
+                fetchQueryBuilder.include({
+                    ...commondQcFlagsAssociationDef,
+                    include: [
+                        { association: 'dataPasses', required: true, where: { id: dataPassId } },
+                        { association: 'effectivePeriods', required: true },
+                    ],
+                });
+            } else if (simulationPassId) {
+                fetchQueryBuilder.include({
+                    ...commondQcFlagsAssociationDef,
+                    include: [
+                        { association: 'simulationPasses', required: true, where: { id: simulationPassId } },
+                        { association: 'effectivePeriods', required: true },
+                    ],
+                });
+            } else {
+                fetchQueryBuilder.include({
+                    ...commondQcFlagsAssociationDef,
+                    include: [
+                        { association: 'simulationPasses', required: false },
+                        { association: 'dataPasses', required: false },
+                        { association: 'effectivePeriods', required: true },
+                    ],
+                    where: {
+                        deleted: false,
+                        [Op.or]: [
+                            { '$qcFlags.id$': null },
+                            {
+                                '$qcFlags.dataPasses.id$': null,
+                                '$qcFlags.simulationPasses.id$': null,
+                            },
+                        ],
+                    },
+                });
+            }
+        }
+
         const runs = (await RunRepository.findAll(fetchQueryBuilder)).map(runAdapter.toEntity);
 
         return { count, runs };

test/api/qcFlags.test.js

@@ -150,9 +150,9 @@ module.exports = () => {
             expect(data).to.be.eql({
                 106: {
                     1: {
-                        missingVerificationsCount: 1,
+                        missingVerificationsCount: 2,
                         mcReproducible: false,
-                        badEffectiveRunCoverage: 0.7222222,
+                        badEffectiveRunCoverage: 0.9288889,
                         explicitlyNotBadEffectiveRunCoverage: 0,
                     },
                 },

test/api/runs.test.js

@@ -21,6 +21,7 @@ const { RunDetectorQualities } = require('../../lib/domain/enums/RunDetectorQual
 const { RunCalibrationStatus } = require('../../lib/domain/enums/RunCalibrationStatus.js');
 const { updateRun } = require('../../lib/server/services/run/updateRun.js');
 const { RunDefinition } = require('../../lib/domain/enums/RunDefinition.js');
+const { qcFlagService } = require('../../lib/server/services/qualityControlFlag/QcFlagService.js');
 
 module.exports = () => {
     before(resetDatabaseContent);
@@ -606,6 +607,39 @@ module.exports = () => {
             expect(runs.every(({ aliceDipoleCurrent, aliceDipolePolarity }) =>
                 Math.round(aliceDipoleCurrent * (aliceDipolePolarity === 'NEGATIVE' ? -1 : 1) / 1000) === 0)).to.be.true;
         });
+
+        it('should successfully handle query including QC flags', async () => {
+            { // Data Passes
+                const response = await request(server).get(`/api/runs?filter[dataPassIds][]=1&include[effectiveQcFlags]=true`)
+                expect(response.status).to.equal(200);
+                const { data: runs } = response.body;
+
+                expect(runs).to.have.lengthOf(3);
+                expect(runs.find(({ runNumber }) => runNumber === 107).qcFlags['1'].map(({ id }) => id)).to.have.all.members([202, 201]);
+                expect(runs.find(({ runNumber }) => runNumber === 107).qcFlags['2'].map(({ id }) => id)).to.have.all.members([203]);
+                expect(runs.find(({ runNumber }) => runNumber === 106).qcFlags['1'].map(({ id }) => id)).to.have.all.members([3, 2, 1]);
+            }
+
+            { // Simulation Passes
+                const response = await request(server).get(`/api/runs?filter[simulationPassIds][]=1&include[effectiveQcFlags]=true`)
+                expect(response.status).to.equal(200);
+                const { data: runs } = response.body;
+                expect(runs).to.have.lengthOf(3);
+
+                console.log('TOBEC', await qcFlagService.getAllPerSimulationPassAndRunAndDetector({ simulationPassId: 1, runNumber: 106, detectorId: 1}));
+
+                expect(runs.find(({ runNumber }) => runNumber === 106).qcFlags['1'].map(({ id }) => id)).to.have.all.members([6, 5]);
+            }
+
+            { // Synchronous flags
+                const response = await request(server).get(`/api/runs?filter[lhcPeriodIds][]=1&include[effectiveQcFlags]=true`)
+                expect(response.status).to.equal(200);
+                const { data: runs } = response.body;
+                expect(runs).to.have.lengthOf(4);
+                expect(runs.find(({ runNumber }) => runNumber === 56).qcFlags['7'].map(({ id }) => id)).to.have.all.members([101, 100]);
+                expect(runs.find(({ runNumber }) => runNumber === 56).qcFlags['4'].map(({ id }) => id)).to.have.all.members([102]);
+            }
+        });
     });
 
     describe('GET /api/runs/reasonTypes', () => {

test/lib/server/services/qualityControlFlag/QcFlagService.test.js

@@ -290,9 +290,9 @@ module.exports = () => {
             expect(await qcFlagSummaryService.getSummary({ simulationPassId: 1 })).to.be.eql({
                 106: {
                     1: {
-                        missingVerificationsCount: 1,
+                        missingVerificationsCount: 2,
                         mcReproducible: false,
-                        badEffectiveRunCoverage: 0.7222222,
+                        badEffectiveRunCoverage: 0.9288889,
                         explicitlyNotBadEffectiveRunCoverage: 0,
                     },
                 },

test/lib/usecases/run/GetAllRunsUseCase.test.js

@@ -17,6 +17,8 @@ const chai = require('chai');
 const { RunQualities } = require('../../../../lib/domain/enums/RunQualities.js');
 const { RunCalibrationStatus } = require('../../../../lib/domain/enums/RunCalibrationStatus.js');
 const { RunDefinition } = require('../../../../lib/domain/enums/RunDefinition.js');
+const assert = require('assert');
+const { BadParameterError } = require('../../../../lib/server/errors/BadParameterError.js');
 
 const { expect } = chai;
 
@@ -795,7 +797,7 @@ module.exports = () => {
             expect(runs).to.have.lengthOf(0);
         }
     });
-
+    
     it('should successfully filter by detectors notBadFraction', async () => {
         const dataPassIds = [1];
         {
@@ -849,4 +851,57 @@ module.exports = () => {
             expect(runs.map(({ runNumber }) => runNumber)).to.have.all.members([107]);
         }
     });
+
+        it('should successfully handle query including QC flags', async () => {
+        {
+            await assert.rejects(() => new GetAllRunsUseCase().execute({
+                query: {
+                    include: { effectiveQcFlags: true },
+                },
+            }), new BadParameterError('If including QC flags, one and exactly one'
+                    + 'of `dataPassId`, `simulationPassId` or `lhcPeriodId` is required'));
+        }
+
+        { // Data Passes
+            const { runs } = await new GetAllRunsUseCase().execute({
+                query: {
+                    filter: {
+                        dataPassIds: [1],
+                    },
+                    include: { effectiveQcFlags: true },
+                },
+            });
+            expect(runs).to.have.lengthOf(3);
+            expect(runs.find(({ runNumber }) => runNumber === 107).qcFlags['1'].map(({ id }) => id)).to.have.all.members([202, 201]);
+            expect(runs.find(({ runNumber }) => runNumber === 107).qcFlags['2'].map(({ id }) => id)).to.have.all.members([203]);
+            expect(runs.find(({ runNumber }) => runNumber === 106).qcFlags['1'].map(({ id }) => id)).to.have.all.members([3, 2, 1]);
+        }
+
+        { // Simulation Passes
+            const { runs } = await new GetAllRunsUseCase().execute({
+                query: {
+                    filter: {
+                        simulationPassIds: [1],
+                    },
+                    include: { effectiveQcFlags: true },
+                },
+            });
+            expect(runs).to.have.lengthOf(3);
+            expect(runs.find(({ runNumber }) => runNumber === 106).qcFlags['1'].map(({ id }) => id)).to.have.all.members([6, 5]);
+        }
+
+        { // Synchronous flags
+            const { runs } = await new GetAllRunsUseCase().execute({
+                query: {
+                    filter: {
+                        lhcPeriodIds: [1],
+                    },
+                    include: { effectiveQcFlags: true },
+                },
+            });
+            expect(runs).to.have.lengthOf(4);
+            expect(runs.find(({ runNumber }) => runNumber === 56).qcFlags['7'].map(({ id }) => id)).to.have.all.members([101, 100]);
+            expect(runs.find(({ runNumber }) => runNumber === 56).qcFlags['4'].map(({ id }) => id)).to.have.all.members([102]);
+        }
+    });
 };