Skip to content

Commit 8c52bff

Browse files
folland87folland87
committed
Merge branch 'staging'
2 parents c1244e0 + 58ec33c commit 8c52bff

5 files changed

Lines changed: 381 additions & 0 deletions

File tree

‎src/api/app.js‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import curiexploreRoutes from "./curiexplore/curiexplore.routes";
2222
import documentTypesRoutes from "./document-types/document-types.routes";
2323
import documentsRoutes from "./documents/documents.routes";
2424
import domainsRoutes from "./domains/domains.routes";
25+
import dumpRoutes from "./dump/dump.routes";
2526
import emailTypesRoutes from "./email-types/email-types.routes";
2627
import followUpsRoutes from "./followups/followups.routes";
2728
import geographicalcategoriesRoutes from "./geographicalcategories/geographicalcategories.routes";
@@ -132,6 +133,7 @@ app.use(curiexploreRoutes);
132133
app.use(documentsRoutes);
133134
app.use(documentTypesRoutes);
134135
app.use(domainsRoutes);
136+
app.use(dumpRoutes);
135137
app.use(emailTypesRoutes);
136138
app.use(followUpsRoutes);
137139
app.use(geographicalcategoriesRoutes);

‎src/api/commons/queries/structures.dump.query.js‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
import currentNameQuery from './current-name.query';
2+
import currentIdentifiersQuery from './current-identifiers.query';
3+
4+
5+
export default [
6+
...currentNameQuery,
7+
...currentIdentifiersQuery,
8+
{
9+
$project: {
10+
_id: 0,
11+
id: 1,
12+
displayName: '$currentName.usualName',
13+
identifiers: { $ifNull: ['$identifiers', []] },
14+
},
15+
},
16+
];

‎src/api/dump/dump.routes.js‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
import express from 'express';
2+
import { createGzip } from 'zlib';
3+
import { Transform } from 'stream';
4+
import { pipeline } from 'stream/promises';
5+
import { db } from '../../services/mongo.service';
6+
import { requireRoles } from '../commons/middlewares/rbac.middlewares';
7+
8+
const router = new express.Router();
9+
10+
/**
11+
* Transform stream that converts MongoDB documents to NDJSON format
12+
* (Newline Delimited JSON - perfect for Python consumption)
13+
*/
14+
class NDJSONTransform extends Transform {
15+
constructor() {
16+
super({ objectMode: true });
17+
}
18+
19+
_transform(doc, encoding, callback) {
20+
try {
21+
const jsonLine = JSON.stringify(doc) + '\n';
22+
callback(null, jsonLine);
23+
} catch (error) {
24+
callback(error);
25+
}
26+
}
27+
}
28+
29+
router.route('/dump/structures')
30+
.get([
31+
requireRoles(['admin']),
32+
async (req, res) => {
33+
try {
34+
// Check if it's a browser download (query param ?download=true for raw .gz file)
35+
const isBrowserDownload = req.query.download === 'true';
36+
37+
if (isBrowserDownload) {
38+
// For browser downloads: send as raw gzip file (not auto-decompressed)
39+
res.setHeader('Content-Type', 'application/gzip');
40+
res.setHeader('Content-Disposition', 'attachment; filename="structures-dump.ndjson.gz"');
41+
} else {
42+
// For programmatic access: browsers will auto-decompress, Python sees compressed
43+
res.setHeader('Content-Type', 'application/x-ndjson');
44+
res.setHeader('Content-Encoding', 'gzip');
45+
res.setHeader('Content-Disposition', 'attachment; filename="structures-dump.ndjson.gz"');
46+
}
47+
48+
res.setHeader('Transfer-Encoding', 'chunked');
49+
res.setHeader('Cache-Control', 'no-cache');
50+
51+
const cursor = db.collection('structures-dump').find({}, {
52+
batchSize: 1000,
53+
noCursorTimeout: false,
54+
});
55+
56+
const ndjsonTransform = new NDJSONTransform();
57+
const gzip = createGzip({ level: 6 });
58+
59+
req.on('close', () => {
60+
cursor.close();
61+
});
62+
63+
await pipeline(
64+
cursor.stream(),
65+
ndjsonTransform,
66+
gzip,
67+
res
68+
);
69+
} catch (error) {
70+
// Only send error if headers haven't been sent
71+
if (!res.headersSent) {
72+
res.status(500).json({ error: 'Failed to generate dump', message: error.message });
73+
} else {
74+
// If streaming has started, we can only close the connection
75+
res.end();
76+
}
77+
}
78+
},
79+
]);
80+
81+
export default router;

‎src/jobs/dump/structures.js‎

Lines changed: 276 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,276 @@
1+
import { db } from "../../services/mongo.service";
2+
3+
import currentCategoryQuery from "../../api/commons/queries/current-category.query";
4+
import currentEmailsQuery from "../../api/commons/queries/current-emails.query";
5+
import currentIdentifiersQuery from "../../api/commons/queries/current-identifiers.query";
6+
import currentLegalCategoryQuery from "../../api/commons/queries/current-legal-category.query";
7+
import currentLocalisationQuery from "../../api/commons/queries/current-localisation.query";
8+
import currentNameQuery from "../../api/commons/queries/current-name.query";
9+
import currentWebsitesQuery from "../../api/commons/queries/current-websites.query";
10+
import currentSocialsQuery from "../../api/commons/queries/current-socials.query";
11+
import relationTypesLightQuery from '../../api/commons/queries/relation-types.light.query';
12+
import categoryLightQuery from '../../api/commons/queries/categories.light.query';
13+
import legalCategoryLightQuery from '../../api/commons/queries/legal-categories.light.query';
14+
import personLightQuery from '../../api/commons/queries/persons.light.query';
15+
import prizeLightQuery from '../../api/commons/queries/prizes.light.query';
16+
import structuresDumpQuery from '../../api/commons/queries/structures.dump.query';
17+
import supervisingMinistersLightQuery from '../../api/commons/queries/supervising-ministers.light.query';
18+
import termsLightQuery from '../../api/commons/queries/terms.light.query';
19+
20+
function getRelatedObject(localField) {
21+
return ([
22+
{
23+
$lookup: {
24+
from: 'categories',
25+
localField,
26+
foreignField: 'id',
27+
pipeline: categoryLightQuery,
28+
as: 'relatedCategories',
29+
},
30+
},
31+
{
32+
$lookup: {
33+
from: 'legalcategories',
34+
localField,
35+
foreignField: 'id',
36+
pipeline: legalCategoryLightQuery,
37+
as: 'relatedLegalCategories',
38+
},
39+
},
40+
{
41+
$lookup: {
42+
from: 'terms',
43+
localField,
44+
foreignField: 'id',
45+
pipeline: termsLightQuery,
46+
as: 'relatedTerms',
47+
},
48+
},
49+
{
50+
$lookup: {
51+
from: 'persons',
52+
localField,
53+
foreignField: 'id',
54+
pipeline: personLightQuery,
55+
as: 'relatedPersons',
56+
},
57+
},
58+
{
59+
$lookup: {
60+
from: 'prizes',
61+
localField,
62+
foreignField: 'id',
63+
pipeline: prizeLightQuery,
64+
as: 'relatedPrizes',
65+
},
66+
},
67+
{
68+
$lookup: {
69+
from: 'structures',
70+
localField,
71+
foreignField: 'id',
72+
pipeline: structuresDumpQuery,
73+
as: 'relatedStructures',
74+
},
75+
},
76+
{
77+
$lookup: {
78+
from: 'supervisingministers',
79+
localField,
80+
foreignField: 'id',
81+
pipeline: supervisingMinistersLightQuery,
82+
as: 'relatedMinisters',
83+
},
84+
},
85+
{
86+
$set: {
87+
related: {
88+
$concatArrays: [
89+
'$relatedLegalCategories',
90+
'$relatedStructures',
91+
'$relatedPrizes',
92+
'$relatedPersons',
93+
'$relatedTerms',
94+
'$relatedCategories',
95+
'$relatedMinisters',
96+
],
97+
},
98+
},
99+
},
100+
]);
101+
}
102+
103+
export const relatedObjectLookup = [
104+
...getRelatedObject('relatedObjectId'),
105+
{ $set: { relatedObject: { $arrayElemAt: ['$related', 0] } } },
106+
];
107+
export const resourceLookup = [
108+
...getRelatedObject('resourceId'),
109+
{ $set: { resource: { $arrayElemAt: ['$related', 0] } } },
110+
];
111+
112+
const relationTypeQuery = [
113+
{
114+
$lookup: {
115+
from: 'relationtypes',
116+
localField: 'relationTypeId',
117+
foreignField: 'id',
118+
pipeline: relationTypesLightQuery,
119+
as: 'relationType',
120+
},
121+
},
122+
{ $set: { relationType: { $arrayElemAt: ['$relationType', 0] } } },
123+
]
124+
const relationGroupQuery = [
125+
{
126+
$lookup: {
127+
from: 'relationsgroups',
128+
localField: 'relationsGroupId',
129+
foreignField: 'id',
130+
pipeline: [
131+
{
132+
$project: {
133+
_id: 0,
134+
id: 1,
135+
resourceId: 1,
136+
name: 1,
137+
accepts: 1,
138+
priority: 1,
139+
},
140+
},
141+
],
142+
as: 'relationGroup',
143+
},
144+
},
145+
{ $set: { relationGroup: { $arrayElemAt: ['$relationGroup', 0] } } },
146+
]
147+
148+
const projection = {
149+
_id: 0,
150+
id: 1,
151+
resourceId: 1,
152+
relatedObjectId: 1,
153+
relationGroup: { $ifNull: ['$relationGroup', null] },
154+
relationType: { $ifNull: ['$relationType', { priority: 99 }] },
155+
relationTag: { $ifNull: ['$relationTag', null] },
156+
startDate: { $ifNull: ['$startDate', null] },
157+
endDate: { $ifNull: ['$endDate', null] },
158+
endDatePrevisional: { $ifNull: ['$endDatePrevisional', null] },
159+
mandatePosition: { $ifNull: ['$mandatePosition', null] },
160+
mandateReason: { $ifNull: ['$mandateReason', null] },
161+
mandateEmail: { $ifNull: ['$mandateEmail', null] },
162+
personalEmail: { $ifNull: ['$personalEmail', null] },
163+
mandatePhonenumber: { $ifNull: ['$mandatePhonenumber', null] },
164+
mandateTemporary: { $ifNull: ['$mandateTemporary', null] },
165+
mandatePrecision: { $ifNull: ['$mandatePrecision', null] },
166+
laureatePrecision: { $ifNull: ['$laureatePrecision', null] },
167+
active: { $ifNull: ['$active', null] },
168+
}
169+
170+
const relationRelatedQuery = [
171+
...relatedObjectLookup,
172+
...relationTypeQuery,
173+
...relationGroupQuery,
174+
{
175+
$project: {...projection, relatedObject: 1 },
176+
},
177+
];
178+
179+
const relationResourceQuery = [
180+
...resourceLookup,
181+
...relationTypeQuery,
182+
...relationGroupQuery,
183+
{
184+
$project: {...projection, resource: 1 },
185+
},
186+
];
187+
188+
189+
190+
const structureDumpQuery = [
191+
...currentCategoryQuery,
192+
...currentEmailsQuery,
193+
...currentIdentifiersQuery,
194+
...currentLegalCategoryQuery,
195+
...currentLocalisationQuery,
196+
...currentNameQuery,
197+
...currentSocialsQuery,
198+
...currentWebsitesQuery,
199+
{
200+
$lookup: {
201+
from: 'relationships',
202+
localField: 'id',
203+
foreignField: 'resourceId',
204+
pipeline: [...relationRelatedQuery],
205+
as: 'rel1'
206+
}
207+
},
208+
{
209+
$lookup: {
210+
from: 'relationships',
211+
localField: 'id',
212+
foreignField: 'relatedObjectId',
213+
pipeline: [...relationResourceQuery],
214+
as: 'rel2'
215+
}
216+
},
217+
{
218+
$set: {
219+
relations: {
220+
$concatArrays: ['$rel1', '$rel2']
221+
}
222+
}
223+
},
224+
{
225+
$project: {
226+
_id: 0,
227+
id: 1,
228+
object: 'structures',
229+
status: { $ifNull: ['$structureStatus', null] },
230+
displayName: '$currentName.usualName',
231+
currentName: { $ifNull: ['$currentName', {}] },
232+
descriptionEn: { $ifNull: ['$descriptionEn', null] },
233+
descriptionFr: { $ifNull: ['$descriptionFr', null] },
234+
currentLocalisation: { $ifNull: ['$currentLocalisation', {}] },
235+
localisations: 1,
236+
category: { $ifNull: ['$category', {}] },
237+
legalcategory: { $ifNull: ['$legalcategory', {}] },
238+
identifiers: { $ifNull: ['$identifiers', []] },
239+
relations: { $ifNull: ['$relations', []] },
240+
socialmedias: { $ifNull: ['$socialmedias', []] },
241+
categories: { $ifNull: ['$categories', []] },
242+
closureDate: { $ifNull: ['$closureDate', null] },
243+
createdAt: 1,
244+
creationDate: { $ifNull: ['$creationDate', null] },
245+
emails: { $ifNull: ['$emails', []] },
246+
websites: { $ifNull: ['$websites', []] },
247+
dumpedAt: new Date()
248+
},
249+
},
250+
{
251+
$out: 'structures-dump',
252+
},
253+
];
254+
255+
256+
export default async function createStructuresDump() {
257+
console.log('--------------------------------------');
258+
console.log('Creating structures dump...');
259+
console.log('--------------------------------------');
260+
try {
261+
const res = await db.collection('structures').aggregate(structureDumpQuery, { allowDiskUse: true }).toArray();
262+
console.log('--------------------------------------');
263+
console.log('Structures dump created successfully', res);
264+
console.log('--------------------------------------');
265+
return {status: 'success', data: res};
266+
} catch (error) {
267+
console.log('--------------------------------------');
268+
console.log('--------------------------------------');
269+
console.log('--------------------------------------');
270+
console.error('Error creating structures dump:', error);
271+
console.log('--------------------------------------');
272+
console.log('--------------------------------------');
273+
console.log('--------------------------------------');
274+
throw error;
275+
}
276+
}

0 commit comments

Comments
 (0)