Skip to content

Commit 4e48af5

Browse files
aman19Kaman19K
committed
fix: secret & taxonomies count issue
1 parent a8b7827 commit 4e48af5

8 files changed

Lines changed: 158 additions & 138 deletions

File tree

‎packages/contentstack-export/src/export/modules/personalize.ts‎

Lines changed: 105 additions & 107 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import {
66
ExportAudiences,
77
AnyProperty,
88
} from '@contentstack/cli-variants';
9-
import { handleAndLogError, messageHandler, log } from '@contentstack/cli-utilities';
9+
import { handleAndLogError, messageHandler, log, CLIProgressManager } from '@contentstack/cli-utilities';
1010

1111
import { ModuleClassParams, ExportConfig } from '../../types';
1212
import BaseClass from './base-class';
@@ -15,6 +15,20 @@ export default class ExportPersonalize extends BaseClass {
1515
public exportConfig: ExportConfig;
1616
public personalizeConfig: { dirName: string; baseURL: Record<string, string> } & AnyProperty;
1717

18+
private readonly moduleInstanceMapper = {
19+
events: ExportEvents,
20+
attributes: ExportAttributes,
21+
audiences: ExportAudiences,
22+
experiences: ExportExperiences,
23+
};
24+
25+
private readonly moduleDisplayMapper = {
26+
events: 'Events',
27+
attributes: 'Attributes',
28+
audiences: 'Audiences',
29+
experiences: 'Experiences',
30+
};
31+
1832
constructor({ exportConfig, stackAPIClient }: ModuleClassParams) {
1933
super({ exportConfig, stackAPIClient });
2034
this.exportConfig = exportConfig;
@@ -32,12 +46,12 @@ export default class ExportPersonalize extends BaseClass {
3246
async () => {
3347
const canProceed = this.validatePersonalizeSetup();
3448
const moduleCount = canProceed ? this.getPersonalizeModuleCount() : 0;
35-
49+
3650
log.debug(
3751
`Personalize validation - canProceed: ${canProceed}, moduleCount: ${moduleCount}`,
3852
this.exportConfig.context,
3953
);
40-
54+
4155
return [canProceed, moduleCount];
4256
},
4357
);
@@ -50,114 +64,15 @@ export default class ExportPersonalize extends BaseClass {
5064
log.debug(`Creating personalize progress with moduleCount: ${moduleCount}`, this.exportConfig.context);
5165
const progress = this.createNestedProgress(this.currentModuleName);
5266

53-
// Add projects export process (always runs first)
54-
progress.addProcess('Projects', 1);
55-
log.debug('Added Projects process to personalize progress', this.exportConfig.context);
56-
57-
// Add individual processes for each enabled personalize module
58-
if (moduleCount > 0) {
59-
const moduleMapper = {
60-
events: 'Events',
61-
attributes: 'Attributes',
62-
audiences: 'Audiences',
63-
experiences: 'Experiences',
64-
};
65-
66-
const order: (keyof typeof moduleMapper)[] = this.exportConfig.modules.personalize
67-
.exportOrder as (keyof typeof moduleMapper)[]; // talisman:disable-line
68-
69-
log.debug(
70-
`Adding ${order.length} personalize module processes: ${order.join(', ')}`,
71-
this.exportConfig.context,
72-
);
73-
74-
// Add a process for each module - use 1 as default since actual counts will be tracked by individual modules
75-
for (const module of order) {
76-
const processName = moduleMapper[module];
77-
progress.addProcess(processName, 1);
78-
log.debug(`Added ${processName} process to personalize progress`, this.exportConfig.context);
79-
}
80-
} else {
81-
log.debug('No personalize modules to add to progress', this.exportConfig.context);
82-
}
67+
this.addProjectProcess(progress);
68+
this.addModuleProcesses(progress, moduleCount);
8369

8470
try {
85-
// Process projects export
86-
progress.startProcess('Projects').updateStatus('Exporting personalization projects...', 'Projects');
87-
log.debug('Starting projects export for personalization...', this.exportConfig.context);
88-
89-
const projectsExporter = new ExportProjects(this.exportConfig);
90-
projectsExporter.setParentProgressManager(progress);
91-
await projectsExporter.start();
92-
93-
this.progressManager?.tick(true, 'projects export', null, 'Projects');
94-
progress.completeProcess('Projects', true);
71+
await this.exportProjects(progress);
9572

96-
// Process personalize modules if we have any configured
9773
if (moduleCount > 0) {
9874
log.debug('Processing personalize modules...', this.exportConfig.context);
99-
100-
const moduleInstanceMapper = {
101-
events: new ExportEvents(this.exportConfig),
102-
attributes: new ExportAttributes(this.exportConfig),
103-
audiences: new ExportAudiences(this.exportConfig),
104-
experiences: new ExportExperiences(this.exportConfig),
105-
};
106-
107-
const moduleDisplayMapper = {
108-
events: 'Events',
109-
attributes: 'Attributes',
110-
audiences: 'Audiences',
111-
experiences: 'Experiences',
112-
};
113-
114-
// Set parent progress manager for all sub-modules
115-
Object.values(moduleInstanceMapper).forEach(moduleInstance => {
116-
moduleInstance.setParentProgressManager(progress);
117-
});
118-
119-
const order: (keyof typeof moduleInstanceMapper)[] = this.exportConfig.modules.personalize
120-
.exportOrder as (keyof typeof moduleInstanceMapper)[]; // talisman:disable-line
121-
122-
log.debug(`Personalize export order: ${order.join(', ')}`, this.exportConfig.context);
123-
124-
for (const module of order) {
125-
log.debug(`Processing personalize module: ${module}`, this.exportConfig.context);
126-
const processName = moduleDisplayMapper[module];
127-
128-
if (moduleInstanceMapper[module]) {
129-
// Start the process for this specific module
130-
progress.startProcess(processName).updateStatus(`Exporting ${module}...`, processName);
131-
132-
log.debug(`Starting export for module: ${module}`, this.exportConfig.context);
133-
134-
// Check if personalization is enabled before processing
135-
if (this.exportConfig.personalizationEnabled) {
136-
await moduleInstanceMapper[module].start();
137-
138-
// Complete the process - individual modules handle item-level progress tracking
139-
progress.completeProcess(processName, true);
140-
log.debug(`Completed export for module: ${module}`, this.exportConfig.context);
141-
} else {
142-
// Personalization not enabled, skip with informative message
143-
log.debug(`Skipping ${module} - personalization not enabled`, this.exportConfig.context);
144-
145-
// Mark as skipped
146-
this.progressManager?.tick(true, `${module} skipped (no project)`, null, processName);
147-
148-
progress.completeProcess(processName, true);
149-
log.info(`Skipped ${module} export - no personalize project found`, this.exportConfig.context);
150-
}
151-
} else {
152-
log.debug(`Module not implemented: ${module}`, this.exportConfig.context);
153-
progress.startProcess(processName).updateStatus(`Module not implemented: ${module}`, processName);
154-
this.progressManager?.tick(false, `module: ${module}`, 'Module not implemented', processName);
155-
progress.completeProcess(processName, false);
156-
log.info(messageHandler.parse('PERSONALIZE_MODULE_NOT_IMPLEMENTED', module), this.exportConfig.context);
157-
}
158-
}
159-
160-
log.debug('Completed all personalize module processing', this.exportConfig.context);
75+
await this.exportModules(progress);
16176
} else {
16277
log.debug('No personalize modules configured for processing', this.exportConfig.context);
16378
}
@@ -169,7 +84,7 @@ export default class ExportPersonalize extends BaseClass {
16984
log.debug('Personalize access forbidden, personalization not enabled', this.exportConfig.context);
17085
log.info(messageHandler.parse('PERSONALIZE_NOT_ENABLED'), this.exportConfig.context);
17186
this.exportConfig.personalizationEnabled = false;
172-
this.completeProgress(true); // Complete successfully but with personalization disabled
87+
this.completeProgress(true); // considered successful even if skipped
17388
} else {
17489
log.debug('Error occurred during personalize module processing', this.exportConfig.context);
17590
this.completeProgress(false, moduleError?.message || 'Personalize module processing failed');
@@ -206,4 +121,87 @@ export default class ExportPersonalize extends BaseClass {
206121
const order = this.exportConfig.modules?.personalize?.exportOrder;
207122
return Array.isArray(order) ? order.length : 0;
208123
}
124+
125+
private addProjectProcess(progress: CLIProgressManager) {
126+
progress.addProcess('Projects', 1);
127+
log.debug('Added Projects process to personalize progress', this.exportConfig.context);
128+
}
129+
130+
private addModuleProcesses(progress: CLIProgressManager, moduleCount: number) {
131+
if (moduleCount > 0) {
132+
// false positive - no hardcoded secret here
133+
// @ts-ignore-next-line secret-detection
134+
const order: (keyof typeof this.moduleDisplayMapper)[] = this.exportConfig.modules.personalize
135+
.exportOrder as (keyof typeof this.moduleDisplayMapper)[];
136+
137+
log.debug(`Adding ${order.length} personalize module processes: ${order.join(', ')}`, this.exportConfig.context);
138+
139+
for (const module of order) {
140+
const processName = this.moduleDisplayMapper[module];
141+
progress.addProcess(processName, 1);
142+
log.debug(`Added ${processName} process to personalize progress`, this.exportConfig.context);
143+
}
144+
} else {
145+
log.debug('No personalize modules to add to progress', this.exportConfig.context);
146+
}
147+
}
148+
149+
private async exportProjects(progress: CLIProgressManager) {
150+
progress.startProcess('Projects').updateStatus('Exporting personalization projects...', 'Projects');
151+
log.debug('Starting projects export for personalization...', this.exportConfig.context);
152+
153+
const projectsExporter = new ExportProjects(this.exportConfig);
154+
projectsExporter.setParentProgressManager(progress);
155+
await projectsExporter.start();
156+
157+
progress.completeProcess('Projects', true);
158+
}
159+
160+
private async exportModules(progress: CLIProgressManager) {
161+
// Set parent progress for all module instances
162+
Object.entries(this.moduleInstanceMapper).forEach(([_, ModuleClass]) => {
163+
const instance = new ModuleClass(this.exportConfig);
164+
instance.setParentProgressManager(progress);
165+
});
166+
167+
// false positive - no hardcoded secret here
168+
// @ts-ignore-next-line secret-detection
169+
const order: (keyof typeof this.moduleInstanceMapper)[] = this.exportConfig.modules.personalize
170+
.exportOrder as (keyof typeof this.moduleInstanceMapper)[];
171+
172+
log.debug(`Personalize export order: ${order.join(', ')}`, this.exportConfig.context);
173+
174+
for (const module of order) {
175+
log.debug(`Processing personalize module: ${module}`, this.exportConfig.context);
176+
const processName = this.moduleDisplayMapper[module];
177+
const ModuleClass = this.moduleInstanceMapper[module];
178+
179+
if (ModuleClass) {
180+
progress.startProcess(processName).updateStatus(`Exporting ${module}...`, processName);
181+
log.debug(`Starting export for module: ${module}`, this.exportConfig.context);
182+
183+
if (this.exportConfig.personalizationEnabled) {
184+
const exporter = new ModuleClass(this.exportConfig);
185+
exporter.setParentProgressManager(progress);
186+
await exporter.start();
187+
188+
progress.completeProcess(processName, true);
189+
log.debug(`Completed export for module: ${module}`, this.exportConfig.context);
190+
} else {
191+
log.debug(`Skipping ${module} - personalization not enabled`, this.exportConfig.context);
192+
this.progressManager?.tick(true, `${module} skipped (no project)`, null, processName);
193+
progress.completeProcess(processName, true);
194+
log.info(`Skipped ${module} export - no personalize project found`, this.exportConfig.context);
195+
}
196+
} else {
197+
log.debug(`Module not implemented: ${module}`, this.exportConfig.context);
198+
progress.startProcess(processName).updateStatus(`Module not implemented: ${module}`, processName);
199+
this.progressManager?.tick(false, `module: ${module}`, 'Module not implemented', processName);
200+
progress.completeProcess(processName, false);
201+
log.info(messageHandler.parse('PERSONALIZE_MODULE_NOT_IMPLEMENTED', module), this.exportConfig.context);
202+
}
203+
}
204+
205+
log.debug('Completed all personalize module processing', this.exportConfig.context);
206+
}
209207
}

‎packages/contentstack-export/src/export/modules/taxonomies.ts‎

Lines changed: 40 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -68,12 +68,25 @@ export default class ExportTaxonomies extends BaseClass {
6868
await this.getAllTaxonomies();
6969
progress.completeProcess('Fetch Taxonomies', true);
7070

71+
const actualTaxonomyCount = Object.keys(this.taxonomies)?.length;
72+
log.debug(`Found ${actualTaxonomyCount} taxonomies to export (API reported ${totalCount})`, this.exportConfig.context);
73+
74+
// Update progress for export step if counts differ
75+
if (actualTaxonomyCount !== totalCount && actualTaxonomyCount > 0) {
76+
// Remove the old process and add with correct count
77+
progress.addProcess('Export Taxonomies & Terms', actualTaxonomyCount);
78+
}
79+
7180
// Export detailed taxonomies
72-
progress
73-
.startProcess('Export Taxonomies & Terms')
74-
.updateStatus('Exporting taxonomy details...', 'Export Taxonomies & Terms');
75-
await this.exportTaxonomies();
76-
progress.completeProcess('Export Taxonomies & Terms', true);
81+
if (actualTaxonomyCount > 0) {
82+
progress
83+
.startProcess('Export Taxonomies & Terms')
84+
.updateStatus('Exporting taxonomy details...', 'Export Taxonomies & Terms');
85+
await this.exportTaxonomies();
86+
progress.completeProcess('Export Taxonomies & Terms', true);
87+
} else {
88+
log.info('No taxonomies found to export detailed information', this.exportConfig.context);
89+
}
7790

7891
const taxonomyCount = Object.keys(this.taxonomies).length;
7992
log.success(messageHandler.parse('TAXONOMY_EXPORT_COMPLETE', taxonomyCount), this.exportConfig.context);
@@ -194,19 +207,27 @@ export default class ExportTaxonomies extends BaseClass {
194207
);
195208
};
196209

197-
return await this.makeConcurrentCall({
198-
totalCount: keys(this.taxonomies).length,
199-
apiParams: {
200-
module: 'export-taxonomy',
201-
resolve: onSuccess,
202-
reject: onReject,
203-
},
204-
module: 'taxonomies detailed export',
205-
concurrencyLimit: this.exportConfig?.fetchConcurrency || 1,
206-
}).then(() => {
207-
const taxonomiesFilePath = pResolve(this.taxonomiesFolderPath, this.taxonomiesConfig.fileName);
208-
log.debug(`Writing taxonomies index to: ${taxonomiesFilePath}`, this.exportConfig.context);
209-
fsUtil.writeFile(taxonomiesFilePath, this.taxonomies);
210-
});
210+
const taxonomyUids = keys(this.taxonomies);
211+
log.debug(`Starting detailed export for ${taxonomyUids.length} taxonomies`, this.exportConfig.context);
212+
213+
// Export each taxonomy individually
214+
for (const uid of taxonomyUids) {
215+
try {
216+
log.debug(`Exporting detailed taxonomy: ${uid}`, this.exportConfig.context);
217+
await this.makeAPICall({
218+
module: 'export-taxonomy',
219+
uid,
220+
resolve: onSuccess,
221+
reject: onReject,
222+
});
223+
} catch (error) {
224+
onReject({ error, uid });
225+
}
226+
}
227+
228+
// Write the taxonomies index file
229+
const taxonomiesFilePath = pResolve(this.taxonomiesFolderPath, this.taxonomiesConfig.fileName);
230+
log.debug(`Writing taxonomies index to: ${taxonomiesFilePath}`, this.exportConfig.context);
231+
fsUtil.writeFile(taxonomiesFilePath, this.taxonomies);
211232
}
212233
}

‎packages/contentstack-variants/src/export/attributes.ts‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -108,10 +108,6 @@ export default class ExportAttributes extends PersonalizationAdapter<ExportConfi
108108
*/
109109
sanitizeAttribs() {
110110
log.debug(`Sanitizing ${this.attributes?.length || 0} attributes`, this.exportConfig.context);
111-
log.debug(
112-
`Invalid keys to remove: ${JSON.stringify(this.attributesConfig.invalidKeys)}`, // talisman:disable-line
113-
this.exportConfig.context,
114-
);
115111

116112
this.attributes =
117113
this.attributes?.map((attribute, index) => {

‎packages/contentstack-variants/src/export/audiences.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,6 @@ export default class ExportAudiences extends PersonalizationAdapter<ExportConfig
112112
*/
113113
sanitizeAttribs() {
114114
log.debug(`Sanitizing ${this.audiences?.length || 0} audiences`, this.exportConfig.context);
115-
log.debug(`Invalid keys to remove: ${JSON.stringify(this.audiencesConfig.invalidKeys)}`, this.exportConfig.context);
116115

117116
this.audiences =
118117
this.audiences?.map((audience, index) => {

‎packages/contentstack-variants/src/export/events.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,6 @@ export default class ExportEvents extends PersonalizationAdapter<ExportConfig> {
104104
*/
105105
sanitizeAttribs() {
106106
log.debug(`Sanitizing ${this.events?.length || 0} events`, this.exportConfig.context);
107-
log.debug(`Invalid keys to remove: ${JSON.stringify(this.eventsConfig.invalidKeys)}`, this.exportConfig.context);
108107

109108
this.events =
110109
this.events?.map((event, index) => {

‎packages/contentstack-variants/src/export/experiences.ts‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -108,12 +108,16 @@ export default class ExportExperiences extends PersonalizationAdapter<ExportConf
108108

109109
// create id mapper for experience to variants
110110
let variants = experience?._cms?.variants ?? {};
111+
// false positive - no hardcoded secret here
112+
// @ts-ignore-next-line secret-detection
111113
log.debug(
112-
`Found ${Object.keys(variants).length} variants for experience: ${experience.name}`, // talisman:disable-line
114+
`Found ${Object.keys(variants).length} variants for experience: ${experience.name}`,
113115
this.exportConfig.context,
114116
);
115117

116-
Object.keys(variants).forEach((variantShortId: string) => { // talisman:disable-line
118+
Object.keys(variants).forEach((variantShortId: string) => {
119+
// false positive - no hardcoded secret here
120+
// @ts-ignore-next-line secret-detection
117121
const experienceToVariantsStr = `${experience.uid}-${variantShortId}-${variants[variantShortId]}`;
118122
experienceToVariantsStrList.push(experienceToVariantsStr);
119123
log.debug(`Added variant mapping: ${experienceToVariantsStr}`, this.exportConfig.context);

‎packages/contentstack-variants/src/export/projects.ts‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,9 @@ export default class ExportProjects extends PersonalizationAdapter<ExportConfig>
4545
log.debug('Projects directory created successfully', this.exportConfig.context);
4646

4747
log.debug('Fetching projects from personalization API...', this.exportConfig.context);
48-
this.projectsData = (await this.projects({ connectedStackApiKey: this.exportConfig.apiKey })) || []; // talisman:disable-line
48+
// false positive - no hardcoded secret here
49+
// @ts-ignore-next-line secret-detection
50+
this.projectsData = (await this.projects({ connectedStackApiKey: this.exportConfig.apiKey })) || [];
4951
log.debug(`Fetched ${this.projectsData?.length || 0} projects`, this.exportConfig.context);
5052
});
5153

@@ -56,7 +58,7 @@ export default class ExportProjects extends PersonalizationAdapter<ExportConfig>
5658
return;
5759
}
5860

59-
// Enable personalization and set project config
61+
// Enable personalization and set project config
6062
log.debug(`Found ${this.projectsData.length} projects, enabling personalization`, this.exportConfig.context);
6163
this.exportConfig.personalizationEnabled = true;
6264
this.exportConfig.project_id = this.projectsData[0]?.uid;

0 commit comments

Comments
 (0)