Feat: Added Check for publish details for Assets and Entries

packages/contentstack-audit/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@contentstack/cli-audit",
-  "version": "1.7.5",
+  "version": "1.8.0",
   "description": "Contentstack audit plugin",
   "author": "Contentstack CLI",
   "homepage": "https://github.com/contentstack/cli",
@@ -21,32 +21,32 @@
     "@contentstack/cli-command": "~1.3.3",
     "@contentstack/cli-utilities": "~1.8.2",
     "@oclif/plugin-help": "^5.2.20",
-    "@oclif/plugin-plugins": "^5.4.24",
+    "@oclif/plugin-plugins": "^5.4.34",
     "chalk": "^4.1.2",
     "fast-csv": "^4.3.6",
-    "fs-extra": "^11.2.0",
+    "fs-extra": "^11.3.0",
     "lodash": "^4.17.21",
     "uuid": "^9.0.1",
     "winston": "^3.17.0"
   },
   "devDependencies": {
-    "@oclif/test": "^4.1.6",
+    "@oclif/test": "^4.1.11",
     "@types/chai": "^4.3.20",
     "@types/fs-extra": "^11.0.4",
     "@types/mocha": "^10.0.10",
-    "@types/node": "^20.17.10",
+    "@types/node": "^20.17.19",
     "@types/uuid": "^9.0.8",
     "chai": "^4.5.0",
     "eslint": "^8.57.1",
     "eslint-config-oclif": "^4.0.0",
-    "eslint-config-oclif-typescript": "^3.1.13",
+    "eslint-config-oclif-typescript": "^3.1.14",
     "mocha": "^10.8.2",
     "nyc": "^15.1.0",
     "oclif": "^3.17.2",
     "shx": "^0.3.4",
     "sinon": "^19.0.2",
     "ts-node": "^10.9.2",
-    "typescript": "^5.7.2"
+    "typescript": "^5.7.3"
   },
   "oclif": {
     "bin": "csdx",

packages/contentstack-audit/src/audit-base-command.ts

@@ -12,7 +12,7 @@ import config from './config';
 import { print } from './util/log';
 import { auditMsg } from './messages';
 import { BaseCommand } from './base-command';
-import { Entries, GlobalField, ContentType, Extensions, Workflows } from './modules';
+import { Entries, GlobalField, ContentType, Extensions, Workflows, Assets } from './modules';
 import {
   CommandNames,
   ContentTypeStruct,
@@ -58,7 +58,9 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
       missingSelectFeild,
       missingMandatoryFields,
       missingTitleFields,
-      missingRefInCustomRoles
+      missingRefInCustomRoles,
+      missingEnvLocalesInAssets,
+      missingEnvLocalesInEntries
     } = await this.scanAndFix();
 
     this.showOutputOnScreen([
@@ -76,6 +78,8 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
       { module: 'Entries Title Field', missingRefs: missingTitleFields },
     ]);
     this.showOutputOnScreenWorkflowsAndExtension([{ module: 'Custom Roles', missingRefs: missingRefInCustomRoles }]);
+    this.showOutputOnScreenWorkflowsAndExtension([{ module: 'Assets', missingRefs: missingEnvLocalesInAssets }]);
+    this.showOutputOnScreenWorkflowsAndExtension([{ module: 'Entries Missing Locale and Environments', missingRefs: missingEnvLocalesInEntries }])
     if (
       !isEmpty(missingCtRefs) ||
       !isEmpty(missingGfRefs) ||
@@ -84,7 +88,9 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
       !isEmpty(missingCtRefsInExtensions) ||
       !isEmpty(missingSelectFeild) ||
       !isEmpty(missingTitleFields) ||
-      !isEmpty(missingRefInCustomRoles) 
+      !isEmpty(missingRefInCustomRoles) ||
+      !isEmpty(missingEnvLocalesInAssets) ||
+      !isEmpty(missingEnvLocalesInEntries)
     ) {
       if (this.currentCommand === 'cm:stacks:audit') {
         this.log(this.$t(auditMsg.FINAL_REPORT_PATH, { path: this.sharedConfig.reportPath }), 'warn');
@@ -112,7 +118,9 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
       !isEmpty(missingCtRefsInWorkflow) ||
       !isEmpty(missingCtRefsInExtensions) ||
       !isEmpty(missingSelectFeild) || 
-      !isEmpty(missingRefInCustomRoles)
+      !isEmpty(missingRefInCustomRoles) ||
+      !isEmpty(missingEnvLocalesInAssets) ||
+      !isEmpty(missingEnvLocalesInEntries)
     );
   }
 
@@ -133,7 +141,9 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
       missingEntry,
       missingMandatoryFields,
       missingTitleFields,
-      missingRefInCustomRoles;
+      missingRefInCustomRoles,
+      missingEnvLocalesInAssets,
+      missingEnvLocalesInEntries;
 
     for (const module of this.sharedConfig.flags.modules || this.sharedConfig.modules) {
       print([
@@ -153,6 +163,10 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
         fix: this.currentCommand === 'cm:stacks:audit:fix',
       };
       switch (module) {
+        case 'assets':
+          missingEnvLocalesInAssets = await new Assets(cloneDeep(constructorParam)).run();
+          await this.prepareReport(module, missingEnvLocalesInAssets);
+          break;
         case 'content-types':
           missingCtRefs = await new ContentType(cloneDeep(constructorParam)).run();
           await this.prepareReport(module, missingCtRefs);
@@ -167,6 +181,7 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
           missingSelectFeild = missingEntry.missingSelectFeild ?? {};
           missingMandatoryFields = missingEntry.missingMandatoryFields ?? {};
           missingTitleFields = missingEntry.missingTitleFields ?? {};
+          missingEnvLocalesInEntries = missingEntry.missingEnvLocale??{};
           await this.prepareReport(module, missingEntryRefs);
 
           await this.prepareReport(`Entries_Select_feild`, missingSelectFeild);
@@ -175,6 +190,8 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
 
           await this.prepareReport('Entries_Title_feild', missingTitleFields);
 
+          await this.prepareReport('Entry_Missing_Locale_and_Env_in_Publish_Details', missingEnvLocalesInEntries);
+
           break;
         case 'workflows':
           missingCtRefsInWorkflow = await new Workflows({
@@ -220,6 +237,8 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
       missingMandatoryFields,
       missingTitleFields,
       missingRefInCustomRoles,
+      missingEnvLocalesInAssets,
+      missingEnvLocalesInEntries
     };
   }
 

packages/contentstack-audit/src/config/index.ts

@@ -2,7 +2,7 @@ const config = {
   showTerminalOutput: true,
   skipRefs: ['sys_assets'],
   skipFieldTypes: ['taxonomy', 'group'],
-  modules: ['content-types', 'global-fields', 'entries', 'extensions', 'workflows', 'custom-roles'],
+  modules: ['content-types', 'global-fields', 'entries', 'extensions', 'workflows', 'custom-roles', 'assets'],
   'fix-fields': ['reference', 'global_field', 'json:rte', 'json:extension', 'blocks', 'group', 'content_types'],
   moduleConfig: {
     'content-types': {
@@ -40,6 +40,16 @@ const config = {
       dirName: 'custom-roles',
       fileName: 'custom-roles.json',
     },
+    'assets': {
+      name: 'assets',
+      dirName: 'assets',
+      fileName: 'assets.json',
+    },
+    'environments': {
+      name: 'environments',
+      dirName: 'environments',
+      fileName: 'environments.json',
+    }
   },
   entries: {
     systemKeys: [
@@ -76,13 +86,23 @@ const config = {
     'min_instance',
     'missingFieldUid',
     'isPublished',
+    'locale',
+    'environment',
+    'ctUid',
+    'ctLocale',
+    'entry_uid',
+    'publish_locale',
+    'publish_environment',
+    'asset_uid'
   ],
   ReportTitleForEntries: {
     Entries_Select_feild: 'Entries_Select_feild',
     Entries_Mandatory_feild: 'Entries_Mandatory_feild',
     Entries_Title_feild: 'Entries_Title_feild',
+    Entry_Missing_Locale_and_Env: 'Entry_Missing_Locale_and_Env',
+    Entry_Missing_Locale_and_Env_in_Publish_Details: 'Entry_Missing_Locale_and_Env_in_Publish_Details'
   },
-  feild_level_modules: ['Entries_Title_feild', 'Entries_Mandatory_feild', 'Entries_Select_feild'],
+  feild_level_modules: ['Entries_Title_feild', 'Entries_Mandatory_feild', 'Entries_Select_feild', 'Entry_Missing_Locale_and_Env_in_Publish_Details'],
 };
 
 export default config;

packages/contentstack-audit/src/messages/index.ts

@@ -37,6 +37,12 @@ const auditMsg = {
   AUDIT_CMD_DESCRIPTION: 'Perform audits and find possible errors in the exported Contentstack data',
   SCAN_WF_SUCCESS_MSG: 'Successfully completed the scanning of workflow with UID {uid} and name {name}.',
   SCAN_CR_SUCCESS_MSG: 'Successfully completed the scanning of custom role with UID {uid} and name {name}.',
+  SCAN_ASSET_SUCCESS_MSG: `Successfully completed the scanning of Asset with UID '{uid}'.`,
+  SCAN_ASSET_WARN_MSG: `The locale '{locale}' or environment '{environment}' are not present for asset with uid '{uid}'`,
+  ENTRY_PUBLISH_DETAILS: `Removing the publish detials for entry '{uid}' of ct '{ctuid}' in locale '{locale}' as locale '{publocale}' or environment '{environment}' does not exist`,
+  CT_REFERENCE_FIELD: `The mentioned Reference Field is not Array field name 'reference_to' having display name 'display_name'`,
+  ASSET_NOT_EXIST: `The publish_details either does not exist or is not an array for asset uid '{uid}'`,
+  ENTRY_PUBLISH_DETAILS_NOT_EXIST: `The publish_details either does not exist or is not an array for entry uid '{uid}'`,
 };
 
 const auditFixMsg = {
@@ -49,6 +55,7 @@ const auditFixMsg = {
   WF_FIX_MSG: 'Successfully removed the workflow {uid} named {name}.',
   ENTRY_MANDATORY_FIELD_FIX: `Removing the publish details from the entry with UID '{uid}' in Locale '{locale}'...`,
   ENTRY_SELECT_FIELD_FIX: `Adding the value '{value}' in the select field of entry UID '{uid}'...`,
+  ASSET_FIX: `Fixed publish detials for Asset with UID '{uid}'`,
 };
 
 const messages: typeof errors &

packages/contentstack-audit/src/modules/assets.ts

@@ -0,0 +1,176 @@
+import { join, resolve } from 'path';
+import { existsSync, readFileSync, writeFileSync } from 'fs';
+
+import { FsUtility, sanitizePath, ux } from '@contentstack/cli-utilities';
+
+import {
+  LogFn,
+  ConfigType,
+  ContentTypeStruct,
+  CtConstructorParam,
+  ModuleConstructorParam,
+  EntryStruct,
+} from '../types';
+import auditConfig from '../config';
+import { $t, auditFixMsg, auditMsg, commonMsg } from '../messages';
+import values from 'lodash/values';
+import { keys } from 'lodash';
+
+/* The `ContentType` class is responsible for scanning content types, looking for references, and
+generating a report in JSON and CSV formats. */
+export default class Assets {
+  public log: LogFn;
+  protected fix: boolean;
+  public fileName: string;
+  public config: ConfigType;
+  public folderPath: string;
+  public currentUid!: string;
+  public currentTitle!: string;
+  public assets!: Record<string, any>;
+  public locales: string[] = [];
+  public environments: string[] = [];
+  protected schema: ContentTypeStruct[] = [];
+  protected missingEnvLocales: Record<string, any> = {};
+  public moduleName: keyof typeof auditConfig.moduleConfig;
+
+  constructor({ log, fix, config, moduleName }: ModuleConstructorParam & CtConstructorParam) {
+    this.log = log;
+    this.config = config;
+    this.fix = fix ?? false;
+    this.moduleName = this.validateModules(moduleName!, this.config.moduleConfig);
+    this.fileName = config.moduleConfig[this.moduleName].fileName;
+    this.folderPath = resolve(
+      sanitizePath(config.basePath),
+      sanitizePath(config.moduleConfig[this.moduleName].dirName),
+    );
+  }
+
+  validateModules(
+    moduleName: keyof typeof auditConfig.moduleConfig,
+    moduleConfig: Record<string, unknown>,
+  ): keyof typeof auditConfig.moduleConfig {
+    if (Object.keys(moduleConfig).includes(moduleName)) {
+      return moduleName;
+    }
+    return 'assets';
+  }
+  /**
+   * The `run` function checks if a folder path exists, sets the schema based on the module name,
+   * iterates over the schema and looks for references, and returns a list of missing references.
+   * @returns the `missingEnvLocales` object.
+   */
+  async run(returnFixSchema = false) {
+    if (!existsSync(this.folderPath)) {
+      this.log(`Skipping ${this.moduleName} audit`, 'warn');
+      this.log($t(auditMsg.NOT_VALID_PATH, { path: this.folderPath }), { color: 'yellow' });
+      return returnFixSchema ? [] : {};
+    }
+
+    await this.prerequisiteData();
+    await this.lookForReference();
+
+    if (returnFixSchema) {
+      return this.schema;
+    }
+
+    for (let propName in this.missingEnvLocales) {
+      if (Array.isArray(this.missingEnvLocales[propName])) {
+        if (!this.missingEnvLocales[propName].length) {
+          delete this.missingEnvLocales[propName];
+        }
+      }
+    }
+
+    return this.missingEnvLocales;
+  }
+
+  /**
+   * @method prerequisiteData
+   * The `prerequisiteData` function reads and parses JSON files to retrieve extension and marketplace
+   * app data, and stores them in the `extensions` array.
+   */
+  async prerequisiteData() {
+    this.log(auditMsg.PREPARING_ENTRY_METADATA, 'info');
+
+    const localesFolderPath = resolve(this.config.basePath, this.config.moduleConfig.locales.dirName);
+    const localesPath = join(localesFolderPath, this.config.moduleConfig.locales.fileName);
+    const masterLocalesPath = join(localesFolderPath, 'master-locale.json');
+    this.locales = existsSync(masterLocalesPath) ? values(JSON.parse(readFileSync(masterLocalesPath, 'utf8'))) : [];
+
+    if (existsSync(localesPath)) {
+      this.locales.push(...values(JSON.parse(readFileSync(localesPath, 'utf8'))));
+    }
+    this.locales = this.locales.map((locale: any) => locale.code);
+    const environmentPath = resolve(
+      this.config.basePath,
+      this.config.moduleConfig.environments.dirName,
+      this.config.moduleConfig.environments.fileName,
+    );
+    this.environments = existsSync(environmentPath) ? keys(JSON.parse(readFileSync(environmentPath, 'utf8'))) : [];
+  }
+
+  /**
+   * The function checks if it can write the fix content to a file and if so, it writes the content as
+   * JSON to the specified file path.
+   */
+  async writeFixContent(filePath: string, schema: Record<string, EntryStruct>) {
+    let canWrite = true;
+
+    if (this.fix) {
+      if (!this.config.flags['copy-dir'] && !this.config.flags['external-config']?.skipConfirm) {
+        canWrite = this.config.flags.yes || (await ux.confirm(commonMsg.FIX_CONFIRMATION));
+      }
+
+      if (canWrite) {
+        writeFileSync(filePath, JSON.stringify(schema));
+      }
+    }
+  }
+
+  /**
+   * This function traverse over the publish detials of the assets and remove the publish details where the locale or environment does not exist
+   */
+  async lookForReference(): Promise<void> {
+    let basePath = join(this.folderPath);
+    let fsUtility = new FsUtility({ basePath, indexFileName: 'assets.json' });
+    let indexer = fsUtility.indexFileContent;
+    for (const fileIndex in indexer) {
+      const assets = (await fsUtility.readChunkFiles.next()) as Record<string, EntryStruct>;
+      this.assets = assets;
+      for (const assetUid in assets) {
+        if (this.assets[assetUid]?.publish_details && !Array.isArray(this.assets[assetUid].publish_details)) {
+          this.log($t(auditMsg.ASSET_NOT_EXIST, { uid: assetUid }), { color: 'red' });
+        }
+
+        this.assets[assetUid].publish_details = this.assets[assetUid]?.publish_details.filter((pd: any) => {
+          if (this.locales?.includes(pd?.locale) && this.environments?.includes(pd?.environment)) {
+            this.log($t(auditMsg.SCAN_ASSET_SUCCESS_MSG, { uid: assetUid }), { color: 'green' });
+            return true;
+          } else {
+            this.log(
+              $t(auditMsg.SCAN_ASSET_WARN_MSG, { uid: assetUid, locale: pd.locale, environment: pd.environment }),
+              { color: 'yellow' },
+            );
+            if (!Object.keys(this.missingEnvLocales).includes(assetUid)) {
+              this.missingEnvLocales[assetUid] = [
+                { asset_uid: assetUid, publish_locale: pd.locale, publish_environment: pd.environment },
+              ];
+            } else {
+              this.missingEnvLocales[assetUid].push({
+                asset_uid: assetUid,
+                publish_locale: pd.locale,
+                publish_environment: pd.environment,
+              });
+            }
+            this.log($t(auditMsg.SCAN_ASSET_SUCCESS_MSG, { uid: assetUid }), { color: 'green' });
+            return false;
+          }
+        });
+        if (this.fix) {
+          this.log($t(auditFixMsg.ASSET_FIX, { uid: assetUid }), { color: 'green' });
+          await this.writeFixContent(`${basePath}/${indexer[fileIndex]}`, this.assets);
+        }
+      }
+    }
+  }
+}