markup-cleanup fix (#239)

markup-cleanup action now unpublished url path that's no longer used.

Author: jkf276

actions/check-product-changes/poller.js

@@ -14,13 +14,13 @@ const { Timings, aggregate } = require('../lib/benchmark');
 const { AdminAPI } = require('../lib/aem');
 const {
   requestSaaS,
-  requestSpreadsheet,
   isValidUrl,
   getProductUrl,
   formatMemoryUsage,
   FILE_PREFIX,
   STATE_FILE_EXT,
   PDP_FILE_EXT,
+  requestPublishedProductsIndex,
 } = require('../utils');
 const { GetLastModifiedQuery } = require('../queries');
 const { generateProductHtml } = require('../pdp-renderer/render');
@@ -293,7 +293,7 @@ async function processDeletedProducts(remainingSkus, state, context, adminApi) {
   const { filesLib } = aioLibs;
 
   try {
-    const deletedProducts = (await requestSpreadsheet('published-products-index', null, context))
+    const deletedProducts = (await requestPublishedProductsIndex(context))
       .data.filter(({ sku }) => remainingSkus.includes(sku));
 
     // Process in batches

actions/mark-up-clean-up/index.js

@@ -14,10 +14,12 @@ const { Core, Files } = require('@adobe/aio-sdk');
 const { ObservabilityClient } = require('../lib/observability');
 const { GetUrlKeyQuery } = require('../queries');
 const { getRuntimeConfig } = require('../lib/runtimeConfig');
+const { AdminAPI } = require('../lib/aem');
 const {
   requestSaaS,
-  requestSpreadsheet,
+  requestPublishedProductsIndex,
   PDP_FILE_EXT,
+  createBatches,
 } = require('../utils');
 
 /**
@@ -43,13 +45,13 @@ function urlkeymatch(publishedProduct, queriedProducts){
  * @param {Object} context - The context object with logger and other utilities
  * @param {Object} filesLib - The files library instance from '@adobe/aio-sdk'
  */
-async function markUpCleanUP(context, filesLib, logger) {
+async function markUpCleanUP(context, filesLib, logger, adminApi) {
 
   try {    
-    const publishedProducts = await requestSpreadsheet('published-products-index', null, context);  
+    const publishedProducts = await requestPublishedProductsIndex(context);  
     const publishedSkus = publishedProducts.data.map((product) => product.sku);
     let queryResult = await requestSaaS(GetUrlKeyQuery, 'getUrlKey', { skus: publishedSkus }, context);
-    queryResult = queryResult.data.products    
+    queryResult = queryResult.data.products;
 
     const redundantpublishedProducts = publishedProducts.data.filter((product) => !urlkeymatch(product, queryResult))
     context.counts.detected = redundantpublishedProducts.length;
@@ -68,6 +70,20 @@ async function markUpCleanUP(context, filesLib, logger) {
         }
       }
     };    
+    const pendingJobs = [];
+    const unpublishJobsBatches = createBatches(redundantpublishedProducts);
+    for (let batchNumber = 0; batchNumber < unpublishJobsBatches.length; batchNumber++) {
+      const batch = unpublishJobsBatches[batchNumber].map((product) => {
+        return {
+          sku: product.sku,
+          path: product.path,
+        };
+      });
+      const pendingJob =  adminApi.unpublishAndDelete(batch, context.locale, batchNumber);
+      context.counts.unpublished += batch.length;
+      pendingJobs.push(pendingJob);
+    }    
+    await Promise.all(pendingJobs);
   } catch (e) {
     logger.error('Error while cleanning up markup storage', e);
   }
@@ -99,7 +115,7 @@ async function main(params) {
     logIngestorEndpoint,
   } = cfg;
 
-  const counts = { detected: 0, deleted: 0 };
+  const counts = { detected: 0, deleted: 0, unpublished: 0 };
 
   const sharedContext = {
     storeUrl,
@@ -113,22 +129,30 @@ async function main(params) {
     logLevel,
     logIngestorEndpoint,
   }
+  const adminApi = new AdminAPI({ org: params.ORG, site: params.SITE }, sharedContext, { authToken: params.AEM_ADMIN_AUTH_TOKEN });
 
   let activationResult = {status: {}};
 
-  for (const locale of locales) {    
-    const context = { ...sharedContext, startTime: new Date() };
-    let tempLocale = 'default';
-    if (locale) {
-      context.locale = locale;
-      tempLocale = locale;
-    } 
+  try {
+    adminApi.startProcessing();
 
-    activationResult.status[tempLocale] = await markUpCleanUP(context, filesLib, logger);
-  }
+    for (const locale of locales) {    
+      const context = { ...sharedContext, startTime: new Date() };
+      let tempLocale = 'default';
+      if (locale) {
+        context.locale = locale;
+        tempLocale = locale;
+      } 
+      activationResult.status[tempLocale] = await markUpCleanUP(context, filesLib, logger);
+   }
 
-  await observabilityClient.sendActivationResult(activationResult);
-  return activationResult;
+    await observabilityClient.sendActivationResult(activationResult);
+    return activationResult;
+  } catch (e) {
+    logger.error(e);
+  } finally {
+    await adminApi.stopProcessing();
+  }
 }
 
 exports.main = main

actions/utils.js

@@ -11,12 +11,29 @@ governing permissions and limitations under the License.
 */
 const deepmerge = require('@fastify/deepmerge')();
 const helixSharedStringLib = require('@adobe/helix-shared-string');
+const BATCH_SIZE = 50;
 
 /* This file exposes some common utilities for your actions */
 
 const FILE_PREFIX = 'check-product-changes';
 const [STATE_FILE_EXT, PDP_FILE_EXT] = ['csv', 'html'];
 
+/**
+ * Creates batches of products for processing
+ * @param products
+ * @param context
+ * @returns {*}
+ */
+function createBatches(products) {
+  return products.reduce((acc, product) => {
+    if (!acc.length || acc[acc.length - 1].length === BATCH_SIZE) {
+      acc.push([]);
+    }
+    acc[acc.length - 1].push(product);
+    return acc;
+  }, []);
+}
+
 /**
  *
  * Returns the list of missing keys giving an object and its required keys.
@@ -176,13 +193,42 @@ async function request(name, url, req, timeout = 60000) {
  *
  * @returns {Promise<object>} spreadsheet data as JSON.
  */
-async function requestSpreadsheet(name, sheet, context) {
+async function requestSpreadsheet(name, sheet, context, offset = 0) {
   const { contentUrl } = context;
   let sheetUrl = `${contentUrl}/${name}.json`
+  const requestURL = new URL(sheetUrl);
+
   if (sheet) {
-    sheetUrl += `?sheet=${sheet}`;
+    requestURL.searchParams.set('sheet', sheet);
   }
-  return request('spreadsheet', sheetUrl);
+
+  if (offset > 0) {
+    requestURL.searchParams.set('offset', offset);
+  }
+
+  return request('spreadsheet', requestURL.toString());
+}
+
+/**
+ * Requests the published products index from the site.
+ *
+ * @param {string} name file name of the spreadsheet.
+ * @param {string} [sheet] optional sheet name.
+ * @param {object} context the context object.
+ *
+ * @returns {Promise<object>} spreadsheet data as JSON.
+ */
+async function requestPublishedProductsIndex(context) {
+  
+  const publishedProductsIndex = await requestSpreadsheet('published-products-index', null, context, 0);
+
+  for (let offset = 1000; offset < publishedProductsIndex.total; offset += 1000) {
+    const tempPublishedProductsIndex = await requestSpreadsheet('published-products-index', null, context, offset);
+    publishedProductsIndex.data.push(...tempPublishedProductsIndex.data);
+  }
+  publishedProductsIndex.limit = publishedProductsIndex.total;
+  
+  return publishedProductsIndex;
 }
 
 async function requestConfigService(context) {
@@ -391,6 +437,7 @@ function formatMemoryUsage(data) {
 }
 
 module.exports = {
+  createBatches,
   errorResponse,
   getBearerToken,
   checkMissingRequestInputs,
@@ -402,6 +449,7 @@ module.exports = {
   getProductUrl,
   getDefaultStoreURL,
   formatMemoryUsage,
+  requestPublishedProductsIndex,
   FILE_PREFIX,
   PDP_FILE_EXT,
   STATE_FILE_EXT,

tools/check-products-count.js

@@ -15,7 +15,7 @@ require('dotenv').config();
 const fs = require('fs');
 const yaml = require('js-yaml');
 const path = require('path');
-const { requestSaaS, requestSpreadsheet } = require('../actions/utils');
+const { requestSaaS, requestPublishedProductsIndex } = require('../actions/utils');
 const { GetAllSkusPaginatedQuery } = require('../actions/queries');
 const filePath = path.resolve(__dirname, '..', 'app.config.yaml');
 
@@ -42,7 +42,7 @@ async function main() {
     } = process.env;
 
     const context = { storeCode: options.storecode, storeUrl: options.url, configName: options.config };
-    const { total: actualCount } = await requestSpreadsheet('published-products-index', context);
+    const { total: actualCount } = await requestPublishedProductsIndex(context);
     let [productsCount, currentPage, expectedCount] = [-1, 1, 0];
     while (productsCount !== 0) {
         const { data: { productSearch: { items: products } } } = await requestSaaS(GetAllSkusPaginatedQuery, 'getAllSkusPaginated', { currentPage }, context);