Working Branch for PLP fixes (#271)

* adds local_fs capability, to debug without aio. bumps timeouts and memory

Signed-off-by: Stephen Rugh <rugh@adobe.com>

* increasing timeout to 3 hours and commenting out triggers for now

Signed-off-by: Stephen Rugh <rugh@adobe.com>

* add claude tool to summarize and check errors in activations

* batch getUrlKey GraphQL queries in mark-up-clean-up to stay within catalog 100-product limit

Passing all published SKUs in a single query fails when the catalog exceeds 100 products
(observed at 23,398 SKUs: "Product count exceeds the maximum allowed (100)"). With no
query result, every published product appears redundant and risks mass unpublishing.

Fix: split SKUs into batches of 50 (reusing createBatches/BATCH_SIZE) and fan out
requests in parallel, then merge results before the redundancy check.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

* add script to get the state file. add note to runbook about log availability. add watchdog exit script to exit early incase of recurring failures at commerce api (504, etc).

* Updates comments on watchdog and ensures we dont exit while publishing batches.

* Apply suggestion from @sirugh

* Apply suggestions from code review

Co-authored-by: Natxo Cabré <natxo.cabre@gmail.com>

* missed loglevel env read.

* extend get-state-csv to download both check-product-changes and render-all-categories; output to local-data/

- adds render-all-categories prefix alongside check-product-changes
- saves files to local-data/{prefix}/ (project-root-anchored via __dirname)
- uses path.basename instead of replace to derive local filenames
- anchors dotenv to project root so script works from any cwd

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

* fix: use CATALOG_BATCH_SIZE=100 for GetUrlKeyQuery batching in mark-up-clean-up

Catalog Service GraphQL rejects queries with >100 SKUs ("Product count exceeds
the maximum allowed (100)"). The batching added in 4625e42 originally used
BATCH_SIZE=50, which was safe, but 22f211d raised BATCH_SIZE to 600 for the
AEM Admin rate limit, inadvertently breaking the catalog query batching.

Fix: add CATALOG_BATCH_SIZE=100 constant, make createBatches accept an optional
size param (defaulting to BATCH_SIZE), and pass CATALOG_BATCH_SIZE explicitly
at the GetUrlKeyQuery call site. The AEM unpublish batches keep BATCH_SIZE=600.

Limit confirmed against live API: 100 SKUs succeed, 101 fails.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

* add comment explaining pLimit reduction from 50 to 20 in enrichProductWithRenderedHash

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

* Move check-errors and add readme for it

* fix tests

---------

Signed-off-by: Stephen Rugh <rugh@adobe.com>
Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
Co-authored-by: Natxo Cabré <natxo.cabre@gmail.com>

Author: 3 people

.claude/commands/check-aio-errors.md

@@ -0,0 +1 @@
+Run `bash tools/check-errors.sh $ARGUMENTS` and evaluate the results, summarizing the error types, affected actions, and likely root causes.

.claude/settings.local.json

@@ -0,0 +1,7 @@
+{
+  "permissions": {
+    "allow": [
+      "WebFetch(domain:www.aem.live)"
+    ]
+  }
+}

.gitignore

@@ -47,3 +47,6 @@ logs
 
 # AI
 .cursor/
+
+# LOCAL_FS=true files
+local-data/

CLAUDE.md

@@ -0,0 +1,5 @@
+# Project Rules
+
+## App Builder Deployments
+
+Use `npm run deploy` instead of `aio app deploy` for all App Builder deployments.

actions/check-product-changes/index.js

@@ -10,12 +10,17 @@ governing permissions and limitations under the License.
 */
 
 const { Core, State, Files } = require('@adobe/aio-sdk');
+const { localFilesLib } = require('../lib/localFilesLib');
 const { poll } = require('./poller');
 const { StateManager } = require('../lib/state');
 const { ObservabilityClient } = require('../lib/observability');
 const { getRuntimeConfig } = require('../lib/runtimeConfig');
 const { handleActionError } = require('../lib/errorHandler');
 
+// Must match timeout in app.config.yaml. The mutex TTL is derived from this so the
+// lock auto-expires if the runtime kills the process before the finally block runs.
+const ACTION_TIMEOUT_MS = 10800000; // 3 hours
+
 /**
  * Entry point for the "Product changes check" action.
  * @param {Object} params
@@ -38,8 +43,11 @@ async function main(params) {
         });
 
         // Init SDK libs and state manager
-        const stateLib = await State.init(params.libInit || {});
-        const filesLib = await Files.init(params.libInit || {});
+        const isLocal = !!params.LOCAL_FS;
+        const stateLib = isLocal
+            ? { get: async () => null, put: async () => {}, delete: async () => {} }
+            : await State.init(params.libInit || {});
+        const filesLib = isLocal ? localFilesLib : await Files.init(params.libInit || {});
         const stateMgr = new StateManager(stateLib, { logger });
 
         let activationResult;
@@ -61,7 +69,7 @@ async function main(params) {
 
         try {
             // Mark job as running with TTL to avoid permanent lock on unexpected failures
-            await stateMgr.put('running', 'true', { ttl: 3600 });
+            await stateMgr.put('running', 'true', { ttl: ACTION_TIMEOUT_MS / 1000 });
 
             // Core logic
             activationResult = await poll(cfg, { stateLib: stateMgr, filesLib }, logger);

actions/check-product-changes/poller.js

@@ -35,6 +35,10 @@ const { JobFailedError, ERROR_CODES } = require('../lib/errorHandler');
 const crypto = require('crypto');
 const BATCH_SIZE = 50;
 const DATA_KEY = 'skus';
+// Exit early rather than burning the full 3-hour timeout when processing stalls.
+// Activity is recorded after each render completes and after each AEM batch completes.
+const WATCHDOG_TIMEOUT_MS = 5 * 60 * 1000;
+const WATCHDOG_CHECK_INTERVAL_MS = 30 * 1000;
 
 function checkParams(params) {
   validateRequiredParams(params, [
@@ -95,8 +99,12 @@ async function enrichProductWithRenderedHash(product, context) {
   const { logger } = context;
   const { sku, urlKey, path } = product;
 
+  // Reduced from 50 → 20: each concurrent render makes a full-detail GraphQL
+  // request to the Commerce API (ProductQuery) plus HTML compilation. At 50
+  // concurrent renders the Commerce API returned 504s, stalling the action and
+  // triggering the watchdog.
   if (!renderLimit$) {
-    renderLimit$ = import('p-limit').then(({ default: pLimit }) => pLimit(50));
+    renderLimit$ = import('p-limit').then(({ default: pLimit }) => pLimit(20));
   }
 
   return (await renderLimit$)(async () => {
@@ -122,6 +130,7 @@ async function enrichProductWithRenderedHash(product, context) {
       logger.error(`Error generating product HTML for SKU ${sku}:`, e);
     }
 
+    context.recordActivity?.();
     return product;
   });
 }
@@ -284,11 +293,31 @@ async function poll(params, aioLibs, logger) {
 
     let stateText = 'completed';
 
+    // Watchdog: fires if no progress (render or AEM batch completion) for WATCHDOG_TIMEOUT_MS.
+    // recordActivity is called after each render (success or failure) and after each AEM batch completes.
+    let lastActivityAt = Date.now();
+    let watchdogIntervalId;
+    const watchdogPromise = new Promise((_, reject) => {
+      watchdogIntervalId = setInterval(() => {
+        const idleMs = Date.now() - lastActivityAt;
+        if (idleMs > WATCHDOG_TIMEOUT_MS) {
+          clearInterval(watchdogIntervalId);
+          const err = Object.assign(
+            new Error(`Watchdog: no processing activity for ${Math.floor(idleMs / 60000)} minutes — aborting`),
+            { isWatchdog: true },
+          );
+          logger.error(err.message);
+          reject(err);
+        }
+      }, WATCHDOG_CHECK_INTERVAL_MS);
+    });
+    sharedContext.recordActivity = () => { lastActivityAt = Date.now(); };
+
     try {
       // start processing preview and publish queues
       await adminApi.startProcessing();
 
-      const results = await Promise.all(
+      const localeProcessing = Promise.all(
         locales.map(async (locale) => {
           const timings = new Timings();
           const context = { ...sharedContext, startTime: new Date() };
@@ -343,13 +372,14 @@ async function poll(params, aioLibs, logger) {
                   const records = products.map(({ sku, path, renderedAt }) => ({ sku, path, renderedAt }));
                   return adminApi
                     .previewAndPublish(records, locale, batchNumber + 1)
-                    .then((publishedBatch) =>
-                      processPublishedBatch(publishedBatch, state, counts, products, aioLibs, {
+                    .then((publishedBatch) => {
+                      context.recordActivity?.();
+                      return processPublishedBatch(publishedBatch, state, counts, products, aioLibs, {
                         dataKey: DATA_KEY,
                         keyField: 'sku',
                         filePrefix: FILE_PREFIX,
-                      }),
-                    )
+                      });
+                    })
                     .catch((error) => {
                       // Handle batch errors gracefully - don't fail the entire job
                       if (error.code === ERROR_CODES.BATCH_ERROR) {
@@ -386,6 +416,8 @@ async function poll(params, aioLibs, logger) {
         }),
       );
 
+      const results = await Promise.race([localeProcessing, watchdogPromise]);
+
       await adminApi.stopProcessing();
 
       // aggregate timings
@@ -406,8 +438,14 @@ async function poll(params, aioLibs, logger) {
         code: e.code,
         stack: e.stack,
       });
-      // wait for queues to finish, even in error case
-      await adminApi.stopProcessing();
+      if (e.isWatchdog) {
+        // Don't drain queues — processing is stalled. Abort immediately so the mutex is
+        // cleared promptly and the next scheduled run can start.
+        adminApi.abortProcessing();
+      } else {
+        // wait for queues to finish, even in error case
+        await adminApi.stopProcessing();
+      }
       stateText = 'failure';
 
       // If it's a JobFailedError, re-throw it
@@ -422,6 +460,8 @@ async function poll(params, aioLibs, logger) {
         e.statusCode || 500,
         { originalError: e.message },
       );
+    } finally {
+      clearInterval(watchdogIntervalId);
     }
 
     // get memory usage

actions/fetch-all-products/index.js

@@ -13,6 +13,7 @@ governing permissions and limitations under the License.
 */
 
 const { Core, Files } = require('@adobe/aio-sdk');
+const { localFilesLib } = require('../lib/localFilesLib');
 const { getConfig, getSiteType, requestSaaS, SITE_TYPES, FILE_PREFIX } = require('../utils');
 const { ProductsQuery, ProductCountQuery } = require('../queries');
 const { Timings } = require('../lib/benchmark');
@@ -224,7 +225,7 @@ async function main(params) {
         const allProducts = await getAllProducts(siteType, context, cfg.categoryFamilies);
 
         timings.sample('getAllProducts');
-        const filesLib = await Files.init(params.libInit || {});
+        const filesLib = params.LOCAL_FS ? localFilesLib : await Files.init(params.libInit || {});
         timings.sample('saveFile');
         const productsFileName = `${FILE_PREFIX}/${stateFilePrefix}-products.json`;
         logger.debug(`Saving products to ${productsFileName}`);

actions/lib/aem.js

@@ -31,10 +31,10 @@ class AdminAPI {
     inflight = [];
     MAX_RETRIES = 3;
     RETRY_DELAY = 5000;
-    /** Max number of pending jobs (queues + inflight). Keep low to avoid "noisy neighbor" effect. */
-    MAX_PENDING_JOBS = 20;
-    /** Poll interval for job status (ms). Avoid polling too frequently (e.g. once every 5 seconds). */
-    JOB_STATUS_POLL_INTERVAL_MS = 5000;
+    /** Max number of pending jobs (queues + inflight). With BATCH_SIZE=600, keep small to avoid large memory backlog. */
+    MAX_PENDING_JOBS = 4;
+    /** Poll interval for job status (ms). At BATCH_SIZE=600, jobs take ~60s so 15s polling is sufficient. */
+    JOB_STATUS_POLL_INTERVAL_MS = 15000;
     /** Resolvers for backpressure: call when pending drops below MAX_PENDING_JOBS */
     _backpressureResolvers = [];
 
@@ -134,6 +134,11 @@ class AdminAPI {
         return this.stopProcessing$;
     }
 
+    abortProcessing() {
+        clearInterval(this.interval);
+        this.interval = null;
+    }
+
     trackInFlight(name, callback) {
         const executeTask = () => {
             const promise = new Promise(callback);

actions/lib/localFilesLib.js

@@ -0,0 +1,33 @@
+const fs = require('fs');
+const path = require('path');
+
+const BASE_DIR = path.resolve(__dirname, '../../local-data');
+
+function resolvePath(filePath) {
+  return path.join(BASE_DIR, filePath);
+}
+
+const localFilesLib = {
+  async write(filePath, content) {
+    const fullPath = resolvePath(filePath);
+    fs.mkdirSync(path.dirname(fullPath), { recursive: true });
+    fs.writeFileSync(fullPath, content);
+  },
+
+  async read(filePath) {
+    return fs.readFileSync(resolvePath(filePath));
+  },
+
+  async delete(filePath) {
+    const fullPath = resolvePath(filePath);
+    if (fs.existsSync(fullPath)) fs.unlinkSync(fullPath);
+  },
+
+  async list(prefix) {
+    const fullPath = resolvePath(prefix);
+    if (!fs.existsSync(fullPath)) return [];
+    return fs.readdirSync(fullPath).map((name) => ({ name: path.join(prefix, name) }));
+  },
+};
+
+module.exports = { localFilesLib };

actions/mark-up-clean-up/index.js

@@ -11,6 +11,7 @@ governing permissions and limitations under the License.
 */
 
 const { Core, Files } = require('@adobe/aio-sdk');
+const { localFilesLib } = require('../lib/localFilesLib');
 const { ObservabilityClient } = require('../lib/observability');
 const { GetUrlKeyQuery } = require('../queries');
 const { getRuntimeConfig } = require('../lib/runtimeConfig');
@@ -19,6 +20,7 @@ const {
   requestSaaS,
   requestPublishedProductsIndex,
   createBatches,
+  CATALOG_BATCH_SIZE,
 } = require('../utils');
 const { getHtmlFilePath } = require('../renderUtils');
 
@@ -66,8 +68,11 @@ async function markUpCleanUP(context, filesLib, logger, adminApi) {
   try {    
     const publishedProducts = await requestPublishedProductsIndex(context);  
     const publishedSkus = publishedProducts.data.map((product) => product.sku);
-    let queryResult = await requestSaaS(GetUrlKeyQuery, 'getUrlKey', { skus: publishedSkus }, context);
-    queryResult = queryResult.data.products;
+    const skuBatches = createBatches(publishedSkus, CATALOG_BATCH_SIZE);
+    const batchResults = await Promise.all(
+      skuBatches.map((batch) => requestSaaS(GetUrlKeyQuery, 'getUrlKey', { skus: batch }, context))
+    );
+    const queryResult = batchResults.flatMap((result) => result.data.products);
 
     const redundantpublishedProducts = publishedProducts.data.filter((product) => !urlkeymatch(product, queryResult, context))
     context.counts.detected = redundantpublishedProducts.length;
@@ -117,7 +122,7 @@ async function main(params) {
     org: cfg.org,
     site: cfg.site
   });
-  const filesLib = await Files.init(params.libInit || {});  
+  const filesLib = params.LOCAL_FS ? localFilesLib : await Files.init(params.libInit || {});
 
   const {
     // required