feat: flags to avoid ads and cookie banners

Author: mukulboro

.gitignore

@@ -30,6 +30,7 @@ Thumbs.db
 dist/
 coverage/
 build/
+test/
 
 # Optional: lock files (if you want to avoid committing them)
 # package-lock.json

README.md

@@ -376,7 +376,10 @@ Arguments:
 Options:
   -o, --output <dir>      Custom output directory (default: domain name)
   --clean                 Remove tracking scripts and analytics
+  --block-ads             Block advertisements using adblocker plugin
+  --block-cookies         Automatically remove and block cookie consent banners
   --ai                    Enable AI-powered analysis (requires OpenAI API key)
+  --ai-model <model>      AI model to use (default: gemini-3-flash-preview)
   --openai-key <key>      OpenAI API key for AI features (or set OPENAI_API_KEY env var)
   --debug                 Enable detailed debug logging
   --timeout <ms>          Page load timeout in milliseconds (default: 120000)

package.json

@@ -58,6 +58,8 @@
     "node-fetch": "^3.3.2",
     "openai": "^4.67.3",
     "puppeteer": "^24.16.2",
+    "puppeteer-extra": "^3.3.6",
+    "puppeteer-extra-plugin-adblocker": "^2.13.6",
     "sharp": "^0.33.5"
   },
   "repository": {

src/cli.js

@@ -127,6 +127,16 @@ program
     'Remove tracking scripts, analytics, and third-party code',
     false,
   )
+  .option(
+    '--block-ads',
+    'Block advertisements using adblocker plugin',
+    false,
+  )
+  .option(
+    '--block-cookies',
+    'Automatically remove and block cookie consent banners',
+    false,
+  )
   .option(
     '--ai',
     'Enable AI-powered website analysis (reads OPENAI_API_KEY or GEMINI_API_KEY from env)',
@@ -162,6 +172,8 @@ program
       const config = {
         outputDir: options.output,
         clean: options.clean,
+        blockAds: options.blockAds,
+        blockCookies: options.blockCookies,
         ai: aiEnabled,
         aiModel: options.aiModel,
         debug: options.debug,

src/core/browser-engine.js

@@ -1,4 +1,5 @@
-import puppeteer from 'puppeteer';
+import puppeteer from 'puppeteer-extra';
+import AdblockerPlugin from 'puppeteer-extra-plugin-adblocker';
 import chalk from 'chalk';
 
 /**
@@ -10,10 +11,17 @@ export class BrowserEngine {
       headless: options.headless !== false,
       timeout: options.timeout || 120000,
       debug: options.debug || false,
+      blockAds: !!options.blockAds,
+      blockCookies: !!options.blockCookies,
       ...options,
     };
     this.browser = null;
     this.page = null;
+
+    // Use adblocker plugin if enabled
+    if (this.options.blockAds) {
+      puppeteer.use(AdblockerPlugin({ blockTrackers: true }));
+    }
   }
 
   /**
@@ -39,7 +47,7 @@ export class BrowserEngine {
 
     try {
       await this.page.setBypassCSP(true);
-    } catch {}
+    } catch { }
 
     await this.page.setViewport({ width: 1366, height: 768 });
     await this.page.setUserAgent(
@@ -49,18 +57,19 @@ export class BrowserEngine {
     await this.page.setDefaultTimeout(this.options.timeout);
     await this.page.setDefaultNavigationTimeout(this.options.timeout);
 
+    // If adblocker is NOT enabled, we still use request interception for basic logic
+    // If it IS enabled, puppeteer-extra-plugin-adblocker handles much of this.
     await this.page.setRequestInterception(true);
 
     this.page.on('request', (request) => {
-      request.continue().catch(() => {});
+      request.continue().catch(() => { });
     });
 
     this.page.on('requestfailed', (request) => {
       if (this.options.debug) {
         console.log(
           chalk.yellow(
-            `⚠️ Failed request: ${request.url()} - ${
-              request.failure()?.errorText || ''
+            `⚠️ Failed request: ${request.url()} - ${request.failure()?.errorText || ''
             }`,
           ),
         );
@@ -70,11 +79,175 @@ export class BrowserEngine {
     return this.page;
   }
 
+  /**
+   * Automatically detect and remove/click-through cookie consent banners
+   */
+  async handleCookieConsent(page) {
+    if (!this.options.blockCookies) return;
+
+    if (this.options.debug) {
+      console.log(chalk.gray('  🍪 Attempting to remove cookie consent banners...'));
+    }
+
+    try {
+      await page.evaluate(() => {
+        // 1. Common selectors for cookie banners and modals
+        const cookieSelectors = [
+          '[id*="cookie" i]', '[class*="cookie" i]',
+          '[id*="consent" i]', '[class*="consent" i]',
+          '[id*="privacy" i]', '[class*="privacy" i]',
+          '[id*="banner" i]', '[class*="banner" i]',
+          '[id*="notice" i]', '[class*="notice" i]',
+          '[id*="modal" i]', '[class*="modal" i]',
+          '.optanon-alert-box-wrapper', '#onetrust-consent-sdk',
+          '.cc_banner', '.cc_container',
+          '#didomi-host'
+        ];
+
+        // 2. Common button text for "Accept"
+        const acceptButtonTexts = [
+          // Core accept
+          'Accept',
+          'Accept All',
+          'Accept all cookies',
+          'Accept Cookies',
+          'Accept all',
+          'Accept & Continue',
+          'Accept and Continue',
+          'Accept and Close',
+          'Accept All Cookies',
+          'Accept all and continue',
+
+          // Agree
+          'Agree',
+          'I Agree',
+          'I agree',
+          'Agree & Continue',
+          'Agree and Continue',
+
+          // Allow
+          'Allow',
+          'Allow All',
+          'Allow all cookies',
+          'Allow Cookies',
+          'Allow all',
+          'Allow & Continue',
+
+          // Consent
+          'Consent',
+          'Give Consent',
+          'Provide Consent',
+          'Yes I Consent',
+
+          // Confirm / continue
+          'Confirm',
+          'Confirm choices',
+          'Confirm selection',
+          'Confirm and continue',
+          'Continue',
+          'Continue to site',
+          'Continue without changes',
+
+          // OK style
+          'OK',
+          'Okay',
+          'Ok',
+          'Got it',
+          'Understood',
+          'I understand',
+          'Sounds good',
+
+          // Close style (many banners accept on close)
+          'Close',
+          'Close and accept',
+          'Dismiss',
+          'Done',
+
+          // Save / approve
+          'Save',
+          'Save and accept',
+          'Save preferences',
+          'Approve',
+          'Approve all',
+
+          // Misc common wording
+          'Yes',
+          'Yes, I agree',
+          'Yes, accept',
+          'Yes, allow',
+          'Enable all',
+          'Enable cookies',
+          'Accept recommended',
+          'Accept suggested',
+
+          // GDPR CMP wording
+          'Accept all purposes',
+          'Accept all cookies and continue',
+          'Agree to all',
+          'Allow all and continue'
+        ];
+
+        // Remove identified banners
+        cookieSelectors.forEach(selector => {
+          try {
+            document.querySelectorAll(selector).forEach(el => {
+              // Only remove if it looks like a popup (likely fixed or absolute)
+              const style = window.getComputedStyle(el);
+              if (style.position === 'fixed' || style.position === 'absolute' || el.tagName === 'DIV') {
+                el.style.display = 'none';
+                el.setAttribute('aria-hidden', 'true');
+              }
+            });
+          } catch (e) { }
+        });
+
+        // Click "Accept" buttons if found (case-insensitive partial match)
+        const buttons = Array.from(document.querySelectorAll('button, a, span, div[role="button"]'));
+        for (const btn of buttons) {
+          const text = (btn.innerText || btn.textContent || '').trim();
+          if (!text || text.length > 50) continue; // Skip empty or too long text (unlikely to be a simple button)
+
+          const lowerText = text.toLowerCase();
+          if (acceptButtonTexts.some(t => lowerText.includes(t.toLowerCase()))) {
+            try {
+              btn.click();
+            } catch (e) { }
+          }
+        }
+
+        // 3. Remove overlay/backdrop if present
+        const overlays = [
+          '.modal-backdrop', '.fade', '.in',
+          '[class*="backdrop" i]', '[id*="backdrop" i]',
+          '[class*="overlay" i]', '[id*="overlay" i]'
+        ];
+        overlays.forEach(selector => {
+          try {
+            document.querySelectorAll(selector).forEach(el => {
+              const style = window.getComputedStyle(el);
+              if (style.position === 'fixed' || style.zIndex > 100) {
+                el.style.display = 'none';
+              }
+            });
+          } catch (e) { }
+        });
+
+        // 4. Restore scrolling if it was disabled by a modal
+        document.body.style.overflow = 'auto';
+        document.documentElement.style.overflow = 'auto';
+      });
+    } catch (error) {
+      if (this.options.debug) {
+        console.log(chalk.gray(`  ⚠️ Cookie removal error: ${error.message}`));
+      }
+    }
+  }
+
   async close() {
     if (this.browser) {
       try {
         await this.browser.close();
-      } catch {}
+      } catch { }
       this.browser = null;
       this.page = null;
     }

src/core/mirror-cloner.js

@@ -569,6 +569,9 @@ export class MirrorCloner {
       timeout: this.options.timeout,
     });
 
+    // Handle cookie consent banners before further processing
+    await this.browserEngine.handleCookieConsent(page);
+
     await this.waitForRootReady(page);
     await this.scrollToBottomAndLoad(page);
     await this.waitForImagesSettled(page, 8000);