Merge pull request #12 from mukulboro/main

AI Enhancements, Ad-Blocking, and Performance Logging

Author: SanjeevSaniel

.gitignore

@@ -30,7 +30,9 @@ Thumbs.db
 dist/
 coverage/
 build/
+test/
 
 # Optional: lock files (if you want to avoid committing them)
 # package-lock.json
-# yarn.lock
+# yarn.lock
+bun.lock

.npmignore

@@ -64,3 +64,5 @@ build/
 # Temporary folders
 tmp/
 temp/
+
+bun.lock

CHANGELOG.md

@@ -1,5 +1,14 @@
 # Changelog
 
+## [Unreleased]
+
+### Breaking Changes
+- **Default AI Model Changed**: The default AI model has been switched from `gpt-4o` to `gemini-3-flash-preview` to make the CLI more accessible out of the box (Google provides a free tier for Gemini API keys). If you previously relied on the default `gpt-4o` and only had an `OPENAI_API_KEY` set, you may need to pass `--ai-model gpt-4o` explicitly.
+
+### Changed
+- Improved `handleCookieConsent()` precision to avoid accidentally hiding non-banner layout elements.
+- Fixed an issue where the Adblocker plugin would register multiples times during `BrowserEngine` initialization.
+
 ## 1.0.2 - 2025-08-18
 
 ### Added

README.md

@@ -104,60 +104,55 @@ npm install
 node src/cli.js https://example.com
 ```
 
-### OpenAI API Key Setup (Required for AI Features)
+### API Key Setup (Required for AI Features)
 
-**🚨 IMPORTANT**: Users must set up their OpenAI API key in their terminal environment before using AI features. The package does NOT include pre-configured API keys.
+**🚨 IMPORTANT**: Users must set up an API key (Gemini or OpenAI) in their terminal environment before using AI features. The tool defaults to `gemini-3-flash-preview` as Google provides free API keys, making it more accessible.
 
-#### **Step 1: Get OpenAI API Key**
+#### **Step 1: Get an API Key**
 
-1. Visit [OpenAI Platform](https://platform.openai.com/api-keys)
-2. Create account and generate API key
-3. Copy your API key (starts with `sk-`)
+- **Gemini (Default)**: Get a free key from [Google AI Studio](https://aistudio.google.com/app/apikey)
+- **OpenAI (Alternative)**: Get a key from [OpenAI Platform](https://platform.openai.com/api-keys)
 
 #### **Step 2: Set Environment Variable (REQUIRED)**
 
 **Windows PowerShell:**
 
 ```powershell
 # Temporary (current session only)
-$env:OPENAI_API_KEY="sk-your-api-key-here"
+$env:GEMINI_API_KEY="AIzaSy..."
+# Or if using OpenAI:
+$env:OPENAI_API_KEY="sk-..."
 
 # Permanent (recommended)
-[System.Environment]::SetEnvironmentVariable('OPENAI_API_KEY', 'sk-your-api-key-here', 'User')
-
-# Verify setup
-echo $env:OPENAI_API_KEY
+[System.Environment]::SetEnvironmentVariable('GEMINI_API_KEY', 'AIzaSy...', 'User')
 ```
 
 **Windows Command Prompt:**
 
 ```cmd
 # Temporary (current session only)
-set OPENAI_API_KEY=sk-your-api-key-here
-
-# Verify setup
-echo %OPENAI_API_KEY%
+set GEMINI_API_KEY=AIzaSy...
 ```
 
 **macOS/Linux (Bash/Zsh):**
 
 ```bash
 # Temporary (current session only)
-export OPENAI_API_KEY="sk-your-api-key-here"
+export GEMINI_API_KEY="AIzaSy..."
 
 # Permanent (add to ~/.bashrc or ~/.zshrc)
-echo 'export OPENAI_API_KEY="sk-your-api-key-here"' >> ~/.bashrc
+echo 'export GEMINI_API_KEY="AIzaSy..."' >> ~/.bashrc
 source ~/.bashrc
-
-# Verify setup
-echo $OPENAI_API_KEY
 ```
 
 #### **Alternative: Command Line Parameter**
 
 ```bash
 # Pass API key directly (not recommended for security)
-mirror-web-cli https://example.com --ai --openai-key "sk-your-key-here"
+# If using Gemini (default)
+mirror-web-cli https://example.com --ai --openai-key "AIzaSy..." 
+# If using OpenAI
+mirror-web-cli https://example.com --ai --ai-model gpt-4o --openai-key "sk-..."
 ```
 
 #### **Step 3: Verify Setup**
@@ -172,9 +167,9 @@ mirror-web-cli https://example.com --ai --debug
 
 **Requirements:**
 
-- ✅ **OpenAI API keys only** (must start with `sk-`)
-- ✅ **GPT-4o model** for intelligent analysis  
-- ✅ **Active OpenAI account** with billing setup
+- ✅ **Gemini AI API Key** (`AIzaSy...`) or **OpenAI API Key** (`sk-...`)
+- ✅ **gemini-3-flash-preview** (Default model) or **GPT-4o**
+- ✅ **Active account** with API access
 - ✅ **Terminal environment setup** (no pre-configured keys)
 
 ### Basic Usage (After Installation)
@@ -203,16 +198,17 @@ mirror-web-cli https://complex-site.com --debug
 
 ```bash
 # FIRST: Set up API key (see above section)
-export OPENAI_API_KEY="sk-your-api-key-here"  # Linux/macOS
+export GEMINI_API_KEY="AIzaSy..."  # Linux/macOS
 # or
-$env:OPENAI_API_KEY="sk-your-api-key-here"    # Windows PowerShell
+$env:GEMINI_API_KEY="AIzaSy..."    # Windows PowerShell
 
-# THEN: Use AI features
+# THEN: Use AI features (defaults to gemini-3-flash-preview)
 mirror-web-cli https://example.com --ai
 # → Creates: ./example.com-ai-enhanced/
 
-# AI + Clean mirroring
-mirror-web-cli https://complex-app.com --ai --clean
+# AI + Clean mirroring with OpenAI explicitly:
+export OPENAI_API_KEY="sk-..."
+mirror-web-cli https://complex-app.com --ai --ai-model gpt-4o --clean
 # → Creates: ./complex-app.com-ai-enhanced/
 ```
 
@@ -223,8 +219,8 @@ mirror-web-cli https://complex-app.com --ai --clean
 mirror-web-cli https://example.com --ai
 
 # You'll see:
-# ⚠️ AI features requested but no OPENAI_API_KEY found
-# Add OPENAI_API_KEY to your environment...
+# ⚠️ AI analysis API key not found. AI analysis will be disabled.
+# Set GEMINI_API_KEY environment variable to enable AI features...
 # Continuing without AI features...
 ```
 
@@ -376,7 +372,10 @@ Arguments:
 Options:
   -o, --output <dir>      Custom output directory (default: domain name)
   --clean                 Remove tracking scripts and analytics
+  --block-ads             Block advertisements using adblocker plugin
+  --block-cookies         Automatically remove and block cookie consent banners
   --ai                    Enable AI-powered analysis (requires OpenAI API key)
+  --ai-model <model>      AI model to use (default: gemini-3-flash-preview)
   --openai-key <key>      OpenAI API key for AI features (or set OPENAI_API_KEY env var)
   --debug                 Enable detailed debug logging
   --timeout <ms>          Page load timeout in milliseconds (default: 120000)
@@ -385,14 +384,13 @@ Options:
   -V, --version           Show version number
 ```
 
-### OpenAI API Key Priority
+### API Key Priority
 
-The tool checks for OpenAI API keys in this order:
+The tool checks for API keys in this order:
 
-1. `--openai-key` command line parameter
-2. `OPENAI_API_KEY` environment variable
+1. `--openai-key` command line parameter (handles both Gemini and OpenAI keys)
+2. `GEMINI_API_KEY` (if default Gemini model is used) or `OPENAI_API_KEY` environment variable
 3. If neither is found, AI features are disabled with a helpful message
-4. Keys must start with `sk-` (validated automatically)
 
 ## 🏗️ Framework Support
 
@@ -443,24 +441,24 @@ mirror-web-cli https://shop.example.com --debug --clean
 # → Creates: ./shop.example.com-standard/ with detailed logging, removes analytics
 ```
 
-### AI-Powered Analysis (OpenAI)
+### AI-Powered Analysis (Gemini/OpenAI)
 
 **Windows PowerShell:**
 
 ```powershell
 # Set environment variable first
-$env:OPENAI_API_KEY="sk-proj-your-openai-key-here"
+$env:GEMINI_API_KEY="AIzaSy..."
 mirror-web-cli https://complex-app.com --ai --clean
-# → Creates: ./complex-app.com-ai-enhanced/ with OpenAI GPT-4o framework analysis
+# → Creates: ./complex-app.com-ai-enhanced/ with AI framework analysis
 ```
 
 **macOS/Linux:**
 
 ```bash
 # Set environment variable first
-export OPENAI_API_KEY="sk-proj-your-openai-key-here"
+export GEMINI_API_KEY="AIzaSy..."
 mirror-web-cli https://complex-app.com --ai --clean
-# → Creates: ./complex-app.com-ai-enhanced/ with OpenAI GPT-4o framework analysis
+# → Creates: ./complex-app.com-ai-enhanced/ with AI framework analysis
 ```
 
 **Cross-platform (using CLI parameter):**

package.json

@@ -58,6 +58,8 @@
     "node-fetch": "^3.3.2",
     "openai": "^4.67.3",
     "puppeteer": "^24.16.2",
+    "puppeteer-extra": "^3.3.6",
+    "puppeteer-extra-plugin-adblocker": "^2.13.6",
     "sharp": "^0.33.5"
   },
   "repository": {

src/ai/ai-analyzer.js

@@ -9,35 +9,52 @@ dotenv.config();
  * AI-Powered Website Analysis using OpenAI GPT-4o
  */
 export class AIAnalyzer {
-  constructor() {
+  constructor(aiModel = 'gpt-4o') {
     this.openai = null;
     this.isEnabled = false;
+    this.aiModel = aiModel;
     this.initializeOpenAI();
   }
 
   initializeOpenAI() {
-    const apiKey = process.env.OPENAI_API_KEY;
+    const isGemini = this.aiModel.toLowerCase().includes('gemini');
+    let apiKey = isGemini ? process.env.GEMINI_API_KEY : process.env.OPENAI_API_KEY;
 
+    // Fallback if specific key is missing
     if (!apiKey) {
+      apiKey = process.env.OPENAI_API_KEY || process.env.GEMINI_API_KEY;
+    }
+
+    if (!apiKey) {
+      const keyName = isGemini ? 'GEMINI_API_KEY' : 'OPENAI_API_KEY';
       console.log(
         chalk.yellow(
-          '⚠️ OpenAI API key not found. AI analysis will be disabled.',
+          `⚠️ AI analysis API key not found. AI analysis will be disabled.`,
         ),
       );
       console.log(
         chalk.gray(
-          '   Set OPENAI_API_KEY environment variable to enable AI features.',
+          `   Set ${keyName} environment variable to enable AI features.`,
         ),
       );
       return;
     }
 
     try {
-      this.openai = new OpenAI({ apiKey });
+      const config = { apiKey };
+      
+      // Use Gemini OpenAI-compatible endpoint if model is gemini or GEMINI_API_KEY is used
+      if (isGemini || process.env.GEMINI_API_KEY === apiKey) {
+        config.baseURL = 'https://generativelanguage.googleapis.com/v1beta/openai/';
+        console.log(chalk.green(`✅ AI analysis enabled with Gemini model: ${this.aiModel}`));
+      } else {
+        console.log(chalk.green(`✅ AI analysis enabled with OpenAI model: ${this.aiModel}`));
+      }
+
+      this.openai = new OpenAI(config);
       this.isEnabled = true;
-      console.log(chalk.green('✅ AI analysis enabled with OpenAI GPT-4o'));
     } catch (error) {
-      console.log(chalk.red('❌ Failed to initialize OpenAI:'), error.message);
+      console.log(chalk.red('❌ Failed to initialize AI client:'), error.message);
     }
   }
 
@@ -72,7 +89,7 @@ export class AIAnalyzer {
     );
 
     const response = await this.openai.chat.completions.create({
-      model: 'gpt-4o',
+      model: this.aiModel,
       messages: [
         {
           role: 'system',
@@ -225,7 +242,7 @@ Provide actionable optimization steps in JSON format:
 }`;
 
       const response = await this.openai.chat.completions.create({
-        model: 'gpt-4o',
+        model: this.aiModel,
         messages: [
           {
             role: 'system',