Added logging and within-island stats and plots

Author: Akshat Gupta

examples/circle_packing/config_adaptive_exploration_full.yaml

@@ -44,6 +44,7 @@ database:
   elite_selection_ratio: 0.3
   exploration_ratio: 0.2
   exploitation_ratio: 0.7
+  feature_bins: 10 
 
   # Adaptive exploration settings
   use_adaptive_search: true

openevolve/database.py

@@ -297,6 +297,35 @@ def add(
                 # Program exists, compare fitness
                 should_replace = self._is_better(program, self.programs[existing_program_id])
 
+                # Log when program is not replaced (same format as replacement logging)
+                if not should_replace:
+                    coords_dict = {
+                        self.config.feature_dimensions[i]: feature_coords[i]
+                        for i in range(len(feature_coords))
+                    }
+                    existing_program = self.programs[existing_program_id]
+                    new_fitness = get_fitness_score(program.metrics, self.config.feature_dimensions)
+                    existing_fitness = get_fitness_score(
+                        existing_program.metrics, self.config.feature_dimensions
+                    )
+                    logger.info(
+                        "Island %d MAP-Elites cell not replaced: %s (fitness: %.3f <= %.3f)",
+                        island_idx,
+                        coords_dict,
+                        new_fitness,
+                        existing_fitness,
+                    )
+                    # Remove program from database since it doesn't replace anything
+                    # This prevents accumulation of programs that don't contribute to MAP-Elites
+                    # if program.id in self.programs:
+                    #     del self.programs[program.id]
+                    # # Also remove from island set and archive if present
+                    # self.islands[island_idx].discard(program.id)
+                    # self.archive.discard(program.id)
+                    # logger.debug(f"Removed program {program.id} from database (did not replace MAP-Elites cell)")
+
+                    #return program.id  # Early return since program wasn't added
+
         if should_replace:
             # Log significant MAP-Elites events
             coords_dict = {

plot_all.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+# Script to run all visualization scripts for a given example folder
+
+# Set the circle packing folder here
+FOLDER="examples/circle_packing10"
+
+# Or use command line argument if provided
+if [ $# -ge 1 ]; then
+    FOLDER="$1"
+fi
+
+echo "Running visualizations for: $FOLDER"
+echo ""
+
+# Run cell usage visualization
+echo "1. Generating cell usage plot..."
+python3 visualize_cell_usage.py "$FOLDER"
+echo ""
+
+# Run checkpoint analysis (different script for circle_packing vs others)
+echo "2. Generating checkpoint analysis plot..."
+if [[ "$FOLDER" == *"circle_packing"* ]]; then
+    python3 visualize_checkpoints.py "$FOLDER"
+else
+    python3 visualize_signal_processing.py "$FOLDER"
+fi
+echo ""
+
+# Run checkpoint retention analysis
+echo "3. Generating checkpoint retention plot..."
+python3 visualize_checkpoint_retention.py "$FOLDER"
+echo ""
+
+echo "All visualizations complete!"
+echo "Output saved to: $FOLDER/openevolve_output/logs/"
+

visualize_cell_usage.py

@@ -0,0 +1,241 @@
+#!/usr/bin/env python3
+"""Visualize MAP-Elites cell usage from log file."""
+
+import sys
+import re
+import yaml
+from pathlib import Path
+from collections import defaultdict
+import numpy as np
+import matplotlib.pyplot as plt
+import seaborn as sns
+
+# Parse command line arguments
+# Usage: visualize_cell_usage.py <folder> [config_file]
+folder = Path(sys.argv[1] if len(sys.argv) > 1 else input("Folder: ").strip())
+config_file = sys.argv[2] if len(sys.argv) > 2 else None
+
+# Find config and log
+if config_file:
+    config_path = folder / config_file
+    if not config_path.exists():
+        print(f"Error: Specified config file not found: {config_path}")
+        exit(1)
+else:
+    # Try default config first, then fall back to any config*.yaml
+    default_config = folder / "config_adaptive_exploration_full.yaml"
+    if default_config.exists():
+        config_path = default_config
+    else:
+        config_path = next(folder.glob("config*.yaml"), None)
+
+log_path = max((folder / "openevolve_output" / "logs").glob("openevolve_*.log"), key=lambda p: p.stat().st_mtime, default=None)
+
+if not config_path or not log_path:
+    print(f"Error: Config or log not found in {folder}")
+    if not config_path:
+        print(f"  Config path: {config_path}")
+    if not log_path:
+        print(f"  Log path: {log_path}")
+    exit(1)
+
+# Load bins and num_islands from config
+with open(config_path) as f:
+    config_data = yaml.safe_load(f)
+    bins = config_data['database']['feature_bins']
+    num_islands = config_data['database'].get('num_islands', 1)
+
+print(f"Using config: {config_path}")
+print(f"Number of islands: {num_islands}")
+print(f"Feature bins: {bins}")
+
+# Parse log - track data per island
+# Data structures: island_id -> defaultdict/dict
+island_usage = {i: defaultdict(int) for i in range(num_islands)}  # All events per island
+island_improved = {i: defaultdict(int) for i in range(num_islands)}  # Only "improved" events per island
+island_not_replaced = {i: defaultdict(int) for i in range(num_islands)}  # Only "not replaced" events per island
+
+# Track cumulative events over iterations per island
+island_cumulative_new = {i: [] for i in range(num_islands)}  # (iteration, count)
+island_cumulative_improved = {i: [] for i in range(num_islands)}  # (iteration, count)
+island_cumulative_not_replaced = {i: [] for i in range(num_islands)}  # (iteration, count)
+
+# Current iteration and counts per island
+current_iteration = 0
+island_new_count = {i: 0 for i in range(num_islands)}
+island_improved_count = {i: 0 for i in range(num_islands)}
+island_not_replaced_count = {i: 0 for i in range(num_islands)}
+
+pattern = re.compile(r"\{'complexity': (\d+), 'diversity': (\d+)\}")
+iteration_pattern = re.compile(r"Iteration (\d+):")
+island_pattern = re.compile(r"island (\d+)")  # Matches "Island 0", "Island 1", etc.
+
+with open(log_path) as f:
+    for line in f:
+        # Check for iteration number - this marks the end of an iteration
+        iter_match = iteration_pattern.search(line)
+        if iter_match:
+            # Save cumulative counts for the iteration that just completed (all islands)
+            iteration_num = int(iter_match.group(1))
+            if iteration_num > 0:
+                for island_id in range(num_islands):
+                    if island_new_count[island_id] > 0 or island_improved_count[island_id] > 0 or island_not_replaced_count[island_id] > 0:
+                        # Check if we already added this iteration for this island
+                        existing = island_cumulative_new[island_id]
+                        if not existing or existing[-1][0] != iteration_num:
+                            island_cumulative_new[island_id].append((iteration_num, island_new_count[island_id]))
+                            island_cumulative_improved[island_id].append((iteration_num, island_improved_count[island_id]))
+                            island_cumulative_not_replaced[island_id].append((iteration_num, island_not_replaced_count[island_id]))
+            current_iteration = iteration_num
+        
+        # Check for MAP-Elites events (these happen during the iteration)
+        if m := pattern.search(line):
+            complexity = int(m.group(1))
+            diversity = int(m.group(2))
+            coord = (complexity, diversity)
+            
+            # Extract island number from the line
+            island_match = island_pattern.search(line)
+            if island_match:
+                island_id = int(island_match.group(1))
+                print(f"Island {island_id} found in line: {line}")
+                if island_id >= num_islands:
+                    # Skip if island ID is out of range
+                    continue
+            else:
+                # If no island found, default to island 0 (backward compatibility)
+                island_id = 0
+            
+            # Count all events for this island
+            island_usage[island_id][coord] += 1
+            
+            # Check for specific event types and update cumulative counts
+            if "New MAP-Elites cell occupied" in line:
+                island_new_count[island_id] += 1
+            elif "improved" in line:
+                island_improved[island_id][coord] += 1
+                island_improved_count[island_id] += 1
+            elif "not replaced" in line:
+                island_not_replaced[island_id][coord] += 1
+                island_not_replaced_count[island_id] += 1
+
+# Add final iteration counts if we have events but no final iteration marker
+if current_iteration > 0:
+    for island_id in range(num_islands):
+        if island_new_count[island_id] > 0 or island_improved_count[island_id] > 0 or island_not_replaced_count[island_id] > 0:
+            existing = island_cumulative_new[island_id]
+            if not existing or existing[-1][0] != current_iteration:
+                island_cumulative_new[island_id].append((current_iteration, island_new_count[island_id]))
+                island_cumulative_improved[island_id].append((current_iteration, island_improved_count[island_id]))
+                island_cumulative_not_replaced[island_id].append((current_iteration, island_not_replaced_count[island_id]))
+
+def create_grid(data_dict, bins):
+    """Create a 2D grid from coordinate dictionary"""
+    grid = np.zeros((bins, bins), dtype=int)
+    for (c, d), count in data_dict.items():
+        if 0 <= c < bins and 0 <= d < bins:
+            grid[d, c] = count
+    return grid
+
+# Font sizes
+label_fontsize = 14
+title_fontsize = 16
+tick_fontsize = 12
+annot_fontsize = 10
+cbar_label_fontsize = 12
+legend_fontsize = 12
+
+# Create plots for each island
+for island_id in range(num_islands):
+    # Create grids for each event type for this island
+    grid_all = create_grid(island_usage[island_id], bins)
+    grid_improved = create_grid(island_improved[island_id], bins)
+    grid_not_replaced = create_grid(island_not_replaced[island_id], bins)
+    
+    # Create four plots (2x2 layout) for this island
+    fig, axes = plt.subplots(2, 2, figsize=(20, 16))
+    axes = axes.flatten()  # Flatten to 1D array for easier indexing
+    
+    # Plot 1: All events
+    sns.heatmap(grid_all, annot=True, fmt='d', cmap='YlOrRd', cbar_kws={'label': 'Count'}, ax=axes[0], annot_kws={'size': annot_fontsize})
+    axes[0].set_xlabel('Complexity Bin', fontsize=label_fontsize)
+    axes[0].set_ylabel('Diversity Bin', fontsize=label_fontsize)
+    axes[0].set_title(f'Island {island_id}: All MAP-Elites Events\nTotal: {int(grid_all.sum())}', fontsize=title_fontsize)
+    axes[0].tick_params(axis='both', labelsize=tick_fontsize)
+    axes[0].invert_yaxis()
+    
+    # Plot 2: Improved events
+    sns.heatmap(grid_improved, annot=True, fmt='d', cmap='Greens', cbar_kws={'label': 'Count'}, ax=axes[1], annot_kws={'size': annot_fontsize})
+    axes[1].set_xlabel('Complexity Bin', fontsize=label_fontsize)
+    axes[1].set_ylabel('Diversity Bin', fontsize=label_fontsize)
+    axes[1].set_title(f'Island {island_id}: Cell Improved Events\nTotal: {int(grid_improved.sum())}', fontsize=title_fontsize)
+    axes[1].tick_params(axis='both', labelsize=tick_fontsize)
+    axes[1].invert_yaxis()
+    
+    # Plot 3: Not replaced events
+    sns.heatmap(grid_not_replaced, annot=True, fmt='d', cmap='Reds', cbar_kws={'label': 'Count'}, ax=axes[2], annot_kws={'size': annot_fontsize})
+    axes[2].set_xlabel('Complexity Bin', fontsize=label_fontsize)
+    axes[2].set_ylabel('Diversity Bin', fontsize=label_fontsize)
+    axes[2].set_title(f'Island {island_id}: Cell Not Replaced Events\nTotal: {int(grid_not_replaced.sum())}', fontsize=title_fontsize)
+    axes[2].tick_params(axis='both', labelsize=tick_fontsize)
+    axes[2].invert_yaxis()
+    
+    # Plot 4: Cumulative events over iterations
+    cumulative_new = island_cumulative_new[island_id]
+    cumulative_improved = island_cumulative_improved[island_id]
+    cumulative_not_replaced = island_cumulative_not_replaced[island_id]
+    
+    if cumulative_new or cumulative_improved or cumulative_not_replaced:
+        # Get all unique iterations
+        all_iterations = set()
+        if cumulative_new:
+            all_iterations.update(x[0] for x in cumulative_new)
+        if cumulative_improved:
+            all_iterations.update(x[0] for x in cumulative_improved)
+        if cumulative_not_replaced:
+            all_iterations.update(x[0] for x in cumulative_not_replaced)
+        
+        iterations = sorted(all_iterations)
+        
+        # Create dictionaries for quick lookup
+        new_dict = dict(cumulative_new)
+        improved_dict = dict(cumulative_improved)
+        not_replaced_dict = dict(cumulative_not_replaced)
+        
+        # Build aligned arrays
+        new_counts = [new_dict.get(iter, 0) for iter in iterations]
+        improved_counts = [improved_dict.get(iter, 0) for iter in iterations]
+        not_replaced_counts = [not_replaced_dict.get(iter, 0) for iter in iterations]
+        
+        axes[3].plot(iterations, new_counts, 'o-', linewidth=2, markersize=4, label='New Cell Occupied', color='orange')
+        axes[3].plot(iterations, improved_counts, 's-', linewidth=2, markersize=4, label='Cell Improved', color='green')
+        axes[3].plot(iterations, not_replaced_counts, '^-', linewidth=2, markersize=4, label='Cell Not Replaced', color='red')
+        axes[3].set_xlabel('Iteration', fontsize=label_fontsize)
+        axes[3].set_ylabel('Cumulative Count', fontsize=label_fontsize)
+        axes[3].set_title(f'Island {island_id}: Cumulative MAP-Elites Events Over Iterations', fontsize=title_fontsize)
+        axes[3].tick_params(axis='both', labelsize=tick_fontsize)
+        axes[3].legend(fontsize=legend_fontsize)
+        axes[3].grid(True, alpha=0.3)
+    else:
+        axes[3].text(0.5, 0.5, 'No iteration data found', ha='center', va='center', fontsize=label_fontsize)
+        axes[3].set_title(f'Island {island_id}: Cumulative MAP-Elites Events Over Iterations', fontsize=title_fontsize)
+    
+    # Set colorbar label and tick font sizes for all colorbars (only for heatmaps)
+    for ax in fig.axes:
+        if hasattr(ax, 'yaxis') and ax.yaxis.label.get_text() == 'Count':
+            ax.yaxis.label.set_fontsize(cbar_label_fontsize)
+            ax.tick_params(labelsize=tick_fontsize)
+    
+    plt.tight_layout()
+    
+    # Save with island-specific filename
+    if num_islands > 1:
+        output_path = folder / "openevolve_output" / "logs" / f"cell_usage_island_{island_id}.png"
+    else:
+        # For single island, use the original filename for backward compatibility
+        output_path = folder / "openevolve_output" / "logs" / "cell_usage.png"
+    
+    plt.savefig(output_path, dpi=150)
+    print(f"Saved Island {island_id} plot to {output_path}")
+    plt.close()  # Close figure to free memory
+