Merge pull request #12 from anhmtk/cleanup/wave-1e-safe

chore(cleanup): Wave-1e tools recreation and analysis pipeline

Author: anhmtk

docs/cleanup/wave-1e.md

@@ -0,0 +1,87 @@
+# Cleanup Wave-1e Report
+
+This document summarizes the activities and outcomes of Cleanup Wave-1e for the StillMe project. The primary objectives for this wave included near-duplicate refactoring with alias shims, pilot canonical imports, and continued controlled quarantine.
+
+## **PREREQUISITE: Wave-1d PR**
+
+**Manual PR Creation Required:**
+- **URL:** https://github.com/anhmtk/stillme_ai_ipc/pull/new/cleanup/wave-1d-safe
+- **Title:** `chore(cleanup): Wave-1d feature-coverage + near-dupe detector + quarantine (score ≥ 65)`
+- **Base:** `main`
+- **Head:** `cleanup/wave-1d-safe`
+
+**Description:**
+```
+Wave-1d cleanup with enhanced coverage, near-duplicate detection, and quarantine of 297 high-score files
+
+## Changes
+- Feature-group coverage: 36 modules tested (52.8% success rate)
+- Near-duplicate detection: 8,007 clusters from 690 files
+- Import graph & dynamic protection: 82 dynamic imports protected
+- Rescore with near-dupe: 801 files analyzed, 539 high-risk
+- Structural normalization: 2 test files moved to tests/legacy/
+- Quarantine: 297 files moved to _attic/ (total 450 files)
+
+## Artifacts
+- artifacts/coverage.json
+- artifacts/near_dupes.json
+- artifacts/import_inbound.json
+- artifacts/dynamic_registry_paths.json
+- artifacts/redundancy_report.csv
+- artifacts/attic_moves.csv
+- docs/cleanup/wave-1d.md
+- docs/cleanup/near_dupes.md
+
+## Safety
+- No permanent deletion, only moves to _attic/
+- __init__.py files hard-protected (score 0)
+- Rollback capability maintained
+- Tracked files only
+```
+
+---
+
+## **PHASE 1: NEAR-DUPE FILTERING**
+
+- **Objective**: Filter near-duplicate clusters to identify pilot candidates for refactoring.
+- **Approach**: Load `artifacts/near_dupes.json`, apply filtering criteria, and generate pilot set.
+
+## **PHASE 2: ALIAS SHIM & IMPORT REWRITE (PILOT)**
+
+- **Objective**: Create alias shims and rewrite imports to canonical files for pilot clusters.
+- **Approach**: 
+  - Create `stillme_compat/` package with deprecation warnings
+  - Generate import rewrite script
+  - Apply canonical imports for pilot clusters
+
+## **PHASE 3: RESCORE & QUARANTINE**
+
+- **Objective**: Recalculate scores and quarantine files with relaxed criteria for near-dupes/backups.
+- **Approach**: 
+  - Run full analysis pipeline
+  - Apply dual scoring: 70 for general, 60 for near-dupe/backup
+  - Execute controlled quarantine
+
+## **PHASE 4: TESTS & CI GUARDS**
+
+- **Objective**: Ensure system stability and add CI gates for near-duplicate prevention.
+- **Approach**: 
+  - Run tests and smoke checks
+  - Update CI with near-dupe gates
+  - Create baseline for future comparisons
+
+## **PHASE 5: BRANCH & PR**
+
+- **Objective**: Create clean branch and PR with clear diff.
+- **Approach**: 
+  - Create `cleanup/wave-1e-safe` from latest main
+  - Commit in logical groups
+  - Push and create PR
+
+## **Safety Measures**
+
+- `ALWAYS_PROTECT_PATHS` and `ALWAYS_PROTECT_FILENAMES` are respected
+- All import rewrites have alias shims for backward compatibility
+- Deprecation warnings for old import paths
+- Rollback capability maintained
+- Tracked files only

tools/inventory/feature_smoke.py

@@ -1,18 +1,115 @@
 #!/usr/bin/env python3
 """
+cleanup/wave-1e-safe
+Feature Smoke Test for Coverage Generation
+Safely imports modules from specific feature groups to generate coverage data.
+"""
+
+import os
+import importlib.util
+import sys
+import pkgutil
 Feature-Group Coverage Smoke Test
 Tests different feature groups to increase coverage
 """
 
 import os
 import sys
 import importlib
+ main
 import logging
 from pathlib import Path
 
 logging.basicConfig(level=logging.INFO, format='%(levelname)s:%(name)s:%(message)s')
 logger = logging.getLogger(__name__)
 
+ cleanup/wave-1e-safe
+def is_excluded(file_path: str, global_excludes: list) -> bool:
+    """Checks if a file path should be excluded based on global exclude patterns."""
+    path = Path(file_path).as_posix()
+    for exclude in global_excludes:
+        if Path(exclude).is_absolute():
+            if Path(path) == Path(exclude) or Path(path).is_relative_to(Path(exclude)):
+                return True
+        else:
+            if exclude.endswith('/') and f"/{exclude}" in path:
+                return True
+            elif exclude.startswith('*') and path.endswith(exclude[1:]):
+                return True
+            elif exclude in path:
+                return True
+    return False
+
+def smoke_import_modules(root_packages: list[str], global_excludes: list[str]):
+    """
+    Attempts to import all Python modules under the given root packages safely.
+    Logs warnings for failures but does not stop execution.
+    """
+    imported_count = 0
+    failed_count = 0
+    total_modules = 0
+    
+    # Add current directory to sys.path to ensure local modules are discoverable
+    sys.path.insert(0, os.getcwd())
+
+    all_modules = set()
+    for root_package in root_packages:
+        try:
+            # Attempt to find the path for the root package
+            spec = importlib.util.find_spec(root_package)
+            if spec and spec.submodule_search_locations:
+                root_path = spec.submodule_search_locations[0]
+            else:
+                # Fallback for packages not yet installed or in sys.path
+                root_path = os.path.join(os.getcwd(), *root_package.split('.'))
+                if not os.path.exists(root_path):
+                    logger.warning(f"Root package path not found for {root_package}. Skipping.")
+                    continue
+
+            for importer, modname, ispkg in pkgutil.walk_packages([root_path], prefix=f"{root_package}."):
+                module_path = os.path.join(importer.path, modname.replace('.', os.sep))
+                if ispkg:
+                    module_path = os.path.join(module_path, '__init__.py')
+                else:
+                    module_path += '.py'
+
+                if is_excluded(module_path, global_excludes):
+                    logger.debug(f"Excluding module: {modname} ({module_path})")
+                    continue
+                all_modules.add(modname)
+        except Exception as e:
+            logger.warning(f"Could not walk packages for {root_package}: {e}")
+
+    total_modules = len(all_modules)
+    logger.info(f"📦 Found {total_modules} modules to import")
+
+    for modname in sorted(list(all_modules)):
+        try:
+            importlib.import_module(modname)
+            imported_count += 1
+        except (ImportError, SyntaxError, Exception) as e:
+            logger.warning(f"Could not import {modname}: {e}")
+            failed_count += 1
+    
+    logger.info(f"✅ Successfully imported: {imported_count}")
+    logger.warning(f"❌ Failed to import: {failed_count}")
+    logger.info(f"📊 Total modules processed: {total_modules}")
+
+if __name__ == "__main__":
+    from pathlib import Path
+    REPO_ROOT = Path(__file__).parent.parent.parent
+    
+    # Define root packages and global excludes as per user's request
+    ROOT_PACKAGES = ["stillme_core", "stillme_ethical_core"]
+    GLOBAL_EXCLUDES = [
+        ".git/", ".github/", ".venv/", "venv/", "env/", "site-packages/", "dist/", 
+        "build/", "node_modules/", "artifacts/", "reports/", "htmlcov", 
+        "__pycache__/", "*.egg-info/", ".sandbox/"
+    ]
+    
+    logger.info("🔥 Starting smoke import for coverage generation...")
+    smoke_import_modules(ROOT_PACKAGES, GLOBAL_EXCLUDES)
+
 def test_feature_group(group_name: str, modules: list, entry_points: list = None):
     """Test a feature group by importing modules and calling entry points"""
     logger.info(f"🧪 Testing feature group: {group_name}")
@@ -125,3 +222,4 @@ def main():
 
 if __name__ == "__main__":
     main()
+ main