indexer.ts

/**
 * Core Indexer - Orchestrates codebase indexing
 * Scans files, delegates to analyzers, creates embeddings, stores in vector DB
 */

import { promises as fs } from 'fs';
import path from 'path';
import { glob } from 'glob';
import ignore from 'ignore';
import {
  CodebaseMetadata,
  CodeChunk,
  IndexingProgress,
  IndexingStats,
  IndexingPhase,
  CodebaseConfig
} from '../types/index.js';
import { analyzerRegistry } from './analyzer-registry.js';
import { isCodeFile, isBinaryFile } from '../utils/language-detection.js';
import { getEmbeddingProvider } from '../embeddings/index.js';
import { getStorageProvider, CodeChunkWithEmbedding } from '../storage/index.js';
import {
  LibraryUsageTracker,
  PatternDetector,
  ImportGraph,
  InternalFileGraph,
  FileExport
} from '../utils/usage-tracker.js';
import { getFileCommitDates } from '../utils/git-dates.js';

export interface IndexerOptions {
  rootPath: string;
  config?: Partial<CodebaseConfig>;
  onProgress?: (progress: IndexingProgress) => void;
}

export class CodebaseIndexer {
  private rootPath: string;
  private config: CodebaseConfig;
  private progress: IndexingProgress;
  private onProgressCallback?: (progress: IndexingProgress) => void;

  constructor(options: IndexerOptions) {
    this.rootPath = path.resolve(options.rootPath);
    this.config = this.mergeConfig(options.config);
    this.onProgressCallback = options.onProgress;

    this.progress = {
      phase: 'initializing',
      percentage: 0,
      filesProcessed: 0,
      totalFiles: 0,
      chunksCreated: 0,
      errors: [],
      startedAt: new Date()
    };
  }

  private mergeConfig(userConfig?: Partial<CodebaseConfig>): CodebaseConfig {
    const defaultConfig: CodebaseConfig = {
      analyzers: {
        angular: { enabled: true, priority: 100 },
        react: { enabled: false, priority: 90 },
        vue: { enabled: false, priority: 90 },
        generic: { enabled: true, priority: 10 }
      },
      include: ['**/*.{ts,tsx,js,jsx,html,css,scss,sass,less}'],
      exclude: ['node_modules/**', 'dist/**', 'build/**', '.git/**', 'coverage/**'],
      respectGitignore: true,
      parsing: {
        maxFileSize: 1048576, // 1MB
        chunkSize: 100,
        chunkOverlap: 10,
        parseTests: true,
        parseNodeModules: false
      },
      styleGuides: {
        autoDetect: true,
        paths: ['STYLE_GUIDE.md', 'docs/style-guide.md', 'ARCHITECTURE.md'],
        parseMarkdown: true
      },
      documentation: {
        autoDetect: true,
        includeReadmes: true,
        includeChangelogs: false
      },
      embedding: {
        provider: 'transformers',
        model: 'Xenova/bge-small-en-v1.5',
        batchSize: 100
      },
      skipEmbedding: false,
      storage: {
        provider: 'lancedb',
        path: './codebase-index'
      }
    };

    return {
      ...defaultConfig,
      ...userConfig,
      analyzers: { ...defaultConfig.analyzers, ...userConfig?.analyzers },
      parsing: { ...defaultConfig.parsing, ...userConfig?.parsing },
      styleGuides: { ...defaultConfig.styleGuides, ...userConfig?.styleGuides },
      documentation: {
        ...defaultConfig.documentation,
        ...userConfig?.documentation
      },
      embedding: { ...defaultConfig.embedding, ...userConfig?.embedding },
      storage: { ...defaultConfig.storage, ...userConfig?.storage }
    };
  }

  async index(): Promise<IndexingStats> {
    const startTime = Date.now();
    const stats: IndexingStats = {
      totalFiles: 0,
      indexedFiles: 0,
      skippedFiles: 0,
      totalChunks: 0,
      totalLines: 0,
      duration: 0,
      avgChunkSize: 0,
      componentsByType: {},
      componentsByLayer: {
        presentation: 0,
        business: 0,
        data: 0,
        state: 0,
        core: 0,
        shared: 0,
        feature: 0,
        infrastructure: 0,
        unknown: 0
      },
      errors: [],
      startedAt: new Date()
    };

    try {
      // Phase 1: Scanning
      this.updateProgress('scanning', 0);
      let files = await this.scanFiles();

      // Memory safety: limit total files to prevent heap exhaustion
      const MAX_FILES = 10000;
      if (files.length > MAX_FILES) {
        console.warn(
          `WARNING: Found ${files.length} files, limiting to ${MAX_FILES} to prevent memory issues.`
        );
        console.warn(
          `Consider using more specific include patterns or excluding large directories.`
        );
        files = files.slice(0, MAX_FILES);
      }

      stats.totalFiles = files.length;
      this.progress.totalFiles = files.length;

      console.error(`Found ${files.length} files to index`);

      // Phase 2: Analyzing & Parsing
      this.updateProgress('analyzing', 0);
      const allChunks: CodeChunk[] = [];
      const libraryTracker = new LibraryUsageTracker();
      const patternDetector = new PatternDetector();
      const importGraph = new ImportGraph();
      const internalFileGraph = new InternalFileGraph(this.rootPath);

      // Fetch git commit dates for pattern momentum analysis
      const fileDates = await getFileCommitDates(this.rootPath);

      for (let i = 0; i < files.length; i++) {
        const file = files[i];
        this.progress.currentFile = file;
        this.progress.filesProcessed = i + 1;
        this.progress.percentage = Math.round(((i + 1) / files.length) * 100);

        try {
          // Normalize line endings to \n for consistent cross-platform output
          const rawContent = await fs.readFile(file, 'utf-8');
          const content = rawContent.replace(/\r\n/g, '\n');
          const result = await analyzerRegistry.analyzeFile(file, content);

          if (result) {
            allChunks.push(...result.chunks);
            stats.indexedFiles++;
            stats.totalLines += content.split('\n').length;

            // Track library usage AND import graph from imports
            for (const imp of result.imports) {
              libraryTracker.track(imp.source, file);
              importGraph.trackImport(imp.source, file, imp.line || 1);

              // Track internal file-to-file imports (relative paths)
              if (imp.source.startsWith('.')) {
                // Resolve the relative import to an absolute path
                const fileDir = path.dirname(file);
                let resolvedPath = path.resolve(fileDir, imp.source);

                // Try common extensions if not already specified
                const ext = path.extname(resolvedPath);
                if (!ext) {
                  for (const tryExt of ['.ts', '.tsx', '.js', '.jsx']) {
                    const withExt = resolvedPath + tryExt;
                    // We don't check if file exists for performance - just track what's referenced
                    resolvedPath = withExt;
                    break;
                  }
                }

                internalFileGraph.trackImport(file, resolvedPath, imp.imports);
              }
            }

            // Track exports for unused export detection
            if (result.exports && result.exports.length > 0) {
              const fileExports: FileExport[] = result.exports.map((exp) => ({
                name: exp.name,
                type: exp.isDefault ? 'default' : (exp.type as FileExport['type']) || 'other'
              }));
              internalFileGraph.trackExports(file, fileExports);
            }

            // Detect generic patterns from code
            patternDetector.detectFromCode(content, file);

            // Helper to extract code snippet around a pattern
            const extractSnippet = (
              pattern: RegExp,
              linesBefore = 1,
              linesAfter = 3
            ): string | undefined => {
              const match = content.match(pattern);
              if (!match) return undefined;
              const lines = content.split('\n');
              const matchIndex = content.substring(0, match.index).split('\n').length - 1;
              const start = Math.max(0, matchIndex - linesBefore);
              const end = Math.min(lines.length, matchIndex + linesAfter + 1);
              return lines.slice(start, end).join('\n').trim();
            };

            const relPath = file.split(/[\\/]/).slice(-3).join('/');

            // Get file date for pattern momentum tracking
            // Try multiple path formats since git uses forward slashes
            const normalizedRelPath = path.relative(this.rootPath, file).replace(/\\/g, '/');
            const fileDate = fileDates.get(normalizedRelPath);

            // GENERIC PATTERN FORWARDING
            // Framework analyzers return detectedPatterns in metadata - we just forward them
            // This keeps the indexer framework-agnostic
            if (result.metadata?.detectedPatterns) {
              for (const pattern of result.metadata.detectedPatterns) {
                // Try to extract a relevant snippet for the pattern
                const snippetPattern = this.getSnippetPatternFor(pattern.category, pattern.name);
                const snippet = snippetPattern ? extractSnippet(snippetPattern) : undefined;
                patternDetector.track(
                  pattern.category,
                  pattern.name,
                  snippet ? { file: relPath, snippet } : undefined,
                  fileDate
                );
              }
            }

            // Track file for Golden File scoring (framework-agnostic based on patterns)
            const detectedPatterns = result.metadata?.detectedPatterns || [];
            const hasPattern = (category: string, name: string) =>
              detectedPatterns.some(
                (p: { category: string; name: string }) =>
                  p.category === category && p.name === name
              );

            const patternScore =
              (hasPattern('dependencyInjection', 'inject() function') ? 1 : 0) +
              (hasPattern('stateManagement', 'Signals') ? 1 : 0) +
              (hasPattern('reactivity', 'Computed') ? 1 : 0) +
              (hasPattern('reactivity', 'Effect') ? 1 : 0) +
              (hasPattern('componentStyle', 'Standalone') ? 1 : 0) +
              (hasPattern('componentInputs', 'Signal-based inputs') ? 1 : 0);
            if (patternScore >= 3) {
              patternDetector.trackGoldenFile(relPath, patternScore, {
                inject: hasPattern('dependencyInjection', 'inject() function'),
                signals: hasPattern('stateManagement', 'Signals'),
                computed: hasPattern('reactivity', 'Computed'),
                effect: hasPattern('reactivity', 'Effect'),
                standalone: hasPattern('componentStyle', 'Standalone'),
                signalInputs: hasPattern('componentInputs', 'Signal-based inputs')
              });
            }

            // Update component statistics
            for (const component of result.components) {
              if (component.componentType) {
                stats.componentsByType[component.componentType] =
                  (stats.componentsByType[component.componentType] || 0) + 1;
              }
              if (component.layer) {
                stats.componentsByLayer[component.layer]++;
              }
            }
          } else {
            stats.skippedFiles++;
          }
        } catch (error) {
          stats.skippedFiles++;
          stats.errors.push({
            filePath: file,
            error: error instanceof Error ? error.message : String(error),
            phase: 'analyzing',
            timestamp: new Date()
          });
        }

        if (this.onProgressCallback) {
          this.onProgressCallback(this.progress);
        }
      }

      stats.totalChunks = allChunks.length;
      stats.avgChunkSize =
        allChunks.length > 0
          ? Math.round(allChunks.reduce((sum, c) => sum + c.content.length, 0) / allChunks.length)
          : 0;

      // Memory safety: limit chunks to prevent embedding memory issues
      const MAX_CHUNKS = 5000;
      let chunksToEmbed = allChunks;
      if (allChunks.length > MAX_CHUNKS) {
        console.warn(
          `WARNING: ${allChunks.length} chunks exceed limit. Indexing first ${MAX_CHUNKS} chunks.`
        );
        chunksToEmbed = allChunks.slice(0, MAX_CHUNKS);
      }

      // Phase 3: Embedding
      const chunksWithEmbeddings: CodeChunkWithEmbedding[] = [];

      if (!this.config.skipEmbedding) {
        this.updateProgress('embedding', 50);
        console.error(`Creating embeddings for ${chunksToEmbed.length} chunks...`);

        // Initialize embedding provider
        const embeddingProvider = await getEmbeddingProvider(this.config.embedding);

        // Generate embeddings for all chunks
        const batchSize = this.config.embedding?.batchSize || 32;

        for (let i = 0; i < chunksToEmbed.length; i += batchSize) {
          const batch = chunksToEmbed.slice(i, i + batchSize);
          const texts = batch.map((chunk) => {
            // Create a searchable text representation
            const parts = [chunk.content];
            if (chunk.metadata?.componentName) {
              parts.unshift(`Component: ${chunk.metadata.componentName}`);
            }
            if (chunk.componentType) {
              parts.unshift(`Type: ${chunk.componentType}`);
            }
            return parts.join('\n');
          });

          const embeddings = await embeddingProvider.embedBatch(texts);

          for (let j = 0; j < batch.length; j++) {
            chunksWithEmbeddings.push({
              ...batch[j],
              embedding: embeddings[j]
            });
          }

          // Update progress
          const embeddingProgress = 50 + Math.round((i / chunksToEmbed.length) * 25);
          this.updateProgress('embedding', embeddingProgress);

          if ((i + batchSize) % 100 === 0 || i + batchSize >= chunksToEmbed.length) {
            console.error(
              `Embedded ${Math.min(i + batchSize, chunksToEmbed.length)}/${
                chunksToEmbed.length
              } chunks`
            );
          }
        }
      } else {
        console.error('Skipping embedding generation (skipEmbedding=true)');
      }

      // Phase 4: Storing
      this.updateProgress('storing', 75);

      if (!this.config.skipEmbedding) {
        console.error(`Storing ${chunksToEmbed.length} chunks...`);

        // Store in LanceDB for vector search
        const storagePath = path.join(this.rootPath, '.codebase-index');
        const storageProvider = await getStorageProvider({ path: storagePath });
        await storageProvider.clear(); // Clear existing index
        await storageProvider.store(chunksWithEmbeddings);
      }

      // Also save JSON for keyword search (Fuse.js) - use chunksToEmbed for consistency
      const indexPath = path.join(this.rootPath, '.codebase-index.json');
      // Write without pretty-printing to save memory
      await fs.writeFile(indexPath, JSON.stringify(chunksToEmbed));

      // Save library usage and pattern stats
      const intelligencePath = path.join(this.rootPath, '.codebase-intelligence.json');
      const libraryStats = libraryTracker.getStats();

      // Extract tsconfig paths for AI to understand import aliases
      let tsconfigPaths: Record<string, string[]> | undefined;
      try {
        const tsconfigPath = path.join(this.rootPath, 'tsconfig.json');
        const tsconfigContent = await fs.readFile(tsconfigPath, 'utf-8');
        const tsconfig = JSON.parse(tsconfigContent);
        if (tsconfig.compilerOptions?.paths) {
          tsconfigPaths = tsconfig.compilerOptions.paths;
          console.error(
            `Found ${Object.keys(tsconfigPaths!).length} path aliases in tsconfig.json`
          );
        }
      } catch (_error) {
        // No tsconfig.json or no paths defined
      }

      const intelligence = {
        libraryUsage: libraryStats,
        patterns: patternDetector.getAllPatterns(),
        goldenFiles: patternDetector.getGoldenFiles(5),
        // tsconfig paths help AI understand import aliases (e.g., @mycompany/* -> libs/*)
        // This reveals which @scoped packages are internal vs external
        tsconfigPaths,
        importGraph: {
          usages: importGraph.getAllUsages(),
          topUsed: importGraph.getTopUsed(30)
        },
        // Internal file graph for circular dependency and unused export detection
        internalFileGraph: internalFileGraph.toJSON(),
        generatedAt: new Date().toISOString()
      };
      await fs.writeFile(intelligencePath, JSON.stringify(intelligence, null, 2));

      // Phase 5: Complete
      this.updateProgress('complete', 100);

      stats.duration = Date.now() - startTime;
      stats.completedAt = new Date();

      console.error(`Indexing complete in ${stats.duration}ms`);
      console.error(`Indexed ${stats.indexedFiles} files, ${stats.totalChunks} chunks`);

      return stats;
    } catch (error) {
      this.progress.phase = 'error';
      stats.errors.push({
        filePath: this.rootPath,
        error: error instanceof Error ? error.message : String(error),
        phase: this.progress.phase,
        timestamp: new Date()
      });
      throw error;
    }
  }

  private async scanFiles(): Promise<string[]> {
    const files: string[] = [];

    // Read .gitignore if respecting it
    let ig: ReturnType<typeof ignore.default> | null = null;
    if (this.config.respectGitignore) {
      try {
        const gitignorePath = path.join(this.rootPath, '.gitignore');
        const gitignoreContent = await fs.readFile(gitignorePath, 'utf-8');
        ig = ignore.default().add(gitignoreContent);
      } catch (_error) {
        // No .gitignore or couldn't read it
      }
    }

    // Scan with glob
    const includePatterns = this.config.include || ['**/*'];
    const excludePatterns = this.config.exclude || [];

    for (const pattern of includePatterns) {
      const matches = await glob(pattern, {
        cwd: this.rootPath,
        absolute: true,
        ignore: excludePatterns,
        nodir: true
      });

      for (const file of matches) {
        const relativePath = path.relative(this.rootPath, file);

        // Check gitignore
        if (ig && ig.ignores(relativePath)) {
          continue;
        }

        // Check if it's a code file
        if (!isCodeFile(file) || isBinaryFile(file)) {
          continue;
        }

        // Check file size
        try {
          const stats = await fs.stat(file);
          if (stats.size > (this.config.parsing?.maxFileSize || 1048576)) {
            console.warn(`Skipping large file: ${file} (${stats.size} bytes)`);
            continue;
          }
        } catch (_error) {
          continue;
        }

        files.push(file);
      }
    }

    return files;
  }

  private updateProgress(phase: IndexingPhase, percentage: number): void {
    this.progress.phase = phase;
    this.progress.percentage = percentage;

    if (this.onProgressCallback) {
      this.onProgressCallback(this.progress);
    }
  }

  async detectMetadata(): Promise<CodebaseMetadata> {
    // Get all registered analyzers (sorted by priority, highest first)
    const analyzers = analyzerRegistry.getAll();

    // Start with base metadata template
    let metadata: CodebaseMetadata = {
      name: path.basename(this.rootPath),
      rootPath: this.rootPath,
      languages: [],
      dependencies: [],
      architecture: {
        type: 'mixed',
        layers: {
          presentation: 0,
          business: 0,
          data: 0,
          state: 0,
          core: 0,
          shared: 0,
          feature: 0,
          infrastructure: 0,
          unknown: 0
        },
        patterns: []
      },
      styleGuides: [],
      documentation: [],
      projectStructure: {
        type: 'single-app'
      },
      statistics: {
        totalFiles: 0,
        totalLines: 0,
        totalComponents: 0,
        componentsByType: {},
        componentsByLayer: {
          presentation: 0,
          business: 0,
          data: 0,
          state: 0,
          core: 0,
          shared: 0,
          feature: 0,
          infrastructure: 0,
          unknown: 0
        }
      },
      customMetadata: {}
    };

    // Loop through all analyzers (highest priority first) and merge their metadata
    // Higher priority analyzers' values win on conflicts
    for (const analyzer of analyzers) {
      try {
        const analyzerMeta = await analyzer.detectCodebaseMetadata(this.rootPath);
        metadata = this.mergeMetadata(metadata, analyzerMeta);
      } catch (error) {
        // Analyzer failed, continue with next
        console.warn(`Analyzer ${analyzer.name} failed to detect metadata:`, error);
      }
    }

    // Load intelligence data if available
    try {
      const intelligencePath = path.join(this.rootPath, '.codebase-intelligence.json');
      const intelligenceContent = await fs.readFile(intelligencePath, 'utf-8');
      const intelligence = JSON.parse(intelligenceContent);

      metadata.customMetadata = {
        ...metadata.customMetadata,
        libraryUsage: intelligence.libraryUsage,
        patterns: intelligence.patterns,
        intelligenceGeneratedAt: intelligence.generatedAt
      };
    } catch (_error) {
      // Intelligence file doesn't exist yet (indexing not run)
    }

    return metadata;
  }

  /**
   * Merge two CodebaseMetadata objects.
   * The 'incoming' metadata takes precedence for non-empty values.
   */
  private mergeMetadata(base: CodebaseMetadata, incoming: CodebaseMetadata): CodebaseMetadata {
    return {
      name: incoming.name || base.name,
      rootPath: incoming.rootPath || base.rootPath,
      languages: [...new Set([...base.languages, ...incoming.languages])], // Merge and deduplicate
      dependencies: this.mergeDependencies(base.dependencies, incoming.dependencies),
      framework: incoming.framework || base.framework, // Framework from higher priority analyzer wins
      architecture: {
        type: incoming.architecture?.type || base.architecture.type,
        layers: this.mergeLayers(base.architecture.layers, incoming.architecture?.layers),
        patterns: [
          ...new Set([
            ...(base.architecture.patterns || []),
            ...(incoming.architecture?.patterns || [])
          ])
        ] // Merge and deduplicate
      },
      styleGuides: [...new Set([...base.styleGuides, ...incoming.styleGuides])], // Merge and deduplicate
      documentation: [...new Set([...base.documentation, ...incoming.documentation])], // Merge and deduplicate
      projectStructure:
        incoming.projectStructure?.type !== 'single-app'
          ? incoming.projectStructure
          : base.projectStructure,
      statistics: this.mergeStatistics(base.statistics, incoming.statistics),
      customMetadata: { ...base.customMetadata, ...incoming.customMetadata }
    };
  }

  private mergeDependencies(base: any[], incoming: any[]): any[] {
    const seen = new Set(base.map((d) => d.name));
    const result = [...base];
    for (const dep of incoming) {
      if (!seen.has(dep.name)) {
        result.push(dep);
        seen.add(dep.name);
      }
    }
    return result;
  }

  private mergeLayers(base: any, incoming?: any): any {
    if (!incoming) return base;
    return {
      presentation: Math.max(base.presentation || 0, incoming.presentation || 0),
      business: Math.max(base.business || 0, incoming.business || 0),
      data: Math.max(base.data || 0, incoming.data || 0),
      state: Math.max(base.state || 0, incoming.state || 0),
      core: Math.max(base.core || 0, incoming.core || 0),
      shared: Math.max(base.shared || 0, incoming.shared || 0),
      feature: Math.max(base.feature || 0, incoming.feature || 0),
      infrastructure: Math.max(base.infrastructure || 0, incoming.infrastructure || 0),
      unknown: Math.max(base.unknown || 0, incoming.unknown || 0)
    };
  }

  private mergeStatistics(base: any, incoming: any): any {
    return {
      totalFiles: Math.max(base.totalFiles || 0, incoming.totalFiles || 0),
      totalLines: Math.max(base.totalLines || 0, incoming.totalLines || 0),
      totalComponents: Math.max(base.totalComponents || 0, incoming.totalComponents || 0),
      componentsByType: { ...base.componentsByType, ...incoming.componentsByType },
      componentsByLayer: this.mergeLayers(base.componentsByLayer, incoming.componentsByLayer)
    };
  }

  /**
   * Get regex pattern for extracting code snippets based on pattern category and name
   * This maps abstract pattern names to actual code patterns
   */
  private getSnippetPatternFor(category: string, name: string): RegExp | null {
    const patterns: Record<string, Record<string, RegExp>> = {
      dependencyInjection: {
        'inject() function': /\binject\s*[<(]/,
        'Constructor injection': /constructor\s*\(/
      },
      stateManagement: {
        RxJS: /BehaviorSubject|ReplaySubject|Subject|Observable/,
        Signals: /\bsignal\s*[<(]/
      },
      reactivity: {
        Effect: /\beffect\s*\(/,
        Computed: /\bcomputed\s*[<(]/
      },
      componentStyle: {
        Standalone: /standalone\s*:\s*true/,
        'NgModule-based': /@(?:Component|Directive|Pipe)\s*\(/
      },
      componentInputs: {
        'Signal-based inputs': /\binput\s*[<(]/,
        'Decorator-based @Input': /@Input\(\)/
      }
    };
    return patterns[category]?.[name] || null;
  }

  getProgress(): IndexingProgress {
    return { ...this.progress };
  }
}