test_tracking_real_data.py

#!/usr/bin/env python3
"""
Quick test of TCC tracking system using all files in realtime folder
"""

import sys
import os
import pandas as pd
import numpy as np
from pathlib import Path
from datetime import datetime, timedelta

# Add the project root to the path
sys.path.insert(0, str(Path(__file__).parent))

# Import necessary modules
from scripts.main import process_insat_tcc_detection, load_config
from src.tracking.tcc_tracker import TCCTracker

def test_tracking_with_real_data():
    """Test tracking system with all real satellite data files"""
    
    print("TCC Tracking Test with Real Satellite Data (AI/ML Enabled)")
    print("=" * 60)
    
    # Load configuration
    config = load_config()
    
    # Initialize tracker
    tracker = TCCTracker(config)
    
    # Find all .h5 files in realtime folder
    data_dir = Path("data/realtime")
    h5_files = list(data_dir.glob("*.h5"))
    
    if not h5_files:
        print("No .h5 files found in realtime folder")
        return False
    
    # Sort files by timestamp extracted from filename
    def extract_timestamp(filename):
        """Extract timestamp from filename format: 3RIMG_05JUL2025_0915_L1B_STD_V01R00.h5"""
        try:
            # Extract date and time parts
            parts = filename.stem.split('_')
            date_part = parts[1]  # e.g., '05JUL2025'
            time_part = parts[2]  # e.g., '0915'
            
            # Parse date
            day = int(date_part[:2])
            month_str = date_part[2:5]
            year = int(date_part[5:])
            
            # Convert month
            month_map = {'JAN': 1, 'FEB': 2, 'MAR': 3, 'APR': 4, 'MAY': 5, 'JUN': 6,
                        'JUL': 7, 'AUG': 8, 'SEP': 9, 'OCT': 10, 'NOV': 11, 'DEC': 12}
            month = month_map[month_str]
            
            # Parse time
            hour = int(time_part[:2])
            minute = int(time_part[2:])
            
            return datetime(year, month, day, hour, minute)
        except:
            return datetime.min
    
    # Sort files chronologically
    h5_files.sort(key=extract_timestamp)
    
    print(f"Found {len(h5_files)} satellite data files:")
    for i, file_path in enumerate(h5_files):
        timestamp = extract_timestamp(file_path)
        print(f"  T{i+1}: {file_path.name} ({timestamp.strftime('%Y-%m-%d %H:%M')})")
    
    print(f"  AI/ML Classification: ENABLED")
    print()
    
    # Process each file
    all_tracked_detections = []
    
    for i, file_path in enumerate(h5_files):
        step_num = i + 1
        timestamp = extract_timestamp(file_path)
        timestamp_str = timestamp.strftime('%Y-%m-%dT%H:%M:%S')
        
        print(f"Step {step_num}: Processing T{step_num} ({timestamp.strftime('%Y-%m-%d %H:%M')}) data with AI/ML...")
        
        try:
            # Process file
            detections = process_insat_tcc_detection(str(file_path), enable_ml=True)
            
            if len(detections) == 0:
                print(f"  No TCCs detected in file {step_num}")
                continue
                
            print(f"  Found {len(detections)} TCCs in T{step_num}")
            
            # Convert to list of dictionaries for tracker
            detection_list = detections.to_dict('records')
            
            # Set timestamp for all detections
            for det in detection_list:
                det['timestamp'] = timestamp_str
            
            # Add detections to tracker
            tracked_detections = tracker.add_detections(detection_list, timestamp_str)
            all_tracked_detections.append((step_num, tracked_detections))
            
            print(f"  Tracked T{step_num}: {len(tracked_detections)} detections with track IDs")
            print("  Sample tracks:")
            for j, det in enumerate(tracked_detections[:3]):  # Show first 3
                print(f"    TCC {j+1}: ({det['center_lat']:.2f}, {det['center_lon']:.2f}) -> Track: {det.get('track_id', 'None')}")
            
        except Exception as e:
            print(f"  Error processing T{step_num}: {e}")
            continue
        
        print()
    
    # Analyze tracking results
    print("Final Analysis: Tracking Results Summary")
    print("=" * 50)
    
    # Get tracking summary
    summary = tracker.get_track_summary()
    print(f"Tracking Summary:")
    print(f"  Active tracks: {summary['active_tracks']}")
    print(f"  Inactive tracks: {summary['inactive_tracks']}")
    print(f"  Total tracks created: {summary['total_tracks']}")
    
    # Analyze tracking continuity across all time points
    print(f"\nTracking Continuity Analysis:")
    
    # Get track IDs from each time point
    track_sets = []
    for step_num, tracked_detections in all_tracked_detections:
        track_ids = set([det.get('track_id', '') for det in tracked_detections])
        track_sets.append((step_num, track_ids))
        print(f"  T{step_num}: {len(track_ids)} unique tracks")
    
    # Analyze continuity between consecutive time points
    if len(track_sets) > 1:
        print(f"\nContinuity between consecutive time points:")
        for i in range(len(track_sets) - 1):
            step1, tracks1 = track_sets[i]
            step2, tracks2 = track_sets[i + 1]
            continued = tracks1.intersection(tracks2)
            new_tracks = tracks2 - tracks1
            
            print(f"  T{step1} → T{step2}: {len(continued)} tracks continued, {len(new_tracks)} new tracks")
            if continued:
                print(f"    Continued: {list(continued)[:5]}")  # Show first 5
    
    # Find tracks that persist across multiple time points
    if len(track_sets) > 2:
        print(f"\nPersistent tracks:")
        all_track_ids = set()
        for _, tracks in track_sets:
            all_track_ids.update(tracks)
        
        persistent_tracks = all_track_ids
        for _, tracks in track_sets:
            persistent_tracks = persistent_tracks.intersection(tracks)
        
        print(f"  Tracks appearing in all {len(track_sets)} time points: {len(persistent_tracks)}")
        if persistent_tracks:
            print(f"    Persistent track IDs: {list(persistent_tracks)}")
    
    # Export tracking data
    print(f"\nExporting tracking data...")
    try:
        tracking_df = tracker.export_tracks()
        output_file = f"tracking_test_results_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv"
        tracking_df.to_csv(output_file, index=False)
        print(f"Tracking results exported to: {output_file}")
        
        # Show some statistics
        if len(tracking_df) > 0:
            unique_tracks = tracking_df['track_id'].nunique()
            avg_detections_per_track = len(tracking_df) / unique_tracks
            print(f"  Total detections: {len(tracking_df)}")
            print(f"  Unique tracks: {unique_tracks}")
            print(f"  Avg detections per track: {avg_detections_per_track:.1f}")
            
            # Time span analysis
            timestamps = pd.to_datetime(tracking_df['timestamp'])
            time_span = (timestamps.max() - timestamps.min()).total_seconds() / 3600
            print(f"  Time span: {time_span:.1f} hours")
            
    except Exception as e:
        print(f"Error exporting data: {e}")
    
    print(f"\n✅ Multi-file tracking test completed successfully!")
    print(f"Search radii used: 3h=450km, 6h=550km, 9h=600km, 12h=650km")
    print(f"Files processed: {len(h5_files)}")
    print(f"AI/ML Classification: ENABLED")
    
    # Print assignment statistics
    assignment_stats = tracker.get_assignment_statistics()
    print(f"\nAssignment Algorithm Performance:")
    print(f"  Hungarian success: {assignment_stats['hungarian_success']}")
    print(f"  Hungarian failures: {assignment_stats['hungarian_failed']}")
    print(f"  Greedy fallbacks: {assignment_stats['greedy_fallback']}")
    print(f"  Threshold assignments: {assignment_stats['threshold_assignments']}")
    if assignment_stats.get('success_rate'):
        print(f"  Success rate: {assignment_stats['success_rate']:.1f}%")
    
    return True

if __name__ == "__main__":
    success = test_tracking_with_real_data()
    sys.exit(0 if success else 1)