precompute_structure.py

#!/usr/bin/env python3
"""
Pre-compute structure analysis - extract unique paths through the JSON hierarchy
"""
import json
import sys
from collections import defaultdict

def extract_paths(obj, path="", paths=None, path_types=None, path_values=None, max_depth=10, current_depth=0):
    """
    Recursively extract all paths through the JSON structure
    Also tracks the type (node/array/tuple) and values for each path
    """
    if paths is None:
        paths = defaultdict(int)
    if path_types is None:
        path_types = defaultdict(lambda: {'node': 0, 'array': 0, 'tuple': 0, 'primitive': 0})
    if path_values is None:
        path_values = defaultdict(set)
    
    if current_depth >= max_depth:
        return paths, path_types, path_values
    
    if isinstance(obj, dict):
        for key, value in obj.items():
            current_path = f"{path}.{key}" if path else key
            
            # Count this path
            paths[current_path] += 1
            
            # Determine type and recurse
            if isinstance(value, dict):
                path_types[current_path]['node'] += 1
                extract_paths(value, current_path, paths, path_types, path_values, max_depth, current_depth + 1)
            elif isinstance(value, list):
                if len(value) > 0:
                    first_item = value[0]
                    if isinstance(first_item, dict):
                        # Array of objects (tuple-like structure)
                        path_types[current_path]['tuple'] += 1
                        # Track unique tuple structures by counting distinct keys
                        tuple_keys = tuple(sorted(first_item.keys()))
                        path_values[current_path].add(str(tuple_keys))
                        extract_paths(first_item, current_path, paths, path_types, path_values, max_depth, current_depth + 1)
                    elif isinstance(first_item, list):
                        # Nested array
                        path_types[current_path]['array'] += 1
                        extract_paths(first_item, current_path, paths, path_types, path_values, max_depth, current_depth + 1)
                    else:
                        # Array of primitives
                        path_types[current_path]['array'] += 1
                        # Track unique values
                        for item in value:
                            if item is not None:
                                path_values[current_path].add(str(item))
                else:
                    # Empty array
                    path_types[current_path]['array'] += 1
            elif value is not None:
                # Primitive value
                path_types[current_path]['primitive'] += 1
                path_values[current_path].add(str(value))
                
    elif isinstance(obj, list):
        if len(obj) > 0:
            first_item = obj[0]
            if isinstance(first_item, dict):
                # Array of objects
                path_types[path]['tuple'] += 1
                tuple_keys = tuple(sorted(first_item.keys()))
                path_values[path].add(str(tuple_keys))
                extract_paths(first_item, path, paths, path_types, path_values, max_depth, current_depth + 1)
            elif isinstance(first_item, list):
                # Nested array
                path_types[path]['array'] += 1
                extract_paths(first_item, path, paths, path_types, path_values, max_depth, current_depth + 1)
            else:
                # Array of primitives
                path_types[path]['array'] += 1
                for item in obj:
                    if item is not None:
                        path_values[path].add(str(item))
    
    return paths, path_types, path_values

def analyze_structure(data):
    """Analyze the structure of all items"""
    
    all_paths = defaultdict(int)
    all_path_types = defaultdict(lambda: {'node': 0, 'array': 0, 'tuple': 0, 'primitive': 0})
    all_path_values = defaultdict(set)
    path_examples = {}  # Store example values for each path
    path_example_indices = {}  # Store example item index for each path
    
    for i, item in enumerate(data):
        paths, path_types, path_values = extract_paths(item, path="", paths=defaultdict(int), 
                                                       path_types=defaultdict(lambda: {'node': 0, 'array': 0, 'tuple': 0, 'primitive': 0}),
                                                       path_values=defaultdict(set))
        
        for path, count in paths.items():
            all_paths[path] += count
            
            # Aggregate path types
            if path in path_types:
                for ptype, pcount in path_types[path].items():
                    all_path_types[path][ptype] += pcount
            
            # Aggregate path values
            if path in path_values:
                all_path_values[path].update(path_values[path])
            
            # Store example value and item index (first occurrence)
            if path not in path_examples:
                path_example_indices[path] = i  # Store the item index
                # Try to get an example value
                try:
                    parts = path.split('.')
                    current = item
                    for part in parts:
                        if part.endswith('[]'):
                            part = part[:-2]
                            if isinstance(current, dict) and part in current:
                                current = current[part]
                                if isinstance(current, list) and len(current) > 0:
                                    current = current[0]
                                else:
                                    current = None
                                    break
                            else:
                                current = None
                                break
                        else:
                            if isinstance(current, dict) and part in current:
                                current = current[part]
                            else:
                                current = None
                                break
                    
                    if current is not None:
                        # Convert to string representation
                        if isinstance(current, (str, int, float, bool)):
                            path_examples[path] = str(current)[:100]  # Limit length
                        elif isinstance(current, dict):
                            path_examples[path] = f"{{object with {len(current)} keys}}"
                        elif isinstance(current, list):
                            path_examples[path] = f"[array with {len(current)} items]"
                except:
                    pass
    
    # Sort paths by depth (number of dots) and then by count
    def path_depth(path):
        return path.count('.')
    
    sorted_paths = sorted(all_paths.items(), key=lambda x: (path_depth(x[0]), -x[1]))
    
    # Determine primary type for each path (most common type)
    path_type_info = {}
    type_counts = {'node': 0, 'array': 0, 'tuple': 0, 'primitive': 0, 'unknown': 0}
    
    for path in all_paths.keys():
        types = all_path_types.get(path, {})
        # Find the dominant type
        if types and sum(types.values()) > 0:
            dominant_type = max(types.items(), key=lambda x: x[1])[0]
        else:
            dominant_type = 'unknown'
        type_counts[dominant_type] = type_counts.get(dominant_type, 0) + 1
        
        unique_values_count = len(all_path_values.get(path, set()))
        
        path_type_info[path] = {
            'type': dominant_type,
            'type_counts': types,
            'unique_values_count': unique_values_count if dominant_type in ['array', 'tuple', 'primitive'] else None
        }
    
    # Group by depth level
    paths_by_depth = defaultdict(dict)
    for path, count in sorted_paths:
        depth = path_depth(path)
        type_info = path_type_info.get(path, {})
        paths_by_depth[depth][path] = {
            'count': count,
            'example': path_examples.get(path, None),
            'example_index': path_example_indices.get(path, None),
            'type': type_info.get('type', 'unknown'),
            'type_counts': type_info.get('type_counts', {}),
            'unique_values_count': type_info.get('unique_values_count')
        }
    
    result = {
        'total_items': len(data),
        'total_unique_paths': len(all_paths),
        'type_counts': type_counts,
        'paths_by_depth': {str(k): v for k, v in sorted(paths_by_depth.items())},
        'all_paths': dict(sorted(all_paths.items(), key=lambda x: -x[1]))
    }
    
    return result

def main():
    print("Loading SearchExport.json...")
    try:
        with open('SearchExport.json', 'r', encoding='utf-8') as f:
            data = json.load(f)
    except FileNotFoundError:
        print("Error: SearchExport.json not found")
        sys.exit(1)
    except json.JSONDecodeError as e:
        print(f"Error parsing JSON: {e}")
        sys.exit(1)
    
    print(f"Analyzing structure of {len(data)} items...")
    result = analyze_structure(data)
    
    output_file = 'structure_analysis_data.json'
    print(f"Writing results to {output_file}...")
    with open(output_file, 'w', encoding='utf-8') as f:
        json.dump(result, f, indent=2, ensure_ascii=False)
    
    print("Analysis complete!")
    print(f"\nSummary:")
    print(f"  Total items: {result['total_items']}")
    print(f"  Total unique paths: {result['total_unique_paths']}")
    print(f"\nPaths by depth:")
    for depth in sorted([int(k) for k in result['paths_by_depth'].keys()]):
        path_count = len(result['paths_by_depth'][str(depth)])
        print(f"  Depth {depth}: {path_count} unique paths")
    
    print(f"\nPath types:")
    for ptype, count in result['type_counts'].items():
        print(f"  {ptype}: {count} paths")
    
    print(f"\nTop 10 paths by usage:")
    for path, count in list(result['all_paths'].items())[:10]:
        type_info = result['paths_by_depth'].get('0', {}).get(path, {}) or result['paths_by_depth'].get('1', {}).get(path, {})
        path_type = type_info.get('type', 'unknown') if type_info else 'unknown'
        print(f"    {path}: {count:,} occurrences ({path_type})")

if __name__ == '__main__':
    main()