final_project.py

#!/usr/bin/env python
# coding: utf-8

# In[27]:


import json
import nltk
import re
import string
import hfst
import math

from fcfg_to_cfg import fcfg_to_cfg


# In[ ]:


# Helper functions
# Utilities

def expr_to_latex(expr):
    """
    Recursively converts an nltk.Expression to a LaTeX string.
    """
    if isinstance(expr, nltk.sem.logic.ApplicationExpression):  # Function application (e.g., char(n, lett))
        # Extract function (predicate) and arguments
        arguments = []
        while isinstance(expr, nltk.sem.logic.ApplicationExpression):
            arguments.append(expr.argument)
            expr = expr.function  # Move up to the function name
        
        # Now expr is the function name (predicate), and arguments contains all its arguments
        function_name = rf"\mathbf{{{expr}}}"  # Boldface the predicate name
        arguments = ", ".join(expr_to_latex(arg) for arg in reversed(arguments))  # Reverse to maintain correct order
        return f"{function_name}({arguments})"
    elif isinstance(expr, nltk.sem.logic.LambdaExpression):  # Lambda abstraction (e.g., \x.P(x))
        return rf"\lambda {expr.variable} . {expr_to_latex(expr.term)}"
    elif isinstance(expr, nltk.sem.logic.QuantifiedExpression):  # Quantifiers (e.g., exists n. P(n))
        quantifier = r"\forall" if expr.getQuantifier() == "all" else r"\exists"
        return rf"{quantifier} {expr.variable} \, {expr_to_latex(expr.term)}"
    elif isinstance(expr, nltk.sem.logic.NegatedExpression):  # Negation (e.g., ¬P)
        return rf"\neg {expr_to_latex(expr.term)}"
    elif isinstance(expr, nltk.sem.logic.AndExpression):  # Conjunction (P & Q)
        return rf"({expr_to_latex(expr.first)} \wedge {expr_to_latex(expr.second)})"
    elif isinstance(expr, nltk.sem.logic.OrExpression):  # Disjunction (P | Q)
        return rf"({expr_to_latex(expr.first)} \vee {expr_to_latex(expr.second)})"
    elif isinstance(expr, nltk.sem.logic.ImpExpression):  # Implication (P -> Q)
        return rf"({expr_to_latex(expr.first)} \rightarrow {expr_to_latex(expr.second)})"
    elif isinstance(expr, nltk.sem.logic.BinaryExpression):  # Handles '<->' equivalence
        if expr.operator == '<->':
            return rf"({expr_to_latex(expr.first)} \leftrightarrow {expr_to_latex(expr.second)})"
        else:
            return rf"({expr_to_latex(expr.first)} {expr.operator} {expr_to_latex(expr.second)})"
    elif isinstance(expr, nltk.sem.logic.IndividualVariableExpression):  # Variables (e.g., x, y, n)
        return str(expr)
    elif isinstance(expr, nltk.sem.logic.ConstantExpression):  # Constants (e.g., lett, numbers)
        expr_str = str(expr)
        if expr_str.isdigit():  # If it's a number, don't boldface it
            return expr_str
        return rf"\mathbf{{{expr_str}}}"  # Boldface non-numeric constants like 'lett'
    else:
        return str(expr)  # Default case for any unhandled expression type
# Authors chatgpt4o and Mats Rooth Feb 8 2025

def emptysets(val:nltk.sem.evaluate.Valuation):
  val.update([(k,set()) for (k,v) in val.items() if v == 'set()'])

# Model construction
from typing import Callable, List, Set

def to_model_str(word: str, special_rels: List[Callable[[str], str]]=[]) -> str:
    """
    Creates the string form of the model for the input word. This string is meant to be passed to `nltk.Valuation.fromstring`.
    By default, the function will only add the relations mapping i => i for i from 1 to the length of `word` and a relation 
    mapping char => the set of tuples (i, word[i]). The `special_rels` function allows you to specify additional relations to 
    be added to the valuation string.
    
    :param word: The word to create a model string for.
    :param special_rels: A list of functions that when called return a string of the form {relation_name} => {relation_contents}. Defaults to the empty list.
    :returns: a string representing the model for word
    """
    n = len(word)
    model_str = []
    char = []
    for i in range(1, n+1):
        model_str.append(f'{i} => {i}')
        char.append((i, word[i-1]))
    model_str.append(f'char => {set(char)}'.lower())
    return '\n'.join(model_str + [rel(word) for rel in special_rels]).replace("'", "")
# Angela Liu

###This code is from CL1 2023

vowels = ['a', 'e', 'i', 'o', 'u']
fricatives = ['v', 'f', 's', 'z', 'h', 'th', 'sh', 'zh']

def capital(word, i):
    return word[i].isupper()

get_capital = lambda word: f'capital => {set([i+1 for i in range(len(word)) if capital(word,i)])}'

def less_than(i, j):
    return i<j
get_less_than = lambda word: f'le => {set([(i+1,j+1) for i in range(len(word)) for j in range(len(word)) if i!=j and less_than(i+1,j+1)])}'

def adjacent(i,j):
    return abs(i-j) == 1

get_adjacent = lambda word: f'ad => {set([(i+1,j+1) for i in range(len(word)) for j in range(len(word)) if adjacent(i+1,j+1)])}'

get_even =  lambda word: f'even => {set([i+1 for i in range(len(word)) if (i+1)%2==0])}'

get_odd = lambda word: f'odd => {set([i+1 for i in range(len(word)) if (i+1)%2!=0])}'

# @323
def voiced(word):
    word=word.lower()
    v=[]
    for i in range(len(word)):
        if i != len(word)-1:
            if (word[i] == 'n' and word[i+1]=='g') or (word[i] == 's' and word[i+1] == 'z'):
                v.append((i+1, word[i]))
                v.append((i+2, word[i+1]))
        if word[i] == 'z':
            if ((i,'s') not in v):
                v.append((i+1, 'z'))
        if word[i] == 'g':
            if ((i,'n') not in v):
                v.append((i+1, 'g'))
        if word[i] == 'n':
            if ((i+2,'g') not in v):
                v.append((i+1,'n'))
        if word[i] in ['a', 'e', 'i', 'o', 'u', 'b', 'd', 'j', 'l', 'm', 'r', 'v', 'w', 'y']:
            v.append((i+1, word[i]))
    return v
            
get_voiced= lambda w: f'voiced => {set([(i+1,w[i].lower()) for i in range(len(w)) if (i+1, w[i].lower()) in voiced(w)])}'

def centered(word,i):
    if (len(word)%2==0):
        return len(word)//2 == i or i == len(word)//2 + 1
    else:
        return len(word)//2 + 1 == i

get_centered = lambda word: f'cent => {set([i+1 for i in range(len(word)) if centered(word,i+1)])}'

get_mirrored = lambda w: f'mirrored => {set(i+1 for i in range(len(w)) if w[i].lower() == w[len(w)-1-i].lower())}'

get_glide =  lambda w: f'glide => {set(i+1 for i in range(len(w)) if w[i].lower() == "w" or w[i].lower() == "y")}'

# from @325
def fricatives(word):
    word.lower()
    frics = []
    for i in range(len(word)):
        if i != len(word)-1:
            if word[i] in ['t', 's', 'z'] and word[i+1] == 'h':
                frics.append((i+1, word[i]))
                frics.append((i+2, 'h'))
        if word[i] == 'h':
            if ((i, 't') not in frics) and ((i, 's') not in frics) and ((i, 'z') not in frics):
                frics.append((i+1, 'h'))
        if word[i] in ['s', 'z']:
            if ((i+2, 'h') not in frics):
                frics.append((i+1, word[i]))
        if word[i] in ['v', 'f']:
            frics.append((i+1, word[i]))
        return frics

get_fricative = lambda w: f'fricative => {set([(i+1,w[i].lower()) for i in range(len(w)) if (i+1, w[i].lower()) in fricatives(w)])}'


# get all the vowels in a word
get_vowel = lambda w: f'vowel => {set(re.findall(r"[AEIOUaeiou]", w))}'.lower()

# get all the consonants in a word
get_cons = lambda w: 'consonant => {}'.format(set(re.findall(r"[^AEIOUaeiouywYW\W0-9]", w))).lower()

# get all the tuple of two numbers (n,m) such that n < m, n&m < len(word) and n!=m
follows = lambda w: f'le => {set([(i+1,j+1) for i in range(len(w)) for j in range(i, len(w)) if i != j])}'

# get all the letters that are capitalized
get_capital = lambda w: 'capital => {}'.format(set(re.findall(r"[A-Z]",w))).lower()
# for A[SEM=<\n.exists c.(capital(n)& char(n,c))>] -> 'capitalized'

# get_capital = lambda w: f'capital => {set([m.span()[0] + 1 for m in re.finditer(r"[A-Z]", w)])}'

# get all the glides in a word
get_glide = lambda w: f'glide => {set(re.findall(r"[ywYW]", w))}'.lower()

# for exactly one
equals = lambda w: f'eq => {set([(i+1,i+1) for i in range(len(w))])}'

# get all alphabetical letters in a word
get_alphabet = lambda w: f'alphabet => {set(re.findall(r"[A-Za-z]", w))}'.lower()

# get all liquids
get_liquid = lambda w: f'liquid => {set(re.findall(r"[lrLR]", w))}'.lower()

# get all nasals
get_nasal = lambda w: f'nasal => {set(re.findall(r"[nmNM]", w))}'.lower()

# get all plosives
get_plosive = lambda w: f'plosive => {set(re.findall(r"[pbtdkgPBTDKG]", w))}'.lower()



letter_funcs = {
    f"let{ch}": (lambda c: lambda w: f"let{c} => {c}")(ch)
    for ch in string.ascii_lowercase
}

all_func = [
    follows, get_capital, get_vowel, equals, get_alphabet,
    get_adjacent, get_voiced, get_fricative, get_glide, get_centered,
    get_mirrored, get_less_than, get_even, get_odd,
    get_liquid, get_nasal, get_plosive, get_cons
] + list(letter_funcs.values())


# In[3]:


# Convert FCFG to PCFG
feature_grammar_file = "filtered_fcfg.fcfg"
cfg_grammar_file = "cfg_grammer.cfg"
pcfg_grammar_file = "pcfg_grammar.pcfg"

# Convert the FCFG grammar to CFG and PCFG and save them to files
fcfg_to_cfg(feature_grammar_file, cfg_grammar_file, pcfg_grammar_file)


# In[4]:


def get_real_root_tree(tree):
    if not len(tree) == 1:
        return tree
    
    if not isinstance(tree[0], nltk.Tree):
        return tree
    
    if not len(tree.label().split('/')) == len(tree[0].label().split('/')):
        return tree
    
    # Check if symbols of the first child are the same as the root
    if not tree.label().split('-')[0] == tree[0].label().split('-')[0]:
        return tree
    
    return tree[0]

def map_pcfg_to_fcfg_trees(pcfg_trees, fcfg_trees):
    """
    Maps PCFG trees to their corresponding FCFG trees and calculates probability sums.
    
    Args:
        pcfg_trees: List of PCFG trees
        fcfg_trees: List of FCFG trees
        
    Returns:
        List of tuples (fcfg_tree, total_probability) where total_probability is the sum
        of probabilities from all PCFG trees that map to this FCFG tree
    """
    pcfg_trees = [get_real_root_tree(tree) for tree in pcfg_trees]
    
    def tree_to_position_and_label(tree):
        return [(a,tree[a]) if isinstance(tree[a],str) else (a, tree[a].label()) for a in tree.treepositions()]
    
    pcfg_list_trees = [tree_to_position_and_label(tree) for tree in pcfg_trees]
    fcfg_list_trees = [tree_to_position_and_label(tree) for tree in fcfg_trees]
    
    def get_fcfg_features(label) -> list:
        features = set()
        recurse = False
        key = None
        for key, value in label.items():
            if isinstance(key, str):
                if key.lower() == 'sem':
                    continue
                if not isinstance(value, str):
                    continue
                
                if value.startswith('<') and value.endswith('>'):
                    features.add((key, value.strip()[1:-1]))
                else:
                    features.add((key, value.strip()))
            else:
                if isinstance(key, nltk.featstruct.SlashFeature):
                    recurse = True
                    key = key
                    
        if recurse:
           return [features] + get_fcfg_features(label[key])       
        
        return [features]
    
    def get_fcfg_symbol(label) -> str:
        symbol = ""
        recurse = False
        key = None
        for key, value in label.items():
            if isinstance(key, nltk.featstruct.Feature):
                if not isinstance(value, str):
                    continue
                symbol += value
            elif isinstance(key, nltk.featstruct.SlashFeature):
                    recurse = True
                    key = key
        if recurse:
            return symbol + '/' + get_fcfg_symbol(label[key])
        
        return symbol
    
    def get_pcfg_symbol(label) -> str:
        pcfg_slash_index = label.strip().split('/')
        
        pcfg_symbol = ""
        for pcfg_part in pcfg_slash_index:
            if '-' in pcfg_part:
                pcfg_part = pcfg_part.split('-')[0]
            pcfg_symbol += pcfg_part + '/'
        pcfg_symbol = pcfg_symbol[:-1]
        return pcfg_symbol
    
    
    def match_fcfg_and_pcfg_symbol(fcfg_label, pcfg_label):
        if isinstance(fcfg_label, str) and isinstance(pcfg_label, str):
            return fcfg_label == pcfg_label
        
        fcfg_symbol = get_fcfg_symbol(fcfg_label)
        fcfg_slash_index = fcfg_symbol.strip().split('/')
        pcfg_slash_index = pcfg_label.strip().split('/')
        
        if len(fcfg_slash_index) != len(pcfg_slash_index):
            return False
        
        pcfg_symbol = get_pcfg_symbol(pcfg_label)

        if fcfg_symbol != pcfg_symbol:
            return False
        
        return True
    
    def match_fcfg_and_pcfg_value(fcfg_label, pcfg_label):
        prod_feature_map = {
            'S': ['NUM'],
            'DP': ['NUM', 'PRED', 'STR'],
            'VP': ['NUM'],
            'NP': ['NUM', 'PRED', 'STR'],
            'CP': ['NUM'],
            'Adv': ['STR'],
            'A': [],
            'P': [],
            'ADV': ['STR'],
            'TV': ['NUM'],
            'Det': ['NUM', 'PRED', 'STR'],
            'N': ['NUM'],
            'NEG': [],
            'THEREP': ['NUM']
        }

        if isinstance(fcfg_label, str) and isinstance(pcfg_label, str):
            return fcfg_label == pcfg_label
        
        pcfg_slash_index = pcfg_label.strip().split('/')
        pcfg_values = [pcfg_label.split('-')[1:] if '-' in pcfg_label else [] for pcfg_label in pcfg_slash_index]
        pcfg_symbol = get_pcfg_symbol(pcfg_label)
        split_symbol = pcfg_symbol.split('/')
        
        fcfg_features = get_fcfg_features(fcfg_label)
        
        for symbol, fcfg_feature_set, pcfg_value_set in zip(split_symbol, fcfg_features, pcfg_values):
            for key, value in fcfg_feature_set:
                features = prod_feature_map.get(symbol, [])
                key_upper = key.upper()
                index = features.index(key_upper) if key_upper in features else -1
                if index == -1:
                    print(f"Warning: Feature '{key}' not found in production map for symbol '{symbol}'.")
                    continue
                if value != pcfg_value_set[index]:
                    return False
            
        return True
        
    
    def tree_match(pcfg_tree, fcfg_tree):
        if len(pcfg_tree) != len(fcfg_tree):
            return False
        
        for pcfg_node, fcfg_node in zip(pcfg_tree, fcfg_tree):
            if not match_fcfg_and_pcfg_symbol(fcfg_node[1], pcfg_node[1]):
                return False
            
            if not match_fcfg_and_pcfg_value(fcfg_node[1], pcfg_node[1]):
                return False
        
        return True
    
    # Initialize results dictionary: {fcfg_tree_index: total_probability}
    fcfg_probability_sums = {i: 0.0 for i in range(len(fcfg_trees))}
    
    # Map each PCFG tree to corresponding FCFG tree and add its probability
    for pcfg_tree, pcfg_list_tree in zip(pcfg_trees, pcfg_list_trees):
        if not isinstance(pcfg_list_tree[0][1], str):
            raise ValueError("The first element of the PCFG tree must be a string.")
        
        probability = pcfg_tree.prob()
        matched = False
        
        for i, fcfg_tree in enumerate(fcfg_list_trees):
            # if not isinstance(fcfg_tree[0][1], str):
            #     raise ValueError("The first element of the FCFG tree must be a string.")
            if tree_match(pcfg_list_tree, fcfg_tree):
                fcfg_probability_sums[i] += probability
                matched = True
                # break
        
        # It seems NLTK FCFT has some cases where if a production is not needed and 
        # acts like an identify function it will be removed. See below:
        # NP[NUM=?b, SEM=<\n.(?A(n) & ?N(n))>] -> A[SEM=?A] N[NUM=?b, SEM=?N]
        # N[NUM=?b, SEM=<\x.(?P(x) & ?Q(x))>] -> A[SEM=?P] N[NUM=?b, SEM=?Q]
        # NP[SEM=?Q, NUM=?b] -> N[SEM=?Q, NUM=?b]
        # It does not create tree (NP-sg-no-no (N-sg (A penultimate) (N-sg letter))))
        # if not matched:
        #     print(f"Warning: No matching FCFG tree found for PCFG tree: {pcfg_tree}")
    
    # Create result list of (fcfg_tree, total_probability) tuples
    result = [(fcfg_trees[i], prob) for i, prob in fcfg_probability_sums.items() if prob > 0]
    return result
    


# # Year 21

# In[ ]:


# Year: 2021
transcripts_file = "21/1000.txt"
data_json_file = "21/parsed_transcripts_with_probs.json"
# Load both parsers
pcfg_parser = nltk.load_parser(pcfg_grammar_file, trace=0)
fcfg_parser = nltk.load_parser(feature_grammar_file, trace=0)

# Load transcripts
with open(transcripts_file, "r") as f:
    transcripts = f.readlines()

data = {} 

# Parse the transcripts and store the results in the data dictionary
count = 0
num_fails = 0
parsed_transcripts = 0
total_transcripts = len(transcripts)
for line in transcripts:
    count += 1
    if count % 5000 == 0:
        print(f"Processed {count} lines...")
        print(f"Number of failed parses: {num_fails}")
        # Save intermediate results every 5000 lines
        with open(data_json_file, "w") as f:
            json.dump(data, f, indent=4)
        print(f"Saved intermediate results to {data_json_file}")
    
    if line.strip():
        parts = line.strip().split(maxsplit=1)
        if len(parts) == 2:
            id, sentence = parts
            tokens = sentence.lower().split()
            
            parts = id.split('-')
            root_id = '-'.join(parts[:-1])
            transcript_id = parts[-1]
            # print(f'root_id: {root_id}, transcript_id: {transcript_id}')
            
            if root_id not in data:
                data[root_id] = {}
                
            data[root_id]["max_transcript_id"] = transcript_id
            try:
                # Parse with both parsers
                pcfg_trees = list(pcfg_parser.parse(tokens))
                fcfg_trees = list(fcfg_parser.parse(tokens))
                
                if pcfg_trees and fcfg_trees:
                    
                    # Map PCFG trees to FCFG trees and assign probabilities
                    mapped_trees = map_pcfg_to_fcfg_trees(pcfg_trees, fcfg_trees)
                    # print(mapped_trees)
                    
                    if len(mapped_trees) != len(fcfg_trees):
                        print(f"Warning: Mapped trees count {len(mapped_trees)} does not match FCFG trees count {len(fcfg_trees)}")
                        print(f"Mapped trees: {mapped_trees}")
                    
                    if mapped_trees:
                        parsed_transcripts += 1
                        # Store the FCFG trees with their assigned probabilities
                        if transcript_id not in data[root_id]:
                            data[root_id][transcript_id] = []
                        
                        for fcfg_tree, prob in mapped_trees:
                            sem = fcfg_tree.label()['SEM']
                            data[root_id][transcript_id].append({
                                "formula": str(sem),
                                "probability": prob
                            })
                        
                    else:
                        # print(f"No mapping found for trees of sentence: {sentence}")
                        num_fails += 1
                else:
                    # print(f"No parse trees found for sentence: {sentence}")
                    num_fails += 1
                    continue
            except Exception as e:
                print(f"Error parsing sentence '{sentence}': {e}")
                num_fails += 1
        else:
            print("Skipping line in text file:", line)
            
# Save the final data to a JSON file
with open(data_json_file, "w") as f:
    json.dump(data, f, indent=4)

print(f"Total number of transcripts: {len(data)}")
print(f"Final number of failed parses: {num_fails}")


# In[28]:


with open("21/1000.txt", "r") as f:
    transcripts = f.readlines()
    
# Count the number of transcripts that are not empty
num_transcripts = 0
for line in transcripts:
    if line.strip():
        parts = line.strip().split(maxsplit=1)
        if len(parts) == 2:
            num_transcripts += 1

with open("21/parsed_transcripts_with_probs.json", "r") as f:
    data = json.load(f)

# parsed transcripts is for each root id num transcripts - 1 for max_transcript id
parsed_transcripts = 0
for root_id, transcripts in data.items():
    parsed_transcripts += len(transcripts) - 1

print(f"Total transcripts processed: {num_transcripts}")
print(f"Total parsable sentences: {parsed_transcripts}")
print(f"Proportion of parsable sentences: {parsed_transcripts / count:.2%}")


# In[ ]:


with open("21/fst.txt", "r") as f:
    dag_data = f.readlines()

transducer_dict = {}

# Create an empty HfstBasicTransducer
curr_id = None
transducer = None
    
count = 0
# # Process each line of the DAG data
for line in dag_data:
    count += 1   
    parts = line.strip().split()
    parst = [p.strip() for p in parts]
    if not line.strip() or len(parts) == 0:
        continue
    if len(parts) == 1 and len(parts[0].split('-')) == 3:
        transducer_dict[curr_id] = transducer
        curr_id = parts[0]
        transducer = hfst.HfstBasicTransducer()
    elif len(parts) == 5:
        source = int(parts[0])
        target = int(parts[1])
        input_symbol = parts[2]
        output_symbol = parts[3]
        weight = float(parts[4])
        transducer.add_transition(source, target, input_symbol, output_symbol, weight)
    elif len(parts) == 2:
        source = int(parts[0])
        final_weight = float(parts[1])
        transducer.set_final_weight(source, final_weight)
    elif len(parts) == 1:
        transducer.set_final_weight(int(parts[0]), 0.0)

print(f"Total number of transducers: {len(transducer_dict)}")

with open("21/parsed_transcripts_with_probs.json", "r") as f:
    data = json.load(f)

for root_id, transcripts in data.items():
    if root_id in transducer_dict:
        transducer = transducer_dict[root_id]
        if transducer:
            max_transcripts = transcripts["max_transcript_id"]
            transducer = hfst.HfstTransducer(transducer)
            paths = transducer.extract_paths(max_number=1000, output='raw')
            for transcript_id, transcript in transcripts.items():
                for item in transcript:
                    if transcript_id == "max_transcript_id":
                        continue
                    item['accoustic_weight'] = float(paths[int(transcript_id)][0])
        else:
            print(f"Warning: No transducer found for root_id {root_id}")
    else:
        # remove the root_id from the data dictionary
        del data[root_id]
        print(f"Warning: No transducer found for root_id {root_id}")
        continue
    
with open("21/parsed_transcripts_with_probs_weights.json", "w") as f:
    json.dump(data, f, indent=4)
print(f"Total number of transcripts: {len(data)}")


# In[ ]:


try:
    with open("21/parsed_transcripts_with_probs_weights.json", "r") as f:
        all_data = json.load(f)
except FileNotFoundError:
    all_data = {}  # Changed to empty dict since the format uses dictionary structure

truth_values = {}
with open("21/pooled_truth", "r") as f:
    for line in f:
        if line.strip():
            parts = line.strip().lower().split()
            if len(parts) >= 2:
                id = parts[0]
                # The rest are word-value pairs
                word_values = []
                for i in range(1, len(parts), 2):
                    if i+1 < len(parts):
                        word = parts[i]
                        value = parts[i+1]
                        if value in ['t', 'f']:  # Skip 's' and 'u' values
                            word_values.append((word, value == 't'))
                truth_values[id] = word_values

total_words = 0
correct_truths = 0
ids_over_85_percent = 0
total_ids = 0

# Function to convert log probability to regular probability
def log_to_prob(log_prob):
    # Scale down the large log probability values
    # Using exp to convert from log space to probability space
    scale_factor = 0.0001
    return math.exp(log_prob * scale_factor)

# Iterate through root_ids in the data
for root_id, root_data in all_data.items():
    if root_id not in truth_values:
        continue
    
    total_ids += 1
    word_values = truth_values[root_id]
    words = [w for w, v in word_values]
    truths = [v for w, v in word_values]
    
    # Initialize expected values for each word across all transcripts
    expected_values = [0.0] * len(words)
    
    # Collect all formulas with their weighted probabilities across all transcripts
    all_formulas = []
    
    # Filter out non-transcript keys (like 'max_transcript_id')
    transcript_keys = [key for key in root_data.keys() if key != 'max_transcript_id']
    
    # Sort transcript IDs numerically
    transcript_ids = sorted(transcript_keys, key=int)
    
    # First pass: collect all formulas with their weighted probabilities
    for transcript_id in transcript_ids:
        formulas_with_prob = root_data[transcript_id]
        
        for formula_data in formulas_with_prob:
            formula = formula_data["formula"]
            tree_prob = formula_data["probability"]
            acoustic_weight = formula_data.get("accoustic_weight", 0.0)
            
            # Convert acoustic weight to regular probability
            acoustic_prob = log_to_prob(acoustic_weight)
            weighted_prob = tree_prob * acoustic_prob
            
            all_formulas.append({
                "formula": formula,
                "weighted_prob": weighted_prob,
                "transcript_id": transcript_id
            })
    
    # Calculate total weighted probability for normalization
    total_weighted_prob = sum(item["weighted_prob"] for item in all_formulas)
    
    if total_weighted_prob == 0:
        print(f"Warning: Total weighted probability is zero for root_id {root_id}")
        continue
    
    # Second pass: evaluate each formula and accumulate expected values
    for formula_data in all_formulas:
        formula = formula_data["formula"]
        weighted_prob = formula_data["weighted_prob"]
        transcript_id = formula_data["transcript_id"]
        
        # Normalize the weighted probability
        normalized_prob = weighted_prob / total_weighted_prob
        
        vals = [nltk.Valuation.fromstring(to_model_str(w, all_func)) for w in words]
        for val in vals: emptysets(val)
        models = [nltk.Model(val.domain, val) for val in vals]
        assignments = [nltk.Assignment(val.domain) for val in vals]
        
        try:
            for idx, truth in enumerate(truths):
                result = models[idx].evaluate(formula, assignments[idx])
                # Add weighted contribution to expected value (1 if True, 0 if False)
                expected_values[idx] += normalized_prob * (1 if result else 0)
        except Exception as e:
            print(f"Error evaluating root_id {root_id}, transcript_id {transcript_id}, formula: {formula}: {e}")
            continue
    
    # Check how many words have correct expected values (threshold at 0.5)
    correct_count = 0
    for idx, truth in enumerate(truths):
        total_words += 1
        predicted_truth = expected_values[idx] >= 0.5
        if predicted_truth == truth:
            correct_truths += 1
            correct_count += 1
    
    # Calculate accuracy for this root_id
    accuracy = correct_count / len(truths) if len(truths) > 0 else 0
    # print(f"Root_id {root_id}: Accuracy = {accuracy:.4f}")
    
    # Check if accuracy is at least 85%
    if accuracy >= 0.85:
        ids_over_85_percent += 1

# Calculate and print final statistics
print(f"\nFinal Statistics for 2021:")
print(f"Total words processed: {total_words}")
print(f"Correct truth evaluations: {correct_truths}")
if total_words > 0:
    print(f"Overall accuracy: {correct_truths / total_words:.4f}")
print(f"IDs with accuracy ≥ 85%: {ids_over_85_percent} out of {total_ids}")


# # Year 23

# In[ ]:


# Year: 2023
transcripts_file = "23/1000.txt"
data_json_file = "23/parsed_transcripts_with_probs.json"
# Load both parsers
pcfg_parser = nltk.load_parser(pcfg_grammar_file, trace=0)
fcfg_parser = nltk.load_parser(feature_grammar_file, trace=0)

# Load transcripts
with open(transcripts_file, "r") as f:
    transcripts = f.readlines()

data = {} 

# Parse the transcripts and store the results in the data dictionary
count = 0
num_fails = 0
parsed_transcripts = 0
total_transcripts = len(transcripts)
for line in transcripts:
    count += 1
    if count % 5000 == 0:
        print(f"Processed {count} lines...")
        print(f"Number of failed parses: {num_fails}")
        # Save intermediate results every 5000 lines
        with open(data_json_file, "w") as f:
            json.dump(data, f, indent=4)
        print(f"Saved intermediate results to {data_json_file}")
    
    if line.strip():
        parts = line.strip().split(maxsplit=1)
        if len(parts) == 2:
            id, sentence = parts
            tokens = sentence.lower().split()
            
            parts = id.split('-')
            root_id = '-'.join(parts[:-1])
            transcript_id = parts[-1]
            # print(f'root_id: {root_id}, transcript_id: {transcript_id}')
            
            if root_id not in data:
                data[root_id] = {}
                
            data[root_id]["max_transcript_id"] = transcript_id
            try:
                # Parse with both parsers
                pcfg_trees = list(pcfg_parser.parse(tokens))
                fcfg_trees = list(fcfg_parser.parse(tokens))
                
                if pcfg_trees and fcfg_trees:
                    
                    # Map PCFG trees to FCFG trees and assign probabilities
                    mapped_trees = map_pcfg_to_fcfg_trees(pcfg_trees, fcfg_trees)
                    # print(mapped_trees)
                    
                    if len(mapped_trees) != len(fcfg_trees):
                        print(f"Warning: Mapped trees count {len(mapped_trees)} does not match FCFG trees count {len(fcfg_trees)}")
                        print(f"Mapped trees: {mapped_trees}")
                    
                    if mapped_trees:
                        parsed_transcripts += 1
                        # Store the FCFG trees with their assigned probabilities
                        if transcript_id not in data[root_id]:
                            data[root_id][transcript_id] = []
                        
                        for fcfg_tree, prob in mapped_trees:
                            sem = fcfg_tree.label()['SEM']
                            data[root_id][transcript_id].append({
                                "formula": str(sem),
                                "probability": prob
                            })
                        
                    else:
                        # print(f"No mapping found for trees of sentence: {sentence}")
                        num_fails += 1
                else:
                    # print(f"No parse trees found for sentence: {sentence}")
                    num_fails += 1
                    continue
            except Exception as e:
                print(f"Error parsing sentence '{sentence}': {e}")
                num_fails += 1
        else:
            print("Skipping line in text file:", line)
            
# Save the final data to a JSON file
with open(data_json_file, "w") as f:
    json.dump(data, f, indent=4)

print(f"Total number of transcripts: {len(data)}")
print(f"Final number of failed parses: {num_fails}")


# In[ ]:


with open("23/1000.txt", "r") as f:
    transcripts = f.readlines()
    
# Count the number of transcripts that are not empty
num_transcripts = 0
for line in transcripts:
    if line.strip():
        parts = line.strip().split(maxsplit=1)
        if len(parts) == 2:
            num_transcripts += 1

with open("23/parsed_transcripts_with_probs.json", "r") as f:
    data = json.load(f)

# parsed transcripts is for each root id num transcripts - 1 for max_transcript id
parsed_transcripts = 0
for root_id, transcripts in data.items():
    parsed_transcripts += len(transcripts) - 1

print(f"Total transcripts processed: {num_transcripts}")
print(f"Total parsable sentences: {parsed_transcripts}")
print(f"Proportion of parsable sentences: {parsed_transcripts / count:.2%}")


# In[ ]:


with open("23/fst.txt", "r") as f:
    dag_data = f.readlines()

transducer_dict = {}

# Create an empty HfstBasicTransducer
curr_id = None
transducer = None
    
count = 0
# # Process each line of the DAG data
for line in dag_data:
    count += 1   
    parts = line.strip().split()
    parst = [p.strip() for p in parts]
    if not line.strip() or len(parts) == 0:
        continue
    if len(parts) == 1 and len(parts[0].split('-')) == 3:
        transducer_dict[curr_id] = transducer
        curr_id = parts[0]
        transducer = hfst.HfstBasicTransducer()
    elif len(parts) == 5:
        source = int(parts[0])
        target = int(parts[1])
        input_symbol = parts[2]
        output_symbol = parts[3]
        weight = float(parts[4])
        transducer.add_transition(source, target, input_symbol, output_symbol, weight)
    elif len(parts) == 2:
        source = int(parts[0])
        final_weight = float(parts[1])
        transducer.set_final_weight(source, final_weight)
    elif len(parts) == 1:
        transducer.set_final_weight(int(parts[0]), 0.0)

print(f"Total number of transducers: {len(transducer_dict)}")

with open("23/parsed_transcripts_with_probs.json", "r") as f:
    data = json.load(f)

for root_id, transcripts in data.items():
    if root_id in transducer_dict:
        transducer = transducer_dict[root_id]
        if transducer:
            max_transcripts = transcripts["max_transcript_id"]
            transducer = hfst.HfstTransducer(transducer)
            paths = transducer.extract_paths(max_number=1000, output='raw')
            for transcript_id, transcript in transcripts.items():
                for item in transcript:
                    if transcript_id == "max_transcript_id":
                        continue
                    item['accoustic_weight'] = float(paths[int(transcript_id)][0])
        else:
            print(f"Warning: No transducer found for root_id {root_id}")
    else:
        # remove the root_id from the data dictionary
        del data[root_id]
        print(f"Warning: No transducer found for root_id {root_id}")
        continue
    
with open("23/parsed_transcripts_with_probs_weights.json", "w") as f:
    json.dump(data, f, indent=4)
print(f"Total number of transcripts: {len(data)}")


# In[ ]:


try:
    with open("23/parsed_transcripts_with_probs_weights.json", "r") as f:
        all_data = json.load(f)
except FileNotFoundError:
    all_data = {}  # Changed to empty dict since the format uses dictionary structure

truth_values = {}
with open("23/pooled_truth", "r") as f:
    for line in f:
        if line.strip():
            parts = line.strip().lower().split()
            if len(parts) >= 2:
                id = parts[0]
                # The rest are word-value pairs
                word_values = []
                for i in range(1, len(parts), 2):
                    if i+1 < len(parts):
                        word = parts[i]
                        value = parts[i+1]
                        if value in ['t', 'f']:  # Skip 's' and 'u' values
                            word_values.append((word, value == 't'))
                truth_values[id] = word_values

total_words = 0
correct_truths = 0
ids_over_85_percent = 0
total_ids = 0

# Function to convert log probability to regular probability
def log_to_prob(log_prob):
    # Scale down the large log probability values
    # Using exp to convert from log space to probability space
    scale_factor = 0.0001
    return math.exp(log_prob * scale_factor)

# Iterate through root_ids in the data
for root_id, root_data in all_data.items():
    if root_id not in truth_values:
        continue
    
    total_ids += 1
    word_values = truth_values[root_id]
    words = [w for w, v in word_values]
    truths = [v for w, v in word_values]
    
    # Initialize expected values for each word across all transcripts
    expected_values = [0.0] * len(words)
    
    # Collect all formulas with their weighted probabilities across all transcripts
    all_formulas = []
    
    # Filter out non-transcript keys (like 'max_transcript_id')
    transcript_keys = [key for key in root_data.keys() if key != 'max_transcript_id']
    
    # Sort transcript IDs numerically
    transcript_ids = sorted(transcript_keys, key=int)
    
    # First pass: collect all formulas with their weighted probabilities
    for transcript_id in transcript_ids:
        formulas_with_prob = root_data[transcript_id]
        
        for formula_data in formulas_with_prob:
            formula = formula_data["formula"]
            tree_prob = formula_data["probability"]
            acoustic_weight = formula_data.get("accoustic_weight", 0.0)
            
            # Convert acoustic weight to regular probability
            acoustic_prob = log_to_prob(acoustic_weight)
            weighted_prob = tree_prob * acoustic_prob
            
            all_formulas.append({
                "formula": formula,
                "weighted_prob": weighted_prob,
                "transcript_id": transcript_id
            })
    
    # Calculate total weighted probability for normalization
    total_weighted_prob = sum(item["weighted_prob"] for item in all_formulas)
    
    if total_weighted_prob == 0:
        print(f"Warning: Total weighted probability is zero for root_id {root_id}")
        continue
    
    # Second pass: evaluate each formula and accumulate expected values
    for formula_data in all_formulas:
        formula = formula_data["formula"]
        weighted_prob = formula_data["weighted_prob"]
        transcript_id = formula_data["transcript_id"]
        
        # Normalize the weighted probability
        normalized_prob = weighted_prob / total_weighted_prob
        
        vals = [nltk.Valuation.fromstring(to_model_str(w, all_func)) for w in words]
        for val in vals: emptysets(val)
        models = [nltk.Model(val.domain, val) for val in vals]
        assignments = [nltk.Assignment(val.domain) for val in vals]
        
        try:
            for idx, truth in enumerate(truths):
                result = models[idx].evaluate(formula, assignments[idx])
                # Add weighted contribution to expected value (1 if True, 0 if False)
                expected_values[idx] += normalized_prob * (1 if result else 0)
        except Exception as e:
            print(f"Error evaluating root_id {root_id}, transcript_id {transcript_id}, formula: {formula}: {e}")
            continue
    
    # Check how many words have correct expected values (threshold at 0.5)
    correct_count = 0
    for idx, truth in enumerate(truths):
        total_words += 1
        predicted_truth = expected_values[idx] >= 0.5
        if predicted_truth == truth:
            correct_truths += 1
            correct_count += 1
    
    # Calculate accuracy for this root_id
    accuracy = correct_count / len(truths) if len(truths) > 0 else 0
    # print(f"Root_id {root_id}: Accuracy = {accuracy:.4f}")
    
    # Check if accuracy is at least 85%
    if accuracy >= 0.85:
        ids_over_85_percent += 1

# Calculate and print final statistics
print(f"\nFinal Statistics for 2023:")
print(f"Total words processed: {total_words}")
print(f"Correct truth evaluations: {correct_truths}")
if total_words > 0:
    print(f"Overall accuracy: {correct_truths / total_words:.4f}")
print(f"IDs with accuracy ≥ 85%: {ids_over_85_percent} out of {total_ids}")