final3.py

from flask import Flask, jsonify, request
import os
import glob
from langchain_neo4j import Neo4jGraph
from langchain_groq import ChatGroq
from langchain_core.documents import Document
from langchain_experimental.graph_transformers import LLMGraphTransformer
import textwrap
import uuid
import re

app = Flask(__name__)

# Neo4j and Groq configuration
NEO4J_URI = ""
NEO4J_USERNAME = "neo4j"
NEO4J_PASSWORD = ""
GROQ_API_KEY = ""

# Set environment variables
os.environ["NEO4J_URI"] = NEO4J_URI
os.environ["NEO4J_USERNAME"] = NEO4J_USERNAME
os.environ["NEO4J_PASSWORD"] = NEO4J_PASSWORD

# Initialize Neo4j graph
try:
    graph = Neo4jGraph(
        url=NEO4J_URI,
        username=NEO4J_USERNAME,
        password=NEO4J_PASSWORD
    )
    print("Neo4j graph initialized successfully")
except Exception as e:
    print(f"Failed to initialize Neo4j graph: {str(e)}")
    graph = None

# Initialize LLM
llm = ChatGroq(groq_api_key=GROQ_API_KEY, model_name="Gemma2-9b-It")
llm_transformer = LLMGraphTransformer(llm=llm)

# Function to read text files from dataset folder
def read_text_files_from_dataset(dataset_folder="dataset"):
    text = ""
    text_files = glob.glob(os.path.join(dataset_folder, "*.txt"))
    
    if not text_files:
        raise FileNotFoundError(f"No .txt files found in the {dataset_folder} folder")
    
    for file_path in text_files:
        with open(file_path, 'r', encoding='utf-8') as file:
            text += file.read() + "\n\n"
    
    return text.strip()

# Split text into chunks
def split_text_into_chunks(text, max_chunk_size=1000):
    paragraphs = text.split('\n\n')
    chunks = []
    current_chunk = ""
    
    for paragraph in paragraphs:
        if len(current_chunk) + len(paragraph) + 1 <= max_chunk_size:
            current_chunk += paragraph + "\n\n"
        else:
            if current_chunk:
                chunks.append(current_chunk.strip())
            current_chunk = paragraph + "\n\n"
    
    if current_chunk:
        chunks.append(current_chunk.strip())
    
    return chunks

# Load dataset and split into chunks
try:
    text = read_text_files_from_dataset()
    chunks = split_text_into_chunks(text)
    print(f"Loaded {len(chunks)} chunks from dataset")
except Exception as e:
    chunks = []
    print(f"Error reading dataset: {str(e)}")

# Store current chunk index
current_chunk_index = 0

# Track changes for undo functionality
change_log = []

# Test route to verify app is running
@app.route('/')
def home():
    return "Flask app is running!"

@app.route('/process_chunk', methods=['GET'])
def process_chunk():
    global current_chunk_index
    
    if not graph:
        return jsonify({"status": "error", "message": "Neo4j graph not initialized"})
    
    if not chunks:
        return jsonify({"status": "error", "message": "No text files found in dataset folder"})
    
    if current_chunk_index >= len(chunks):
        return jsonify({"status": "error", "message": "No more chunks to process"})
    
    chunk_text = chunks[current_chunk_index]
    document = [Document(page_content=chunk_text)]
    
    try:
        graph_doc = llm_transformer.convert_to_graph_documents(document)
        
        # Track nodes and relationships added in this chunk
        added_nodes = []
        added_relationships = []
        
        for node in graph_doc[0].nodes:
            merge_query = f"""
            MERGE (n:`{node.type}` {{id: $id}})
            SET n += $properties
            """
            properties = node.properties.copy()
            properties['id'] = node.id
            graph.query(merge_query, params={'id': node.id, 'properties': properties})
            added_nodes.append(node.id)
        
        for rel in graph_doc[0].relationships:
            if rel.source.id == rel.target.id:
                continue
            rel_query = f"""
            MATCH (source:`{rel.source.type}` {{id: $source_id}}),
                  (target:`{rel.target.type}` {{id: $target_id}})
            CREATE (source)-[r:`{rel.type}`]->(target)
            SET r += $properties
            """
            properties = rel.properties
            graph.query(rel_query, params={'source_id': rel.source.id, 'target_id': rel.target.id, 'properties': properties})
            added_relationships.append((rel.source.id, rel.type, rel.target.id))
        
        # Log the changes for undo
        change_log.append({
            'chunk_index': current_chunk_index,
            'added_nodes': added_nodes,
            'added_relationships': added_relationships
        })
        
        current_chunk_index += 1
        
        return jsonify({
            "status": "success",
            "chunk_index": current_chunk_index - 1,
            "chunk_content": chunk_text[:100] + "..." if len(chunk_text) > 100 else chunk_text,
            "nodes_added": len(added_nodes),
            "relationships_added": len(added_relationships)
        })
    
    except Exception as e:
        return jsonify({"status": "error", "message": str(e)})

@app.route('/undo', methods=['GET'])
def undo():
    if not graph:
        return jsonify({"status": "error", "message": "Neo4j graph not initialized"})
    
    if not change_log:
        return jsonify({"status": "error", "message": "No changes to undo"})
    
    try:
        last_change = change_log.pop()
        
        # Delete relationships added in the last chunk
        for rel in last_change['added_relationships']:
            source_id, rel_type, target_id = rel
            delete_rel_query = f"""
            MATCH (source {{id: $source_id}})-[r:`{rel_type}`]->(target {{id: $target_id}})
            DELETE r
            """
            graph.query(delete_rel_query, params={'source_id': source_id, 'target_id': target_id})
        
        # Delete nodes added in the last chunk (if they have no relationships)
        for node_id in last_change['added_nodes']:
            delete_node_query = f"""
            MATCH (n {{id: $node_id}})
            WHERE NOT EXISTS ((n)--())
            DELETE n
            """
            graph.query(delete_node_query, params={'node_id': node_id})
        
        return jsonify({
            "status": "success",
            "undone_chunk_index": last_change['chunk_index']
        })
    
    except Exception as e:
        return jsonify({"status": "error", "message": str(e)})

@app.route('/get_graph', methods=['GET'])
def get_graph():
    if not graph:
        return jsonify({"status": "error", "message": "Neo4j graph not initialized"})
    
    try:
        # Query all nodes
        nodes_query = """
        MATCH (n)
        RETURN n.id AS id, labels(n) AS labels, properties(n) AS properties
        """
        nodes_result = graph.query(nodes_query)
        
        # Query all relationships
        relationships_query = """
        MATCH (source)-[r]->(target)
        RETURN source.id AS source_id, type(r) AS type, target.id AS target_id, properties(r) AS properties
        """
        relationships_result = graph.query(relationships_query)
        
        # Format nodes
        nodes = [
            {
                "id": record["id"],
                "labels": record["labels"],
                "properties": record["properties"]
            }
            for record in nodes_result
        ]
        
        # Format relationships
        relationships = [
            {
                "source_id": record["source_id"],
                "type": record["type"],
                "target_id": record["target_id"],
                "properties": record["properties"]
            }
            for record in relationships_result
        ]
        
        return jsonify({
            "status": "success",
            "graph": {
                "nodes": nodes,
                "relationships": relationships
            }
        })
    
    except Exception as e:
        return jsonify({"status": "error", "message": str(e)})
@app.route('/process_fanfiction', methods=['POST'])
def process_fanfiction():
    if not graph:
        return jsonify({"status": "error", "message": "Neo4j graph not initialized"}), 500
    
    data = request.json
    if not data or 'text' not in data:
        return jsonify({"status": "error", "message": "Text input is required"}), 400
    
    input_text = data['text']
    if not input_text.strip():
        return jsonify({"status": "error", "message": "Input text is empty"}), 400
    
    try:
        # Split text into chunks
        chunks = split_text_into_chunks(input_text)
        contradictions = []
        nodes_to_add = []
        relationships_to_add = []
        
        # Process each chunk
        for chunk in chunks:
            document = [Document(page_content=chunk)]
            graph_doc = llm_transformer.convert_to_graph_documents(document)[0]
            
            # Check nodes for contradictions
            for node in graph_doc.nodes:
                existing_node = graph.query(
                    f"MATCH (n:`{node.type}` {{id: $id}}) RETURN n",
                    params={'id': node.id}
                )
                if existing_node:
                    existing_properties = existing_node[0]['n']
                    for key, value in node.properties.items():
                        if key in existing_properties and existing_properties[key] != value:
                            contradictions.append({
                                "type": "node",
                                "entity": node.id,
                                "property": key,
                                "original_value": existing_properties[key],
                                "fanfiction_value": value
                            })
                else:
                    # No contradiction, mark node for addition
                    nodes_to_add.append(node)
            
            # Check relationships for contradictions
            for rel in graph_doc.relationships:
                existing_rel = graph.query(
                    f"""
                    MATCH (source:`{rel.source.type}` {{id: $source_id}})-[r:`{rel.type}`]->(target:`{rel.target.type}` {{id: $target_id}})
                    RETURN r
                    """,
                    params={'source_id': rel.source.id, 'target_id': rel.target.id}
                )
                if not existing_rel:
                    conflicting_rel = graph.query(
                        f"""
                        MATCH (source:`{rel.source.type}` {{id: $source_id}})-[r]->(target:`{rel.target.type}` {{id: $target_id}})
                        WHERE type(r) <> $rel_type
                        RETURN type(r) as rel_type
                        """,
                        params={'source_id': rel.source.id, 'target_id': rel.target.id, 'rel_type': rel.type}
                    )
                    if conflicting_rel:
                        contradictions.append({
                            "type": "relationship",
                            "source": rel.source.id,
                            "target": rel.target.id,
                            "fanfiction_relationship": rel.type,
                            "original_relationship": conflicting_rel[0]['rel_type']
                        })
                    else:
                        # No contradiction, mark relationship for addition
                        relationships_to_add.append(rel)
        
        # If no contradictions, update the graph
        if not contradictions:
            added_nodes = []
            added_relationships = []
            
            # Add nodes
            for node in nodes_to_add:
                merge_query = f"""
                MERGE (n:`{node.type}` {{id: $id}})
                SET n += $properties
                """
                properties = node.properties.copy()
                properties['id'] = node.id
                graph.query(merge_query, params={'id': node.id, 'properties': properties})
                added_nodes.append(node.id)
            
            # Add relationships
            for rel in relationships_to_add:
                if rel.source.id == rel.target.id:
                    continue
                rel_query = f"""
                MATCH (source:`{rel.source.type}` {{id: $source_id}}),
                      (target:`{rel.target.type}` {{id: $target_id}})
                CREATE (source)-[r:`{rel.type}`]->(target)
                SET r += $properties
                """
                properties = rel.properties
                graph.query(rel_query, params={'source_id': rel.source.id, 'target_id': rel.target.id, 'properties': properties})
                added_relationships.append((rel.source.id, rel.type, rel.target.id))
            
            # Log the changes for undo
            change_log.append({
                'chunk_index': current_chunk_index,
                'added_nodes': added_nodes,
                'added_relationships': added_relationships
            })
            
            return jsonify({
                "status": "success",
                "message": "Graph updated with fanfiction content",
                "graph_updated": True,
                "nodes_added": len(added_nodes),
                "relationships_added": len(added_relationships)
            })
        else:
            return jsonify({
                "status": "contradictions_found",
                "graph_updated": False,
                "contradictions": contradictions
            })
    
    except Exception as e:
        return jsonify({"status": "error", "message": f"Error processing fanfiction text: {str(e)}"}), 500
if __name__ == '__main__':
    app.run(debug=True, host='localhost', port=5000)