study-assistant.py

# -*- coding: utf-8 -*-
"""NLP Miniproject (updated)

Automatically generated by Colab.

Original file is located at
    https://colab.research.google.com/drive/1fwoE5oYvmKxWKM2D25Mioy3etSbKJw9n
"""

# @title Install all the Libraries
!pip install googletrans==3.1.0a0
!pip install pymupdf
!pip install -q transformers==4.43.1 einops accelerate langchain bitsandbytes sentencepiece langchain_community langchain_huggingface
!pip install spacy
!pip install Together
!pip install deep-translator
!pip install -q transformers==4.43.1 einops accelerate langchain bitsandbytes sentencepiece langchain_community langchain_huggingface
!pip install spacy

"""##Model Implementation"""

# @title Load the model
from transformers import AutoTokenizer
import transformers
import torch
import fitz  # PyMuPDF for handling PDFs
import spacy
import os
from googletrans import Translator  # For multilingual translation
from IPython.display import Markdown

model = "meta-llama/Meta-Llama-3.1-8B-Instruct"

pipeline = transformers.pipeline(
    "text-generation",
    model=model,
    model_kwargs={"torch_dtype": torch.bfloat16},
    device_map="auto",
)

# @title All the functions
def printmd(string):
    display(Markdown(string))

def generate_study_plan(days, syllabus):
    input_prompt = f"Create a detailed roadmap to complete the following syllabus in {days} days:\n{syllabus}"
    prompt = """
    You are an expert in making study plans. You are an academic assistant who helps users plan their studies.
    You have to make a detailed study plan. Share the study resources for each topic, the number of focused hours to study.
    Dont give me any outline for the study plan, I need the detailed brief study plan for the specified days.
    Cover the Important topics from the syllabus first and then cover rest of the things.
    """
    messages = [
        {"role": "system", "content": prompt},
        {"role": "user", "content": input_prompt},
    ]

    outputs = pipeline(
        messages,
        max_new_tokens=1024,
        do_sample=True,
        top_p=0.9,
        temperature=0.6,
        return_full_text=False,
    )
    printmd(outputs[0]["generated_text"])

# Translate the plan to user's preferred language
def translate_study_plan(plan_text, target_lang):
    translator = Translator()
    translated = translator.translate(plan_text, dest=target_lang)
    return translated.text

# Read PDF content using PyMuPDF
def read_pdf_input(pdf_path):
    doc = fitz.open(pdf_path)
    text = ""
    for page_num in range(len(doc)):
        page = doc.load_page(page_num)  # Load each page
        text += page.get_text("text")  # Extract text from page
    return text


# Simple function to read syllabus input if it's in plain text
def read_text_input(text_path):
    if os.path.exists(text_path):
        with open(text_path, 'r') as file:
            return file.read()
    return text_path


# NLP function to extract key topics using NER (Named Entity Recognition)
def extract_topics(text):
    nlp = spacy.load("en_core_web_sm")  # Load a small English model
    doc = nlp(text)
    topics = set()
    for ent in doc.ents:
        if ent.label_ in {"ORG", "GPE", "PERSON", "WORK_OF_ART", "EVENT"}:  # Extract named entities
            topics.add(ent.text)
    return list(topics)

def generate_summary(syllabus):
    input_prompt = f"Generate a simple summary for this syllabus: {syllabus}"
    prompt = """
    You are an expert in genearting summaries and getting the important topics for the user syllabus.
    Be friendly with the user, suggest the summary with proper points.
    """
    messages = [
        {"role": "system", "content": prompt},
        {"role": "user", "content": input_prompt},
    ]

    outputs = pipeline(
        messages,
        max_new_tokens=1024,
        do_sample=True,
        top_p=0.9,
        temperature=0.6,
        return_full_text=False,
    )
    printmd(outputs[0]["generated_text"])

def split_text(text, max_length=5000):
    # Split text into chunks with a max length of 5000 characters
    chunks = []
    while len(text) > max_length:
        # Find the last space within the max_length to avoid splitting words
        split_point = text[:max_length].rfind(' ')
        chunks.append(text[:split_point])
        text = text[split_point:]
    chunks.append(text)
    return chunks

# @title Main function using Model
def main():
    print("Enter syllabus (text or PDF path):")
    input_syllabus = input().strip()

    # Handle PDF input if provided
    if input_syllabus.endswith('.pdf'):
        syllabus_text = read_pdf_input(input_syllabus)
    else:
        syllabus_text = read_text_input(input_syllabus)

    print("Enter the number of days to complete the syllabus (or press Enter to skip):")
    days_input = input().strip()
    days = int(days_input) if days_input else 30  # Default to 30 days

    # NLP Step 1: Extract Key Topics using NER
    key_topics = extract_topics(syllabus_text)
    print(f"Key Topics extracted from the syllabus: {key_topics}")

    # NLP Step 2: Summarize the syllabus for an overview
    syllabus_summary = generate_summary(syllabus_text)
    print(f"\nSummary of the syllabus: {syllabus_summary}")

    # Generate a study roadmap using LLaMA
    study_plan = generate_study_plan(days, syllabus_text)

    print("\nGenerated Study Plan:")
    print(study_plan)

    # Translation: Translate the roadmap to the desired language
    print("Enter the target language code (e.g., 'es' for Spanish):")
    target_lang = input().strip()

    translated_plan = translate_study_plan(study_plan, target_lang)

    # Output the translated study plan
    print(f"\nTranslated Study Plan in {target_lang}:")
    print(translated_plan)

if __name__ == "__main__":
    main()

"""## API Implementation"""

# @title Load the API Key and Model
import os
from together import Together
from deep_translator import GoogleTranslator
from google.colab import userdata
import torch
import spacy
import fitz
import time
import ipywidgets as widgets
from IPython.display import Markdown

Lkey = userdata.get('Llama_Key')
client = Together(api_key=Lkey)
model = "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo"

# @title All the functions
def printmd(string):
    display(Markdown(string))

def generate_study_plan(days, syllabus):
    input_prompt = f"Create a detailed roadmap to complete the following syllabus in {days} days:\n{syllabus}, not before the days i specified. It should cover {days}"
    prompt = """
    You are an expert in making study plans. You are an academic assistant who helps users plan their studies.
    You have to make a detailed study plan. Share the study resources for each topic, the number of focused hours to study.
    Dont give me any outline for the study plan, I need the detailed brief study plan with covering the important topics first and taking proper time to cover them.
    Cover the Important topics from the syllabus first and then cover rest of the things.
    Dont respond with anything else other than the study plan. When you tell the day along with the day also tell what should be done for the day.
    Share some important resources for each topic like youtube videos with Youtube video title, books, etc. Dont share if the resources or youtube videos are wrong, if wrong then try sharing the topic name that they can search and get better results.
    """
    messages = [
        {"role": "system", "content": prompt},
        {"role": "user", "content": input_prompt},
    ]
    response = client.chat.completions.create(
        model=model,
        messages=messages,
        max_tokens=4096,
        temperature=0.6,
        top_p=0.7,
        top_k=50,
        repetition_penalty=1,
        stop=["<|eot_id|>","<|eom_id|>"],
        stream=False
    )
    return (response.choices[0].message.content)

# Translate the plan to user's preferred language
def translate_study_plan(plan_text, target_lang):
    translator = GoogleTranslator(source='auto', target=target_lang)
    translated_chunks = []
    # if target_lang == 'en':
    #       chunks = split_text(plan_text, max_length=5000)
    #       # Translate each chunk and store the result
    #       for chunk in chunks:
    #           translated_chunks.append(translator.translate(chunk))
    #       # Join the translated chunks back into a single string
    #       translated_plan = " ".join(translated_chunks)
    #       return translated_plan

    chunks = split_text(plan_text, max_length=1000)  # Try smaller chunks
    for chunk in chunks:
        attempts = 3
        for attempt in range(attempts):
            try:
                translated_chunks.append(translator.translate(chunk))
                time.sleep(1)  # Short sleep to avoid rate limit
                break  # Exit loop on success
            except RequestError as e:
                print(f"Attempt {attempt + 1} failed: {e}")
                time.sleep(2 ** attempt)  # Exponential backoff
        else:
            print("All attempts failed for chunk:", chunk)  # Log failure

    # Join the translated chunks back into a single string
    translated_plan = " ".join(translated_chunks)
    return translated_plan

# Read PDF content using PyMuPDF
def read_pdf_input(pdf_path):
    doc = fitz.open(pdf_path)
    text = ""
    for page_num in range(len(doc)):
        page = doc.load_page(page_num)  # Load each page
        text += page.get_text("text")  # Extract text from page
    return text


def read_text_input(text_path):
    if os.path.exists(text_path):
        with open(text_path, 'r') as file:
            return file.read()
    return text_path


def extract_topics(text):
    nlp = spacy.load("en_core_web_sm")
    doc = nlp(text)
    topics = set()
    for ent in doc.ents:
        if ent.label_ in {"ORG", "GPE", "PERSON", "WORK_OF_ART", "EVENT"}:
            topics.add(ent.text)
    return list(topics)

def generate_summary(syllabus):
    input_prompt = f"Generate a simple summary for this syllabus: {syllabus}"
    prompt = """
    You are an expert in genearting summaries and getting the important topics for the user syllabus.
    Be friendly with the user, suggest the summary with proper points.
    Dont respond with anything else than the summary.
    """
    messages = [
        {"role": "system", "content": prompt},
        {"role": "user", "content": input_prompt},
    ]
    response = client.chat.completions.create(
        model=model,
        messages=messages,
        max_tokens=512,
        temperature=0.6,
        top_p=0.7,
        top_k=50,
        repetition_penalty=1,
        stop=["<|eot_id|>","<|eom_id|>"],
        stream=False
    )
    return (response.choices[0].message.content)

def split_text(text, max_length):
    chunks = []
    # print("slippting")
    while len(text) > max_length:
        # Find the last space within the max_length to avoid splitting words
        split_point = text[:max_length].rfind(' ')
        chunks.append(text[:split_point])
        text = text[split_point:]
    chunks.append(text)
    return chunks

import ipywidgets as widgets

def get_languages():
    languages = {
      'English': 'en',
      'Hindi': 'hi',
      'Marathi': 'mr',
      'Bengali': 'bn',
      'Gujarati': 'gu',
      'Tamil': 'ta',
      'Telugu': 'te',
      'Kannada': 'kn',
      'Malayalam': 'ml',
      'Punjabi': 'pa',
      'Odia': 'or',
      'Assamese': 'as',
      'Nepali': 'ne',
      'Urdu': 'ur',
      'Sindhi': 'sd',
      'Bhojpuri': 'bho',
      'Manipuri': 'mni',
      'Spanish': 'es',
      'French': 'fr',
      'German': 'de',
      'Italian': 'it',
      'Japanese': 'ja',
      'Korean': 'ko',
      'Russian': 'ru',
    }

    # Create a dropdown for input language selection
    input_langauge = widgets.Dropdown(
        options=languages.keys(),
        description='Select Input Language:',
    )

    # Create a dropdown for input language selection
    output_langauge = widgets.Dropdown(
        options=languages.keys(),
        description='Select Output Language:',
    )

    display(input_langauge)
    display(output_langauge)
    return languages, input_langauge, output_langauge

def get_code(languages, input_langauge, output_langauge):
    for key, value in languages.items():
        if key == input_langauge.value:
            input_lang = value
        if key == output_langauge.value:
            output_lang = value
    return input_lang, output_lang

"""## Main working"""

languages, input_langauge, output_langauge = get_languages()

input_code, output_code = get_code(languages, input_langauge, output_langauge)
print(input_code, " to ", output_code)

# Get syllabus input from the user
# print("Enter syllabus (text or PDF path):")
print(translate_study_plan("Enter syllabus (text or PDF path):", input_code))
input_syllabus = input().strip()

# Handle PDF input if provided
syllabus = ""
try:
    if input_syllabus.endswith('.pdf'):
        syllabus = read_pdf_input(input_syllabus)
    else:
        syllabus = read_text_input(input_syllabus)

except FileNotFoundError:
    printmd("**Error:** The specified file was not found. Please check the path and try again.")
    raise  # Optional: re-raise the exception if you want to stop the program

except Exception as e:
    printmd(f"**Error:** An unexpected error: {e}")
    raise  # Optional: re-raise the exception if you want to stop the program

# Convert syllabus text to English
if syllabus:
    syllabus_text = translate_study_plan(syllabus, 'en')
    printmd("---")
    printmd("### Syllabus Content")
    printmd(syllabus_text)
    printmd("---")

    # Get the number of days to complete the syllabus
    # print("Enter the number of days to complete the syllabus (or press Enter to skip):")
    print(translate_study_plan("Enter the number of days to complete the syllabus (or press Enter to skip):", input_code))
    days_input = input().strip()
    days = int(days_input) if days_input else 7  # Default to 7 days

    printmd("---")
    printmd("### Summary of the syllabus:")
    summary = generate_summary(syllabus_text)

    # Translate the summary to the user's preferred language
    translated_summ = translate_study_plan(summary, output_code)
    printmd(translated_summ)
    printmd("---")

    printmd("### Study Plan:")
    study_plan = generate_study_plan(days, syllabus_text)

    # Output the translated study plan
    translated_plan = translate_study_plan(study_plan, output_code)
    printmd(translated_plan)
    printmd("---")

    printmd("### Key Topics")
    array_topics = extract_topics(syllabus_text)
    key_topics = ", ".join(array_topics)
    # printmd(key_topics)

    # Translate key topics to the user's preferred language
    translated_topics = translate_study_plan(key_topics, output_code)
    printmd(translated_topics)
    printmd("---")