python-design-pattern-rag/tasks.py at main · taggedzi/python-design-pattern-rag · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
"""The task file for the proect, to be used by invoke"""
# pylint: disable=line-too-long
import logging
from pathlib import Path
from invoke import task

logging.basicConfig(level=logging.INFO, format='[%(asctime)s] %(levelname)s: %(message)s')

@task
def venv(c):
    """Create virtual environment and install dependencies."""
    logging.info("🟢 Starting to create the working virtual environment...")
    if not Path("venv").exists():
        c.run("python -m venv venv")
        logging.info("  ✅ Virtual environment created successfully!")
    else:
        logging.info("  ⏭️  Virtual environment already exists, skipping creation.")

    logging.info("  📥 Starting to install dependencies...")
    c.run(r".\\venv\\Scripts\\pip install -r requirements.txt")
    logging.info("  ✅ Dependencies installed successfully!")

@task
def setup_ollama(c):
    """Setup Ollama environment."""
    create_rag_model(c)
    create_lessons_model(c)

@task
def create_rag_model(c):
    """Create a custom Ollama model."""
    logging.info("📥 Downloading model for rag generation...")
    c.run("ollama pull deepseek-coder:6.7b")
    logging.info("✅ Model downloaded successfully!")

    logging.info("🛠️ Starting to create the custom model...")
    c.run("ollama create pattern-rag-gen -f ollama/pattern-rag-gen.Modelfile")
    logging.info("✅ Custom model created successfully!")

@task
def create_lessons_model(c):
    """Create a custom Ollama model."""
    logging.info("📥 Downloading model for lesson creation...")
    c.run("ollama pull falcon3:7b")
    logging.info("✅ Model downloaded successfully!")

    logging.info("🛠️ Starting to create the custom model...")
    c.run("ollama create lesson-planner -f ollama/lesson-planner.Modelfile")
    logging.info("✅ Custom model created successfully!")

@task
def build_chunks(c):
    """Run chunk_all_patterns script."""
    logging.info("🧩 Starting to chunk all patterns...")
    c.run(r".\\venv\\Scripts\\python scripts/rag_chunker.py")
    logging.info("✅ Chunks generated successfully!")

@task
def build_lessons(c, model="lesson-planner:latest"):
    """Generate Markdown lessons."""
    logging.info("📚 Starting to generate lessons...")
    c.run(rf".\\venv\\Scripts\\python scripts/generate_lessons.py")
    logging.info("✅ Lessons generated successfully!")
    build_doc_index(c)

@task
def build_search_index(c):
    """Generate summary index."""
    logging.info("🟢 Starting to generate summary index...")
    c.run(r".\\venv\\Scripts\\python scripts/summary_index_generator.py chunks/ ./summary_index.json")
    logging.info("✅ Summary index generated successfully!")

@task
def build_doc_index(c):
    """Generate the index.md file for the docs folder."""
    logging.info("🟢 Starting to generate docs index...")
    c.run(r".\\venv\\Scripts\\python scripts/build_doc_index.py --docs docs --output index.md")
    logging.info("✅ Docs index generated successfully!")

@task
def build_all(c):
    """Run the build pipeline."""
    build_chunks(c)
    build_search_index(c)
    build_lessons(c)

@task
def full(c):
    """Run the full pipeline."""
    venv(c)
    setup_ollama(c)
    build_all(c)