Merge pull request #2 from RndmCodeGuy20/temporary

Migrate to Github Directory Loader

Author: RndmCodeGuy20

api.py

@@ -315,7 +315,7 @@ async def reload_chroma():
         # First check if we can write to the directory
         logging.info("Checking ChromaDB directory permissions: %s", CHROMA_DB_PATH)
 
-        return {"message": "Reloading ChromaDB..."}
+        # return {"message": "Reloading ChromaDB..."}
 
         # Create parent directory if it doesn't exist
         os.makedirs(os.path.dirname(CHROMA_DB_PATH), exist_ok=True)

embeddings.py

@@ -3,6 +3,7 @@
 import logging
 from dotenv import load_dotenv
 from langchain_community.document_loaders import DirectoryLoader
+from langchain_community.document_loaders import GithubFileLoader
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain.schema import Document
 from langchain_google_genai import GoogleGenerativeAIEmbeddings
@@ -17,6 +18,7 @@
 
 CHROMA_DB_PATH = os.getenv("CHROMA_PATH")
 DATA_STORE_PATH = os.getenv("DATA_STORE_PATH")
+GITHUB_TOKEN = os.getenv("GITHUB_TOKEN")
 EMBEDDING_MODEL_NAME = "models/text-embedding-004"
 CHUNK_SIZE = 1000
 CHUNK_OVERLAP = 500
@@ -36,7 +38,13 @@ def load_documents(directory: str) -> list[Document]:
     logging.info(f"Loading documents from: {directory}")
     if not os.path.exists(directory):
         raise FileNotFoundError(f"Directory not found: {directory}")
-    loader = DirectoryLoader(directory, glob="**/*.md")
+    # loader = DirectoryLoader(directory, glob="**/*.md")
+    loader = GithubFileLoader(
+        repo="rndmcodeguy20/model-builder",
+        branch="master",
+        file_filter=lambda file_path: file_path.endswith(".md") and file_path.startswith("docs/"),
+        access_token=GITHUB_TOKEN,
+    )
     documents = loader.load()
     logging.info(f"Loaded {len(documents)} documents from: {directory}")
     return documents

requirements.txt

@@ -1,14 +1,17 @@
+## The following requirements were added by pip freeze:
 aiofiles==24.1.0
 aiohappyeyeballs==2.6.1
 aiohttp==3.11.16
 aiosignal==1.3.2
 annotated-types==0.7.0
 anyio==4.9.0
 asgiref==3.8.1
+astroid==3.3.9
 attrs==25.3.0
 backoff==2.2.1
 bcrypt==4.3.0
 beautifulsoup4==4.13.3
+black==25.1.0
 build==1.2.2.post1
 cachetools==5.5.2
 certifi==2025.1.31
@@ -22,6 +25,7 @@ coloredlogs==15.0.1
 cryptography==44.0.2
 dataclasses-json==0.6.7
 Deprecated==1.2.18
+dill==0.3.9
 distro==1.9.0
 durationpy==0.9
 emoji==2.14.1
@@ -50,6 +54,7 @@ humanfriendly==10.0
 idna==3.10
 importlib_metadata==8.6.1
 importlib_resources==6.5.2
+isort==6.0.1
 joblib==1.4.2
 jsonpatch==1.33
 jsonpointer==3.0.0
@@ -68,6 +73,7 @@ lxml==5.3.2
 Markdown==3.7
 markdown-it-py==3.0.0
 marshmallow==3.26.1
+mccabe==0.7.0
 mdurl==0.1.2
 mmh3==5.1.0
 monotonic==1.6
@@ -79,7 +85,7 @@ nltk==3.9.1
 numpy==1.26.4
 oauthlib==3.2.2
 olefile==0.47
-onnxruntime==1.21.0
+onnxruntime==1.19.0
 opentelemetry-api==1.31.1
 opentelemetry-exporter-otlp-proto-common==1.31.1
 opentelemetry-exporter-otlp-proto-grpc==1.31.1
@@ -93,6 +99,8 @@ opentelemetry-util-http==0.52b1
 orjson==3.10.16
 overrides==7.7.0
 packaging==24.2
+pathspec==0.12.1
+platformdirs==4.3.7
 posthog==3.23.0
 propcache==0.3.1
 proto-plus==1.26.1
@@ -105,6 +113,7 @@ pydantic==2.11.3
 pydantic-settings==2.8.1
 pydantic_core==2.33.1
 Pygments==2.19.1
+pylint==3.3.6
 pypdf==5.4.0
 PyPika==0.48.9
 pyproject_hooks==1.2.0
@@ -132,6 +141,7 @@ starlette==0.45.3
 sympy==1.13.3
 tenacity==9.1.2
 tokenizers==0.21.1
+tomlkit==0.13.2
 tqdm==4.67.1
 typer==0.15.2
 typing-inspect==0.9.0