better handling of global_temp_dir

Author: thiswillbeyourgithub

DocToolsLLM/utils/loaders.py

@@ -45,7 +45,7 @@
 
 from .misc import (doc_loaders_cache, html_to_text, hasher,
                    file_hasher, get_splitter, check_docs_tkn_length,
-                   average_word_length, wpm)
+                   average_word_length, wpm, global_temp_dir)
 from .typechecker import optional_typecheck
 from .logger import whi, yel, red, log
 from .flags import is_verbose, is_linux
@@ -173,8 +173,6 @@ def load(self):
     ["norm"],
 ]
 
-global_temp_dir = [None]  # will be replaced when load_one_doc is called
-
 
 @optional_typecheck
 def load_one_doc(

DocToolsLLM/utils/misc.py

@@ -31,6 +31,9 @@
 TextSplitter = lazy_import.lazy_class('langchain.text_splitter.TextSplitter')
 RecursiveCharacterTextSplitter = lazy_import.lazy_class('langchain.text_splitter.RecursiveCharacterTextSplitter')
 
+# will be replaced when load_one_doc is called, by the path to the file where the loaders can store temporary file
+global_temp_dir = [None]
+
 try:
     import ftlangdetect
 except Exception as err: