Merge pull request #21 from MEITREX/comments

Add comments and logging info

Author: ZombieAlienRobot

fileextractlib/DocumentProcessor.py

@@ -1,11 +1,13 @@
 import io
+import logging
 
 import requests
 from fileextractlib.DocumentData import DocumentData
 from fileextractlib.PdfProcessor import PdfProcessor
 from fileextractlib.PowerPointProcessor import PowerPointProcessor
 
 
+_logger = logging.getLogger(__name__)
 class DocumentProcessor:
     def __init__(self):
         self.pdf_processor = PdfProcessor()
@@ -21,6 +23,8 @@ def process(self, file_url: str) -> DocumentData:
             raise ValueError("Content type header not found")
 
         if content_type_header == "application/pdf":
+            _logger.info("Processing PDF")
             return self.pdf_processor.process_from_io(file_bytes)
         elif content_type_header == "application/vnd.openxmlformats-officedocument.presentationml.presentation":
+            _logger.info("Processing Powerpoint")
             return self.powerpoint_processor.process_from_io(file_bytes)

fileextractlib/PdfProcessor.py

@@ -1,3 +1,4 @@
+import logging
 import io
 import pdf2image
 import argparse
@@ -7,6 +8,7 @@
 from pypdf import PdfWriter, PdfReader
 from fileextractlib.DocumentData import DocumentData, PageData
 
+_logger = logging.getLogger(__name__)
 
 class PdfProcessor:
     """
@@ -18,9 +20,11 @@ def __init__(self):
 
     def process_from_io(self, file: typing.BinaryIO) -> DocumentData:
         # create thumbnail images for each page
+        _logger.info("Creating thumbnails")
         page_images = pdf2image.convert_from_bytes(file.read())
 
         # split the pdf into pages, so we can extract text for each page separately
+        _logger.info("Splitting document into pages")
         file.seek(0)
         pdf_reader = PdfReader(file)
 
@@ -40,6 +44,8 @@ def process_from_io(self, file: typing.BinaryIO) -> DocumentData:
 
                 pages.append(PageData(page_index, page_text, page_images[page_index], None))
 
+        _logger.info("Finished processing file.")
+
         return DocumentData(pages, [])
 
 

fileextractlib/TopicModel.py

@@ -26,8 +26,13 @@ def __init__(self, record_segments: list[VideoSegmentEntity | DocumentSegmentEnt
         self.docs = []
 
     def create_topic_model(self):
+        """
+        This method creates the topic model from which the suggested tags are generated.
+
+        """
         embeddings = []
 
+        # find the appropriate fields to use depending on the segmentEntity
         for entity in self.record_segments:
             if isinstance(entity, DocumentSegmentEntity):
                 self.docs.append(entity.text)
@@ -39,19 +44,25 @@ def create_topic_model(self):
                 self.docs.append(entity.textual_representation)
                 embeddings.append(entity.embedding)
 
+        # check to ensure enough segments are available to run the topic model
         if len(self.docs) < 11:
             _logger.info("More documents needed to create topic model.")
             return
 
         embeddings = np.array(embeddings)
+        # set stop_words to remove stop words, ngram_range defines how many words the terms can contain
         vectorizer_model = CountVectorizer(stop_words="english", ngram_range=(1, 3))
+        # reduce_frequent_words to further reduce common words,
+        # bm25_weighting changes the weighting to a more robust one for small datasets
+        # more info: https://maartengr.github.io/BERTopic/getting_started/ctfidf/ctfidf.html
         ctfidf_model = ClassTfidfTransformer(reduce_frequent_words=True, bm25_weighting=True)
+        # change diversity to further improve results of the keywords, lower values means less diverse
         mmr = MaximalMarginalRelevance(diversity=0.3)
 
         representation_models = mmr
 
         self.model = BERTopic(
-            min_topic_size=7,
+            min_topic_size=7, # set the min topic size lower to work better with small datasets
             vectorizer_model=vectorizer_model,
             ctfidf_model=ctfidf_model,
             representation_model=representation_models
@@ -60,6 +71,10 @@ def create_topic_model(self):
         self.model.fit_transform(self.docs, embeddings)
 
     def add_tags_to_media_records(self, segments):
+        """
+        This method adds tags to all media records. Replaces old tags when run.
+
+        """
         if len(self.docs) < 11:
             _logger.info("Topic model wasn't created. More documents needed.")
             return
@@ -94,6 +109,10 @@ def add_tags_to_media_records(self, segments):
         return mediarecords_with_tags
 
     def add_tags_to_assessments(self, segments):
+        """
+       This method adds tags to all assessments. Replaces old tags when run.
+
+       """
         if len(self.docs) < 11:
             _logger.info("Topic model wasn't created. More documents needed.")
             return

service/DocProcAiService.py

@@ -99,9 +99,12 @@ async def ingest_media_record_task():
             self.segment_database.delete_document_segments_by_media_record_id([media_record_id])
             self.segment_database.delete_video_segments_by_media_record_id([media_record_id])
 
+            _logger.info("Processing file of Type: " + record_type)
             if record_type == "PRESENTATION" or record_type == "DOCUMENT":
+                _logger.info("Starting document processor for " + str(media_record_id))
                 document_processor = DocumentProcessor()
                 document_data = document_processor.process(download_url)
+                _logger.info("Generating embeddings for " + str(media_record_id))
                 self.__lecture_pdf_embedding_generator.generate_embeddings(document_data.pages)
                 for segment in document_data.pages:
                     thumbnail_bytes = io.BytesIO()
@@ -116,10 +119,12 @@ async def ingest_media_record_task():
 
                 if config.current["lecture_llm_generator"]["document_summary_generator"]["enabled"]:
                     # generate and store a summary of this media record
+                    _logger.info("Generating summary for " + str(media_record_id))
                     self.__lecture_llm_generator.generate_summary_for_document(document_data)
 
                 self.media_record_info_database.upsert_media_record_info(media_record_id, document_data.summary, None)
             elif record_type == "VIDEO":
+                _logger.info("Starting video processor for " + str(media_record_id))
                 video_processor = VideoProcessor(
                     segment_image_similarity_threshold=
                     config.current["video_segmentation"]["segment_image_similarity_threshold"],
@@ -128,13 +133,16 @@ async def ingest_media_record_task():
                 del video_processor
 
                 # generate text embeddings for the segments of the video
+                _logger.info("Generating embeddings for " + str(media_record_id))
                 self.__lecture_video_embedding_generator.generate_embeddings(video_data.segments)
 
                 # generate titles for the video's segments if llm features enabled
                 if config.current["lecture_llm_generator"]["segment_title_generator"]["enabled"]:
+                    _logger.info("Generating title for " + str(media_record_id))
                     self.__lecture_llm_generator.generate_titles_for_video(video_data)
                 else:
                     # otherwise set empty data/placeholders
+                    _logger.info("LLM generator disabled. Setting placeholders.")
                     video_data.summary = []
                     for i, segment in enumerate(video_data.segments, start=1):
                         segment.title = "Section " + str(i)
@@ -177,6 +185,9 @@ async def ingest_media_record_task():
                                                                             priority))
 
     def __generate_tags(self):
+        """
+        Generates the suggested tags for all media records and assessments. This will recreate all suggested tags
+        """
         segments = self.segment_database.get_all_entity_segments()
 
         topic_model = TopicModel(segments)
@@ -188,6 +199,10 @@ def __generate_tags(self):
         self.__generate_tags_for_assessments(segments, topic_model)
 
     def __generate_tags_for_media_records(self, segments, topic_model):
+        """
+        Generates the suggested tags for all media records. This will recreate all suggested tags.
+        This step will be skipped if no media records are found.
+        """
         _logger.info("Generating tags for media records.")
         media_records = self.media_record_info_database.get_all_media_records()
         if not media_records: # check if media_records is empty
@@ -201,6 +216,10 @@ def __generate_tags_for_media_records(self, segments, topic_model):
             _logger.info("Generated tags for media records.")
 
     def __generate_tags_for_assessments(self, segments, topic_model):
+        """
+        Generates the suggested tags for all assessments. This will recreate all suggested tags.
+        This step will be skipped if no assessments are found.
+        """
         _logger.info("Generating tags for assesments.")
         assesments = self.assesment_database.get_all_assessments()
         if not assesments: # check if assessments is empty