record.set_text_from_pdf(): add first_pages param

Author: Gerit Wagner

colrev/ops/pdf_prep.py

@@ -137,7 +137,7 @@ def prepare_pdf(self, item: dict) -> dict:
             record_dict, path=self.review_manager.path
         )
         if record_dict[Fields.FILE].endswith(".pdf"):
-            record.set_text_from_pdf()
+            record.set_text_from_pdf(first_pages=True)
         original_filename = record_dict[Fields.FILE]
 
         self.review_manager.logger.debug(f"Start PDF prep of {record_dict[Fields.ID]}")

colrev/packages/files_dir/src/files_dir.py

@@ -315,7 +315,7 @@ def _get_grobid_metadata(self, *, file_path: Path) -> dict:
                     record = colrev.record.record_pdf.PDFRecord(
                         record_dict, path=self.review_manager.path
                     )
-                    record.set_text_from_pdf()
+                    record.set_text_from_pdf(first_pages=True)
                     record_dict = record.get_data()
                     if Fields.TEXT_FROM_PDF in record_dict:
                         text: str = record_dict[Fields.TEXT_FROM_PDF]
@@ -676,7 +676,7 @@ def _add_doi_from_pdf_if_not_available(self, record_dict: dict) -> None:
                 record_dict, path=self.review_manager.path
             )
             if Fields.DOI not in record_dict:
-                record.set_text_from_pdf()
+                record.set_text_from_pdf(first_pages=True)
                 res = re.findall(self._doi_regex, record.data[Fields.TEXT_FROM_PDF])
                 if res:
                     record.data[Fields.DOI] = res[0].upper()

colrev/packages/genai/src/genai_data.py

@@ -110,7 +110,7 @@ def _run_prompt(self) -> None:
             record = colrev.record.record_pdf.PDFRecord(
                 record_dict, path=self.review_manager.path
             )
-            record.set_text_from_pdf(all_pages=True)
+            record.set_text_from_pdf()
 
             user_message = self.settings.prompt.format(
                 record_id=record_dict[Fields.ID],

colrev/packages/ocrmypdf/src/ocrmypdf.py

@@ -82,7 +82,7 @@ def _apply_ocr(
         record.add_field_provenance_note(
             key=Fields.FILE, note="pdf_processed with OCRMYPDF"
         )
-        record.set_text_from_pdf()
+        record.set_text_from_pdf(first_pages=True)
         return record
 
     def prep_pdf(

colrev/record/qm/quality_model.py

@@ -86,7 +86,7 @@ def run(self, *, record: colrev.record.record.Record) -> None:
             ):
                 # The following should be improved.
                 record = colrev.record.record_pdf.PDFRecord(record.data, path=self.path)
-                record.set_text_from_pdf()
+                record.set_text_from_pdf(first_pages=True)
 
         for checker in self.checkers:
             if checker.msg in self.defects_to_ignore:

colrev/record/record_pdf.py

@@ -98,11 +98,17 @@ def set_nr_pages_in_pdf(self) -> None:
             pages_in_file = doc.page_count
         self.data[Fields.NR_PAGES_IN_FILE] = pages_in_file
 
-    def set_text_from_pdf(self) -> None:
+    def set_text_from_pdf(self, *, first_pages: bool = False) -> None:
         """Set the text_from_pdf field based on the PDF"""
         self.data[Fields.TEXT_FROM_PDF] = ""
         self.set_nr_pages_in_pdf()
-        text = self.extract_text_by_page(pages=[0, 1, 2])
+
+        if first_pages:
+            pages = [0, 1, 2]
+        else:
+            pages = list(range(self.data[Fields.NR_PAGES_IN_FILE]))
+
+        text = self.extract_text_by_page(pages=pages)
         text_from_pdf = text.replace("\n", " ").replace("\x0c", "")
         self.data[Fields.TEXT_FROM_PDF] = text_from_pdf