Add onboarding_report.csv generation

silnlp/common/extract_corpora.py

@@ -1,7 +1,7 @@
 import argparse
 import logging
 from pathlib import Path
-from typing import Optional, Set
+from typing import List, Optional, Set, Tuple
 
 from ..common.corpus import count_lines
 from ..common.environment import SilNlpEnv
@@ -76,7 +76,7 @@ def extract_corpora(
     parent_project: Optional[str] = None,
     versification_error_output_path: Optional[str] = None,
     environment: SilNlpEnv = SilNlpEnv.create_standard_environment(),
-) -> Path | None:
+) -> Tuple[Path | None, List[int], int, int]:
     # Process the projects that have data and tell the user.
     if len(projects) > 0:
         expected_verse_count = count_lines(environment.assets_dir / "vref.txt")
@@ -85,11 +85,17 @@ def extract_corpora(
         for project in projects:
             LOGGER.info(f"Extracting {project}...")
             project_dir = environment.get_paratext_project_dir(project)
-            parent_project_dir = get_parent_project_dir(project_dir, environment)
+
+            if parent_project is None:
+                parent_project_dir = get_parent_project_dir(project_dir, environment)
+            else:
+                parent_project_dir = environment.get_paratext_project_dir(parent_project)
             if parent_project_dir is not None:
                 LOGGER.info(f"Identified parent project {parent_project_dir.name}.")
 
-            check_versification(project_dir, parent_project_dir, versification_error_output_path, environment)
+            matching, detected_versification, versification_error_count = check_versification(
+                project_dir, parent_project_dir, versification_error_output_path, environment
+            )
             corpus_filename, verse_count, line_count = extract_project(
                 project_dir,
                 environment.mt_scripture_dir,
@@ -115,10 +121,10 @@ def extract_corpora(
             )
             LOGGER.info(f"# of Terms: {terms_count}")
             LOGGER.info("Done.")
-            return corpus_filename
+            return corpus_filename, [v.value for v in detected_versification], versification_error_count, terms_count
     else:
         LOGGER.warning(f"Couldn't find any data to process for any project in {environment.pt_projects_dir}.")
-        return None
+        return None, [], 0, 0
 
 
 if __name__ == "__main__":

silnlp/common/onboard_project.py

@@ -1,4 +1,6 @@
 import argparse
+import csv
+import pandas as pd
 import getpass
 import hashlib
 import logging
@@ -19,20 +21,71 @@
 from silnlp.common.analyze import analyze
 from silnlp.common.clean_projects import process_single_project_for_cleaning
 from silnlp.nmt.clearml_connection import TAGS_LIST, SILClearML
-from silnlp.nmt.config import Config
+from silnlp.nmt.config import SUPPORTED_GLOSS_ISOS, Config
 
 from ..nmt.config_utils import create_config
 from .collect_verse_counts import collect_verse_counts
 from .environment import SilNlpEnv
 from .extract_corpora import extract_corpora
-from .iso_info import ALT_ISO, NLLB_TAG_FROM_ISO
+from .iso_info import ALT_ISO, NLLB_SCRIPT_SET, NLLB_TAG_FROM_ISO
 
 LOGGER = logging.getLogger(__package__ + ".onboard_project")
 
 
+class OnboardingReport:
+
+    def __init__(self, project_type: str) -> None:
+        self.project_type: str = project_type
+        self.name_on_bucket: str = ""
+        self.short_name: str = ""
+        self.name: str = ""
+        self.iso_code: str = ""
+        self.language: str = ""
+        self.lang_in_nllb: bool = False
+        self.script: str = ""
+        self.script_in_nllb: bool = False
+        self.normalization: str = ""
+        self.versification: List[int] = []
+        self.versification_error_count: int = 0
+        self.verse_count: int = 0
+        self.alignment: str = ""
+        self.key_terms_type: str = ""
+        self.key_terms_count: int = 0
+        self.key_terms_glosses_exist: bool = False
+        self.mean_tokens_per_verse: float = 0.0
+        self.mean_char_per_token: float = 0.0
+        self.num_added_tokens: int = 0
+        self.num_verses_truncated: int = 0
+
+    def generate_dict(self) -> dict:
+        return {
+            "Project Type": self.project_type,
+            "Name on Bucket": self.name_on_bucket,
+            "Short Name": self.short_name,
+            "Name": self.name,
+            "ISO Code": self.iso_code,
+            "Language": self.language,
+            "Language in NLLB": "yes" if self.lang_in_nllb else "no",
+            "Script": self.script,
+            "Script in NLLB": "yes" if self.script_in_nllb else "no",
+            "Normalization": self.normalization,
+            "Versification": ";".join(str(v) for v in self.versification),
+            "Versification Error Count": self.versification_error_count if self.versification_error_count else "",
+            "Verse Count": self.verse_count,
+            "Alignment with Main": self.alignment,
+            "Key Terms Type": self.key_terms_type,
+            "Key Terms Count": self.key_terms_count,
+            "Key Terms Glosses Exist": "yes" if self.key_terms_glosses_exist else "no",
+            "Mean Tokens Per Verse": f"{self.mean_tokens_per_verse:.2f}" if self.mean_tokens_per_verse else "",
+            "Mean Char Per Token": f"{self.mean_char_per_token:.3f}" if self.mean_char_per_token else "",
+            "Number of Added Tokens": self.num_added_tokens,
+            "Number of Verses Truncated": self.num_verses_truncated,
+        }
+
+
 class OnboardingProject:
 
-    def __init__(self, project_name: str, overwrite: bool, environment: SilNlpEnv) -> None:
+    def __init__(self, project_name: str, project_type: str, overwrite: bool, environment: SilNlpEnv) -> None:
         self.project_name: str = project_name
         self.local_project_path: Path | None = None
         self.output_folder: Path | None = None
@@ -41,6 +94,7 @@ def __init__(self, project_name: str, overwrite: bool, environment: SilNlpEnv) -
         self.resource: bool | None = None
         self.overwrite: bool = overwrite
         self.environment: SilNlpEnv = environment
+        self.report: OnboardingReport = OnboardingReport(project_type)
 
     def get_extract_path(self) -> Path | None:
         if self.extract_file is not None:
@@ -59,7 +113,7 @@ def extract_corpora_wrapper(self, extract_config: dict) -> None:
         LOGGER.info(f"Extracting corpora for project '{self.project_name}'")
 
         versification_error_output_path = Path(self.output_folder / f"versification_errors_{self.project_name}.txt")
-        extract_path = extract_corpora(
+        extract_path, detected_versification, versification_error_count, terms_count = extract_corpora(
             projects={self.project_name},
             include_markers=extract_config.get("markers", False),
             extract_lemmas=extract_config.get("lemmas", False),
@@ -71,6 +125,10 @@ def extract_corpora_wrapper(self, extract_config: dict) -> None:
         )
         self.extract_file = extract_path
 
+        self.report.versification = detected_versification
+        self.report.versification_error_count = versification_error_count
+        self.report.key_terms_count = terms_count
+
     def wildebeest_analysis_wrapper(self, wildebeest_config: dict) -> None:
         extract_path = self.get_extract_path()
         if extract_path is None:
@@ -244,11 +302,11 @@ def align_wrapper(
         )
         exp_name = f"{self.output_folder.stem}/{self.project_name}/alignments"
         analyze(config=align_config, exp_name=exp_name, create_summaries=True, environment=self.environment)
-        corpus_stats_csv = align_output_dir / "corpus_stats.csv"
+        corpus_stats_csv = align_output_dir / "corpus-stats.csv"
         if corpus_stats_csv.exists():
             shutil.move(
                 str(corpus_stats_csv),
-                str(self.output_folder / "corpus_stats.csv"),
+                str(self.output_folder / "corpus-stats.csv"),
             )
 
     def check_for_project_errors(self) -> None:
@@ -369,6 +427,54 @@ def rename_project(self, project_name: str, datestamp: bool) -> str:
             project_name = append_datestamp(project_name)
         return project_name
 
+    def generate_report(self) -> dict:
+        self.report.name_on_bucket = self.project_name
+
+        stats_df = pd.read_csv(self.output_folder / "tokenization_stats.csv", header=[0, 1])
+
+        target_stats = stats_df[stats_df[(" ", "Translation Side")] == "Target"]
+
+        self.report.mean_tokens_per_verse = target_stats[("Tokens/Verse", "Mean")].values[0]
+        self.report.mean_char_per_token = target_stats[("Characters/Token", "Mean")].values[0]
+        self.report.num_added_tokens = target_stats[(" ", "Num Tokens Added to Vocab")].values[0]
+        self.report.num_verses_truncated = target_stats[("Tokens/Verse", "Num Verses >= 200 Tokens")].values[0]
+
+        verse_counts_df = pd.read_csv(self.output_folder / "verse_counts.csv")
+
+        extract_verse_counts = verse_counts_df[verse_counts_df["file"] == self.extract_file.stem]
+
+        self.report.verse_count = extract_verse_counts["Total"].values[0]
+
+        settings = FileParatextProjectSettingsParser(self.environment.pt_projects_dir / self.project_name).parse()
+
+        self.report.name = settings.full_name
+        self.report.short_name = settings.name
+        self.report.iso_code = settings.language_code
+        self.report.key_terms_glosses_exist = (
+            self.report.iso_code in SUPPORTED_GLOSS_ISOS
+            or ALT_ISO.get_alternative(self.report.iso_code) in SUPPORTED_GLOSS_ISOS
+        )
+        self.report.key_terms_type = settings.biblical_terms_list_type
+
+        corpus_stats_csv = self.output_folder / "corpus-stats.csv"
+        if corpus_stats_csv.exists():
+            corpus_stats_df = pd.read_csv(corpus_stats_csv)
+            alignment_row = corpus_stats_df[corpus_stats_df["trg_project"] == self.extract_file.stem]
+            if not alignment_row.empty:
+                self.report.alignment = alignment_row["align_score"].values[0]
+                self.report.script = alignment_row["trg_script"].values[0]
+            else:
+                alignment_row = corpus_stats_df[corpus_stats_df["src_project"] == self.extract_file.stem].head(1)
+                if not alignment_row.empty:
+                    self.report.script = alignment_row["src_script"].values[0]
+
+        nllb_tag = NLLB_TAG_FROM_ISO.get(self.report.iso_code, None)
+        self.report.lang_in_nllb = nllb_tag is not None
+        self.report.script = nllb_tag.split("_")[1] if nllb_tag and self.report.script == "" else self.report.script
+        self.report.script_in_nllb = self.report.script in NLLB_SCRIPT_SET
+
+        return self.report.generate_dict()
+
 
 class OnboardingRequest:
 
@@ -383,13 +489,19 @@ def __init__(
         self.overwrite = onboarding_config.get("overwrite", False)
         main_project_name = onboarding_config.get("main_project", None)
         self.main_project: OnboardingProject = OnboardingProject(
-            project_name=main_project_name, overwrite=self.overwrite, environment=self.environment
+            project_name=main_project_name,
+            project_type="Request Project",
+            overwrite=self.overwrite,
+            environment=self.environment,
         )
         reference_project_names = onboarding_config.get("ref_projects", [])
         self.reference_projects: List[OnboardingProject] = []
         for ref_project_name in reference_project_names:
             reference_project = OnboardingProject(
-                project_name=ref_project_name, overwrite=self.overwrite, environment=self.environment
+                project_name=ref_project_name,
+                project_type="Reference Project",
+                overwrite=self.overwrite,
+                environment=self.environment,
             )
             self.reference_projects.append(reference_project)
         self.no_clean: bool = onboarding_config.get("no_clean", False)
@@ -437,6 +549,8 @@ def process_onboarding_request(self) -> None:
             if self.align:
                 self.align_main_project()
 
+            self.create_report()
+
     def align_main_project(self) -> None:
         iso_codes = set()
         if self.align_isos:
@@ -472,6 +586,10 @@ def prepare_and_upload_projects(self) -> None:
                     LOGGER.error(f"Error occurred while uploading reference project '{project.project_name}': {e}")
                     LOGGER.error(f"Continuing with onboarding without reference project '{project.project_name}'.")
                     reference_projects_to_remove.append(project)
+                else:
+                    LOGGER.error(f"Error occurred while uploading main project '{project.project_name}': {e}")
+                    LOGGER.error("Main project upload failed. Stopping onboarding process.")
+                    raise e
 
         for project in reference_projects_to_remove:
             self.reference_projects.remove(project)
@@ -583,6 +701,16 @@ def create_log_file(self) -> Iterator[None]:
         finally:
             close_logger(log_file_path)
 
+    def create_report(self) -> None:
+        LOGGER.info("Creating onboarding report.")
+        report_path = self.output_folder / "onboarding_report.csv"
+
+        projects = [self.main_project] + self.reference_projects
+        formatted_reports = [p.generate_report() for p in projects]
+
+        report_df = pd.DataFrame(formatted_reports)
+        report_df.to_csv(report_path, index=False)
+
 
 def create_paratext_project_folder_if_not_exists(project_name: str, environment: SilNlpEnv) -> Path:
     pt_project_path = environment.get_paratext_project_dir(project_name)

silnlp/common/paratext.py

@@ -441,7 +441,7 @@ def check_versification(
     parent_project_dir: Optional[str],
     versification_error_output_path: str,
     environment: SilNlpEnv,
-) -> Tuple[bool, List[VersificationType]]:
+) -> Tuple[bool, List[VersificationType], int]:
 
     parent_settings = None
     if parent_project_dir is not None:
@@ -463,13 +463,13 @@ def check_versification(
         ot_versification, key_ot_verses = detect_OT_versification(project_dir, environment)
         if ot_versification == VersificationType.UNKNOWN:
             LOGGER.warning(f"Unknown versification detected for {project_dir}.")
-            return (matching, [ot_versification])
+            return (matching, [ot_versification], 0)
     if check_nt:
         nt_versification: List[VersificationType]
         nt_versification, key_nt_verses = detect_NT_versification(project_dir, environment)
         if nt_versification[0] == VersificationType.UNKNOWN:
             LOGGER.warning(f"Unknown versification detected for {project_dir}.")
-            return (matching, nt_versification)
+            return (matching, nt_versification, 0)
 
     detected_versification: List[VersificationType] = [VersificationType.UNKNOWN]
     if check_ot and check_nt:
@@ -480,7 +480,7 @@ def check_versification(
                 f"The detected versifications were based on {', '.join(key_ot_verses + key_nt_verses)} "
                 "being the last verse of their respective chapters."
             )
-            return (matching, [ot_versification] + nt_versification)
+            return (matching, [ot_versification] + nt_versification, 0)
         detected_versification = [ot_versification]
         key_verses = key_ot_verses + key_nt_verses
     elif not check_ot and check_nt:
@@ -495,7 +495,7 @@ def check_versification(
             "Versification detection for the OT requires the book of Daniel. "
             "Versification detection for the NT requires the books of John, Acts, and Romans."
         )
-        return (matching, detected_versification)
+        return (matching, detected_versification, 0)
 
     if settings.versification.type not in detected_versification:
         if not (
@@ -509,7 +509,7 @@ def check_versification(
                 f"being the last verse of {'their' if len(key_verses)>=2 else 'its'} "
                 f"respective chapter{'s' if len(key_verses)>=2 else ''}."
             )
-            return (matching, detected_versification)
+            return (matching, detected_versification, 0)
 
     errors = FileParatextProjectVersificationErrorDetector(project_dir).get_usfm_versification_errors()
     if len(errors) > 0:
@@ -532,4 +532,4 @@ def check_versification(
                     )
 
     matching = True
-    return (matching, detected_versification)
+    return (matching, detected_versification, len(errors))