diff --git a/silnlp/common/extract_corpora.py b/silnlp/common/extract_corpora.py index 7f2fd2cb..61f43137 100644 --- a/silnlp/common/extract_corpora.py +++ b/silnlp/common/extract_corpora.py @@ -1,7 +1,7 @@ import argparse import logging from pathlib import Path -from typing import Optional, Set +from typing import List, Optional, Set, Tuple from ..common.corpus import count_lines from ..common.environment import SilNlpEnv @@ -76,7 +76,7 @@ def extract_corpora( parent_project: Optional[str] = None, versification_error_output_path: Optional[str] = None, environment: SilNlpEnv = SilNlpEnv.create_standard_environment(), -) -> Path | None: +) -> Tuple[Path | None, List[int], int, int]: # Process the projects that have data and tell the user. if len(projects) > 0: expected_verse_count = count_lines(environment.assets_dir / "vref.txt") @@ -85,11 +85,17 @@ def extract_corpora( for project in projects: LOGGER.info(f"Extracting {project}...") project_dir = environment.get_paratext_project_dir(project) - parent_project_dir = get_parent_project_dir(project_dir, environment) + + if parent_project is None: + parent_project_dir = get_parent_project_dir(project_dir, environment) + else: + parent_project_dir = environment.get_paratext_project_dir(parent_project) if parent_project_dir is not None: LOGGER.info(f"Identified parent project {parent_project_dir.name}.") - check_versification(project_dir, parent_project_dir, versification_error_output_path, environment) + matching, detected_versification, versification_error_count = check_versification( + project_dir, parent_project_dir, versification_error_output_path, environment + ) corpus_filename, verse_count, line_count = extract_project( project_dir, environment.mt_scripture_dir, @@ -115,10 +121,10 @@ def extract_corpora( ) LOGGER.info(f"# of Terms: {terms_count}") LOGGER.info("Done.") - return corpus_filename + return corpus_filename, [v.value for v in detected_versification], versification_error_count, terms_count else: LOGGER.warning(f"Couldn't find any data to process for any project in {environment.pt_projects_dir}.") - return None + return None, [], 0, 0 if __name__ == "__main__": diff --git a/silnlp/common/onboard_project.py b/silnlp/common/onboard_project.py index 1eef23f6..0b2ee9b4 100644 --- a/silnlp/common/onboard_project.py +++ b/silnlp/common/onboard_project.py @@ -1,4 +1,6 @@ import argparse +import csv +import pandas as pd import getpass import hashlib import logging @@ -19,20 +21,71 @@ from silnlp.common.analyze import analyze from silnlp.common.clean_projects import process_single_project_for_cleaning from silnlp.nmt.clearml_connection import TAGS_LIST, SILClearML -from silnlp.nmt.config import Config +from silnlp.nmt.config import SUPPORTED_GLOSS_ISOS, Config from ..nmt.config_utils import create_config from .collect_verse_counts import collect_verse_counts from .environment import SilNlpEnv from .extract_corpora import extract_corpora -from .iso_info import ALT_ISO, NLLB_TAG_FROM_ISO +from .iso_info import ALT_ISO, NLLB_SCRIPT_SET, NLLB_TAG_FROM_ISO LOGGER = logging.getLogger(__package__ + ".onboard_project") +class OnboardingReport: + + def __init__(self, project_type: str) -> None: + self.project_type: str = project_type + self.name_on_bucket: str = "" + self.short_name: str = "" + self.name: str = "" + self.iso_code: str = "" + self.language: str = "" + self.lang_in_nllb: bool = False + self.script: str = "" + self.script_in_nllb: bool = False + self.normalization: str = "" + self.versification: List[int] = [] + self.versification_error_count: int = 0 + self.verse_count: int = 0 + self.alignment: str = "" + self.key_terms_type: str = "" + self.key_terms_count: int = 0 + self.key_terms_glosses_exist: bool = False + self.mean_tokens_per_verse: float = 0.0 + self.mean_char_per_token: float = 0.0 + self.num_added_tokens: int = 0 + self.num_verses_truncated: int = 0 + + def generate_dict(self) -> dict: + return { + "Project Type": self.project_type, + "Name on Bucket": self.name_on_bucket, + "Short Name": self.short_name, + "Name": self.name, + "ISO Code": self.iso_code, + "Language": self.language, + "Language in NLLB": "yes" if self.lang_in_nllb else "no", + "Script": self.script, + "Script in NLLB": "yes" if self.script_in_nllb else "no", + "Normalization": self.normalization, + "Versification": ";".join(str(v) for v in self.versification), + "Versification Error Count": self.versification_error_count if self.versification_error_count else "", + "Verse Count": self.verse_count, + "Alignment with Main": self.alignment, + "Key Terms Type": self.key_terms_type, + "Key Terms Count": self.key_terms_count, + "Key Terms Glosses Exist": "yes" if self.key_terms_glosses_exist else "no", + "Mean Tokens Per Verse": f"{self.mean_tokens_per_verse:.2f}" if self.mean_tokens_per_verse else "", + "Mean Char Per Token": f"{self.mean_char_per_token:.3f}" if self.mean_char_per_token else "", + "Number of Added Tokens": self.num_added_tokens, + "Number of Verses Truncated": self.num_verses_truncated, + } + + class OnboardingProject: - def __init__(self, project_name: str, overwrite: bool, environment: SilNlpEnv) -> None: + def __init__(self, project_name: str, project_type: str, overwrite: bool, environment: SilNlpEnv) -> None: self.project_name: str = project_name self.local_project_path: Path | None = None self.output_folder: Path | None = None @@ -41,6 +94,7 @@ def __init__(self, project_name: str, overwrite: bool, environment: SilNlpEnv) - self.resource: bool | None = None self.overwrite: bool = overwrite self.environment: SilNlpEnv = environment + self.report: OnboardingReport = OnboardingReport(project_type) def get_extract_path(self) -> Path | None: if self.extract_file is not None: @@ -59,7 +113,7 @@ def extract_corpora_wrapper(self, extract_config: dict) -> None: LOGGER.info(f"Extracting corpora for project '{self.project_name}'") versification_error_output_path = Path(self.output_folder / f"versification_errors_{self.project_name}.txt") - extract_path = extract_corpora( + extract_path, detected_versification, versification_error_count, terms_count = extract_corpora( projects={self.project_name}, include_markers=extract_config.get("markers", False), extract_lemmas=extract_config.get("lemmas", False), @@ -71,6 +125,10 @@ def extract_corpora_wrapper(self, extract_config: dict) -> None: ) self.extract_file = extract_path + self.report.versification = detected_versification + self.report.versification_error_count = versification_error_count + self.report.key_terms_count = terms_count + def wildebeest_analysis_wrapper(self, wildebeest_config: dict) -> None: extract_path = self.get_extract_path() if extract_path is None: @@ -244,11 +302,11 @@ def align_wrapper( ) exp_name = f"{self.output_folder.stem}/{self.project_name}/alignments" analyze(config=align_config, exp_name=exp_name, create_summaries=True, environment=self.environment) - corpus_stats_csv = align_output_dir / "corpus_stats.csv" + corpus_stats_csv = align_output_dir / "corpus-stats.csv" if corpus_stats_csv.exists(): shutil.move( str(corpus_stats_csv), - str(self.output_folder / "corpus_stats.csv"), + str(self.output_folder / "corpus-stats.csv"), ) def check_for_project_errors(self) -> None: @@ -369,6 +427,54 @@ def rename_project(self, project_name: str, datestamp: bool) -> str: project_name = append_datestamp(project_name) return project_name + def generate_report(self) -> dict: + self.report.name_on_bucket = self.project_name + + stats_df = pd.read_csv(self.output_folder / "tokenization_stats.csv", header=[0, 1]) + + target_stats = stats_df[stats_df[(" ", "Translation Side")] == "Target"] + + self.report.mean_tokens_per_verse = target_stats[("Tokens/Verse", "Mean")].values[0] + self.report.mean_char_per_token = target_stats[("Characters/Token", "Mean")].values[0] + self.report.num_added_tokens = target_stats[(" ", "Num Tokens Added to Vocab")].values[0] + self.report.num_verses_truncated = target_stats[("Tokens/Verse", "Num Verses >= 200 Tokens")].values[0] + + verse_counts_df = pd.read_csv(self.output_folder / "verse_counts.csv") + + extract_verse_counts = verse_counts_df[verse_counts_df["file"] == self.extract_file.stem] + + self.report.verse_count = extract_verse_counts["Total"].values[0] + + settings = FileParatextProjectSettingsParser(self.environment.pt_projects_dir / self.project_name).parse() + + self.report.name = settings.full_name + self.report.short_name = settings.name + self.report.iso_code = settings.language_code + self.report.key_terms_glosses_exist = ( + self.report.iso_code in SUPPORTED_GLOSS_ISOS + or ALT_ISO.get_alternative(self.report.iso_code) in SUPPORTED_GLOSS_ISOS + ) + self.report.key_terms_type = settings.biblical_terms_list_type + + corpus_stats_csv = self.output_folder / "corpus-stats.csv" + if corpus_stats_csv.exists(): + corpus_stats_df = pd.read_csv(corpus_stats_csv) + alignment_row = corpus_stats_df[corpus_stats_df["trg_project"] == self.extract_file.stem] + if not alignment_row.empty: + self.report.alignment = alignment_row["align_score"].values[0] + self.report.script = alignment_row["trg_script"].values[0] + else: + alignment_row = corpus_stats_df[corpus_stats_df["src_project"] == self.extract_file.stem].head(1) + if not alignment_row.empty: + self.report.script = alignment_row["src_script"].values[0] + + nllb_tag = NLLB_TAG_FROM_ISO.get(self.report.iso_code, None) + self.report.lang_in_nllb = nllb_tag is not None + self.report.script = nllb_tag.split("_")[1] if nllb_tag and self.report.script == "" else self.report.script + self.report.script_in_nllb = self.report.script in NLLB_SCRIPT_SET + + return self.report.generate_dict() + class OnboardingRequest: @@ -383,13 +489,19 @@ def __init__( self.overwrite = onboarding_config.get("overwrite", False) main_project_name = onboarding_config.get("main_project", None) self.main_project: OnboardingProject = OnboardingProject( - project_name=main_project_name, overwrite=self.overwrite, environment=self.environment + project_name=main_project_name, + project_type="Request Project", + overwrite=self.overwrite, + environment=self.environment, ) reference_project_names = onboarding_config.get("ref_projects", []) self.reference_projects: List[OnboardingProject] = [] for ref_project_name in reference_project_names: reference_project = OnboardingProject( - project_name=ref_project_name, overwrite=self.overwrite, environment=self.environment + project_name=ref_project_name, + project_type="Reference Project", + overwrite=self.overwrite, + environment=self.environment, ) self.reference_projects.append(reference_project) self.no_clean: bool = onboarding_config.get("no_clean", False) @@ -437,6 +549,8 @@ def process_onboarding_request(self) -> None: if self.align: self.align_main_project() + self.create_report() + def align_main_project(self) -> None: iso_codes = set() if self.align_isos: @@ -472,6 +586,10 @@ def prepare_and_upload_projects(self) -> None: LOGGER.error(f"Error occurred while uploading reference project '{project.project_name}': {e}") LOGGER.error(f"Continuing with onboarding without reference project '{project.project_name}'.") reference_projects_to_remove.append(project) + else: + LOGGER.error(f"Error occurred while uploading main project '{project.project_name}': {e}") + LOGGER.error("Main project upload failed. Stopping onboarding process.") + raise e for project in reference_projects_to_remove: self.reference_projects.remove(project) @@ -583,6 +701,16 @@ def create_log_file(self) -> Iterator[None]: finally: close_logger(log_file_path) + def create_report(self) -> None: + LOGGER.info("Creating onboarding report.") + report_path = self.output_folder / "onboarding_report.csv" + + projects = [self.main_project] + self.reference_projects + formatted_reports = [p.generate_report() for p in projects] + + report_df = pd.DataFrame(formatted_reports) + report_df.to_csv(report_path, index=False) + def create_paratext_project_folder_if_not_exists(project_name: str, environment: SilNlpEnv) -> Path: pt_project_path = environment.get_paratext_project_dir(project_name) diff --git a/silnlp/common/paratext.py b/silnlp/common/paratext.py index bb628bb1..674f450c 100644 --- a/silnlp/common/paratext.py +++ b/silnlp/common/paratext.py @@ -441,7 +441,7 @@ def check_versification( parent_project_dir: Optional[str], versification_error_output_path: str, environment: SilNlpEnv, -) -> Tuple[bool, List[VersificationType]]: +) -> Tuple[bool, List[VersificationType], int]: parent_settings = None if parent_project_dir is not None: @@ -463,13 +463,13 @@ def check_versification( ot_versification, key_ot_verses = detect_OT_versification(project_dir, environment) if ot_versification == VersificationType.UNKNOWN: LOGGER.warning(f"Unknown versification detected for {project_dir}.") - return (matching, [ot_versification]) + return (matching, [ot_versification], 0) if check_nt: nt_versification: List[VersificationType] nt_versification, key_nt_verses = detect_NT_versification(project_dir, environment) if nt_versification[0] == VersificationType.UNKNOWN: LOGGER.warning(f"Unknown versification detected for {project_dir}.") - return (matching, nt_versification) + return (matching, nt_versification, 0) detected_versification: List[VersificationType] = [VersificationType.UNKNOWN] if check_ot and check_nt: @@ -480,7 +480,7 @@ def check_versification( f"The detected versifications were based on {', '.join(key_ot_verses + key_nt_verses)} " "being the last verse of their respective chapters." ) - return (matching, [ot_versification] + nt_versification) + return (matching, [ot_versification] + nt_versification, 0) detected_versification = [ot_versification] key_verses = key_ot_verses + key_nt_verses elif not check_ot and check_nt: @@ -495,7 +495,7 @@ def check_versification( "Versification detection for the OT requires the book of Daniel. " "Versification detection for the NT requires the books of John, Acts, and Romans." ) - return (matching, detected_versification) + return (matching, detected_versification, 0) if settings.versification.type not in detected_versification: if not ( @@ -509,7 +509,7 @@ def check_versification( f"being the last verse of {'their' if len(key_verses)>=2 else 'its'} " f"respective chapter{'s' if len(key_verses)>=2 else ''}." ) - return (matching, detected_versification) + return (matching, detected_versification, 0) errors = FileParatextProjectVersificationErrorDetector(project_dir).get_usfm_versification_errors() if len(errors) > 0: @@ -532,4 +532,4 @@ def check_versification( ) matching = True - return (matching, detected_versification) + return (matching, detected_versification, len(errors))