commit many fixes/update: broken state

Author: clovis

api/text_pair.py

@@ -15,7 +15,11 @@
 
 
 def table_exists(user, password, table_name):
-    conn = psycopg2.connect(database=GLOBAL_CONFIG.get("DATABASE", "database_name"), user=user, password=password)
+    conn = psycopg2.connect(
+        database=GLOBAL_CONFIG.get("DATABASE", "database_name"),
+        user=user,
+        password=password,
+    )
     with conn.cursor() as cursor:
         cursor.execute("SELECT 1 FROM information_schema.tables WHERE table_name=%s", (table_name,))
         result = cursor.fetchone()
@@ -31,13 +35,13 @@ def back_up_philo_db_data(philo_db_path, output_path):
     text_path = output_path / "TEXT"
     text_path.mkdir()
     print("    - Copying TEXT files...")
-    for file in os.scandir(philo_db_path / 'data/TEXT/'):
+    for file in os.scandir(philo_db_path / "data/TEXT/"):
         shutil.copy(file.path, text_path)
 
     # Copy db related files:
     print("    - Copying database files...")
-    shutil.copy(philo_db_path / 'data/toms.db', output_path)
-    shutil.copy(philo_db_path / 'data/db.locals.py', output_path)
+    shutil.copy(philo_db_path / "data/toms.db", output_path)
+    shutil.copy(philo_db_path / "data/db.locals.py", output_path)
     print("    ✓ PhiloLogic database backup complete")
 
 
@@ -112,12 +116,12 @@ def extract_textpair_database(table, web_app_path, output_path):
 
         print("  - Compressing with LZ4...")
         # Read the tar file and compress with lz4
-        with open(temp_tar, 'rb') as f:
+        with open(temp_tar, "rb") as f:
             tar_data = f.read()
         compressed_data = lz4.frame.compress(tar_data, compression_level=3)
 
         # Write the compressed data
-        with open(tar_path, 'wb') as f:
+        with open(tar_path, "wb") as f:
             f.write(compressed_data)
 
         # Clean up temporary tar file
@@ -144,4 +148,4 @@ def extract_textpair_database(table, web_app_path, output_path):
     output_path = args.output_path or os.getcwd()
     web_app_path = args.web_app_path or os.path.join(GLOBAL_CONFIG.get("WEB_APP", "web_app_path"), args.db_name)
     web_app_path = web_app_path.rstrip("/")
-    extract_textpair_database(args.db_name, web_app_path, output_path)
+    extract_textpair_database(args.db_name, web_app_path, output_path)

extras/backup_database.py

@@ -21,7 +21,7 @@ def check_database_connection(user, password):
         conn = psycopg2.connect(
             database=GLOBAL_CONFIG.get("DATABASE", "database_name"),
             user=user,
-            password=password
+            password=password,
         )
         conn.close()
         return True
@@ -48,28 +48,28 @@ def update_app_config(web_app_path):
 
         # Update the apiServer value
         api_server = GLOBAL_CONFIG.get("WEB_APP", "api_server")
-        config['apiServer'] = api_server
+        config["apiServer"] = api_server
 
         # Update PhiloLogic paths to point to the restored data
         source_data_path = web_app_path / "source_data"
         if source_data_path.exists():
-            config['sourcePhiloDBPath'] = str(source_data_path.absolute())
+            config["sourcePhiloDBPath"] = str(source_data_path.absolute())
 
         target_data_path = web_app_path / "target_data"
         if target_data_path.exists():
-            config['targetPhiloDBPath'] = str(target_data_path.absolute())
-        elif 'targetPhiloDBPath' in config:
+            config["targetPhiloDBPath"] = str(target_data_path.absolute())
+        elif "targetPhiloDBPath" in config:
             # If target_data doesn't exist and there was a target path, remove it
-            config['targetPhiloDBPath'] = ""
+            config["targetPhiloDBPath"] = ""
 
         # Write the updated config back
-        with open(config_path, 'w') as f:
+        with open(config_path, "w") as f:
             json.dump(config, f, indent=2)
 
         print(f"Updated appConfig.json:")
         print(f"  - apiServer: {api_server}")
         print(f"  - sourcePhiloDBPath: {config['sourcePhiloDBPath']}")
-        if config.get('targetPhiloDBPath'):
+        if config.get("targetPhiloDBPath"):
             print(f"  - targetPhiloDBPath: {config['targetPhiloDBPath']}")
 
         return True
@@ -91,11 +91,11 @@ def run_npm_build(web_app_path):
 
         # Run npm install
         print("Running npm install...")
-        subprocess.run(['npm', 'install'], check=True)
+        subprocess.run(["npm", "install"], check=True)
 
         # Run npm build
         print("Running npm run build...")
-        subprocess.run(['npm', 'run', 'build'], check=True)
+        subprocess.run(["npm", "run", "build"], check=True)
 
         return True
 
@@ -117,12 +117,12 @@ def check_existing_resources(db_name, db_user, db_password, web_app_dest, backup
     # Check for existing tables
     sql_files = list(backup_dir.glob("textpair_*.sql"))
     for sql_file in sql_files:
-        table_name = sql_file.stem.replace('textpair_', '')
+        table_name = sql_file.stem.replace("textpair_", "")
         with psycopg2.connect(database=db_name, user=db_user, password=db_password) as conn:
             with conn.cursor() as cursor:
                 cursor.execute(
                     "SELECT 1 FROM information_schema.tables WHERE table_name = %s",
-                    (table_name,)
+                    (table_name,),
                 )
                 if cursor.fetchone() is not None:
                     existing_resources.append(f"database table '{table_name}'")
@@ -176,12 +176,12 @@ def restore_textpair_database(backup_path, web_app_dest=None, force=False):
         # Extract the tarball using lz4 module
         print("\nExtracting backup archive...")
         print("  - Decompressing with LZ4...")
-        with open(backup_path, 'rb') as f:
+        with open(backup_path, "rb") as f:
             compressed_data = f.read()
         decompressed_data = lz4.frame.decompress(compressed_data)
         print("  - Extracting files...")
         temp_tar = temp_dir / "temp.tar"
-        with open(temp_tar, 'wb') as f:
+        with open(temp_tar, "wb") as f:
             f.write(decompressed_data)
         os.system(f"tar xf {temp_tar} -C {temp_dir}")
         os.remove(temp_tar)
@@ -209,8 +209,10 @@ def restore_textpair_database(backup_path, web_app_dest=None, force=False):
                 print("\nWARNING: The following resources will be overwritten:")
                 for resource in existing:
                     print(f"  - {resource}")
-                response = input("\nDo you want to proceed with the restoration? This will replace all existing resources (y/n): ")
-                if response.lower() != 'y':
+                response = input(
+                    "\nDo you want to proceed with the restoration? This will replace all existing resources (y/n): "
+                )
+                if response.lower() != "y":
                     print("Restoration cancelled")
                     return
                 print("")  # Empty line for better readability
@@ -224,7 +226,7 @@ def restore_textpair_database(backup_path, web_app_dest=None, force=False):
         print(f"Found {len(sql_files)} tables to restore")
 
         for sql_file in sql_files:
-            table_name = sql_file.stem.replace('textpair_', '')
+            table_name = sql_file.stem.replace("textpair_", "")
 
             # Drop existing table if it exists
             print(f"  - Processing {table_name}:")
@@ -236,7 +238,7 @@ def restore_textpair_database(backup_path, web_app_dest=None, force=False):
 
             # Restore table
             print(f"    • Restoring table data...")
-            os.system(f'PGPASSWORD={db_password} psql -U {db_user} -d {db_name} -f {sql_file}')
+            os.system(f"PGPASSWORD={db_password} psql -U {db_user} -d {db_name} -f {sql_file}")
             print(f"    ✓ Table {table_name} restored")
 
         print("✓ Database restoration complete")
@@ -290,10 +292,17 @@ def restore_textpair_database(backup_path, web_app_dest=None, force=False):
 if __name__ == "__main__":
     parser = ArgumentParser()
     parser.add_argument("backup_path", type=str, help="Path to the backup tarball file")
-    parser.add_argument("--web_app_dest", type=str, default="",
-                      help="Optional destination path for web app files")
-    parser.add_argument("--force", action="store_true",
-                      help="Overwrite existing files/tables without prompting")
+    parser.add_argument(
+        "--web_app_dest",
+        type=str,
+        default="",
+        help="Optional destination path for web app files",
+    )
+    parser.add_argument(
+        "--force",
+        action="store_true",
+        help="Overwrite existing files/tables without prompting",
+    )
     args = parser.parse_args()
 
-    restore_textpair_database(args.backup_path, args.web_app_dest, args.force)
+    restore_textpair_database(args.backup_path, args.web_app_dest, args.force)

extras/restore_database.py

@@ -1,7 +1,13 @@
 """Global imports for main textpair function"""
+
 from .parse_config import get_config
 from .passage_classifier import classify_passages
-from .sequence_alignment import Ngrams, banality_auto_detect, merge_alignments, phrase_matcher
+from .sequence_alignment import (
+    Ngrams,
+    banality_auto_detect,
+    merge_alignments,
+    phrase_matcher,
+)
 from .text_parser import parse_files
 from .utils import get_text
 from .vector_space_alignment import run_vsa

lib/textpair/__init__.py

@@ -34,14 +34,25 @@ def build_graph_and_labels(alignments_file: str, embedding_model: str, llm_param
     # Run graph building in separate environment
     try:
         result = subprocess.run(
-            [graph_python, "-m", "textpair_graph", "build",
-             alignments_file, output_dir, "--model", embedding_model],
+            [
+                graph_python,
+                "-m",
+                "textpair_graph",
+                "build",
+                alignments_file,
+                output_dir,
+                "--model",
+                embedding_model,
+            ],
             check=True,
-            capture_output=False
+            capture_output=False,
         )
         print("✓ Graph model built successfully!")
     except subprocess.CalledProcessError as e:
-        print(f"ERROR: Graph model generation failed with exit code {e.returncode}", file=sys.stderr)
+        print(
+            f"ERROR: Graph model generation failed with exit code {e.returncode}",
+            file=sys.stderr,
+        )
         return
     except FileNotFoundError:
         print(f"ERROR: Graph environment not found at {graph_python}", file=sys.stderr)
@@ -55,15 +66,27 @@ def build_graph_and_labels(alignments_file: str, embedding_model: str, llm_param
         if os.path.exists(graph_data_path):
             try:
                 result = subprocess.run(
-                    [graph_python, "-m", "textpair_graph", "label",
-                     graph_data_path, "--model", llm_params["llm_model"],
-                     "--context_window", str(llm_params["llm_context_window"]),
-                     "--port", str(llm_params["llm_port"])],
+                    [
+                        graph_python,
+                        "-m",
+                        "textpair_graph",
+                        "label",
+                        graph_data_path,
+                        "--model",
+                        llm_params["llm_model"],
+                        "--context_window",
+                        str(llm_params["llm_context_window"]),
+                        "--port",
+                        str(llm_params["llm_port"]),
+                    ],
                     check=True,
-                    capture_output=False
+                    capture_output=False,
                 )
             except subprocess.CalledProcessError as e:
-                print(f"WARNING: Cluster labeling failed with exit code {e.returncode}", file=sys.stderr)
+                print(
+                    f"WARNING: Cluster labeling failed with exit code {e.returncode}",
+                    file=sys.stderr,
+                )
 
 
 def delete_database(dbname: str) -> None:
@@ -200,14 +223,24 @@ async def run_alignment(params):
         filename = os.listdir(result_batch_path)[0]
         os.system(f"mv {result_batch_path}/{filename} {results_file} && rm -rf {result_batch_path}")
     else:
-        print("Merging alignments into one file (this may take a while)... ", end="", flush=True)
+        print(
+            "Merging alignments into one file (this may take a while)... ",
+            end="",
+            flush=True,
+        )
         merge_command = f"find {result_batch_path} -type f | sort -V | xargs lz4cat --rm | lz4 -q > {results_file}; rm -rf {result_batch_path}"
         os.system(merge_command)
         print("done.")
     count = get_count(os.path.join(params.output_path, "results/count.txt"))
 
     # Postprocessing steps
-    if any([params.matching_params["phrase_filter"], params.matching_params["banality_auto_detection"], params.matching_params["banality_llm_eval"]]):
+    if any(
+        [
+            params.matching_params["phrase_filter"],
+            params.matching_params["banality_auto_detection"],
+            params.matching_params["banality_llm_eval"],
+        ]
+    ):
         print(f"\n### Postprocessing {count} pairwise alignments ###")
         if params.matching_params["phrase_filter"]:
             filtered_passages = phrase_matcher(results_file, params.matching_params["phrase_filter"], count)
@@ -219,7 +252,7 @@ async def run_alignment(params):
             banalities_found = banality_auto_detect(
                 results_file,
                 params.paths["source"]["common_ngrams"],
-                f'{params.paths["source"]["ngram_output_path"]}/ngrams_in_order',
+                f"{params.paths['source']['ngram_output_path']}/ngrams_in_order",
                 params.matching_params["store_banalities"],
                 count,
                 params.matching_params["most_common_ngram_proportion"],
@@ -261,7 +294,7 @@ async def run_alignment(params):
             params.passage_classification["classes"],
             min_confidence=0.3,
             top_k=3,
-            batch_size=32
+            batch_size=32,
         )
 
     # Passage merger
@@ -293,7 +326,6 @@ async def run_alignment(params):
         )
 
 
-
 async def run_vsa_similarity(params) -> None:
     """Run vsa similarity"""
     if params.paths["target"]["ngram_output_path"] == "":  # if path not defined make target like source
@@ -331,7 +363,7 @@ async def run_vsa_similarity(params) -> None:
         params.workers,
         {**params.preprocessing_params, **params.matching_params},
         params.output_path,
-        params.llm_params
+        params.llm_params,
     )
 
     # Passage classification (if enabled)
@@ -344,7 +376,7 @@ async def run_vsa_similarity(params) -> None:
             params.passage_classification["classes"],
             min_confidence=0.5,
             top_k=3,
-            batch_size=32
+            batch_size=32,
         )
 
     if params.web_app_config["skip_web_app"] is False:
@@ -368,7 +400,7 @@ async def run_vsa_similarity(params) -> None:
             params.web_app_config["source_philo_db_path"],
             params.web_app_config["target_philo_db_path"],
             params.matching_params["matching_algorithm"],
-            params
+            params,
         )
 
 
@@ -432,4 +464,5 @@ async def main():
 
 if __name__ == "__main__":
     import asyncio
+
     asyncio.run(main())