Migrate pipeline and services from TensorFlow to PyTorch

Replace TFAutoModel/Keras with PyTorch across the BERT predictor and qgen modules. Pin all dependencies with exact versions and add spaCy language model URLs to requirements. Add integration tests for complexity indices and vector models. Update rb submodule to matching PyTorch migration commit.

Author: stefanruseti

pipeline/management/commands/pipeline_worker.py

@@ -20,7 +20,6 @@
 
 def process(job: Job):
     import torch
-    import tensorflow as tf
     from rb import Lang
     from pipeline.tuning import hyperparameter_search
     from pipeline.parallel import build_features
@@ -84,7 +83,6 @@ def process(job: Job):
 
 def predict(job: Job):
     import torch
-    import tensorflow as tf
     from rb import Lang
     from pipeline.enums import ModelTypeEnum
     from pipeline.models import Model
@@ -149,7 +147,6 @@ def predict(job: Job):
 
 def compute_all_indices(job: Job):
     import torch
-    import tensorflow as tf
     from rb import Lang
     from pipeline.parallel import build_features
 

pipeline/migrations/0001_initial.py

@@ -6,11 +6,12 @@
 
 
 def init_model_types(apps, schema_editor):
-    from pipeline.enums import ModelTypeEnum
+    # Hardcoded to avoid importing pipeline.enums (which triggers heavy ML dependencies)
+    # Keep in sync with pipeline.enums.ModelTypeEnum
     from pipeline.models import ModelType
-    for mt in ModelTypeEnum:
+    for label in ['TRANSFORMER', 'XGBOOST']:
         obj = ModelType()
-        obj.label = mt.name
+        obj.label = label
         obj.save()
 
 def delete_model_types(apps, schema_editor):

pipeline/parallel.py

@@ -4,7 +4,6 @@
 
 import pyphen
 import torch
-import tensorflow as tf
 from rb import Document, Lang
 from rb.cna.cna_graph import CnaGraph
 from rb.complexity.complexity_index import compute_indices
@@ -15,8 +14,6 @@
 
 def build_features(text: str, lang: Lang, all_elements=False) -> Dict[str, float]:
     Logger.get_logger().setLevel(logging.WARNING)
-    tf.config.run_functions_eagerly(True)
-    tf.config.set_visible_devices([], 'GPU')
     doc = Document(lang, text)
     model = create_vector_model(lang, VectorModelType.TRANSFORMER, "")
     model.encode(doc)