Merge pull request #114 from pavi2707/main

Add model testing results to pytest xml report

Author: JRosenkranz

tests/models/test_decoders.py

@@ -473,7 +473,12 @@ def persistent_model():
     "model_path,batch_size,seq_length,max_new_tokens", common_shapes
 )
 def test_common_shapes(
-    model_path, batch_size, seq_length, max_new_tokens, persistent_model
+    model_path,
+    batch_size,
+    seq_length,
+    max_new_tokens,
+    persistent_model,
+    record_property,
 ):
     torch.manual_seed(42)
     torch.set_grad_enabled(False)
@@ -735,6 +740,9 @@ def _metric_calculator(r: torch.Tensor, t: torch.Tensor):
         ce_failure_rate = len(ce_fail_responses_list) / total_tokens
         dprint(f"mean diff failure rate: {diff_failure_rate}")
         dprint(f"cross entropy loss failure rate: {ce_failure_rate}")
+        # Add failure rates to xml report
+        record_property("mean_diff_failure_rate", diff_failure_rate)
+        record_property("cross_entropy_loss_failure_rate", ce_failure_rate)
         if "mean_diff" not in skip_assertions:
             assert diff_failure_rate < failure_rate_threshold, (
                 f"failure rate for mean diff was too high: {diff_failure_rate}"

tests/models/test_encoders.py

@@ -104,7 +104,7 @@ def reset_compiler():
 
 
 @pytest.mark.parametrize("model_path,batch_size,seq_length", common_shapes)
-def test_common_shapes(model_path, batch_size, seq_length):
+def test_common_shapes(model_path, batch_size, seq_length, record_property):
     os.environ["COMPILATION_MODE"] = "offline"
 
     dprint(
@@ -187,5 +187,7 @@ def test_common_shapes(model_path, batch_size, seq_length):
 
     abs_mean_diff = sum(diffs) / len(diffs)
     print(f"absolute mean diff: {abs_mean_diff}")
+    # Add value to xml report
+    record_property("absolute_mean_diff", float(abs_mean_diff))
 
     assert abs_mean_diff < validation_diff_threshold