feat: Update docs & adding reason to comparer

Author: aliyuldashev

docs/1.validator.md

@@ -147,11 +147,7 @@ To set up a validator node on the RedTeam Subnet, follow these steps:
 
     You will be prompted to enter your Hugging Face access token. Visit [Hugging Face Access Tokens](https://huggingface.co/settings/tokens) to generate one if you don't have it already.
 
-6. Custom Setup for Specific Challenges
-
-    For setup instructions related to specific challenges, please refer to the [Validator Custom Setup](./4.validator_custom.md).
-
-7. Start the validator node:
+6. Start the validator node:
 
     ```sh
     # Activate the virtual environment if not already activated
@@ -173,15 +169,3 @@ To set up a validator node on the RedTeam Subnet, follow these steps:
 
     - `--logging.trace` - Enable trace logging
     - `--logging.debug` - Enable debug logging
-
-8. (Optional but Recommended) Start the Auto-Update Script
-
-    ```sh
-    # Activate the virtual environment if not already activated
-    source .venv/bin/activate
-
-    # Start auto-updater
-    pm2 start python --name "validator_autoupdate" \
-        -- -m scripts.validator_auto_update \
-        -- --process-name "validator_snxxx"
-    ```

docs/5.docker.md docs/4.docker.mddocs/5.docker.md renamed to docs/4.docker.md

@@ -167,7 +167,7 @@ def _compare_within_batch(self, miner_commit: MinerChallengeCommit):
 
     def _compare_outputs(
         self, miner_input: dict, miner_output: dict, reference_output: dict
-    ) -> float:
+    ) -> dict:
         """
         Send comparison request to challenge container's /compare endpoint.
 
@@ -177,7 +177,7 @@ def _compare_outputs(
             reference_output: The output from the reference miner
 
         Returns:
-            float: Comparison score between 0 and 1
+            dict: Comparison score between 0 and 1, and reason for the score
         """
         _protocol, _ssl_verify = self._check_protocol(is_challenger=True)
 
@@ -198,15 +198,19 @@ def _compare_outputs(
             response_data = response.json()
             data = response_data.get("data", {})
             similarity_score = data.get("similarity_score", 1.0)
+            similarity_reason = data.get("reason", "Unknown")
 
             # Normalize score to float between 0 and 1
             if isinstance(similarity_score, int):
                 similarity_score = float(similarity_score)
             elif not isinstance(similarity_score, float):
                 similarity_score = 1.0
 
-            return max(0.0, min(1.0, similarity_score))
+            return {
+                "similarity_score": max(0.0, min(1.0, similarity_score)),
+                "reason": similarity_reason,
+            }
 
         except Exception as e:
             bt.logging.error(f"Error in comparison request: {str(e)}")
-            return 0.0
+            return {"similarity_score": 0.0, "reason": f"Error: {str(e)}"}

docs/4.validator_custom.md

@@ -207,7 +207,7 @@ def _compare_within_batch(self, miner_commit: MinerChallengeCommit):
 
     def _compare_outputs(
         self, miner_input: dict, miner_output: dict, reference_output: dict
-    ) -> float:
+    ) -> dict:
         """
         Send comparison request to challenge container's /compare endpoint.
 
@@ -217,7 +217,7 @@ def _compare_outputs(
             reference_output: The output from the reference miner
 
         Returns:
-            float: Comparison score between 0 and 1
+            dict: Comparison score between 0 and 1, and reason for the score
         """
         _protocol, _ssl_verify = self._check_protocol(is_challenger=True)
 
@@ -235,19 +235,24 @@ def _compare_outputs(
                 json=payload,
             )
 
-            similarity_score = response.json()
+            _result = response.json().get("data", {})
+            similarity_score = _result.get("similarity_score", 0.0)
+            similarity_reason = _result.get("reason", "Unknown")
 
             # Normalize score to float between 0 and 1
             if isinstance(similarity_score, int):
                 similarity_score = float(similarity_score)
             elif not isinstance(similarity_score, float):
                 similarity_score = 0.0
 
-            return max(0.0, min(1.0, similarity_score))
+            return {
+                "similarity_score": max(0.0, min(1.0, similarity_score)),
+                "reason": similarity_reason,
+            }
 
         except Exception as e:
             bt.logging.error(f"Error in comparison request: {str(e)}")
-            return 0.0
+            return {"similarity_score": 0.0, "reason": f"Error: {str(e)}"}
 
     def _setup_challenge(self):
         """

redteam_core/challenge_pool/ab_sniffer_v3/comparer.py

@@ -164,7 +164,7 @@ def _compare_within_batch(self, miner_commit: MinerChallengeCommit):
 
     def _compare_outputs(
         self, miner_input: dict, miner_output: dict, reference_output: dict
-    ) -> float:
+    ) -> dict:
         """
         Send comparison request to challenge container's /compare endpoint.
 
@@ -174,7 +174,7 @@ def _compare_outputs(
             reference_output: The output from the reference miner
 
         Returns:
-            float: Comparison score between 0 and 1
+            dict: Comparison score between 0 and 1, and reason for the score
         """
         _protocol, _ssl_verify = self._check_protocol(is_challenger=True)
 
@@ -195,15 +195,19 @@ def _compare_outputs(
             response_data = response.json()
             data = response_data.get("data", {})
             similarity_score = data.get("similarity_score", 1.0)
+            similarity_reason = data.get("reason", "Unknown")
 
             # Normalize score to float between 0 and 1
             if isinstance(similarity_score, int):
                 similarity_score = float(similarity_score)
             elif not isinstance(similarity_score, float):
                 similarity_score = 1.0
 
-            return max(0.0, min(1.0, similarity_score))
+            return {
+                "similarity_score": max(0.0, min(1.0, similarity_score)),
+                "reason": similarity_reason,
+            }
 
         except Exception as e:
             bt.logging.error(f"Error in comparison request: {str(e)}")
-            return 0.0
+            return {"similarity_score": 0.0, "reason": f"Error: {str(e)}"}

redteam_core/challenge_pool/comparer.py

@@ -213,7 +213,7 @@ def _compare_within_batch(self, miner_commit: MinerChallengeCommit):
 
     def _compare_outputs(
         self, miner_input: dict, miner_output: dict, reference_output: dict
-    ) -> float:
+    ) -> dict:
         """
         Send comparison request to challenge container's /compare endpoint.
 
@@ -223,7 +223,7 @@ def _compare_outputs(
             reference_output: The output from the reference miner
 
         Returns:
-            float: Comparison score between 0 and 1
+            dict: Comparison score between 0 and 1, and reason for the score
         """
         _protocol, _ssl_verify = self._check_protocol(is_challenger=True)
 
@@ -244,15 +244,19 @@ def _compare_outputs(
             response_data = response.json()
             data = response_data.get("data", {})
             similarity_score = data.get("similarity_score", 1.0)
+            similarity_reason = data.get("reason", "Unknown")
 
             # Normalize score to float between 0 and 1
             if isinstance(similarity_score, int):
                 similarity_score = float(similarity_score)
             elif not isinstance(similarity_score, float):
                 similarity_score = 1.0
 
-            return max(0.0, min(1.0, similarity_score))
+            return {
+                "similarity_score": max(0.0, min(1.0, similarity_score)),
+                "reason": similarity_reason,
+            }
 
         except Exception as e:
             bt.logging.error(f"Error in comparison request: {str(e)}")
-            return 0.0
+            return {"similarity_score": 0.0, "reason": f"Error: {str(e)}"}