Merge branch 'develop' of https://github.com/ReversecLabs/spikee into spikee/develop

Author: ThomasCross

spikee/attacks/echo_chamber.py

@@ -1082,7 +1082,6 @@ def attack(
                     last_message_id = sentences_message_id
                     continue
 
-
             return (
                 conversation.get_attempt_total(),
                 False,

spikee/cli.py

@@ -100,7 +100,7 @@ def main():
 
     subparsers = parser.add_subparsers(dest="command", help="Sub-commands")
 
-    #region === [INIT] Sub-command (NEW) ==============================================
+    # region === [INIT] Sub-command (NEW) ==============================================
     parser_init = subparsers.add_parser(
         "init", help="Initialize a local SPIKEE workspace"
     )
@@ -120,9 +120,9 @@ def main():
         action="store_true",
         help="Include the built-in web viewer in the local workspace",
     )
-    #endregion
+    # endregion
 
-    #region === [GENERATE] Sub-command ===============================================
+    # region === [GENERATE] Sub-command ===============================================
     parser_generate = subparsers.add_parser("generate", help="Generate a dataset")
     subparsers_generate = parser_generate.add_subparsers(
         dest="generate_command", help="Generation sub-commands"
@@ -261,9 +261,9 @@ def main():
         default=None,
         help='Plugin-specific options as "plugin1:option1,option2;plugin2:option2"',
     )
-    #endregion
+    # endregion
 
-    #region === [TEST] Sub-command ===================================================
+    # region === [TEST] Sub-command ===================================================
     parser_test = subparsers.add_parser(
         "test", help="Test the dataset against a target"
     )
@@ -373,9 +373,9 @@ def main():
         action="store_true",
         help="Create new results file, do not attempt to resume",
     )
-    #endregion
+    # endregion
 
-    #region === [RESULTS] Sub-command ================================================
+    # region === [RESULTS] Sub-command ================================================
     parser_results = subparsers.add_parser("results", help="Analyze or convert results")
     subparsers_results = parser_results.add_subparsers(
         dest="results_command", help="Results sub-commands"
@@ -526,9 +526,9 @@ def main():
     parser_dataset_comparison.add_argument(
         "--tag", default=None, help="Include a tag at the end of the results filename"
     )
-    #endregion
+    # endregion
 
-    #region === [Viewer] Sub-command ================================================
+    # region === [Viewer] Sub-command ================================================
     parser_viewer = subparsers.add_parser("viewer", help="Launch local web viewers")
     subparsers_viewer = parser_viewer.add_subparsers(
         dest="viewer_command", help="Viewer sub-commands"
@@ -587,9 +587,9 @@ def main():
     parser_convert_to_excel.add_argument(
         "--result-file", type=str, required=True, help="Path to the results JSONL file"
     )
-    #endregion
+    # endregion
 
-    #region === [LIST] Sub-command ================================================
+    # region === [LIST] Sub-command ================================================
     parser_list = subparsers.add_parser(
         "list",
         help="List seeds, datasets, judges, targets, plugins, attacks, or providers",
@@ -614,38 +614,60 @@ def main():
             action="store_true",
             help="Include descriptions of modules where available",
         )
-    #endregion
+    # endregion
 
-    #region === [DEBUG] Sub-command ================================================
-    parser_debug = subparsers.add_parser("debug", help="Debugging and testing utilities")
+    # region === [DEBUG] Sub-command ================================================
+    parser_debug = subparsers.add_parser(
+        "debug", help="Debugging and testing utilities"
+    )
+
+    debug_subparsers = parser_debug.add_subparsers(
+        dest="debug_command", help="Debugging sub-commands"
+    )
 
-    debug_subparsers = parser_debug.add_subparsers(dest="debug_command", help="Debugging sub-commands")
-    
     # -- module debug --
     module_subparser = debug_subparsers.add_parser("module", help="Debug module")
-    module_subparsers = module_subparser.add_subparsers(dest="module_type", help="Type of module to debug")
+    module_subparsers = module_subparser.add_subparsers(
+        dest="module_type", help="Type of module to debug"
+    )
 
-    target_subparser = module_subparsers.add_parser("targets", help="Debug target module")
+    target_subparser = module_subparsers.add_parser(
+        "targets", help="Debug target module"
+    )
     judge_subparser = module_subparsers.add_parser("judges", help="Debug judge module")
-    plugin_subparser = module_subparsers.add_parser("plugins", help="Debug plugin module")
-    attack_subparser = module_subparsers.add_parser("attacks", help="Debug attack module")
-    provider_subparser = module_subparsers.add_parser("providers", help="Debug provider module")
+    plugin_subparser = module_subparsers.add_parser(
+        "plugins", help="Debug plugin module"
+    )
+    attack_subparser = module_subparsers.add_parser(
+        "attacks", help="Debug attack module"
+    )
+    provider_subparser = module_subparsers.add_parser(
+        "providers", help="Debug provider module"
+    )
 
-    for subparser in [target_subparser, judge_subparser, plugin_subparser, attack_subparser, provider_subparser]:
+    for subparser in [
+        target_subparser,
+        judge_subparser,
+        plugin_subparser,
+        attack_subparser,
+        provider_subparser,
+    ]:
         subparser.add_argument(
-            "-m", "--module",
+            "-m",
+            "--module",
             type=str,
             required=True,
             help="Name of the module to debug",
         )
         subparser.add_argument(
-            "-i", "--input",
+            "-i",
+            "--input",
             type=str,
             default="",
             required=True,
             help="Input string for the module (if applicable)",
         )
-    
+
     # Target-specific arguments
     target_subparser.add_argument(
         "--system-message",
@@ -674,7 +696,8 @@ def main():
         help="Additional arguments for the judge module (if applicable)",
     )
     judge_subparser.add_argument(
-        "-o", "--output",
+        "-o",
+        "--output",
         type=str,
         default="",
         required=True,
@@ -715,7 +738,7 @@ def main():
         default=None,
         help="Options to pass to the attack module (if applicable)",
     )
-    
+
     # Provider-specific arguments
     provider_subparser.add_argument(
         "--max-tokens",
@@ -730,7 +753,7 @@ def main():
         help="Temperature for the provider module (if applicable)",
     )
 
-    #endregion
+    # endregion
 
     args = convert_to_new_args(parser.parse_args())
 
@@ -744,8 +767,11 @@ def main():
     if args.command == "init":
         # Guard against running init inside the spikee source package directory
         cwd = Path(os.getcwd())
-        if (cwd / "cli.py").exists() and (cwd / "tester.py").exists()  and (cwd / "generator.py").exists():
-
+        if (
+            (cwd / "cli.py").exists()
+            and (cwd / "tester.py").exists()
+            and (cwd / "generator.py").exists()
+        ):
             if not args.force:
                 print(
                     "[init] ERROR: It looks like you are running 'spikee init' within the Spikee sourcecode directory (e.g. ./spikee/)."
@@ -759,7 +785,7 @@ def main():
                 print("[init]   spikee init\n")
                 print("[init] Use --force to override warnings.")
                 sys.exit(1)
-            
+
             else:
                 print(
                     "[init] WARNING: You are running 'spikee init' inside the Spikee sourcecode directory with --force. This will overwrite Spikee's source code with workspace files. Proceeding with initialization..."
@@ -816,7 +842,7 @@ def main():
             list_providers(args)
         else:
             parser_list.print_help()
-    
+
     elif args.command == "debug":
         if args.debug_command == "module":
             if args.module_type == "targets":
@@ -954,4 +980,4 @@ def copy_builtin_modules(include_option, force=False):
                 print(f"[init] No built-in {module_type} were copied")
 
         except Exception as e:
-            print(f"[init] Error processing {module_type}: {e}")
+            print(f"[init] Error processing {module_type}: {e}")

spikee/data/workspace/providers/sample_provider.py

@@ -50,9 +50,7 @@ def setup(
     def get_description(self) -> ModuleDescriptionHint:
         return [ModuleTag.LLM], "Sample provider for OpenAI API based AnyLLM providers."
 
-    def _invoke(
-        self, messages: MessageHint
-    ) -> AIMessage:
+    def _invoke(self, messages: MessageHint) -> AIMessage:
         """Return Mock Message"""
 
         # upgraded_messages = agent_framework_message_translation(upgrade_messages(messages))

spikee/debug.py

@@ -22,6 +22,7 @@
 from spikee.judge import call_judge
 from spikee.utilities.llm import get_llm
 
+
 def debug_module_target(args):
     """
     Debug a target module by sending a single input and printing the response.
@@ -41,13 +42,16 @@ def debug_module_target(args):
 
     if args.system_message:
         target_args["system_message"] = args.system_message
-    
+
     if args.target_options:
         target_args["target_options"] = args.target_options
 
     response = target.process_input(args.input, **target_args)
 
-    print(f"[{target.__class__.__name__}] Response: {get_content(response) if response else 'No response'}")
+    print(
+        f"[{target.__class__.__name__}] Response: {get_content(response) if response else 'No response'}"
+    )
+
 
 def debug_module_judge(args):
     """
@@ -72,7 +76,7 @@ def debug_module_judge(args):
 
     if args.output:
         judge_args["llm_output"] = args.output
-    
+
     else:
         raise ValueError("Judges require an output argument to evaluate the response.")
 
@@ -81,11 +85,12 @@ def debug_module_judge(args):
 
     if args.judge_args:
         judge_args["judge_args"] = args.judge_args
-    
+
     response = judge.judge(args.input, **judge_args)
 
     print(f"[{judge.__class__.__name__}] Judge Result: {response}")
 
+
 def debug_module_plugin(args):
     """
     Debug a plugin module by transforming an input and printing the result.
@@ -112,7 +117,10 @@ def debug_module_plugin(args):
 
     response = plugin.transform(args.input, **plugin_args)
 
-    print(f"[{plugin.__class__.__name__}] Plugin Response: {get_content(response) if response else 'No response'}")
+    print(
+        f"[{plugin.__class__.__name__}] Plugin Response: {get_content(response) if response else 'No response'}"
+    )
+
 
 def debug_module_attack(args):
     """
@@ -151,14 +159,16 @@ def debug_module_attack(args):
         entry = json.loads(base64.b64decode(args.input).decode("utf-8"))
     except (json.JSONDecodeError, base64.binascii.Error):
         raise ValueError("Attack input must be a valid base64-encoded JSON entry.")
-    
+
     missing_entry_args = []
     for arg in ["id", "long_id", "content", "content_type", "judge_name", "judge_args"]:
         if arg not in entry:
             missing_entry_args.append(arg)
-    
+
     if len(missing_entry_args) > 0:
-        raise ValueError(f"Missing required entry arguments for attack: {', '.join(missing_entry_args)}")
+        raise ValueError(
+            f"Missing required entry arguments for attack: {', '.join(missing_entry_args)}"
+        )
 
     attack_args = {}
 
@@ -173,7 +183,10 @@ def debug_module_attack(args):
 
     response = attack.attack(entry, **attack_args)
 
-    print(f"[{attack.__class__.__name__}] Attack Response: {str(response) if response else 'No response'}")
+    print(
+        f"[{attack.__class__.__name__}] Attack Response: {str(response) if response else 'No response'}"
+    )
+
 
 def debug_module_provider(args):
     """
@@ -195,7 +208,9 @@ def debug_module_provider(args):
     provider_args = {}
 
     if args.module is None:
-        raise ValueError("Providers require a model argument to specify the LLM model to use.")
+        raise ValueError(
+            "Providers require a model argument to specify the LLM model to use."
+        )
 
     provider_args["max_tokens"] = args.max_tokens
     provider_args["temperature"] = args.temperature
@@ -206,11 +221,14 @@ def debug_module_provider(args):
     )
 
     if provider is None:
-        raise ValueError("No provider could be loaded. Please check the module name and ensure it is a valid LLM provider.")
+        raise ValueError(
+            "No provider could be loaded. Please check the module name and ensure it is a valid LLM provider."
+        )
 
     response = provider.invoke(args.input)
 
     response = response.content
 
-    print(f"[{provider.__class__.__name__}] Provider Response: {get_content(response) if response else 'No response'}")
-
+    print(
+        f"[{provider.__class__.__name__}] Provider Response: {get_content(response) if response else 'No response'}"
+    )

spikee/plugins/homoglyph.py

@@ -125,7 +125,9 @@ def plugin_transform(self, text: str, plugin_option: str = "") -> str:
 
         rng = random.Random(seed)
         return "".join(
-            self.CONFUSABLES[c] if (c in self.CONFUSABLES and rng.random() < ratio) else c
+            self.CONFUSABLES[c]
+            if (c in self.CONFUSABLES and rng.random() < ratio)
+            else c
             for c in text
         )
 

spikee/providers/aws_transcribe_stt.py

@@ -160,9 +160,7 @@ async def _write_chunks():
 
         return " ".join(transcript_parts).strip()
 
-    def _invoke(
-        self, messages: MessageHint
-    ) -> AIMessage:
+    def _invoke(self, messages: MessageHint) -> AIMessage:
         """Invoke AWS Transcribe streaming STT with base64-encoded audio. Returns transcribed text."""
 
         msg, _ = single_message(messages)

spikee/providers/azure_openai.py

@@ -62,9 +62,7 @@ def setup(
     def get_description(self) -> ModuleDescriptionHint:
         return [ModuleTag.LLM], "LLM Provider for Azure OpenAI models via any-llm."
 
-    def _invoke(
-        self, messages: MessageHint
-    ) -> AIMessage:
+    def _invoke(self, messages: MessageHint) -> AIMessage:
         """Invoke AnyLLM Azure OpenAI LLM with the provided messages."""
 
         formatted_messages = format_messages(messages)

spikee/providers/bedrock.py

@@ -112,9 +112,7 @@ def setup(
     def get_description(self) -> ModuleDescriptionHint:
         return [ModuleTag.LLM], "LLM Provider for AWS Bedrock models via any-llm."
 
-    def _invoke(
-        self, messages: MessageHint
-    ) -> AIMessage:
+    def _invoke(self, messages: MessageHint) -> AIMessage:
         """Invoke AnyLLM Bedrock LLM with the provided messages."""
 
         formatted_messages = format_messages(messages)