Move benchmarks to .codeflash/benchmarks/ and auto-discover

Move codeflash's own benchmarks to .codeflash/benchmarks/. Add
auto-discovery of .codeflash/benchmarks/ in codeflash compare and
benchmark mode -- when benchmarks-root is not explicitly configured,
the CLI checks for .codeflash/benchmarks/ before erroring.

Backwards compatible: users with existing benchmarks-root config
are unaffected. Docs continue to show tests/benchmarks as the
example path.

Author: KRRT7

tests/benchmarks/__init__.py .codeflash/benchmarks/__init__.pytests/benchmarks/__init__.py renamed to .codeflash/benchmarks/__init__.py

@@ -16,7 +16,7 @@
             "tests/",
             "-vv",
             "--ignore",
-            "tests/benchmarks/"
+            ".codeflash/benchmarks/"
         ],
     },
     "launch": {

…t_benchmark_code_extract_code_context.py …t_benchmark_code_extract_code_context.pytests/benchmarks/test_benchmark_code_extract_code_context.py renamed to .codeflash/benchmarks/test_benchmark_code_extract_code_context.py tests/benchmarks/test_benchmark_code_extract_code_context.py renamed to .codeflash/benchmarks/test_benchmark_code_extract_code_context.py

@@ -156,7 +156,14 @@ def process_pyproject_config(args: Namespace) -> Namespace:
             raise AssertionError("--tests-root must be specified")
     assert Path(args.tests_root).is_dir(), f"--tests-root {args.tests_root} must be a valid directory"
     if args.benchmark:
-        assert args.benchmarks_root is not None, "--benchmarks-root must be specified when running with --benchmark"
+        if args.benchmarks_root is None:
+            # Auto-discover .codeflash/benchmarks/ convention
+            candidate = Path.cwd() / ".codeflash" / "benchmarks"
+            if candidate.is_dir():
+                args.benchmarks_root = str(candidate)
+            else:
+                msg = "--benchmarks-root must be specified when running with --benchmark, or .codeflash/benchmarks/ must exist"
+                raise AssertionError(msg)
         assert Path(args.benchmarks_root).is_dir(), (
             f"--benchmarks-root {args.benchmarks_root} must be a valid directory"
         )

…/benchmarks/test_benchmark_comparator.py …/benchmarks/test_benchmark_comparator.pytests/benchmarks/test_benchmark_comparator.py renamed to .codeflash/benchmarks/test_benchmark_comparator.py tests/benchmarks/test_benchmark_comparator.py renamed to .codeflash/benchmarks/test_benchmark_comparator.py

@@ -87,8 +87,14 @@ def run_compare(args: Namespace) -> None:
     benchmarks_root_str = pyproject_config.get("benchmarks_root")
 
     if not benchmarks_root_str:
-        logger.error("benchmarks-root must be configured in [tool.codeflash] to use compare")
-        sys.exit(1)
+        # Auto-discover .codeflash/benchmarks/ if it exists
+        candidate = project_root / ".codeflash" / "benchmarks"
+        if candidate.is_dir():
+            benchmarks_root_str = str(candidate)
+            logger.info(f"Auto-discovered benchmarks at {candidate}")
+        else:
+            logger.error("benchmarks-root must be configured in [tool.codeflash] or .codeflash/benchmarks/ must exist")
+            sys.exit(1)
 
     benchmarks_root = Path(benchmarks_root_str).resolve()
     if not benchmarks_root.is_dir():