Skip to content

Commit 3308446

Browse files
sjarmakclaude
andcommitted
Remove bustub-hyperloglog-impl-001 from active selection + fix teleport instruction
bustub requires TAC-specific infrastructure (DinD, proprietary eval.py, external server init) incompatible with our standard pipeline. Feature suite drops to 22 tasks, matching the Neyman-optimal allocation. Also fix bare $DEVICE_NAME shell variable in teleport instruction.md (replace with <device-name> placeholder). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 726f746 commit 3308446

File tree

2 files changed

+2
-106
lines changed

2 files changed

+2
-106
lines changed

benchmarks/csb_sdlc_fix/teleport-users-can-delete-fix-001/instruction.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ Deletion of a user's last MFA device should be prevented when the security polic
1818
## Reproduction Steps
1919
1.Set `second_factor: on` on the `auth_service`
2020
2.Create a user with 1 MFA device
21-
3.Run `tsh mfa rm $DEVICE_NAME`
21+
3.Run `tsh mfa rm <device-name>`
2222

2323
## Bug details
2424
- Teleport version: v6.0.0-rc.1

configs/selected_benchmark_tasks.json

Lines changed: 1 addition & 105 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
"csb_sdlc_debug": 19,
2727
"csb_sdlc_design": 15,
2828
"csb_sdlc_document": 15,
29-
"csb_sdlc_feature": 23,
29+
"csb_sdlc_feature": 22,
3030
"csb_sdlc_fix": 34,
3131
"csb_sdlc_refactor": 18,
3232
"csb_sdlc_secure": 15,
@@ -836,110 +836,6 @@
836836
"suite": "csb_sdlc_refactor",
837837
"loc_band": "2M-8M"
838838
},
839-
{
840-
"task_id": "bustub-hyperloglog-impl-001",
841-
"benchmark": "csb_sdlc_feature",
842-
"sdlc_phase": "Implementation (feature)",
843-
"language": "cpp",
844-
"difficulty": "hard",
845-
"category": "implement",
846-
"repo": "cmu-db/bustub",
847-
"mcp_benefit_score": 0.49,
848-
"mcp_breakdown": {
849-
"context_complexity": 0.5,
850-
"cross_file_deps": 0.3,
851-
"semantic_search_potential": 0.5,
852-
"task_category_weight": 0.7
853-
},
854-
"selection_rationale": "All ccb_tac tasks selected (small benchmark)",
855-
"task_dir": "csb_sdlc_feature/bustub-hyperloglog-impl-001",
856-
"context_length": 500000,
857-
"context_length_source": "task_metrics_run",
858-
"files_count": 6,
859-
"files_count_source": "task_metrics_run",
860-
"repo_size_bytes": 10676224,
861-
"repo_size_mb": 10.182,
862-
"repo_file_count": 1248,
863-
"repo_directory_count": 191,
864-
"repo_approx_loc": 262071,
865-
"repo_languages": [
866-
{
867-
"language": "cpp",
868-
"files": 450,
869-
"pct_files": 36.06
870-
},
871-
{
872-
"language": "c_cpp_headers",
873-
"files": 297,
874-
"pct_files": 23.8
875-
},
876-
{
877-
"language": "c",
878-
"files": 51,
879-
"pct_files": 4.09
880-
},
881-
{
882-
"language": "python",
883-
"files": 44,
884-
"pct_files": 3.53
885-
},
886-
{
887-
"language": "shell",
888-
"files": 8,
889-
"pct_files": 0.64
890-
},
891-
{
892-
"language": "makefile",
893-
"files": 7,
894-
"pct_files": 0.56
895-
},
896-
{
897-
"language": "javascript",
898-
"files": 1,
899-
"pct_files": 0.08
900-
},
901-
{
902-
"language": "ruby",
903-
"files": 1,
904-
"pct_files": 0.08
905-
}
906-
],
907-
"repo_primary_language": "cpp",
908-
"repo_complexity": 0.67,
909-
"repo_complexity_label": "hard",
910-
"repo_complexity_source": "cached_registry",
911-
"task_complexity": 0.391,
912-
"task_complexity_label": "medium",
913-
"task_complexity_source": "ground_truth_meta_plus_registry",
914-
"repo_approx_loc_source": "cloc",
915-
"repo_cloc_total_files": 1114,
916-
"repo_cloc_top_languages": [
917-
{
918-
"language": "C++",
919-
"code_lines": 109869
920-
},
921-
{
922-
"language": "C/C++ Header",
923-
"code_lines": 70314
924-
},
925-
{
926-
"language": "C",
927-
"code_lines": 34476
928-
},
929-
{
930-
"language": "Markdown",
931-
"code_lines": 21089
932-
},
933-
{
934-
"language": "Python",
935-
"code_lines": 8280
936-
}
937-
],
938-
"n_repos": 1,
939-
"task_type": "implementation",
940-
"suite": "csb_sdlc_feature",
941-
"loc_band": "<400K"
942-
},
943839
{
944840
"task_id": "calcom-code-review-001",
945841
"benchmark": "csb_sdlc_test",

0 commit comments

Comments
 (0)