fix: replace git fetch --depth 1 origin <sha> with full clone + checkout

sjarmak · claude · sjarmak · commit 3dd763247490 · 2026-02-21T02:00:53.000Z
GitHub doesn't serve arbitrary commit SHAs via shallow fetch (git fetch
--depth 1 origin &lt;sha&gt;). This fails with "upload-pack: not our ref".

Replace all 14 affected Dockerfiles (7 standard + 7 artifact_only) to
use: git clone &lt;url&gt; + git checkout &lt;sha&gt;

Also extends run_selected_tasks.sh to support artifact configs:
- Add --full-config flag (default: mcp-remote-direct)
- Derive BASELINE_CONFIG using baseline_config_for() from _common.sh
- Add Dockerfile.artifact_only swap/restore in run_benchmark() for *-artifact configs
- Update main loop to use dynamic config vars

Co-Authored-By: Claude Sonnet 4.6 &lt;noreply@anthropic.com&gt;
diff --git a/benchmarks/ccb_mcp_crossrepo_tracing/ccx-config-trace-010/environment/Dockerfile b/benchmarks/ccb_mcp_crossrepo_tracing/ccx-config-trace-010/environment/Dockerfile
@@ -15,8 +15,8 @@ WORKDIR /workspace
 
 # Clone local checkout repos (baseline config: agent has local access to these)
 RUN git clone --depth 1 --branch v1.32.0 https://github.com/kubernetes/kubernetes /workspace/kubernetes
-RUN git clone https://github.com/kubernetes/client-go /workspace/client-go --no-checkout && \
-    cd /workspace/client-go && git fetch --depth 1 origin 8020fc4fcf89965904a5f43689f169d6e01d1e80 && git checkout FETCH_HEAD
+RUN git clone https://github.com/kubernetes/client-go /workspace/client-go && \
+    cd /workspace/client-go && git checkout 8020fc4fcf89965904a5f43689f169d6e01d1e80
 
 # Initialize git identity for agent commits
 RUN git config --global user.email "agent@example.com" && \
diff --git a/benchmarks/ccb_mcp_crossrepo_tracing/ccx-config-trace-010/environment/Dockerfile.artifact_only b/benchmarks/ccb_mcp_crossrepo_tracing/ccx-config-trace-010/environment/Dockerfile.artifact_only
@@ -20,8 +20,8 @@ WORKDIR /workspace
 
 # Clone local checkout repos (baseline config: agent has local access to these)
 RUN git clone --depth 1 --branch v1.32.0 https://github.com/kubernetes/kubernetes /workspace/kubernetes
-RUN git clone https://github.com/kubernetes/client-go /workspace/client-go --no-checkout && \
-    cd /workspace/client-go && git fetch --depth 1 origin 8020fc4fcf89965904a5f43689f169d6e01d1e80 && git checkout FETCH_HEAD
+RUN git clone https://github.com/kubernetes/client-go /workspace/client-go && \
+    cd /workspace/client-go && git checkout 8020fc4fcf89965904a5f43689f169d6e01d1e80
 
 # Initialize git identity for agent commits
 RUN git config --global user.email "agent@example.com" && \
diff --git a/benchmarks/ccb_mcp_crossrepo_tracing/ccx-dep-trace-001/environment/Dockerfile b/benchmarks/ccb_mcp_crossrepo_tracing/ccx-dep-trace-001/environment/Dockerfile
@@ -15,8 +15,8 @@ WORKDIR /workspace
 
 # Clone local checkout repos (baseline config: agent has local access to these)
 RUN git clone --depth 1 --branch v1.32.0 https://github.com/kubernetes/kubernetes /workspace/kubernetes
-RUN git clone https://github.com/kubernetes/client-go /workspace/client-go --no-checkout && \
-    cd /workspace/client-go && git fetch --depth 1 origin 8020fc4fcf89965904a5f43689f169d6e01d1e80 && git checkout FETCH_HEAD
+RUN git clone https://github.com/kubernetes/client-go /workspace/client-go && \
+    cd /workspace/client-go && git checkout 8020fc4fcf89965904a5f43689f169d6e01d1e80
 
 # Initialize git identity for agent commits
 RUN git config --global user.email "agent@example.com" && \
diff --git a/benchmarks/ccb_mcp_crossrepo_tracing/ccx-dep-trace-001/environment/Dockerfile.artifact_only b/benchmarks/ccb_mcp_crossrepo_tracing/ccx-dep-trace-001/environment/Dockerfile.artifact_only
@@ -20,8 +20,8 @@ WORKDIR /workspace
 
 # Clone local checkout repos (baseline config: agent has local access to these)
 RUN git clone --depth 1 --branch v1.32.0 https://github.com/kubernetes/kubernetes /workspace/kubernetes
-RUN git clone https://github.com/kubernetes/client-go /workspace/client-go --no-checkout && \
-    cd /workspace/client-go && git fetch --depth 1 origin 8020fc4fcf89965904a5f43689f169d6e01d1e80 && git checkout FETCH_HEAD
+RUN git clone https://github.com/kubernetes/client-go /workspace/client-go && \
+    cd /workspace/client-go && git checkout 8020fc4fcf89965904a5f43689f169d6e01d1e80
 
 # Initialize git identity for agent commits
 RUN git config --global user.email "agent@example.com" && \
diff --git a/benchmarks/ccb_mcp_crossrepo_tracing/ccx-dep-trace-004/environment/Dockerfile b/benchmarks/ccb_mcp_crossrepo_tracing/ccx-dep-trace-004/environment/Dockerfile
@@ -15,8 +15,8 @@ WORKDIR /workspace
 
 # Clone local checkout repos (baseline config: agent has local access to these)
 RUN git clone --depth 1 --branch v11.4.0 https://github.com/grafana/grafana /workspace/grafana
-RUN git clone https://github.com/grafana/loki /workspace/loki --no-checkout && \
-    cd /workspace/loki && git fetch --depth 1 origin a3af38d4da899032d3ee46a30932d072d37e1b9c && git checkout FETCH_HEAD
+RUN git clone https://github.com/grafana/loki /workspace/loki && \
+    cd /workspace/loki && git checkout a3af38d4da899032d3ee46a30932d072d37e1b9c
 
 # Initialize git identity for agent commits
 RUN git config --global user.email "agent@example.com" && \
diff --git a/benchmarks/ccb_mcp_crossrepo_tracing/ccx-dep-trace-004/environment/Dockerfile.artifact_only b/benchmarks/ccb_mcp_crossrepo_tracing/ccx-dep-trace-004/environment/Dockerfile.artifact_only
@@ -20,8 +20,8 @@ WORKDIR /workspace
 
 # Clone local checkout repos (baseline config: agent has local access to these)
 RUN git clone --depth 1 --branch v11.4.0 https://github.com/grafana/grafana /workspace/grafana
-RUN git clone https://github.com/grafana/loki /workspace/loki --no-checkout && \
-    cd /workspace/loki && git fetch --depth 1 origin a3af38d4da899032d3ee46a30932d072d37e1b9c && git checkout FETCH_HEAD
+RUN git clone https://github.com/grafana/loki /workspace/loki && \
+    cd /workspace/loki && git checkout a3af38d4da899032d3ee46a30932d072d37e1b9c
 
 # Initialize git identity for agent commits
 RUN git config --global user.email "agent@example.com" && \
diff --git a/benchmarks/ccb_mcp_onboarding/ccx-onboard-050-ds/environment/Dockerfile b/benchmarks/ccb_mcp_onboarding/ccx-onboard-050-ds/environment/Dockerfile
@@ -16,8 +16,8 @@ WORKDIR /workspace
 # Clone local checkout repos (baseline config: agent has local access to these)
 RUN git clone --depth 1 --branch v1.32.0 https://github.com/kubernetes/kubernetes /workspace/kubernetes
 RUN git clone --depth 1 --branch v3.5.17 https://github.com/etcd-io/etcd /workspace/etcd
-RUN git clone https://github.com/kubernetes/client-go /workspace/client-go --no-checkout && \
-    cd /workspace/client-go && git fetch --depth 1 origin 8020fc4fcf89965904a5f43689f169d6e01d1e80 && git checkout FETCH_HEAD
+RUN git clone https://github.com/kubernetes/client-go /workspace/client-go && \
+    cd /workspace/client-go && git checkout 8020fc4fcf89965904a5f43689f169d6e01d1e80
 
 # Initialize git identity for agent commits
 RUN git config --global user.email "agent@example.com" && \
diff --git a/benchmarks/ccb_mcp_onboarding/ccx-onboard-050-ds/environment/Dockerfile.artifact_only b/benchmarks/ccb_mcp_onboarding/ccx-onboard-050-ds/environment/Dockerfile.artifact_only
@@ -21,8 +21,8 @@ WORKDIR /workspace
 # Clone local checkout repos (baseline config: agent has local access to these)
 RUN git clone --depth 1 --branch v1.32.0 https://github.com/kubernetes/kubernetes /workspace/kubernetes
 RUN git clone --depth 1 --branch v3.5.17 https://github.com/etcd-io/etcd /workspace/etcd
-RUN git clone https://github.com/kubernetes/client-go /workspace/client-go --no-checkout && \
-    cd /workspace/client-go && git fetch --depth 1 origin 8020fc4fcf89965904a5f43689f169d6e01d1e80 && git checkout FETCH_HEAD
+RUN git clone https://github.com/kubernetes/client-go /workspace/client-go && \
+    cd /workspace/client-go && git checkout 8020fc4fcf89965904a5f43689f169d6e01d1e80
 
 # Initialize git identity for agent commits
 RUN git config --global user.email "agent@example.com" && \
diff --git a/benchmarks/ccb_mcp_platform/ccx-explore-091-ds/environment/Dockerfile b/benchmarks/ccb_mcp_platform/ccx-explore-091-ds/environment/Dockerfile
@@ -15,10 +15,10 @@ WORKDIR /workspace
 
 # Clone local checkout repos (baseline config: agent has local access to these)
 RUN git clone --depth 1 --branch v1.32.0 https://github.com/kubernetes/kubernetes /workspace/kubernetes
-RUN git clone https://github.com/kubernetes/client-go /workspace/client-go --no-checkout && \
-    cd /workspace/client-go && git fetch --depth 1 origin 8020fc4fcf89965904a5f43689f169d6e01d1e80 && git checkout FETCH_HEAD
-RUN git clone https://github.com/kubernetes/api /workspace/api --no-checkout && \
-    cd /workspace/api && git fetch --depth 1 origin fa23dd302759dbb681c1a41f09d24190a38c1d58 && git checkout FETCH_HEAD
+RUN git clone https://github.com/kubernetes/client-go /workspace/client-go && \
+    cd /workspace/client-go && git checkout 8020fc4fcf89965904a5f43689f169d6e01d1e80
+RUN git clone https://github.com/kubernetes/api /workspace/api && \
+    cd /workspace/api && git checkout fa23dd302759dbb681c1a41f09d24190a38c1d58
 
 # Initialize git identity for agent commits
 RUN git config --global user.email "agent@example.com" && \
diff --git a/benchmarks/ccb_mcp_platform/ccx-explore-091-ds/environment/Dockerfile.artifact_only b/benchmarks/ccb_mcp_platform/ccx-explore-091-ds/environment/Dockerfile.artifact_only
@@ -20,10 +20,10 @@ WORKDIR /workspace
 
 # Clone local checkout repos (baseline config: agent has local access to these)
 RUN git clone --depth 1 --branch v1.32.0 https://github.com/kubernetes/kubernetes /workspace/kubernetes
-RUN git clone https://github.com/kubernetes/client-go /workspace/client-go --no-checkout && \
-    cd /workspace/client-go && git fetch --depth 1 origin 8020fc4fcf89965904a5f43689f169d6e01d1e80 && git checkout FETCH_HEAD
-RUN git clone https://github.com/kubernetes/api /workspace/api --no-checkout && \
-    cd /workspace/api && git fetch --depth 1 origin fa23dd302759dbb681c1a41f09d24190a38c1d58 && git checkout FETCH_HEAD
+RUN git clone https://github.com/kubernetes/client-go /workspace/client-go && \
+    cd /workspace/client-go && git checkout 8020fc4fcf89965904a5f43689f169d6e01d1e80
+RUN git clone https://github.com/kubernetes/api /workspace/api && \
+    cd /workspace/api && git checkout fa23dd302759dbb681c1a41f09d24190a38c1d58
 
 # Initialize git identity for agent commits
 RUN git config --global user.email "agent@example.com" && \
diff --git a/benchmarks/ccb_mcp_security/ccx-vuln-remed-011/environment/Dockerfile b/benchmarks/ccb_mcp_security/ccx-vuln-remed-011/environment/Dockerfile
@@ -14,8 +14,8 @@ WORKDIR /workspace
 
 # Clone local checkout repos (baseline config: agent has local access to these)
 RUN git clone --depth 1 --branch v22.13.0 https://github.com/nodejs/node /workspace/node
-RUN git clone https://github.com/expressjs/express /workspace/express --no-checkout && \
-    cd /workspace/express && git fetch --depth 1 origin 9de5890d0dc6128b7f7eb15469d76aa60dacc48f && git checkout FETCH_HEAD
+RUN git clone https://github.com/expressjs/express /workspace/express && \
+    cd /workspace/express && git checkout 9de5890d0dc6128b7f7eb15469d76aa60dacc48f
 
 # Initialize git identity for agent commits
 RUN git config --global user.email "agent@example.com" && \
diff --git a/benchmarks/ccb_mcp_security/ccx-vuln-remed-011/environment/Dockerfile.artifact_only b/benchmarks/ccb_mcp_security/ccx-vuln-remed-011/environment/Dockerfile.artifact_only
@@ -19,8 +19,8 @@ WORKDIR /workspace
 
 # Clone local checkout repos (baseline config: agent has local access to these)
 RUN git clone --depth 1 --branch v22.13.0 https://github.com/nodejs/node /workspace/node
-RUN git clone https://github.com/expressjs/express /workspace/express --no-checkout && \
-    cd /workspace/express && git fetch --depth 1 origin 9de5890d0dc6128b7f7eb15469d76aa60dacc48f && git checkout FETCH_HEAD
+RUN git clone https://github.com/expressjs/express /workspace/express && \
+    cd /workspace/express && git checkout 9de5890d0dc6128b7f7eb15469d76aa60dacc48f
 
 # Initialize git identity for agent commits
 RUN git config --global user.email "agent@example.com" && \
diff --git a/benchmarks/ccb_mcp_security/ccx-vuln-remed-014/environment/Dockerfile b/benchmarks/ccb_mcp_security/ccx-vuln-remed-014/environment/Dockerfile
@@ -14,8 +14,8 @@ WORKDIR /workspace
 
 # Clone local checkout repos (baseline config: agent has local access to these)
 RUN git clone --depth 1 --branch v11.4.0 https://github.com/grafana/grafana /workspace/grafana
-RUN git clone https://github.com/grafana/loki /workspace/loki --no-checkout && \
-    cd /workspace/loki && git fetch --depth 1 origin a3af38d4da899032d3ee46a30932d072d37e1b9c && git checkout FETCH_HEAD
+RUN git clone https://github.com/grafana/loki /workspace/loki && \
+    cd /workspace/loki && git checkout a3af38d4da899032d3ee46a30932d072d37e1b9c
 
 # Initialize git identity for agent commits
 RUN git config --global user.email "agent@example.com" && \
diff --git a/benchmarks/ccb_mcp_security/ccx-vuln-remed-014/environment/Dockerfile.artifact_only b/benchmarks/ccb_mcp_security/ccx-vuln-remed-014/environment/Dockerfile.artifact_only
@@ -19,8 +19,8 @@ WORKDIR /workspace
 
 # Clone local checkout repos (baseline config: agent has local access to these)
 RUN git clone --depth 1 --branch v11.4.0 https://github.com/grafana/grafana /workspace/grafana
-RUN git clone https://github.com/grafana/loki /workspace/loki --no-checkout && \
-    cd /workspace/loki && git fetch --depth 1 origin a3af38d4da899032d3ee46a30932d072d37e1b9c && git checkout FETCH_HEAD
+RUN git clone https://github.com/grafana/loki /workspace/loki && \
+    cd /workspace/loki && git checkout a3af38d4da899032d3ee46a30932d072d37e1b9c
 
 # Initialize git identity for agent commits
 RUN git config --global user.email "agent@example.com" && \
diff --git a/configs/run_selected_tasks.sh b/configs/run_selected_tasks.sh
@@ -17,6 +17,8 @@
 #   --use-case-category CATEGORY    Filter by MCP-unique use case category (A-J), only valid with --selection-file
 #   --baseline-only                 Run only baseline (no MCP)
 #   --full-only                     Run only MCP-Full (mcp-remote-direct)
+#   --full-config CONFIG            Full config name (default: mcp-remote-direct)
+#                                   Use mcp-remote-artifact for artifact-based evaluation
 #   --model MODEL                   Override model (default: claude-opus-4-6)
 #   --concurrency N                 Concurrent tasks (default: 2)
 #   --category CATEGORY             Run category (default: staging)
@@ -53,6 +55,7 @@ TIMEOUT_MULTIPLIER=10
 RUN_BASELINE=true
 RUN_FULL=true
 CATEGORY="${CATEGORY:-staging}"
+FULL_CONFIG="${FULL_CONFIG:-mcp-remote-direct}"
 DRY_RUN=false
 SKIP_COMPLETED=false
 AGENT_PATH="agents.claude_baseline_agent:BaselineClaudeCodeAgent"
@@ -91,6 +94,10 @@ while [[ $# -gt 0 ]]; do
             CATEGORY="$2"
             shift 2
             ;;
+        --full-config)
+            FULL_CONFIG="$2"
+            shift 2
+            ;;
         --skip-completed)
             SKIP_COMPLETED=true
             shift
@@ -127,6 +134,11 @@ fi
 
 ensure_fresh_token
 
+# Derive baseline config and mcp_type values from FULL_CONFIG
+BASELINE_CONFIG=$(baseline_config_for "$FULL_CONFIG")
+BL_MCP_TYPE=$(config_to_mcp_type "$BASELINE_CONFIG")
+FULL_MCP_TYPE=$(config_to_mcp_type "$FULL_CONFIG")
+
 # ============================================
 # EXTRACT TASKS FROM SELECTION FILE
 # ============================================
@@ -199,7 +211,7 @@ echo "Source:        $SELECTION_FILE"
 echo "Model:         $MODEL"
 echo "Total tasks:   $TOTAL_TASKS"
 echo "Concurrency:   $CONCURRENCY"
-echo "Configs:       baseline-local-direct=$RUN_BASELINE mcp-remote-direct=$RUN_FULL"
+echo "Configs:       $BASELINE_CONFIG=$RUN_BASELINE $FULL_CONFIG=$RUN_FULL"
 echo "Skip done:     $SKIP_COMPLETED"
 [ -n "$USE_CASE_CATEGORY_FILTER" ] && echo "Category:      $USE_CASE_CATEGORY_FILTER"
 echo ""
@@ -301,6 +313,17 @@ run_benchmark() {
             echo "WARNING: Task directory not found: $abs_path"
             continue
         fi
+
+        # Swap Dockerfile for artifact configs (both baseline-local-artifact and mcp-remote-artifact)
+        local _df="${abs_path}/environment/Dockerfile"
+        local _df_artifact="${abs_path}/environment/Dockerfile.artifact_only"
+        local _df_swapped=false
+        if [[ "$mcp_mode" == *-artifact ]] && [ -f "$_df_artifact" ]; then
+            cp "$_df" "${_df}.run_bak"
+            cp "$_df_artifact" "$_df"
+            _df_swapped=true
+        fi
+
         BASELINE_MCP_TYPE=$mcp_type harbor run \
             --path "$abs_path" \
             --agent-import-path "$AGENT_PATH" \
@@ -310,6 +333,11 @@ run_benchmark() {
             --timeout-multiplier $TIMEOUT_MULTIPLIER \
             2>&1 | tee -a "${jobs_dir}.log" \
             || echo "WARNING: Task failed: $task_id (continuing...)"
+
+        # Restore original Dockerfile after run
+        if [ "$_df_swapped" = true ]; then
+            mv "${_df}.run_bak" "$_df"
+        fi
     done <<< "$task_ids" 3<<< "$task_dirs"
 
     unset SOURCEGRAPH_REPO_NAME 2>/dev/null || true
@@ -324,10 +352,10 @@ run_benchmark() {
 # ============================================
 for bm in $(echo "${!BENCHMARK_COUNTS[@]}" | tr ' ' '\n' | sort); do
     if [ "$RUN_BASELINE" = true ]; then
-        run_benchmark "$bm" "baseline-local-direct" "none"
+        run_benchmark "$bm" "$BASELINE_CONFIG" "$BL_MCP_TYPE"
     fi
     if [ "$RUN_FULL" = true ]; then
-        run_benchmark "$bm" "mcp-remote-direct" "sourcegraph_full"
+        run_benchmark "$bm" "$FULL_CONFIG" "$FULL_MCP_TYPE"
     fi
 done
 
