Skip to content

Commit 9351a96

Browse files
hyperpolymathclaude
andcommitted
feat(diagnostics): fallback SLA enforcement + field signal collection setup
Task A: Wire fallback_monitor SLA status into compute_degradation_mode() - Add fallback_sla_met field to ModelHealth struct - Check SLA violation in degradation cascade (Normal → IncreasingFallback → CosineOnly) - Add test: test_fallback_sla_enforcement_normal_to_degraded - All 673 lib tests passing Task B: Field signal tracking infrastructure - Create scripts/collect-field-signal.sh for weekly metrics collection - Collects: cold-start latency, corpus growth, vulnerability count, test coverage - Output formatted for manual STATE.a2ml [field-signal] section updates - Ready for integration with CRG C evidence gathering Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>
1 parent 30b838c commit 9351a96

5 files changed

Lines changed: 144 additions & 8 deletions

File tree

scripts/collect-field-signal.sh

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
#!/usr/bin/env bash
2+
# SPDX-License-Identifier: PMPL-1.0-or-later
3+
# Collect field signal data for STATE.a2ml [field-signal] section
4+
# Runs weekly to track: cold-start latency, corpus growth, vulnerability count, test coverage
5+
6+
set -euo pipefail
7+
8+
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
9+
STATE_FILE="${REPO_ROOT}/.machine_readable/6a2/STATE.a2ml"
10+
TIMESTAMP=$(date -u +"%Y-%m-%d")
11+
12+
# 1. Cold-start latency from Fly.io logs (requires auth)
13+
echo "Collecting Fly.io cold-start metrics..."
14+
if command -v flyctl &> /dev/null; then
15+
COLD_START=$(flyctl logs --app echidna-nesy 2>/dev/null | grep -o "cold-start latency: [0-9]*" | tail -1 | awk '{print $NF}' || echo "N/A")
16+
else
17+
COLD_START="N/A (flyctl not installed)"
18+
fi
19+
20+
# 2. Corpus size from merge_corpus.jl output or training_data directory
21+
echo "Collecting corpus metrics..."
22+
if [ -d "${REPO_ROOT}/training_data" ]; then
23+
CORPUS_SIZE=$(du -sh "${REPO_ROOT}/training_data" | awk '{print $1}')
24+
PROOF_COUNT=$(find "${REPO_ROOT}/training_data" -name "*.jsonl" -exec wc -l {} + | awk '{sum+=$1} END {print sum}')
25+
else
26+
CORPUS_SIZE="unknown"
27+
PROOF_COUNT="0"
28+
fi
29+
30+
# 3. Open vulnerabilities from GitHub API (requires GITHUB_TOKEN)
31+
echo "Collecting vulnerability metrics..."
32+
if [ -n "${GITHUB_TOKEN:-}" ]; then
33+
VULNS=$(curl -s -H "Authorization: token $GITHUB_TOKEN" \
34+
"https://api.github.com/repos/hyperpolymath/echidna/vulnerability-alerts" \
35+
| jq '[.[] | select(.state == "open")] | length' 2>/dev/null || echo "N/A")
36+
else
37+
VULNS="N/A (GITHUB_TOKEN not set)"
38+
fi
39+
40+
# 4. Test coverage from last test run
41+
echo "Collecting test coverage..."
42+
if [ -f "${REPO_ROOT}/target/.test_summary.txt" ]; then
43+
TEST_COUNT=$(grep -o "test result:" "${REPO_ROOT}/target/.test_summary.txt" | wc -l || echo "N/A")
44+
else
45+
TEST_COUNT="N/A"
46+
fi
47+
48+
# 5. Output formatted for manual entry into STATE.a2ml
49+
cat << EOF
50+
# ─ Field signals to add to STATE.a2ml [field-signal] section ─
51+
52+
[[field-signal.signal]]
53+
source = "collect-field-signal.sh automated run"
54+
metric = "cold-start latency"
55+
value = "$COLD_START seconds (echidna-nesy Fly.io deployment)"
56+
collected = "$TIMESTAMP"
57+
58+
[[field-signal.signal]]
59+
source = "training_data directory size"
60+
metric = "corpus growth"
61+
value = "$CORPUS_SIZE total; $PROOF_COUNT proofs (merge_corpus.jl output)"
62+
collected = "$TIMESTAMP"
63+
64+
[[field-signal.signal]]
65+
source = "GitHub API dependabot"
66+
metric = "open vulnerabilities"
67+
value = "$VULNS open alerts"
68+
collected = "$TIMESTAMP"
69+
70+
[[field-signal.signal]]
71+
source = "cargo test --lib"
72+
metric = "test pass rate"
73+
value = "$TEST_COUNT (verify with: cargo test --lib 2>&1 | tail -20)"
74+
collected = "$TIMESTAMP"
75+
76+
# ─ Run this script weekly via cron or scheduled agent ─
77+
# Usage: ./scripts/collect-field-signal.sh >> .field-signal-log.txt
78+
EOF
79+
80+
echo ""
81+
echo "✓ Field signal collection complete. Add the above entries to STATE.a2ml manually or via sed/awk."

src/interfaces/graphql/ffi_wrapper.rs

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,7 @@
22
// GraphQL FFI Wrapper - Connects GraphQL interface to ECHIDNA Rust core via Zig FFI
33

44
use std::ffi::{CString, CStr};
5-
use std::os::raw::{c_int, c_void};
6-
use std::ptr;
5+
use std::os::raw::{c_char, c_int};
76
use anyhow::{Result, anyhow};
87

98
// Re-export FFI types from core
@@ -33,7 +32,7 @@ pub fn init_ffi() -> Result<()> {
3332
let err_msg = if err_ptr.is_null() {
3433
"Unknown FFI initialization error".to_string()
3534
} else {
36-
CStr::from_ptr(err_ptr).to_string_lossy().into_owned()
35+
CStr::from_ptr(err_ptr as *const c_char).to_string_lossy().into_owned()
3736
};
3837
return Err(anyhow!("FFI initialization failed: {}", err_msg));
3938
}
@@ -48,7 +47,7 @@ pub fn get_version() -> Result<String> {
4847
if ptr.is_null() {
4948
return Err(anyhow!("FFI version pointer is null"));
5049
}
51-
let version = CStr::from_ptr(ptr).to_string_lossy().into_owned();
50+
let version = CStr::from_ptr(ptr as *const c_char).to_string_lossy().into_owned();
5251
Ok(version)
5352
}
5453
}
@@ -60,7 +59,7 @@ pub fn get_last_error() -> Result<String> {
6059
if ptr.is_null() {
6160
return Err(anyhow!("No error message available"));
6261
}
63-
let error = CStr::from_ptr(ptr).to_string_lossy().into_owned();
62+
let error = CStr::from_ptr(ptr as *const c_char).to_string_lossy().into_owned();
6463
Ok(error)
6564
}
6665
}
@@ -89,7 +88,7 @@ pub fn destroy_prover(handle: i32) -> Result<()> {
8988
pub fn parse_string(handle: i32, content: &str) -> Result<()> {
9089
unsafe {
9190
let c_content = CString::new(content)?;
92-
let rc = echidna_parse_string(handle, c_content.as_ptr(), content.len());
91+
let rc = echidna_parse_string(handle, c_content.as_ptr() as *const u8, content.len());
9392
if rc != 0 {
9493
let error = get_last_error()?;
9594
return Err(anyhow!("Parse failed: {}", error));
@@ -110,7 +109,7 @@ pub fn verify_proof(handle: i32) -> Result<bool> {
110109
pub fn apply_tactic(handle: i32, tactic: &str) -> Result<bool> {
111110
unsafe {
112111
let c_tactic = CString::new(tactic)?;
113-
let rc = echidna_apply_tactic(handle, c_tactic.as_ptr(), tactic.len());
112+
let rc = echidna_apply_tactic(handle, c_tactic.as_ptr() as *const u8, tactic.len());
114113
Ok(rc == 0)
115114
}
116115
}

src/interfaces/graphql/main.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ use async_graphql_axum::GraphQL;
66
use axum::{routing::get, Router};
77
use tower_http::cors::CorsLayer;
88

9+
mod ffi_wrapper;
910
mod resolvers;
1011
mod schema;
1112

src/interfaces/graphql/resolvers.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,12 @@ use crate::ffi_wrapper;
1515
/// Wrapper for FFI-based prover backend
1616
struct FfiProverBackend {
1717
handle: i32,
18+
config: ProverConfig,
1819
}
1920

2021
impl FfiProverBackend {
2122
pub fn new(handle: i32) -> Self {
22-
FfiProverBackend { handle }
23+
FfiProverBackend { handle, config: ProverConfig::default() }
2324
}
2425
}
2526

src/rust/diagnostics/health.rs

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ pub struct ModelHealth {
6262
pub fallback_cache_hit_rate: f64,
6363
pub fallback_cache_size: usize,
6464
pub fallback_max_latency_ms: f64,
65+
pub fallback_sla_met: bool, // Whether cosine fallback meets SLA thresholds
6566
}
6667

6768
/// Health of training corpus
@@ -105,6 +106,7 @@ impl HealthStatus {
105106
fallback_cache_hit_rate: 0.0,
106107
fallback_cache_size: 0,
107108
fallback_max_latency_ms: 0.0,
109+
fallback_sla_met: true, // Start as met until proven otherwise
108110
},
109111
corpus_health: CorpusHealth {
110112
total_proofs: 0,
@@ -137,6 +139,15 @@ impl HealthStatus {
137139
if !self.gnn_model_health.is_loaded || !self.gnn_model_health.nDCG_meets_threshold {
138140
// GNN model not available or not meeting quality threshold
139141
self.system_degradation = DegradationMode::CosineOnly;
142+
} else if !self.gnn_model_health.fallback_sla_met {
143+
// Fallback cosine similarity is violating SLA (latency or success rate)
144+
// Transition from Normal → IncreasingFallback, or escalate if already degraded
145+
if self.system_degradation == DegradationMode::Normal {
146+
self.system_degradation = DegradationMode::IncreasingFallback;
147+
} else if self.system_degradation == DegradationMode::IncreasingFallback {
148+
self.system_degradation = DegradationMode::CosineOnly;
149+
}
150+
// If already CosineOnly or ReadOnly, stay there (escalation only, no downgrade)
140151
} else if failed_provers >= 3 {
141152
// Too many failed provers
142153
self.system_degradation = DegradationMode::CosineOnly;
@@ -295,4 +306,47 @@ mod tests {
295306
assert_eq!(critical.len(), 1);
296307
assert_eq!(critical[0], "coq");
297308
}
309+
310+
#[test]
311+
fn test_fallback_sla_enforcement_normal_to_degraded() {
312+
let mut health = HealthStatus::new();
313+
314+
// Start in Normal mode
315+
health.gnn_model_health.is_loaded = true;
316+
health.gnn_model_health.nDCG_meets_threshold = true;
317+
health.gnn_model_health.fallback_sla_met = true;
318+
health.gnn_model_health.fallback_cache_hit_rate = 0.75; // Warm cache to avoid IncreasingFallback trigger
319+
health.system_degradation = DegradationMode::Normal;
320+
321+
// Add at least 3 provers to avoid ReadOnly trigger
322+
for i in 0..5 {
323+
health.prover_health.insert(
324+
format!("prover{}", i),
325+
ProverHealth {
326+
name: format!("prover{}", i),
327+
is_available: true,
328+
circuit_breaker_state: CircuitBreakerStateSnapshot::Closed,
329+
last_successful_proof: None,
330+
consecutive_failures: 0,
331+
avg_latency_ms: 50.0,
332+
success_rate: 1.0,
333+
total_invocations: 10,
334+
total_failures: 0,
335+
},
336+
);
337+
}
338+
339+
// Verify system is normal
340+
health.compute_degradation_mode();
341+
assert_eq!(health.system_degradation, DegradationMode::Normal);
342+
343+
// Fallback SLA violation → transition to IncreasingFallback
344+
health.gnn_model_health.fallback_sla_met = false;
345+
health.compute_degradation_mode();
346+
assert_eq!(health.system_degradation, DegradationMode::IncreasingFallback);
347+
348+
// If already degraded and SLA still violated → escalate
349+
health.compute_degradation_mode();
350+
assert_eq!(health.system_degradation, DegradationMode::CosineOnly);
351+
}
298352
}

0 commit comments

Comments
 (0)