Skip to content

Commit 8e9fb06

Browse files
committed
fix: commit six_layer_pipeline.go (was untracked, breaking CI)
The pipeline_process.go refactor references runLayer1..6 methods defined in six_layer_pipeline.go. The file existed locally but was never staged, so CI builds on a clean checkout failed at internal/filter/pipeline_process.go:34 with: runLayer1Preprocess undefined (type *PipelineCoordinator has no field or method runLayer1Preprocess) Adding it restores compilation.
1 parent 319d89a commit 8e9fb06

1 file changed

Lines changed: 237 additions & 0 deletions

File tree

Lines changed: 237 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,237 @@
1+
package filter
2+
3+
// Six-layer pipeline orchestration.
4+
//
5+
// All 50+ filter implementations are unchanged; this file only reorganizes
6+
// how they are sequenced. Each layer has a single clear responsibility:
7+
//
8+
// Layer 1 Preprocess — noise removal before any analysis (TOML, routing, dedup)
9+
// Layer 2 Structural — statistical signal/noise decisions (entropy, perplexity, AST)
10+
// Layer 3 Semantic — meaning-level compression (gist, compaction, attribution)
11+
// Layer 4 LLM-Specific — KV-cache aware techniques (H2O, attention sink, anchoring)
12+
// Layer 5 ContentType — format-specific passes (diff, log, JSON, agent memory)
13+
// Layer 6 Budget — hard enforcement (budget, session tracking)
14+
15+
// runLayer1Preprocess applies TOML filters, adaptive routing, extractive prefilter,
16+
// and adaptive learning (session-learned patterns applied earliest for best ROI).
17+
func (p *PipelineCoordinator) runLayer1Preprocess(input string, stats *PipelineStats) string {
18+
output := p.applyAdaptiveRouting(input, stats)
19+
if p.shouldEarlyExit(stats) {
20+
return output
21+
}
22+
23+
if p.tomlFilterWrapper != nil && p.config.EnableTOMLFilter {
24+
filtered, saved := p.tomlFilterWrapper.Apply(output, ModeMinimal)
25+
if saved > 0 {
26+
stats.LayerStats["0_toml_filter"] = LayerStat{TokensSaved: saved}
27+
stats.runningSaved += saved
28+
output = filtered
29+
if p.shouldEarlyExit(stats) {
30+
return output
31+
}
32+
}
33+
}
34+
35+
if p.adaptiveLearning != nil && p.config.EnableAdaptiveLearning && len(output) > 1000 {
36+
output = p.processLayer(filterLayer{p.adaptiveLearning, "1_adaptive_learning"}, output, stats)
37+
}
38+
39+
return output
40+
}
41+
42+
// runLayer2Structural removes statistically low-value content: entropy pruning,
43+
// perplexity scoring, AST preservation, n-gram abbreviation.
44+
func (p *PipelineCoordinator) runLayer2Structural(input string, stats *PipelineStats) string {
45+
output := input
46+
47+
if p.entropyFilter != nil && p.config.EnableEntropy && !p.shouldSkipEntropy(output) {
48+
output = p.processLayer(p.layers[0], output, stats)
49+
if p.shouldEarlyExit(stats) {
50+
return output
51+
}
52+
}
53+
54+
if p.perplexityFilter != nil && p.config.EnablePerplexity && !p.shouldSkipPerplexity(output) {
55+
output = p.processLayer(p.layers[1], output, stats)
56+
if p.shouldEarlyExit(stats) {
57+
return output
58+
}
59+
}
60+
61+
if p.goalDrivenFilter != nil && p.config.EnableGoalDriven && !p.shouldSkipQueryDependent() {
62+
output = p.processLayer(p.layers[2], output, stats)
63+
if p.shouldEarlyExit(stats) {
64+
return output
65+
}
66+
}
67+
68+
if p.astPreserveFilter != nil && p.config.EnableAST {
69+
output = p.processLayer(p.layers[3], output, stats)
70+
if p.shouldEarlyExit(stats) {
71+
return output
72+
}
73+
}
74+
75+
if p.contrastiveFilter != nil && p.config.EnableContrastive && !p.shouldSkipQueryDependent() {
76+
output = p.processLayer(p.layers[4], output, stats)
77+
if p.shouldEarlyExit(stats) {
78+
return output
79+
}
80+
}
81+
82+
if p.ngramAbbreviator != nil && !p.shouldSkipNgram(output) {
83+
output = p.processLayer(p.layers[5], output, stats)
84+
}
85+
86+
return output
87+
}
88+
89+
// runLayer3Semantic compresses at the meaning level: evaluator heads, gist extraction,
90+
// hierarchical summarization, conversation compaction, attribution pruning,
91+
// meta-token lossless encoding, and semantic chunking.
92+
func (p *PipelineCoordinator) runLayer3Semantic(input string, stats *PipelineStats) string {
93+
output := input
94+
95+
if p.evaluatorHeadsFilter != nil && p.config.EnableEvaluator {
96+
output = p.processLayer(p.layers[6], output, stats)
97+
if p.shouldEarlyExit(stats) {
98+
return output
99+
}
100+
}
101+
102+
if p.gistFilter != nil && p.config.EnableGist {
103+
output = p.processLayer(p.layers[7], output, stats)
104+
if p.shouldEarlyExit(stats) {
105+
return output
106+
}
107+
}
108+
109+
if p.hierarchicalSummaryFilter != nil && p.config.EnableHierarchical {
110+
output = p.processLayer(p.layers[8], output, stats)
111+
if p.shouldEarlyExit(stats) {
112+
return output
113+
}
114+
}
115+
116+
if p.compactionLayer != nil && !p.shouldSkipCompaction(output) {
117+
output = p.processLayer(p.layers[9], output, stats)
118+
if p.shouldEarlyExit(stats) {
119+
return output
120+
}
121+
}
122+
123+
if p.attributionFilter != nil {
124+
output = p.processLayer(p.layers[10], output, stats)
125+
if p.shouldEarlyExit(stats) {
126+
return output
127+
}
128+
}
129+
130+
if p.metaTokenFilter != nil && !p.shouldSkipMetaToken(output) {
131+
output = p.processLayer(p.layers[13], output, stats)
132+
if p.shouldEarlyExit(stats) {
133+
return output
134+
}
135+
}
136+
137+
if p.semanticChunkFilter != nil && !p.shouldSkipSemanticChunk(output) {
138+
output = p.processLayer(p.layers[14], output, stats)
139+
}
140+
141+
return output
142+
}
143+
144+
// runLayer4LLMSpecific applies KV-cache aware techniques: QuantumLock alignment,
145+
// H2O heavy-hitter eviction, attention sink preservation, sketch store,
146+
// lazy pruning, and semantic anchor compression.
147+
func (p *PipelineCoordinator) runLayer4LLMSpecific(input string, stats *PipelineStats) string {
148+
output := input
149+
150+
if p.quantumLockFilter != nil && p.config.EnableQuantumLock {
151+
output = p.processLayer(filterLayer{p.quantumLockFilter, "4_quantum_lock"}, output, stats)
152+
if p.shouldEarlyExit(stats) {
153+
return output
154+
}
155+
}
156+
157+
if p.photonFilter != nil && p.config.EnablePhoton {
158+
output = p.processLayer(filterLayer{p.photonFilter, "4_photon"}, output, stats)
159+
if p.shouldEarlyExit(stats) {
160+
return output
161+
}
162+
}
163+
164+
if p.h2oFilter != nil && !p.shouldSkipH2O(output) {
165+
output = p.processLayer(p.layers[11], output, stats)
166+
if p.shouldEarlyExit(stats) {
167+
return output
168+
}
169+
}
170+
171+
if p.attentionSinkFilter != nil && !p.shouldSkipAttentionSink(output) {
172+
output = p.processLayer(p.layers[12], output, stats)
173+
if p.shouldEarlyExit(stats) {
174+
return output
175+
}
176+
}
177+
178+
if p.sketchStoreFilter != nil && !p.shouldSkipBudgetDependent() {
179+
output = p.processLayer(p.layers[15], output, stats)
180+
if p.shouldEarlyExit(stats) {
181+
return output
182+
}
183+
}
184+
185+
if p.lazyPrunerFilter != nil && !p.shouldSkipBudgetDependent() {
186+
output = p.processLayer(p.layers[16], output, stats)
187+
if p.shouldEarlyExit(stats) {
188+
return output
189+
}
190+
}
191+
192+
if p.semanticAnchorFilter != nil {
193+
output = p.processLayer(p.layers[17], output, stats)
194+
}
195+
196+
return output
197+
}
198+
199+
// runLayer5ContentType applies content-format aware passes: agent memory consolidation,
200+
// edge-case handling (L21-25), reasoning trace compression (L26-30),
201+
// and advanced research techniques (L31-45: diff, log, JSON, search, structural collapse).
202+
func (p *PipelineCoordinator) runLayer5ContentType(input string, stats *PipelineStats) string {
203+
output := input
204+
205+
if p.agentMemoryFilter != nil {
206+
output = p.processLayer(p.layers[18], output, stats)
207+
if p.shouldEarlyExit(stats) {
208+
return output
209+
}
210+
}
211+
212+
if p.edgeCaseFilter != nil {
213+
output = p.processLayer(p.layers[19], output, stats)
214+
if p.shouldEarlyExit(stats) {
215+
return output
216+
}
217+
}
218+
219+
if p.reasoningFilter != nil {
220+
output = p.processLayer(p.layers[20], output, stats)
221+
if p.shouldEarlyExit(stats) {
222+
return output
223+
}
224+
}
225+
226+
if p.advancedFilter != nil {
227+
output = p.processLayer(p.layers[21], output, stats)
228+
}
229+
230+
return output
231+
}
232+
233+
// runLayer6BudgetQuality enforces token budget and session tracking.
234+
// Quality guardrail and feedback are handled in Process() after all layers complete.
235+
func (p *PipelineCoordinator) runLayer6BudgetQuality(input string, stats *PipelineStats) string {
236+
return p.processBudgetLayer(input, stats)
237+
}

0 commit comments

Comments
 (0)