feat: add MATERIALIZED CTE support and improve metrics

Features:
- Add MATERIALIZED/NOT MATERIALIZED parsing for CTEs (SQL standard)
- Add comprehensive parser and AST metrics tracking
- Add pool metrics for AST, statement, and expression objects

Fixes:
- Fix tokenizer negative column numbers in toSQLPosition
- Correct column calculation to be consistently 1-based

Metrics additions:
- RecordParse() for parser operation tracking
- RecordASTPoolGet/Put() for AST pool metrics
- RecordStatementPoolGet/Put() for statement pool metrics
- RecordExpressionPoolGet/Put() for expression pool metrics
- Extended Stats struct with parser and pool metrics

Tests:
- Add TestParser_MaterializedCTE test cases
- Update tokenizer error location test for correct column value
- Add column validation to fuzz tests

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

Author: 2 people

pkg/metrics/metrics.go

@@ -15,7 +15,22 @@ type Metrics struct {
 	tokenizeDuration   int64 // Total tokenization time in nanoseconds
 	lastTokenizeTime   int64 // Last tokenization timestamp
 
-	// Memory metrics
+	// Parser metrics
+	parseOperations   int64 // Total parse operations
+	parseErrors       int64 // Total parse errors
+	parseDuration     int64 // Total parse time in nanoseconds
+	lastParseTime     int64 // Last parse timestamp
+	statementsCreated int64 // Total statements parsed
+
+	// AST pool metrics
+	astPoolGets  int64 // AST pool retrievals
+	astPoolPuts  int64 // AST pool returns
+	stmtPoolGets int64 // Statement pool retrievals
+	stmtPoolPuts int64 // Statement pool returns
+	exprPoolGets int64 // Expression pool retrievals
+	exprPoolPuts int64 // Expression pool returns
+
+	// Memory metrics (tokenizer pool)
 	poolGets   int64 // Total pool retrievals
 	poolPuts   int64 // Total pool returns
 	poolMisses int64 // Pool misses (had to create new)
@@ -119,23 +134,120 @@ func RecordPoolPut() {
 	atomic.AddInt64(&globalMetrics.poolPuts, 1)
 }
 
+// RecordParse records a parse operation
+func RecordParse(duration time.Duration, statementCount int, err error) {
+	if atomic.LoadInt32(&globalMetrics.enabled) == 0 {
+		return
+	}
+
+	// Record operation
+	atomic.AddInt64(&globalMetrics.parseOperations, 1)
+	atomic.AddInt64(&globalMetrics.parseDuration, int64(duration))
+	atomic.StoreInt64(&globalMetrics.lastParseTime, time.Now().UnixNano())
+	atomic.AddInt64(&globalMetrics.statementsCreated, int64(statementCount))
+
+	// Record errors
+	if err != nil {
+		atomic.AddInt64(&globalMetrics.parseErrors, 1)
+
+		// Record error by type
+		errorType := "parse:" + err.Error()
+		globalMetrics.errorsMutex.Lock()
+		globalMetrics.errorsByType[errorType]++
+		globalMetrics.errorsMutex.Unlock()
+	}
+}
+
+// RecordASTPoolGet records an AST pool retrieval
+func RecordASTPoolGet() {
+	if atomic.LoadInt32(&globalMetrics.enabled) == 0 {
+		return
+	}
+	atomic.AddInt64(&globalMetrics.astPoolGets, 1)
+}
+
+// RecordASTPoolPut records an AST pool return
+func RecordASTPoolPut() {
+	if atomic.LoadInt32(&globalMetrics.enabled) == 0 {
+		return
+	}
+	atomic.AddInt64(&globalMetrics.astPoolPuts, 1)
+}
+
+// RecordStatementPoolGet records a statement pool retrieval
+func RecordStatementPoolGet() {
+	if atomic.LoadInt32(&globalMetrics.enabled) == 0 {
+		return
+	}
+	atomic.AddInt64(&globalMetrics.stmtPoolGets, 1)
+}
+
+// RecordStatementPoolPut records a statement pool return
+func RecordStatementPoolPut() {
+	if atomic.LoadInt32(&globalMetrics.enabled) == 0 {
+		return
+	}
+	atomic.AddInt64(&globalMetrics.stmtPoolPuts, 1)
+}
+
+// RecordExpressionPoolGet records an expression pool retrieval
+func RecordExpressionPoolGet() {
+	if atomic.LoadInt32(&globalMetrics.enabled) == 0 {
+		return
+	}
+	atomic.AddInt64(&globalMetrics.exprPoolGets, 1)
+}
+
+// RecordExpressionPoolPut records an expression pool return
+func RecordExpressionPoolPut() {
+	if atomic.LoadInt32(&globalMetrics.enabled) == 0 {
+		return
+	}
+	atomic.AddInt64(&globalMetrics.exprPoolPuts, 1)
+}
+
 // Stats represents current performance statistics
 type Stats struct {
-	// Basic counts
+	// Tokenization counts
 	TokenizeOperations int64   `json:"tokenize_operations"`
 	TokenizeErrors     int64   `json:"tokenize_errors"`
-	ErrorRate          float64 `json:"error_rate"`
+	TokenizeErrorRate  float64 `json:"tokenize_error_rate"`
 
-	// Performance metrics
-	AverageDuration     time.Duration `json:"average_duration"`
-	OperationsPerSecond float64       `json:"operations_per_second"`
+	// Parser counts
+	ParseOperations   int64   `json:"parse_operations"`
+	ParseErrors       int64   `json:"parse_errors"`
+	ParseErrorRate    float64 `json:"parse_error_rate"`
+	StatementsCreated int64   `json:"statements_created"`
 
-	// Memory/Pool metrics
+	// Tokenization performance metrics
+	AverageTokenizeDuration     time.Duration `json:"average_tokenize_duration"`
+	TokenizeOperationsPerSecond float64       `json:"tokenize_operations_per_second"`
+
+	// Parser performance metrics
+	AverageParseDuration     time.Duration `json:"average_parse_duration"`
+	ParseOperationsPerSecond float64       `json:"parse_operations_per_second"`
+
+	// Tokenizer pool metrics
 	PoolGets     int64   `json:"pool_gets"`
 	PoolPuts     int64   `json:"pool_puts"`
 	PoolBalance  int64   `json:"pool_balance"`
 	PoolMissRate float64 `json:"pool_miss_rate"`
 
+	// AST pool metrics
+	ASTPoolGets    int64 `json:"ast_pool_gets"`
+	ASTPoolPuts    int64 `json:"ast_pool_puts"`
+	ASTPoolBalance int64 `json:"ast_pool_balance"`
+
+	// Statement pool metrics
+	StmtPoolGets    int64 `json:"stmt_pool_gets"`
+	StmtPoolPuts    int64 `json:"stmt_pool_puts"`
+	StmtPoolBalance int64 `json:"stmt_pool_balance"`
+
+	// Expression pool metrics
+	ExprPoolGets    int64 `json:"expr_pool_gets"`
+	ExprPoolPuts    int64 `json:"expr_pool_puts"`
+	ExprPoolBalance int64 `json:"expr_pool_balance"`
+
 	// Query size metrics
 	MinQuerySize        int64   `json:"min_query_size"`
 	MaxQuerySize        int64   `json:"max_query_size"`
@@ -148,6 +260,9 @@ type Stats struct {
 
 	// Error breakdown
 	ErrorsByType map[string]int64 `json:"errors_by_type"`
+
+	// Legacy field for backwards compatibility
+	ErrorRate float64 `json:"error_rate"`
 }
 
 // GetStats returns current performance statistics
@@ -156,49 +271,115 @@ func GetStats() Stats {
 		return Stats{}
 	}
 
-	operations := atomic.LoadInt64(&globalMetrics.tokenizeOperations)
-	errors := atomic.LoadInt64(&globalMetrics.tokenizeErrors)
-	duration := atomic.LoadInt64(&globalMetrics.tokenizeDuration)
+	// Tokenization metrics
+	tokenizeOps := atomic.LoadInt64(&globalMetrics.tokenizeOperations)
+	tokenizeErrs := atomic.LoadInt64(&globalMetrics.tokenizeErrors)
+	tokenizeDur := atomic.LoadInt64(&globalMetrics.tokenizeDuration)
+	lastTokenizeTime := atomic.LoadInt64(&globalMetrics.lastTokenizeTime)
+
+	// Parser metrics
+	parseOps := atomic.LoadInt64(&globalMetrics.parseOperations)
+	parseErrs := atomic.LoadInt64(&globalMetrics.parseErrors)
+	parseDur := atomic.LoadInt64(&globalMetrics.parseDuration)
+	lastParseTime := atomic.LoadInt64(&globalMetrics.lastParseTime)
+	stmtsCreated := atomic.LoadInt64(&globalMetrics.statementsCreated)
+
+	// Pool metrics
 	poolGets := atomic.LoadInt64(&globalMetrics.poolGets)
 	poolPuts := atomic.LoadInt64(&globalMetrics.poolPuts)
 	poolMisses := atomic.LoadInt64(&globalMetrics.poolMisses)
+
+	// AST pool metrics
+	astPoolGets := atomic.LoadInt64(&globalMetrics.astPoolGets)
+	astPoolPuts := atomic.LoadInt64(&globalMetrics.astPoolPuts)
+	stmtPoolGets := atomic.LoadInt64(&globalMetrics.stmtPoolGets)
+	stmtPoolPuts := atomic.LoadInt64(&globalMetrics.stmtPoolPuts)
+	exprPoolGets := atomic.LoadInt64(&globalMetrics.exprPoolGets)
+	exprPoolPuts := atomic.LoadInt64(&globalMetrics.exprPoolPuts)
+
+	// Query size metrics
 	minSize := atomic.LoadInt64(&globalMetrics.minQuerySize)
 	maxSize := atomic.LoadInt64(&globalMetrics.maxQuerySize)
 	totalBytes := atomic.LoadInt64(&globalMetrics.totalQueryBytes)
-	lastOpTime := atomic.LoadInt64(&globalMetrics.lastTokenizeTime)
 
 	// Load start time atomically
 	startTime := globalMetrics.startTime.Load().(time.Time)
 
 	stats := Stats{
-		TokenizeOperations:  operations,
-		TokenizeErrors:      errors,
-		PoolGets:            poolGets,
-		PoolPuts:            poolPuts,
-		PoolBalance:         poolGets - poolPuts,
+		// Tokenization
+		TokenizeOperations: tokenizeOps,
+		TokenizeErrors:     tokenizeErrs,
+
+		// Parser
+		ParseOperations:   parseOps,
+		ParseErrors:       parseErrs,
+		StatementsCreated: stmtsCreated,
+
+		// Tokenizer pool
+		PoolGets:    poolGets,
+		PoolPuts:    poolPuts,
+		PoolBalance: poolGets - poolPuts,
+
+		// AST pools
+		ASTPoolGets:     astPoolGets,
+		ASTPoolPuts:     astPoolPuts,
+		ASTPoolBalance:  astPoolGets - astPoolPuts,
+		StmtPoolGets:    stmtPoolGets,
+		StmtPoolPuts:    stmtPoolPuts,
+		StmtPoolBalance: stmtPoolGets - stmtPoolPuts,
+		ExprPoolGets:    exprPoolGets,
+		ExprPoolPuts:    exprPoolPuts,
+		ExprPoolBalance: exprPoolGets - exprPoolPuts,
+
+		// Query size
 		MinQuerySize:        minSize,
 		MaxQuerySize:        maxSize,
 		TotalBytesProcessed: totalBytes,
 		Uptime:              time.Since(startTime),
 	}
 
-	// Calculate rates and averages
-	if operations > 0 {
-		stats.ErrorRate = float64(errors) / float64(operations)
-		stats.AverageDuration = time.Duration(duration / operations)
-		stats.AverageQuerySize = float64(totalBytes) / float64(operations)
+	// Calculate tokenization rates and averages
+	if tokenizeOps > 0 {
+		stats.TokenizeErrorRate = float64(tokenizeErrs) / float64(tokenizeOps)
+		stats.AverageTokenizeDuration = time.Duration(tokenizeDur / tokenizeOps)
+		stats.AverageQuerySize = float64(totalBytes) / float64(tokenizeOps)
 
 		// Operations per second
 		uptime := time.Since(startTime).Seconds()
 		if uptime > 0 {
-			stats.OperationsPerSecond = float64(operations) / uptime
+			stats.TokenizeOperationsPerSecond = float64(tokenizeOps) / uptime
 		}
 	}
 
+	// Calculate parse rates and averages
+	if parseOps > 0 {
+		stats.ParseErrorRate = float64(parseErrs) / float64(parseOps)
+		stats.AverageParseDuration = time.Duration(parseDur / parseOps)
+
+		// Operations per second
+		uptime := time.Since(startTime).Seconds()
+		if uptime > 0 {
+			stats.ParseOperationsPerSecond = float64(parseOps) / uptime
+		}
+	}
+
+	// Calculate pool miss rate
 	if poolGets > 0 {
 		stats.PoolMissRate = float64(poolMisses) / float64(poolGets)
 	}
 
+	// Legacy error rate (combined)
+	totalOps := tokenizeOps + parseOps
+	totalErrs := tokenizeErrs + parseErrs
+	if totalOps > 0 {
+		stats.ErrorRate = float64(totalErrs) / float64(totalOps)
+	}
+
+	// Determine last operation time (most recent of tokenize or parse)
+	lastOpTime := lastTokenizeTime
+	if lastParseTime > lastOpTime {
+		lastOpTime = lastParseTime
+	}
 	if lastOpTime > 0 {
 		stats.LastOperationTime = time.Unix(0, lastOpTime)
 	}
@@ -216,17 +397,38 @@ func GetStats() Stats {
 
 // Reset clears all metrics (useful for testing)
 func Reset() {
+	// Tokenization metrics
 	atomic.StoreInt64(&globalMetrics.tokenizeOperations, 0)
 	atomic.StoreInt64(&globalMetrics.tokenizeErrors, 0)
 	atomic.StoreInt64(&globalMetrics.tokenizeDuration, 0)
 	atomic.StoreInt64(&globalMetrics.lastTokenizeTime, 0)
+
+	// Parser metrics
+	atomic.StoreInt64(&globalMetrics.parseOperations, 0)
+	atomic.StoreInt64(&globalMetrics.parseErrors, 0)
+	atomic.StoreInt64(&globalMetrics.parseDuration, 0)
+	atomic.StoreInt64(&globalMetrics.lastParseTime, 0)
+	atomic.StoreInt64(&globalMetrics.statementsCreated, 0)
+
+	// Tokenizer pool metrics
 	atomic.StoreInt64(&globalMetrics.poolGets, 0)
 	atomic.StoreInt64(&globalMetrics.poolPuts, 0)
 	atomic.StoreInt64(&globalMetrics.poolMisses, 0)
+
+	// AST pool metrics
+	atomic.StoreInt64(&globalMetrics.astPoolGets, 0)
+	atomic.StoreInt64(&globalMetrics.astPoolPuts, 0)
+	atomic.StoreInt64(&globalMetrics.stmtPoolGets, 0)
+	atomic.StoreInt64(&globalMetrics.stmtPoolPuts, 0)
+	atomic.StoreInt64(&globalMetrics.exprPoolGets, 0)
+	atomic.StoreInt64(&globalMetrics.exprPoolPuts, 0)
+
+	// Query size metrics
 	atomic.StoreInt64(&globalMetrics.minQuerySize, -1)
 	atomic.StoreInt64(&globalMetrics.maxQuerySize, 0)
 	atomic.StoreInt64(&globalMetrics.totalQueryBytes, 0)
 
+	// Error tracking
 	globalMetrics.errorsMutex.Lock()
 	globalMetrics.errorsByType = make(map[string]int64)
 	globalMetrics.errorsMutex.Unlock()

pkg/metrics/metrics_test.go

@@ -95,12 +95,12 @@ func TestMetricsBasicFunctionality(t *testing.T) {
 	}
 
 	// Verify timing
-	if stats.AverageDuration <= 0 {
-		t.Error("Average duration should be positive")
+	if stats.AverageTokenizeDuration <= 0 {
+		t.Error("Average tokenize duration should be positive")
 	}
 
-	if stats.OperationsPerSecond <= 0 {
-		t.Error("Operations per second should be positive")
+	if stats.TokenizeOperationsPerSecond <= 0 {
+		t.Error("Tokenize operations per second should be positive")
 	}
 
 	if stats.Uptime <= 0 {

pkg/sql/ast/ast.go

@@ -56,11 +56,12 @@ func (w WithClause) Children() []Node {
 // CommonTableExpr represents a single Common Table Expression in a WITH clause.
 // It supports optional column specifications and any statement type as the CTE query.
 // Phase 2 Complete: Full parser support with column specifications.
+// Phase 2.6: Added MATERIALIZED/NOT MATERIALIZED support for query optimization hints.
 type CommonTableExpr struct {
 	Name         string
 	Columns      []string
 	Statement    Statement
-	Materialized *bool // TODO: Add MATERIALIZED/NOT MATERIALIZED parsing support
+	Materialized *bool // nil = default, true = MATERIALIZED, false = NOT MATERIALIZED
 }
 
 func (c *CommonTableExpr) statementNode()      {}

pkg/sql/parser/cte.go

@@ -129,6 +129,23 @@ func (p *Parser) parseCommonTableExpr() (*ast.CommonTableExpr, error) {
 	}
 	p.advance()
 
+	// Parse optional MATERIALIZED / NOT MATERIALIZED
+	// Syntax: AS [NOT] MATERIALIZED (query)
+	var materialized *bool
+	if p.isType(models.TokenTypeNot) {
+		p.advance() // Consume NOT
+		if !p.isType(models.TokenTypeMaterialized) {
+			return nil, p.expectedError("MATERIALIZED after NOT")
+		}
+		p.advance() // Consume MATERIALIZED
+		notMaterialized := false
+		materialized = &notMaterialized
+	} else if p.isType(models.TokenTypeMaterialized) {
+		p.advance() // Consume MATERIALIZED
+		isMaterialized := true
+		materialized = &isMaterialized
+	}
+
 	// Parse the CTE query (must be in parentheses)
 	if !p.isType(models.TokenTypeLParen) {
 		return nil, p.expectedError("( before CTE query")
@@ -147,9 +164,10 @@ func (p *Parser) parseCommonTableExpr() (*ast.CommonTableExpr, error) {
 	p.advance() // Consume )
 
 	return &ast.CommonTableExpr{
-		Name:      name,
-		Columns:   columns,
-		Statement: stmt,
+		Name:         name,
+		Columns:      columns,
+		Statement:    stmt,
+		Materialized: materialized,
 	}, nil
 }