Add regenerate-explain tool for multi-statement test cases (#65)

Author: kyleconroy

CLAUDE.md

@@ -2,13 +2,13 @@
 
 ## Next Steps
 
-To find the next test to work on, run:
+To find the next explain test to work on (fewest pending statements first), run:
 
 ```bash
-go run ./cmd/next-test
+go run ./cmd/next-test -explain
 ```
 
-This tool finds all tests with `todo: true` in their metadata and returns the one with the shortest `query.sql` file.
+This finds tests with `explain_todo` entries in their metadata.
 
 To find the next format roundtrip test to work on, run:
 
@@ -18,18 +18,6 @@ go run ./cmd/next-test -format
 
 This finds tests with `todo_format: true` in their metadata.
 
-## Workflow
-
-1. Run `go run ./cmd/next-test` to find the next test to implement
-2. Check the test's `query.sql` to understand what ClickHouse SQL needs parsing
-3. Check the test's `explain.txt` to understand the expected EXPLAIN output
-4. Implement the necessary AST types in `ast/`
-5. Add parser logic in `parser/parser.go`
-6. Update the `Explain()` function if needed to match ClickHouse's output format
-7. Enable the test by removing `todo: true` from its `metadata.json` (set it to `{}`)
-8. Run `go test ./parser/... -timeout 5s` to verify
-9. Check if other todo tests now pass (see below)
-
 ## Running Tests
 
 Always run parser tests with a 5 second timeout:
@@ -40,38 +28,39 @@ go test ./parser/... -timeout 5s
 
 The tests are very fast. If a test is timing out, it indicates a bug (likely an infinite loop in the parser).
 
-## Checking for Newly Passing Todo Tests
+## Checking for Newly Passing Format Tests
 
-After implementing parser changes, run:
+After implementing format changes, run:
 
 ```bash
-go test ./parser/... -check-skipped -v 2>&1 | grep "PASSES NOW"
+go test ./parser/... -check-format -v 2>&1 | grep "FORMAT PASSES NOW"
 ```
 
-Tests that output `PASSES NOW` can have their `todo` flag removed from `metadata.json`. This helps identify when parser improvements fix multiple tests at once.
+Tests that output `FORMAT PASSES NOW` can have their `todo_format` flag removed from `metadata.json`.
 
-## Checking for Newly Passing Format Tests
+## Checking for Newly Passing Explain Tests
 
-After implementing format changes, run:
+After implementing parser/explain changes, run:
 
 ```bash
-go test ./parser/... -check-format -v 2>&1 | grep "FORMAT PASSES NOW"
+go test ./parser/... -check-explain -v 2>&1 | grep "EXPLAIN PASSES NOW"
 ```
 
-Tests that output `FORMAT PASSES NOW` can have their `todo_format` flag removed from `metadata.json`.
+Tests that output `EXPLAIN PASSES NOW` can have their statement removed from `explain_todo` in `metadata.json`.
 
 ## Test Structure
 
 Each test in `parser/testdata/` contains:
 
-- `metadata.json` - `{}` for enabled tests, `{"todo": true}` for pending tests
+- `metadata.json` - `{}` for enabled tests
 - `query.sql` - ClickHouse SQL to parse
 - `explain.txt` - Expected EXPLAIN AST output (matches ClickHouse's format)
+- `explain_N.txt` - Expected EXPLAIN AST output for Nth statement (N >= 2)
 
 ### Metadata Options
 
-- `todo: true` - Test is pending parser/explain implementation
 - `todo_format: true` - Format roundtrip test is pending implementation
+- `explain_todo: {"stmt2": true}` - Skip specific statement subtests
 - `skip: true` - Skip test entirely (e.g., causes infinite loop)
 - `explain: false` - Skip test (e.g., ClickHouse couldn't parse it)
 - `parse_error: true` - Query is intentionally invalid SQL

cmd/next-test/main.go

@@ -9,24 +9,33 @@ import (
 	"sort"
 )
 
-var formatFlag = flag.Bool("format", false, "Find tests with todo_format instead of todo")
+var formatFlag = flag.Bool("format", false, "Find tests with todo_format: true")
+var explainFlag = flag.Bool("explain", false, "Find tests with explain_todo entries (fewest first)")
 
 type testMetadata struct {
-	Todo       bool  `json:"todo,omitempty"`
-	TodoFormat bool  `json:"todo_format,omitempty"`
-	Explain    *bool `json:"explain,omitempty"`
-	Skip       bool  `json:"skip,omitempty"`
-	ParseError bool  `json:"parse_error,omitempty"`
+	TodoFormat  bool            `json:"todo_format,omitempty"`
+	ExplainTodo map[string]bool `json:"explain_todo,omitempty"`
+	Explain     *bool           `json:"explain,omitempty"`
+	Skip        bool            `json:"skip,omitempty"`
+	ParseError  bool            `json:"parse_error,omitempty"`
 }
 
 type todoTest struct {
-	name      string
-	querySize int
+	name           string
+	querySize      int
+	explainTodoLen int
 }
 
 func main() {
 	flag.Parse()
 
+	if !*formatFlag && !*explainFlag {
+		fmt.Fprintf(os.Stderr, "Usage: go run ./cmd/next-test [-format | -explain]\n")
+		fmt.Fprintf(os.Stderr, "  -format   Find tests with todo_format: true\n")
+		fmt.Fprintf(os.Stderr, "  -explain  Find tests with explain_todo entries (fewest first)\n")
+		os.Exit(1)
+	}
+
 	testdataDir := "parser/testdata"
 	entries, err := os.ReadDir(testdataDir)
 	if err != nil {
@@ -55,22 +64,22 @@ func main() {
 			continue
 		}
 
-		// Check for todo or todo_format based on flag
+		// Skip tests with skip or explain=false or parse_error
+		if metadata.Skip || (metadata.Explain != nil && !*metadata.Explain) || metadata.ParseError {
+			continue
+		}
+
+		// Check based on flag
 		if *formatFlag {
 			if !metadata.TodoFormat {
 				continue
 			}
-		} else {
-			if !metadata.Todo {
+		} else if *explainFlag {
+			if len(metadata.ExplainTodo) == 0 {
 				continue
 			}
 		}
 
-		// Skip tests with skip or explain=false or parse_error
-		if metadata.Skip || (metadata.Explain != nil && !*metadata.Explain) || metadata.ParseError {
-			continue
-		}
-
 		// Read query to get its size
 		queryPath := filepath.Join(testDir, "query.sql")
 		queryBytes, err := os.ReadFile(queryPath)
@@ -79,31 +88,47 @@ func main() {
 		}
 
 		todoTests = append(todoTests, todoTest{
-			name:      entry.Name(),
-			querySize: len(queryBytes),
+			name:           entry.Name(),
+			querySize:      len(queryBytes),
+			explainTodoLen: len(metadata.ExplainTodo),
 		})
 	}
 
-	todoType := "todo"
-	if *formatFlag {
-		todoType = "todo_format"
+	todoType := "todo_format"
+	if *explainFlag {
+		todoType = "explain_todo"
 	}
 
 	if len(todoTests) == 0 {
 		fmt.Printf("No %s tests found!\n", todoType)
 		return
 	}
 
-	// Sort by query size (shortest first)
-	sort.Slice(todoTests, func(i, j int) bool {
-		return todoTests[i].querySize < todoTests[j].querySize
-	})
+	// Sort based on mode
+	if *explainFlag {
+		// Sort by explain_todo count (fewest first), then by query size
+		sort.Slice(todoTests, func(i, j int) bool {
+			if todoTests[i].explainTodoLen != todoTests[j].explainTodoLen {
+				return todoTests[i].explainTodoLen < todoTests[j].explainTodoLen
+			}
+			return todoTests[i].querySize < todoTests[j].querySize
+		})
+	} else {
+		// Sort by query size (shortest first)
+		sort.Slice(todoTests, func(i, j int) bool {
+			return todoTests[i].querySize < todoTests[j].querySize
+		})
+	}
 
-	// Print the shortest one
+	// Print the best candidate
 	next := todoTests[0]
 	testDir := filepath.Join(testdataDir, next.name)
 
-	fmt.Printf("Next %s test: %s\n\n", todoType, next.name)
+	if *explainFlag {
+		fmt.Printf("Next %s test: %s (%d pending statements)\n\n", todoType, next.name, next.explainTodoLen)
+	} else {
+		fmt.Printf("Next %s test: %s\n\n", todoType, next.name)
+	}
 
 	// Print query.sql contents
 	queryPath := filepath.Join(testDir, "query.sql")
@@ -116,5 +141,19 @@ func main() {
 		fmt.Printf("\nExpected EXPLAIN output:\n%s\n", string(explainBytes))
 	}
 
+	// Print explain_todo entries if in explain mode
+	if *explainFlag {
+		metadataPath := filepath.Join(testDir, "metadata.json")
+		if metadataBytes, err := os.ReadFile(metadataPath); err == nil {
+			var metadata testMetadata
+			if json.Unmarshal(metadataBytes, &metadata) == nil {
+				fmt.Printf("\nPending statements (explain_todo):\n")
+				for stmt := range metadata.ExplainTodo {
+					fmt.Printf("  - %s\n", stmt)
+				}
+			}
+		}
+	}
+
 	fmt.Printf("\nRemaining %s tests: %d\n", todoType, len(todoTests))
 }