Fix 23 todo tests: BETWEEN transformation and float literal parsing (#25)

Author: kyleconroy

ast/ast.go

@@ -911,6 +911,7 @@ type CastExpr struct {
 	TypeExpr       Expression     `json:"type_expr,omitempty"` // For dynamic type like CAST(x, if(cond, 'Type1', 'Type2'))
 	Alias          string         `json:"alias,omitempty"`
 	OperatorSyntax bool           `json:"operator_syntax,omitempty"` // true if using :: syntax
+	UsedASSyntax   bool           `json:"-"`                         // true if CAST(x AS Type) syntax used (not CAST(x, 'Type'))
 }
 
 func (c *CastExpr) Pos() token.Position { return c.Position }

internal/explain/expressions.go

@@ -198,7 +198,11 @@ func explainUnaryExpr(sb *strings.Builder, n *ast.UnaryExpr, indent string, dept
 
 func explainSubquery(sb *strings.Builder, n *ast.Subquery, indent string, depth int) {
 	children := 1
-	fmt.Fprintf(sb, "%sSubquery (children %d)\n", indent, children)
+	if n.Alias != "" {
+		fmt.Fprintf(sb, "%sSubquery (alias %s) (children %d)\n", indent, n.Alias, children)
+	} else {
+		fmt.Fprintf(sb, "%sSubquery (children %d)\n", indent, children)
+	}
 	Node(sb, n.Query, depth+1)
 }
 
@@ -270,8 +274,12 @@ func explainAliasedExpr(sb *strings.Builder, n *ast.AliasedExpr, depth int) {
 		Node(sb, e.Then, depth+2)
 		Node(sb, e.Else, depth+2)
 	case *ast.CastExpr:
-		// CAST expressions - ClickHouse doesn't show aliases on CAST in EXPLAIN AST
-		explainCastExpr(sb, e, indent, depth)
+		// CAST expressions - show alias only for CAST(x AS Type) syntax, not CAST(x, 'Type')
+		if e.UsedASSyntax {
+			explainCastExprWithAlias(sb, e, n.Alias, indent, depth)
+		} else {
+			explainCastExpr(sb, e, indent, depth)
+		}
 	case *ast.ArrayAccess:
 		// Array access - ClickHouse doesn't show aliases on arrayElement in EXPLAIN AST
 		explainArrayAccess(sb, e, indent, depth)

internal/explain/format.go

@@ -2,6 +2,7 @@ package explain
 
 import (
 	"fmt"
+	"math"
 	"strconv"
 	"strings"
 
@@ -10,6 +11,16 @@ import (
 
 // FormatFloat formats a float value for EXPLAIN AST output
 func FormatFloat(val float64) string {
+	// Handle special float values - ClickHouse uses lowercase
+	if math.IsInf(val, 1) {
+		return "inf"
+	}
+	if math.IsInf(val, -1) {
+		return "-inf"
+	}
+	if math.IsNaN(val) {
+		return "nan"
+	}
 	// Use 'f' format to avoid scientific notation, -1 precision for smallest representation
 	return strconv.FormatFloat(val, 'f', -1, 64)
 }
@@ -142,6 +153,21 @@ func formatTupleLiteral(val interface{}) string {
 	return fmt.Sprintf("Tuple_(%s)", strings.Join(parts, ", "))
 }
 
+// formatInListAsTuple formats an IN expression's value list as a tuple literal
+func formatInListAsTuple(list []ast.Expression) string {
+	var parts []string
+	for _, e := range list {
+		if lit, ok := e.(*ast.Literal); ok {
+			parts = append(parts, FormatLiteral(lit))
+		} else if ident, ok := e.(*ast.Identifier); ok {
+			parts = append(parts, ident.Name())
+		} else {
+			parts = append(parts, formatExprAsString(e))
+		}
+	}
+	return fmt.Sprintf("Tuple_(%s)", strings.Join(parts, ", "))
+}
+
 // FormatDataType formats a DataType for EXPLAIN AST output
 func FormatDataType(dt *ast.DataType) string {
 	if dt == nil {
@@ -161,6 +187,9 @@ func FormatDataType(dt *ast.DataType) string {
 			}
 		} else if nested, ok := p.(*ast.DataType); ok {
 			params = append(params, FormatDataType(nested))
+		} else if ntp, ok := p.(*ast.NameTypePair); ok {
+			// Named tuple field: "name Type"
+			params = append(params, ntp.Name+" "+FormatDataType(ntp.Type))
 		} else {
 			params = append(params, fmt.Sprintf("%v", p))
 		}

internal/explain/functions.go

@@ -83,8 +83,19 @@ func explainCastExpr(sb *strings.Builder, n *ast.CastExpr, indent string, depth
 }
 
 func explainCastExprWithAlias(sb *strings.Builder, n *ast.CastExpr, alias string, indent string, depth int) {
+	// For :: operator syntax, ClickHouse hides alias only when expression is
+	// an array/tuple with complex content that gets formatted as string
+	hideAlias := false
+	if n.OperatorSyntax {
+		if lit, ok := n.Expr.(*ast.Literal); ok {
+			if lit.Type == ast.LiteralArray || lit.Type == ast.LiteralTuple {
+				hideAlias = !containsOnlyPrimitives(lit)
+			}
+		}
+	}
+
 	// CAST is represented as Function CAST with expr and type as arguments
-	if alias != "" {
+	if alias != "" && !hideAlias {
 		fmt.Fprintf(sb, "%sFunction CAST (alias %s) (children %d)\n", indent, alias, 1)
 	} else {
 		fmt.Fprintf(sb, "%sFunction CAST (children %d)\n", indent, 1)
@@ -205,6 +216,18 @@ func explainArrayAccess(sb *strings.Builder, n *ast.ArrayAccess, indent string,
 	Node(sb, n.Index, depth+2)
 }
 
+func explainArrayAccessWithAlias(sb *strings.Builder, n *ast.ArrayAccess, alias string, indent string, depth int) {
+	// Array access is represented as Function arrayElement
+	if alias != "" {
+		fmt.Fprintf(sb, "%sFunction arrayElement (alias %s) (children %d)\n", indent, alias, 1)
+	} else {
+		fmt.Fprintf(sb, "%sFunction arrayElement (children %d)\n", indent, 1)
+	}
+	fmt.Fprintf(sb, "%s ExpressionList (children %d)\n", indent, 2)
+	Node(sb, n.Array, depth+2)
+	Node(sb, n.Index, depth+2)
+}
+
 func explainTupleAccess(sb *strings.Builder, n *ast.TupleAccess, indent string, depth int) {
 	// Tuple access is represented as Function tupleElement
 	fmt.Fprintf(sb, "%sFunction tupleElement (children %d)\n", indent, 1)
@@ -213,6 +236,18 @@ func explainTupleAccess(sb *strings.Builder, n *ast.TupleAccess, indent string,
 	Node(sb, n.Index, depth+2)
 }
 
+func explainTupleAccessWithAlias(sb *strings.Builder, n *ast.TupleAccess, alias string, indent string, depth int) {
+	// Tuple access is represented as Function tupleElement
+	if alias != "" {
+		fmt.Fprintf(sb, "%sFunction tupleElement (alias %s) (children %d)\n", indent, alias, 1)
+	} else {
+		fmt.Fprintf(sb, "%sFunction tupleElement (children %d)\n", indent, 1)
+	}
+	fmt.Fprintf(sb, "%s ExpressionList (children %d)\n", indent, 2)
+	Node(sb, n.Tuple, depth+2)
+	Node(sb, n.Index, depth+2)
+}
+
 func explainLikeExpr(sb *strings.Builder, n *ast.LikeExpr, indent string, depth int) {
 	// LIKE is represented as Function like
 	fnName := "like"
@@ -229,17 +264,37 @@ func explainLikeExpr(sb *strings.Builder, n *ast.LikeExpr, indent string, depth
 }
 
 func explainBetweenExpr(sb *strings.Builder, n *ast.BetweenExpr, indent string, depth int) {
-	// BETWEEN is represented as Function and with two comparisons
-	// But for explain, we can use a simpler form
-	fnName := "between"
 	if n.Not {
-		fnName = "notBetween"
+		// NOT BETWEEN is transformed to: expr < low OR expr > high
+		// Represented as: Function or with two comparisons: less and greater
+		fmt.Fprintf(sb, "%sFunction or (children %d)\n", indent, 1)
+		fmt.Fprintf(sb, "%s ExpressionList (children %d)\n", indent, 2)
+		// less(expr, low)
+		fmt.Fprintf(sb, "%s  Function less (children %d)\n", indent, 1)
+		fmt.Fprintf(sb, "%s   ExpressionList (children %d)\n", indent, 2)
+		Node(sb, n.Expr, depth+4)
+		Node(sb, n.Low, depth+4)
+		// greater(expr, high)
+		fmt.Fprintf(sb, "%s  Function greater (children %d)\n", indent, 1)
+		fmt.Fprintf(sb, "%s   ExpressionList (children %d)\n", indent, 2)
+		Node(sb, n.Expr, depth+4)
+		Node(sb, n.High, depth+4)
+	} else {
+		// BETWEEN is represented as Function and with two comparisons
+		// expr >= low AND expr <= high
+		fmt.Fprintf(sb, "%sFunction and (children %d)\n", indent, 1)
+		fmt.Fprintf(sb, "%s ExpressionList (children %d)\n", indent, 2)
+		// greaterOrEquals(expr, low)
+		fmt.Fprintf(sb, "%s  Function greaterOrEquals (children %d)\n", indent, 1)
+		fmt.Fprintf(sb, "%s   ExpressionList (children %d)\n", indent, 2)
+		Node(sb, n.Expr, depth+4)
+		Node(sb, n.Low, depth+4)
+		// lessOrEquals(expr, high)
+		fmt.Fprintf(sb, "%s  Function lessOrEquals (children %d)\n", indent, 1)
+		fmt.Fprintf(sb, "%s   ExpressionList (children %d)\n", indent, 2)
+		Node(sb, n.Expr, depth+4)
+		Node(sb, n.High, depth+4)
 	}
-	fmt.Fprintf(sb, "%sFunction %s (children %d)\n", indent, fnName, 1)
-	fmt.Fprintf(sb, "%s ExpressionList (children %d)\n", indent, 3)
-	Node(sb, n.Expr, depth+2)
-	Node(sb, n.Low, depth+2)
-	Node(sb, n.High, depth+2)
 }
 
 func explainIsNullExpr(sb *strings.Builder, n *ast.IsNullExpr, indent string, depth int) {
@@ -325,6 +380,7 @@ func explainExtractExpr(sb *strings.Builder, n *ast.ExtractExpr, indent string,
 func explainWindowSpec(sb *strings.Builder, n *ast.WindowSpec, indent string, depth int) {
 	// Window spec is represented as WindowDefinition
 	// For simple cases like OVER (), just output WindowDefinition without children
+	// Note: ClickHouse's EXPLAIN AST does not output frame info (ROWS BETWEEN etc)
 	children := 0
 	if n.Name != "" {
 		children++
@@ -335,9 +391,6 @@ func explainWindowSpec(sb *strings.Builder, n *ast.WindowSpec, indent string, de
 	if len(n.OrderBy) > 0 {
 		children++
 	}
-	if n.Frame != nil {
-		children++
-	}
 	if children > 0 {
 		fmt.Fprintf(sb, "%sWindowDefinition (children %d)\n", indent, children)
 		if n.Name != "" {
@@ -352,7 +405,7 @@ func explainWindowSpec(sb *strings.Builder, n *ast.WindowSpec, indent string, de
 		if len(n.OrderBy) > 0 {
 			fmt.Fprintf(sb, "%s ExpressionList (children %d)\n", indent, len(n.OrderBy))
 			for _, o := range n.OrderBy {
-				Node(sb, o.Expression, depth+2)
+				explainOrderByElement(sb, o, strings.Repeat(" ", depth+2), depth+2)
 			}
 		}
 		// Frame handling would go here if needed

internal/explain/statements.go

@@ -316,8 +316,12 @@ func explainExplainQuery(sb *strings.Builder, n *ast.ExplainQuery, indent string
 }
 
 func explainShowQuery(sb *strings.Builder, n *ast.ShowQuery, indent string) {
-	// Capitalize ShowType correctly for display
+	// ClickHouse maps certain SHOW types to ShowTables in EXPLAIN AST
 	showType := strings.Title(strings.ToLower(string(n.ShowType)))
+	// SHOW SETTINGS is displayed as ShowTables in ClickHouse
+	if showType == "Settings" {
+		showType = "Tables"
+	}
 	fmt.Fprintf(sb, "%sShow%s\n", indent, showType)
 }
 
@@ -327,13 +331,14 @@ func explainUseQuery(sb *strings.Builder, n *ast.UseQuery, indent string) {
 
 func explainDescribeQuery(sb *strings.Builder, n *ast.DescribeQuery, indent string) {
 	if n.TableFunction != nil {
-		// DESCRIBE on a table function
+		// DESCRIBE on a table function - wrap in TableExpression
 		children := 1
 		if len(n.Settings) > 0 {
 			children++
 		}
 		fmt.Fprintf(sb, "%sDescribeQuery (children %d)\n", indent, children)
-		explainFunctionCall(sb, n.TableFunction, indent+" ", 1)
+		fmt.Fprintf(sb, "%s TableExpression (children 1)\n", indent)
+		explainFunctionCall(sb, n.TableFunction, indent+"  ", 2)
 		if len(n.Settings) > 0 {
 			fmt.Fprintf(sb, "%s Set\n", indent)
 		}

lexer/lexer.go

@@ -594,16 +594,21 @@ func (l *Lexer) readNumber() Item {
 		}
 	}
 
-	// Check for decimal point
-	if l.ch == '.' && unicode.IsDigit(l.peekChar()) {
-		sb.WriteRune(l.ch)
-		l.readChar()
-		for unicode.IsDigit(l.ch) {
+	// Check for decimal point (either followed by digit or end of number like 1.)
+	if l.ch == '.' {
+		nextCh := l.peekChar()
+		// Allow 1. (trailing dot with no digits) and 1.5 (dot with digits)
+		// But not 1.something (identifier-like)
+		if unicode.IsDigit(nextCh) || (!isIdentStart(nextCh) && nextCh != '.') {
 			sb.WriteRune(l.ch)
 			l.readChar()
-			// Handle underscore separators
-			for l.ch == '_' && unicode.IsDigit(l.peekChar()) {
+			for unicode.IsDigit(l.ch) {
+				sb.WriteRune(l.ch)
 				l.readChar()
+				// Handle underscore separators
+				for l.ch == '_' && unicode.IsDigit(l.peekChar()) {
+					l.readChar()
+				}
 			}
 		}
 	}
@@ -675,15 +680,20 @@ func (l *Lexer) readNumberOrIdent() Item {
 		}
 	}
 
-	// Check for decimal point
-	if l.ch == '.' && unicode.IsDigit(l.peekChar()) {
-		sb.WriteRune(l.ch)
-		l.readChar()
-		for unicode.IsDigit(l.ch) {
+	// Check for decimal point (either followed by digit or end of number like 1.)
+	if l.ch == '.' {
+		nextCh := l.peekChar()
+		// Allow 1. (trailing dot with no digits) and 1.5 (dot with digits)
+		// But not 1.something (identifier-like)
+		if unicode.IsDigit(nextCh) || (!isIdentStart(nextCh) && nextCh != '.') {
 			sb.WriteRune(l.ch)
 			l.readChar()
-			for l.ch == '_' && unicode.IsDigit(l.peekChar()) {
+			for unicode.IsDigit(l.ch) {
+				sb.WriteRune(l.ch)
 				l.readChar()
+				for l.ch == '_' && unicode.IsDigit(l.peekChar()) {
+					l.readChar()
+				}
 			}
 		}
 	}

parser/expression.go

@@ -958,6 +958,7 @@ func (p *Parser) parseCast() ast.Expression {
 	if p.currentIs(token.AS) {
 		p.nextToken()
 		expr.Type = p.parseDataType()
+		expr.UsedASSyntax = true
 	} else if p.currentIs(token.COMMA) {
 		p.nextToken()
 		// Type can be given as a string literal or an expression (e.g., if(cond, 'Type1', 'Type2'))
@@ -1564,6 +1565,18 @@ func (p *Parser) parseAlias(left ast.Expression) ast.Expression {
 	case *ast.Subquery:
 		e.Alias = alias
 		return e
+	case *ast.CastExpr:
+		// For :: operator syntax, set alias directly on CastExpr
+		// For function-style CAST(), wrap in AliasedExpr
+		if e.OperatorSyntax {
+			e.Alias = alias
+			return e
+		}
+		return &ast.AliasedExpr{
+			Position: left.Pos(),
+			Expr:     left,
+			Alias:    alias,
+		}
 	case *ast.CaseExpr:
 		e.Alias = alias
 		return e