Add CREATE EXTERNAL FILE FORMAT FormatType and format options parsing (#32)

Author: kyleconroy

ast/external_statements.go

@@ -17,20 +17,35 @@ type ExternalDataSourceOption struct {
 
 // CreateExternalFileFormatStatement represents CREATE EXTERNAL FILE FORMAT statement
 type CreateExternalFileFormatStatement struct {
-	Name    *Identifier
-	Options []*ExternalFileFormatOption
+	Name                      *Identifier
+	FormatType                string
+	ExternalFileFormatOptions []ExternalFileFormatOption
 }
 
 func (s *CreateExternalFileFormatStatement) node()      {}
 func (s *CreateExternalFileFormatStatement) statement() {}
 
-// ExternalFileFormatOption represents an option for external file format
-type ExternalFileFormatOption struct {
+// ExternalFileFormatOption is an interface for external file format options
+type ExternalFileFormatOption interface {
+	externalFileFormatOption()
+}
+
+// ExternalFileFormatContainerOption represents a container option with suboptions
+type ExternalFileFormatContainerOption struct {
 	OptionKind string
-	Value      ScalarExpression
-	SubOptions []*ExternalFileFormatOption
+	Suboptions []ExternalFileFormatOption
 }
 
+func (o *ExternalFileFormatContainerOption) externalFileFormatOption() {}
+
+// ExternalFileFormatLiteralOption represents a literal value option
+type ExternalFileFormatLiteralOption struct {
+	OptionKind string
+	Value      *StringLiteral
+}
+
+func (o *ExternalFileFormatLiteralOption) externalFileFormatOption() {}
+
 // CreateExternalTableStatement represents CREATE EXTERNAL TABLE statement
 type CreateExternalTableStatement struct {
 	SchemaObjectName *SchemaObjectName

parser/marshal.go

@@ -6067,9 +6067,48 @@ func createExternalFileFormatStatementToJSON(s *ast.CreateExternalFileFormatStat
 	if s.Name != nil {
 		node["Name"] = identifierToJSON(s.Name)
 	}
+	if s.FormatType != "" {
+		node["FormatType"] = s.FormatType
+	}
+	if len(s.ExternalFileFormatOptions) > 0 {
+		var options []jsonNode
+		for _, opt := range s.ExternalFileFormatOptions {
+			options = append(options, externalFileFormatOptionToJSON(opt))
+		}
+		node["ExternalFileFormatOptions"] = options
+	}
 	return node
 }
 
+func externalFileFormatOptionToJSON(opt ast.ExternalFileFormatOption) jsonNode {
+	switch o := opt.(type) {
+	case *ast.ExternalFileFormatContainerOption:
+		node := jsonNode{
+			"$type":      "ExternalFileFormatContainerOption",
+			"OptionKind": o.OptionKind,
+		}
+		if len(o.Suboptions) > 0 {
+			var subs []jsonNode
+			for _, sub := range o.Suboptions {
+				subs = append(subs, externalFileFormatOptionToJSON(sub))
+			}
+			node["Suboptions"] = subs
+		}
+		return node
+	case *ast.ExternalFileFormatLiteralOption:
+		node := jsonNode{
+			"$type":      "ExternalFileFormatLiteralOption",
+			"OptionKind": o.OptionKind,
+		}
+		if o.Value != nil {
+			node["Value"] = stringLiteralToJSON(o.Value)
+		}
+		return node
+	default:
+		return jsonNode{"$type": "UnknownExternalFileFormatOption"}
+	}
+}
+
 func createExternalTableStatementToJSON(s *ast.CreateExternalTableStatement) jsonNode {
 	node := jsonNode{
 		"$type": "CreateExternalTableStatement",

parser/parse_statements.go

@@ -4248,19 +4248,44 @@ func (p *Parser) parseCreateExternalFileFormatStatement() (*ast.CreateExternalFi
 		p.nextToken() // consume (
 
 		for p.curTok.Type != TokenRParen && p.curTok.Type != TokenEOF {
-			opt := &ast.ExternalFileFormatOption{
-				OptionKind: p.curTok.Literal,
-			}
+			optName := strings.ToUpper(p.curTok.Literal)
 			p.nextToken() // consume option name
-			if p.curTok.Type == TokenEquals {
-				p.nextToken()
-				val, err := p.parseScalarExpression()
-				if err != nil {
-					return nil, err
+
+			if optName == "FORMAT_TYPE" {
+				if p.curTok.Type == TokenEquals {
+					p.nextToken() // consume =
+				}
+				// Parse format type value and convert to PascalCase
+				stmt.FormatType = p.formatTypeToPascalCase(p.curTok.Literal)
+				p.nextToken() // consume value
+			} else if optName == "FORMAT_OPTIONS" {
+				// Parse container option with suboptions
+				opt := &ast.ExternalFileFormatContainerOption{
+					OptionKind: "FormatOptions",
+				}
+				if p.curTok.Type == TokenLParen {
+					p.nextToken() // consume (
+					for p.curTok.Type != TokenRParen && p.curTok.Type != TokenEOF {
+						subOpt := p.parseExternalFileFormatSuboption()
+						if subOpt != nil {
+							opt.Suboptions = append(opt.Suboptions, subOpt)
+						}
+						if p.curTok.Type == TokenComma {
+							p.nextToken()
+						}
+					}
+					if p.curTok.Type == TokenRParen {
+						p.nextToken() // consume )
+					}
+				}
+				stmt.ExternalFileFormatOptions = append(stmt.ExternalFileFormatOptions, opt)
+			} else {
+				// Skip other options for now
+				if p.curTok.Type == TokenEquals {
+					p.nextToken() // consume =
+					p.nextToken() // consume value
 				}
-				opt.Value = val
 			}
-			stmt.Options = append(stmt.Options, opt)
 			if p.curTok.Type == TokenComma {
 				p.nextToken()
 			}
@@ -4276,6 +4301,67 @@ func (p *Parser) parseCreateExternalFileFormatStatement() (*ast.CreateExternalFi
 	return stmt, nil
 }
 
+func (p *Parser) formatTypeToPascalCase(s string) string {
+	upper := strings.ToUpper(s)
+	switch upper {
+	case "DELTA":
+		return "Delta"
+	case "DELIMITEDTEXT":
+		return "DelimitedText"
+	case "PARQUET":
+		return "Parquet"
+	case "ORC":
+		return "Orc"
+	case "RCFILE":
+		return "RcFile"
+	case "JSON":
+		return "Json"
+	default:
+		return s
+	}
+}
+
+func (p *Parser) parseExternalFileFormatSuboption() ast.ExternalFileFormatOption {
+	optName := strings.ToUpper(p.curTok.Literal)
+	p.nextToken() // consume option name
+
+	// Map to option kind
+	optionKind := p.externalFileFormatOptionKind(optName)
+
+	if p.curTok.Type == TokenEquals {
+		p.nextToken() // consume =
+		val, _ := p.parseStringLiteral()
+		return &ast.ExternalFileFormatLiteralOption{
+			OptionKind: optionKind,
+			Value:      val,
+		}
+	}
+	return nil
+}
+
+func (p *Parser) externalFileFormatOptionKind(name string) string {
+	switch strings.ToUpper(name) {
+	case "PARSER_VERSION":
+		return "ParserVersion"
+	case "FIELD_TERMINATOR":
+		return "FieldTerminator"
+	case "STRING_DELIMITER":
+		return "StringDelimiter"
+	case "DATE_FORMAT":
+		return "DateFormat"
+	case "USE_TYPE_DEFAULT":
+		return "UseTypeDefault"
+	case "ENCODING":
+		return "Encoding"
+	case "DATA_COMPRESSION":
+		return "DataCompression"
+	case "FIRST_ROW":
+		return "FirstRow"
+	default:
+		return name
+	}
+}
+
 func (p *Parser) parseCreateExternalTableStatement() (*ast.CreateExternalTableStatement, error) {
 	// TABLE name - skip rest of statement for now
 	p.nextToken() // consume TABLE

parser/testdata/Baselines160_CreateExternalFileFormatStatementTests160/metadata.json

@@ -1 +1 @@
-{"todo": true}
+{}

parser/testdata/CreateExternalFileFormatStatementTests160/metadata.json

@@ -1 +1 @@
-{"todo": true}
+{}