Add full CREATE EXTERNAL DATA SOURCE support with options (#34)

Author: kyleconroy

ast/external_statements.go

@@ -2,17 +2,19 @@ package ast
 
 // CreateExternalDataSourceStatement represents CREATE EXTERNAL DATA SOURCE statement
 type CreateExternalDataSourceStatement struct {
-	Name    *Identifier
-	Options []*ExternalDataSourceOption
+	Name                      *Identifier
+	DataSourceType            string // HADOOP, RDBMS, SHARD_MAP_MANAGER, BLOB_STORAGE, EXTERNAL_GENERICS
+	Location                  *StringLiteral
+	ExternalDataSourceOptions []*ExternalDataSourceLiteralOrIdentifierOption
 }
 
 func (s *CreateExternalDataSourceStatement) node()      {}
 func (s *CreateExternalDataSourceStatement) statement() {}
 
-// ExternalDataSourceOption represents an option for external data source
-type ExternalDataSourceOption struct {
-	OptionKind string
-	Value      ScalarExpression
+// ExternalDataSourceLiteralOrIdentifierOption represents an option for external data source
+type ExternalDataSourceLiteralOrIdentifierOption struct {
+	OptionKind string // Credential, ResourceManagerLocation, DatabaseName, ShardMapName
+	Value      *IdentifierOrValueExpression
 }
 
 // CreateExternalFileFormatStatement represents CREATE EXTERNAL FILE FORMAT statement
@@ -97,8 +99,8 @@ type ExternalLibraryOption struct {
 
 // AlterExternalDataSourceStatement represents ALTER EXTERNAL DATA SOURCE statement
 type AlterExternalDataSourceStatement struct {
-	Name    *Identifier
-	Options []*ExternalDataSourceOption
+	Name                      *Identifier
+	ExternalDataSourceOptions []*ExternalDataSourceLiteralOrIdentifierOption
 }
 
 func (s *AlterExternalDataSourceStatement) node()      {}

parser/marshal.go

@@ -6057,6 +6057,32 @@ func createExternalDataSourceStatementToJSON(s *ast.CreateExternalDataSourceStat
 	if s.Name != nil {
 		node["Name"] = identifierToJSON(s.Name)
 	}
+	if s.DataSourceType != "" {
+		node["DataSourceType"] = s.DataSourceType
+	}
+	if s.Location != nil {
+		node["Location"] = stringLiteralToJSON(s.Location)
+	}
+	if len(s.ExternalDataSourceOptions) > 0 {
+		var options []jsonNode
+		for _, opt := range s.ExternalDataSourceOptions {
+			options = append(options, externalDataSourceOptionToJSON(opt))
+		}
+		node["ExternalDataSourceOptions"] = options
+	}
+	return node
+}
+
+func externalDataSourceOptionToJSON(opt *ast.ExternalDataSourceLiteralOrIdentifierOption) jsonNode {
+	node := jsonNode{
+		"$type": "ExternalDataSourceLiteralOrIdentifierOption",
+	}
+	if opt.Value != nil {
+		node["Value"] = identifierOrValueExpressionToJSON(opt.Value)
+	}
+	if opt.OptionKind != "" {
+		node["OptionKind"] = opt.OptionKind
+	}
 	return node
 }
 
@@ -7124,29 +7150,16 @@ func alterExternalDataSourceStatementToJSON(s *ast.AlterExternalDataSourceStatem
 	if s.Name != nil {
 		node["Name"] = identifierToJSON(s.Name)
 	}
-	if len(s.Options) > 0 {
-		opts := make([]jsonNode, len(s.Options))
-		for i, o := range s.Options {
+	if len(s.ExternalDataSourceOptions) > 0 {
+		opts := make([]jsonNode, len(s.ExternalDataSourceOptions))
+		for i, o := range s.ExternalDataSourceOptions {
 			opts[i] = externalDataSourceOptionToJSON(o)
 		}
 		node["ExternalDataSourceOptions"] = opts
 	}
 	return node
 }
 
-func externalDataSourceOptionToJSON(o *ast.ExternalDataSourceOption) jsonNode {
-	node := jsonNode{
-		"$type": "ExternalDataSourceLiteralOrIdentifierOption",
-	}
-	if o.OptionKind != "" {
-		node["OptionKind"] = o.OptionKind
-	}
-	if o.Value != nil {
-		node["Value"] = scalarExpressionToJSON(o.Value)
-	}
-	return node
-}
-
 func alterExternalLanguageStatementToJSON(s *ast.AlterExternalLanguageStatement) jsonNode {
 	node := jsonNode{
 		"$type": "AlterExternalLanguageStatement",

parser/parse_ddl.go

@@ -3819,32 +3819,33 @@ func (p *Parser) parseAlterExternalDataSourceStatement() (*ast.AlterExternalData
 			break
 		}
 
-		opt := &ast.ExternalDataSourceOption{}
-		opt.OptionKind = strings.ToUpper(p.curTok.Literal)
+		optName := strings.ToUpper(p.curTok.Literal)
 		p.nextToken()
 
 		// Expect =
 		if p.curTok.Type == TokenEquals {
 			p.nextToken()
 		}
 
+		opt := &ast.ExternalDataSourceLiteralOrIdentifierOption{
+			OptionKind: externalDataSourceOptionKindToPascalCase(optName),
+			Value:      &ast.IdentifierOrValueExpression{},
+		}
+
 		// Parse value
 		if p.curTok.Type == TokenString {
-			val, _ := p.parseStringLiteral()
-			opt.Value = val
+			strLit, _ := p.parseStringLiteral()
+			opt.Value.Value = strLit.Value
+			opt.Value.ValueExpression = strLit
 		} else if p.curTok.Type == TokenIdent {
-			opt.Value = &ast.ColumnReferenceExpression{
-				ColumnType: "Regular",
-				MultiPartIdentifier: &ast.MultiPartIdentifier{
-					Count:       1,
-					Identifiers: []*ast.Identifier{p.parseIdentifier()},
-				},
-			}
+			ident := p.parseIdentifier()
+			opt.Value.Value = ident.Value
+			opt.Value.Identifier = ident
 		} else {
 			p.nextToken()
 		}
 
-		stmt.Options = append(stmt.Options, opt)
+		stmt.ExternalDataSourceOptions = append(stmt.ExternalDataSourceOptions, opt)
 
 		if p.curTok.Type == TokenComma {
 			p.nextToken()

parser/parse_statements.go

@@ -4192,26 +4192,57 @@ func (p *Parser) parseCreateExternalDataSourceStatement() (*ast.CreateExternalDa
 
 	// Parse WITH clause
 	if p.curTok.Type == TokenWith {
+		// Default to EXTERNAL_GENERICS if WITH clause exists but no TYPE specified
+		stmt.DataSourceType = "EXTERNAL_GENERICS"
 		p.nextToken() // consume WITH
 		if p.curTok.Type != TokenLParen {
 			return nil, fmt.Errorf("expected ( after WITH, got %s", p.curTok.Literal)
 		}
 		p.nextToken() // consume (
 
 		for p.curTok.Type != TokenRParen && p.curTok.Type != TokenEOF {
-			opt := &ast.ExternalDataSourceOption{
-				OptionKind: p.curTok.Literal,
-			}
+			optName := strings.ToUpper(p.curTok.Literal)
 			p.nextToken() // consume option name
 			if p.curTok.Type == TokenEquals {
+				p.nextToken() // consume =
+			}
+
+			switch optName {
+			case "TYPE":
+				// TYPE sets DataSourceType
+				stmt.DataSourceType = strings.ToUpper(p.curTok.Literal)
 				p.nextToken()
-				val, err := p.parseScalarExpression()
+			case "LOCATION":
+				// LOCATION sets Location as StringLiteral
+				strLit, err := p.parseStringLiteral()
 				if err != nil {
 					return nil, err
 				}
-				opt.Value = val
+				stmt.Location = strLit
+			default:
+				// All other options go into ExternalDataSourceOptions
+				opt := &ast.ExternalDataSourceLiteralOrIdentifierOption{
+					OptionKind: externalDataSourceOptionKindToPascalCase(optName),
+					Value:      &ast.IdentifierOrValueExpression{},
+				}
+
+				// Determine if value is identifier or string literal
+				if p.curTok.Type == TokenString {
+					strLit, err := p.parseStringLiteral()
+					if err != nil {
+						return nil, err
+					}
+					opt.Value.Value = strLit.Value
+					opt.Value.ValueExpression = strLit
+				} else {
+					// It's an identifier
+					ident := p.parseIdentifier()
+					opt.Value.Value = ident.Value
+					opt.Value.Identifier = ident
+				}
+				stmt.ExternalDataSourceOptions = append(stmt.ExternalDataSourceOptions, opt)
 			}
-			stmt.Options = append(stmt.Options, opt)
+
 			if p.curTok.Type == TokenComma {
 				p.nextToken()
 			}
@@ -4227,6 +4258,26 @@ func (p *Parser) parseCreateExternalDataSourceStatement() (*ast.CreateExternalDa
 	return stmt, nil
 }
 
+// externalDataSourceOptionKindToPascalCase converts option names to PascalCase
+func externalDataSourceOptionKindToPascalCase(optName string) string {
+	switch strings.ToUpper(optName) {
+	case "CREDENTIAL":
+		return "Credential"
+	case "RESOURCE_MANAGER_LOCATION":
+		return "ResourceManagerLocation"
+	case "DATABASE_NAME":
+		return "DatabaseName"
+	case "SHARD_MAP_NAME":
+		return "ShardMapName"
+	case "CONNECTION_OPTIONS":
+		return "ConnectionOptions"
+	case "PUSHDOWN":
+		return "Pushdown"
+	default:
+		return optName
+	}
+}
+
 func (p *Parser) parseCreateExternalFileFormatStatement() (*ast.CreateExternalFileFormatStatement, error) {
 	// FILE FORMAT name WITH (options)
 	p.nextToken() // consume FILE

parser/testdata/Baselines130_CreateExternalDataSourceStatementTests130/metadata.json

@@ -1 +1 @@
-{"todo": true}
+{}

parser/testdata/Baselines140_CreateExternalDataSource140/metadata.json

@@ -1 +1 @@
-{"todo": true}
+{}

parser/testdata/Baselines160_CreateExternalDataSourceStatementTests160/metadata.json

@@ -1 +1 @@
-{"todo": true}
+{}

parser/testdata/CreateExternalDataSource140/metadata.json

@@ -1 +1 @@
-{"todo": true}
+{}

parser/testdata/CreateExternalDataSourceStatementTests130/metadata.json

@@ -1 +1 @@
-{"todo": true}
+{}

parser/testdata/CreateExternalDataSourceStatementTests160/metadata.json

@@ -1 +1 @@
-{"todo": true}
+{}