Catalog from engine plugin

Author: asmyasnikov

docs/guides/engine-plugins.md

@@ -123,6 +123,8 @@ Return `statements`: one `Statement` per query block. Each `Statement` has:
 - `parameters` — Parameters for this statement.
 - `columns` — Result columns (names, types, nullability, etc.) for this statement.
 
+You may also return **`catalog`** (optional). When present, sqlc passes it to the codegen plugin so codegen can emit model structs from the schema (tables/columns), not only per-query row types. Build an `engine.Catalog` from your schema (e.g. from `schema_sql` or DB metadata): one or more **CatalogSchema**, each with **CatalogTable** (rel = Identifier, columns = **CatalogColumn**). See `protos/engine/engine.proto` for the `Catalog`, `CatalogSchema`, `CatalogTable`, `CatalogColumn`, and `Identifier` messages.
+
 The engine package provides helpers (optional) to split `query.sql` and parse `"-- name: X :cmd"` lines in the same way as the built-in engines:
 
 - `engine.CommentSyntax` — Which comment styles to accept (`Dash`, `SlashStar`, `Hash`).
@@ -151,14 +153,22 @@ func handleParse(req *engine.ParseRequest) (*engine.ParseResponse, error) {
         st.Columns = extractColumns(b.SQL, schema)
         statements = append(statements, st)
     }
-    return &engine.ParseResponse{Statements: statements}, nil
+    resp := &engine.ParseResponse{Statements: statements}
+    if cat := buildCatalogFromSchema(schema); cat != nil {
+        resp.Catalog = cat
+    }
+    return resp, nil
 }
 ```
 
-Parameter and column types use the `Parameter` and `Column` messages in `engine.proto` (name, position, data_type, nullable, is_array, array_dims; for columns, table_name and schema_name are optional).
+Parameter and column types use the `Parameter` and `Column` messages in `engine.proto` (name, position, data_type, nullable, is_array, array_dims; for columns, table_name and schema_name are optional). If you return a **Catalog** (tables/columns from schema or DB), sqlc forwards it to the codegen plugin so generated code can include model types from the schema.
 
 Support for sqlc placeholders (`sqlc.arg()`, `sqlc.narg()`, `sqlc.slice()`, `sqlc.embed()`) is up to the plugin: it can parse and map them into `parameters` (and schema usage) as needed.
 
+#### Catalog (optional)
+
+If the plugin returns a non-empty **`catalog`** in `ParseResponse`, sqlc converts it to the codegen plugin’s Catalog and passes it in the GenerateRequest. Codegen plugins can then emit model structs from the schema (e.g. `type Author struct`) in addition to per-query row types. Build the catalog from `schema_sql` or from live DB metadata when using `connection_params`. The `engine.proto` defines `Catalog`, `CatalogSchema`, `CatalogTable`, `CatalogColumn`, and `Identifier` for this purpose.
+
 ### 3. Build and run
 
 ```bash
@@ -188,7 +198,7 @@ The protocol and Go SDK are in this repository: `protos/engine/engine.proto` and
 
 ## Architecture
 
-For each `sql[]` block, `sqlc generate` branches on the configured engine: built-in (postgresql, mysql, sqlite) use the compiler and catalog; any engine listed under `engines:` in sqlc.yaml uses the plugin path (no compiler). For the plugin path, sqlc calls Parse **once per query file**, sending the full file contents and schema (or connection params). The plugin returns **N statements** (one per query block); sqlc passes each statement to codegen as a separate query.
+For each `sql[]` block, `sqlc generate` branches on the configured engine: built-in (postgresql, mysql, sqlite) use the compiler and catalog; any engine listed under `engines:` in sqlc.yaml uses the plugin path (no compiler). For the plugin path, sqlc calls Parse **once per query file**, sending the full file contents and schema (or connection params). The plugin returns **N statements** (one per query block) and optionally a **catalog** (tables/columns from schema or DB). sqlc passes each statement to codegen as a separate query and, when present, passes the catalog so codegen can emit model types from the schema.
 
 ```
 ┌─────────────────────────────────────────────────────────────────┐

internal/cmd/plugin_engine.go

@@ -17,6 +17,7 @@ import (
 	"github.com/sqlc-dev/sqlc/internal/config"
 	"github.com/sqlc-dev/sqlc/internal/metadata"
 	"github.com/sqlc-dev/sqlc/internal/multierr"
+	"github.com/sqlc-dev/sqlc/internal/plugin"
 	"github.com/sqlc-dev/sqlc/internal/sql/ast"
 	"github.com/sqlc-dev/sqlc/internal/sql/catalog"
 	"github.com/sqlc-dev/sqlc/internal/sql/sqlpath"
@@ -164,6 +165,7 @@ func runPluginQuerySet(ctx context.Context, rp resultProcessor, name, dir string
 	}
 
 	var queries []*compiler.Query
+	var pluginCatalog *plugin.Catalog
 	merr := multierr.New()
 	set := map[string]struct{}{}
 
@@ -179,6 +181,9 @@ func runPluginQuerySet(ctx context.Context, rp resultProcessor, name, dir string
 			merr.Add(filename, queryContent, 0, err)
 			continue
 		}
+		if pluginCatalog == nil && resp.GetCatalog() != nil {
+			pluginCatalog = engineCatalogToPlugin(resp.GetCatalog())
+		}
 		baseName := filepath.Base(filename)
 		stmts := resp.GetStatements()
 		for _, st := range stmts {
@@ -204,8 +209,9 @@ func runPluginQuerySet(ctx context.Context, rp resultProcessor, name, dir string
 	}
 
 	result := &compiler.Result{
-		Catalog: catalog.New(""),
-		Queries: queries,
+		Catalog:       catalog.New(""),
+		Queries:       queries,
+		PluginCatalog: pluginCatalog,
 	}
 	return rp.ProcessResult(ctx, combo, sql, result)
 }
@@ -231,6 +237,56 @@ func loadSchemaSQL(schemaPaths []string, readFile func(string) ([]byte, error))
 	return strings.Join(parts, "\n"), nil
 }
 
+// engineCatalogToPlugin converts engine.Catalog (from ParseResponse) to plugin.Catalog for codegen.
+func engineCatalogToPlugin(engineCat *pb.Catalog) *plugin.Catalog {
+	if engineCat == nil {
+		return nil
+	}
+	var schemas []*plugin.Schema
+	for _, es := range engineCat.GetSchemas() {
+		var tables []*plugin.Table
+		for _, et := range es.GetTables() {
+			rel := et.GetRel()
+			if rel == nil {
+				continue
+			}
+			var cols []*plugin.Column
+			for _, col := range et.GetColumns() {
+				cols = append(cols, &plugin.Column{
+					Name:      col.GetName(),
+					NotNull:   col.GetNotNull(),
+					IsArray:   col.GetIsArray(),
+					ArrayDims: col.GetArrayDims(),
+					Type:      engineIDToPlugin(col.GetType()),
+					Table:     engineIDToPlugin(rel),
+				})
+			}
+			tables = append(tables, &plugin.Table{
+				Rel:     engineIDToPlugin(rel),
+				Columns: cols,
+				Comment: et.GetComment(),
+			})
+		}
+		schemas = append(schemas, &plugin.Schema{
+			Name:    es.GetName(),
+			Tables:  tables,
+			Comment: es.GetComment(),
+		})
+	}
+	return &plugin.Catalog{
+		Comment:       engineCat.GetComment(),
+		DefaultSchema: engineCat.GetDefaultSchema(),
+		Name:          engineCat.GetName(),
+		Schemas:       schemas,
+	}
+}
+
+func engineIDToPlugin(e *pb.Identifier) *plugin.Identifier {
+	if e == nil {
+		return nil
+	}
+	return &plugin.Identifier{Catalog: e.GetCatalog(), Schema: e.GetSchema(), Name: e.GetName()}
+
 // statementToCompilerQuery converts one engine.Statement from the plugin into a compiler.Query.
 func statementToCompilerQuery(st *pb.Statement, filename string) *compiler.Query {
 	if st == nil {

internal/cmd/shim.go

@@ -224,9 +224,13 @@ func pluginQueryParam(p compiler.Parameter) *plugin.Parameter {
 }
 
 func codeGenRequest(r *compiler.Result, settings config.CombinedSettings) *plugin.GenerateRequest {
+	cat := pluginCatalog(r.Catalog)
+	if r.PluginCatalog != nil {
+		cat = r.PluginCatalog
+	}
 	return &plugin.GenerateRequest{
 		Settings:    pluginSettings(r, settings),
-		Catalog:     pluginCatalog(r.Catalog),
+		Catalog:     cat,
 		Queries:     pluginQueries(r),
 		SqlcVersion: info.Version,
 	}

internal/compiler/result.go

@@ -1,10 +1,14 @@
 package compiler
 
 import (
+	"github.com/sqlc-dev/sqlc/internal/plugin"
 	"github.com/sqlc-dev/sqlc/internal/sql/catalog"
 )
 
 type Result struct {
-	Catalog *catalog.Catalog
-	Queries []*Query
+	Catalog      *catalog.Catalog
+	Queries      []*Query
+	// PluginCatalog is set by plugin engines (e.g. sqlc-engine-ydb) from ParseResponse.Catalog.
+	// When non-nil, it is passed to codegen instead of converting Catalog.
+	PluginCatalog *plugin.Catalog
 }

protos/engine/engine.proto

@@ -64,10 +64,51 @@ message Statement {
   repeated Column columns = 5;
 }
 
-// ParseResponse contains the processed statements. The plugin receives the full query file
-// and schema (or connection params); it returns one Statement per query block.
+// ParseResponse contains the processed statements and optional catalog. The plugin receives
+// the full query file and schema (or connection params); it returns one Statement per query
+// block. Catalog is derived from schema_sql or connection_params and is passed to codegen.
 message ParseResponse {
   repeated Statement statements = 1;
+  Catalog catalog = 2;
+}
+
+// Identifier identifies a catalog/schema/name (used in Catalog types).
+message Identifier {
+  string catalog = 1;
+  string schema = 2;
+  string name = 3;
+}
+
+// Catalog mirrors plugin.Catalog so engine plugins can return schema without importing plugin.
+// The sqlc app converts it to plugin.Catalog when calling codegen.
+message Catalog {
+  string comment = 1;
+  string default_schema = 2;
+  string name = 3;
+  repeated CatalogSchema schemas = 4;
+}
+
+message CatalogSchema {
+  string comment = 1;
+  string name = 2;
+  repeated CatalogTable tables = 3;
+}
+
+message CatalogTable {
+  Identifier rel = 1;
+  repeated CatalogColumn columns = 2;
+  string comment = 3;
+}
+
+message CatalogColumn {
+  string name = 1;
+  bool not_null = 3;
+  bool is_array = 4;
+  string comment = 5;
+  int32 length = 6;
+  int32 array_dims = 17;
+  Identifier type = 12;
+  Identifier table = 10;
 }
 
 // Parameter represents a query parameter.