feat: Add transformer to update table description with its table options

docs/table_options.go

@@ -0,0 +1,52 @@
+package docs
+
+import (
+	"encoding/json"
+	"errors"
+	"reflect"
+	"regexp"
+	"strings"
+
+	schemaDocs "github.com/cloudquery/codegen/jsonschema/docs"
+	"github.com/cloudquery/plugin-sdk/v4/schema"
+	invoschema "github.com/invopop/jsonschema"
+)
+
+func TableOptionsDescriptionTransformer(tableOptions any, jsonSchema string) (schema.Transform, error) {
+	var sc invoschema.Schema
+	if err := json.Unmarshal([]byte(jsonSchema), &sc); err != nil {
+		return nil, err
+	}
+	tableNamesToOptionsDocs := make(map[string]string)
+	tableOptionsType := reflect.ValueOf(tableOptions).Elem().Type()
+	for i := range tableOptionsType.NumField() {
+		field := tableOptionsType.Field(i)
+		fieldType := field.Type.String()
+		if strings.Contains(fieldType, ".") {
+			fieldType = strings.Split(fieldType, ".")[1]
+		}
+		defValue, ok := sc.Definitions[fieldType]
+		if !ok {
+			return nil, errors.New("definition not found for " + field.Name)
+		}
+		tableName := strings.Split(field.Tag.Get("json"), ",")[0]
+		if tableName == "" {
+			return nil, errors.New("json tag not found for table " + field.Name)
+		}
+		newRoot := sc
+		newRoot.ID = "Table Options"
+		newRoot.Ref = "#/$defs/" + "Table Options"
+		newRoot.Definitions["Table Options"] = defValue
+		sch, _ := json.Marshal(newRoot)
+		doc, _ := schemaDocs.Generate(sch, 1)
+		tocRegex := regexp.MustCompile(`# Table of contents[\s\S]+?##`)
+		tableNamesToOptionsDocs[tableName] = tocRegex.ReplaceAllString(doc, "##")
+	}
+
+	return func(table *schema.Table) error {
+		if tableNamesToOptionsDocs[table.Name] != "" {
+			table.Description = table.Description + "\n\n" + tableNamesToOptionsDocs[table.Name]
+		}
+		return nil
+	}, nil
+}

docs/table_options_test.go

@@ -0,0 +1,63 @@
+package docs
+
+import (
+	_ "embed"
+	"testing"
+
+	"github.com/cloudquery/plugin-sdk/v4/docs/testdata"
+	"github.com/cloudquery/plugin-sdk/v4/schema"
+	"github.com/stretchr/testify/require"
+)
+
+//go:embed testdata/schema.json
+var testSchema string
+
+type testTableOptions struct {
+	Dummy testdata.DummyTableOptions `json:"dummy,omitempty"`
+}
+
+var testTable = &schema.Table{
+	Name:        "dummy",
+	Description: "This is a dummy table",
+}
+
+func TestTableOptionsDescriptionTransformer(t *testing.T) {
+	type args struct {
+		tableOptions any
+		jsonSchema   string
+		table        *schema.Table
+	}
+	tests := []struct {
+		name     string
+		args     args
+		wantDesc string
+		wantErr  bool
+	}{
+		{
+			name:     "adds table options to description",
+			args:     args{tableOptions: &testTableOptions{}, jsonSchema: testSchema, table: testTable},
+			wantDesc: "This is a dummy table\n\n## <a name=\"Table Options\"></a>Table Options\n\n  DummyTableOptions contains configuration for the dummy table\n\n* `filter` (`string`)\n",
+		},
+		{
+			name:     "leaves description unchanged when table doesn't have options",
+			args:     args{tableOptions: &testTableOptions{}, jsonSchema: testSchema, table: &schema.Table{Description: "Foobar"}},
+			wantDesc: "Foobar",
+		},
+		{
+			name:    "errors out when table options don't match schema",
+			wantErr: true,
+			args:    args{tableOptions: &testTableOptions{}, jsonSchema: "", table: testTable},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			transformer, err := TableOptionsDescriptionTransformer(tt.args.tableOptions, tt.args.jsonSchema)
+			if tt.wantErr {
+				require.Error(t, err)
+				return
+			}
+			require.NoError(t, transformer(tt.args.table))
+			require.Equal(t, tt.wantDesc, tt.args.table.Description)
+		})
+	}
+}

docs/testdata/schema.json

@@ -0,0 +1,44 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "test/spec",
+  "$ref": "#/$defs/Spec",
+  "$defs": {
+    "Spec": {
+      "properties": {
+        "table_options": {
+          "$ref": "#/$defs/TableOptions",
+          "description": "TableOptions is a set of options to override the defaults for certain tables."
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "TableOptions": {
+      "properties": {
+        "dummy": {
+          "oneOf": [
+            {
+              "$ref": "#/$defs/DummyTableOptions",
+              "description": "Options for the dummy table."
+            },
+            {
+              "type": "null"
+            }
+          ]
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "DummyTableOptions": {
+      "properties": {
+        "filter": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object",
+      "description": "DummyTableOptions contains configuration for the dummy table"
+    }
+  }
+}

docs/testdata/tableoptions.go

@@ -0,0 +1,5 @@
+package testdata
+
+type DummyTableOptions struct {
+	Filter string `json:"filter,omitempty"`
+}

examples/simple_plugin/go.mod

@@ -1,6 +1,6 @@
 module github.com/cloudquery/plugin-sdk/examples/simple_plugin
 
-go 1.23.0
+go 1.23.4
 
 toolchain go1.24.1
 
@@ -33,6 +33,7 @@ require (
 	github.com/buger/jsonparser v1.1.1 // indirect
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/cloudquery/cloudquery-api-go v1.13.8 // indirect
+	github.com/cloudquery/codegen v0.3.26 // indirect
 	github.com/cloudquery/plugin-pb-go v1.26.9 // indirect
 	github.com/cloudquery/plugin-sdk/v2 v2.7.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect

examples/simple_plugin/go.sum

@@ -52,6 +52,8 @@ github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK3
 github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/cloudquery/cloudquery-api-go v1.13.8 h1:8n5D0G2wynbUdexr1GS8ND8i0uOwm0gXKNipJsijPe0=
 github.com/cloudquery/cloudquery-api-go v1.13.8/go.mod h1:ZhEjPkDGDL2KZKlQLUnsgQ0mPz3qC7qftr37q3q+IcA=
+github.com/cloudquery/codegen v0.3.26 h1:cWORVpObYW5/0LnjC0KO/Ocg1+vbZivJfFd+sMpb5ZY=
+github.com/cloudquery/codegen v0.3.26/go.mod h1:bg/M1JxFvNVABMLMFb/uAQmTGAyI2L/E4zL4kho9RFs=
 github.com/cloudquery/plugin-pb-go v1.26.9 h1:lkgxqIzabD6yvDm7D7oJvgO/T/bYIh7SSOojEgbMpnA=
 github.com/cloudquery/plugin-pb-go v1.26.9/go.mod h1:euhtVJKRtmWzukBxOjJyCKHPU9O9Gs5vasiBCaZVFRA=
 github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U=

go.mod

@@ -1,6 +1,6 @@
 module github.com/cloudquery/plugin-sdk/v4
 
-go 1.23.0
+go 1.23.4
 
 toolchain go1.24.1
 
@@ -65,6 +65,7 @@ require (
 	github.com/bahlo/generic-list-go v0.2.0 // indirect
 	github.com/buger/jsonparser v1.1.1 // indirect
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
+	github.com/cloudquery/codegen v0.3.26 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect
@@ -97,3 +98,5 @@ require (
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
+
+replace github.com/invopop/jsonschema => github.com/cloudquery/jsonschema v0.0.0-20240220124159-92878faa2a66

go.sum

@@ -52,6 +52,10 @@ github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK3
 github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/cloudquery/cloudquery-api-go v1.13.8 h1:8n5D0G2wynbUdexr1GS8ND8i0uOwm0gXKNipJsijPe0=
 github.com/cloudquery/cloudquery-api-go v1.13.8/go.mod h1:ZhEjPkDGDL2KZKlQLUnsgQ0mPz3qC7qftr37q3q+IcA=
+github.com/cloudquery/codegen v0.3.26 h1:cWORVpObYW5/0LnjC0KO/Ocg1+vbZivJfFd+sMpb5ZY=
+github.com/cloudquery/codegen v0.3.26/go.mod h1:bg/M1JxFvNVABMLMFb/uAQmTGAyI2L/E4zL4kho9RFs=
+github.com/cloudquery/jsonschema v0.0.0-20240220124159-92878faa2a66 h1:OZLPSIBYEfvkAUeOeM8CwTgVQy5zhayI99ishCrsFV0=
+github.com/cloudquery/jsonschema v0.0.0-20240220124159-92878faa2a66/go.mod h1:0SoZ/U7yJlNOR+fWsBSeTvTbGXB6DK01tzJ7m2Xfg34=
 github.com/cloudquery/plugin-pb-go v1.26.9 h1:lkgxqIzabD6yvDm7D7oJvgO/T/bYIh7SSOojEgbMpnA=
 github.com/cloudquery/plugin-pb-go v1.26.9/go.mod h1:euhtVJKRtmWzukBxOjJyCKHPU9O9Gs5vasiBCaZVFRA=
 github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U=
@@ -99,8 +103,6 @@ github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISH
 github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
-github.com/invopop/jsonschema v0.13.0 h1:KvpoAJWEjR3uD9Kbm2HWJmqsEaHt8lBUpd0qHcIi21E=
-github.com/invopop/jsonschema v0.13.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE=
 github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4=