chore: add struct tags and update docs (#477)

Chief-Rishab · web-flow · commit 4c182109bfd0 · 2023-03-22T12:35:21.000-05:00
diff --git a/plugins/extractors/bigquery/bigquery.go b/plugins/extractors/bigquery/bigquery.go
@@ -34,25 +34,25 @@ var summary string
 
 // Config holds the set of configuration for the bigquery extractor
 type Config struct {
-	ProjectID string `mapstructure:"project_id" validate:"required"`
+	ProjectID string `json:"project_id" yaml:"project_id" mapstructure:"project_id" validate:"required"`
 	// ServiceAccountBase64 takes precedence over ServiceAccountJSON field
-	ServiceAccountBase64 string   `mapstructure:"service_account_base64"`
-	ServiceAccountJSON   string   `mapstructure:"service_account_json"`
-	MaxPageSize          int      `mapstructure:"max_page_size"`
-	TablePattern         string   `mapstructure:"table_pattern"`
-	Exclude              Exclude  `mapstructure:"exclude"`
-	IncludeColumnProfile bool     `mapstructure:"include_column_profile"`
-	MaxPreviewRows       int      `mapstructure:"max_preview_rows" default:"30"`
-	IsCollectTableUsage  bool     `mapstructure:"collect_table_usage" default:"false"`
-	UsagePeriodInDay     int64    `mapstructure:"usage_period_in_day" default:"7"`
-	UsageProjectIDs      []string `mapstructure:"usage_project_ids"`
+	ServiceAccountBase64 string   `json:"service_account_base64" yaml:"service_account_base64" mapstructure:"service_account_base64"`
+	ServiceAccountJSON   string   `json:"service_account_json" yaml:"service_account_json" mapstructure:"service_account_json"`
+	MaxPageSize          int      `json:"max_page_size" yaml:"max_page_size" mapstructure:"max_page_size"`
+	TablePattern         string   `json:"table_pattern" yaml:"table_pattern" mapstructure:"table_pattern"`
+	Exclude              Exclude  `json:"exclude" yaml:"exclude" mapstructure:"exclude"`
+	IncludeColumnProfile bool     `json:"include_column_profile" yaml:"include_column_profile" mapstructure:"include_column_profile"`
+	MaxPreviewRows       int      `json:"max_preview_rows" yaml:"max_preview_rows" mapstructure:"max_preview_rows" default:"30"`
+	IsCollectTableUsage  bool     `json:"collect_table_usage" yaml:"collect_table_usage" mapstructure:"collect_table_usage" default:"false"`
+	UsagePeriodInDay     int64    `json:"usage_period_in_day" yaml:"usage_period_in_day" mapstructure:"usage_period_in_day" default:"7"`
+	UsageProjectIDs      []string `json:"usage_project_ids" yaml:"usage_project_ids" mapstructure:"usage_project_ids"`
 }
 
 type Exclude struct {
 	// list of datasetIDs
-	Datasets []string `mapstructure:"datasets"`
+	Datasets []string `json:"datasets" yaml:"datasets" mapstructure:"datasets"`
 	// list of tableNames in format - datasetID.tableID
-	Tables []string `mapstructure:"tables"`
+	Tables []string `json:"tables" yaml:"tables" mapstructure:"tables"`
 }
 
 const (
diff --git a/plugins/extractors/bigtable/bigtable.go b/plugins/extractors/bigtable/bigtable.go
@@ -27,7 +27,7 @@ const (
 
 // Config holds the configurations for the bigtable extractor
 type Config struct {
-	ProjectID string `mapstructure:"project_id" validate:"required"`
+	ProjectID string `json:"project_id" yaml:"project_id" mapstructure:"project_id" validate:"required"`
 }
 
 var info = plugins.Info{
diff --git a/plugins/extractors/caramlstore/caramlstore.go b/plugins/extractors/caramlstore/caramlstore.go
@@ -29,9 +29,9 @@ var summary string
 
 // Config holds the set of configuration for the CaraML Store extractor
 type Config struct {
-	URL            string        `mapstructure:"url" validate:"required"`
-	MaxSizeInMB    int           `mapstructure:"max_size_in_mb"`
-	RequestTimeout time.Duration `mapstructure:"request_timeout" validate:"min=1ms" default:"10s"`
+	URL            string        `json:"url" yaml:"url" mapstructure:"url" validate:"required"`
+	MaxSizeInMB    int           `json:"max_size_in_mb" yaml:"max_size_in_mb" mapstructure:"max_size_in_mb"`
+	RequestTimeout time.Duration `json:"request_timeout" yaml:"request_timeout" mapstructure:"request_timeout" validate:"min=1ms" default:"10s"`
 }
 
 var sampleConfig = `url: caraml-store.com:80`
diff --git a/plugins/extractors/cassandra/cassandra.go b/plugins/extractors/cassandra/cassandra.go
@@ -37,10 +37,10 @@ const (
 
 // Config holds the set of configuration for the cassandra extractor
 type Config struct {
-	UserID   string `mapstructure:"user_id" validate:"required"`
-	Password string `mapstructure:"password" validate:"required"`
-	Host     string `mapstructure:"host" validate:"required"`
-	Port     int    `mapstructure:"port" validate:"required"`
+	UserID   string `json:"user_id" yaml:"user_id" mapstructure:"user_id" validate:"required"`
+	Password string `json:"password" yaml:"password" mapstructure:"password" validate:"required"`
+	Host     string `json:"host" yaml:"host" mapstructure:"host" validate:"required"`
+	Port     int    `json:"port" yaml:"port" mapstructure:"port" validate:"required"`
 }
 
 var sampleConfig = `
diff --git a/plugins/extractors/clickhouse/clickhouse.go b/plugins/extractors/clickhouse/clickhouse.go
@@ -22,7 +22,7 @@ var summary string
 
 // Config holds the connection URL for the extractor
 type Config struct {
-	ConnectionURL string `mapstructure:"connection_url" validate:"required"`
+	ConnectionURL string `json:"connection_url" yaml:"connection_url" mapstructure:"connection_url" validate:"required"`
 }
 
 var sampleConfig = `connection_url: "tcp://localhost:3306?username=admin&password=pass123&debug=true"`
diff --git a/plugins/extractors/couchdb/couchdb.go b/plugins/extractors/couchdb/couchdb.go
@@ -27,7 +27,7 @@ var defaultDBList = []string{
 
 // Config holds the connection URL for the extractor
 type Config struct {
-	ConnectionURL string `mapstructure:"connection_url" validate:"required"`
+	ConnectionURL string `json:"connection_url" yaml:"connection_url" mapstructure:"connection_url" validate:"required"`
 }
 
 var sampleConfig = `connection_url: http://admin:pass123@localhost:3306/`
@@ -139,10 +139,10 @@ func (e *Extractor) processTable(ctx context.Context, dbName string, docID strin
 	// push table to channel
 	e.emit(models.NewRecord(&v1beta2.Asset{
 		Urn:     models.NewURN("couchdb", e.UrnScope, "table", fmt.Sprintf("%s.%s", dbName, docID)),
-		Name: docID,
-		Type: "table",
+		Name:    docID,
+		Type:    "table",
 		Service: "couchdb",
-		Data: table,
+		Data:    table,
 	}))
 
 	return
diff --git a/plugins/extractors/elastic/elastic.go b/plugins/extractors/elastic/elastic.go
@@ -22,9 +22,9 @@ import (
 var summary string
 
 type Config struct {
-	User     string `mapstructure:"user"`
-	Password string `mapstructure:"password"`
-	Host     string `mapstructure:"host" validate:"required"`
+	User     string `json:"user" yaml:"user" mapstructure:"user"`
+	Password string `json:"password" yaml:"password" mapstructure:"password"`
+	Host     string `json:"host" yaml:"host" mapstructure:"host" validate:"required"`
 }
 
 var sampleConfig = `
diff --git a/plugins/extractors/gcs/gcs.go b/plugins/extractors/gcs/gcs.go
@@ -25,9 +25,9 @@ var summary string
 
 // Config holds the set of configuration for the extractor
 type Config struct {
-	ProjectID          string `mapstructure:"project_id" validate:"required"`
-	ServiceAccountJSON string `mapstructure:"service_account_json"`
-	ExtractBlob        bool   `mapstructure:"extract_blob"`
+	ProjectID          string `json:"project_id" yaml:"project_id" mapstructure:"project_id" validate:"required"`
+	ServiceAccountJSON string `json:"service_account_json" yaml:"service_account_json" mapstructure:"service_account_json"`
+	ExtractBlob        bool   `json:"extract_blob" yaml:"extract_blob" mapstructure:"extract_blob"`
 }
 
 var sampleConfig = `
diff --git a/plugins/extractors/github/github.go b/plugins/extractors/github/github.go
@@ -21,8 +21,8 @@ var summary string
 
 // Config holds the set of configuration for the extractor
 type Config struct {
-	Org   string `mapstructure:"org" validate:"required"`
-	Token string `mapstructure:"token" validate:"required"`
+	Org   string `json:"org" yaml:"org" mapstructure:"org" validate:"required"`
+	Token string `json:"token" yaml:"token" mapstructure:"token" validate:"required"`
 }
 
 var sampleConfig = `
@@ -96,8 +96,8 @@ func (e *Extractor) Extract(ctx context.Context, emit plugins.Emit) (err error)
 		emit(models.NewRecord(&v1beta2.Asset{
 			Urn:     models.NewURN("github", e.UrnScope, "user", usr.GetNodeID()),
 			Service: "github",
-			Type: "user",
-			Data: u,
+			Type:    "user",
+			Data:    u,
 		}))
 	}
 
diff --git a/plugins/extractors/grafana/grafana.go b/plugins/extractors/grafana/grafana.go
@@ -21,8 +21,8 @@ var summary string
 
 // Config holds the set of configuration for the grafana extractor
 type Config struct {
-	BaseURL string `mapstructure:"base_url" validate:"required"`
-	APIKey  string `mapstructure:"api_key" validate:"required"`
+	BaseURL string `json:"base_url" yaml:"base_url" mapstructure:"base_url" validate:"required"`
+	APIKey  string `json:"api_key" yaml:"api_key" mapstructure:"api_key" validate:"required"`
 }
 
 var sampleConfig = `
diff --git a/plugins/extractors/gsuite/gsuite.go b/plugins/extractors/gsuite/gsuite.go
@@ -23,8 +23,8 @@ import (
 var summary string
 
 type Config struct {
-	ServiceAccountJSON string `mapstructure:"service_account_json" validate:"required"`
-	UserEmail          string `mapstructure:"user_email" validate:"required"`
+	ServiceAccountJSON string `json:"service_account_json" yaml:"service_account_json" mapstructure:"service_account_json" validate:"required"`
+	UserEmail          string `json:"user_email" yaml:"user_email" mapstructure:"user_email" validate:"required"`
 }
 
 var sampleConfig = `
diff --git a/plugins/extractors/kafka/kafka.go b/plugins/extractors/kafka/kafka.go
@@ -30,8 +30,8 @@ var defaultTopics = map[string]byte{
 
 // Config holds the set of configuration for the kafka extractor
 type Config struct {
-	Broker string     `mapstructure:"broker" validate:"required"`
-	Auth   AuthConfig `mapstructure:"auth_config"`
+	Broker string     `json:"broker" yaml:"broker" mapstructure:"broker" validate:"required"`
+	Auth   AuthConfig `json:"auth_config" yaml:"auth_config" mapstructure:"auth_config"`
 }
 
 type AuthConfig struct {
diff --git a/plugins/extractors/mariadb/mariadb.go b/plugins/extractors/mariadb/mariadb.go
@@ -31,7 +31,7 @@ var defaultDBList = []string{
 
 // Config holds the connection URL for the extractor
 type Config struct {
-	ConnectionURL string `mapstructure:"connection_url" validate:"required"`
+	ConnectionURL string `json:"connection_url" yaml:"connection_url" mapstructure:"connection_url" validate:"required"`
 }
 
 var sampleConfig = `connection_url: "admin:pass123@tcp(localhost:3306)/"`
@@ -139,10 +139,10 @@ func (e *Extractor) processTable(database string, tableName string) (err error)
 	// push table to channel
 	e.emit(models.NewRecord(&v1beta2.Asset{
 		Urn:     models.NewURN("mariadb", e.UrnScope, "table", fmt.Sprintf("%s.%s", database, tableName)),
-		Name: tableName,
-		Type: "table",
+		Name:    tableName,
+		Type:    "table",
 		Service: "mariadb",
-		Data: data,
+		Data:    data,
 	}))
 	return
 }
diff --git a/plugins/extractors/merlin/merlin.go b/plugins/extractors/merlin/merlin.go
@@ -33,10 +33,10 @@ var summary string
 
 // Config holds the set of configuration for the Merlin extractor.
 type Config struct {
-	URL                  string        `mapstructure:"url" validate:"required"`
-	ServiceAccountBase64 string        `mapstructure:"service_account_base64"`
-	RequestTimeout       time.Duration `mapstructure:"request_timeout" validate:"min=1ms" default:"10s"`
-	WorkerCount          int           `mapstructure:"worker_count" validate:"min=1" default:"5"`
+	URL                  string        `json:"url" yaml:"url" mapstructure:"url" validate:"required"`
+	ServiceAccountBase64 string        `json:"service_account_base64" yaml:"service_account_base64" mapstructure:"service_account_base64"`
+	RequestTimeout       time.Duration `json:"request_timeout" yaml:"request_timeout" mapstructure:"request_timeout" validate:"min=1ms" default:"10s"`
+	WorkerCount          int           `json:"worker_count" yaml:"worker_count" mapstructure:"worker_count" validate:"min=1" default:"5"`
 }
 
 var sampleConfig = heredoc.Doc(`
diff --git a/plugins/extractors/metabase/metabase.go b/plugins/extractors/metabase/metabase.go
@@ -34,11 +34,11 @@ var info = plugins.Info{
 
 // Config holds the set of configuration for the metabase extractor
 type Config struct {
-	Host          string `mapstructure:"host" validate:"required"`
-	InstanceLabel string `mapstructure:"instance_label" validate:"required"`
-	Username      string `mapstructure:"username" validate:"required_without=SessionID"`
-	Password      string `mapstructure:"password"`
-	SessionID     string `mapstructure:"session_id"`
+	Host          string `json:"host" yaml:"host" mapstructure:"host" validate:"required"`
+	InstanceLabel string `json:"instance_label" yaml:"instance_label" mapstructure:"instance_label" validate:"required"`
+	Username      string `json:"username" yaml:"username" mapstructure:"username" validate:"required_without=SessionID"`
+	Password      string `json:"password" yaml:"password" mapstructure:"password"`
+	SessionID     string `json:"session_id" yaml:"session_id" mapstructure:"session_id"`
 }
 
 // Extractor manages the extraction of data
diff --git a/plugins/extractors/mongodb/mongodb.go b/plugins/extractors/mongodb/mongodb.go
@@ -31,7 +31,7 @@ var defaultCollections = []string{
 
 // Config holds the connection URL for the extractor
 type Config struct {
-	ConnectionURL string `mapstructure:"connection_url" validate:"required"`
+	ConnectionURL string `json:"connection_url" yaml:"connection_url" mapstructure:"connection_url" validate:"required"`
 }
 
 var sampleConfig = `connection_url: "mongodb://admin:pass123@localhost:3306"`
diff --git a/plugins/extractors/mssql/mssql.go b/plugins/extractors/mssql/mssql.go
@@ -33,7 +33,7 @@ var defaultDBList = []string{
 
 // Config holds the connection URL for the extractor
 type Config struct {
-	ConnectionURL string `mapstructure:"connection_url" validate:"required"`
+	ConnectionURL string `json:"connection_url" yaml:"connection_url" mapstructure:"connection_url" validate:"required"`
 }
 
 var sampleConfig = `connection_url: "sqlserver://admin:pass123@localhost:3306/"`
diff --git a/plugins/extractors/mysql/README.md b/plugins/extractors/mysql/README.md
@@ -7,7 +7,6 @@ source:
   name: mysql
   config:
     connection_url: admin:pass123@tcp(localhost:3306)/
-    identifier: my-mysql
 ```
 
 ## Inputs
diff --git a/plugins/extractors/mysql/mysql.go b/plugins/extractors/mysql/mysql.go
@@ -32,7 +32,7 @@ var defaultDBList = []string{
 
 // Config holds the connection URL for the extractor
 type Config struct {
-	ConnectionURL string `mapstructure:"connection_url" validate:"required"`
+	ConnectionURL string `json:"connection_url" yaml:"connection_url" mapstructure:"connection_url" validate:"required"`
 }
 
 var sampleConfig = `connection_url: "admin:pass123@tcp(localhost:3306)/"`
diff --git a/plugins/extractors/optimus/optimus.go b/plugins/extractors/optimus/optimus.go
@@ -22,8 +22,8 @@ var summary string
 
 // Config holds the set of configuration for the bigquery extractor
 type Config struct {
-	Host        string `mapstructure:"host" validate:"required"`
-	MaxSizeInMB int    `mapstructure:"max_size_in_mb"`
+	Host        string `json:"host" yaml:"host" mapstructure:"host" validate:"required"`
+	MaxSizeInMB int    `json:"max_size_in_mb" yaml:"max_size_in_mb" mapstructure:"max_size_in_mb"`
 }
 
 var sampleConfig = `host: optimus.com:80`
diff --git a/plugins/extractors/oracle/oracle.go b/plugins/extractors/oracle/oracle.go
@@ -23,7 +23,7 @@ var summary string
 
 // Config holds the set of configuration options for the extractor
 type Config struct {
-	ConnectionURL string `mapstructure:"connection_url" validate:"required"`
+	ConnectionURL string `json:"connection_url" yaml:"connection_url" mapstructure:"connection_url" validate:"required"`
 }
 
 var sampleConfig = `connection_url: oracle://username:passwd@localhost:1521/xe`
diff --git a/plugins/extractors/postgres/postgres.go b/plugins/extractors/postgres/postgres.go
@@ -31,8 +31,8 @@ var defaultDBList = []string{"information_schema", "root", "postgres"}
 
 // Config holds the set of configuration options for the extractor
 type Config struct {
-	ConnectionURL string `mapstructure:"connection_url" validate:"required"`
-	Exclude       string `mapstructure:"exclude"`
+	ConnectionURL string `json:"connection_url" yaml:"connection_url" mapstructure:"connection_url" validate:"required"`
+	Exclude       string `json:"exclude" yaml:"exclude" mapstructure:"exclude"`
 }
 
 var sampleConfig = `
diff --git a/plugins/extractors/presto/presto.go b/plugins/extractors/presto/presto.go
@@ -25,8 +25,8 @@ var summary string
 
 // Config holds the set of configuration options for the extractor
 type Config struct {
-	ConnectionURL string `mapstructure:"connection_url" validate:"required"`
-	Exclude       string `mapstructure:"exclude_catalog"`
+	ConnectionURL string `json:"connection_url" yaml:"connection_url" mapstructure:"connection_url" validate:"required"`
+	Exclude       string `json:"exclude_catalog" yaml:"exclude_catalog" mapstructure:"exclude_catalog"`
 }
 
 var sampleConfig = `
diff --git a/plugins/extractors/redash/redash.go b/plugins/extractors/redash/redash.go
@@ -25,8 +25,8 @@ var summary string
 
 // Config holds the set of configuration for the redash extractor
 type Config struct {
-	BaseURL string `mapstructure:"base_url" validate:"required"`
-	ApiKey  string `mapstructure:"api_key" validate:"required"`
+	BaseURL string `json:"base_url" yaml:"base_url" mapstructure:"base_url" validate:"required"`
+	ApiKey  string `json:"api_key" yaml:"api_key" mapstructure:"api_key" validate:"required"`
 }
 
 var sampleConfig = `
diff --git a/plugins/extractors/redshift/redshift.go b/plugins/extractors/redshift/redshift.go
@@ -22,11 +22,11 @@ var summary string
 
 // Config holds the set of configuration for the metabase extractor
 type Config struct {
-	ClusterID string `mapstructure:"cluster_id" validate:"required"`
-	DBName    string `mapstructure:"db_name" validate:"required"`
-	DBUser    string `mapstructure:"db_user" validate:"required"`
-	AWSRegion string `mapstructure:"aws_region" validate:"required"`
-	Exclude   string `mapstructure:"exclude"`
+	ClusterID string `json:"cluster_id" yaml:"cluster_id" mapstructure:"cluster_id" validate:"required"`
+	DBName    string `json:"db_name" yaml:"db_name" mapstructure:"db_name" validate:"required"`
+	DBUser    string `json:"db_user" yaml:"db_user" mapstructure:"db_user" validate:"required"`
+	AWSRegion string `json:"aws_region" yaml:"aws_region" mapstructure:"aws_region" validate:"required"`
+	Exclude   string `json:"exclude" yaml:"exclude" mapstructure:"exclude"`
 }
 
 var sampleConfig = `
diff --git a/plugins/extractors/snowflake/snowflake.go b/plugins/extractors/snowflake/snowflake.go
@@ -23,7 +23,7 @@ var summary string
 
 // Config holds the connection URL for the extractor
 type Config struct {
-	ConnectionURL string `mapstructure:"connection_url" validate:"required"`
+	ConnectionURL string `json:"connection_url" yaml:"connection_url" mapstructure:"connection_url" validate:"required"`
 }
 
 var sampleConfig = `connection_url: "user:password@my_organization-my_account/mydb"`
diff --git a/plugins/extractors/superset/superset.go b/plugins/extractors/superset/superset.go
@@ -25,10 +25,10 @@ var summary string
 
 // Config holds the set of configuration for the superset extractor
 type Config struct {
-	Username string `mapstructure:"username" validate:"required"`
-	Password string `mapstructure:"password" validate:"required"`
-	Host     string `mapstructure:"host" validate:"required"`
-	Provider string `mapstructure:"provider" validate:"required"`
+	Username string `json:"username" yaml:"username" mapstructure:"username" validate:"required"`
+	Password string `json:"password" yaml:"password" mapstructure:"password" validate:"required"`
+	Host     string `json:"host" yaml:"host" mapstructure:"host" validate:"required"`
+	Provider string `json:"provider" yaml:"provider" mapstructure:"provider" validate:"required"`
 }
 
 var sampleConfig = `
diff --git a/plugins/extractors/tableau/README.md b/plugins/extractors/tableau/README.md
@@ -12,7 +12,6 @@ source:
   config:
     host: http://server.tableau.com
     version: 3.12
-    identifier: my-tableau
     username: meteor_user
     password: xxxxxxxxxx
 ```
diff --git a/plugins/extractors/tableau/tableau.go b/plugins/extractors/tableau/tableau.go
diff --git a/plugins/sinks/compass/README.md b/plugins/sinks/compass/README.md
diff --git a/plugins/sinks/compass/sink.go b/plugins/sinks/compass/sink.go
diff --git a/test/e2e/mysql_kafka.yml b/test/e2e/mysql_kafka.yml

Original file line number	Diff line number	Diff line change
`@@ -27,7 +27,7 @@ const (`
`27`	`27`
`28`	`28`	`// Config holds the configurations for the bigtable extractor`
`29`	`29`	`type Config struct {`
`30`		- ProjectID string `mapstructure:"project_id" validate:"required"`
	`30`	+ ProjectID string `json:"project_id" yaml:"project_id" mapstructure:"project_id" validate:"required"`
`31`	`31`	`}`
`32`	`32`
`33`	`33`	`var info = plugins.Info{`
Original file line number	Diff line number	Diff line change
`@@ -22,7 +22,7 @@ var summary string`
`22`	`22`
`23`	`23`	`// Config holds the connection URL for the extractor`
`24`	`24`	`type Config struct {`
`25`		- ConnectionURL string `mapstructure:"connection_url" validate:"required"`
	`25`	+ ConnectionURL string `json:"connection_url" yaml:"connection_url" mapstructure:"connection_url" validate:"required"`
`26`	`26`	`}`
`27`	`27`
`28`	`28`	var sampleConfig = `connection_url: "tcp://localhost:3306?username=admin&password=pass123&debug=true"`
Original file line number	Diff line number	Diff line change
`@@ -21,8 +21,8 @@ var summary string`
`21`	`21`
`22`	`22`	`// Config holds the set of configuration for the grafana extractor`
`23`	`23`	`type Config struct {`
`24`		- BaseURL string `mapstructure:"base_url" validate:"required"`
`25`		- APIKey string `mapstructure:"api_key" validate:"required"`
	`24`	+ BaseURL string `json:"base_url" yaml:"base_url" mapstructure:"base_url" validate:"required"`
	`25`	+ APIKey string `json:"api_key" yaml:"api_key" mapstructure:"api_key" validate:"required"`
`26`	`26`	`}`
`27`	`27`
`28`	`28`	var sampleConfig = `