Skip to content

Commit a36d64c

Browse files
mildbytemildbyte
authored
Merge pull request #612 from splitgraph/feature/CU-1zv5kbu-jsonschema-fixes
Add titles to builtin plugin JSONSchemas
2 parents 16e31f8 + 21cc2ec commit a36d64c

3 files changed

Lines changed: 122 additions & 33 deletions

File tree

‎splitgraph/hooks/data_source/fdw.py‎

Lines changed: 52 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -445,17 +445,35 @@ def get_description(cls) -> str:
445445

446446
credentials_schema = {
447447
"type": "object",
448-
"properties": {"username": {"type": "string"}, "password": {"type": "string"}},
448+
"properties": {
449+
"username": {
450+
"type": "string",
451+
"title": "Username",
452+
},
453+
"password": {
454+
"type": "string",
455+
"title": "Password",
456+
},
457+
},
449458
"required": ["username", "password"],
450459
}
451460

452461
params_schema = {
453462
"type": "object",
454463
"properties": {
455-
"host": {"type": "string", "description": "Remote hostname"},
456-
"port": {"type": "integer", "description": "Port"},
457-
"dbname": {"type": "string", "description": "Database name"},
458-
"remote_schema": {"type": "string", "description": "Remote schema name"},
464+
"host": {
465+
"type": "string",
466+
"title": "Host",
467+
"description": "Hostname or IP address of the PostgreSQL instance",
468+
},
469+
"port": {"type": "integer", "title": "Port", "description": "Port of the database"},
470+
"dbname": {"type": "string", "title": "Database", "description": "Database name"},
471+
"remote_schema": {
472+
"type": "string",
473+
"title": "Schema name",
474+
"description": "Schema name to import data from",
475+
"default": "public",
476+
},
459477
},
460478
"required": ["host", "port", "dbname", "remote_schema"],
461479
}
@@ -503,24 +521,27 @@ def get_remote_schema_name(self) -> str:
503521
class MongoDataSource(ForeignDataWrapperDataSource):
504522
credentials_schema = {
505523
"type": "object",
506-
"properties": {"username": {"type": "string"}, "password": {"type": "string"}},
524+
"properties": {
525+
"username": {"type": "string", "title": "Username"},
526+
"password": {"type": "string", "title": "Password"},
527+
},
507528
"required": ["username", "password"],
508529
}
509530

510531
params_schema = {
511532
"type": "object",
512533
"properties": {
513-
"host": {"type": "string"},
514-
"port": {"type": "integer"},
534+
"host": {"type": "string", "title": "Host"},
535+
"port": {"type": "integer", "title": "Port"},
515536
},
516537
"required": ["host", "port"],
517538
}
518539

519540
table_params_schema = {
520541
"type": "object",
521542
"properties": {
522-
"database": {"type": "string"},
523-
"collection": {"type": "string"},
543+
"database": {"type": "string", "title": "Database name"},
544+
"collection": {"type": "string", "title": "Collection name"},
524545
},
525546
"required": ["database", "collection"],
526547
}
@@ -573,16 +594,19 @@ def get_table_schema(self, table_name, table_schema):
573594
class MySQLDataSource(ForeignDataWrapperDataSource):
574595
credentials_schema = {
575596
"type": "object",
576-
"properties": {"username": {"type": "string"}, "password": {"type": "string"}},
597+
"properties": {
598+
"username": {"type": "string", "title": "Username"},
599+
"password": {"type": "string", "title": "Password"},
600+
},
577601
"required": ["username", "password"],
578602
}
579603

580604
params_schema = {
581605
"type": "object",
582606
"properties": {
583-
"host": {"type": "string"},
584-
"port": {"type": "integer"},
585-
"dbname": {"type": "string"},
607+
"host": {"type": "string", "title": "Host"},
608+
"port": {"type": "integer", "title": "Port"},
609+
"dbname": {"type": "string", "title": "Database name"},
586610
},
587611
"required": ["host", "port", "dbname"],
588612
}
@@ -634,16 +658,16 @@ class ElasticSearchDataSource(ForeignDataWrapperDataSource):
634658
credentials_schema = {
635659
"type": "object",
636660
"properties": {
637-
"username": {"type": "string"},
638-
"password": {"type": "string"},
661+
"username": {"type": "string", "title": "Username"},
662+
"password": {"type": "string", "title": "Password"},
639663
},
640664
}
641665

642666
params_schema = {
643667
"type": "object",
644668
"properties": {
645-
"host": {"type": "string"},
646-
"port": {"type": "integer"},
669+
"host": {"type": "string", "title": "Host"},
670+
"port": {"type": "integer", "title": "Port"},
647671
},
648672
"required": ["host", "port"],
649673
}
@@ -653,27 +677,36 @@ class ElasticSearchDataSource(ForeignDataWrapperDataSource):
653677
"properties": {
654678
"index": {
655679
"type": "string",
680+
"title": "Index name or pattern",
656681
"description": 'ES index name or pattern to use, for example, "events-*"',
657682
},
658683
"type": {
659684
"type": "string",
685+
"title": "doc_type (ES6 or earlier)",
660686
"description": "Pre-ES7 doc_type, not required in ES7 or later",
661687
},
662688
"query_column": {
663689
"type": "string",
664-
"description": "Name of the column to use to pass queries in",
690+
"title": "Query column name",
691+
"description": "Allows you to do `WHERE query = '{" "match" ": ...}'`",
692+
"default": "query",
665693
},
666694
"score_column": {
667695
"type": "string",
696+
"title": "Score column name",
668697
"description": "Name of the column with the document score",
669698
},
670699
"scroll_size": {
671700
"type": "integer",
701+
"title": "Scroll size",
672702
"description": "Fetch size, default 1000",
703+
"default": 1000,
673704
},
674705
"scroll_duration": {
675706
"type": "string",
707+
"title": "Scroll duration",
676708
"description": "How long to hold the scroll context open for, default 10m",
709+
"default": "10m",
677710
},
678711
},
679712
"required": ["index"],

‎splitgraph/ingestion/csv/__init__.py‎

Lines changed: 41 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -82,49 +82,71 @@ def query_to_csv(engine: "PsycopgEngine", query, buffer, schema: Optional[str] =
8282
class CSVDataSource(ForeignDataWrapperDataSource):
8383
credentials_schema: Dict[str, Any] = {
8484
"type": "object",
85-
"properties": {"s3_access_key": {"type": "string"}, "s3_secret_key": {"type": "string"}},
85+
"properties": {
86+
"s3_access_key": {"type": "string", "title": "AWS Access Key"},
87+
"s3_secret_key": {"type": "string", "title": "AWS Secret Access Key"},
88+
},
8689
}
8790

8891
params_schema: Dict[str, Any] = {
8992
"type": "object",
9093
"properties": {
9194
"connection": {
9295
"type": "object",
96+
"title": "Connection",
9397
"oneOf": [
9498
{
9599
"type": "object",
96100
"required": ["connection_type", "url"],
97101
"properties": {
98-
"connection_type": {"type": "string", "const": "http"},
99-
"url": {"type": "string", "description": "HTTP URL to the CSV file"},
102+
"connection_type": {
103+
"type": "string",
104+
"const": "http",
105+
"title": "Connection type",
106+
},
107+
"url": {
108+
"type": "string",
109+
"description": "HTTP URL to the CSV file",
110+
"title": "URL",
111+
},
100112
},
101113
},
102114
{
103115
"type": "object",
104116
"required": ["connection_type", "s3_endpoint", "s3_bucket"],
105117
"properties": {
106-
"connection_type": {"type": "string", "const": "s3"},
118+
"connection_type": {
119+
"type": "string",
120+
"const": "s3",
121+
"title": "Connection type",
122+
},
107123
"s3_endpoint": {
108124
"type": "string",
125+
"title": "S3 endpoint",
109126
"description": "S3 endpoint (including port if required)",
110127
},
111128
"s3_region": {
112129
"type": "string",
130+
"title": "S3 region",
113131
"description": "Region of the S3 bucket",
114132
},
115133
"s3_secure": {
134+
"title": "Secure",
116135
"type": "boolean",
117136
"description": "Whether to use HTTPS for S3 access",
118137
},
119138
"s3_bucket": {
139+
"title": "Bucket name",
120140
"type": "string",
121141
"description": "Bucket the object is in",
122142
},
123143
"s3_object": {
144+
"title": "S3 Object name",
124145
"type": "string",
125146
"description": "Limit the import to a single object",
126147
},
127148
"s3_object_prefix": {
149+
"title": "S3 Object prefix",
128150
"type": "string",
129151
"description": "Prefix for object in S3 bucket",
130152
},
@@ -134,36 +156,43 @@ class CSVDataSource(ForeignDataWrapperDataSource):
134156
},
135157
"autodetect_header": {
136158
"type": "boolean",
159+
"title": "Autodetect header",
137160
"description": "Detect whether the CSV file has a header automatically",
138161
"default": True,
139162
},
140163
"autodetect_dialect": {
141164
"type": "boolean",
165+
"title": "Autodetect dialect",
142166
"description": "Detect the CSV file's dialect (separator, quoting characters etc) automatically",
143167
"default": True,
144168
},
145169
"autodetect_encoding": {
146170
"type": "boolean",
171+
"title": "Autodetect encoding",
147172
"description": "Detect the CSV file's encoding automatically",
148173
"default": True,
149174
},
150175
"autodetect_sample_size": {
151176
"type": "integer",
177+
"title": "Sample size",
152178
"description": "Sample size, in bytes, for encoding/dialect/header detection",
153179
"default": 65536,
154180
},
155181
"schema_inference_rows": {
156182
"type": "integer",
183+
"title": "Schema inference rows",
157184
"description": "Number of rows to use for schema inference",
158185
"default": 100000,
159186
},
160187
"encoding": {
161188
"type": "string",
189+
"title": "Encoding",
162190
"description": "Encoding of the CSV file",
163191
"default": "utf-8",
164192
},
165193
"ignore_decode_errors": {
166194
"type": "boolean",
195+
"title": "Ignore decoding errors",
167196
"description": "Ignore errors when decoding the file",
168197
"default": False,
169198
},
@@ -173,11 +202,13 @@ class CSVDataSource(ForeignDataWrapperDataSource):
173202
"default": True,
174203
},
175204
"delimiter": {
205+
"title": "Delimiter",
176206
"type": "string",
177207
"description": "Character used to separate fields in the file",
178208
"default": ",",
179209
},
180210
"quotechar": {
211+
"title": "Quote character",
181212
"type": "string",
182213
"description": "Character used to quote fields",
183214
"default": '"',
@@ -188,8 +219,12 @@ class CSVDataSource(ForeignDataWrapperDataSource):
188219
table_params_schema: Dict[str, Any] = {
189220
"type": "object",
190221
"properties": {
191-
"url": {"type": "string", "description": "HTTP URL to the CSV file"},
192-
"s3_object": {"type": "string", "description": "S3 object of the CSV file"},
222+
"url": {"type": "string", "description": "HTTP URL to the CSV file", "title": "URL"},
223+
"s3_object": {
224+
"type": "string",
225+
"description": "S3 object of the CSV file",
226+
"title": "S3 object",
227+
},
193228
# Add various CSV dialect overrides to the table params schema.
194229
**{k: v for k, v in params_schema["properties"].items() if k != "connection"},
195230
},

‎splitgraph/ingestion/snowflake/__init__.py‎

Lines changed: 29 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -37,25 +37,39 @@ class SnowflakeDataSource(ForeignDataWrapperDataSource):
3737
credentials_schema: Dict[str, Any] = {
3838
"type": "object",
3939
"properties": {
40-
"username": {"type": "string", "description": "Username"},
40+
"username": {"type": "string", "title": "Username", "description": "Username"},
4141
"secret": {
4242
"type": "object",
43+
"title": "Connection secret",
4344
"oneOf": [
4445
{
4546
"type": "object",
4647
"required": ["secret_type", "password"],
4748
"properties": {
48-
"secret_type": {"type": "string", "const": "password"},
49-
"password": {"type": "string", "description": "Password"},
49+
"secret_type": {
50+
"type": "string",
51+
"const": "password",
52+
"title": "Secret type",
53+
},
54+
"password": {
55+
"type": "string",
56+
"title": "Password",
57+
"description": "Password",
58+
},
5059
},
5160
},
5261
{
5362
"type": "object",
5463
"required": ["secret_type", "private_key"],
5564
"properties": {
56-
"secret_type": {"type": "string", "const": "private_key"},
65+
"secret_type": {
66+
"type": "string",
67+
"const": "private_key",
68+
"title": "Secret type",
69+
},
5770
"private_key": {
5871
"type": "string",
72+
"title": "Private key",
5973
"description": "Private key in PEM format",
6074
},
6175
},
@@ -73,16 +87,22 @@ class SnowflakeDataSource(ForeignDataWrapperDataSource):
7387
params_schema = {
7488
"type": "object",
7589
"properties": {
76-
"database": {"type": "string", "description": "Snowflake database name"},
77-
"schema": {"type": "string", "description": "Snowflake schema"},
78-
"warehouse": {"type": "string", "description": "Warehouse name"},
79-
"role": {"type": "string", "description": "Role"},
90+
"database": {
91+
"type": "string",
92+
"title": "Database",
93+
"description": "Snowflake database name",
94+
},
95+
"schema": {"type": "string", "title": "Schema", "description": "Snowflake schema"},
96+
"warehouse": {"type": "string", "title": "Warehouse", "description": "Warehouse name"},
97+
"role": {"type": "string", "title": "Role", "description": "Role"},
8098
"batch_size": {
8199
"type": "integer",
100+
"title": "Batch size",
82101
"description": "Default fetch size for remote queries",
83102
},
84103
"envvars": {
85104
"type": "object",
105+
"title": "Environment variables",
86106
"description": "Environment variables to set on the engine side",
87107
},
88108
},
@@ -94,6 +114,7 @@ class SnowflakeDataSource(ForeignDataWrapperDataSource):
94114
"properties": {
95115
"subquery": {
96116
"type": "string",
117+
"title": "Subquery",
97118
"description": "Subquery for this table to run on the server side",
98119
}
99120
},

0 commit comments

Comments
 (0)