Skip to content

Commit 35fa4cd

Browse files
bryanckBryan Keller
bryanck
and
Bryan Keller
authored
feat: add storage options to responses (#148)
This PR adds a `storage_options` field to `CreateTableResponse` and `DescribeTableResponse`. This is a string map that can be used by a namespace server implementation to return storage parameters such as vended credentials for storage access by the client. --------- Co-authored-by: Bryan Keller <bkeller@netflix.com>
1 parent a81f637 commit 35fa4cd

20 files changed

Lines changed: 274 additions & 18 deletions

File tree

‎docs/src/spec/rest.yaml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1319,6 +1319,14 @@ components:
13191319
type: object
13201320
additionalProperties:
13211321
type: string
1322+
storage_options:
1323+
type: object
1324+
description: |
1325+
Configuration options to be used to access storage. The available
1326+
options depend on the type of storage in use. These will be
1327+
passed directly to Lance to initialize storage access.
1328+
additionalProperties:
1329+
type: string
13221330

13231331
CountTableRowsRequest:
13241332
type: object
@@ -1972,6 +1980,14 @@ components:
19721980
type: object
19731981
additionalProperties:
19741982
type: string
1983+
storage_options:
1984+
type: object
1985+
description: |
1986+
Configuration options to be used to access storage. The available
1987+
options depend on the type of storage in use. These will be
1988+
passed directly to Lance to initialize storage access.
1989+
additionalProperties:
1990+
type: string
19751991

19761992
TableExistsRequest:
19771993
type: object

‎java/lance-namespace-apache-client/api/openapi.yaml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1875,6 +1875,8 @@ components:
18751875
version: 0
18761876
properties:
18771877
key: properties
1878+
storage_options:
1879+
key: storage_options
18781880
properties:
18791881
version:
18801882
format: int64
@@ -1887,6 +1889,13 @@ components:
18871889
properties:
18881890
additionalProperties:
18891891
type: string
1892+
storage_options:
1893+
additionalProperties:
1894+
type: string
1895+
description: |
1896+
Configuration options to be used to access storage. The available
1897+
options depend on the type of storage in use. These will be
1898+
passed directly to Lance to initialize storage access.
18901899
required:
18911900
- location
18921901
- schema
@@ -2679,6 +2688,8 @@ components:
26792688
location: location
26802689
properties:
26812690
key: properties
2691+
storage_options:
2692+
key: storage_options
26822693
properties:
26832694
name:
26842695
type: string
@@ -2691,6 +2702,13 @@ components:
26912702
properties:
26922703
additionalProperties:
26932704
type: string
2705+
storage_options:
2706+
additionalProperties:
2707+
type: string
2708+
description: |
2709+
Configuration options to be used to access storage. The available
2710+
options depend on the type of storage in use. These will be
2711+
passed directly to Lance to initialize storage access.
26942712
required:
26952713
- location
26962714
- name

‎java/lance-namespace-apache-client/docs/CreateTableResponse.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
|**namespace** | **List&lt;String&gt;** | | |
1212
|**location** | **String** | | |
1313
|**properties** | **Map&lt;String, String&gt;** | | [optional] |
14+
|**storageOptions** | **Map&lt;String, String&gt;** | Configuration options to be used to access storage. The available options depend on the type of storage in use. These will be passed directly to Lance to initialize storage access. | [optional] |
1415

1516

1617

‎java/lance-namespace-apache-client/docs/DescribeTableResponse.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
|**location** | **String** | | |
1212
|**schema** | [**JsonSchema**](JsonSchema.md) | | |
1313
|**properties** | **Map&lt;String, String&gt;** | | [optional] |
14+
|**storageOptions** | **Map&lt;String, String&gt;** | Configuration options to be used to access storage. The available options depend on the type of storage in use. These will be passed directly to Lance to initialize storage access. | [optional] |
1415

1516

1617

‎java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/CreateTableResponse.java‎

Lines changed: 65 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,8 @@
3131
CreateTableResponse.JSON_PROPERTY_NAME,
3232
CreateTableResponse.JSON_PROPERTY_NAMESPACE,
3333
CreateTableResponse.JSON_PROPERTY_LOCATION,
34-
CreateTableResponse.JSON_PROPERTY_PROPERTIES
34+
CreateTableResponse.JSON_PROPERTY_PROPERTIES,
35+
CreateTableResponse.JSON_PROPERTY_STORAGE_OPTIONS
3536
})
3637
@javax.annotation.Generated(
3738
value = "org.openapitools.codegen.languages.JavaClientCodegen",
@@ -49,6 +50,9 @@ public class CreateTableResponse {
4950
public static final String JSON_PROPERTY_PROPERTIES = "properties";
5051
@javax.annotation.Nullable private Map<String, String> properties = new HashMap<>();
5152

53+
public static final String JSON_PROPERTY_STORAGE_OPTIONS = "storage_options";
54+
@javax.annotation.Nullable private Map<String, String> storageOptions = new HashMap<>();
55+
5256
public CreateTableResponse() {}
5357

5458
public CreateTableResponse name(@javax.annotation.Nonnull String name) {
@@ -163,6 +167,40 @@ public void setProperties(@javax.annotation.Nullable Map<String, String> propert
163167
this.properties = properties;
164168
}
165169

170+
public CreateTableResponse storageOptions(
171+
@javax.annotation.Nullable Map<String, String> storageOptions) {
172+
173+
this.storageOptions = storageOptions;
174+
return this;
175+
}
176+
177+
public CreateTableResponse putStorageOptionsItem(String key, String storageOptionsItem) {
178+
if (this.storageOptions == null) {
179+
this.storageOptions = new HashMap<>();
180+
}
181+
this.storageOptions.put(key, storageOptionsItem);
182+
return this;
183+
}
184+
185+
/**
186+
* Configuration options to be used to access storage. The available options depend on the type of
187+
* storage in use. These will be passed directly to Lance to initialize storage access.
188+
*
189+
* @return storageOptions
190+
*/
191+
@javax.annotation.Nullable
192+
@JsonProperty(JSON_PROPERTY_STORAGE_OPTIONS)
193+
@JsonInclude(value = JsonInclude.Include.USE_DEFAULTS)
194+
public Map<String, String> getStorageOptions() {
195+
return storageOptions;
196+
}
197+
198+
@JsonProperty(JSON_PROPERTY_STORAGE_OPTIONS)
199+
@JsonInclude(value = JsonInclude.Include.USE_DEFAULTS)
200+
public void setStorageOptions(@javax.annotation.Nullable Map<String, String> storageOptions) {
201+
this.storageOptions = storageOptions;
202+
}
203+
166204
@Override
167205
public boolean equals(Object o) {
168206
if (this == o) {
@@ -175,12 +213,13 @@ public boolean equals(Object o) {
175213
return Objects.equals(this.name, createTableResponse.name)
176214
&& Objects.equals(this.namespace, createTableResponse.namespace)
177215
&& Objects.equals(this.location, createTableResponse.location)
178-
&& Objects.equals(this.properties, createTableResponse.properties);
216+
&& Objects.equals(this.properties, createTableResponse.properties)
217+
&& Objects.equals(this.storageOptions, createTableResponse.storageOptions);
179218
}
180219

181220
@Override
182221
public int hashCode() {
183-
return Objects.hash(name, namespace, location, properties);
222+
return Objects.hash(name, namespace, location, properties, storageOptions);
184223
}
185224

186225
@Override
@@ -191,6 +230,7 @@ public String toString() {
191230
sb.append(" namespace: ").append(toIndentedString(namespace)).append("\n");
192231
sb.append(" location: ").append(toIndentedString(location)).append("\n");
193232
sb.append(" properties: ").append(toIndentedString(properties)).append("\n");
233+
sb.append(" storageOptions: ").append(toIndentedString(storageOptions)).append("\n");
194234
sb.append("}");
195235
return sb.toString();
196236
}
@@ -311,6 +351,28 @@ public String toUrlQueryString(String prefix) {
311351
}
312352
}
313353

354+
// add `storage_options` to the URL query string
355+
if (getStorageOptions() != null) {
356+
for (String _key : getStorageOptions().keySet()) {
357+
try {
358+
joiner.add(
359+
String.format(
360+
"%sstorage_options%s%s=%s",
361+
prefix,
362+
suffix,
363+
"".equals(suffix)
364+
? ""
365+
: String.format("%s%d%s", containerPrefix, _key, containerSuffix),
366+
getStorageOptions().get(_key),
367+
URLEncoder.encode(String.valueOf(getStorageOptions().get(_key)), "UTF-8")
368+
.replaceAll("\\+", "%20")));
369+
} catch (UnsupportedEncodingException e) {
370+
// Should never happen, UTF-8 is always supported
371+
throw new RuntimeException(e);
372+
}
373+
}
374+
}
375+
314376
return joiner.toString();
315377
}
316378
}

‎java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/DescribeTableResponse.java‎

Lines changed: 65 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,8 @@
2929
DescribeTableResponse.JSON_PROPERTY_VERSION,
3030
DescribeTableResponse.JSON_PROPERTY_LOCATION,
3131
DescribeTableResponse.JSON_PROPERTY_SCHEMA,
32-
DescribeTableResponse.JSON_PROPERTY_PROPERTIES
32+
DescribeTableResponse.JSON_PROPERTY_PROPERTIES,
33+
DescribeTableResponse.JSON_PROPERTY_STORAGE_OPTIONS
3334
})
3435
@javax.annotation.Generated(
3536
value = "org.openapitools.codegen.languages.JavaClientCodegen",
@@ -47,6 +48,9 @@ public class DescribeTableResponse {
4748
public static final String JSON_PROPERTY_PROPERTIES = "properties";
4849
@javax.annotation.Nullable private Map<String, String> properties = new HashMap<>();
4950

51+
public static final String JSON_PROPERTY_STORAGE_OPTIONS = "storage_options";
52+
@javax.annotation.Nullable private Map<String, String> storageOptions = new HashMap<>();
53+
5054
public DescribeTableResponse() {}
5155

5256
public DescribeTableResponse version(@javax.annotation.Nullable Long version) {
@@ -154,6 +158,40 @@ public void setProperties(@javax.annotation.Nullable Map<String, String> propert
154158
this.properties = properties;
155159
}
156160

161+
public DescribeTableResponse storageOptions(
162+
@javax.annotation.Nullable Map<String, String> storageOptions) {
163+
164+
this.storageOptions = storageOptions;
165+
return this;
166+
}
167+
168+
public DescribeTableResponse putStorageOptionsItem(String key, String storageOptionsItem) {
169+
if (this.storageOptions == null) {
170+
this.storageOptions = new HashMap<>();
171+
}
172+
this.storageOptions.put(key, storageOptionsItem);
173+
return this;
174+
}
175+
176+
/**
177+
* Configuration options to be used to access storage. The available options depend on the type of
178+
* storage in use. These will be passed directly to Lance to initialize storage access.
179+
*
180+
* @return storageOptions
181+
*/
182+
@javax.annotation.Nullable
183+
@JsonProperty(JSON_PROPERTY_STORAGE_OPTIONS)
184+
@JsonInclude(value = JsonInclude.Include.USE_DEFAULTS)
185+
public Map<String, String> getStorageOptions() {
186+
return storageOptions;
187+
}
188+
189+
@JsonProperty(JSON_PROPERTY_STORAGE_OPTIONS)
190+
@JsonInclude(value = JsonInclude.Include.USE_DEFAULTS)
191+
public void setStorageOptions(@javax.annotation.Nullable Map<String, String> storageOptions) {
192+
this.storageOptions = storageOptions;
193+
}
194+
157195
@Override
158196
public boolean equals(Object o) {
159197
if (this == o) {
@@ -166,12 +204,13 @@ public boolean equals(Object o) {
166204
return Objects.equals(this.version, describeTableResponse.version)
167205
&& Objects.equals(this.location, describeTableResponse.location)
168206
&& Objects.equals(this.schema, describeTableResponse.schema)
169-
&& Objects.equals(this.properties, describeTableResponse.properties);
207+
&& Objects.equals(this.properties, describeTableResponse.properties)
208+
&& Objects.equals(this.storageOptions, describeTableResponse.storageOptions);
170209
}
171210

172211
@Override
173212
public int hashCode() {
174-
return Objects.hash(version, location, schema, properties);
213+
return Objects.hash(version, location, schema, properties, storageOptions);
175214
}
176215

177216
@Override
@@ -182,6 +221,7 @@ public String toString() {
182221
sb.append(" location: ").append(toIndentedString(location)).append("\n");
183222
sb.append(" schema: ").append(toIndentedString(schema)).append("\n");
184223
sb.append(" properties: ").append(toIndentedString(properties)).append("\n");
224+
sb.append(" storageOptions: ").append(toIndentedString(storageOptions)).append("\n");
185225
sb.append("}");
186226
return sb.toString();
187227
}
@@ -286,6 +326,28 @@ public String toUrlQueryString(String prefix) {
286326
}
287327
}
288328

329+
// add `storage_options` to the URL query string
330+
if (getStorageOptions() != null) {
331+
for (String _key : getStorageOptions().keySet()) {
332+
try {
333+
joiner.add(
334+
String.format(
335+
"%sstorage_options%s%s=%s",
336+
prefix,
337+
suffix,
338+
"".equals(suffix)
339+
? ""
340+
: String.format("%s%d%s", containerPrefix, _key, containerSuffix),
341+
getStorageOptions().get(_key),
342+
URLEncoder.encode(String.valueOf(getStorageOptions().get(_key)), "UTF-8")
343+
.replaceAll("\\+", "%20")));
344+
} catch (UnsupportedEncodingException e) {
345+
// Should never happen, UTF-8 is always supported
346+
throw new RuntimeException(e);
347+
}
348+
}
349+
}
350+
289351
return joiner.toString();
290352
}
291353
}

‎java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/api/TableApi.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -373,7 +373,7 @@ default ResponseEntity<CreateTableResponse> createTable(
373373
for (MediaType mediaType : MediaType.parseMediaTypes(request.getHeader("Accept"))) {
374374
if (mediaType.isCompatibleWith(MediaType.valueOf("application/json"))) {
375375
String exampleString =
376-
"{ \"name\" : \"name\", \"namespace\" : [ \"namespace\", \"namespace\" ], \"location\" : \"location\", \"properties\" : { \"key\" : \"properties\" } }";
376+
"{ \"name\" : \"name\", \"namespace\" : [ \"namespace\", \"namespace\" ], \"location\" : \"location\", \"properties\" : { \"key\" : \"properties\" }, \"storage_options\" : { \"key\" : \"storage_options\" } }";
377377
ApiUtil.setExampleResponse(request, "application/json", exampleString);
378378
break;
379379
}
@@ -1059,7 +1059,7 @@ default ResponseEntity<DescribeTableResponse> describeTable(
10591059
for (MediaType mediaType : MediaType.parseMediaTypes(request.getHeader("Accept"))) {
10601060
if (mediaType.isCompatibleWith(MediaType.valueOf("application/json"))) {
10611061
String exampleString =
1062-
"{ \"schema\" : { \"metadata\" : { \"key\" : \"metadata\" }, \"fields\" : [ { \"metadata\" : { \"key\" : \"metadata\" }, \"nullable\" : true, \"name\" : \"name\", \"type\" : { \"length\" : 0, \"fields\" : [ null, null ], \"type\" : \"type\" } }, { \"metadata\" : { \"key\" : \"metadata\" }, \"nullable\" : true, \"name\" : \"name\", \"type\" : { \"length\" : 0, \"fields\" : [ null, null ], \"type\" : \"type\" } } ] }, \"location\" : \"location\", \"version\" : 0, \"properties\" : { \"key\" : \"properties\" } }";
1062+
"{ \"schema\" : { \"metadata\" : { \"key\" : \"metadata\" }, \"fields\" : [ { \"metadata\" : { \"key\" : \"metadata\" }, \"nullable\" : true, \"name\" : \"name\", \"type\" : { \"length\" : 0, \"fields\" : [ null, null ], \"type\" : \"type\" } }, { \"metadata\" : { \"key\" : \"metadata\" }, \"nullable\" : true, \"name\" : \"name\", \"type\" : { \"length\" : 0, \"fields\" : [ null, null ], \"type\" : \"type\" } } ] }, \"location\" : \"location\", \"version\" : 0, \"properties\" : { \"key\" : \"properties\" }, \"storage_options\" : { \"key\" : \"storage_options\" } }";
10631063
ApiUtil.setExampleResponse(request, "application/json", exampleString);
10641064
break;
10651065
}

0 commit comments

Comments
 (0)