Skip to content

Commit 22c5979

Browse files
SalvatoreTSalvatoreT
committed
feat: support STORAGE_EMULATOR_HOST environment variable
Allow pointing the Storage client at a local emulator via the STORAGE_EMULATOR_HOST environment variable, the emulator_host config field, or an emulator_host keyword argument. When set, the client connects to the emulator anonymously and does not require credentials or a project ID. Fixes #34022
1 parent a97550e commit 22c5979

3 files changed

Lines changed: 157 additions & 11 deletions

File tree

‎google-cloud-storage/lib/google-cloud-storage.rb‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,8 @@ module Cloud
7777
# readonly_storage = gcloud.storage scope: readonly_scope
7878
#
7979
def storage scope: nil, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil,
80-
max_elapsed_time: nil, base_interval: nil, max_interval: nil, multiplier: nil, upload_chunk_size: nil
80+
max_elapsed_time: nil, base_interval: nil, max_interval: nil, multiplier: nil, upload_chunk_size: nil,
81+
emulator_host: nil
8182
Google::Cloud.storage @project, @keyfile, scope: scope,
8283
retries: retries || @retries,
8384
timeout: timeout || @timeout,
@@ -88,7 +89,8 @@ def storage scope: nil, retries: nil, timeout: nil, open_timeout: nil, read_time
8889
base_interval: base_interval,
8990
max_interval: max_interval,
9091
multiplier: multiplier,
91-
upload_chunk_size: upload_chunk_size
92+
upload_chunk_size: upload_chunk_size,
93+
emulator_host: emulator_host
9294
end
9395

9496
##
@@ -142,7 +144,7 @@ def storage scope: nil, retries: nil, timeout: nil, open_timeout: nil, read_time
142144
def self.storage project_id = nil, credentials = nil, scope: nil,
143145
retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil,
144146
max_elapsed_time: nil, base_interval: nil, max_interval: nil, multiplier: nil,
145-
upload_chunk_size: nil
147+
upload_chunk_size: nil, emulator_host: nil
146148
require "google/cloud/storage"
147149
Google::Cloud::Storage.new project_id: project_id,
148150
credentials: credentials,
@@ -156,7 +158,8 @@ def self.storage project_id = nil, credentials = nil, scope: nil,
156158
base_interval: base_interval,
157159
max_interval: max_interval,
158160
multiplier: multiplier,
159-
upload_chunk_size: upload_chunk_size
161+
upload_chunk_size: upload_chunk_size,
162+
emulator_host: emulator_host
160163
end
161164
end
162165
end
@@ -172,6 +175,9 @@ def self.storage project_id = nil, credentials = nil, scope: nil,
172175
"STORAGE_KEYFILE", "STORAGE_KEYFILE_JSON"
173176
)
174177
end
178+
default_emulator = Google::Cloud::Config.deferred do
179+
ENV["STORAGE_EMULATOR_HOST"]
180+
end
175181

176182
config.add_field! :project_id, default_project, match: String, allow_nil: true
177183
config.add_alias! :project, :project_id
@@ -192,5 +198,6 @@ def self.storage project_id = nil, credentials = nil, scope: nil,
192198
config.add_field! :send_timeout, nil, match: Integer
193199
config.add_field! :upload_chunk_size, nil, match: Integer
194200
config.add_field! :endpoint, nil, match: String, allow_nil: true
201+
config.add_field! :emulator_host, default_emulator, match: String, allow_nil: true
195202
config.add_field! :universe_domain, nil, match: String, allow_nil: true
196203
end

‎google-cloud-storage/lib/google/cloud/storage.rb‎

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,11 @@ module Storage
7070
# @param [Integer] send_timeout How long, in seconds, before receiving response from server times out. Optional.
7171
# @param [String] endpoint Override of the endpoint host name. Optional.
7272
# If the param is nil, uses the default endpoint.
73+
# @param [String] emulator_host Address of a Storage emulator to connect
74+
# to, including the scheme (e.g. `http://localhost:9000`). Optional.
75+
# Defaults to the `STORAGE_EMULATOR_HOST` environment variable when set.
76+
# When present, the client connects to the emulator and does not require
77+
# credentials (it connects anonymously unless credentials are provided).
7378
# @param universe_domain [String] Override of the universe domain. Optional.
7479
# If unset or nil, uses the default unvierse domain
7580
# @param [Integer] upload_chunk_size The chunk size of storage upload, in bytes.
@@ -97,28 +102,36 @@ def self.new project_id: nil, credentials: nil, scope: nil, retries: nil,
97102
timeout: nil, open_timeout: nil, read_timeout: nil,
98103
send_timeout: nil, endpoint: nil, project: nil, keyfile: nil,
99104
max_elapsed_time: nil, base_interval: nil, max_interval: nil,
100-
multiplier: nil, upload_chunk_size: nil, universe_domain: nil
105+
multiplier: nil, upload_chunk_size: nil, universe_domain: nil,
106+
emulator_host: nil
101107
scope ||= configure.scope
102108
retries ||= configure.retries
103109
timeout ||= configure.timeout
104110
open_timeout ||= configure.open_timeout || timeout
105111
read_timeout ||= configure.read_timeout || timeout
106112
send_timeout ||= configure.send_timeout || timeout
107113
endpoint ||= configure.endpoint
108-
credentials ||= keyfile || default_credentials(scope: scope)
109114
max_elapsed_time ||= configure.max_elapsed_time
110115
base_interval ||= configure.base_interval
111116
max_interval ||= configure.max_interval
112117
multiplier ||= configure.multiplier
113118
upload_chunk_size ||= configure.upload_chunk_size
114119
universe_domain ||= configure.universe_domain
120+
emulator_host ||= configure.emulator_host
115121

116-
unless credentials.is_a? Google::Auth::Credentials
122+
if emulator_host
123+
endpoint = emulator_host
124+
credentials ||= keyfile
125+
else
126+
credentials ||= keyfile || default_credentials(scope: scope)
127+
end
128+
129+
if credentials && !credentials.is_a?(Google::Auth::Credentials)
117130
credentials = Storage::Credentials.new credentials, scope: scope
118131
end
119132

120133
project_id = resolve_project_id(project_id || project, credentials)
121-
raise ArgumentError, "project_id is missing" if project_id.empty?
134+
raise ArgumentError, "project_id is missing" if project_id.empty? && !emulator_host
122135

123136
Storage::Project.new(
124137
Storage::Service.new(
@@ -203,6 +216,10 @@ def self.anonymous retries: nil, timeout: nil, open_timeout: nil,
203216
# parameter `keyfile` is considered deprecated, but may also be used.)
204217
# * `endpoint` - (String) Override of the endpoint host name, or `nil`
205218
# to use the default endpoint.
219+
# * `emulator_host` - (String) Address of a Storage emulator to connect to,
220+
# including the scheme (e.g. `http://localhost:9000`), or `nil` to connect
221+
# to the live service. Defaults to the `STORAGE_EMULATOR_HOST` environment
222+
# variable when set.
206223
# * `scope` - (String, Array<String>) The OAuth 2.0 scopes controlling
207224
# the set of resources and operations that the connection can access.
208225
# * `retries` - (Integer) Number of times to retry requests on server

‎google-cloud-storage/test/google/cloud/storage_test.rb‎

Lines changed: 125 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
describe "#storage" do
2020
it "calls out to Google::Cloud.storage" do
2121
gcloud = Google::Cloud.new
22-
stubbed_storage = ->(project, keyfile, scope: nil, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, max_elapsed_time: nil, base_interval: nil, max_interval: nil, multiplier: nil, upload_chunk_size: nil) {
22+
stubbed_storage = ->(project, keyfile, scope: nil, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, max_elapsed_time: nil, base_interval: nil, max_interval: nil, multiplier: nil, upload_chunk_size: nil, emulator_host: nil) {
2323
_(project).must_be :nil?
2424
_(keyfile).must_be :nil?
2525
_(scope).must_be :nil?
@@ -44,7 +44,7 @@
4444

4545
it "passes project and keyfile to Google::Cloud.storage" do
4646
gcloud = Google::Cloud.new "project-id", "keyfile-path"
47-
stubbed_storage = ->(project, keyfile, scope: nil, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, max_elapsed_time: nil, base_interval: nil, max_interval: nil, multiplier: nil, upload_chunk_size: nil) {
47+
stubbed_storage = ->(project, keyfile, scope: nil, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, max_elapsed_time: nil, base_interval: nil, max_interval: nil, multiplier: nil, upload_chunk_size: nil, emulator_host: nil) {
4848
_(project).must_equal "project-id"
4949
_(keyfile).must_equal "keyfile-path"
5050
_(scope).must_be :nil?
@@ -69,7 +69,7 @@
6969

7070
it "passes project and keyfile and options to Google::Cloud.storage" do
7171
gcloud = Google::Cloud.new "project-id", "keyfile-path"
72-
stubbed_storage = ->(project, keyfile, scope: nil, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, max_elapsed_time: nil, base_interval: nil, max_interval: nil, multiplier: nil, upload_chunk_size: nil) {
72+
stubbed_storage = ->(project, keyfile, scope: nil, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, max_elapsed_time: nil, base_interval: nil, max_interval: nil, multiplier: nil, upload_chunk_size: nil, emulator_host: nil) {
7373
_(project).must_equal "project-id"
7474
_(keyfile).must_equal "keyfile-path"
7575
_(scope).must_equal "http://example.com/scope"
@@ -387,6 +387,103 @@ def creds.is_a? target
387387
end
388388
end
389389

390+
describe "Storage.emulator" do
391+
let(:emulator_host) { "http://localhost:9000" }
392+
let(:default_credentials) do
393+
creds = OpenStruct.new empty: true
394+
def creds.is_a? target
395+
target == Google::Auth::Credentials
396+
end
397+
creds
398+
end
399+
400+
it "uses STORAGE_EMULATOR_HOST environment variable" do
401+
emulator_check = ->(name) { (name == "STORAGE_EMULATOR_HOST") ? emulator_host : nil }
402+
stubbed_service = ->(project, credentials, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, quota_project: nil, max_elapsed_time: nil, base_interval: nil,
403+
max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil) {
404+
_(project).must_equal "project-id"
405+
_(credentials).must_be :nil?
406+
_(host).must_equal emulator_host
407+
OpenStruct.new project: project
408+
}
409+
410+
# Clear all environment variables, except STORAGE_EMULATOR_HOST
411+
ENV.stub :[], emulator_check do
412+
# Get project_id from Google Compute Engine
413+
Google::Cloud.stub :env, OpenStruct.new(project_id: "project-id") do
414+
Google::Cloud::Storage::Service.stub :new, stubbed_service do
415+
storage = Google::Cloud::Storage.new
416+
_(storage).must_be_kind_of Google::Cloud::Storage::Project
417+
_(storage.project).must_equal "project-id"
418+
end
419+
end
420+
end
421+
end
422+
423+
it "allows emulator_host to be set" do
424+
stubbed_service = ->(project, credentials, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, quota_project: nil, max_elapsed_time: nil, base_interval: nil,
425+
max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil) {
426+
_(project).must_equal "project-id"
427+
_(credentials).must_be :nil?
428+
_(host).must_equal emulator_host
429+
OpenStruct.new project: project
430+
}
431+
432+
# Clear all environment variables
433+
ENV.stub :[], nil do
434+
Google::Cloud.stub :env, OpenStruct.new(project_id: "project-id") do
435+
Google::Cloud::Storage::Service.stub :new, stubbed_service do
436+
storage = Google::Cloud::Storage.new emulator_host: emulator_host
437+
_(storage).must_be_kind_of Google::Cloud::Storage::Project
438+
_(storage.project).must_equal "project-id"
439+
end
440+
end
441+
end
442+
end
443+
444+
it "does not require a project_id when using an emulator" do
445+
stubbed_service = ->(project, credentials, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, quota_project: nil, max_elapsed_time: nil, base_interval: nil,
446+
max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil) {
447+
_(project).must_equal ""
448+
_(credentials).must_be :nil?
449+
_(host).must_equal emulator_host
450+
OpenStruct.new project: project
451+
}
452+
453+
# Clear all environment variables, and provide no project_id from any source
454+
ENV.stub :[], nil do
455+
Google::Cloud.stub :env, OpenStruct.new(project_id: nil) do
456+
Google::Cloud::Storage::Service.stub :new, stubbed_service do
457+
storage = Google::Cloud::Storage.new emulator_host: emulator_host
458+
_(storage).must_be_kind_of Google::Cloud::Storage::Project
459+
_(storage.project).must_equal ""
460+
end
461+
end
462+
end
463+
end
464+
465+
it "honors provided credentials when using an emulator" do
466+
stubbed_service = ->(project, credentials, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, quota_project: nil, max_elapsed_time: nil, base_interval: nil,
467+
max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil) {
468+
_(project).must_equal "project-id"
469+
_(credentials).must_equal default_credentials
470+
_(host).must_equal emulator_host
471+
OpenStruct.new project: project
472+
}
473+
474+
# Clear all environment variables
475+
ENV.stub :[], nil do
476+
Google::Cloud.stub :env, OpenStruct.new(project_id: "project-id") do
477+
Google::Cloud::Storage::Service.stub :new, stubbed_service do
478+
storage = Google::Cloud::Storage.new credentials: default_credentials, emulator_host: emulator_host
479+
_(storage).must_be_kind_of Google::Cloud::Storage::Project
480+
_(storage.project).must_equal "project-id"
481+
end
482+
end
483+
end
484+
end
485+
end
486+
390487
describe "Storage.configure" do
391488
let(:found_credentials) { "{}" }
392489

@@ -660,6 +757,31 @@ def creds.is_a? target
660757
end
661758
end
662759

760+
it "uses storage config for emulator_host" do
761+
stubbed_service = ->(project, credentials, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, quota_project: nil, max_elapsed_time: nil, base_interval: nil,
762+
max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil) {
763+
_(project).must_equal "project-id"
764+
_(credentials).must_be :nil?
765+
_(host).must_equal "http://localhost:9000"
766+
OpenStruct.new project: project
767+
}
768+
769+
# Clear all environment variables
770+
ENV.stub :[], nil do
771+
# Set new configuration
772+
Google::Cloud::Storage.configure do |config|
773+
config.project_id = "project-id"
774+
config.emulator_host = "http://localhost:9000"
775+
end
776+
777+
Google::Cloud::Storage::Service.stub :new, stubbed_service do
778+
storage = Google::Cloud::Storage.new
779+
_(storage).must_be_kind_of Google::Cloud::Storage::Project
780+
_(storage.project).must_equal "project-id"
781+
end
782+
end
783+
end
784+
663785
it "uses storage config for quota project" do
664786
stubbed_credentials = ->(keyfile, scope: nil) {
665787
_(keyfile).must_equal "path/to/keyfile.json"

0 commit comments

Comments
 (0)