Skip to content

Commit 0f02ee2

Browse files
jessicaKoehnkejessicaKoehnke
authored
Merge pull request opentripplanner#6648 from HBTGmbH/gcs-host-configurable
Make Google Cloud Storage host configurable
2 parents 29834b5 + 54cba2a commit 0f02ee2

15 files changed

Lines changed: 324 additions & 63 deletions

File tree

‎application/src/ext-test/java/org/opentripplanner/ext/datastore/gs/GsConfigurationDocTest.java‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
package org.opentripplanner.ext.datastore.gs;
2+
3+
import static org.opentripplanner.framework.application.OtpFileNames.BUILD_CONFIG_FILENAME;
4+
import static org.opentripplanner.framework.io.FileUtils.assertFileEquals;
5+
import static org.opentripplanner.framework.io.FileUtils.readFile;
6+
import static org.opentripplanner.framework.io.FileUtils.writeFile;
7+
import static org.opentripplanner.generate.doc.framework.TemplateUtil.replaceSection;
8+
import static org.opentripplanner.utils.text.MarkdownFormatter.HEADER_4;
9+
10+
import java.io.File;
11+
import org.junit.jupiter.api.Test;
12+
import org.opentripplanner.generate.doc.framework.DocBuilder;
13+
import org.opentripplanner.generate.doc.framework.DocsTestConstants;
14+
import org.opentripplanner.generate.doc.framework.GeneratesDocumentation;
15+
import org.opentripplanner.generate.doc.framework.ParameterDetailsList;
16+
import org.opentripplanner.generate.doc.framework.ParameterSummaryTable;
17+
import org.opentripplanner.generate.doc.framework.SkipNodes;
18+
import org.opentripplanner.generate.doc.framework.TemplateUtil;
19+
import org.opentripplanner.standalone.config.BuildConfig;
20+
import org.opentripplanner.standalone.config.framework.json.JsonSupport;
21+
import org.opentripplanner.standalone.config.framework.json.NodeAdapter;
22+
import org.opentripplanner.test.support.ResourceLoader;
23+
24+
@GeneratesDocumentation
25+
public class GsConfigurationDocTest implements DocsTestConstants {
26+
27+
private static final File TEMPLATE = new File(TEMPLATE_PATH, "GoogleCloudStorage.md");
28+
private static final File OUT_FILE = new File(SANDBOX_USER_DOC_PATH, "GoogleCloudStorage.md");
29+
private static final SkipNodes SKIP_NODES = SkipNodes.of().build();
30+
31+
@Test
32+
public void updateMapGcStorageConfigDoc() {
33+
NodeAdapter node = readMapGcStorageConfigConfig();
34+
35+
String template = readFile(TEMPLATE);
36+
String original = readFile(OUT_FILE);
37+
38+
template = replaceSection(template, "config", updaterDoc(node));
39+
40+
writeFile(OUT_FILE, template);
41+
assertFileEquals(original, OUT_FILE);
42+
}
43+
44+
private NodeAdapter readMapGcStorageConfigConfig() {
45+
var buildConfigFile = ResourceLoader.of(GsConfigurationDocTest.class).extTestResourceFile(
46+
BUILD_CONFIG_FILENAME
47+
);
48+
49+
var json = JsonSupport.jsonNodeFromPath(buildConfigFile.toPath());
50+
var conf = new BuildConfig(json, buildConfigFile.toString(), false);
51+
return conf.asNodeAdapter();
52+
}
53+
54+
private String updaterDoc(NodeAdapter node) {
55+
DocBuilder buf = new DocBuilder();
56+
addExample(buf, node);
57+
addParameterSummaryTable(buf, node.child("gsConfig"));
58+
addDetailsSection(buf, node.child("gsConfig"));
59+
return buf.toString();
60+
}
61+
62+
private void addParameterSummaryTable(DocBuilder buf, NodeAdapter node) {
63+
buf
64+
.header(3, "Overview", null)
65+
.addSection(new ParameterSummaryTable(SKIP_NODES).createTable(node).toMarkdownTable());
66+
}
67+
68+
private void addDetailsSection(DocBuilder buf, NodeAdapter node) {
69+
buf
70+
.header(3, "Details", null)
71+
.addSection(ParameterDetailsList.listParametersWithDetails(node, SKIP_NODES, HEADER_4));
72+
}
73+
74+
private void addExample(DocBuilder buf, NodeAdapter node) {
75+
var root = TemplateUtil.jsonExampleBuilder(node.rawNode()).build();
76+
buf.header(3, "Example configuration", null).addExample("build-config.json", root);
77+
}
78+
}

‎application/src/ext-test/java/org/opentripplanner/ext/datastore/gs/GsDataSourceRepositoryTest.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,13 @@
66
import java.net.URI;
77
import org.junit.jupiter.api.Test;
88
import org.opentripplanner.datastore.api.FileType;
9+
import org.opentripplanner.datastore.api.GsParameters;
910

1011
public class GsDataSourceRepositoryTest {
1112

12-
private final GsDataSourceRepository subject = new GsDataSourceRepository(null);
13+
private final GsDataSourceRepository subject = new GsDataSourceRepository(
14+
GsParameters.defaultValues()
15+
);
1316

1417
@Test
1518
public void description() {

‎application/src/ext-test/java/org/opentripplanner/ext/datastore/gs/GsIntegrationTest.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717
import org.opentripplanner.datastore.api.CompositeDataSource;
1818
import org.opentripplanner.datastore.api.DataSource;
1919
import org.opentripplanner.datastore.api.FileType;
20+
import org.opentripplanner.datastore.api.GsParameters;
21+
import org.opentripplanner.ext.datastore.gs.config.GsConfig;
2022

2123
/**
2224
* This is a manual integration test to test the Google Cloud Storage integration. To set up the
@@ -35,16 +37,18 @@ public class GsIntegrationTest {
3537

3638
private static final String CREDENTIALS_FILE =
3739
"<Insert path to local Google Service Credential file here>";
40+
private static final String HOST = "<Insert host here>";
3841
private static final String BUCKET_NAME = "<Insert bucket name here>";
3942
private static final URI GTFS_URI = toUri(BUCKET_NAME, "gtfs.zip");
4043
private static final String DATA = "{ \"key\" : \"data\" }";
4144

45+
private static final GsParameters GS_PARAMETERS = new GsConfig(HOST, CREDENTIALS_FILE);
4246
private GsDataSourceRepository repo;
4347

4448
@BeforeEach
4549
public void setUp() {
4650
// Open a repository
47-
repo = new GsDataSourceRepository(CREDENTIALS_FILE);
51+
repo = new GsDataSourceRepository(GS_PARAMETERS);
4852
repo.open();
4953
}
5054

‎application/src/ext-test/resources/org/opentripplanner/ext/datastore/gs/build-config.json‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
{
2+
"gsConfig": {
3+
"cloudServiceHost": "http://fake-gcp:4443",
4+
"credentialFile": "/path/to/file"
5+
},
6+
7+
"graph": "gs://otp-test-bucket/a/b/graph.obj",
8+
"buildReportDir": "gs://otp-test-bucket/a/b/np-report",
9+
"osm": [
10+
{
11+
"source": "gs://otp-test-bucket/a/b/northpole.pbf"
12+
}
13+
],
14+
"dem": [
15+
{
16+
"source": "gs://otp-test-bucket/a/b/northpole.dem.tif"
17+
}
18+
],
19+
"transitFeeds": [
20+
{
21+
"type": "gtfs",
22+
"source": "gs://otp-test-bucket/a/b/gtfs.zip"
23+
}
24+
]
25+
}

‎application/src/ext/java/org/opentripplanner/ext/datastore/gs/GsDataSourceModule.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,6 @@ DataSourceRepository provideGoogleStorageDataSourceRepository(OtpDataStoreConfig
2525
return null;
2626
}
2727
LOG.info("Google Cloud Store Repository enabled - GS resources detected.");
28-
return new GsDataSourceRepository(config.gsCredentials());
28+
return new GsDataSourceRepository(config.gsParameters());
2929
}
3030
}

‎application/src/ext/java/org/opentripplanner/ext/datastore/gs/GsDataSourceRepository.java‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import org.opentripplanner.datastore.api.CompositeDataSource;
1313
import org.opentripplanner.datastore.api.DataSource;
1414
import org.opentripplanner.datastore.api.FileType;
15+
import org.opentripplanner.datastore.api.GsParameters;
1516
import org.opentripplanner.datastore.base.DataSourceRepository;
1617
import org.opentripplanner.datastore.file.ZipStreamDataSourceDecorator;
1718

@@ -21,10 +22,12 @@
2122
public class GsDataSourceRepository implements DataSourceRepository {
2223

2324
private final String credentialsFilename;
25+
private final String host;
2426
private Storage storage;
2527

26-
public GsDataSourceRepository(String credentialsFilename) {
27-
this.credentialsFilename = credentialsFilename;
28+
public GsDataSourceRepository(GsParameters gsParameters) {
29+
this.credentialsFilename = gsParameters.credentialFile();
30+
this.host = gsParameters.host();
2831
}
2932

3033
@Override
@@ -90,8 +93,13 @@ private CompositeDataSource createCompositeSource(BlobId blobId, FileType type)
9093
}
9194

9295
private Storage connectToStorage() {
96+
StorageOptions.Builder builder;
9397
try {
94-
StorageOptions.Builder builder = StorageOptions.getDefaultInstance().toBuilder();
98+
if (host != null) {
99+
builder = StorageOptions.newBuilder().setHost(host);
100+
} else {
101+
builder = StorageOptions.getDefaultInstance().toBuilder();
102+
}
95103

96104
if (credentialsFilename != null) {
97105
GoogleCredentials credentials = GoogleCredentials.fromStream(

‎application/src/ext/java/org/opentripplanner/ext/datastore/gs/config/GsConfig.java‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
package org.opentripplanner.ext.datastore.gs.config;
2+
3+
import org.opentripplanner.datastore.api.GsParameters;
4+
import org.opentripplanner.standalone.config.framework.json.NodeAdapter;
5+
import org.opentripplanner.standalone.config.framework.json.OtpVersion;
6+
7+
public class GsConfig implements GsParameters {
8+
9+
private final String host;
10+
private final String credentialFile;
11+
12+
public GsConfig(String host, String credentialFile) {
13+
this.host = host;
14+
this.credentialFile = credentialFile;
15+
}
16+
17+
public static GsConfig fromConfig(NodeAdapter root, String parameterName) {
18+
NodeAdapter gsRoot = root
19+
.of(parameterName)
20+
.since(OtpVersion.V2_8)
21+
.summary("Configuration for Google Cloud Storage")
22+
.asObject();
23+
24+
String host = gsRoot
25+
.of("cloudServiceHost")
26+
.since(OtpVersion.V2_8)
27+
.summary("Host of the Google Cloud Storage Server")
28+
.description(
29+
"""
30+
Host of the Google Cloud Storage server. In case of a real GCS Bucket this parameter can be
31+
omitted. When the host differs from the usual GCS host, for example when emulating GCS in a
32+
docker container for testing purposes, the host has to be specified including the port.
33+
Eg: http://localhost:4443"""
34+
)
35+
.asString(null);
36+
37+
String credentialFile = gsRoot
38+
.of("credentialFile")
39+
.since(OtpVersion.V2_8)
40+
.summary("Local file system path to Google Cloud Platform service accounts credentials file.")
41+
.description(
42+
"""
43+
The credentials are used to access GCS URLs. When using GCS from outside of Google Cloud you
44+
need to provide a path the the service credentials. Environment variables in the path are
45+
resolved.
46+
47+
This is a path to a file on the local file system, not an URI.
48+
"""
49+
)
50+
.asString(null);
51+
52+
return new GsConfig(host, credentialFile);
53+
}
54+
55+
@Override
56+
public String host() {
57+
return host;
58+
}
59+
60+
@Override
61+
public String credentialFile() {
62+
return credentialFile;
63+
}
64+
}

‎application/src/main/java/org/opentripplanner/datastore/api/GsParameters.java‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
package org.opentripplanner.datastore.api;
2+
3+
import javax.annotation.Nullable;
4+
5+
public interface GsParameters {
6+
/**
7+
* Host of the Google Cloud Services, including the port.
8+
* <p>
9+
* Optional. May return {@code null}. If the host is not set, the connection to the Google Cloud
10+
* Platform is done via the default host {@code https://storage.googleapis.com:4443}.
11+
*
12+
*/
13+
@Nullable
14+
String host();
15+
16+
/**
17+
* Local file system path to Google Cloud Platform service accounts credentials file. The
18+
* credentials are used to access GCS urls. When using GCS from outside of the bucket cluster you
19+
* need to provide a path to the service credentials.
20+
* <p>
21+
* This is a path to a file on the local file system, not an URI.
22+
* <p>
23+
* Optional. May return {@code null}. If the credentials are not set, the connection to the Google
24+
* Cloud Platform is done without any authorization.
25+
*/
26+
@Nullable
27+
String credentialFile();
28+
29+
static GsParameters defaultValues() {
30+
return new GsParameters() {
31+
@Override
32+
public String host() {
33+
return null;
34+
}
35+
36+
@Override
37+
public String credentialFile() {
38+
return null;
39+
}
40+
};
41+
}
42+
}

‎application/src/main/java/org/opentripplanner/datastore/api/OtpDataStoreConfig.java‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -18,15 +18,9 @@ public interface OtpDataStoreConfig {
1818
URI reportDirectory();
1919

2020
/**
21-
* Local file system path to Google Cloud Platform service accounts credentials file. The
22-
* credentials is used to access GCS urls. When using GCS from outside of the bucket cluster you
23-
* need to provide a path the the service credentials.
24-
* <p>
25-
* This is a path to a file on the local file system, not an URI.
26-
* <p>
27-
* Optional. May return {@code null}.
21+
* Parameters used to connect to Google Cloud Storage.
2822
*/
29-
String gsCredentials();
23+
GsParameters gsParameters();
3024

3125
/**
3226
* List of URIs to the open street map pbf files (the pbf format is the only one supported).

‎application/src/main/java/org/opentripplanner/standalone/config/BuildConfig.java‎

Lines changed: 6 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import javax.annotation.Nullable;
2222
import org.opentripplanner.datastore.api.OtpDataStoreConfig;
2323
import org.opentripplanner.ext.dataoverlay.configuration.DataOverlayConfig;
24+
import org.opentripplanner.ext.datastore.gs.config.GsConfig;
2425
import org.opentripplanner.ext.emission.config.EmissionConfig;
2526
import org.opentripplanner.ext.emission.parameters.EmissionParameters;
2627
import org.opentripplanner.ext.fares.FaresConfiguration;
@@ -137,8 +138,6 @@ public class BuildConfig implements OtpDataStoreConfig {
137138

138139
private final Pattern demLocalFilePattern;
139140

140-
private final String gsCredentials;
141-
142141
private final URI streetGraph;
143142

144143
private final URI graph;
@@ -187,6 +186,7 @@ public class BuildConfig implements OtpDataStoreConfig {
187186
public final ZoneId transitModelTimeZone;
188187
private final List<FeedScopedId> transitRouteToStationCentroid;
189188
public final URI stopConsolidation;
189+
private final GsConfig gsConfig;
190190

191191
/**
192192
* Set all parameters from the given Jackson JSON tree, applying defaults. Supplying
@@ -557,20 +557,6 @@ that we support remote input files (cloud storage or arbitrary URLs) not all dat
557557
)
558558
.asPattern(DEFAULT_DEM_PATTERN);
559559

560-
gsCredentials = root
561-
.of("gsCredentials")
562-
.since(V2_0)
563-
.summary("Local file system path to Google Cloud Platform service accounts credentials file.")
564-
.description(
565-
"""
566-
The credentials is used to access GCS urls. When using GCS from outside of Google Cloud you
567-
need to provide a path the the service credentials. Environment variables in the path are
568-
resolved.
569-
570-
This is a path to a file on the local file system, not an URI.
571-
"""
572-
)
573-
.asString(null);
574560
graph = root
575561
.of("graph")
576562
.since(V2_0)
@@ -625,6 +611,8 @@ that we support remote input files (cloud storage or arbitrary URLs) not all dat
625611

626612
transferRequests = TransferRequestConfig.map(root, "transferRequests");
627613

614+
gsConfig = GsConfig.fromConfig(root, "gsConfig");
615+
628616
if (logUnusedParams && LOG.isWarnEnabled()) {
629617
root.logAllWarnings(LOG::warn);
630618
}
@@ -636,8 +624,8 @@ public URI reportDirectory() {
636624
}
637625

638626
@Override
639-
public String gsCredentials() {
640-
return gsCredentials;
627+
public GsConfig gsParameters() {
628+
return gsConfig;
641629
}
642630

643631
@Override

0 commit comments

Comments
 (0)