Skip to content

Commit 71640c4

Browse files
feat(Spanner): Unified source config parser:
1. Implemented a source config parser. 2. Parser returns abstract SourceConnectionConfig. 3. SourceConnectionConfig is dependent on the source type. 4. For JDBC the password is resolved using secret manager.
1 parent f66b280 commit 71640c4

14 files changed

Lines changed: 972 additions & 88 deletions

File tree

v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/shard/Shard.java

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -22,15 +22,15 @@
2222

2323
public class Shard implements Serializable {
2424

25-
private String logicalShardId;
26-
private String host;
27-
private String port;
28-
private String user;
29-
private String password;
30-
private String dbName;
31-
private String namespace;
32-
private String secretManagerUri;
33-
private String connectionProperties;
25+
private String logicalShardId = "";
26+
private String host = "";
27+
private String port = "";
28+
private String user = "";
29+
private String password = "";
30+
private String dbName = "";
31+
private String namespace = "";
32+
private String secretManagerUri = "";
33+
private String connectionProperties = "";
3434

3535
private Map<String, String> dbNameToLogicalShardIdMap = new HashMap<>();
3636

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
/*
2+
* Copyright (C) 2026 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy of
6+
* the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations under
14+
* the License.
15+
*/
16+
package com.google.cloud.teleport.v2.spanner.migrations.source.config;
17+
18+
/** Represents the connection configuration for an Astra DB source. */
19+
public class AstraConnectionConfig implements SourceConnectionConfig {
20+
21+
private String databaseId = "";
22+
private String astraToken = "";
23+
private String keySpace = "";
24+
private String astraDbRegion = "";
25+
26+
public String getDatabaseId() {
27+
return databaseId;
28+
}
29+
30+
public void setDatabaseId(String databaseId) {
31+
this.databaseId = databaseId;
32+
}
33+
34+
public String getAstraToken() {
35+
return astraToken;
36+
}
37+
38+
public void setAstraToken(String astraToken) {
39+
this.astraToken = astraToken;
40+
}
41+
42+
public String getKeySpace() {
43+
return keySpace;
44+
}
45+
46+
public void setKeySpace(String keySpace) {
47+
this.keySpace = keySpace;
48+
}
49+
50+
public String getAstraDbRegion() {
51+
return astraDbRegion;
52+
}
53+
54+
public void setAstraDbRegion(String astraDbRegion) {
55+
this.astraDbRegion = astraDbRegion;
56+
}
57+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
/*
2+
* Copyright (C) 2026 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy of
6+
* the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations under
14+
* the License.
15+
*/
16+
package com.google.cloud.teleport.v2.spanner.migrations.source.config;
17+
18+
import com.datastax.oss.driver.api.core.config.OptionsMap;
19+
20+
/** Represents the connection configuration for a Cassandra source. */
21+
public class CassandraConnectionConfig implements SourceConnectionConfig {
22+
23+
private final OptionsMap optionsMap;
24+
25+
public CassandraConnectionConfig(OptionsMap optionsMap) {
26+
this.optionsMap = optionsMap;
27+
}
28+
29+
public OptionsMap getOptionsMap() {
30+
return optionsMap;
31+
}
32+
}
Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
/*
2+
* Copyright (C) 2026 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy of
6+
* the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations under
14+
* the License.
15+
*/
16+
package com.google.cloud.teleport.v2.spanner.migrations.source.config;
17+
18+
import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard;
19+
import java.util.List;
20+
21+
/**
22+
* Represents the connection configuration for a sharded RDBMS source (MySQL/PG).
23+
*
24+
* <p>For non-sharded RDBMS, shardConfigs will contain single element.
25+
*/
26+
public class JdbcShardConfig implements SourceConnectionConfig {
27+
private List<Shard> shardConfigs;
28+
29+
public List<Shard> getShardConfigs() {
30+
return shardConfigs;
31+
}
32+
33+
public void setShardConfigs(List<Shard> shardConfigs) {
34+
this.shardConfigs = shardConfigs;
35+
}
36+
}
Lines changed: 176 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,176 @@
1+
/*
2+
* Copyright (C) 2026 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy of
6+
* the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations under
14+
* the License.
15+
*/
16+
package com.google.cloud.teleport.v2.spanner.migrations.source.config;
17+
18+
import com.fasterxml.jackson.annotation.JsonSetter;
19+
import com.fasterxml.jackson.annotation.Nulls;
20+
import com.fasterxml.jackson.databind.DeserializationFeature;
21+
import com.fasterxml.jackson.databind.ObjectMapper;
22+
import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard;
23+
import com.google.cloud.teleport.v2.spanner.migrations.utils.CassandraDriverConfigLoader;
24+
import com.google.cloud.teleport.v2.spanner.migrations.utils.ISecretManagerAccessor;
25+
import com.google.common.annotations.VisibleForTesting;
26+
import com.typesafe.config.Config;
27+
import com.typesafe.config.ConfigFactory;
28+
import com.typesafe.config.ConfigResolveOptions;
29+
import java.io.IOException;
30+
import java.io.InputStream;
31+
import java.nio.channels.Channels;
32+
import java.nio.charset.StandardCharsets;
33+
import java.util.Comparator;
34+
import java.util.Map;
35+
import org.apache.beam.sdk.io.FileSystems;
36+
import org.apache.commons.io.IOUtils;
37+
import org.slf4j.Logger;
38+
import org.slf4j.LoggerFactory;
39+
40+
/**
41+
* Parses the source configuration from JSON/HOCON and resolves secrets from Secret Manager.
42+
*
43+
* <p>Supports the following source types: MySQL, PostgreSQL, Cassandra, and Astra DB.
44+
*/
45+
public class SourceConfigParser {
46+
private static final Logger LOG = LoggerFactory.getLogger(SourceConfigParser.class);
47+
private final ObjectMapper mapper;
48+
49+
private final ISecretManagerAccessor secretManagerAccessor;
50+
51+
/**
52+
* Constructs a new {@code SourceConfigParser} with the specified secret manager accessor.
53+
*
54+
* @param secretManagerAccessor the accessor used to resolve secrets from Secret Manager
55+
*/
56+
public SourceConfigParser(ISecretManagerAccessor secretManagerAccessor) {
57+
this.secretManagerAccessor = secretManagerAccessor;
58+
this.mapper =
59+
new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
60+
mapper
61+
.configOverride(String.class)
62+
.setSetterInfo(JsonSetter.Value.forValueNulls(Nulls.AS_EMPTY));
63+
}
64+
65+
/**
66+
* Parses the configuration file from GCS into the appropriate {@link SourceConnectionConfig}
67+
* implementing class.
68+
*
69+
* @param sourceTypeStr The source database type ("mysql", "postgresql", "cassandra", "astra_db").
70+
* @param sourceConfigFilePath The URI to the HOCON or JSON config file.
71+
* @return A populated implementation of {@link SourceConnectionConfig}.
72+
*/
73+
public SourceConnectionConfig parseConfiguration(
74+
String sourceTypeStr, String sourceConfigFilePath) throws Exception {
75+
76+
SourceType sourceType = SourceType.parseSourceType(sourceTypeStr);
77+
switch (SourceType.parseSourceType(sourceTypeStr)) {
78+
case CASSANDRA:
79+
// Maps directly to the DataStax OptionsMap
80+
return new CassandraConnectionConfig(
81+
CassandraDriverConfigLoader.getOptionsMapFromFile(sourceConfigFilePath));
82+
case ASTRA_DB:
83+
String astraFileContent = readConfigFilePath(sourceConfigFilePath);
84+
Map<String, Object> astraConfigMap = parseConfigToConfigMap(astraFileContent);
85+
return mapper.convertValue(astraConfigMap, AstraConnectionConfig.class);
86+
case MYSQL:
87+
case PG:
88+
String jdbcFileContent = readConfigFilePath(sourceConfigFilePath);
89+
Map<String, Object> jdbcConfigMap = parseConfigToConfigMap(jdbcFileContent);
90+
JdbcShardConfig jdbcShardConfig = mapper.convertValue(jdbcConfigMap, JdbcShardConfig.class);
91+
// Returns ordered list of shards
92+
jdbcShardConfig.getShardConfigs().sort(Comparator.comparing(Shard::getLogicalShardId));
93+
resolveShardSecret(jdbcShardConfig, sourceConfigFilePath);
94+
return jdbcShardConfig;
95+
default:
96+
throw new IllegalArgumentException("Unsupported source type: " + sourceType);
97+
}
98+
}
99+
100+
/**
101+
* Resolves password secrets for all shard configurations inside the given {@link JdbcShardConfig}
102+
* using the {@link ISecretManagerAccessor}.
103+
*
104+
* @param jdbcShardConfig the JDBC shard configuration object containing shard configs to be
105+
* updated
106+
* @param sourceShardsFilePath the path to the source shards configuration file, used for error
107+
* reporting
108+
* @throws RuntimeException if neither the password nor secretManagerUri is found for any shard
109+
*/
110+
@VisibleForTesting
111+
void resolveShardSecret(JdbcShardConfig jdbcShardConfig, String sourceShardsFilePath) {
112+
if (jdbcShardConfig.getShardConfigs() == null) {
113+
throw new IllegalArgumentException(
114+
"The configuration file is missing the 'shardConfigs' field.");
115+
}
116+
for (Shard shard : jdbcShardConfig.getShardConfigs()) {
117+
LOG.info("Processing shard: {}", shard.getLogicalShardId());
118+
String password =
119+
secretManagerAccessor.resolvePassword(
120+
shard.getSecretManagerUri(), shard.getLogicalShardId(), shard.getPassword());
121+
if (password == null || password.isEmpty()) {
122+
throw new RuntimeException(
123+
"Neither password nor secretManagerUri was found in the shard file "
124+
+ sourceShardsFilePath
125+
+ " for shard "
126+
+ shard.getLogicalShardId());
127+
}
128+
shard.setPassword(password);
129+
}
130+
}
131+
132+
/**
133+
* Parses a configuration string (either HOCON or JSON format) and converts it into a standard
134+
* Java {@link Map}.
135+
*
136+
* @param configContent the HOCON or JSON configuration string
137+
* @return a map representing the resolved configuration properties
138+
*/
139+
@VisibleForTesting
140+
static Map<String, Object> parseConfigToConfigMap(String configContent) {
141+
142+
// Parse HOCON/JSON content
143+
// ConfigFactory.parseString handles both HOCON and JSON formats seamlessly.
144+
// resolve() handles HOCON's inbuilt inheritance (${...} substitutions).
145+
Config rawConfig =
146+
ConfigFactory.parseString(configContent).resolve(ConfigResolveOptions.defaults());
147+
148+
// Convert the resolved Typesafe Config object into a standard Java Map for Jackson mapping
149+
return rawConfig.root().unwrapped();
150+
}
151+
152+
/**
153+
* Reads the content of a configuration file from a specified file path (e.g., GCS or local).
154+
*
155+
* @param sourceConfigFilePath the path or URI to the configuration file
156+
* @return the content of the file as a string
157+
* @throws Exception if an error occurs while reading the file
158+
*/
159+
@VisibleForTesting
160+
static String readConfigFilePath(String sourceConfigFilePath) throws Exception {
161+
try (InputStream stream =
162+
Channels.newInputStream(
163+
FileSystems.open(FileSystems.matchNewResource(sourceConfigFilePath, false)))) {
164+
165+
return IOUtils.toString(stream, StandardCharsets.UTF_8);
166+
} catch (IOException e) {
167+
String errorMessage =
168+
"Failed to read configuration input file at "
169+
+ sourceConfigFilePath
170+
+ ". Make sure it is ASCII or UTF-8 encoded and contains a"
171+
+ " well-formed HOCON/JSON string.";
172+
LOG.error(errorMessage, e);
173+
throw new RuntimeException(errorMessage, e);
174+
}
175+
}
176+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
/*
2+
* Copyright (C) 2026 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy of
6+
* the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations under
14+
* the License.
15+
*/
16+
package com.google.cloud.teleport.v2.spanner.migrations.source.config;
17+
18+
/**
19+
* Interface representing the connection configuration for a migration source database.
20+
*
21+
* <p>Implementing classes provide specific configuration details for different database engines
22+
* (e.g., Cassandra, Astra DB, JDBC/MySQL/PostgreSQL).
23+
*
24+
* <p>This interface will be expanded in Phase 2 during the platformization phase.
25+
*/
26+
public interface SourceConnectionConfig {}

0 commit comments

Comments
 (0)