|
| 1 | +/* |
| 2 | + * Copyright (C) 2026 Google LLC |
| 3 | + * |
| 4 | + * Licensed under the Apache License, Version 2.0 (the "License"); you may not |
| 5 | + * use this file except in compliance with the License. You may obtain a copy of |
| 6 | + * the License at |
| 7 | + * |
| 8 | + * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | + * |
| 10 | + * Unless required by applicable law or agreed to in writing, software |
| 11 | + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| 12 | + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| 13 | + * License for the specific language governing permissions and limitations under |
| 14 | + * the License. |
| 15 | + */ |
| 16 | +package com.google.cloud.teleport.v2.spanner.migrations.source.config; |
| 17 | + |
| 18 | +import com.fasterxml.jackson.annotation.JsonSetter; |
| 19 | +import com.fasterxml.jackson.annotation.Nulls; |
| 20 | +import com.fasterxml.jackson.databind.DeserializationFeature; |
| 21 | +import com.fasterxml.jackson.databind.ObjectMapper; |
| 22 | +import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; |
| 23 | +import com.google.cloud.teleport.v2.spanner.migrations.utils.CassandraDriverConfigLoader; |
| 24 | +import com.google.cloud.teleport.v2.spanner.migrations.utils.ISecretManagerAccessor; |
| 25 | +import com.google.common.annotations.VisibleForTesting; |
| 26 | +import com.typesafe.config.Config; |
| 27 | +import com.typesafe.config.ConfigFactory; |
| 28 | +import com.typesafe.config.ConfigResolveOptions; |
| 29 | +import java.io.IOException; |
| 30 | +import java.io.InputStream; |
| 31 | +import java.nio.channels.Channels; |
| 32 | +import java.nio.charset.StandardCharsets; |
| 33 | +import java.util.Comparator; |
| 34 | +import java.util.Map; |
| 35 | +import org.apache.beam.sdk.io.FileSystems; |
| 36 | +import org.apache.commons.io.IOUtils; |
| 37 | +import org.slf4j.Logger; |
| 38 | +import org.slf4j.LoggerFactory; |
| 39 | + |
| 40 | +/** |
| 41 | + * Parses the source configuration from JSON/HOCON and resolves secrets from Secret Manager. |
| 42 | + * |
| 43 | + * <p>Supports the following source types: MySQL, PostgreSQL, Cassandra, and Astra DB. |
| 44 | + */ |
| 45 | +public class SourceConfigParser { |
| 46 | + private static final Logger LOG = LoggerFactory.getLogger(SourceConfigParser.class); |
| 47 | + private final ObjectMapper mapper; |
| 48 | + |
| 49 | + private final ISecretManagerAccessor secretManagerAccessor; |
| 50 | + |
| 51 | + /** |
| 52 | + * Constructs a new {@code SourceConfigParser} with the specified secret manager accessor. |
| 53 | + * |
| 54 | + * @param secretManagerAccessor the accessor used to resolve secrets from Secret Manager |
| 55 | + */ |
| 56 | + public SourceConfigParser(ISecretManagerAccessor secretManagerAccessor) { |
| 57 | + this.secretManagerAccessor = secretManagerAccessor; |
| 58 | + this.mapper = |
| 59 | + new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); |
| 60 | + mapper |
| 61 | + .configOverride(String.class) |
| 62 | + .setSetterInfo(JsonSetter.Value.forValueNulls(Nulls.AS_EMPTY)); |
| 63 | + } |
| 64 | + |
| 65 | + /** |
| 66 | + * Parses the configuration file from GCS into the appropriate {@link SourceConnectionConfig} |
| 67 | + * implementing class. |
| 68 | + * |
| 69 | + * @param sourceTypeStr The source database type ("mysql", "postgresql", "cassandra", "astra_db"). |
| 70 | + * @param sourceConfigFilePath The URI to the HOCON or JSON config file. |
| 71 | + * @return A populated implementation of {@link SourceConnectionConfig}. |
| 72 | + */ |
| 73 | + public SourceConnectionConfig parseConfiguration( |
| 74 | + String sourceTypeStr, String sourceConfigFilePath) throws Exception { |
| 75 | + |
| 76 | + SourceType sourceType = SourceType.parseSourceType(sourceTypeStr); |
| 77 | + switch (SourceType.parseSourceType(sourceTypeStr)) { |
| 78 | + case CASSANDRA: |
| 79 | + // Maps directly to the DataStax OptionsMap |
| 80 | + return new CassandraConnectionConfig( |
| 81 | + CassandraDriverConfigLoader.getOptionsMapFromFile(sourceConfigFilePath)); |
| 82 | + case ASTRA_DB: |
| 83 | + String astraFileContent = readConfigFilePath(sourceConfigFilePath); |
| 84 | + Map<String, Object> astraConfigMap = parseConfigToConfigMap(astraFileContent); |
| 85 | + return mapper.convertValue(astraConfigMap, AstraConnectionConfig.class); |
| 86 | + case MYSQL: |
| 87 | + case PG: |
| 88 | + String jdbcFileContent = readConfigFilePath(sourceConfigFilePath); |
| 89 | + Map<String, Object> jdbcConfigMap = parseConfigToConfigMap(jdbcFileContent); |
| 90 | + JdbcShardConfig jdbcShardConfig = mapper.convertValue(jdbcConfigMap, JdbcShardConfig.class); |
| 91 | + // Returns ordered list of shards |
| 92 | + jdbcShardConfig.getShardConfigs().sort(Comparator.comparing(Shard::getLogicalShardId)); |
| 93 | + resolveShardSecret(jdbcShardConfig, sourceConfigFilePath); |
| 94 | + return jdbcShardConfig; |
| 95 | + default: |
| 96 | + throw new IllegalArgumentException("Unsupported source type: " + sourceType); |
| 97 | + } |
| 98 | + } |
| 99 | + |
| 100 | + /** |
| 101 | + * Resolves password secrets for all shard configurations inside the given {@link JdbcShardConfig} |
| 102 | + * using the {@link ISecretManagerAccessor}. |
| 103 | + * |
| 104 | + * @param jdbcShardConfig the JDBC shard configuration object containing shard configs to be |
| 105 | + * updated |
| 106 | + * @param sourceShardsFilePath the path to the source shards configuration file, used for error |
| 107 | + * reporting |
| 108 | + * @throws RuntimeException if neither the password nor secretManagerUri is found for any shard |
| 109 | + */ |
| 110 | + @VisibleForTesting |
| 111 | + void resolveShardSecret(JdbcShardConfig jdbcShardConfig, String sourceShardsFilePath) { |
| 112 | + if (jdbcShardConfig.getShardConfigs() == null) { |
| 113 | + throw new IllegalArgumentException( |
| 114 | + "The configuration file is missing the 'shardConfigs' field."); |
| 115 | + } |
| 116 | + for (Shard shard : jdbcShardConfig.getShardConfigs()) { |
| 117 | + LOG.info("Processing shard: {}", shard.getLogicalShardId()); |
| 118 | + String password = |
| 119 | + secretManagerAccessor.resolvePassword( |
| 120 | + shard.getSecretManagerUri(), shard.getLogicalShardId(), shard.getPassword()); |
| 121 | + if (password == null || password.isEmpty()) { |
| 122 | + throw new RuntimeException( |
| 123 | + "Neither password nor secretManagerUri was found in the shard file " |
| 124 | + + sourceShardsFilePath |
| 125 | + + " for shard " |
| 126 | + + shard.getLogicalShardId()); |
| 127 | + } |
| 128 | + shard.setPassword(password); |
| 129 | + } |
| 130 | + } |
| 131 | + |
| 132 | + /** |
| 133 | + * Parses a configuration string (either HOCON or JSON format) and converts it into a standard |
| 134 | + * Java {@link Map}. |
| 135 | + * |
| 136 | + * @param configContent the HOCON or JSON configuration string |
| 137 | + * @return a map representing the resolved configuration properties |
| 138 | + */ |
| 139 | + @VisibleForTesting |
| 140 | + static Map<String, Object> parseConfigToConfigMap(String configContent) { |
| 141 | + |
| 142 | + // Parse HOCON/JSON content |
| 143 | + // ConfigFactory.parseString handles both HOCON and JSON formats seamlessly. |
| 144 | + // resolve() handles HOCON's inbuilt inheritance (${...} substitutions). |
| 145 | + Config rawConfig = |
| 146 | + ConfigFactory.parseString(configContent).resolve(ConfigResolveOptions.defaults()); |
| 147 | + |
| 148 | + // Convert the resolved Typesafe Config object into a standard Java Map for Jackson mapping |
| 149 | + return rawConfig.root().unwrapped(); |
| 150 | + } |
| 151 | + |
| 152 | + /** |
| 153 | + * Reads the content of a configuration file from a specified file path (e.g., GCS or local). |
| 154 | + * |
| 155 | + * @param sourceConfigFilePath the path or URI to the configuration file |
| 156 | + * @return the content of the file as a string |
| 157 | + * @throws Exception if an error occurs while reading the file |
| 158 | + */ |
| 159 | + @VisibleForTesting |
| 160 | + static String readConfigFilePath(String sourceConfigFilePath) throws Exception { |
| 161 | + try (InputStream stream = |
| 162 | + Channels.newInputStream( |
| 163 | + FileSystems.open(FileSystems.matchNewResource(sourceConfigFilePath, false)))) { |
| 164 | + |
| 165 | + return IOUtils.toString(stream, StandardCharsets.UTF_8); |
| 166 | + } catch (IOException e) { |
| 167 | + String errorMessage = |
| 168 | + "Failed to read configuration input file at " |
| 169 | + + sourceConfigFilePath |
| 170 | + + ". Make sure it is ASCII or UTF-8 encoded and contains a" |
| 171 | + + " well-formed HOCON/JSON string."; |
| 172 | + LOG.error(errorMessage, e); |
| 173 | + throw new RuntimeException(errorMessage, e); |
| 174 | + } |
| 175 | + } |
| 176 | +} |
0 commit comments