Skip to content

Commit 403f38e

Browse files
ryannedolanclaudeCopilot
authored
Add CREATE DATABASE DDL with Deployer mechanism for K8s Database objects (#214)
* Add CREATE DATABASE DDL with Deployer mechanism for K8s Database objects Introduces CREATE [OR REPLACE] DATABASE <name> [WITH (...)] syntax that deploys a Database CRD to Kubernetes via the Deployer pattern. Includes parser, executor, shared processCreateDatabase util, K8sDatabaseDeployer, and quidem test with !specify support. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * Fix case-insensitive option lookup in deployers SQL parser uppercases unquoted identifiers, so WITH (url '...') produces key "URL". Use a case-insensitive TreeMap in K8sDatabaseDeployer and fix expected config key casing in the CREATE JOB quidem test. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * Fix expected dialect enum serialization in quidem test SnakeYAML serializes Java enums by name (MYSQL) not value (MySQL). Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * Add unit tests for K8sDatabaseDeployer, processCreateDatabase, and K8sDeployerProvider Agent-Logs-Url: https://github.com/linkedin/Hoptimator/sessions/b448a07f-580a-40c3-a885-a6b2a53fcf25 Co-authored-by: ryannedolan <1387539+ryannedolan@users.noreply.github.com> * Address code review feedback: rename test methods and fix comment grammar Agent-Logs-Url: https://github.com/linkedin/Hoptimator/sessions/b448a07f-580a-40c3-a885-a6b2a53fcf25 Co-authored-by: ryannedolan <1387539+ryannedolan@users.noreply.github.com> * Remove unused V1alpha1DatabaseSpec import from K8sDatabaseDeployerTest Agent-Logs-Url: https://github.com/linkedin/Hoptimator/sessions/975eee55-ade8-4ede-b3d8-70015a063e28 Co-authored-by: ryannedolan <1387539+ryannedolan@users.noreply.github.com> --------- Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: ryannedolan <1387539+ryannedolan@users.noreply.github.com>
1 parent aff89e1 commit 403f38e

14 files changed

Lines changed: 622 additions & 0 deletions

File tree

‎hoptimator-api/src/main/java/com/linkedin/hoptimator/DatabaseDeployable.java‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
package com.linkedin.hoptimator;
2+
3+
import java.util.Map;
4+
5+
6+
/** Represents a CREATE DATABASE request. */
7+
public class DatabaseDeployable implements Deployable {
8+
9+
private final String name;
10+
private final Map<String, String> options;
11+
12+
public DatabaseDeployable(String name, Map<String, String> options) {
13+
this.name = name;
14+
this.options = options;
15+
}
16+
17+
public String name() {
18+
return name;
19+
}
20+
21+
public Map<String, String> options() {
22+
return options;
23+
}
24+
25+
@Override
26+
public String toString() {
27+
return "Database[" + name + "]";
28+
}
29+
}

‎hoptimator-jdbc/src/main/codegen/config.fmpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ data: {
3232
"org.apache.calcite.sql.SqlTruncate"
3333
"org.apache.calcite.sql.ddl.SqlCreateTableLike"
3434
"org.apache.calcite.sql.ddl.SqlDdlNodes"
35+
"com.linkedin.hoptimator.jdbc.ddl.SqlCreateDatabase"
3536
"com.linkedin.hoptimator.jdbc.ddl.SqlCreateFunction"
3637
"com.linkedin.hoptimator.jdbc.ddl.SqlCreateMaterializedView"
3738
"com.linkedin.hoptimator.jdbc.ddl.SqlCreateTable"
@@ -91,6 +92,7 @@ data: {
9192
# Each must accept arguments "(SqlParserPos pos, boolean replace)".
9293
# Example: "SqlCreateForeignSchema".
9394
createStatementParserMethods: [
95+
"SqlCreateDatabase"
9496
"SqlCreateMaterializedView"
9597
"SqlCreateTrigger"
9698
"SqlCreateTable"

‎hoptimator-jdbc/src/main/codegen/includes/parserImpls.ftl‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -334,6 +334,20 @@ SqlCreate SqlCreateTrigger(Span s, boolean replace) :
334334
}
335335
}
336336

337+
SqlCreate SqlCreateDatabase(Span s, boolean replace) :
338+
{
339+
final boolean ifNotExists;
340+
final SqlIdentifier id;
341+
SqlNodeList optionList = null;
342+
}
343+
{
344+
<DATABASE> ifNotExists = IfNotExistsOpt() id = CompoundIdentifier()
345+
[ optionList = Options() ]
346+
{
347+
return new SqlCreateDatabase(s.end(this), replace, ifNotExists, id, optionList);
348+
}
349+
}
350+
337351
SqlCreate SqlCreateFunction(Span s, boolean replace) :
338352
{
339353
final boolean ifNotExists;

‎hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/HoptimatorDdlExecutor.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import com.linkedin.hoptimator.UserJob;
2626
import com.linkedin.hoptimator.View;
2727
import com.linkedin.hoptimator.jdbc.ddl.HoptimatorDdlParserImpl;
28+
import com.linkedin.hoptimator.jdbc.ddl.SqlCreateDatabase;
2829
import com.linkedin.hoptimator.jdbc.ddl.SqlCreateMaterializedView;
2930
import com.linkedin.hoptimator.jdbc.ddl.SqlCreateTable;
3031
import com.linkedin.hoptimator.jdbc.ddl.SqlCreateTrigger;
@@ -268,6 +269,19 @@ public void execute(SqlCreateTable create, CalcitePrepare.Context context) {
268269
logger.info("CREATE TABLE {} completed", create.name);
269270
}
270271

272+
/** Executes a {@code CREATE DATABASE} command. */
273+
public void execute(SqlCreateDatabase create, CalcitePrepare.Context context) {
274+
HoptimatorDdlUtils.DdlMode mode = create.getReplace()
275+
? HoptimatorDdlUtils.DdlMode.UPDATE : HoptimatorDdlUtils.DdlMode.CREATE;
276+
try {
277+
HoptimatorDdlUtils.processCreateDatabase(connection, create, mode);
278+
} catch (SQLException | RuntimeException e) {
279+
logger.info("Failed to deploy database {}", create.name);
280+
throw new DdlException(create, e.getMessage(), e);
281+
}
282+
logger.info("CREATE DATABASE {} completed", create.name);
283+
}
284+
271285
/** Executes a {@code PAUSE TRIGGER} command. */
272286
public void execute(SqlPauseTrigger pause, CalcitePrepare.Context context) {
273287
updateTriggerPausedState(pause, pause.name, true);

‎hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/HoptimatorDdlUtils.java‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,12 @@
2121

2222
import com.google.common.collect.ImmutableList;
2323
import com.linkedin.hoptimator.Database;
24+
import com.linkedin.hoptimator.DatabaseDeployable;
2425
import com.linkedin.hoptimator.Deployer;
2526
import com.linkedin.hoptimator.MaterializedView;
2627
import com.linkedin.hoptimator.Pipeline;
2728
import com.linkedin.hoptimator.Source;
29+
import com.linkedin.hoptimator.jdbc.ddl.SqlCreateDatabase;
2830
import com.linkedin.hoptimator.jdbc.ddl.SqlCreateMaterializedView;
2931
import com.linkedin.hoptimator.jdbc.ddl.SqlCreateTable;
3032
import com.linkedin.hoptimator.util.DeploymentService;
@@ -621,6 +623,55 @@ public RexNode newColumnDefaultValue(RelOptTable table, int iColumn,
621623
}
622624
}
623625

626+
/**
627+
* Shared implementation of the {@code CREATE DATABASE} pipeline for both real deployment
628+
* and dry-run (SPECIFY) modes.
629+
*
630+
* @param conn the JDBC connection
631+
* @param create the parsed DDL node
632+
* @param mode whether to CREATE, UPDATE, or SPECIFY
633+
* @return a SpecifyResult (specs are empty for CREATE/UPDATE, YAML for SPECIFY)
634+
* @throws SQLException on validation or deployment errors
635+
*/
636+
static SpecifyResult processCreateDatabase(HoptimatorConnection conn,
637+
SqlCreateDatabase create, DdlMode mode) throws SQLException {
638+
HoptimatorConnection.HoptimatorConnectionDualLogger logger = conn.getLogger(HoptimatorDdlUtils.class);
639+
640+
logger.info("Validating statement: {}", create);
641+
ValidationService.validateOrThrow(create);
642+
643+
if (create.name.names.size() > 1) {
644+
throw new SQLException("Database names cannot be compound identifiers.");
645+
}
646+
String name = create.name.names.get(0);
647+
648+
Map<String, String> dbOptions = options(create.options);
649+
DatabaseDeployable database = new DatabaseDeployable(name, dbOptions);
650+
651+
Collection<Deployer> deployers = null;
652+
try {
653+
logger.info("Validating database {}", name);
654+
ValidationService.validateOrThrow(database);
655+
deployers = DeploymentService.deployers(database, conn);
656+
ValidationService.validateOrThrow(deployers);
657+
658+
List<String> specs = mode.executeDeployers(deployers, conn);
659+
if (mode.mutable()) {
660+
logger.info("Deployed database {}", name);
661+
} else {
662+
DeploymentService.restore(deployers);
663+
}
664+
return new SpecifyResult(specs, null, Collections.singletonList(name));
665+
} catch (SQLException | RuntimeException e) {
666+
logger.info("Failed to deploy database {}", name);
667+
if (deployers != null) {
668+
DeploymentService.restore(deployers);
669+
logger.info("Restored deployable resources for database {}", name);
670+
}
671+
throw e;
672+
}
673+
}
674+
624675
/**
625676
* Returns the YAML specs that would be created for any supported SQL statement —
626677
* {@code CREATE TABLE}, {@code CREATE MATERIALIZED VIEW}, or {@code INSERT INTO}.
@@ -643,6 +694,10 @@ public RexNode newColumnDefaultValue(RelOptTable table, int iColumn,
643694
public static SpecifyResult specifyFromSql(String sql, HoptimatorConnection conn) throws SQLException {
644695
SqlNode sqlNode = HoptimatorDriver.parseQuery(conn, sql);
645696

697+
if (sqlNode instanceof SqlCreateDatabase) {
698+
return processCreateDatabase(conn, (SqlCreateDatabase) sqlNode, DdlMode.SPECIFY);
699+
}
700+
646701
if (sqlNode instanceof SqlCreateTable) {
647702
return processCreateTable(conn.createPrepareContext(), conn, (SqlCreateTable) sqlNode, DdlMode.SPECIFY);
648703
}

‎hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/ddl/HoptimatorDdlParserImpl.java‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import org.apache.calcite.sql.SqlTruncate;
88
import org.apache.calcite.sql.ddl.SqlCreateTableLike;
99
import org.apache.calcite.sql.ddl.SqlDdlNodes;
10+
import com.linkedin.hoptimator.jdbc.ddl.SqlCreateDatabase;
1011
import com.linkedin.hoptimator.jdbc.ddl.SqlCreateFunction;
1112
import com.linkedin.hoptimator.jdbc.ddl.SqlCreateMaterializedView;
1213
import com.linkedin.hoptimator.jdbc.ddl.SqlCreateTable;
@@ -6730,6 +6731,24 @@ final public SqlCreate SqlCreateTrigger(Span s, boolean replace) throws ParseExc
67306731
throw new Error("Missing return statement in function");
67316732
}
67326733

6734+
final public SqlCreate SqlCreateDatabase(Span s, boolean replace) throws ParseException {
6735+
final boolean ifNotExists;
6736+
final SqlIdentifier id;
6737+
SqlNodeList optionList = null;
6738+
jj_consume_token(DATABASE);
6739+
ifNotExists = IfNotExistsOpt();
6740+
id = CompoundIdentifier();
6741+
switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
6742+
case WITH:
6743+
optionList = Options();
6744+
break;
6745+
default:
6746+
;
6747+
}
6748+
{if (true) return new SqlCreateDatabase(s.end(this), replace, ifNotExists, id, optionList);}
6749+
throw new Error("Missing return statement in function");
6750+
}
6751+
67336752
final public SqlCreate SqlCreateFunction(Span s, boolean replace) throws ParseException {
67346753
final boolean ifNotExists;
67356754
final SqlIdentifier id;
@@ -24461,6 +24480,9 @@ final public SqlCreate SqlCreate() throws ParseException {
2446124480
;
2446224481
}
2446324482
switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
24483+
case DATABASE:
24484+
create = SqlCreateDatabase(s, replace);
24485+
break;
2446424486
case MATERIALIZED:
2446524487
create = SqlCreateMaterializedView(s, replace);
2446624488
break;

‎hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/ddl/SqlCreateDatabase.java‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to you under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package com.linkedin.hoptimator.jdbc.ddl;
18+
19+
import org.apache.calcite.sql.SqlCreate;
20+
import org.apache.calcite.sql.SqlIdentifier;
21+
import org.apache.calcite.sql.SqlKind;
22+
import org.apache.calcite.sql.SqlNode;
23+
import org.apache.calcite.sql.SqlNodeList;
24+
import org.apache.calcite.sql.SqlOperator;
25+
import org.apache.calcite.sql.SqlSpecialOperator;
26+
import org.apache.calcite.sql.SqlWriter;
27+
import org.apache.calcite.sql.parser.SqlParserPos;
28+
import org.apache.calcite.util.ImmutableNullableList;
29+
30+
import org.checkerframework.checker.nullness.qual.Nullable;
31+
32+
import java.util.List;
33+
34+
import static java.util.Objects.requireNonNull;
35+
36+
/**
37+
* Parse tree for {@code CREATE DATABASE} statement.
38+
*/
39+
public class SqlCreateDatabase extends SqlCreate {
40+
public final SqlIdentifier name;
41+
public final @Nullable SqlNodeList options;
42+
43+
private static final SqlOperator OPERATOR =
44+
new SqlSpecialOperator("CREATE DATABASE", SqlKind.OTHER_DDL);
45+
46+
/** Creates a SqlCreateDatabase. */
47+
protected SqlCreateDatabase(SqlParserPos pos, boolean replace, boolean ifNotExists,
48+
SqlIdentifier name, @Nullable SqlNodeList options) {
49+
super(OPERATOR, pos, replace, ifNotExists);
50+
this.name = requireNonNull(name, "name");
51+
this.options = options;
52+
}
53+
54+
@SuppressWarnings("nullness")
55+
@Override public List<SqlNode> getOperandList() {
56+
return ImmutableNullableList.of(name, options);
57+
}
58+
59+
@Override public void unparse(SqlWriter writer, int leftPrec, int rightPrec) {
60+
writer.keyword("CREATE");
61+
writer.keyword("DATABASE");
62+
if (ifNotExists) {
63+
writer.keyword("IF NOT EXISTS");
64+
}
65+
name.unparse(writer, leftPrec, rightPrec);
66+
if (options != null) {
67+
writer.keyword("WITH");
68+
SqlWriter.Frame frame = writer.startList("(", ")");
69+
for (SqlNode c : options) {
70+
writer.sep(",");
71+
c.unparse(writer, 0, 0);
72+
}
73+
writer.endList(frame);
74+
}
75+
}
76+
}

0 commit comments

Comments
 (0)