feat: Add flag to allow ignoring proto statements (#232)

Author: mlee2012

README.md

@@ -135,6 +135,13 @@ foreign key relationship is
 which would also be dropped and recreated. Therefore this option is disabled by
 default, and FOREIGN KEY differences will cause the tool to fail.
 
+## Note on Proto Bundles
+
+This tool does not support diffing of `PROTO BUNDLE` statements. By default, the tool will fail if it encounters a `PROTO BUNDLE` statement in the DDL.
+If you are confident that you do not need to diff `PROTO BUNDLE` statements, you can use the `--ignoreProtoBundles` flag. When this flag is set, the tool will ignore any `PROTO BUNDLE` statements in the DDL and will not generate any `ALTER` statements for them.
+
+**Warning:** This is a feature for a specific use case. Only use this flag if you are confident in your knowledge of how it will work.
+
 ## Unsupported Spanner DDL features
 
 This tool by neccessity will lag behind the implementation of new DDL features
@@ -172,6 +179,7 @@ mvn generate-resources compile exec:java \
   -Dexec.args="\
       --allowRecreateIndexes
       --allowRecreateForeignKeys
+      --ignoreProtoBundles
       --originalDdlFile original.ddl
       --newDdlFile new.ddl
       --outputDdlFile alter.ddl
@@ -185,6 +193,7 @@ mvn clean generate-resources compile package
 
 java -jar target/spanner-ddl-diff-*-jar-with-dependencies.jar \
       --allowRecreateIndexes \
+      --ignoreProtoBundles \
       --originalDdlFile original.ddl \
       --newDdlFile new.ddl \
       --outputDdlFile alter.ddl
@@ -334,6 +343,7 @@ generating statements to drop and recreate the constraint.
     --allowRecreateIndexes        Allows dropping and recreating secondary
                                   Indexes to apply changes.
     --help                        Show help.
+    --ignoreProtoBundles          Ignores proto bundle definitions.
     --newDdlFile <FILE>           File path to the new DDL definition.
     --originalDdlFile <FILE>      File path to the original DDL definition.
     --outputDdlFile <FILE>        File path to the output DDL to write.

src/main/java/com/google/cloud/solutions/spannerddl/diff/DatabaseDefinition.java

@@ -35,6 +35,7 @@
 import com.google.common.collect.ImmutableMap;
 import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 
 /**
@@ -54,7 +55,8 @@ public abstract class DatabaseDefinition {
    * @param statements List of parsed DDL statements
    * @return DatabaseDefinition instance
    */
-  public static DatabaseDefinition create(List<ASTddl_statement> statements) {
+  public static DatabaseDefinition create(
+      List<ASTddl_statement> statements, Map<String, Boolean> options) {
     // Use LinkedHashMap to preserve creation order in original DDL.
     LinkedHashMap<String, ASTcreate_table_statement> tablesInCreationOrder = new LinkedHashMap<>();
     LinkedHashMap<String, ASTcreate_index_statement> indexes = new LinkedHashMap<>();
@@ -128,6 +130,13 @@ public static DatabaseDefinition create(List<ASTddl_statement> statements) {
               (ASTcreate_change_stream_statement) statement);
           break;
 
+        case DdlParserTreeConstants.JJTCREATE_PROTO_BUNDLE_STATEMENT:
+        case DdlParserTreeConstants.JJTALTER_PROTO_BUNDLE_STATEMENT:
+          if (!options.get(DdlDiff.IGNORE_PROTO_BUNDLES_OPT)) {
+            throw new UnsupportedOperationException("Not Implemented");
+          }
+          break;
+
         case DdlParserTreeConstants.JJTCREATE_OR_REPLACE_STATEMENT:
           // can be one of several types.
           switch (((ASTcreate_or_replace_statement) statement).getSchemaObject().getId()) {

src/main/java/com/google/cloud/solutions/spannerddl/diff/DdlDiff.java

@@ -71,14 +71,14 @@
  *
  * <p>Example usage:
  *
- * <p>Pass the original and new DDL text to the {@link #build(String, String)} function, and call
- * {@link #generateDifferenceStatements(Map)} to generate the list of {@code ALTER} statements.
+ * <p>Pass the original and new DDL text to the {@link #build(String, String, Map)} function, and
+ * call {@link #generateDifferenceStatements(Map)} to generate the list of {@code ALTER} statements.
  *
  * <p>eg:
  *
  * <pre>
- * List&lt;String&gt; statements = DdlDiff.build(originalDDL, newDDL)
- *    .generateDifferenceStatements(true, true);
+ * List&lt;String&gt; statements = DdlDiff.build(originalDDL, newDDL,
+ * java.util.Collections.emptyMap()).generateDifferenceStatements(java.util.Collections.emptyMap());
  * </pre>
  *
  * <p>or execute the {@link #main(String[]) main()} function with the {@link
@@ -93,6 +93,7 @@ public class DdlDiff {
   public static final String ALLOW_RECREATE_INDEXES_OPT = "allowRecreateIndexes";
   public static final String ALLOW_RECREATE_CONSTRAINTS_OPT = "allowRecreateConstraints";
   public static final String ALLOW_DROP_STATEMENTS_OPT = "allowDropStatements";
+  public static final String IGNORE_PROTO_BUNDLES_OPT = "ignoreProtoBundles";
   public static final String HELP_OPT = "help";
 
   private final DatabaseDefinition originalDb;
@@ -670,7 +671,8 @@ private static String generateOptionsUpdates(MapDifference<String, String> optio
    * @return DdlDiff instance
    * @throws DdlDiffException if there is an error in paring the DDL
    */
-  public static DdlDiff build(String originalDdl, String newDdl) throws DdlDiffException {
+  public static DdlDiff build(String originalDdl, String newDdl, Map<String, Boolean> options)
+      throws DdlDiffException {
     List<ASTddl_statement> originalStatements;
     List<ASTddl_statement> newStatements;
     try {
@@ -684,8 +686,8 @@ public static DdlDiff build(String originalDdl, String newDdl) throws DdlDiffExc
       throw new DdlDiffException("Failed parsing NEW DDL: " + e.getMessage(), e);
     }
 
-    DatabaseDefinition originalDb = DatabaseDefinition.create(originalStatements);
-    DatabaseDefinition newDb = DatabaseDefinition.create(newStatements);
+    DatabaseDefinition originalDb = DatabaseDefinition.create(originalStatements, options);
+    DatabaseDefinition newDb = DatabaseDefinition.create(newStatements, options);
 
     return new DdlDiff(
         originalDb, newDb, getDatabaseNameFromAlterDatabase(originalStatements, newStatements));
@@ -818,6 +820,8 @@ public static List<ASTddl_statement> parseDdl(String original, boolean parseAnno
           case DdlParserTreeConstants.JJTALTER_DATABASE_STATEMENT:
           case DdlParserTreeConstants.JJTCREATE_CHANGE_STREAM_STATEMENT:
           case DdlParserTreeConstants.JJTCREATE_SEARCH_INDEX_STATEMENT:
+          case DdlParserTreeConstants.JJTCREATE_PROTO_BUNDLE_STATEMENT:
+          case DdlParserTreeConstants.JJTALTER_PROTO_BUNDLE_STATEMENT:
             // no-op - allowed
             break;
           case DdlParserTreeConstants.JJTCREATE_OR_REPLACE_STATEMENT:
@@ -860,9 +864,9 @@ public static void main(String[] args) {
       String originalDdl = new String(Files.readAllBytes(options.originalDdlPath()), UTF_8);
       String newDdl = new String(Files.readAllBytes(options.newDdlPath()), UTF_8);
 
-      validateDdl(newDdl);
+      validateDdl(newDdl, options.args());
 
-      DdlDiff ddlDiff = DdlDiff.build(originalDdl, newDdl);
+      DdlDiff ddlDiff = DdlDiff.build(originalDdl, newDdl, options.args());
 
       List<String> alterStatements = ddlDiff.generateDifferenceStatements(options.args());
 
@@ -890,14 +894,14 @@ public static void main(String[] args) {
    * @param ddl new DDL to parse
    * @throws DdlDiffException if there is an error in parsing the DDL
    */
-  public static void validateDdl(String ddl) throws DdlDiffException {
+  public static void validateDdl(String ddl, Map<String, Boolean> options) throws DdlDiffException {
     List<ASTddl_statement> statements;
     try {
       statements = parseDdl(Strings.nullToEmpty(ddl));
     } catch (DdlDiffException e) {
       throw new DdlDiffException("Failed parsing DDL: " + e.getMessage(), e);
     }
-    DatabaseDefinition db = DatabaseDefinition.create(statements);
+    DatabaseDefinition db = DatabaseDefinition.create(statements, options);
     validateReferences(db);
   }
 

src/main/java/com/google/cloud/solutions/spannerddl/diff/DdlDiffOptions.java

@@ -92,6 +92,11 @@ static Options buildOptions() {
             .longOpt(DdlDiff.ALLOW_DROP_STATEMENTS_OPT)
             .desc("Enables output of DROP commands to delete objects not used in the new DDL file.")
             .build());
+    options.addOption(
+        Option.builder()
+            .longOpt(DdlDiff.IGNORE_PROTO_BUNDLES_OPT)
+            .desc("Ignores proto bundle definitions when parsing DDLs and generating diffs.")
+            .build());
     options.addOption(Option.builder().longOpt(DdlDiff.HELP_OPT).desc("Show help").build());
     return options;
   }
@@ -155,7 +160,9 @@ public static DdlDiffOptions parseCommandLine(String[] args) {
               DdlDiff.ALLOW_DROP_STATEMENTS_OPT,
                   commandLine.hasOption(DdlDiff.ALLOW_DROP_STATEMENTS_OPT),
               DdlDiff.ALLOW_RECREATE_CONSTRAINTS_OPT,
-                  commandLine.hasOption(DdlDiff.ALLOW_RECREATE_CONSTRAINTS_OPT));
+                  commandLine.hasOption(DdlDiff.ALLOW_RECREATE_CONSTRAINTS_OPT),
+              DdlDiff.IGNORE_PROTO_BUNDLES_OPT,
+                  commandLine.hasOption(DdlDiff.IGNORE_PROTO_BUNDLES_OPT));
 
       return new AutoValue_DdlDiffOptions(originalDdlPath, newDdlPath, outputDdlPath, argsMap);
     } catch (InvalidPathException e) {

src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTalter_proto_bundle_statement.java

@@ -18,11 +18,9 @@
 public class ASTalter_proto_bundle_statement extends SimpleNode {
   public ASTalter_proto_bundle_statement(int id) {
     super(id);
-    throw new UnsupportedOperationException("Not Implemented");
   }
 
   public ASTalter_proto_bundle_statement(DdlParser p, int id) {
     super(p, id);
-    throw new UnsupportedOperationException("Not Implemented");
   }
 }

src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTcreate_proto_bundle_statement.java

@@ -18,11 +18,9 @@
 public class ASTcreate_proto_bundle_statement extends SimpleNode {
   public ASTcreate_proto_bundle_statement(int id) {
     super(id);
-    throw new UnsupportedOperationException("Not Implemented");
   }
 
   public ASTcreate_proto_bundle_statement(DdlParser p, int id) {
     super(p, id);
-    throw new UnsupportedOperationException("Not Implemented");
   }
 }

src/test/java/com/google/cloud/solutions/spannerddl/diff/DdlDiffFromFilesTest.java

@@ -62,7 +62,11 @@ public void compareDddTextFiles() throws IOException {
                 : Collections.emptyList();
 
         LOG.info("Processing segment (with drops): " + segmentName);
-        DdlDiff ddlDiff = DdlDiff.build(originalSegment.getValue(), newSegment.getValue());
+        DdlDiff ddlDiff =
+            DdlDiff.build(
+                originalSegment.getValue(),
+                newSegment.getValue(),
+                ImmutableMap.of(DdlDiff.IGNORE_PROTO_BUNDLES_OPT, false));
         // Run diff with allowRecreateIndexes and allowDropStatements
         List<String> diff =
             ddlDiff.generateDifferenceStatements(