Fixes for merged logic

Author: vish-cs

pipeline/ingestion/cloudbuild.yaml

@@ -18,27 +18,7 @@ steps:
       - DC_API_KEY
     id: 'build-jar'
 
-  # 2. Run Dataflow Integration Tests
-  - name: 'maven:3.9-eclipse-temurin-17'
-    entrypoint: 'mvn'
-    args:
-      - 'test'
-      - '-Dmaven.repo.local=/workspace/.m2/repository'
-      - '-Dtest=GraphIngestionPipelineIntegrationTest'
-      - '-DprojectId=$PROJECT_ID'
-      - '-DinstanceId=$_INSTANCE_ID'
-      - '-DdatabaseId=$_DATABASE_ID'
-      - '-DgcsBucket=$_GCS_BUCKET'
-      - '-Dregion=$_REGION'
-      - '-pl'
-      - 'pipeline/ingestion'
-      - '-DtestMode=DATAFLOW'
-      - '-Drevision=${_VERSION}'
-    secretEnv:
-      - DC_API_KEY
-    id: 'run-dataflow-integration-test'
-
-  # 3. Build the Dataflow Flex Template
+  # 2. Build the Dataflow Flex Template
   # This step uses the built JAR to create the Flex Template image and spec file.
   - name: 'gcr.io/google.com/cloudsdktool/cloud-sdk'
     entrypoint: 'gcloud'
@@ -61,15 +41,6 @@ steps:
       - 'FLEX_TEMPLATE_JAVA_MAIN_CLASS=org.datacommons.ingestion.pipeline.GraphIngestionPipeline'
     id: 'build-flex-template'
 
-  # 4. Update the "latest" pointer for the workflow default
-  - name: 'gcr.io/google.com/cloudsdktool/cloud-sdk'
-    entrypoint: 'gsutil'
-    args:
-      - 'cp'
-      - 'gs://$_TEMPLATE_BUCKET/templates/flex/ingestion-${_VERSION}.json'
-      - 'gs://$_TEMPLATE_BUCKET/templates/flex/ingestion.json'
-    id: 'update-latest-template'
-
 availableSecrets:
   secretManager:
     - versionName: projects/$PROJECT_ID/secrets/dc-api-key/versions/latest

pipeline/spanner/pom.xml

@@ -76,4 +76,15 @@
             <scope>test</scope>
         </dependency>
     </dependencies>
+
+    <build>
+        <resources>
+            <resource>
+                <directory>../workflow/ingestion-helper</directory>
+                <includes>
+                    <include>schema.sql</include>
+                </includes>
+            </resource>
+        </resources>
+    </build>
 </project>

pipeline/spanner/src/main/java/org/datacommons/ingestion/spanner/SpannerClient.java

@@ -231,17 +231,17 @@ public void validateOrInitializeDatabase() {
     }
   }
 
-  /** Reads DDL statements from the spanner_schema.sql file in the resources directory. */
+  /** Reads DDL statements from the schema.sql file in the resources directory. */
   List<String> readDdlStatements() {
-    InputStream inputStream = getClass().getClassLoader().getResourceAsStream("spanner_schema.sql");
+    InputStream inputStream = getClass().getClassLoader().getResourceAsStream("schema.sql");
     if (inputStream == null) {
-      throw new IllegalStateException("Could not find spanner_schema.sql in resources.");
+      throw new IllegalStateException("Could not find schema.sql in resources.");
     }
     try (BufferedReader reader =
         new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) {
       return parseDdlStatements(reader);
     } catch (IOException e) {
-      throw new IllegalStateException("Failed to read spanner_schema.sql", e);
+      throw new IllegalStateException("Failed to read schema.sql", e);
     }
   }
 

pipeline/spanner/src/main/resources/spanner_schema.sql

@@ -30,21 +30,30 @@ substitutions:
   _AR_REPO_URL: 'us-docker.pkg.dev/datcom-ci/gcr.io'
   _BQ_SPANNER_CONN_ID: 'projects/datcom-ci/locations/us-central1/connections/bq_spanner_conn_test'
   _VERSION: '${SHORT_SHA}'
-  _DATAFLOW_TEMPLATE_PATH: 'gs://datcom-templates/templates/flex/ingestion.json'
-  _PROD_TAG: 'latest' # TODO: rename to prod
+  _DATAFLOW_TEMPLATE_PATH: 'gs://datcom-templates/templates/flex/ingestion-${_VERSION}.json'
+  _PROD_TAG: 'stable'
 
 steps:
 
 # 1. Build and push helper images
 - id: 'build-ingestion-helper'
   name: 'gcr.io/cloud-builders/gcloud'
   args: ['builds', 'submit', 'ingestion-helper', '--config', 'ingestion-helper/cloudbuild.yaml', '--substitutions', '_AR_REPO_URL=${_AR_REPO_URL},_VERSION=${_VERSION}']
-  dir: 'import-automation/workflow'
+  dir: 'pipeline/workflow'
 
 - id: 'build-import-helper'
   name: 'gcr.io/cloud-builders/gcloud'
   args: ['builds', 'submit', 'import-helper', '--config', 'import-helper/cloudbuild.yaml', '--substitutions', '_AR_REPO_URL=${_AR_REPO_URL},_VERSION=${_VERSION}']
-  dir: 'import-automation/workflow'
+  dir: 'pipeline/workflow'
+
+- id: 'build-dataflow-template'
+  name: 'gcr.io/cloud-builders/gcloud'
+  args:
+    - 'builds'
+    - 'submit'
+    - '.'
+    - '--config=pipeline/ingestion/cloudbuild.yaml'
+    - '--substitutions=_VERSION=${_VERSION},_IMAGE_GCR_PATH=${_AR_REPO_URL}/dataflow-templates/ingestion,_INSTANCE_ID=${_SPANNER_INSTANCE_ID},_DATABASE_ID=${_SPANNER_DATABASE_ID},_GCS_BUCKET=${_GCS_BUCKET_ID},_REGION=${_LOCATION}'
 
 # 2. Trigger Staging Build (Child)
 # Overrides default (Production) values with Staging values.
@@ -57,7 +66,7 @@ steps:
     - '--config=deploy-services.yaml'
     - '--project=${_PROJECT_ID}'
     - '--substitutions=_PROJECT_ID=${_PROJECT_ID},_SPANNER_PROJECT_ID=${_SPANNER_PROJECT_ID},_SPANNER_INSTANCE_ID=${_SPANNER_INSTANCE_ID},_SPANNER_DATABASE_ID=${_SPANNER_DATABASE_ID},_SPANNER_GRAPH_DATABASE_ID=${_SPANNER_GRAPH_DATABASE_ID},_GCS_BUCKET_ID=${_GCS_BUCKET_ID},_LOCATION=${_LOCATION},_GCS_MOUNT_BUCKET=${_GCS_MOUNT_BUCKET},_BQ_DATASET_ID=${_BQ_DATASET_ID},_PROJECT_NUMBER=${_PROJECT_NUMBER},_BQ_SPANNER_CONN_ID=${_BQ_SPANNER_CONN_ID},_VERSION=${_VERSION},_DATAFLOW_TEMPLATE_PATH=${_DATAFLOW_TEMPLATE_PATH}'
-  dir: 'import-automation/workflow'
+  dir: 'pipeline/workflow'
 
 # 2. Run E2E Tests on Staging
 - id: 'e2e-test-staging'
@@ -75,12 +84,12 @@ steps:
     - 'SPANNER_INSTANCE_ID=${_SPANNER_INSTANCE_ID}'
     - 'SPANNER_DATABASE_ID=${_SPANNER_DATABASE_ID}'
     - 'GCS_BUCKET_ID=${_GCS_BUCKET_ID}'
-  dir: 'import-automation/workflow'
+  dir: 'pipeline/workflow'
 
 - id: 'tag-prod'
   name: 'gcr.io/cloud-builders/gcloud'
   args: ['builds', 'submit', '.', '--config', 'tag-prod.yaml', '--substitutions', '_AR_REPO_URL=${_AR_REPO_URL},_PROD_TAG=${_PROD_TAG},_VERSION=${_VERSION}']
-  dir: 'import-automation/workflow'
+  dir: 'pipeline/workflow'
 
 # 3. Trigger Production Build (Child)
 # Uses default (Production) values defined in cloudbuild.yaml.
@@ -93,7 +102,7 @@ steps:
     - '--config=deploy-services.yaml'
     - '--project=${_PROJECT_ID}'  # Build runs in CI project, deploys to Prod
     - '--substitutions=_VERSION=${_PROD_TAG}'
-  dir: 'import-automation/workflow'
+  dir: 'pipeline/workflow'
 
 options:
   logging: CLOUD_LOGGING_ONLY