Release 6.3.3 - Deprecate Data Mesh, update SAP & Cloud Build (#109)

Author: TXZebra

config/config.json

@@ -4,7 +4,6 @@
     "deploySFDC": true,
     "deployMarketing": true,
     "deployOracleEBS": true,
-    "deployDataMesh": true,
     "enableTaskDependencies": false,
     "turboMode": true,
     "projectIdSource": "",
@@ -72,12 +71,6 @@
         "region": "us-central1",
         "processingDataset": "CORTEX_VERTEX_AI_PROCESSING"
     },
-    "DataMesh": {
-        "deployDescriptions": true,
-        "deployLakes": false,
-        "deployCatalog": false,
-        "deployACLs": false
-    },
     "SAP": {
         "deployCDC": true,
         "SQLFlavor": "ecc",
@@ -202,4 +195,4 @@
             "reporting": "REPORTING_OracleEBS"
         }
     }
-}
+}

config/config_default.json

@@ -4,7 +4,6 @@
     "deploySFDC": true,
     "deployMarketing": true,
     "deployOracleEBS": true,
-    "deployDataMesh": true,
     "enableTaskDependencies": false,
     "turboMode": true,
     "projectIdSource": "",
@@ -72,12 +71,6 @@
         "region": "us-central1",
         "processingDataset": "CORTEX_VERTEX_AI_PROCESSING"
     },
-    "DataMesh": {
-        "deployDescriptions": true,
-        "deployLakes": false,
-        "deployCatalog": false,
-        "deployACLs": false
-    },
     "SAP": {
         "deployCDC": true,
         "SQLFlavor": "ecc",

docs/deprecated/README_Marketing.md

@@ -100,9 +100,9 @@ The following configurations are required to be in place to successfully bring d
 
     > **NOTE**: Cortex Data Foundations will automatically detect and ingest data from all customers (accounts) that are accessible to the credentials used to generate the token.
 
-4.  Now create a secret using Google Cloud Secret Manager (Security -> Secret Manager) with name “`cortex-framework-google-ads-yaml`” using the following format:
+4.  Now create a secret using Google Cloud Secret Manager (Security -> Secret Manager) with name “airflow-connections-googleads_raw_dataflow`” using the following format:
     ```
-    {"developer_token": "developer_token_value", "refresh_token": "refresh_token_value", "client_id": "client_id_value", "client_secret": "client_secret_value", "use_proto_plus": False}
+    {"conn_type":"google_ads","extra": {"developer_token": "developer_token_value", "refresh_token": "refresh_token_value", "client_id": "client_id_value", "client_secret": "client_secret_value", "use_proto_plus": false}}
     ```
     For the values:
     * `developer_token` value : Available in Google Ads account

src/OracleEBS/cloudbuild.oracle_ebs.yaml

@@ -76,26 +76,6 @@ steps:
           --materializer_settings_file config/reporting_settings.yaml \
           "${_WORKER_POOL_OPTIONS[@]}"
 
-  # Deploy Data Mesh.
-  - name: "gcr.io/kittycorn-public/deploy-kittycorn:v2.0"
-    id: "ebs_datamesh"
-    waitFor: ["ebs_deploy"]
-    script: |
-      #!/usr/bin/env bash
-      set -e
-      _DEPLOY_DATA_MESH_=$(jq -r ."deployDataMesh" ${_CONFIG_FILE})
-
-      if [[ ${_DEPLOY_DATA_MESH_} == "true" ]]; then
-        python3 src/common/data_mesh/deploy_data_mesh.py \
-            --config-file ${_CONFIG_FILE} \
-            --tag-template-directories config/tag_templates \
-            --policy-directories config/policy_taxonomies \
-            --lake-directories config/lakes \
-            --annotation-directories config/annotations/reporting
-      else
-        echo "==Skipping Data Mesh=="
-      fi
-
 logsBucket: gs://${_GCS_LOGS_BUCKET}
 serviceAccount: "${_BUILD_ACCOUNT}"
 substitutions:

src/OracleEBS/config/config.json

@@ -1,7 +1,6 @@
 {
     "testData": true,
     "deployOracleEBS": true,
-    "deployDataMesh": true,
     "turboMode": true,
     "projectIdSource": "",
     "projectIdTarget": "",
@@ -14,12 +13,6 @@
             "reporting": "K9_REPORTING"
         }
     },
-    "DataMesh": {
-        "deployDescriptions": true,
-        "deployLakes": false,
-        "deployCatalog": false,
-        "deployACLs": false
-    },
     "OracleEBS": {
         "itemCategorySetIds": [],
         "currencyConversionType": "Corporate",