Release/0.4.0 (#240)

* GCP first round of updates to use SP and not use service account

* GCP use-sp-auth fixes

* ran dabs successfully

* ran dabs successfully with sp

* ran dabs successfully with sp

* fixed sp in config

* Tested gcp calls

* add pagination support

* deleted job_runs_client.py

* matching the notebooks to the library version and functions

* setup.py

* removed pip install from initialize.py

* removed pip install from initialize.pyv2

* Redoing changes that seem to have disappeared since last commit

re implementing some changes that are not present

* updated the regex for azuredatabricks matching

* changed setup.py

* adding tags to SAT workflows

* issue with accounts api

* Added ../Includes/install_sat_sdk to these notebook.

* Added comment for gov-3

* GCP token fixes

* Added code to handle domain name based on the environment we are in

* changes to logic and gov cloud

* Log delivery config changes

Made changes to log delivery config changes between azure(workspace level) and other clouds (account level)

* diagnostic log library changes

* diagnostic log library changes - setup.py

* Added domain names to other clouds and the build number for tests.

* regex issue for domain check

* Clean run on aws after these fixes

* ignore schema lists and treat as pattern 1

* Fixed initial run issue

* Fixed check for the uc share

* throttle calls between pages and fix empty responses

* added maxpages and timebetweencalls to initialize json

* bug fix for empty records

* Updated build number

* fixed to carry azure account id

* Cleanup and gov cloud diagnostics (#238)

Co-authored-by: arunpamulapati <arunpamulapati>

* Configured the official package

* Fixed tag value

---------

Co-authored-by: arunpamulapati <arunpamulapati>
Co-authored-by: arunpamulapati <arun.pamulapati@databricks.com>
Co-authored-by: ramdas.murali <ramdas.murali@databricks.com>
Co-authored-by: ramdaskmdb <ramdaskmdb>
Co-authored-by: Shreel Shah <shreelshah12@gmail.com>
Co-authored-by: Arun Pamulapati <39059536+arunpamulapati@users.noreply.github.com>

Author: 4 people

dabs/dabs_template/databricks_template_schema.json

@@ -9,10 +9,6 @@
             "type": "string",
             "description": "Cloud type"
         },
-        "google_service_account": {
-            "type": "string",
-            "description": "Google service account"
-        },
         "latest_lts": {
             "type": "string",
             "description": "Latest LTS version"

dabs/dabs_template/template/tmp/resources/sat_driver_job.yml.tmpl

@@ -2,6 +2,8 @@ resources:
   jobs:
     sat_driver:
       name: "SAT Driver Notebook"
+      tags:
+        Application: SAT
       schedule:
         quartz_cron_expression: "0 0 8 ? * Mon,Wed,Fri"
         timezone_id: "America/New_York"
@@ -25,8 +27,4 @@ resources:
             spark_version: {{.latest_lts}}
             runtime_engine: "PHOTON"
             node_type_id: {{.node_type}}
-            {{- if eq .cloud "gcp" }}
-            gcp_attributes:
-              google_service_account: {{.google_service_account}}
-            {{- end }}
       {{- end }}

dabs/dabs_template/template/tmp/resources/sat_initiliazer_job.yml.tmpl

@@ -2,6 +2,8 @@ resources:
   jobs:
     sat_initializer:
       name: "SAT Initializer Notebook (one-time)"
+      tags:
+        Application: SAT
 
       tasks:
         - task_key: "sat_initializer"
@@ -25,8 +27,4 @@ resources:
             spark_version: {{.latest_lts}}
             runtime_engine: "PHOTON"
             node_type_id: {{.node_type}}
-            {{- if eq .cloud "gcp" }}
-            gcp_attributes:
-              google_service_account: {{.google_service_account}}
-            {{- end }}
       {{- end }}

dabs/main.py

@@ -13,7 +13,6 @@ def install(client: WorkspaceClient, answers: dict, profile: str):
     config = {
         "catalog": answers.get("catalog", None),
         "cloud": cloud,
-        "google_service_account": answers.get("gcp-impersonate-service-account", None),
         "latest_lts": client.clusters.select_spark_version(
             long_term_support=True,
             latest=True,

dabs/sat/config.py

@@ -108,15 +108,15 @@ def cloud_specific_questions(client: WorkspaceClient):
     ]
     gcp = [
         Text(
-            name="gcp-gs-path-to-json",
-            message="Path to JSON key file",
+            name="gcp-client-id",
+            message="Client ID",
             ignore=cloud_validation(client, "gcp"),
         ),
-        Text(
-            name="gcp-impersonate-service-account",
-            message="Impersonate Service Account",
+        Password(
+            name="gcp-client-secret",
+            message="Client Secret",
             ignore=cloud_validation(client, "gcp"),
-            default="",
+            echo="",
         ),
     ]
     aws = [
@@ -179,7 +179,7 @@ def generate_secrets(client: WorkspaceClient, answers: dict, cloud_type: str):
             string_value="{}",
         )
 
-    if cloud_type == "aws":
+    if cloud_type == "aws" or cloud_type == "gcp":
         client.secrets.put_secret(
             scope=scope_name,
             key="use-sp-auth",

notebooks/Includes/install_sat_sdk.py

@@ -29,7 +29,7 @@
 
 # COMMAND ----------
 
-SDK_VERSION='0.0.102' 
+SDK_VERSION='0.1.38'
 
 # COMMAND ----------
 

notebooks/Includes/workspace_analysis.py

@@ -981,9 +981,8 @@ def cc_check(df):
 # COMMAND ----------
 
 # DBTITLE 1,Get all audit log delivery configurations. Should be enabled.
-check_id='8' #Log delivery configurations
+check_id='8' #GOV-3 Log delivery configurations
 enabled, sbp_rec = getSecurityBestPracticeRecord(check_id, cloud_type)
-workspaceId = workspace_id
 
 def log_check(df):
     if df is not None and not isEmpty(df) and len(df.collect())>=1:
@@ -997,14 +996,15 @@ def log_check(df):
         return (check_id, 1, {})   
 
 if enabled:   
-    tbl_name = 'acctlogdelivery' 
+    tbl_name = 'acctlogdelivery' if cloud_type != 'azure' else 'acctlogdelivery' + '_' + workspace_id 
     sql=f'''
         SELECT config_name, config_id  
         FROM {tbl_name} 
         WHERE log_type="AUDIT_LOGS" and status="ENABLED" 
         '''
     sqlctrl(workspace_id, sql, log_check)
 
+
 # COMMAND ----------
 
 # DBTITLE 1,How long since the last cluster restart
@@ -1140,7 +1140,7 @@ def uc_delta_share_ip_accesslist(df):
     else:
         return (check_id, 0, {})   
 if enabled:    
-    tbl_name = 'unitycatalogsharerecipients' + '_' + workspace_id
+    tbl_name = 'delta_sharing_recepients_list' + '_' + workspace_id
     sql=f'''
         SELECT name, owner
         FROM {tbl_name} 
@@ -1162,7 +1162,7 @@ def uc_delta_share_expiration_time(df):
     else:
         return (check_id, 0, {})   
 if enabled:    
-    tbl_name = 'unitycatalogsharerecipients' + '_' + workspace_id
+    tbl_name = 'delta_sharing_recepients_list' + '_' + workspace_id
     sql=f'''
         SELECT tokens.* FROM (select explode(tokens) as tokens, full_name, owner
         FROM {tbl_name} 

notebooks/Setup/1. list_account_workspaces_to_conf_file.py

@@ -38,14 +38,28 @@
 )
 
 
+# COMMAND ----------
+
+from urllib.parse import urlparse
+domain = "com"
+try:
+    parsed_url = urlparse(hostname)
+    domain_parts = parsed_url.netloc.split('.')
+    if len(domain_parts) < 2:
+        raise ValueError("Invalid hostname: cannot extract domain part.")
+    domain = domain_parts[-1]
+except Exception as e:
+    print(f"Error extracting domain: {e}")
+print(domain)
+
 # COMMAND ----------
 
 import json
 
 dbutils.notebook.run(
     f"{basePath()}/notebooks/Utils/accounts_bootstrap",
     300,
-    {"json_": json.dumps(json_)},
+    {"json_": json.dumps(json_), "origin": "initializer"},
 )
 
 # COMMAND ----------
@@ -55,7 +69,6 @@
 # easily modify the new lines for new workspaces.
 def generateWorkspaceConfigFile():
     from pyspark.sql.functions import col, concat, lit
-
     dfexist = readWorkspaceConfigFile()
     excluded_configured_workspace = ""
     header_value = True
@@ -66,29 +79,33 @@ def generateWorkspaceConfigFile():
         header_value = False
     else:
         excluded_configured_workspace = ""  # running first time
+
     # get current workspaces that are not yet configured for analysis
     spsql = f"""select workspace_id, deployment_name as deployment_url, workspace_name, workspace_status from `acctworkspaces` 
-            where workspace_status = "RUNNING" {excluded_configured_workspace}"""
+            where trim(workspace_status) = "RUNNING" {excluded_configured_workspace}"""
+    #print(spsql)
+
     df = spark.sql(spsql)
+    #display(df)
     if len(df.take(1)) > 0:
         if cloud_type == "azure":
             df = df.withColumn(
                 "deployment_url",
-                concat(col("deployment_url"), lit(".azuredatabricks.net")),
+                concat(col("deployment_url"), lit(".azuredatabricks."), lit(domain)),
             )  # Azure
         elif cloud_type == "aws":
             df = df.withColumn(
                 "deployment_url",
-                concat(col("deployment_url"), lit(".cloud.databricks.com")),
+                concat(col("deployment_url"), lit(".cloud.databricks."), lit(domain)),
             )  # AWS
         else:
             df = df.withColumn(
                 "deployment_url",
-                concat(col("deployment_url"), lit(".gcp.databricks.com")),
+                concat(col("deployment_url"), lit(".gcp.databricks."), lit(domain)),
             )  # GCP
 
-        #both azure and gcp require sso
-        if cloud_type == "azure" or cloud_type == "gcp" :
+        # both azure and gcp require sso
+        if cloud_type == "azure" or cloud_type == "gcp":
             df = df.withColumn("sso_enabled", lit(True))
         else:
             df = df.withColumn("sso_enabled", lit(False))

notebooks/Setup/3. test_connections.py

@@ -28,16 +28,13 @@
 if cloud_type =='azure': #use client secret
   client_secret = dbutils.secrets.get(json_['master_name_scope'], json_["client_secret_key"])
   json_.update({'token':token, 'client_secret': client_secret})
-elif (cloud_type =='aws' and json_['use_sp_auth'].lower() == 'true'):  
+elif ((cloud_type =='aws' or cloud_type =='gcp') and json_['use_sp_auth'].lower() == 'true'):  
     client_secret = dbutils.secrets.get(json_['master_name_scope'], json_["client_secret_key"])
     json_.update({'token':'dapijedi', 'client_secret': client_secret})
     mastername =' ' # this will not be present when using SPs
     masterpwd = ' '  # we still need to send empty user/pwd.
     json_.update({'token':'dapijedi', 'mastername':mastername, 'masterpwd':masterpwd})
-else: #lets populate master key for accounts api
-    mastername = dbutils.secrets.get(json_['master_name_scope'], json_['master_name_key'])
-    masterpwd = dbutils.secrets.get(json_['master_pwd_scope'], json_['master_pwd_key'])
-    json_.update({'token':'dapijedi', 'mastername':mastername, 'masterpwd':masterpwd})
+
 json_.update({'url':hostname, 'workspace_id': 'accounts', 'cloud_type': cloud_type, 'clusterid':clusterid})
 
 
@@ -141,18 +138,8 @@ def modifyWorkspaceConfigFile(input_connection_arr):
 
 # COMMAND ----------
 
-def renewWorkspaceTokens():
-  if cloud_type == "gcp":
-    # refesh workspace level tokens if PAT tokens are not used as the temp tokens expire in 10 hours
-    gcp_status2 = dbutils.notebook.run("../Setup/gcp/configure_tokens_for_worksaces", 3000)
-    if gcp_status2 != "OK":
-      loggr.exception("Error Encountered in GCP Step#2", gcp_status2)
-      dbutils.notebook.exit()
-
-# COMMAND ----------
-
 input_status_arr=[]
-renewWorkspaceTokens()
+
 for ws in workspaces:
   import json
 
@@ -165,16 +152,12 @@ def renewWorkspaceTokens():
   if cloud_type =='azure': #use client secret
     client_secret = dbutils.secrets.get(json_['master_name_scope'], json_["client_secret_key"])
     json_.update({'token':token, 'client_secret': client_secret})
-  elif (cloud_type =='aws' and json_['use_sp_auth'].lower() == 'true'):  
+  elif ((cloud_type =='aws' or cloud_type =='gcp') and json_['use_sp_auth'].lower() == 'true'):  
     client_secret = dbutils.secrets.get(json_['master_name_scope'], json_["client_secret_key"])
     json_.update({'token':token, 'client_secret': client_secret})
     mastername =' ' # this will not be present when using SPs
     masterpwd = ' '  # we still need to send empty user/pwd.
     json_.update({'token':token, 'mastername':mastername, 'masterpwd':masterpwd})
-  else: #lets populate master key for accounts api
-    mastername = dbutils.secrets.get(json_['master_name_scope'], json_['master_name_key'])
-    masterpwd = dbutils.secrets.get(json_['master_pwd_scope'], json_['master_pwd_key'])
-    json_.update({'token':token, 'mastername':mastername, 'masterpwd':masterpwd})
 
   if (json_['use_mastercreds']) is False:
     tokenscope = json_['workspace_pat_scope']

notebooks/Setup/4. enable_workspaces_for_sat.py

@@ -5,6 +5,10 @@
 
 # COMMAND ----------
 
+# MAGIC %run ../Includes/install_sat_sdk
+
+# COMMAND ----------
+
 # MAGIC %run ../Utils/initialize
 
 # COMMAND ----------