Dev

1- Databricks Lakehouse Platform/1.0 - Creating Clusters.py

@@ -0,0 +1,75 @@
+# Databricks notebook source
+# MAGIC %md
+# MAGIC
+# MAGIC ## Creating Clusters
+
+# COMMAND ----------
+
+# MAGIC %md
+# MAGIC #### Creating a Demo Cluster
+# MAGIC
+# MAGIC Create a cluster with the following configurations:
+# MAGIC
+# MAGIC | Setting | Instructions |
+# MAGIC |--|--|
+# MAGIC |Cluster name|**Demo Cluster**|
+# MAGIC |Cluster mode|**Signle node**|
+# MAGIC |Runtime version|Select the Databricks runtime version 13.3 LTS|
+# MAGIC |Photon Acceleration| Uncheck the option |
+# MAGIC |Node type|4 cores|
+# MAGIC |Auto termination|30 minutes|
+# MAGIC
+
+# COMMAND ----------
+
+# MAGIC %md
+# MAGIC 1-	Navigate to the **Compute** tab in the left side bar.
+# MAGIC
+# MAGIC 2-	Under **All-purpose compute** tab, click **Create compute**.
+# MAGIC
+
+# COMMAND ----------
+
+# MAGIC %md-sandbox
+# MAGIC
+# MAGIC <div  style="text-align: center; line-height: 0; padding-top: 9px;">
+# MAGIC   <img src="https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/main/Labs/Includes/images/cluster_par1.png">
+# MAGIC </div>
+
+# COMMAND ----------
+
+# MAGIC %md
+# MAGIC 3-	On top, click on the default name to change it. Name your cluster as **Demo Cluster**
+# MAGIC
+# MAGIC 4-	Select **Single node** cluster
+# MAGIC
+
+# COMMAND ----------
+
+# MAGIC %md-sandbox
+# MAGIC
+# MAGIC <div  style="text-align: center; line-height: 0; padding-top: 9px;">
+# MAGIC   <img src="https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/main/Labs/Includes/images/cluster_par2.png">
+# MAGIC </div>
+
+# COMMAND ----------
+
+# MAGIC %md
+# MAGIC 5-	Select the Databricks runtime version 13.3 LTS (Long Term Support)
+# MAGIC
+# MAGIC 6-	Uncheck the option for the **Use Photon Acceleration**
+# MAGIC
+# MAGIC 7-	Select a Node type of 4 cores
+# MAGIC
+# MAGIC 8-	Set the auto termination of the cluster to 30 minutes
+# MAGIC
+# MAGIC 9-	Lastly, click **Create compute**.
+# MAGIC
+
+# COMMAND ----------
+
+# MAGIC %md-sandbox
+# MAGIC
+# MAGIC <div  style="text-align: center; line-height: 0; padding-top: 9px;">
+# MAGIC   <img src="https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/main/Labs/Includes/images/cluster_par3.png">
+# MAGIC </div>

1- Databricks Lakehouse Platform/1.1 - Notebook Basics.py

@@ -12,31 +12,31 @@
 # MAGIC # Title 1
 # MAGIC ## Title 2
 # MAGIC ### Title 3
-# MAGIC 
+# MAGIC
 # MAGIC text with a **bold** and *italicized* in it.
-# MAGIC 
+# MAGIC
 # MAGIC Ordered list
 # MAGIC 1. first
 # MAGIC 1. second
 # MAGIC 1. third
-# MAGIC 
+# MAGIC
 # MAGIC Unordered list
 # MAGIC * coffee
 # MAGIC * tea
-# MAGIC * tea
-# MAGIC 
-# MAGIC 
+# MAGIC * milk
+# MAGIC
+# MAGIC
 # MAGIC Images:
 # MAGIC ![Associate-badge](https://www.databricks.com/wp-content/uploads/2022/04/associate-badge-eng.svg)
-# MAGIC 
+# MAGIC
 # MAGIC And of course, tables:
-# MAGIC 
+# MAGIC
 # MAGIC | user_id | user_name |
 # MAGIC |---------|-----------|
 # MAGIC |    1    |    Adam   |
 # MAGIC |    2    |    Sarah  |
 # MAGIC |    3    |    John   |
-# MAGIC 
+# MAGIC
 # MAGIC Links (or Embedded HTML): <a href="https://docs.databricks.com/notebooks/notebooks-manage.html" target="_blank"> Managing Notebooks documentation</a>
 
 # COMMAND ----------
@@ -70,4 +70,9 @@
 
 # COMMAND ----------
 
+print("End of the Notebook")
+
+# COMMAND ----------
 
+# MAGIC %md
+# MAGIC ## End of Notebook

1- Databricks Lakehouse Platform/1.2 - Understanding Delta Tables.sql

@@ -1,16 +1,52 @@
 -- Databricks notebook source
+-- MAGIC %md
+-- MAGIC ## Creating Delta Lake Tables
+
+-- COMMAND ----------
+
+-- MAGIC %md
+-- MAGIC **Note:** If your workspace does not support the `hive_metastore` catalog, switch to the **unity-catalog** branch in this Git Folder.
+
+-- COMMAND ----------
+
+USE CATALOG hive_metastore
+
+-- COMMAND ----------
+
 CREATE TABLE employees
   (id INT, name STRING, salary DOUBLE);
 
 -- COMMAND ----------
 
+-- MAGIC %md
+-- MAGIC
+-- MAGIC ## Catalog Explorer
+-- MAGIC
+-- MAGIC Check the created **employees** table in the **Catalog** explorer.
+
+-- COMMAND ----------
+
+-- MAGIC %md
+-- MAGIC ## Inserting Data
+
+-- COMMAND ----------
+
 INSERT INTO employees
 VALUES 
   (1, "Adam", 3500.0),
-  (2, "Sarah", 4020.5),
+  (2, "Sarah", 4020.5);
+
+INSERT INTO employees
+VALUES
   (3, "John", 2999.3),
-  (4, "Thomas", 4000.3),
-  (5, "Anna", 2500.0),
+  (4, "Thomas", 4000.3);
+
+INSERT INTO employees
+VALUES
+  (5, "Anna", 2500.0);
+
+INSERT INTO employees
+VALUES
   (6, "Kim", 6200.3)
 
 -- COMMAND ----------
@@ -19,14 +55,29 @@ SELECT * FROM employees
 
 -- COMMAND ----------
 
+-- MAGIC %md
+-- MAGIC ## Exploring Table Metadata
+
+-- COMMAND ----------
+
 DESCRIBE DETAIL employees
 
 -- COMMAND ----------
 
+-- MAGIC %md
+-- MAGIC ## Exploring Table Directory
+
+-- COMMAND ----------
+
 -- MAGIC %fs ls 'dbfs:/user/hive/warehouse/employees'
 
 -- COMMAND ----------
 
+-- MAGIC %md
+-- MAGIC ## Updating Table
+
+-- COMMAND ----------
+
 UPDATE employees 
 SET salary = salary + 100
 WHERE name LIKE "A%"
@@ -49,6 +100,11 @@ SELECT * FROM employees
 
 -- COMMAND ----------
 
+-- MAGIC %md
+-- MAGIC ## Exploring Table History
+
+-- COMMAND ----------
+
 DESCRIBE HISTORY employees
 
 -- COMMAND ----------
@@ -57,7 +113,7 @@ DESCRIBE HISTORY employees
 
 -- COMMAND ----------
 
--- MAGIC %fs head 'dbfs:/user/hive/warehouse/employees/_delta_log/00000000000000000002.json'
+-- MAGIC %fs head 'dbfs:/user/hive/warehouse/employees/_delta_log/00000000000000000005.json'
 
 -- COMMAND ----------
 

1- Databricks Lakehouse Platform/1.3 - Advanced Delta Lake Features (1).sql → 1- Databricks Lakehouse Platform/1.3 - Advanced Delta Lake Features.sql

@@ -1,14 +1,24 @@
 -- Databricks notebook source
+-- MAGIC %md
+-- MAGIC
+-- MAGIC ## Delta Time Travel
+
+-- COMMAND ----------
+
+USE CATALOG hive_metastore
+
+-- COMMAND ----------
+
 DESCRIBE HISTORY employees
 
 -- COMMAND ----------
 
 SELECT * 
-FROM employees VERSION AS OF 1
+FROM employees VERSION AS OF 4
 
 -- COMMAND ----------
 
-SELECT * FROM employees@v1
+SELECT * FROM employees@v4
 
 -- COMMAND ----------
 
@@ -20,7 +30,7 @@ SELECT * FROM employees
 
 -- COMMAND ----------
 
-RESTORE TABLE employees TO VERSION AS OF 2 
+RESTORE TABLE employees TO VERSION AS OF 5
 
 -- COMMAND ----------
 
@@ -32,6 +42,12 @@ DESCRIBE HISTORY employees
 
 -- COMMAND ----------
 
+-- MAGIC %md
+-- MAGIC
+-- MAGIC ## OPTIMIZE Command
+
+-- COMMAND ----------
+
 DESCRIBE DETAIL employees
 
 -- COMMAND ----------
@@ -53,6 +69,12 @@ DESCRIBE HISTORY employees
 
 -- COMMAND ----------
 
+-- MAGIC %md
+-- MAGIC
+-- MAGIC ## VACUUM Command
+
+-- COMMAND ----------
+
 VACUUM employees
 
 -- COMMAND ----------
@@ -81,16 +103,18 @@ SELECT * FROM employees@v1
 
 -- COMMAND ----------
 
-DROP TABLE employees
+-- MAGIC %md
+-- MAGIC
+-- MAGIC ## Dropping Tables
 
 -- COMMAND ----------
 
-SELECT * FROM employees
+DROP TABLE employees
 
 -- COMMAND ----------
 
--- MAGIC %fs ls 'dbfs:/user/hive/warehouse/employees'
+SELECT * FROM employees
 
 -- COMMAND ----------
 
-
+-- MAGIC %fs ls 'dbfs:/user/hive/warehouse/employees'

1- Databricks Lakehouse Platform/1.4 - Databases and Tables on Databricks.sql

@@ -1,4 +1,11 @@
 -- Databricks notebook source
+-- MAGIC %md
+-- MAGIC ## Managed Tables
+
+-- COMMAND ----------
+
+USE CATALOG hive_metastore;
+
 CREATE TABLE managed_default
   (width INT, length INT, height INT);
 
@@ -11,6 +18,12 @@ DESCRIBE EXTENDED managed_default
 
 -- COMMAND ----------
 
+-- MAGIC %md
+-- MAGIC
+-- MAGIC ## External Tables
+
+-- COMMAND ----------
+
 CREATE TABLE external_default
   (width INT, length INT, height INT)
 LOCATION 'dbfs:/mnt/demo/external_default';
@@ -24,6 +37,12 @@ DESCRIBE EXTENDED external_default
 
 -- COMMAND ----------
 
+-- MAGIC %md
+-- MAGIC
+-- MAGIC ## Dropping Tables
+
+-- COMMAND ----------
+
 DROP TABLE managed_default
 
 -- COMMAND ----------
@@ -40,6 +59,11 @@ DROP TABLE external_default
 
 -- COMMAND ----------
 
+-- MAGIC %md
+-- MAGIC ## Creating Schemas
+
+-- COMMAND ----------
+
 CREATE SCHEMA new_default
 
 -- COMMAND ----------
@@ -80,14 +104,19 @@ DROP TABLE external_new_default;
 
 -- COMMAND ----------
 
--- MAGIC %fs ls 'dbfs:/user/hive/warehouse/managed_new_default'
+-- MAGIC %fs ls 'dbfs:/user/hive/warehouse/new_default.db/managed_new_default'
 
 -- COMMAND ----------
 
 -- MAGIC %fs ls 'dbfs:/mnt/demo/external_new_default'
 
 -- COMMAND ----------
 
+-- MAGIC %md
+-- MAGIC ## Creating Schemas in Custom Location
+
+-- COMMAND ----------
+
 CREATE SCHEMA custom
 LOCATION 'dbfs:/Shared/schemas/custom.db'
 
@@ -134,7 +163,3 @@ DROP TABLE external_custom;
 -- COMMAND ----------
 
 -- MAGIC %fs ls 'dbfs:/mnt/demo/external_custom'
-
--- COMMAND ----------
-
-