Skip to content

Commit 24db8a8

Browse files
DataWithBaraaDataWithBaraa
committed
adding orchestrations
1 parent 431e97e commit 24db8a8

10 files changed

Lines changed: 386 additions & 18 deletions

‎script/gold/gold_fact_sales.ipynb‎

Lines changed: 215 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,215 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "markdown",
5+
"metadata": {
6+
"application/vnd.databricks.v1+cell": {
7+
"cellMetadata": {
8+
"byteLimit": 2048000,
9+
"rowLimit": 10000
10+
},
11+
"inputWidgets": {},
12+
"nuid": "a3495f99-90ec-472d-a12a-49ab7d16b3bd",
13+
"showTitle": false,
14+
"tableResultSettingsMap": {},
15+
"title": ""
16+
}
17+
},
18+
"source": [
19+
"#Initialization"
20+
]
21+
},
22+
{
23+
"cell_type": "markdown",
24+
"metadata": {
25+
"application/vnd.databricks.v1+cell": {
26+
"cellMetadata": {
27+
"byteLimit": 2048000,
28+
"rowLimit": 10000
29+
},
30+
"inputWidgets": {},
31+
"nuid": "dff4b590-576c-4a13-b547-083d7a31b669",
32+
"showTitle": false,
33+
"tableResultSettingsMap": {},
34+
"title": ""
35+
}
36+
},
37+
"source": [
38+
"#The Transformation Logic"
39+
]
40+
},
41+
{
42+
"cell_type": "code",
43+
"execution_count": 0,
44+
"metadata": {
45+
"application/vnd.databricks.v1+cell": {
46+
"cellMetadata": {
47+
"byteLimit": 2048000,
48+
"rowLimit": 10000
49+
},
50+
"inputWidgets": {},
51+
"nuid": "7801ecff-c5b2-4d1c-89ac-d6acd24848a5",
52+
"showTitle": false,
53+
"tableResultSettingsMap": {},
54+
"title": ""
55+
}
56+
},
57+
"outputs": [],
58+
"source": [
59+
"query = \"\"\"\n",
60+
"SELECT\n",
61+
" ROW_NUMBER() OVER (ORDER BY ci.customer_id) AS customer_key,\n",
62+
" ci.customer_id,\n",
63+
" ci.customer_number,\n",
64+
" ci.first_name,\n",
65+
" ci.last_name,\n",
66+
" la.country,\n",
67+
" ci.marital_status,\n",
68+
" CASE\n",
69+
" WHEN ci.gender <> 'n/a' THEN ci.gender\n",
70+
" ELSE COALESCE(ca.gender, 'n/a')\n",
71+
" END AS gender,\n",
72+
" ca.birth_date AS birthdate,\n",
73+
" ci.created_date AS create_date\n",
74+
"FROM silver.crm_customers ci\n",
75+
"LEFT JOIN silver.erp_customers ca\n",
76+
" ON ci.customer_number = ca.customer_id\n",
77+
"LEFT JOIN silver.erp_customer_location la\n",
78+
" ON ci.customer_number = la.customer_id\n",
79+
"\"\"\"\n",
80+
"df = spark.sql(query)\n"
81+
]
82+
},
83+
{
84+
"cell_type": "code",
85+
"execution_count": 0,
86+
"metadata": {
87+
"application/vnd.databricks.v1+cell": {
88+
"cellMetadata": {
89+
"byteLimit": 2048000,
90+
"rowLimit": 10000
91+
},
92+
"inputWidgets": {},
93+
"nuid": "f59bb479-889d-4f6e-b0f2-8ebc1ee41d6d",
94+
"showTitle": false,
95+
"tableResultSettingsMap": {},
96+
"title": ""
97+
}
98+
},
99+
"outputs": [],
100+
"source": [
101+
"df.display()"
102+
]
103+
},
104+
{
105+
"cell_type": "markdown",
106+
"metadata": {
107+
"application/vnd.databricks.v1+cell": {
108+
"cellMetadata": {
109+
"byteLimit": 2048000,
110+
"rowLimit": 10000
111+
},
112+
"inputWidgets": {},
113+
"nuid": "3aab6a15-eb4d-4b3b-8e8f-18acb7099ab3",
114+
"showTitle": false,
115+
"tableResultSettingsMap": {},
116+
"title": ""
117+
}
118+
},
119+
"source": [
120+
"#Writing Gold Table"
121+
]
122+
},
123+
{
124+
"cell_type": "code",
125+
"execution_count": 0,
126+
"metadata": {
127+
"application/vnd.databricks.v1+cell": {
128+
"cellMetadata": {
129+
"byteLimit": 2048000,
130+
"rowLimit": 10000
131+
},
132+
"inputWidgets": {},
133+
"nuid": "045b25e1-2aed-4afb-b9d7-e088473dc71e",
134+
"showTitle": false,
135+
"tableResultSettingsMap": {},
136+
"title": ""
137+
}
138+
},
139+
"outputs": [],
140+
"source": [
141+
"df.write.mode(\"overwrite\").format(\"delta\").saveAsTable(\"workspace.gold.dim_customers\")"
142+
]
143+
},
144+
{
145+
"cell_type": "markdown",
146+
"metadata": {
147+
"application/vnd.databricks.v1+cell": {
148+
"cellMetadata": {
149+
"byteLimit": 2048000,
150+
"rowLimit": 10000
151+
},
152+
"inputWidgets": {},
153+
"nuid": "56f1fb8a-d6a4-4a79-8f2e-726db7ba2d9f",
154+
"showTitle": false,
155+
"tableResultSettingsMap": {},
156+
"title": ""
157+
}
158+
},
159+
"source": [
160+
"## Sanity checks of Gold table"
161+
]
162+
},
163+
{
164+
"cell_type": "code",
165+
"execution_count": 0,
166+
"metadata": {
167+
"application/vnd.databricks.v1+cell": {
168+
"cellMetadata": {
169+
"byteLimit": 2048000,
170+
"implicitDf": true,
171+
"rowLimit": 10000
172+
},
173+
"inputWidgets": {},
174+
"nuid": "a98632c2-f3ab-4100-b942-7f93ab187a3a",
175+
"showTitle": false,
176+
"tableResultSettingsMap": {},
177+
"title": ""
178+
}
179+
},
180+
"outputs": [],
181+
"source": [
182+
"%sql\n",
183+
"SELECT * FROM workspace.silver.erp_customers"
184+
]
185+
}
186+
],
187+
"metadata": {
188+
"application/vnd.databricks.v1+notebook": {
189+
"computePreferences": null,
190+
"dashboards": [],
191+
"environmentMetadata": {
192+
"base_environment": "",
193+
"environment_version": "4"
194+
},
195+
"inputWidgetPreferences": null,
196+
"language": "python",
197+
"notebookMetadata": {
198+
"mostRecentlyExecutedCommandWithImplicitDF": {
199+
"commandId": 5051243675007213,
200+
"dataframes": [
201+
"_sqldf"
202+
]
203+
},
204+
"pythonIndentUnit": 4
205+
},
206+
"notebookName": "gold_fact_sales",
207+
"widgets": {}
208+
},
209+
"language_info": {
210+
"name": "python"
211+
}
212+
},
213+
"nbformat": 4,
214+
"nbformat_minor": 0
215+
}

‎script/gold/gold_orchestration.ipynb‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "code",
5+
"execution_count": 0,
6+
"metadata": {
7+
"application/vnd.databricks.v1+cell": {
8+
"cellMetadata": {},
9+
"inputWidgets": {},
10+
"nuid": "c966f7ff-82d4-459b-9514-6026c1c8aef8",
11+
"showTitle": false,
12+
"tableResultSettingsMap": {},
13+
"title": ""
14+
}
15+
},
16+
"outputs": [],
17+
"source": [
18+
"notebooks = [\n",
19+
" \"./crm/silver_crm_cust_info\",\n",
20+
" \"./crm/silver_crm_prd_info\",\n",
21+
" \"./crm/silver_crm_sales_details\",\n",
22+
" \"./erp/silver_erp_cust_az12\",\n",
23+
" \"./erp/silver_erp_loc_a101\",\n",
24+
" \"./erp/silver_erp_px_cat_g1v2\"\n",
25+
"]\n",
26+
"\n",
27+
"for nb in notebooks:\n",
28+
" print(f\"Running {nb}\")\n",
29+
" dbutils.notebook.run(nb, timeout_seconds=0)"
30+
]
31+
}
32+
],
33+
"metadata": {
34+
"application/vnd.databricks.v1+notebook": {
35+
"computePreferences": null,
36+
"dashboards": [],
37+
"environmentMetadata": {
38+
"base_environment": "",
39+
"environment_version": "4"
40+
},
41+
"inputWidgetPreferences": null,
42+
"language": "python",
43+
"notebookMetadata": {
44+
"pythonIndentUnit": 4
45+
},
46+
"notebookName": "gold_orchestration",
47+
"widgets": {}
48+
},
49+
"language_info": {
50+
"name": "python"
51+
}
52+
},
53+
"nbformat": 4,
54+
"nbformat_minor": 0
55+
}

‎script/init_lakehouse.ipynb‎

Lines changed: 39 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,11 @@
122122
"execution_count": 0,
123123
"metadata": {
124124
"application/vnd.databricks.v1+cell": {
125-
"cellMetadata": {},
125+
"cellMetadata": {
126+
"byteLimit": 2048000,
127+
"implicitDf": true,
128+
"rowLimit": 10000
129+
},
126130
"inputWidgets": {},
127131
"nuid": "21daaf85-bbb3-4f12-89a4-3e979c7e8bfc",
128132
"showTitle": false,
@@ -133,7 +137,39 @@
133137
"outputs": [],
134138
"source": [
135139
"%sql\n",
136-
"drop table workspace.bronze.crm_cust_info;"
140+
"drop table if exists workspace.bronze.crm_prd_info;\n",
141+
"drop table if exists workspace.bronze.crm_sales_details;\n",
142+
"drop table if exists workspace.bronze.erp_cust_az12;\n",
143+
"drop table if exists workspace.bronze.erp_loc_a101;\n",
144+
"drop table if exists workspace.bronze.erp_px_cat_g1v2"
145+
]
146+
},
147+
{
148+
"cell_type": "code",
149+
"execution_count": 0,
150+
"metadata": {
151+
"application/vnd.databricks.v1+cell": {
152+
"cellMetadata": {
153+
"byteLimit": 2048000,
154+
"implicitDf": true,
155+
"rowLimit": 10000
156+
},
157+
"inputWidgets": {},
158+
"nuid": "6a5cbc94-d6d1-40f4-91e2-fdc4a8f606bb",
159+
"showTitle": false,
160+
"tableResultSettingsMap": {},
161+
"title": ""
162+
}
163+
},
164+
"outputs": [],
165+
"source": [
166+
"%sql\n",
167+
"drop table if exists workspace.silver.crm_customers;\n",
168+
"drop table if exists workspace.silver.crm_products;\n",
169+
"drop table if exists workspace.silver.crm_sales;\n",
170+
"drop table if exists workspace.silver.erp_customer_location;\n",
171+
"drop table if exists workspace.silver.erp_customers;\n",
172+
"drop table if exists workspace.silver.erp_product_category;"
137173
]
138174
}
139175
],
@@ -149,7 +185,7 @@
149185
"language": "python",
150186
"notebookMetadata": {
151187
"mostRecentlyExecutedCommandWithImplicitDF": {
152-
"commandId": 8634005043532649,
188+
"commandId": 5051243675007331,
153189
"dataframes": [
154190
"_sqldf"
155191
]

‎script/silver/crm/silver_crm_cust_info.ipynb‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -319,13 +319,20 @@
319319
"inputWidgets": {},
320320
"nuid": "7f935436-181c-498e-8455-141a23472d20",
321321
"showTitle": false,
322-
"tableResultSettingsMap": {},
322+
"tableResultSettingsMap": {
323+
"0": {
324+
"dataGridStateBlob": "{\"version\":1,\"tableState\":{\"columnPinning\":{\"left\":[\"#row_number#\"],\"right\":[]},\"columnSizing\":{},\"columnVisibility\":{}},\"settings\":{\"columns\":{}},\"syncTimestamp\":1768815776809}",
325+
"filterBlob": null,
326+
"queryPlanFiltersBlob": null,
327+
"tableResultIndex": 0
328+
}
329+
},
323330
"title": ""
324331
}
325332
},
326333
"outputs": [],
327334
"source": [
328-
"df.display()"
335+
"df.limit(10).display()"
329336
]
330337
},
331338
{
@@ -407,7 +414,7 @@
407414
"outputs": [],
408415
"source": [
409416
"%sql\n",
410-
"SELECT * FROM workspace.silver.crm_customers"
417+
"SELECT * FROM workspace.silver.crm_customers LIMIT 10"
411418
]
412419
}
413420
],

‎script/silver/crm/silver_crm_prd_info.ipynb‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -356,7 +356,7 @@
356356
"RENAME_MAP = {\n",
357357
" \"prd_id\": \"product_id\",\n",
358358
" \"cat_id\": \"category_id\",\n",
359-
" \"prd_key\": \"product_key\",\n",
359+
" \"prd_key\": \"product_number\",\n",
360360
" \"prd_nm\": \"product_name\",\n",
361361
" \"prd_cost\": \"product_cost\",\n",
362362
" \"prd_line\": \"product_line\",\n",
@@ -404,7 +404,7 @@
404404
},
405405
"outputs": [],
406406
"source": [
407-
"df.display()"
407+
"df.limit(10).display()"
408408
]
409409
},
410410
{
@@ -486,7 +486,7 @@
486486
"outputs": [],
487487
"source": [
488488
"%sql\n",
489-
"SELECT * FROM workspace.silver.crm_sales"
489+
"SELECT * FROM workspace.silver.crm_products LIMIT 10"
490490
]
491491
}
492492
],

0 commit comments

Comments
 (0)