Skip to content

Commit f55e9dc

Browse files
Ritik574-coderRitik574-coder
authored
Merge pull request #74 from Ritik574-coder/dbt_branch
doing cleaning and transformation in transactions table
2 parents 11dfff1 + 8979e90 commit f55e9dc

2 files changed

Lines changed: 175 additions & 4 deletions

File tree

‎explore_database/stores/stores.sql‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
--================================== stores table overview ====================================
77
--=============================================================================================
88

9-
SELECT TOP (1000)
9+
SELECT
1010
[store_id]
1111
,[store_name]
1212
,[store_type]
@@ -27,7 +27,7 @@ SELECT TOP (1000)
2727
,[is_active]
2828
,[has_parking]
2929
,[has_cafe]
30-
FROM .[bronze].[stores]
30+
FROM [bronze].[stores]
3131

3232
--=============================================================================================
3333
--================================== store_id column cleaning =================================
@@ -1079,5 +1079,5 @@ FROM
10791079
WHEN REPLACE(REPLACE(TRIM(LOWER(has_cafe)), CHAR(13), ''),CHAR(10), '') IN ('0', 'no', 'n','false') THEN 'False'
10801080
ELSE 'Unknown'
10811081
END as has_cafe
1082-
FROM .[bronze].[stores]
1082+
FROM [bronze].[stores]
10831083
)t ;

‎explore_database/transactions/transactions.sql‎

Lines changed: 172 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,10 @@
1+
--#############################################################################################
2+
--#################################### TRANSACTION DATA #######################################
3+
--#############################################################################################
4+
5+
--=============================================================================================
6+
--================================== transactions table overview ==============================
7+
--=============================================================================================
18
SELECT [transaction_id]
29
,[order_id]
310
,[order_line_number]
@@ -59,4 +66,168 @@ SELECT [transaction_id]
5966
,[data_source]
6067
,[record_created_ts]
6168
,[last_modified_ts]
62-
FROM .[bronze].[sales_transactions]
69+
FROM .[bronze].[sales_transactions]
70+
--=============================================================================================
71+
--=============================== transaction_id cleaning overview ============================
72+
--=============================================================================================
73+
-- transaction_id data overview
74+
SELECT
75+
transaction_id
76+
FROM bronze.sales_transactions ;
77+
78+
-- transaction_id data profiling
79+
SELECT
80+
transaction_id
81+
FROM bronze.sales_transactions
82+
WHERE transaction_id IS NULL
83+
OR transaction_id NOT LIKE 'TXN%'
84+
OR transaction_id != TRIM(transaction_id)
85+
OR transaction_id != UPPER(transaction_id)
86+
OR TRIM(transaction_id) = ''
87+
OR LEN(TRIM(transaction_id)) < 13 ;
88+
89+
-- Check for duplicates in transaction_id
90+
SELECT
91+
*
92+
FROM
93+
(
94+
SELECT
95+
transaction_id,
96+
ROW_NUMBER() OVER(PARTITION BY transaction_id ORDER BY transaction_id) as flag
97+
FROM bronze.sales_transactions
98+
) t
99+
WHERE flag > 1 ;
100+
101+
-- inspect duplicate transaction_id records
102+
SELECT *
103+
FROM bronze.sales_transactions
104+
WHERE transaction_id IN (
105+
SELECT transaction_id
106+
FROM bronze.sales_transactions
107+
GROUP BY transaction_id
108+
HAVING COUNT(*) > 1
109+
)
110+
ORDER BY transaction_id, order_line_number;
111+
112+
-- final cleaning and standardization transaction_id
113+
WITH clean_transaction_id AS
114+
(
115+
SELECT
116+
*
117+
FROM
118+
(
119+
SELECT
120+
transaction_id,
121+
ROW_NUMBER() OVER(PARTITION BY transaction_id ORDER BY transaction_id) as flag
122+
FROM bronze.sales_transactions
123+
) t
124+
WHERE flag = 1
125+
)
126+
SELECT
127+
*
128+
FROM clean_transaction_id ;
129+
130+
--=============================================================================================
131+
--================================= order_id cleaning overview ================================
132+
--=============================================================================================
133+
-- order_id data overview
134+
SELECT
135+
order_id
136+
FROM bronze.sales_transactions ;
137+
138+
-- order_id data profiling
139+
SELECT
140+
order_id
141+
FROM bronze.sales_transactions
142+
WHERE order_id IS NULL
143+
OR TRY_CONVERT(INT, order_id) IS NULL
144+
OR LEN(order_id) < 5 ;
145+
146+
-- duplicate check in order id
147+
SELECT
148+
*
149+
FROM
150+
(
151+
SELECT
152+
order_id ,
153+
ROW_NUMBER() OVER(PARTITION BY order_id ORDER BY order_id) as flag
154+
FROM bronze.sales_transactions
155+
)t WHERE flag > 1
156+
ORDER BY flag DESC ;
157+
158+
-- final cleaning and standardization order_id
159+
SELECT
160+
CASE
161+
WHEN TRY_CONVERT(INT, order_id) IS NULL OR LEN(order_id) < 5 THEN NULL
162+
ELSE TRY_CONVERT(INT, order_id)
163+
END order_id
164+
FROM bronze.sales_transactions ;
165+
166+
--#############################################################################################
167+
--#################################### TRANSACTION CLEAN DATA #################################
168+
--#############################################################################################
169+
SELECT
170+
[transaction_id]
171+
,[order_id]
172+
,[order_line_number]
173+
,[order_date]
174+
,[order_year]
175+
,[order_month]
176+
,[order_month_name]
177+
,[order_quarter]
178+
,[order_day_of_week]
179+
,[ship_date]
180+
,[delivery_date]
181+
,[customer_id]
182+
,[customer_full_name]
183+
,[customer_first_name]
184+
,[customer_last_name]
185+
,[customer_email]
186+
,[customer_phone]
187+
,[customer_city]
188+
,[customer_state]
189+
,[customer_zip]
190+
,[customer_region]
191+
,[customer_segment]
192+
,[customer_gender]
193+
,[customer_age]
194+
,[customer_age_group]
195+
,[product_id]
196+
,[product_name]
197+
,[sku]
198+
,[brand]
199+
,[category]
200+
,[sub_category]
201+
,[department]
202+
,[quantity_ordered]
203+
,[unit_list_price]
204+
,[discount_pct]
205+
,[unit_selling_price]
206+
,[line_total_before_tax]
207+
,[tax_rate_pct]
208+
,[tax_amount]
209+
,[line_total_with_tax]
210+
,[store_id]
211+
,[store_name]
212+
,[store_city]
213+
,[store_state]
214+
,[store_region]
215+
,[store_type]
216+
,[employee_id]
217+
,[employee_name]
218+
,[employee_job_title]
219+
,[promo_id]
220+
,[promo_name]
221+
,[sales_channel]
222+
,[payment_method]
223+
,[shipping_method]
224+
,[order_status]
225+
,[is_returned]
226+
,[cost_price]
227+
,[gross_profit]
228+
,[data_source]
229+
,[record_created_ts]
230+
,[last_modified_ts]
231+
FROM .[bronze].[sales_transactions]
232+
233+
SELECT * from bronze.sales_transactions ;

0 commit comments

Comments
 (0)