Skip to content

Commit 543dc98

Browse files
authored
Merge pull request #1557 from Adez017/dicord
added blog on pyspark performance optimizations
2 parents 15ef044 + 21291a9 commit 543dc98

7 files changed

Lines changed: 783 additions & 22 deletions

File tree

1.17 MB
Loading
932 KB
Loading
1.08 MB
Loading
1.02 MB
Loading

blog/spark-performance-optimizations/index.md

Lines changed: 749 additions & 0 deletions
Large diffs are not rendered by default.

src/database/blogs/index.tsx

Lines changed: 34 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -89,17 +89,6 @@ const blogs: Blog[] = [
8989
category: "data engineering",
9090
tags: ["Microsoft", "Azure", "Data Engineering", "Certification"],
9191
},
92-
{
93-
id: 9,
94-
title: "How SSO Actually Works",
95-
image: "/img/blogs/sso_cover.png",
96-
description:
97-
"SSO lets you log into dozens of apps with a single set of credentials. But how does it actually work under the hood? A beginner-friendly walkthrough of the full flow — from clicking 'Sign in with Google' to getting access — step by step.",
98-
slug: "single-sign-on",
99-
authors: ["Aditya-Singh-Rathore", "sanjay-kv"],
100-
category: "security",
101-
tags: ["SSO", "Authentication", "Security", "OAuth", "OpenID Connect", "SAML"],
102-
},
10392
{
10493
id: 8,
10594
title: "Microsoft Fabric: One Platform, One Lake, Every Data Workload",
@@ -111,6 +100,17 @@ const blogs: Blog[] = [
111100
category: "data engineering",
112101
tags: ["Microsoft", "Azure", "Data Engineering", "Certification", "Fabric", "OneLake", "Data Workloads", "Unified Analytics"],
113102
},
103+
{
104+
id: 9,
105+
title: "How SSO Actually Works",
106+
image: "/img/blogs/sso_cover.png",
107+
description:
108+
"SSO lets you log into dozens of apps with a single set of credentials. But how does it actually work under the hood? A beginner-friendly walkthrough of the full flow — from clicking 'Sign in with Google' to getting access — step by step.",
109+
slug: "single-sign-on",
110+
authors: ["Aditya-Singh-Rathore", "sanjay-kv"],
111+
category: "security",
112+
tags: ["SSO", "Authentication", "Security", "OAuth", "OpenID Connect", "SAML"],
113+
},
114114
{
115115
id: 10,
116116
title: "Lakehouse vs Data Warehouse: A Comprehensive Comparison",
@@ -166,17 +166,6 @@ const blogs: Blog[] = [
166166
category: "data engineering",
167167
tags: ["Medallion Architecture", "Data Pipeline", "Data Management", "Data Quality", "Data Governance", "Scalability", "Data Engineering"],
168168
},
169-
{
170-
id: 16,
171-
title: "Why We Rolled Back Our Kafka Pipeline to Batch After 6 Months",
172-
image: "/img/blogs/batch-vs-stream-cover.png",
173-
description:
174-
"Streaming pipelines are powerful for real-time data processing, but they come with hidden costs that are often overlooked. These costs include increased complexity, higher resource consumption, and potential challenges in maintaining data consistency and reliability. This article explores these hidden costs and provides insights on how to mitigate them.",
175-
slug: "batch-vs-stream-processing",
176-
authors: ["Aditya-Singh-Rathore"],
177-
category: "data engineering",
178-
tags: ["Streaming Pipelines", "Real-Time Data Processing", "Data Consistency", "Data Reliability", "Resource Consumption", "Complexity", "Data Engineering"],
179-
},
180169
{
181170
id: 15,
182171
title: "Azure Synapse Analytics: When to Use It (And When to Choose Fabric Instead)",
@@ -188,6 +177,18 @@ const blogs: Blog[] = [
188177
category: "data engineering",
189178
tags: ["Azure", "Synapse Analytics", "Data Warehousing", "Big Data", "Unified Analytics", "Fabric", "Data Engineering"],
190179
},
180+
{
181+
id: 16,
182+
title: "Why We Rolled Back Our Kafka Pipeline to Batch After 6 Months",
183+
image: "/img/blogs/batch-vs-stream-cover.png",
184+
description:
185+
"Streaming pipelines are powerful for real-time data processing, but they come with hidden costs that are often overlooked. These costs include increased complexity, higher resource consumption, and potential challenges in maintaining data consistency and reliability. This article explores these hidden costs and provides insights on how to mitigate them.",
186+
slug: "batch-vs-stream-processing",
187+
authors: ["Aditya-Singh-Rathore"],
188+
category: "data engineering",
189+
tags: ["Streaming Pipelines", "Real-Time Data Processing", "Data Consistency", "Data Reliability", "Resource Consumption", "Complexity", "Data Engineering"],
190+
},
191+
191192
{
192193
id: 17,
193194
title: "Azure Data Pipeline Cost Optimization: How We Cut a $4,200 Bill by 73%",
@@ -199,6 +200,17 @@ const blogs: Blog[] = [
199200
category: "data engineering",
200201
tags: ["Azure", "Data Pipeline", "Cost Optimization", "Data Engineering"],
201202
},
203+
{
204+
id: 18,
205+
title: "PySpark Optimization Techniques: 6 Mistakes That Slow Down Every Beginner's Pipeline",
206+
image: "/img/blogs/pyspark_optimization_cover.png",
207+
description:
208+
"PySpark is a powerful tool for big data processing, but it can be challenging to optimize for performance. In this article, we discuss six common mistakes that beginners make when optimizing their PySpark pipelines, which can lead to slow performance and increased costs. We provide practical tips and techniques to help you avoid these pitfalls and improve the efficiency of your PySpark applications.",
209+
slug: "spark-performance-optimizations",
210+
authors: ["Aditya-Singh-Rathore"],
211+
category: "data engineering",
212+
tags: ["PySpark", "Optimization", "Big Data", "Performance", "Data Engineering"],
213+
},
202214

203215
];
204216

1.11 MB
Loading

0 commit comments

Comments
 (0)