KeGG-KnowlEdgeGraphGrokker/masterclasses/airflow_mastery_guide.html at main · hagopjay/KeGG-KnowlEdgeGraphGrokker · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Airflow Mastery — Production Field Guide</title>
<link rel="preconnect" href="https://fonts.googleapis.com">
<link href="https://fonts.googleapis.com/css2?family=Barlow+Condensed:wght@400;600;700;800&family=JetBrains+Mono:wght@400;500;700&family=Barlow:wght@300;400;500&display=swap" rel="stylesheet">
<style>
  :root {
    --bg:       #080c0a;
    --bg1:      #0d1210;
    --bg2:      #111a14;
    --bg3:      #162019;
    --surface:  #1c2820;
    --border:   #1f3028;
    --border2:  #274036;
    --green:    #39ff8a;
    --green2:   #20c060;
    --green3:   #0d7a3e;
    --green4:   #093d20;
    --amber:    #ffb830;
    --amber2:   #cc8a00;
    --red:      #ff4f4f;
    --red2:     #c02020;
    --cyan:     #30e8d0;
    --purple:   #b070ff;
    --blue:     #4090ff;
    --text:     #c8ddd0;
    --text2:    #7aaa8a;
    --text3:    #4a7060;
    --dim:      #2a4030;
  }
  *, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
  html { scroll-behavior: smooth; }
  body {
    background: var(--bg);
    color: var(--text);
    font-family: 'Barlow', sans-serif;
    font-size: 15px;
    line-height: 1.6;
    min-height: 100vh;
  }

  /* ── SCANLINE OVERLAY ─────────────────────────────────────── */
  body::before {
    content: '';
    position: fixed; inset: 0; z-index: 9999; pointer-events: none;
    background: repeating-linear-gradient(0deg, transparent, transparent 2px, rgba(0,0,0,0.03) 2px, rgba(0,0,0,0.03) 4px);
  }

  /* ── HEADER ──────────────────────────────────────────────── */
  header {
    border-bottom: 1px solid var(--border2);
    padding: 28px 48px 22px;
    position: sticky; top: 0; z-index: 100;
    background: rgba(8,12,10,0.96);
    backdrop-filter: blur(8px);
    display: flex; align-items: flex-end; justify-content: space-between;
  }
  .logo-area { display: flex; align-items: flex-end; gap: 16px; }
  .logo-glyph {
    font-family: 'Barlow Condensed', sans-serif;
    font-size: 52px; font-weight: 800; line-height: 1;
    color: var(--green);
    letter-spacing: -2px;
    text-shadow: 0 0 40px rgba(57,255,138,0.25);
  }
  .logo-sub {
    font-family: 'Barlow Condensed', sans-serif;
    font-size: 13px; font-weight: 600; letter-spacing: 4px;
    color: var(--text3); text-transform: uppercase;
    padding-bottom: 6px;
  }
  .header-meta {
    text-align: right;
    font-family: 'JetBrains Mono', monospace;
    font-size: 11px;
    color: var(--text3);
    line-height: 1.8;
  }
  .status-live {
    display: inline-flex; align-items: center; gap: 6px;
    color: var(--green2);
  }
  .pulse { width: 7px; height: 7px; border-radius: 50%; background: var(--green); animation: pulse 1.8s infinite; }
  @keyframes pulse { 0%,100%{opacity:1;box-shadow:0 0 0 0 rgba(57,255,138,0.4)} 50%{opacity:.7;box-shadow:0 0 0 5px rgba(57,255,138,0)} }

  /* ── TABS ────────────────────────────────────────────────── */
  .tab-bar {
    display: flex; gap: 0;
    background: var(--bg1);
    border-bottom: 1px solid var(--border2);
    padding: 0 48px;
    overflow-x: auto;
    position: sticky; top: 95px; z-index: 99;
  }
  .tab-bar::-webkit-scrollbar { height: 3px; }
  .tab-bar::-webkit-scrollbar-track { background: transparent; }
  .tab-bar::-webkit-scrollbar-thumb { background: var(--border2); }
  .tab-btn {
    font-family: 'Barlow Condensed', sans-serif;
    font-size: 13px; font-weight: 600; letter-spacing: 1.5px;
    text-transform: uppercase;
    color: var(--text3);
    background: none; border: none; cursor: pointer;
    padding: 14px 20px;
    border-bottom: 2px solid transparent;
    transition: all .2s; white-space: nowrap;
  }
  .tab-btn:hover { color: var(--text); }
  .tab-btn.active { color: var(--green); border-bottom-color: var(--green); }

  /* ── MAIN LAYOUT ─────────────────────────────────────────── */
  main { max-width: 1280px; margin: 0 auto; padding: 0 48px 80px; }
  .tab-content { display: none; padding-top: 48px; }
  .tab-content.active { display: block; }

  /* ── SECTION HEADERS ─────────────────────────────────────── */
  .sect-label {
    font-family: 'JetBrains Mono', monospace;
    font-size: 10px; font-weight: 700; letter-spacing: 4px;
    color: var(--green3); text-transform: uppercase;
    margin-bottom: 8px;
  }
  h2.sect-title {
    font-family: 'Barlow Condensed', sans-serif;
    font-size: 36px; font-weight: 800; letter-spacing: -0.5px;
    color: #e8f5ee;
    line-height: 1; margin-bottom: 12px;
  }
  h3.sub-title {
    font-family: 'Barlow Condensed', sans-serif;
    font-size: 22px; font-weight: 700;
    color: var(--green2); margin: 36px 0 10px;
    display: flex; align-items: center; gap: 10px;
  }
  h3.sub-title::before {
    content: ''; display: block;
    width: 20px; height: 2px; background: var(--green);
  }
  h4 {
    font-family: 'Barlow Condensed', sans-serif;
    font-size: 17px; font-weight: 600; letter-spacing: 0.5px;
    color: var(--cyan); margin: 24px 0 8px;
  }
  p { color: var(--text2); margin-bottom: 12px; font-weight: 300; line-height: 1.75; }
  strong { color: var(--text); font-weight: 500; }
  code { font-family: 'JetBrains Mono', monospace; font-size: 12px; color: var(--green); background: var(--green4); padding: 1px 6px; border-radius: 3px; }

  /* ── CODE BLOCKS ─────────────────────────────────────────── */
  .code-block {
    background: var(--bg1);
    border: 1px solid var(--border2);
    border-left: 3px solid var(--green3);
    border-radius: 6px;
    margin: 16px 0 24px;
    overflow: hidden;
  }
  .code-header {
    display: flex; align-items: center; justify-content: space-between;
    padding: 8px 16px;
    background: var(--bg2);
    border-bottom: 1px solid var(--border);
    font-family: 'JetBrains Mono', monospace;
    font-size: 11px;
  }
  .code-lang {
    color: var(--green3); font-weight: 700; letter-spacing: 2px; text-transform: uppercase;
  }
  .code-file { color: var(--text3); }
  pre {
    padding: 20px 24px;
    overflow-x: auto;
    font-family: 'JetBrains Mono', monospace;
    font-size: 12.5px;
    line-height: 1.75;
    color: var(--text);
  }
  pre::-webkit-scrollbar { height: 4px; }
  pre::-webkit-scrollbar-track { background: var(--bg1); }
  pre::-webkit-scrollbar-thumb { background: var(--border2); border-radius: 2px; }

  /* Syntax highlight classes */
  .k  { color: var(--purple); }   /* keyword */
  .s  { color: var(--amber); }    /* string */
  .c  { color: var(--text3); font-style: italic; } /* comment */
  .f  { color: var(--cyan); }     /* function / method */
  .n  { color: var(--green); }    /* number / constant */
  .d  { color: #88ccff; }         /* decorator */
  .t  { color: #ff8888; }         /* type */
  .bi { color: var(--blue); }     /* builtin */

  /* ── CALLOUT BOXES ───────────────────────────────────────── */
  .callout {
    padding: 16px 20px;
    border-radius: 6px;
    margin: 16px 0;
    border-left: 3px solid;
    font-size: 14px;
  }
  .callout.warn  { background: rgba(255,184,48,0.06); border-color: var(--amber); color: #d4a020; }
  .callout.info  { background: rgba(48,232,208,0.06); border-color: var(--cyan);  color: var(--cyan); }
  .callout.crit  { background: rgba(255,79,79,0.06);  border-color: var(--red);   color: var(--red); }
  .callout.tip   { background: rgba(57,255,138,0.05); border-color: var(--green3);color: var(--green2); }
  .callout strong { color: inherit; font-weight: 600; }

  /* ── GRID / CARDS ────────────────────────────────────────── */
  .card-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(280px, 1fr)); gap: 16px; margin: 20px 0; }
  .card {
    background: var(--bg2);
    border: 1px solid var(--border2);
    border-radius: 8px;
    padding: 20px;
  }
  .card-title {
    font-family: 'Barlow Condensed', sans-serif;
    font-size: 18px; font-weight: 700;
    color: var(--text); margin-bottom: 8px;
  }
  .card p { font-size: 13px; margin-bottom: 0; }
  .tag {
    display: inline-block;
    font-family: 'JetBrains Mono', monospace;
    font-size: 10px; font-weight: 700; letter-spacing: 1.5px;
    text-transform: uppercase; padding: 2px 8px; border-radius: 3px; margin-bottom: 10px;
  }
  .tag.green { background: var(--green4); color: var(--green); border: 1px solid var(--green3); }
  .tag.amber { background: rgba(255,184,48,0.1); color: var(--amber); border: 1px solid var(--amber2); }
  .tag.red   { background: rgba(255,79,79,0.1);  color: var(--red);   border: 1px solid var(--red2); }
  .tag.cyan  { background: rgba(48,232,208,0.08); color: var(--cyan);  border: 1px solid rgba(48,232,208,0.3); }
  .tag.purple{ background: rgba(176,112,255,0.1); color: var(--purple);border: 1px solid rgba(176,112,255,0.3); }

  /* ── METRIC ROW ──────────────────────────────────────────── */
  .metric-row { display: flex; gap: 12px; margin: 20px 0; flex-wrap: wrap; }
  .metric {
    flex: 1; min-width: 120px;
    background: var(--bg2); border: 1px solid var(--border2); border-radius: 6px;
    padding: 14px 16px;
  }
  .metric-val {
    font-family: 'Barlow Condensed', sans-serif;
    font-size: 32px; font-weight: 800;
    line-height: 1;
  }
  .metric-label { font-size: 11px; color: var(--text3); margin-top: 4px; text-transform: uppercase; letter-spacing: 1.5px; }

  /* ── TABLE ───────────────────────────────────────────────── */
  .data-table { width: 100%; border-collapse: collapse; margin: 16px 0; font-size: 13px; }
  .data-table th {
    background: var(--bg2); color: var(--text3);
    font-family: 'JetBrains Mono', monospace; font-size: 10px; font-weight: 700;
    letter-spacing: 2px; text-transform: uppercase;
    padding: 10px 14px; text-align: left; border-bottom: 1px solid var(--border2);
  }
  .data-table td { padding: 10px 14px; border-bottom: 1px solid var(--border); color: var(--text2); }
  .data-table tr:hover td { background: var(--bg2); }
  .data-table td:first-child { font-family: 'JetBrains Mono', monospace; color: var(--green2); font-size: 12px; }

  /* ── DIVIDER ─────────────────────────────────────────────── */
  .divider { height: 1px; background: var(--border); margin: 40px 0; }

  /* ── ARCH DIAGRAM ────────────────────────────────────────── */
  .arch-box {
    display: flex; gap: 8px; margin: 20px 0; align-items: center; flex-wrap: wrap;
  }
  .arch-node {
    background: var(--surface); border: 1px solid var(--border2); border-radius: 6px;
    padding: 12px 18px; font-family: 'Barlow Condensed', sans-serif;
    font-size: 15px; font-weight: 700; text-align: center;
  }
  .arch-arrow { color: var(--green3); font-size: 20px; }
  .arch-node.highlight { border-color: var(--green); color: var(--green); }
  .arch-node.warn-node { border-color: var(--amber); color: var(--amber); }
  .arch-node.crit-node { border-color: var(--red); color: var(--red); }

  /* ── CHECKLIST ───────────────────────────────────────────── */
  .checklist { list-style: none; margin: 12px 0; }
  .checklist li {
    padding: 8px 0 8px 28px;
    border-bottom: 1px solid var(--border);
    position: relative; font-size: 13.5px; color: var(--text2);
  }
  .checklist li::before {
    content: '▸'; position: absolute; left: 0;
    color: var(--green3); font-size: 12px;
  }
  .checklist li strong { color: var(--text); }

  /* ── FOOTER ──────────────────────────────────────────────── */
  footer {
    margin-top: 80px; padding: 32px 48px;
    border-top: 1px solid var(--border);
    display: flex; justify-content: space-between; align-items: center;
    font-family: 'JetBrains Mono', monospace; font-size: 11px; color: var(--text3);
  }
</style>
</head>
<body>

<header>
  <div class="logo-area">
    <div class="logo-glyph">AF</div>
    <div>
      <div style="font-family:'Barlow Condensed',sans-serif;font-size:24px;font-weight:800;color:#e8f5ee;letter-spacing:-0.5px;">AIRFLOW MASTERY</div>
      <div class="logo-sub">Production Field Guide — v2.x / v3.x</div>
    </div>
  </div>
  <div class="header-meta">
    <div class="status-live"><span class="pulse"></span> System Operational</div>
    <div>Scheduler · Workers · Webserver · Triggerer</div>
    <div style="color:var(--text3)">Python 3.11+ · Celery / K8s Executors</div>
  </div>
</header>

<div class="tab-bar">
  <button class="tab-btn active" onclick="switchTab('arch')">Architecture</button>
  <button class="tab-btn" onclick="switchTab('dags')">DAG Design</button>
  <button class="tab-btn" onclick="switchTab('operators')">Operators</button>
  <button class="tab-btn" onclick="switchTab('monitoring')">Monitoring</button>
  <button class="tab-btn" onclick="switchTab('debugging')">Debugging</button>
  <button class="tab-btn" onclick="switchTab('performance')">Performance</button>
  <button class="tab-btn" onclick="switchTab('production')">Production</button>
  <button class="tab-btn" onclick="switchTab('recipes')">Recipes</button>
</div>

<main>

<!-- ════════════════════════════════════════════════════════════
     TAB: ARCHITECTURE
═══════════════════════════════════════════════════════════════ -->
<section class="tab-content active" id="arch">
  <div class="sect-label">// 01 — Core Concepts</div>
  <h2 class="sect-title">Architecture Deep Dive</h2>
  <p>Airflow is a <strong>workflow orchestration platform</strong> — not an ETL tool, not a data pipeline tool. It schedules and monitors arbitrary directed acyclic graphs of tasks. Understanding its internal mechanics is non-negotiable for production reliability.</p>

  <div class="arch-box">
    <div class="arch-node highlight">DAG Files<br><small style="font-size:11px;color:var(--text3)">~/dags/</small></div>
    <div class="arch-arrow">→</div>
    <div class="arch-node warn-node">Scheduler<br><small style="font-size:11px;color:var(--text3)">DagBag parse</small></div>
    <div class="arch-arrow">→</div>
    <div class="arch-node">Metastore DB<br><small style="font-size:11px;color:var(--text3)">Postgres / MySQL</small></div>
    <div class="arch-arrow">→</div>
    <div class="arch-node">Executor<br><small style="font-size:11px;color:var(--text3)">Celery / K8s</small></div>
    <div class="arch-arrow">→</div>
    <div class="arch-node highlight">Workers<br><small style="font-size:11px;color:var(--text3)">run tasks</small></div>
  </div>

  <h3 class="sub-title">Component Breakdown</h3>

  <div class="card-grid">
    <div class="card">
      <div class="tag green">Scheduler</div>
      <div class="card-title">The Brain</div>
      <p>Continuously scans DAG files. Evaluates schedule intervals, triggers DAG runs, creates TaskInstances, and queues them to the executor. The <strong>heartbeat loop</strong> runs every ~5s. In Airflow 2+, multiple schedulers can run simultaneously (HA mode) using row-level DB locking.</p>
    </div>
    <div class="card">
      <div class="tag amber">Executor</div>
      <div class="card-title">The Dispatcher</div>
      <p>Determines <em>how</em> tasks run. <strong>LocalExecutor</strong> (subprocess), <strong>CeleryExecutor</strong> (distributed queue via Redis/RabbitMQ), <strong>KubernetesExecutor</strong> (pod-per-task), <strong>CeleryKubernetesExecutor</strong> (hybrid). Never a bottleneck itself — tasks run in workers, not the scheduler.</p>
    </div>
    <div class="card">
      <div class="tag red">Metastore DB</div>
      <div class="card-title">The Source of Truth</div>
      <p>All state lives here. DAG runs, task instances, connections, variables, XCOM values, logs (optionally). Use <strong>Postgres 13+</strong> in production. Never use SQLite beyond local dev. Connection pool sizing is critical — each scheduler + webserver + worker has its own pool.</p>
    </div>
    <div class="card">
      <div class="tag cyan">Triggerer</div>
      <div class="card-title">Async Deferral (2.2+)</div>
      <p>Runs <strong>asyncio event loop</strong> for deferred tasks. Enables sensors and operators to yield control back to the executor while waiting (I/O bound waits), dramatically reducing worker slot consumption. Critical for any workflow with polling sensors.</p>
    </div>
  </div>

  <h3 class="sub-title">The DagBag Parse Cycle — Critical to Understand</h3>
  <p>The scheduler parses every file in <code>dags_folder</code> every <code>dag_dir_list_interval</code> seconds (default 300s). Each parse is a full Python import of the module. This is why <strong>top-level code in DAG files is catastrophic</strong> — it runs on every parse, not just on execution.</p>

  <div class="callout crit">
    <strong>NEVER do this at module level:</strong> database connections, API calls, file I/O, heavy imports, environment variable fetches that call external systems. The scheduler will call this code hundreds of times per hour across all its workers.
  </div>

  <div class="code-block">
    <div class="code-header"><span class="code-lang">python</span><span class="code-file">dag_parse_pitfalls.py</span></div>
    <pre><span class="c"># ✗ BAD: top-level import of a slow library — runs on EVERY parse</span>
<span class="k">import</span> tensorflow <span class="k">as</span> tf          <span class="c"># 3s import. Scheduler grinds to halt.</span>
records = <span class="f">db.fetch_all</span>()         <span class="c"># ✗ live DB call on every parse!</span>
config = <span class="f">requests.get</span>(<span class="s">'http://config-svc'</span>).json()  <span class="c"># ✗ network call!</span>

<span class="c"># ✓ GOOD: defer everything into callables</span>
<span class="k">from</span> airflow.decorators <span class="k">import</span> dag, task
<span class="k">from</span> airflow.models <span class="k">import</span> Variable
<span class="k">from</span> datetime <span class="k">import</span> datetime
<span class="k">import</span> pendulum

<span class="d">@dag</span>(
    dag_id=<span class="s">'safe_dag'</span>,
    schedule=<span class="s">'@daily'</span>,
    start_date=pendulum.datetime(<span class="n">2024</span>, <span class="n">1</span>, <span class="n">1</span>, tz=<span class="s">'UTC'</span>),
    catchup=<span class="k">False</span>,
    tags=[<span class="s">'production'</span>],
    default_args={
        <span class="s">'owner'</span>: <span class="s">'data-eng'</span>,
        <span class="s">'retries'</span>: <span class="n">3</span>,
        <span class="s">'retry_delay'</span>: pendulum.duration(minutes=<span class="n">5</span>),
        <span class="s">'retry_exponential_backoff'</span>: <span class="k">True</span>,
        <span class="s">'max_retry_delay'</span>: pendulum.duration(hours=<span class="n">2</span>),
    }
)
<span class="k">def</span> <span class="f">safe_dag</span>():
    <span class="d">@task</span>
    <span class="k">def</span> <span class="f">load_config</span>():
        <span class="c"># ✓ Variable.get uses cache — still prefer Secrets backend</span>
        config_json = Variable.get(<span class="s">'pipeline_config'</span>, deserialize_json=<span class="k">True</span>)
        <span class="k">return</span> config_json

    <span class="d">@task</span>
    <span class="k">def</span> <span class="f">fetch_records</span>(config: dict):
        <span class="c"># ✓ DB connection is created inside the task — at execution time</span>
        <span class="k">from</span> airflow.providers.postgres.hooks.postgres <span class="k">import</span> PostgresHook
        hook = <span class="f">PostgresHook</span>(postgres_conn_id=<span class="s">'prod_postgres'</span>)
        <span class="k">return</span> hook.<span class="f">get_records</span>(<span class="s">"SELECT * FROM events LIMIT 1000"</span>)

    cfg = <span class="f">load_config</span>()
    <span class="f">fetch_records</span>(cfg)

safe_dag_instance = <span class="f">safe_dag</span>()</pre>
  </div>

  <h3 class="sub-title">Executor Comparison Matrix</h3>
  <table class="data-table">
    <thead><tr><th>Executor</th><th>Concurrency</th><th>Isolation</th><th>Overhead</th><th>Best For</th></tr></thead>
    <tbody>
      <tr><td>SequentialExecutor</td><td>1 task at a time</td><td>None</td><td>Zero</td><td>Dev/debug only</td></tr>
      <tr><td>LocalExecutor</td><td>N subprocesses</td><td>Process</td><td>Low</td><td>Small teams, single node</td></tr>
      <tr><td>CeleryExecutor</td><td>Unlimited workers</td><td>Process</td><td>Medium</td><td>Production, horizontal scale</td></tr>
      <tr><td>KubernetesExecutor</td><td>Unlimited pods</td><td>Container</td><td>High (pod start)</td><td>True isolation, variable workloads</td></tr>
      <tr><td>CeleryK8sExecutor</td><td>Both modes</td><td>Mixed</td><td>Medium-High</td><td>Hybrid: fast small + isolated large</td></tr>
    </tbody>
  </table>
</section>

<!-- ════════════════════════════════════════════════════════════
     TAB: DAG DESIGN
═══════════════════════════════════════════════════════════════ -->
<section class="tab-content" id="dags">
  <div class="sect-label">// 02 — Patterns & Anti-Patterns</div>
  <h2 class="sect-title">DAG Design Mastery</h2>

  <h3 class="sub-title">Dynamic DAG Generation — The Right Way</h3>
  <p>Dynamic DAGs are one of Airflow's most powerful features and also the most commonly abused. The key insight: <strong>generate structure at parse time, not execution time</strong>.</p>

  <div class="code-block">
    <div class="code-header"><span class="code-lang">python</span><span class="code-file">dynamic_dags.py</span></div>
    <pre><span class="c">"""
Pattern 1: Config-driven DAG factory
Generate N DAGs from a single config — one DAG per tenant/environment.
"""</span>
<span class="k">from</span> airflow.decorators <span class="k">import</span> dag, task
<span class="k">from</span> airflow.operators.empty <span class="k">import</span> EmptyOperator
<span class="k">import</span> pendulum
<span class="k">import</span> yaml
<span class="k">from</span> pathlib <span class="k">import</span> Path

<span class="c"># ✓ Read config at parse time — fast file I/O is acceptable</span>
CONFIG_PATH = Path(<span class="s">"/opt/airflow/config/pipelines.yaml"</span>)
PIPELINE_CONFIGS: list[dict] = yaml.<span class="f">safe_load</span>(CONFIG_PATH.<span class="f">read_text</span>())

<span class="k">for</span> pipeline_cfg <span class="k">in</span> PIPELINE_CONFIGS:
    tenant = pipeline_cfg[<span class="s">'tenant'</span>]
    schedule = pipeline_cfg[<span class="s">'schedule'</span>]
    tables = pipeline_cfg[<span class="s">'tables'</span>]       <span class="c"># list of tables to process</span>
    sla_minutes = pipeline_cfg.get(<span class="s">'sla_minutes'</span>, <span class="n">60</span>)

    <span class="d">@dag</span>(
        dag_id=<span class="f">f"pipeline_{tenant}"</span>,
        schedule=schedule,
        start_date=pendulum.datetime(<span class="n">2024</span>, <span class="n">1</span>, <span class="n">1</span>, tz=<span class="s">'UTC'</span>),
        catchup=<span class="k">False</span>,
        tags=[<span class="s">'auto-generated'</span>, tenant],
        default_args={
            <span class="s">'owner'</span>: pipeline_cfg.get(<span class="s">'owner'</span>, <span class="s">'platform'</span>),
            <span class="s">'retries'</span>: <span class="n">2</span>,
            <span class="s">'sla'</span>: pendulum.duration(minutes=sla_minutes),
        }
    )
    <span class="k">def</span> <span class="f">tenant_pipeline</span>(<span class="f">cfg</span>=pipeline_cfg):
        start = <span class="f">EmptyOperator</span>(task_id=<span class="s">'start'</span>)
        end   = <span class="f">EmptyOperator</span>(task_id=<span class="s">'end'</span>, trigger_rule=<span class="s">'none_failed_min_one_success'</span>)

        <span class="d">@task</span>(task_id=<span class="s">'validate_source'</span>)
        <span class="k">def</span> <span class="f">validate_source</span>():
            <span class="c"># runs validation query against source system</span>
            <span class="k">pass</span>

        table_tasks = []
        <span class="k">for</span> table <span class="k">in</span> cfg[<span class="s">'tables'</span>]:
            <span class="d">@task</span>(task_id=<span class="f">f"process_{table}"</span>, retries=<span class="n">3</span>)
            <span class="k">def</span> <span class="f">process_table</span>(table_name=table):
                <span class="k">from</span> airflow.providers.google.cloud.hooks.bigquery <span class="k">import</span> BigQueryHook
                hook = <span class="f">BigQueryHook</span>(gcp_conn_id=<span class="s">'gcp_prod'</span>)
                <span class="c"># Execute table-specific transformation</span>
                hook.<span class="f">run_query</span>(<span class="f">f"""
                    INSERT INTO <span class="s">`prod.{cfg['tenant']}.{table_name}`</span>
                    SELECT * FROM <span class="s">`staging.{table_name}`</span>
                    WHERE _partitiontime = '{{ ds }}'
                """</span>)
            table_tasks.<span class="f">append</span>(<span class="f">process_table</span>())

        start >> <span class="f">validate_source</span>() >> table_tasks >> end

    <span class="c"># ✓ Inject into globals — this is the canonical way to expose dynamic DAGs</span>
    globals()[<span class="f">f"pipeline_{tenant}"</span>] = <span class="f">tenant_pipeline</span>()</pre>
  </div>

  <div class="code-block">
    <div class="code-header"><span class="code-lang">python</span><span class="code-file">taskflow_advanced.py</span></div>
    <pre><span class="c">"""
Pattern 2: TaskFlow API with branching, XCom, and dynamic task mapping.
Dynamic task mapping (Airflow 2.3+) is a game-changer — fan-out
without knowing N at parse time.
"""</span>
<span class="k">from</span> airflow.decorators <span class="k">import</span> dag, task, task_group
<span class="k">from</span> airflow.operators.python <span class="k">import</span> BranchPythonOperator
<span class="k">from</span> airflow.utils.trigger_rule <span class="k">import</span> TriggerRule
<span class="k">import</span> pendulum

<span class="d">@dag</span>(
    dag_id=<span class="s">'advanced_taskflow'</span>,
    schedule=<span class="s">'0 4 * * *'</span>,
    start_date=pendulum.datetime(<span class="n">2024</span>, <span class="n">1</span>, <span class="n">1</span>, tz=<span class="s">'UTC'</span>),
    catchup=<span class="k">False</span>,
    max_active_tasks=<span class="n">16</span>,          <span class="c"># per-DAG concurrency cap</span>
    max_active_runs=<span class="n">1</span>,             <span class="c"># prevent overlapping runs</span>
)
<span class="k">def</span> <span class="f">advanced_pipeline</span>():

    <span class="d">@task</span>
    <span class="k">def</span> <span class="f">get_work_items</span>() -> list[dict]:
        <span class="s">"""Returns variable-length list — dynamic map target"""</span>
        <span class="k">from</span> airflow.providers.postgres.hooks.postgres <span class="k">import</span> PostgresHook
        hook = <span class="f">PostgresHook</span>(<span class="s">'prod'</span>)
        rows = hook.<span class="f">get_records</span>(<span class="s">"SELECT id, payload FROM work_queue WHERE status='pending'"</span>)
        <span class="k">return</span> [{<span class="s">'id'</span>: r[<span class="n">0</span>], <span class="s">'payload'</span>: r[<span class="n">1</span>]} <span class="k">for</span> r <span class="k">in</span> rows]

    <span class="c"># ✓ Dynamic task mapping — creates N parallel tasks at runtime</span>
    <span class="d">@task</span>(
        max_active_tis_per_dag=<span class="n">8</span>,    <span class="c"># throttle parallelism</span>
        retries=<span class="n">2</span>,
    )
    <span class="k">def</span> <span class="f">process_item</span>(item: dict) -> dict:
        <span class="s">"""Runs once per item in work_items list"""</span>
        result = <span class="f">heavy_computation</span>(item[<span class="s">'payload'</span>])
        <span class="k">return</span> {<span class="s">'id'</span>: item[<span class="s">'id'</span>], <span class="s">'status'</span>: <span class="s">'ok'</span>, <span class="s">'result'</span>: result}

    <span class="d">@task</span>(trigger_rule=TriggerRule.<span class="t">ALL_DONE</span>)   <span class="c"># run even if some mapped tasks fail</span>
    <span class="k">def</span> <span class="f">aggregate_results</span>(results: list[dict]) -> dict:
        ok = [r <span class="k">for</span> r <span class="k">in</span> results <span class="k">if</span> r <span class="k">and</span> r.get(<span class="s">'status'</span>) == <span class="s">'ok'</span>]
        <span class="k">return</span> {<span class="s">'total'</span>: <span class="f">len</span>(results), <span class="s">'ok'</span>: <span class="f">len</span>(ok), <span class="s">'failed'</span>: <span class="f">len</span>(results) - <span class="f">len</span>(ok)}

    <span class="d">@task_group</span>(group_id=<span class="s">'notifications'</span>)
    <span class="k">def</span> <span class="f">notify</span>(summary: dict):
        <span class="d">@task</span>
        <span class="k">def</span> <span class="f">send_slack</span>(s=summary):
            <span class="k">from</span> airflow.providers.slack.operators.slack_webhook <span class="k">import</span> SlackWebhookOperator
            <span class="k">if</span> s[<span class="s">'failed'</span>] > <span class="n">0</span>:
                msg = <span class="f">f"⚠️ Pipeline complete: {s['ok']}/{s['total']} ok, {s['failed']} failed"</span>
            <span class="k">else</span>:
                msg = <span class="f">f"✅ Pipeline complete: all {s['total']} items processed"</span>
            <span class="f">SlackWebhookOperator</span>(task_id=<span class="s">'_inner'</span>, slack_webhook_conn_id=<span class="s">'slack_ops'</span>, message=msg).<span class="f">execute</span>({})

        <span class="d">@task</span>
        <span class="k">def</span> <span class="f">update_dashboard</span>(s=summary):
            <span class="c"># POST metrics to internal observability service</span>
            <span class="k">import</span> httpx
            httpx.<span class="f">post</span>(<span class="s">'https://metrics.internal/ingest'</span>, json=s, timeout=<span class="n">10</span>)

        <span class="f">send_slack</span>()
        <span class="f">update_dashboard</span>()

    items = <span class="f">get_work_items</span>()
    <span class="c"># .expand() is the magic — one call generates N mapped task instances</span>
    processed = process_item.<span class="f">expand</span>(item=items)
    summary = <span class="f">aggregate_results</span>(processed)
    <span class="f">notify</span>(summary)

<span class="f">advanced_pipeline</span>()</pre>
  </div>

  <h3 class="sub-title">Trigger Rules — The Full Picture</h3>
  <table class="data-table">
    <thead><tr><th>Trigger Rule</th><th>When Task Runs</th><th>Common Use Case</th></tr></thead>
    <tbody>
      <tr><td>ALL_SUCCESS</td><td>All upstreams succeeded</td><td>Default — normal pipeline steps</td></tr>
      <tr><td>ALL_FAILED</td><td>All upstreams failed</td><td>Failure-path cleanup tasks</td></tr>
      <tr><td>ALL_DONE</td><td>All upstreams finished (any state)</td><td>Always-run summaries, notifications</td></tr>
      <tr><td>ONE_SUCCESS</td><td>At least one upstream succeeded</td><td>Fan-in where partial success is OK</td></tr>
      <tr><td>ONE_FAILED</td><td>At least one upstream failed</td><td>Early alerting before all complete</td></tr>
      <tr><td>NONE_FAILED</td><td>No upstreams failed (skips allowed)</td><td>Branching — tasks after a branch</td></tr>
      <tr><td>NONE_SKIPPED</td><td>No upstreams were skipped</td><td>Strict flows with no optional paths</td></tr>
      <tr><td>NONE_FAILED_MIN_ONE_SUCCESS</td><td>No failures + at least one success</td><td>Join after optional parallel tasks</td></tr>
    </tbody>
  </table>

  <div class="callout warn">
    <strong>Branching gotcha:</strong> When using <code>BranchPythonOperator</code>, all tasks NOT on the chosen branch receive a <code>skipped</code> state. Any downstream join task must use <code>NONE_FAILED_MIN_ONE_SUCCESS</code> or <code>NONE_FAILED</code> — otherwise the join task itself gets skipped too.
  </div>
</section>

<!-- ════════════════════════════════════════════════════════════
     TAB: OPERATORS
═══════════════════════════════════════════════════════════════ -->
<section class="tab-content" id="operators">
  <div class="sect-label">// 03 — Custom Operators & Hooks</div>
  <h2 class="sect-title">Building Production-Grade Operators</h2>

  <p>When built-in operators don't fit, write your own. A well-designed custom operator encapsulates retry logic, connection management, observability hooks, and clean error handling. Here's the full pattern.</p>

  <div class="code-block">
    <div class="code-header"><span class="code-lang">python</span><span class="code-file">operators/base_api_operator.py</span></div>
    <pre><span class="k">from</span> __future__ <span class="k">import</span> annotations
<span class="k">import</span> time
<span class="k">import</span> logging
<span class="k">from</span> typing <span class="k">import</span> Any, Sequence
<span class="k">from</span> airflow.models.baseoperator <span class="k">import</span> BaseOperator
<span class="k">from</span> airflow.utils.context <span class="k">import</span> Context
<span class="k">from</span> airflow.exceptions <span class="k">import</span> AirflowException, AirflowSkipException

log = logging.<span class="f">getLogger</span>(__name__)

<span class="k">class</span> <span class="t">RobustAPIOperator</span>(BaseOperator):
    <span class="s">"""
    Production-grade API operator with:
      - Exponential backoff with jitter
      - Circuit breaker pattern
      - OpenTelemetry span injection
      - Response schema validation
      - Graceful degradation to cache
    """</span>

    <span class="c"># Tells Airflow which template fields to render with Jinja</span>
    template_fields: Sequence[str] = (<span class="s">'endpoint'</span>, <span class="s">'payload'</span>, <span class="s">'headers'</span>)
    template_fields_renderers = {<span class="s">'payload'</span>: <span class="s">'json'</span>, <span class="s">'headers'</span>: <span class="s">'json'</span>}
    ui_color = <span class="s">'#1c7a4a'</span>     <span class="c"># custom color in Airflow UI graph view</span>
    ui_fgcolor = <span class="s">'#ffffff'</span>

    <span class="k">def</span> <span class="f">__init__</span>(
        self,
        *,
        conn_id: str,
        endpoint: str,
        method: str = <span class="s">'GET'</span>,
        payload: dict | None = <span class="k">None</span>,
        headers: dict | None = <span class="k">None</span>,
        response_check: callable | None = <span class="k">None</span>,
        max_retries: int = <span class="n">3</span>,
        backoff_factor: float = <span class="n">2.0</span>,
        timeout: int = <span class="n">30</span>,
        do_xcom_push: bool = <span class="k">True</span>,
        skip_on_empty: bool = <span class="k">False</span>,
        **kwargs,
    ):
        super().__init__(**kwargs)
        self.conn_id = conn_id
        self.endpoint = endpoint
        self.method = method.<span class="f">upper</span>()
        self.payload = payload or {}
        self.headers = headers or {}
        self.response_check = response_check
        self.max_retries = max_retries
        self.backoff_factor = backoff_factor
        self.timeout = timeout
        self.do_xcom_push = do_xcom_push
        self.skip_on_empty = skip_on_empty

    <span class="k">def</span> <span class="f">execute</span>(self, context: Context) -> Any:
        <span class="k">from</span> airflow.hooks.http_hook <span class="k">import</span> HttpHook
        <span class="k">import</span> random

        hook = <span class="f">HttpHook</span>(method=self.method, http_conn_id=self.conn_id)

        attempt = <span class="n">0</span>
        last_exc = <span class="k">None</span>

        <span class="k">while</span> attempt <= self.max_retries:
            <span class="k">try</span>:
                log.<span class="f">info</span>(<span class="s">"[%s] Attempt %d/%d → %s %s"</span>,
                         self.task_id, attempt + <span class="n">1</span>, self.max_retries + <span class="n">1</span>,
                         self.method, self.endpoint)

                response = hook.<span class="f">run</span>(
                    endpoint=self.endpoint,
                    data=self.payload,
                    headers=self.headers,
                    extra_options={<span class="s">'timeout'</span>: self.timeout, <span class="s">'verify'</span>: <span class="k">True</span>}
                )
                response.<span class="f">raise_for_status</span>()

                result = response.<span class="f">json</span>()

                <span class="c"># Optional: caller-provided validation function</span>
                <span class="k">if</span> self.response_check <span class="k">and not</span> self.response_check(result):
                    <span class="k">raise</span> <span class="t">AirflowException</span>(<span class="f">f"response_check() failed on: {result}"</span>)

                <span class="k">if</span> self.skip_on_empty <span class="k">and not</span> result:
                    log.<span class="f">info</span>(<span class="s">"Empty response with skip_on_empty=True — skipping"</span>)
                    <span class="k">raise</span> <span class="t">AirflowSkipException</span>(<span class="s">"Empty response"</span>)

                <span class="c"># Push to XCom for downstream tasks</span>
                context[<span class="s">'ti'</span>].<span class="f">xcom_push</span>(key=<span class="s">'response'</span>, value=result)
                log.<span class="f">info</span>(<span class="s">"[%s] Success on attempt %d"</span>, self.task_id, attempt + <span class="n">1</span>)
                <span class="k">return</span> result

            <span class="k">except</span> <span class="t">AirflowSkipException</span>:
                <span class="k">raise</span>
            <span class="k">except</span> <span class="t">Exception</span> <span class="k">as</span> exc:
                last_exc = exc
                attempt += <span class="n">1</span>
                <span class="k">if</span> attempt > self.max_retries:
                    <span class="k">break</span>
                <span class="c"># Exponential backoff with full jitter</span>
                sleep_time = self.backoff_factor ** attempt + random.<span class="f">uniform</span>(<span class="n">0</span>, <span class="n">1</span>)
                log.<span class="f">warning</span>(<span class="s">"[%s] Attempt %d failed (%s). Retrying in %.1fs"</span>,
                            self.task_id, attempt, exc, sleep_time)
                time.<span class="f">sleep</span>(sleep_time)

        <span class="k">raise</span> <span class="t">AirflowException</span>(
            <span class="f">f"All {self.max_retries + 1} attempts failed. Last error: {last_exc}"</span>
        ) <span class="k">from</span> last_exc

    <span class="k">def</span> <span class="f">on_kill</span>(self):
        <span class="s">"""Called when task is externally killed — clean up resources here."""</span>
        log.<span class="f">warning</span>(<span class="s">"[%s] Task killed — performing cleanup"</span>, self.task_id)
        <span class="c"># Cancel any in-flight requests, release connections, etc.</span></pre>
  </div>

  <div class="code-block">
    <div class="code-header"><span class="code-lang">python</span><span class="code-file">operators/deferred_sensor.py</span></div>
    <pre><span class="c">"""
Deferrable Sensor — uses Airflow Triggerer for async polling.
Does NOT occupy a worker slot while waiting. Scales to thousands
of concurrent sensors with minimal resource usage.
"""</span>
<span class="k">from</span> __future__ <span class="k">import</span> annotations
<span class="k">import</span> asyncio
<span class="k">from</span> datetime <span class="k">import</span> timedelta
<span class="k">from</span> typing <span class="k">import</span> Any, AsyncIterator
<span class="k">from</span> airflow.triggers.base <span class="k">import</span> BaseTrigger, TriggerEvent
<span class="k">from</span> airflow.sensors.base <span class="k">import</span> BaseSensorOperator
<span class="k">from</span> airflow.utils.context <span class="k">import</span> Context

<span class="k">class</span> <span class="t">S3KeyTrigger</span>(BaseTrigger):
    <span class="s">"""Async trigger that polls S3 for a key — runs in Triggerer process."""</span>
    <span class="k">def</span> <span class="f">__init__</span>(self, bucket: str, key: str, poll_interval: float = <span class="n">30.0</span>):
        super().__init__()
        self.bucket = bucket
        self.key = key
        self.poll_interval = poll_interval

    <span class="k">def</span> <span class="f">serialize</span>(self) -> tuple[str, dict]:
        <span class="c"># Must be serializable — stored in DB while trigger awaits</span>
        <span class="k">return</span> (
            <span class="s">"my_package.triggers.S3KeyTrigger"</span>,
            {<span class="s">"bucket"</span>: self.bucket, <span class="s">"key"</span>: self.key, <span class="s">"poll_interval"</span>: self.poll_interval}
        )

    <span class="k">async def</span> <span class="f">run</span>(self) -> AsyncIterator[TriggerEvent]:
        <span class="k">import</span> aiobotocore.session <span class="k">as</span> aioboto
        session = aioboto.<span class="f">get_session</span>()
        <span class="k">async with</span> session.<span class="f">create_client</span>(<span class="s">'s3'</span>) <span class="k">as</span> s3:
            <span class="k">while</span> <span class="k">True</span>:
                <span class="k">try</span>:
                    <span class="k">await</span> s3.<span class="f">head_object</span>(Bucket=self.bucket, Key=self.key)
                    <span class="k">yield</span> <span class="f">TriggerEvent</span>({<span class="s">"status"</span>: <span class="s">"found"</span>, <span class="s">"key"</span>: self.key})
                    <span class="k">return</span>
                <span class="k">except</span> s3.exceptions.ClientError:
                    <span class="k">await</span> asyncio.<span class="f">sleep</span>(self.poll_interval)  <span class="c"># yields control to event loop!</span>

<span class="k">class</span> <span class="t">DeferrableS3Sensor</span>(BaseSensorOperator):
    <span class="k">def</span> <span class="f">__init__</span>(self, *, bucket: str, key: str, poll_interval: float = <span class="n">30.0</span>, **kwargs):
        super().__init__(**kwargs)
        self.bucket = bucket
        self.key = key
        self.poll_interval = poll_interval

    <span class="k">def</span> <span class="f">execute</span>(self, context: Context):
        <span class="c"># Immediately defer — do NOT poll in execute()</span>
        self.<span class="f">defer</span>(
            trigger=<span class="f">S3KeyTrigger</span>(self.bucket, self.key, self.poll_interval),
            method_name=<span class="s">"execute_complete"</span>,
            timeout=timedelta(hours=<span class="n">6</span>),
        )

    <span class="k">def</span> <span class="f">execute_complete</span>(self, context: Context, event: dict) -> str:
        <span class="c"># Called by Triggerer when TriggerEvent fires — resumes in a worker slot</span>
        <span class="k">if</span> event[<span class="s">"status"</span>] != <span class="s">"found"</span>:
            <span class="k">raise</span> <span class="t">AirflowException</span>(<span class="f">f"Unexpected trigger event: {event}"</span>)
        self.log.<span class="f">info</span>(<span class="s">"S3 key found: s3://%s/%s"</span>, self.bucket, event[<span class="s">"key"</span>])
        <span class="k">return</span> event[<span class="s">"key"</span>]</pre>
  </div>
</section>

<!-- ════════════════════════════════════════════════════════════
     TAB: MONITORING
═══════════════════════════════════════════════════════════════ -->
<section class="tab-content" id="monitoring">
  <div class="sect-label">// 04 — Observability in Production</div>
  <h2 class="sect-title">Monitoring, Alerting & SLAs</h2>

  <div class="metric-row">
    <div class="metric"><div class="metric-val" style="color:var(--green)">99.2%</div><div class="metric-label">Target DAG Success Rate</div></div>
    <div class="metric"><div class="metric-val" style="color:var(--amber)">&lt;30s</div><div class="metric-label">Max Scheduler Heartbeat</div></div>
    <div class="metric"><div class="metric-val" style="color:var(--red)">0</div><div class="metric-label">Zombie Tasks Tolerated</div></div>
    <div class="metric"><div class="metric-val" style="color:var(--cyan)">&lt;60s</div><div class="metric-label">Task Queue Wait P95</div></div>
    <div class="metric"><div class="metric-val" style="color:var(--purple)">&lt;5s</div><div class="metric-label">DAG Parse Time Max</div></div>
  </div>

  <h3 class="sub-title">Callback-Based Alerting System</h3>
  <div class="code-block">
    <div class="code-header"><span class="code-lang">python</span><span class="code-file">callbacks/alerting.py</span></div>
    <pre><span class="c">"""
Centralized callback system — attach to default_args to alert on
every DAG failure, retry, SLA miss, and task success event.
"""</span>
<span class="k">from</span> __future__ <span class="k">import</span> annotations
<span class="k">import</span> json
<span class="k">import</span> logging
<span class="k">from</span> typing <span class="k">import</span> TYPE_CHECKING
<span class="k">from</span> airflow.models <span class="k">import</span> TaskInstance, DagRun
<span class="k">from</span> airflow.utils.state <span class="k">import</span> State

<span class="k">if</span> TYPE_CHECKING:
    <span class="k">from</span> airflow.utils.context <span class="k">import</span> Context

log = logging.<span class="f">getLogger</span>(__name__)

<span class="k">def</span> <span class="f">_build_slack_blocks</span>(title: str, color: str, fields: list[dict]) -> list[dict]:
    <span class="s">"""Build Slack Block Kit message."""</span>
    <span class="k">return</span> [
        {<span class="s">"type"</span>: <span class="s">"header"</span>, <span class="s">"text"</span>: {<span class="s">"type"</span>: <span class="s">"plain_text"</span>, <span class="s">"text"</span>: title}},
        {<span class="s">"type"</span>: <span class="s">"section"</span>, <span class="s">"fields"</span>: [
            {<span class="s">"type"</span>: <span class="s">"mrkdwn"</span>, <span class="s">"text"</span>: <span class="f">f"*{f['label']}*\n{f['value']}"</span>}
            <span class="k">for</span> f <span class="k">in</span> fields
        ]},
    ]

<span class="k">def</span> <span class="f">on_failure_callback</span>(context: Context) -> <span class="k">None</span>:
    <span class="s">"""Fires when a task fails. Sends structured Slack alert + Pagerduty if high priority."""</span>
    <span class="k">import</span> httpx
    <span class="k">from</span> airflow.models <span class="k">import</span> Variable

    ti: TaskInstance = context[<span class="s">'task_instance'</span>]
    dag_run: DagRun   = context[<span class="s">'dag_run'</span>]
    exception         = context.get(<span class="s">'exception'</span>)
    log_url           = ti.<span class="f">log_url</span>

    severity = ti.dag.<span class="f">tags</span> <span class="k">and</span> <span class="s">'critical'</span> <span class="k">in</span> ti.dag.tags

    blocks = <span class="f">_build_slack_blocks</span>(
        title=<span class="f">f"{'🔴 CRITICAL' if severity else '🟠 FAILURE'}: {ti.dag_id}.{ti.task_id}"</span>,
        color=<span class="s">'danger'</span>,
        fields=[
            {<span class="s">'label'</span>: <span class="s">'DAG'</span>,          <span class="s">'value'</span>: ti.dag_id},
            {<span class="s">'label'</span>: <span class="s">'Task'</span>,         <span class="s">'value'</span>: ti.task_id},
            {<span class="s">'label'</span>: <span class="s">'Run ID'</span>,       <span class="s">'value'</span>: dag_run.run_id},
            {<span class="s">'label'</span>: <span class="s">'Try #'</span>,        <span class="s">'value'</span>: str(ti.try_number)},
            {<span class="s">'label'</span>: <span class="s">'Error'</span>,        <span class="s">'value'</span>: <span class="f">f"`{str(exception)[:200]}`"</span>},
            {<span class="s">'label'</span>: <span class="s">'Logs'</span>,         <span class="s">'value'</span>: <span class="f">f"<{log_url}|View logs>"</span>},
        ]
    )

    slack_token = Variable.<span class="f">get</span>(<span class="s">'slack_bot_token'</span>)
    slack_channel = Variable.<span class="f">get</span>(<span class="s">'slack_alerts_channel'</span>, <span class="s">'#data-alerts'</span>)

    httpx.<span class="f">post</span>(
        <span class="s">'https://slack.com/api/chat.postMessage'</span>,
        headers={<span class="s">'Authorization'</span>: <span class="f">f'Bearer {slack_token}'</span>},
        json={<span class="s">'channel'</span>: slack_channel, <span class="s">'blocks'</span>: blocks},
        timeout=<span class="n">10</span>
    )

    <span class="k">if</span> severity:
        <span class="f">_trigger_pagerduty</span>(ti, exception, context)

<span class="k">def</span> <span class="f">on_retry_callback</span>(context: Context) -> <span class="k">None</span>:
    ti: TaskInstance = context[<span class="s">'task_instance'</span>]
    log.<span class="f">warning</span>(<span class="s">"RETRY: %s.%s attempt %d/%d — %s"</span>,
        ti.dag_id, ti.task_id, ti.try_number, ti.max_tries + <span class="n">1</span>,
        context.get(<span class="s">'exception'</span>)
    )
    <span class="c"># Lightweight — just log on retries, Slack only on final failure</span>

<span class="k">def</span> <span class="f">on_sla_miss_callback</span>(
    dag, task_list, blocking_task_list, slas, blocking_tis
) -> <span class="k">None</span>:
    <span class="s">"""SLA miss fires when a task hasn't completed by dag.sla relative to execution_date."""</span>
    <span class="k">import</span> httpx
    <span class="k">from</span> airflow.models <span class="k">import</span> Variable
    sla_str = <span class="s">", "</span>.<span class="f">join</span>([<span class="f">f"{s.dag_id}.{s.task_id}"</span> <span class="k">for</span> s <span class="k">in</span> slas])
    log.<span class="f">error</span>(<span class="s">"SLA MISS: %s"</span>, sla_str)
    <span class="c"># Post to ops channel — SLA misses are high priority</span>

<span class="k">def</span> <span class="f">_trigger_pagerduty</span>(ti: TaskInstance, exc, context: Context):
    <span class="k">import</span> httpx
    <span class="k">from</span> airflow.models <span class="k">import</span> Variable
    pd_key = Variable.<span class="f">get</span>(<span class="s">'pagerduty_routing_key'</span>)
    httpx.<span class="f">post</span>(<span class="s">'https://events.pagerduty.com/v2/enqueue'</span>, json={
        <span class="s">'routing_key'</span>: pd_key,
        <span class="s">'event_action'</span>: <span class="s">'trigger'</span>,
        <span class="s">'payload'</span>: {
            <span class="s">'summary'</span>: <span class="f">f"Airflow CRITICAL: {ti.dag_id}.{ti.task_id} failed"</span>,
            <span class="s">'severity'</span>: <span class="s">'critical'</span>,
            <span class="s">'source'</span>: <span class="s">'airflow'</span>,
            <span class="s">'custom_details'</span>: {<span class="s">'run_id'</span>: ti.run_id, <span class="s">'error'</span>: <span class="f">str(exc)[:500]</span>},
        }
    }, timeout=<span class="n">10</span>)

<span class="c"># ── Usage in DAG default_args ────────────────────────────────────</span>
DEFAULT_ARGS = {
    <span class="s">'owner'</span>: <span class="s">'data-engineering'</span>,
    <span class="s">'retries'</span>: <span class="n">3</span>,
    <span class="s">'on_failure_callback'</span>: on_failure_callback,
    <span class="s">'on_retry_callback'</span>: on_retry_callback,
    <span class="s">'sla'</span>: __import__(<span class="s">'pendulum'</span>).duration(hours=<span class="n">2</span>),
}</pre>
  </div>

  <h3 class="sub-title">StatsD / Prometheus Metrics</h3>
  <div class="code-block">
    <div class="code-header"><span class="code-lang">ini</span><span class="code-file">airflow.cfg — metrics config</span></div>
    <pre><span class="c"># airflow.cfg or env vars (AIRFLOW__METRICS__STATSD_*)</span>
[metrics]
statsd_on = True
statsd_host = statsd-exporter.monitoring.svc.cluster.local
statsd_port = 9125
statsd_prefix = airflow
statsd_allow_list = dag.,scheduler.,executor.,pool.
<span class="c"># statsd_allow_list filters metrics sent — reduces cardinality</span>

[scheduler]
<span class="c"># How often scheduler emits its own heartbeat metric</span>
scheduler_heartbeat_sec = 5
<span class="c"># Alert externally if this metric goes stale ></span>
<span class="c"># airflow_scheduler_heartbeat{} for 30+ seconds</span></pre>
  </div>

  <div class="code-block">
    <div class="code-header"><span class="code-lang">yaml</span><span class="code-file">prometheus/airflow_alerts.yaml</span></div>
    <pre><span class="k">groups:</span>
<span class="k">  - name:</span> airflow.critical
<span class="k">    rules:</span>

    <span class="c"># Scheduler died or stalled</span>
<span class="k">    - alert:</span> AirflowSchedulerDead
<span class="k">      expr:</span> time() - airflow_scheduler_heartbeat > <span class="n">60</span>
<span class="k">      for:</span> 2m
<span class="k">      labels:</span>
<span class="k">        severity:</span> critical
<span class="k">      annotations:</span>
<span class="k">        summary:</span> <span class="s">"Airflow scheduler heartbeat missing for {{ $value }}s"</span>

    <span class="c"># Zombie tasks accumulating</span>
<span class="k">    - alert:</span> AirflowZombieTasksHigh
<span class="k">      expr:</span> airflow_zombies_killed_total > <span class="n">5</span>
<span class="k">      for:</span> 10m
<span class="k">      labels:</span>
<span class="k">        severity:</span> warning
<span class="k">      annotations:</span>
<span class="k">        summary:</span> <span class="s">"Zombie tasks detected: {{ $value }}"</span>

    <span class="c"># Task queue depth too high (workers overwhelmed)</span>
<span class="k">    - alert:</span> AirflowTaskQueueDepth
<span class="k">      expr:</span> airflow_executor_queued_tasks > <span class="n">200</span>
<span class="k">      for:</span> 5m
<span class="k">      labels:</span>
<span class="k">        severity:</span> warning

    <span class="c"># DAG import errors present</span>
<span class="k">    - alert:</span> AirflowDagImportErrors
<span class="k">      expr:</span> airflow_dag_processing_import_errors > <span class="n">0</span>
<span class="k">      for:</span> 1m
<span class="k">      labels:</span>
<span class="k">        severity:</span> critical
<span class="k">      annotations:</span>
<span class="k">        summary:</span> <span class="s">"{{ $value }} DAG(s) failing to parse"</span>

    <span class="c"># DB connection pool saturation</span>
<span class="k">    - alert:</span> AirflowDBPoolSaturation
<span class="k">      expr:</span> airflow_pool_starving_tasks > <span class="n">0</span>
<span class="k">      for:</span> 5m
<span class="k">      labels:</span>
<span class="k">        severity:</span> warning</pre>
  </div>
</section>

<!-- ════════════════════════════════════════════════════════════
     TAB: DEBUGGING
═══════════════════════════════════════════════════════════════ -->
<section class="tab-content" id="debugging">
  <div class="sect-label">// 05 — Diagnosing Live Systems</div>
  <h2 class="sect-title">Debugging & Incident Response</h2>