11/**
2- * Backward-pass walkthroughs per brick. Each entry describes:
3- * - differentiates: what scalar function and w.r.t. which params/inputs
4- * - chain_rule: how the upstream cotangent ḡ = dL/dy propagates
5- * through the brick's local Jacobian (named tensors,
6- * explicit reverse steps, MLX `mx.grad` invocation)
7- * - key_identity: one-line equation capturing the central VJP
2+ * Backward-pass walkthroughs per brick. Framework-agnostic — the
3+ * `chain_rule` text never assumes MLX/PyTorch/JAX; concrete
4+ * reverse-mode AD invocations live in the `api` map and HelpModal
5+ * renders the row matching the active platform (MLX on Apple Silicon,
6+ * PyTorch on CUDA/CPU, JAX on TPU).
87 *
9- * Sourced from a research agent (Exa-verified canonical forms) +
10- * cross-checked against textbook derivations. Reverse-mode AD in MLX
11- * = `mx.grad(fn, argnums=[...])` returns a function that maps the
12- * primal arguments to dL/d{arg_i}. The cotangent ḡ shown below is
13- * the gradient signal arriving from layers downstream of this brick.
8+ * Fields:
9+ * - differentiates: scalar function + w.r.t. which params/inputs
10+ * - chain_rule: how ḡ = dL/dy propagates through the local Jacobian
11+ * - key_identity: one-line equation capturing the central VJP
12+ * - api: {mlx, pytorch, jax} — minimal reverse-mode AD call
13+ * producing dL/d{args} for this brick's signature
1414 */
1515
1616
1717export interface BackwardEntry {
1818 differentiates : string ;
1919 chain_rule : string ;
2020 key_identity : string ;
21+ api ?: {
22+ mlx ?: string ;
23+ pytorch ?: string ;
24+ jax ?: string ;
25+ } ;
26+ }
27+
28+
29+ /** Wrap an attention-shaped argnums list as the three canonical calls. */
30+ function attentionLike ( ) : NonNullable < BackwardEntry [ "api" ] > {
31+ return {
32+ mlx :
33+ "mx.grad(fn, argnums=[0,1,2,3,4])(X, W_q, W_k, W_v, W_o)" ,
34+ pytorch :
35+ "loss.backward() # X, W_q, W_k, W_v, W_o each get .grad set\n"
36+ + "# or: torch.autograd.grad(loss, [X, W_q, W_k, W_v, W_o])" ,
37+ jax :
38+ "jax.grad(fn, argnums=(0,1,2,3,4))(X, W_q, W_k, W_v, W_o)" ,
39+ } ;
2140}
2241
2342
@@ -32,11 +51,11 @@ export const BACKWARD_TOPICS: Record<string, BackwardEntry> = {
3251 "Then dL/dP = dL/dC · Vᵀ and dL/dV = Pᵀ · dL/dC. The softmax step " +
3352 "uses its Jacobian per row: dL/dS_ij = P_ij · (dL/dP_ij − Σ_k " +
3453 "P_ik · dL/dP_ik). Finally dL/dQ = (dL/dS · K)/√d_k and dL/dK = " +
35- "(dL/dSᵀ · Q)/√d_k, then propagate through W_{q,k,v}. In MLX: " +
36- "`mx.grad(fn, argnums=[0,1,2,3])(X, W_q, W_k, W_v, ...)`." ,
54+ "(dL/dSᵀ · Q)/√d_k, then propagate through W_{q,k,v}." ,
3755 key_identity :
3856 "dL/dS = P ⊙ (dL/dP − rowsum(P ⊙ dL/dP)·1ᵀ) " +
3957 "(softmax row-Jacobian)" ,
58+ api : attentionLike ( ) ,
4059 } ,
4160
4261 brick_gated_attention : {
@@ -51,6 +70,14 @@ export const BACKWARD_TOPICS: Record<string, BackwardEntry> = {
5170 "and dL/dW_g = Xᵀ·dL/dG; the dL/dCtx branch enters the standard " +
5271 "SDPA backward (softmax-Jacobian, dL/dQ, dL/dK, dL/dV)." ,
5372 key_identity : "∂σ(G)/∂G = σ(G) ⊙ (1 − σ(G))" ,
73+ api : {
74+ mlx :
75+ "mx.grad(fn, argnums=[0,1,2,3,4,5])(X, W_q, W_k, W_v, W_g, W_o)" ,
76+ pytorch :
77+ "loss.backward() # X, W_q, W_k, W_v, W_g, W_o each get .grad" ,
78+ jax :
79+ "jax.grad(fn, argnums=(0,1,2,3,4,5))(X, W_q, W_k, W_v, W_g, W_o)" ,
80+ } ,
5481 } ,
5582
5683 brick_mla : {
@@ -65,12 +92,20 @@ export const BACKWARD_TOPICS: Record<string, BackwardEntry> = {
6592 "dL/dQ_rot — just rotate by the negative angle. Up-projection " +
6693 "VJPs: dL/dW_uq = C_qᵀ·dL/dQ and dL/dC_q = dL/dQ·W_uqᵀ, sym " +
6794 "for K,V (sum both into dL/dC_kv). Then dL/dW_dq = Xᵀ·dL/dC_q, " +
68- "dL/dW_dkv = Xᵀ·dL/dC_kv; dL/dX accumulates from both. MLX: " +
69- "`mx.grad(mla_fn, argnums=[1,2,3,4,5])(X, W_dq, W_uq, W_dkv, " +
70- "W_uk, W_uv)`." ,
95+ "dL/dW_dkv = Xᵀ·dL/dC_kv; dL/dX accumulates from both." ,
7196 key_identity :
7297 "RoPE backward: dL/dQ = R(−θ) · dL/dQ_rot " +
7398 "(rotation transpose = inverse rotation)" ,
99+ api : {
100+ mlx :
101+ "mx.grad(mla_fn, argnums=[1,2,3,4,5])(X, W_dq, W_uq, W_dkv, " +
102+ "W_uk, W_uv)" ,
103+ pytorch :
104+ "loss.backward() # X, W_dq, W_uq, W_dkv, W_uk, W_uv each .grad" ,
105+ jax :
106+ "jax.grad(mla_fn, argnums=(1,2,3,4,5))(X, W_dq, W_uq, W_dkv, " +
107+ "W_uk, W_uv)" ,
108+ } ,
74109 } ,
75110
76111 brick_mlp : {
@@ -83,9 +118,16 @@ export const BACKWARD_TOPICS: Record<string, BackwardEntry> = {
83118 "dL/d(silu(a)) = dL/dh ⊙ b. Then dL/da = dL/d(silu(a)) ⊙ " +
84119 "silu'(a) with silu'(a) = σ(a) + a·σ(a)(1−σ(a)). Finally " +
85120 "dL/dW_gate = Xᵀ·dL/da, dL/dW_up = Xᵀ·dL/db, and dL/dX = " +
86- "dL/da·W_gateᵀ + dL/db·W_upᵀ. MLX: `mx.grad(swiglu_fn, " +
87- "argnums=[1,2,3])(X, W_gate, W_up, W_down)`." ,
121+ "dL/da·W_gateᵀ + dL/db·W_upᵀ." ,
88122 key_identity : "silu'(a) = σ(a) · (1 + a·(1 − σ(a)))" ,
123+ api : {
124+ mlx :
125+ "mx.grad(swiglu_fn, argnums=[0,1,2,3])(X, W_gate, W_up, W_down)" ,
126+ pytorch :
127+ "loss.backward() # X, W_gate, W_up, W_down each get .grad set" ,
128+ jax :
129+ "jax.grad(swiglu_fn, argnums=(0,1,2,3))(X, W_gate, W_up, W_down)" ,
130+ } ,
89131 } ,
90132
91133 brick_moe : {
@@ -107,6 +149,14 @@ export const BACKWARD_TOPICS: Record<string, BackwardEntry> = {
107149 key_identity :
108150 "STE for top-k: ∂Y/∂g_e ≈ Expert_e(X) on selected experts, " +
109151 "0 elsewhere (mask = stop-grad)" ,
152+ api : {
153+ mlx :
154+ "mx.grad(moe_fn, argnums=[0,1,2])(X, W_router, [W_E1,...,W_EN])" ,
155+ pytorch :
156+ "loss.backward() # plus loss_aux.backward() with retain_graph" ,
157+ jax :
158+ "jax.grad(moe_fn, argnums=(0,1,2))(X, W_router, experts)" ,
159+ } ,
110160 } ,
111161
112162 brick_mamba3 : {
@@ -122,12 +172,18 @@ export const BACKWARD_TOPICS: Record<string, BackwardEntry> = {
122172 "ḡ_t = dL/dy_t: run a reverse scan for adjoint states " +
123173 "s_t = C_tᵀ·ḡ_t + Ā_{t+1}ᵀ·s_{t+1}; then dL/dB_t = s_t · " +
124174 "x_tᵀ · Δ_t, dL/dC_t = ḡ_t · h_tᵀ, dL/dx_t = B̄_tᵀ·s_t, and " +
125- "dL/dΔ_t, dL/dA accumulate through exp(Δ·A). MLX: " +
126- "`mx.grad(ssm_fn, argnums=[1,2,3,4])(x, A, B, C, Δ)` — dual " +
127- "form makes this O(T log T) instead of sequential." ,
175+ "dL/dΔ_t, dL/dA accumulate through exp(Δ·A). Dual form makes " +
176+ "this O(T log T) instead of sequential." ,
128177 key_identity :
129178 "Adjoint scan: s_t = Ā_{t+1}ᵀ · s_{t+1} + C_tᵀ · ḡ_t " +
130179 "(reverse-time linear recurrence)" ,
180+ api : {
181+ mlx : "mx.grad(ssm_fn, argnums=[0,1,2,3,4])(x, A, B, C, Δ)" ,
182+ pytorch :
183+ "loss.backward() # x, A, B, C, Δ each get .grad set\n"
184+ + "# nvidia/mamba uses a custom CUDA backward kernel" ,
185+ jax : "jax.grad(ssm_fn, argnums=(0,1,2,3,4))(x, A, B, C, Δ)" ,
186+ } ,
131187 } ,
132188
133189 brick_mlstm : {
@@ -149,6 +205,13 @@ export const BACKWARD_TOPICS: Record<string, BackwardEntry> = {
149205 key_identity :
150206 "Matrix-state adjoint: Ĉ_t = f_{t+1}·Ĉ_{t+1} + q_t · " +
151207 "∂L/∂(C_t·q_t)ᵀ" ,
208+ api : {
209+ mlx :
210+ "mx.grad(mlstm_fn, argnums=[0,1,2,3,4,5,6])(x, W_i, W_f, " +
211+ "W_o, W_q, W_k, W_v)" ,
212+ pytorch : "loss.backward() # BPTT over the matrix state" ,
213+ jax : "jax.grad(mlstm_fn, argnums=tuple(range(7)))(x, *Ws)" ,
214+ } ,
152215 } ,
153216
154217 brick_gdn : {
@@ -163,13 +226,19 @@ export const BACKWARD_TOPICS: Record<string, BackwardEntry> = {
163226 "β_{t+1}·k_{t+1}·k_{t+1}ᵀ)ᵀ · Ŝ_{t+1} (reverse-time, " +
164227 "mirroring the gated identity update). Then dL/dv_t = " +
165228 "Ŝ_t·k_t·β_t, dL/dk_t combines both rank-1 update terms, " +
166- "dL/dβ_t = ⟨Ŝ_t, (v_t − S_{t-1}·k_t)·k_tᵀ⟩. Route q,k,v,β,g " +
167- "through their projections to dL/dW_*. MLX: " +
168- "`mx.grad(gdn_fn, argnums=[1,2,3,4,5])(x, W_q, W_k, W_v, " +
169- "W_β, W_g)`." ,
229+ "dL/dβ_t = ⟨Ŝ_t, (v_t − S_{t-1}·k_t)·k_tᵀ⟩." ,
170230 key_identity :
171231 "Ŝ_t = (I − β_{t+1}·k_{t+1}·k_{t+1}ᵀ)ᵀ · Ŝ_{t+1} + " +
172232 "q_t · ∂L/∂(S_t·q_t)ᵀ" ,
233+ api : {
234+ mlx :
235+ "mx.grad(gdn_fn, argnums=[0,1,2,3,4,5])(x, W_q, W_k, W_v, " +
236+ "W_β, W_g)" ,
237+ pytorch : "loss.backward() # reverse-time scan over delta updates" ,
238+ jax :
239+ "jax.grad(gdn_fn, argnums=(0,1,2,3,4,5))(x, W_q, W_k, W_v, " +
240+ "W_β, W_g)" ,
241+ } ,
173242 } ,
174243
175244 adapter_rmsnorm : {
@@ -182,13 +251,19 @@ export const BACKWARD_TOPICS: Record<string, BackwardEntry> = {
182251 "batch/seq. For dL/dx_i we need the Jacobian of x_i/rms — " +
183252 "both the direct term and the dependence of rms on every x_j. " +
184253 "Working it out: dL/dx_i = (γ_i·ḡ_i)/rms − x_i · (Σ_j " +
185- "γ_j·ḡ_j·x_j) / (d · rms³). Vectorized with u = γ ⊙ ḡ: " +
186- "dL/dx = u/rms − x · ⟨u, x⟩ / (d · rms³). The famous " +
187- "'norm cancels out' part is that after substituting y = " +
188- "γ·x/rms back, the second term becomes a clean per-feature " +
189- "subtraction — what makes ‖y‖_RMS = 1 stable under SGD." ,
254+ "γ_j·ḡ_j·x_j) / (d · rms³). The famous 'norm cancels out' part " +
255+ "is that after substituting y = γ·x/rms back, the second term " +
256+ "becomes a clean per-feature subtraction — what makes " +
257+ "‖y‖_RMS = 1 stable under SGD." ,
190258 key_identity :
191259 "dL/dx = (γ ⊙ ḡ)/rms − x · ⟨γ ⊙ ḡ, x⟩ / (d · rms³)" ,
260+ api : {
261+ mlx : "mx.grad(rmsnorm_fn, argnums=[0,1])(x, γ)" ,
262+ pytorch :
263+ "loss.backward() # x, γ each get .grad set\n"
264+ + "# torch.nn.RMSNorm fused-kernel backward (PyTorch 2.4+)" ,
265+ jax : "jax.grad(rmsnorm_fn, argnums=(0,1))(x, γ)" ,
266+ } ,
192267 } ,
193268
194269 brick_abs_pos_embed : {
@@ -205,11 +280,16 @@ export const BACKWARD_TOPICS: Record<string, BackwardEntry> = {
205280 key_identity :
206281 "dL/dx = ḡ; dL/dW_pos[t] = Σ_b ḡ[b, t, :] " +
207282 "(scatter-add by position)" ,
283+ api : {
284+ mlx : "mx.grad(embed_fn, argnums=[0,1])(x, W_pos)" ,
285+ pytorch :
286+ "loss.backward() # nn.Embedding backward = scatter_add by index" ,
287+ jax : "jax.grad(embed_fn, argnums=(0,1))(x, W_pos)" ,
288+ } ,
208289 } ,
209290
210291 adapter_residual : {
211- differentiates :
212- "L w.r.t. dL/dx; residual adds no parameters." ,
292+ differentiates : "L w.r.t. dL/dx; residual adds no parameters." ,
213293 chain_rule :
214294 "Forward: y = x + F(x). Jacobian = I + ∂F/∂x. Given ḡ = " +
215295 "dL/dy, the VJP splits and sums: dL/dx_skip = ḡ (identity " +
@@ -221,6 +301,12 @@ export const BACKWARD_TOPICS: Record<string, BackwardEntry> = {
221301 key_identity :
222302 "dL/dx = ḡ + ∂F(x)/∂xᵀ · ḡ " +
223303 "(gradient flows through both branches additively)" ,
304+ api : {
305+ mlx :
306+ "# residual is plumbing — ḡ flows through both branches in any AD" ,
307+ pytorch : "# loss.backward() — autograd handles the additive split" ,
308+ jax : "# jax.grad — additive split is fully implicit" ,
309+ } ,
224310 } ,
225311
226312 metric_perplexity : {
@@ -233,5 +319,11 @@ export const BACKWARD_TOPICS: Record<string, BackwardEntry> = {
233319 "J_ij = p_i·(δ_ij − p_j) combined with d/dp(−log p_y) = " +
234320 "−1/p_y telescopes into a clean (p − y)." ,
235321 key_identity : "softmax+CE-VJP: dL/dz = (p − y) / B" ,
322+ api : {
323+ mlx : "mx.grad(ce_loss, argnums=[0])(z, y_onehot)" ,
324+ pytorch :
325+ "loss = F.cross_entropy(z, y); loss.backward() # z.grad = (p-y)/B" ,
326+ jax : "jax.grad(ce_loss)(z, y_onehot)" ,
327+ } ,
236328 } ,
237329} ;
0 commit comments