@@ -39,8 +39,8 @@ EnzymeCore.EnzymeRules.inactive_type(::Type{NoCacheStorage}) = true
3939# `ForwardMode` matching the outer config so the delegated call inherits
4040# those flags.
4141@inline function _fwd_mode (
42- :: Val{NeedsPrimal} , :: Val{RuntimeActivity} , :: Val{StrongZero}
43- ) where {NeedsPrimal, RuntimeActivity, StrongZero}
42+ :: Val{NeedsPrimal} , :: Val{RuntimeActivity} , :: Val{StrongZero}
43+ ) where {NeedsPrimal, RuntimeActivity, StrongZero}
4444 mode = NeedsPrimal ? ForwardWithPrimal : Forward
4545 RuntimeActivity && (mode = Enzyme. set_runtime_activity (mode))
4646 StrongZero && (mode = Enzyme. set_strong_zero (mode))
6363# meaningful tangent — so the function shadow is ignored and the inner
6464# `Enzyme.autodiff` call uses `Const(f_orig)`.
6565function EnzymeRules. forward (
66- :: EnzymeRules.FwdConfig{false, true, W, RuntimeActivity, StrongZero} ,
67- func:: EnzymeCore.Annotation{<:FunctionWrappersWrapper} ,
68- RT:: Type{<:EnzymeCore.Annotation{T}} ,
69- args:: Vararg{EnzymeCore.Annotation, N}
70- ) where {T, W, N, RuntimeActivity, StrongZero}
66+ :: EnzymeRules.FwdConfig{false, true, W, RuntimeActivity, StrongZero} ,
67+ func:: EnzymeCore.Annotation{<:FunctionWrappersWrapper} ,
68+ RT:: Type{<:EnzymeCore.Annotation{T}} ,
69+ args:: Vararg{EnzymeCore.Annotation, N}
70+ ) where {T, W, N, RuntimeActivity, StrongZero}
7171 f_orig = unwrap (func. val)
7272 mode = _fwd_mode (Val (false ), Val (RuntimeActivity), Val (StrongZero))
7373 if W == 1
8686
8787# Both primal and shadow (ForwardWithPrimal mode)
8888function EnzymeRules. forward (
89- :: EnzymeRules.FwdConfig{true, true, W, RuntimeActivity, StrongZero} ,
90- func:: EnzymeCore.Annotation{<:FunctionWrappersWrapper} ,
91- RT:: Type{<:EnzymeCore.Annotation{T}} ,
92- args:: Vararg{EnzymeCore.Annotation, N}
93- ) where {T, W, N, RuntimeActivity, StrongZero}
89+ :: EnzymeRules.FwdConfig{true, true, W, RuntimeActivity, StrongZero} ,
90+ func:: EnzymeCore.Annotation{<:FunctionWrappersWrapper} ,
91+ RT:: Type{<:EnzymeCore.Annotation{T}} ,
92+ args:: Vararg{EnzymeCore.Annotation, N}
93+ ) where {T, W, N, RuntimeActivity, StrongZero}
9494 f_orig = unwrap (func. val)
9595 pargs = ntuple (i -> args[i]. val, Val (N))
9696 primal = f_orig (pargs... ):: T
@@ -113,11 +113,11 @@ end
113113
114114# Primal only (Const return type) — width-independent
115115function EnzymeRules. forward (
116- :: EnzymeRules.FwdConfig{true, false, W, RuntimeActivity, StrongZero} ,
117- func:: EnzymeCore.Annotation{<:FunctionWrappersWrapper} ,
118- RT:: Type{<:EnzymeCore.Annotation} ,
119- args:: Vararg{EnzymeCore.Annotation, N}
120- ) where {W, N, RuntimeActivity, StrongZero}
116+ :: EnzymeRules.FwdConfig{true, false, W, RuntimeActivity, StrongZero} ,
117+ func:: EnzymeCore.Annotation{<:FunctionWrappersWrapper} ,
118+ RT:: Type{<:EnzymeCore.Annotation} ,
119+ args:: Vararg{EnzymeCore.Annotation, N}
120+ ) where {W, N, RuntimeActivity, StrongZero}
121121 f_orig = unwrap (func. val)
122122 pargs = ntuple (i -> args[i]. val, Val (N))
123123 return f_orig (pargs... )
@@ -137,13 +137,13 @@ end
137137# IMPORTANT: forward the `RuntimeActivity` and `StrongZero` flags from the
138138# outer config into the delegated `Enzyme.autodiff` call. Prior to this
139139# fix the rule hard-coded `Forward`, silently dropping
140- # `set_runtime_activity(Forward)` on the way down into `f_orig`.
140+ # `set_runtime_activity(Forward)` on the way down into `f_orig`.
141141function EnzymeRules. forward (
142- :: EnzymeRules.FwdConfig{false, false, W, RuntimeActivity, StrongZero} ,
143- func:: EnzymeCore.Annotation{<:FunctionWrappersWrapper} ,
144- RT:: Type{<:EnzymeCore.Annotation} ,
145- args:: Vararg{EnzymeCore.Annotation, N}
146- ) where {W, N, RuntimeActivity, StrongZero}
142+ :: EnzymeRules.FwdConfig{false, false, W, RuntimeActivity, StrongZero} ,
143+ func:: EnzymeCore.Annotation{<:FunctionWrappersWrapper} ,
144+ RT:: Type{<:EnzymeCore.Annotation} ,
145+ args:: Vararg{EnzymeCore.Annotation, N}
146+ ) where {W, N, RuntimeActivity, StrongZero}
147147 f_orig = unwrap (func. val)
148148 mode = _fwd_mode (Val (false ), Val (RuntimeActivity), Val (StrongZero))
149149 Enzyme. autodiff (mode, Const (f_orig), Const, args... )
@@ -155,11 +155,11 @@ end
155155# =============================================================================
156156
157157function EnzymeRules. augmented_primal (
158- config:: EnzymeRules.RevConfig ,
159- func:: EnzymeCore.Annotation{<:FunctionWrappersWrapper} ,
160- RT:: Type{<:EnzymeCore.Active{T}} ,
161- args:: Vararg{EnzymeCore.Annotation, N}
162- ) where {T, N}
158+ config:: EnzymeRules.RevConfig ,
159+ func:: EnzymeCore.Annotation{<:FunctionWrappersWrapper} ,
160+ RT:: Type{<:EnzymeCore.Active{T}} ,
161+ args:: Vararg{EnzymeCore.Annotation, N}
162+ ) where {T, N}
163163 f_orig = unwrap (func. val)
164164 pargs = ntuple (i -> args[i]. val, Val (N))
165165 result = f_orig (pargs... ):: T
@@ -175,11 +175,11 @@ end
175175# return). Just run the primal for its side effects; no tape is needed because
176176# the reverse pass has nothing to propagate back from the return.
177177function EnzymeRules. augmented_primal (
178- config:: EnzymeRules.RevConfig ,
179- func:: EnzymeCore.Annotation{<:FunctionWrappersWrapper} ,
180- RT:: Type{<:EnzymeCore.Const} ,
181- args:: Vararg{EnzymeCore.Annotation, N}
182- ) where {N}
178+ config:: EnzymeRules.RevConfig ,
179+ func:: EnzymeCore.Annotation{<:FunctionWrappersWrapper} ,
180+ RT:: Type{<:EnzymeCore.Const} ,
181+ args:: Vararg{EnzymeCore.Annotation, N}
182+ ) where {N}
183183 f_orig = unwrap (func. val)
184184 pargs = ntuple (i -> args[i]. val, Val (N))
185185 f_orig (pargs... )
@@ -189,11 +189,11 @@ end
189189# Duplicated / BatchDuplicated return: record the primal so that reverse has
190190# it available when propagating dret through the arguments.
191191function EnzymeRules. augmented_primal (
192- config:: EnzymeRules.RevConfig ,
193- func:: EnzymeCore.Annotation{<:FunctionWrappersWrapper} ,
194- RT:: Type{<:EnzymeCore.Duplicated{T}} ,
195- args:: Vararg{EnzymeCore.Annotation, N}
196- ) where {T, N}
192+ config:: EnzymeRules.RevConfig ,
193+ func:: EnzymeCore.Annotation{<:FunctionWrappersWrapper} ,
194+ RT:: Type{<:EnzymeCore.Duplicated{T}} ,
195+ args:: Vararg{EnzymeCore.Annotation, N}
196+ ) where {T, N}
197197 f_orig = unwrap (func. val)
198198 pargs = ntuple (i -> args[i]. val, Val (N))
199199 primal = f_orig (pargs... ):: T
@@ -205,11 +205,11 @@ function EnzymeRules.augmented_primal(
205205end
206206
207207function EnzymeRules. augmented_primal (
208- config:: EnzymeRules.RevConfig ,
209- func:: EnzymeCore.Annotation{<:FunctionWrappersWrapper} ,
210- RT:: Type{<:EnzymeCore.BatchDuplicated{T, W}} ,
211- args:: Vararg{EnzymeCore.Annotation, N}
212- ) where {T, W, N}
208+ config:: EnzymeRules.RevConfig ,
209+ func:: EnzymeCore.Annotation{<:FunctionWrappersWrapper} ,
210+ RT:: Type{<:EnzymeCore.BatchDuplicated{T, W}} ,
211+ args:: Vararg{EnzymeCore.Annotation, N}
212+ ) where {T, W, N}
213213 f_orig = unwrap (func. val)
214214 pargs = ntuple (i -> args[i]. val, Val (N))
215215 primal = f_orig (pargs... ):: T
@@ -235,41 +235,43 @@ end
235235# function, then scale by dret. This avoids type-inference issues that arise
236236# from calling autodiff(Reverse, Const{Any}(...), ...).
237237@generated function _fww_reverse_grads (
238- mode, f_orig, dret_val:: T , args:: Vararg{EnzymeCore.Active, N}
239- ) where {T, N}
238+ mode, f_orig, dret_val:: T , args:: Vararg{EnzymeCore.Active, N}
239+ ) where {T, N}
240240 # Build forward-mode calls for each partial derivative
241241 exprs = []
242242 for i in 1 : N
243243 seeds = [j == i ? :(one (eltype (typeof (args[$ j])))) : :(zero (eltype (typeof (args[$ j])))) for j in 1 : N]
244244 dups = [:(Duplicated (args[$ j]. val, $ (seeds[j]))) for j in 1 : N]
245245 Ti = :(eltype (typeof (args[$ i])))
246- push! (exprs, quote
247- fwd = Enzyme. autodiff (mode, Const (f_orig), Duplicated{$ T}, $ (dups... ))
248- $ Ti (fwd[1 ] * dret_val):: $Ti
249- end )
246+ push! (
247+ exprs, quote
248+ fwd = Enzyme. autodiff (mode, Const (f_orig), Duplicated{$ T}, $ (dups... ))
249+ $ Ti (fwd[1 ] * dret_val):: $Ti
250+ end
251+ )
250252 end
251253 return Expr (:tuple , exprs... )
252254end
253255
254256function EnzymeRules. reverse (
255- config:: EnzymeRules.RevConfig ,
256- func:: EnzymeCore.Annotation{<:FunctionWrappersWrapper} ,
257- dret:: EnzymeCore.Active{T} ,
258- tape,
259- args:: Vararg{EnzymeCore.Active, N}
260- ) where {T, N}
257+ config:: EnzymeRules.RevConfig ,
258+ func:: EnzymeCore.Annotation{<:FunctionWrappersWrapper} ,
259+ dret:: EnzymeCore.Active{T} ,
260+ tape,
261+ args:: Vararg{EnzymeCore.Active, N}
262+ ) where {T, N}
261263 f_orig = unwrap (func. val)
262264 return _fww_reverse_grads (_fwd_mode_from_rev (config), f_orig, dret. val, args... )
263265end
264266
265267# Handle mixed Active/Const args: return nothing for Const, gradient for Active
266268function EnzymeRules. reverse (
267- config:: EnzymeRules.RevConfig ,
268- func:: EnzymeCore.Annotation{<:FunctionWrappersWrapper} ,
269- dret:: EnzymeCore.Active ,
270- tape,
271- args:: Vararg{EnzymeCore.Annotation, N}
272- ) where {N}
269+ config:: EnzymeRules.RevConfig ,
270+ func:: EnzymeCore.Annotation{<:FunctionWrappersWrapper} ,
271+ dret:: EnzymeCore.Active ,
272+ tape,
273+ args:: Vararg{EnzymeCore.Annotation, N}
274+ ) where {N}
273275 f_orig = unwrap (func. val)
274276 dret_val = dret. val
275277 mode = _fwd_mode_from_rev (config)
@@ -303,12 +305,12 @@ end
303305# `BatchDuplicated` args return `nothing` because their gradients are
304306# accumulated in-place by the `Enzyme.autodiff(Reverse, …)` call above.
305307function EnzymeRules. reverse (
306- config:: EnzymeRules.RevConfig ,
307- func:: EnzymeCore.Annotation{<:FunctionWrappersWrapper} ,
308- dret:: Type{<:EnzymeCore.Const} ,
309- tape,
310- args:: Vararg{EnzymeCore.Annotation, N}
311- ) where {N}
308+ config:: EnzymeRules.RevConfig ,
309+ func:: EnzymeCore.Annotation{<:FunctionWrappersWrapper} ,
310+ dret:: Type{<:EnzymeCore.Const} ,
311+ tape,
312+ args:: Vararg{EnzymeCore.Annotation, N}
313+ ) where {N}
312314 f_orig = unwrap (func. val)
313315 # Only worth invoking Enzyme.autodiff when at least one arg is
314316 # Duplicated/BatchDuplicated — otherwise there's nothing to accumulate.
0 commit comments