File tree Expand file tree Collapse file tree
transformer_engine/pytorch/attention/dot_product_attention Expand file tree Collapse file tree Original file line number Diff line number Diff line change 162162except PackageNotFoundError :
163163 flash_attn_func_v4 = None
164164 flash_attn_varlen_func_v4 = None
165- # flash_attn_combine_v4 = None
166- _flash_attn_fwd_v4 = None
167- _flash_attn_bwd_v4 = None
165+ # TODO: for context parallel
166+ # _flash_attn_fwd_v4 = None
167+ # _flash_attn_bwd_v4 = None
168168else :
169- from flash_attn .cute .interface import ( # pylint: disable=ungrouped-imports
169+ from flash_attn .cute .interface import ( # pylint: disable=ungrouped-imports,no-name-in-module
170170 flash_attn_func as flash_attn_func_v4 ,
171171 flash_attn_varlen_func as flash_attn_varlen_func_v4 ,
172- _flash_attn_fwd as _flash_attn_fwd_v4 ,
173- _flash_attn_bwd as _flash_attn_bwd_v4 ,
172+ # TODO: for context parallel
173+ # _flash_attn_fwd as _flash_attn_fwd_v4,
174+ # _flash_attn_bwd as _flash_attn_bwd_v4,
174175 )
175176
176177 fa_utils .set_flash_attention_4_params ()
You can’t perform that action at this time.
0 commit comments