+
    œ°³j`  ã                   óx   € ^ RI t ^RIHtHt ^RIHt ]P                  ! ]4      t]! 4       t	R R lt
RR R lltR# )	é    N)Ú_flash_attention_forwardÚ!flash_attn_supports_top_left_mask)Úloggingc                óŒ   € V ^8„  d   QhR\         P                  R\         P                  P                  R\         P                  /# )é   ÚqueryÚmoduleÚreturn)ÚtorchÚTensorÚnnÚModuleÚdtype)Úformats   "Új/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/integrations/flash_attention.pyÚ__annotate__r      s1   € ÷ ñ œEŸL™Lð ´%·(±(·/±/ð ÄeÇkÁkñ ó    c                ó   € V P                   \        P                  8X  d¯   V P                  P                  p\        P
                  ! V4      '       d   \        P                  ! V4      # \        VP                  R4      '       d   VP                  P                   # \        R VP                  4        4       4      P                  P                   # R# )ziIf the query is in float32, return a target dtype compatible with flash attention. Return None otherwise.Ú_is_quantizedc              3   ó~   "  € T F3  p\        V\        P                  P                  4      '       g   K/  Vx € K5  	  R # 5i)N)Ú
isinstancer   r   ÚLinear)Ú.0Úlayers   & r   Ú	<genexpr>Ú#get_target_dtype.<locals>.<genexpr>   s*   é € ÐbÑ+; %¼zÈ%ÔQV×QYÑQY×Q`ÑQ`×?aŸšÓ+;ùs   ‚,=³
=N)r   r   Úfloat32ÚdeviceÚtypeÚis_autocast_enabledÚget_autocast_dtypeÚhasattrÚconfigÚnextÚmodulesÚweight)r   r	   Údevice_types   && r   Úget_target_dtyper(      s‹   € à‡{�{”e—m‘mÔ#Ø—l‘l×'Ñ'ˆÜ×$Ò$ [×1Ò1Ü×+Ò+¨KÓ8Ð8ä�V—]‘] O×4Ò4Ø—=‘=×&Ñ&Ð&äÑb¨6¯>©>Ô+;ÓbÓb×iÑi×oÑoÐoÙr   c                ó¶  € V ^8„  d   QhR\         P                  P                  R\         P                  R\         P                  R\         P                  R\         P                  R,          R\        R\        R,          R	\
        R,          R
\        R,          R\        R,          R\         P                  R,          R\        \         P                  R3,          /# )r   r	   r   ÚkeyÚvalueÚattention_maskNÚdropoutÚscalingÚsliding_windowÚsoftcapÚ	is_causalÚs_auxr
   )r   r   r   r   ÚfloatÚintÚboolÚtuple)r   s   "r   r   r      sÒ   € ÷ Lñ LÜ�H‰H�O‰OðLä�<‰<ðLô 
�‰ðLô �<‰<ð	Lô
 —L‘L 4Õ'ðLô ðLô �T�\ðLô ˜$•JðLô �T�\ðLô �d�{ðLô �<‰<˜$ÕðLô Œ5�<‰<˜ÐÕñLr   c                 ó¶  € VP                  R R4      '       d   \        P                  R4       VP                  ^,          p\        ;QJ d&    R VP                   4       F  '       g   K   RM	  RM! R VP                   4       4      '       d   \        R4      hVP                  ^^4      pVP                  ^^4      pVP                  ^^4      pVP                  R,          VP                  R,          ríWí8w  d3   \        P                  P                  P                  V^ WÞ,
          .4      p\        W4      pV	e   T	MV P                  p	\        VVVV3RTRT	R	TR
TRTRTR\        RTRV P                  P                   R\#        V R4      '       d   V P$                  MRRV
e   V
P'                  VP(                  4      MR/VB pWí8w  d   VRRV13,          pVR3# )Úoutput_attentionsFz‚Flash Attention does not support `output_attentions=True`. Please set your attention to `eager` if you want any of these features.c              3   ó*   "  € T F	  q^ 8H  x € K  	  R# 5i)r   N© )r   Údims   & r   r   Ú*flash_attention_forward.<locals>.<genexpr>1   s   é € Ð
+™{˜�!Ž8›{ùs   ‚Tz•Tensor query has shape  with a zero dimension.
FlashAttention does not support inputs with dim=0.
Please check your input shapes or use SDPA instead.NÚquery_lengthr1   r-   Úsoftmax_scaler/   r0   Úuse_top_left_maskÚtarget_dtypeÚattn_implementationÚ	layer_idxr2   .éÿÿÿÿ)ÚgetÚloggerÚwarning_onceÚshapeÚanyÚ
ValueErrorÚ	transposer   r   Ú
functionalÚpadr(   r1   r   Ú_use_top_left_maskr#   Ú_attn_implementationr"   rB   Útor   )r	   r   r*   r+   r,   r-   r.   r/   r0   r1   r2   ÚkwargsÚseq_lenÚhead_dimÚ
v_head_dimr@   Úattn_outputs   &&&&&&&&&&&,     r   Úflash_attention_forwardrU      sô  € ð ‡z�zÐ% u×-Ò-Ü×ÑðWô	
ð �k‰k˜!�n€Gç
ƒsÑ
+˜uŸ{š{Ó
+‡s‡s‚sÑ
+˜uŸ{š{Ó
+×+Ò+ÜðBó
ð 	
ð �O‰O˜A˜qÓ!€EØ
�-‰-˜˜1Ó
€CØ�O‰O˜A˜qÓ!€Eð !Ÿ;™; r�?¨E¯K©K¸­OˆjØÔÜ—‘×#Ñ#×'Ñ'¨°°8Õ3HÐ/IÓJˆô $ EÓ2€Lð 'Ò2‘	¸×8HÑ8H€Iä*ØØØØñ	ð
 ðð ðð ðð ðð &ðð ðô -ðð "ðð #ŸM™M×>Ñ>ðô '.¨f°k×&BÒ&B�&×"Ò"Èðð" Ò ð �H‰H�U—[‘[Ô!àð%ð( ñ)€Kð. ÔØ! # {¨
 {Ð"2Õ3ˆà˜ÐÐr   )g        NNNNN)r   Úmodeling_flash_attention_utilsr   r   Úutilsr   Ú
get_loggerÚ__name__rE   rM   r(   rU   r:   r   r   Ú<module>rZ      s:   ðÛ ç hÝ ð 
×	Ò	˜HÓ	%€á6Ó8Ð õ÷Lñ Lr   