+
    œ°³jŽ"  ã                   ó¶   € ^ RI t ^RIHtHtHt ^RIHt ]P                  ! ]4      t	]! RRR7      t
]! 4       t]! 4       tR R ltR	 R
 ltR R ltRR R lltR# )é    N)Úis_torch_npu_availableÚis_torch_xpu_availableÚlogging)Úis_torch_greater_or_equalz2.8T)Ú
accept_devc                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )é   Úhidden_statesÚn_repÚreturn)ÚtorchÚTensorÚint)Úformats   "Úi/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/integrations/sdpa_attention.pyÚ__annotate__r      s.   € ÷ 	Uñ 	UœUŸ\™\ð 	U´#ð 	U¼%¿,¹,ñ 	Uó    c                ó˜   € V P                   w  r#rEV^8X  d   V # V R,          P                  W#WV4      p V P                  W#V,          WE4      # )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
)ºNNNr   Nr   r   )ÚshapeÚexpandÚreshape)r
   r   ÚbatchÚnum_key_value_headsÚslenÚhead_dims   &&    r   Ú	repeat_kvr      sU   € ð
 2?×1DÑ1DÑ.€E Ø�„zØÐØ!Ð"2Õ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÕ(CÀTÓTÐTr   c                ó’   € V ^8„  d   QhR\         P                  R,          R\         P                  R\         P                  R\        /# )r	   Úattention_maskNÚkeyÚvaluer   ©r   r   Úbool)r   s   "r   r   r      sC   € ÷ 	Nñ 	N¤E§L¡L°4Õ$7ð 	N¼e¿l¹lð 	NÔSX×S_ÑS_ð 	NÔdhñ 	Nr   c                 ó®   € \         '       d   \        # V R J ;'       d9    VP                  R,          VP                  R,          u;8H  ;'       d    R8*  # u # )Né   éÿÿÿÿ)Ú_is_torch_xpu_availableÚ#_is_torch_greater_or_equal_than_2_8r   )r   r    r!   s   &&&r   Úuse_gqa_in_sdpar)      sF   € ÷ ÓÜ2Ð2Ø˜TÐ!×MÐM c§i¡i°¥m°u·{±{À2µ×&MÔ&MÈ#Ñ&MÐMÑ&MÐMr   c                óÒ   € V ^8„  d   QhR\         P                  R\         P                  R,          R\        R\         P                  R\         P                  R\         P                  /# )r	   Úposition_biasr   NÚ	is_causalÚqueryr    r   r"   )r   s   "r   r   r   '   s]   € ÷ %ñ %Ü—<‘<ð%ä—L‘L 4Õ'ð%ô ð%ô �<‰<ð	%ô
 
�‰ð%ô ‡\�\ñ%r   c                ó0  € \         P                  ! VP                  4      P                  pVf¨   V'       d›   VP                  pVP
                  ^,          VP
                  ^,          r‡\         P                  ! WvR7      R,          \         P                  ! W†R7      R,          8¬  p	V	P                  ^^Wx4      p	\         P                  ! W�V4      p
V
# T p
 V
# VP                  \         P                  8X  d   \         P                  ! WV4      p
V
# W,           p
V
# )a  
Create a floating-point dtype mask to use with sdpa. The mask contains the values of `position_bias` to positions where we should
attend to tokens, and -inf where we should not. It will be added to the QK^T result in the attention, before the softmax. Note
that using such a mask will usually prevent sdpa from dispatching to the most efficient kernel implementations.

Note that we cannot create this in advance when we create the mask in the model, as the position_bias is usually learned
differently in every layer.
)Údevice)r   N)Nr   )
r   ÚfinfoÚdtypeÚminr/   r   ÚarangeÚviewÚwherer#   )r+   r   r,   r-   r    Ú	min_dtyper/   Úq_lengthÚ	kv_lengthÚcausal_maskÚposition_bias_masks   &&&&&      r   Úcreate_position_bias_maskr;   '   sì   € ô —’˜CŸI™IÓ&×*Ñ*€IàÒçØ—Z‘ZˆFØ"'§+¡+¨a¥.°#·)±)¸Aµ,�iä—’˜XÔ5°gÕ>Ä%Ç,Â,ÈyÔBhÐipÕBqÑqð ð &×*Ñ*¨1¨a°ÓEˆKÜ!&§¢¨[ÈÓ!SÐð Ðð "/Ñð Ðð 
×	Ñ	¤§¡Ô	+ä"Ÿ[š[¨È	ÓRÐð
 Ðð +Õ;ÐàÐr   c                ó‚  € V ^8„  d   QhR\         P                  P                  R\         P                  R\         P                  R\         P                  R\         P                  R,          R\        R\        R,          R	\
        R,          R
\         P                  R,          R\        \         P                  R3,          /
# )r	   Úmoduler-   r    r!   r   NÚdropoutÚscalingr,   r+   r   )r   ÚnnÚModuler   Úfloatr#   Útuple)r   s   "r   r   r   O   s´   € ÷ [ñ [Ü�H‰H�O‰Oð[ä�<‰<ð[ô 
�‰ð[ô �<‰<ð	[ô
 —L‘L 4Õ'ð[ô ð[ô �T�\ð[ô �d�{ð[ô —<‘< $Õ&ð[ô Œ5�<‰<˜ÐÕñ[r   c	                 ó¼  € V	P                  R R4      '       d   \        P                  R4       / p
\        V R4      '       dS   V P                  ^8”  dB   \        WBV4      '       g,   \        W P                  4      p\        W0P                  4      pMRR/p
VP                  ^,          pVP                  ^,          pVe   TM\        V RR4      pV^8„  ;'       d    VRJ ;'       d    Tp\        P                  P                  4       '       d1   \        V\        P                  4      '       d   VP                  4       p\        '       da   Ve]   VP                   \        P"                  8w  d>   \        P$                  ! VP#                  4       4      P'                  VP(                  4      pV'       dC   Vf?   V^8”  d8   WË8”  d2   VRRRV1R3,          pVRRRV1R3,          pVe   VRRRRV13,          pVe   \+        W„WqV4      pRp\        P,                  P.                  P0                  ! VVV3R	VR
VRVRV/V
B pVP3                  ^^4      P5                  4       pVR3# )Úoutput_attentionsFzƒ`sdpa` attention does not support `output_attentions=True`. Please set your attention to `eager` if you want any of these features.Únum_key_value_groupsÚ
enable_gqaTNr,   r   Ú	attn_maskÚ	dropout_pÚscale)ÚgetÚloggerÚwarning_onceÚhasattrrF   r)   r   r   Úgetattrr   ÚjitÚ
is_tracingÚ
isinstancer   ÚitemÚ_is_torch_npu_availabler1   r#   Úlogical_notÚtor/   r;   r@   Ú
functionalÚscaled_dot_product_attentionÚ	transposeÚ
contiguous)r=   r-   r    r!   r   r>   r?   r,   r+   ÚkwargsÚsdpa_kwargsr7   r8   Úattn_outputs   &&&&&&&&&,    r   Úsdpa_attention_forwardr^   O   sH  € ð ‡z�zÐ% u×-Ò-Ü×ÑðWô	
ð €KÜˆvÐ-×.Ò.°6×3NÑ3NÐQRÔ3RÜ˜~°E×:Ò:Ü˜C×!<Ñ!<Ó=ˆCÜ˜e×%@Ñ%@ÓA‰Eà'¨Ð.ˆKà�{‰{˜1�~€HØ—	‘	˜!•€Ið 'Ò2‘	¼ÀÈÐUYÓ8Z€Ið  ˜1‘×EÐE °4Ð!7×EÐE¸I€Iô ‡y�y×Ñ×Ò¤*¨Y¼¿¹×"EÒ"EØ—N‘NÓ$ˆ	÷
 ÓØÒ%¨.×*>Ñ*>Ä%Ç*Á*Ô*Lä"×.Ò.¨~×/BÑ/BÓ/DÓE×HÑHÈÏÉÓVˆN÷ �^Ò+°¸1´ÀÔAUØ�!�Q˜	˜˜	 1Ð$Õ%ˆØ�a˜˜I˜X˜I qÐ(Õ)ˆàÒ$Ø)¨!¨Q°°9°H°9Ð*<Õ=ˆMð Ò Ü2°=ÐR[ÐdgÓhˆØˆ	ä—(‘(×%Ñ%×BÒBØØØñ	ð !ð		ð
 ð	ð ð	ð ð	ð ñ	€Kð ×'Ñ'¨¨1Ó-×8Ñ8Ó:€Kà˜ÐÐr   )g        NNN)r   Úutilsr   r   r   Úutils.import_utilsr   Ú
get_loggerÚ__name__rL   r(   r'   rT   r   r)   r;   r^   © r   r   Ú<module>rd      sd   ðÛ ç KÑ KÝ :ð 
×	Ò	˜HÓ	%€ñ '@ÀÐRVÔ&WÐ #Ù0Ó2Ð Ù0Ó2Ð õ	Uõ	Nõ%÷P[ñ [r   