+
    œ°³j³!  ã                   óÌ  € ^ RI t ^ RIHt ^ RIt^ RIHt ^RIHt ^RIH	t	 ^RI
HtHt ^RIHt ^RIHt ^R	IHtHtHtHtHtHtHtHtHtHt ^R
IHt ^RIHt ]P@                  ! ]!4      t"Rt#Rt$ ! R R]4      t%R t& ! R R]4      t' ! R R]4      t( ! R R]4      t) ! R R]4      t* ! R R]4      t+ ! R R]4      t, ! R R]4      t- ! R R ]4      t. ! R! R"]4      t/. R#Ot0R# )$é    N)ÚCallable)Únn)Úinitialization)ÚCache)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)Úlogging)ÚGemmaForCausalLM)
ÚLlamaAttentionÚLlamaDecoderLayerÚLlamaForQuestionAnsweringÚLlamaForSequenceClassificationÚLlamaForTokenClassificationÚ
LlamaModelÚLlamaPreTrainedModelÚLlamaRotaryEmbeddingÚapply_rotary_pos_embÚeager_attention_forward)Ú
MistralMLP)ÚDiffLlamaConfigzkajuma/DiffLlama-0.3B-handcutr   c                   ó   € ] tR t^2tRtR# )ÚDiffLlamaMLP© N©Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__static_attributes__r   ó    Úp/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/diffllama/modular_diffllama.pyr   r   2   ó   † Ûr    r   c                 óX   € R R\         P                  ! RV ,          4      ,          ,
          # )gš™™™™™é?g333333ã?g333333Ó¿)ÚmathÚexp)Ú	layer_idxs   &r!   Úlambda_init_fnr'   6   s    € Ø�”t—x’x  yÕ 0Ó1Õ1Õ1Ð1r    c                   ó   € ] tR t^:tRtR# )ÚDiffLlamaRotaryEmbeddingr   Nr   r   r    r!   r)   r)   :   r"   r    r)   c                   ó\   a a€ ] tR t^>t oRtRV3R lV 3R llltRV3R lR lltRtVtV ;t	# )	ÚDiffLlamaAttentionuç  Multi-headed differential attention (https://huggingface.co/papers/2410.05258).

Computes ``(softmax(Q1 K1áµ€) - Î» Â· softmax(Q2 K2áµ€)) Â· V`` as two standard attention calls
sharing Q and K over the two halves of V. The two-call structure is ~30% faster than the
V-doubling shortcut at production shapes, since asymmetric V (``head_dim_v != head_dim_q``)
forces SDPA off Flash/cuDNN onto the memory-efficient/math kernel; Flash Attention 2 also
requires ``head_dim_v == head_dim_q``.
c                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# )é   Úconfigr&   N©r   Úint)ÚformatÚ__classdict__s   "€r!   Ú__annotate__ÚDiffLlamaAttention.__annotate__H   s#   ø€ ÷ jñ j™ð j¹3À½:ñ jr    c                ó¦  <€ \         SV `  W4       VP                  R 8”  d   \        R4      hVP                  e   VP                  ^,          ^ 8w  d   \        RVP                   R24      h\        V4      V n        \        P                  ! \        P                  ! ^ VP                  V P                  3R7      4      V n        \        P                  ! \        P                  ! ^ VP                  V P                  3R7      4      V n        \        P                  ! \        P                  ! ^ VP                  V P                  3R7      4      V n        \        P                  ! \        P                  ! ^ VP                  V P                  3R7      4      V n        \        P"                  ! ^V P                  ,          VP$                  RR7      V n        R# )ç        z€DiffLlama does not support `attention_dropout > 0`: the differential attention mechanism has no paper-defined dropout semantics.Nz–DiffLlama requires `num_key_value_heads` to be even (and at least 2): the two-call differential attention splits the value tensor along KV heads, got Ú.)ÚsizeF)ÚepsÚelementwise_affine)ÚsuperÚ__init__Úattention_dropoutÚ
ValueErrorÚnum_key_value_headsr'   Úlambda_initr   Ú	ParameterÚtorchÚnormalÚlambda_std_devÚhead_dimÚ	lambda_q1Ú	lambda_k1Ú	lambda_q2Ú	lambda_k2ÚRMSNormÚrms_norm_epsÚ	groupnorm©Úselfr.   r&   Ú	__class__s   &&&€r!   r<   ÚDiffLlamaAttention.__init__H   sc  ø€ Ü‰Ñ˜Ô+ð
 ×#Ñ# cÔ)ÜðDóð ð ×%Ñ%Ò-°×1KÑ1KÈaÕ1OÐSTÔ1TÜðVØV\×VpÑVpÐUqÐqrðtóð ô *¨)Ó4ˆÔÜŸš¤e§l¢l°1°f×6KÑ6KÐSW×S`ÑS`ÐRbÔ&cÓdˆŒÜŸš¤e§l¢l°1°f×6KÑ6KÐSW×S`ÑS`ÐRbÔ&cÓdˆŒÜŸš¤e§l¢l°1°f×6KÑ6KÐSW×S`ÑS`ÐRbÔ&cÓdˆŒÜŸš¤e§l¢l°1°f×6KÑ6KÐSW×S`ÑS`ÐRbÔ&cÓdˆŒÜŸš A¨¯©Õ$5¸6×;NÑ;NÐchÔiˆŽr    c                óø   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          RS[ P                  R,          RS[R,          RS[S[ P                  S[ P                  3,          /# )r-   Úhidden_statesÚposition_embeddingsÚattention_maskNÚpast_key_valuesÚreturn)rB   ÚTensorÚtupler   )r1   r2   s   "€r!   r3   r4   a   sy   ø€ ÷ C)ñ C)á—|‘|ðC)ñ #¡5§<¡<±·±Ð#=Õ>ðC)ñ Ÿ™ tÕ+ð	C)ñ
  �ðC)ñ 
‰u�|‰|™UŸ\™\Ð)Õ	*ñC)r    c                ó
  € VP                   R R p. VORNV P                  N5pV P                  V4      P                  V4      P	                  ^^4      pV P                  V4      P                  V4      P	                  ^^4      p	V P                  V4      P                  V4      P	                  ^^4      p
Vw  r¼\        W‰W¼4      w  r‰Ve   VP                  WšV P                  4      w  ršR \        P                  ! V
^^R7       4       w  rÞ\        P                  ! V P                  P                  \         4      pV! V VV	VV3RRRV P"                  /VB w  ppV! V VV	VV3RRRV P"                  /VB w  pp\        P$                  ! VV.RR7      p\        P                  ! V^^R7      w  pp\        P&                  ! \        P(                  ! V P*                  V P,                  ,          R\        P.                  R7      4      P1                  VP2                  4      p\        P&                  ! \        P(                  ! V P4                  V P6                  ,          R\        P.                  R7      4      P1                  VP2                  4      pVV,
          V P8                  ,           pVVV,          ,
          p^V P8                  ,
          V P;                  V4      ,          pVP<                  ! . VORN5!  pV P?                  V4      pVV3# )Nc              3   óH   "  € T F  qP                  ^^^^4      x € K  	  R# 5i)é   N)Úrepeat)Ú.0Úvs   & r!   Ú	<genexpr>Ú-DiffLlamaAttention.forward.<locals>.<genexpr>x   s"   é € Ð'jÑFiÀ¯©°°A°q¸!×(<Ð(<ÓFiùs   ‚ ")ÚdimÚdropoutr6   Úscaling)ra   Údtypeéÿÿÿÿ) ÚshaperE   Úq_projÚviewÚ	transposeÚk_projÚv_projr   Úupdater&   rB   Úchunkr   Úget_interfacer.   Ú_attn_implementationr   rc   Úcatr%   ÚsumrF   rG   Úfloat32Útord   rH   rI   r@   rL   ÚreshapeÚo_proj)rN   rR   rS   rT   rU   ÚkwargsÚinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚcosÚsinÚvalue_states1Úvalue_states2Úattention_interfaceÚattn_output1Úattn_weightsÚattn_output2Ú_Úattn_outputÚlambda_1Úlambda_2Úlambda_fulls   &&&&&,                  r!   ÚforwardÚDiffLlamaAttention.forwarda   sÍ  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJñ (kÄeÇkÂkÐR^Ð`aÐghÕFiÓ'jÑ$ˆä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ &9ØØØØØñ	&
ð ð	&
ð —L‘Lð	&
ð ñ	&
Ñ"ˆ�lñ .ØØØØØñ	
ð ð	
ð —L‘Lð	
ð ñ	
‰ˆ�aô —i’i ¨|Ð <À"ÔEˆô &+§[¢[°¸aÀQÔ%GÑ"ˆ�lÜ—9’9œUŸYšY t§~¡~¸¿¹Õ'FÈBÔV[×VcÑVcÔdÓe×hÑhØ×Ñó
ˆô —9’9œUŸYšY t§~¡~¸¿¹Õ'FÈBÔV[×VcÑVcÔdÓe×hÑhØ×Ñó
ˆð  Õ)¨D×,<Ñ,<Õ<ˆØ" [°<Õ%?Õ?ˆØ˜4×+Ñ+Õ+¨t¯~©~¸kÓ/JÕJˆØ!×)Ò)Ð;¨;Ð;¸Ó;ˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r    )rL   r@   rG   rI   rF   rH   )N)NN)
r   r   r   r   Ú__doc__r<   r‰   r   Ú__classdictcell__Ú__classcell__©rO   r2   s   @@r!   r+   r+   >   s(   ù‡ € ñ÷jõ j÷2C)÷ C)ò C)r    r+   c                   ó>   a a€ ] tR t^§t oV3R lV 3R lltRtVtV ;t# )ÚDiffLlamaDecoderLayerc                ó&   <€ V ^8„  d   QhRS[ RS[/# )r-   r.   r&   r/   )r1   r2   s   "€r!   r3   Ú"DiffLlamaDecoderLayer.__annotate__¨   s   ø€ ÷ Pñ P™ð P¹3ñ Pr    c                óH   <€ \         SV `  W4       \        WR 7      V n        R# ))r.   r&   N)r;   r<   r+   Ú	self_attnrM   s   &&&€r!   r<   ÚDiffLlamaDecoderLayer.__init__¨   s   ø€ Ü‰Ñ˜Ô+ä+°6ÔOˆŽr    )r”   )r   r   r   r   r<   r   rŒ   r�   rŽ   s   @@r!   r�   r�   §   s   ù‡ € ÷P÷ Pó Pr    r�   c                   óN   a € ] tR t^®t o ]P
                  ! 4       R 4       tRtV tR# )ÚDiffLlamaPreTrainedModelc                ó  € \         P                  ! W4       \        V\        4      '       dÛ   \        P
                  ! VP                  ^ V P                  P                  4       \        P
                  ! VP                  ^ V P                  P                  4       \        P
                  ! VP                  ^ V P                  P                  4       \        P
                  ! VP                  ^ V P                  P                  4       R# R# )r   N)r   Ú_init_weightsÚ
isinstancer+   ÚinitÚnormal_rF   r.   rD   rG   rH   rI   )rN   Úmodules   &&r!   r™   Ú&DiffLlamaPreTrainedModel._init_weights¯   s£   € ä×%Ò% dÔ3Ü�fÔ0×1Ò1Ü�LŠL˜×)Ñ)¨1¨d¯k©k×.HÑ.HÔIÜ�LŠL˜×)Ñ)¨1¨d¯k©k×.HÑ.HÔIÜ�LŠL˜×)Ñ)¨1¨d¯k©k×.HÑ.HÔIÜ�LŠL˜×)Ñ)¨1¨d¯k©k×.HÑ.HÖIñ	 2r    r   N)	r   r   r   r   rB   Úno_gradr™   r   rŒ   )r2   s   @r!   r—   r—   ®   s   ø‡ € Ø
‡]‚]ƒ_ñJó öJr    r—   c                   ó   € ] tR t^¹tRtR# )ÚDiffLlamaModelr   Nr   r   r    r!   r¡   r¡   ¹   r"   r    r¡   c                   ó   € ] tR t^½tRtR# )ÚDiffLlamaForCausalLMr   Nr   r   r    r!   r£   r£   ½   r"   r    r£   c                   ó   € ] tR t^ÁtRtR# )Ú"DiffLlamaForSequenceClassificationr   Nr   r   r    r!   r¥   r¥   Á   r"   r    r¥   c                   ó   € ] tR t^ÅtRtR# )ÚDiffLlamaForQuestionAnsweringr   Nr   r   r    r!   r§   r§   Å   r"   r    r§   c                   ó   € ] tR t^ÉtRtR# )ÚDiffLlamaForTokenClassificationr   Nr   r   r    r!   r©   r©   É   r"   r    r©   )r—   r¡   r£   r¥   r§   r©   )1r$   Úcollections.abcr   rB   r   Ú r   r›   Úcache_utilsr   Úmodeling_utilsr   r   Úutilsr	   Úgemma.modeling_gemmar
   Úllama.modeling_llamar   r   r   r   r   r   r   r   r   r   Úmistral.modeling_mistralr   Úconfiguration_diffllamar   Ú
get_loggerr   ÚloggerÚ_CHECKPOINT_FOR_DOCÚ_CONFIG_FOR_DOCr   r'   r)   r+   r�   r—   r¡   r£   r¥   r§   r©   Ú__all__r   r    r!   Ú<module>r¸      sè   ðó" Ý $ã Ý å &Ý  ß FÝ Ý 3÷÷ ÷ õ 2Ý 4ð 
×	Ò	˜HÓ	%€à5Ð Ø#€ô	�:ô 	ò2ô	Ð3ô 	ôf)˜ô f)ôRPÐ-ô PôJÐ3ô Jô	�Zô 	ô	Ð+ô 	ô	Ð)Gô 	ô	Ð$=ô 	ô	Ð&Aô 	ò‚r    