+
    œ°³je  ã                   óþ  € ^ RI t ^ RIHt ^ RIt^ RIHt ^RIHt ^RIH	t	 ^RI
HtHt ^RIHt ^RIHtHt ^R	IHt ^R
IHtHtHtHt ^RIHtHt ^RIHtHt ^RIH t H!t! ^RI"H#t# ^RI$H%t%H&t&H't' ^RI(H)t) ^RI*H+t+H,t, ^RI-H.t. ^RI/H0t0  ! R R]Pb                  4      t2 ! R R]Pb                  4      t3R t4]! R4      R3R l4       t5R R lt6R4R R llt7R t8]! ]54       ! R  R!]Pb                  4      4       t9]! R"4       ! R# R$]Pb                  4      4       t: ! R% R&]4      t;]& ! R' R(]!4      4       t<]& ! R) R*]<4      4       t=]& ! R+ R,]<]4      4       t> ! R- R.]]<4      t? ! R/ R0]]<4      t@ ! R1 R2]]<4      tA. R5OtBR# )6é    N)ÚCallable)Únn)Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_kernel_forward_from_hubÚuse_kernelized_func)Úcreate_causal_mask)ÚGenericForQuestionAnsweringÚ GenericForSequenceClassificationÚGenericForTokenClassificationÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputs©ÚDiffLlamaConfigc                   ó8   a a€ ] tR t^4t oV 3R ltR tRtVtV ;t# )ÚDiffLlamaMLPc                óØ  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        \
        P                  ! V P                  V P                  R R7      V n        \
        P                  ! V P                  V P                  R R7      V n        \
        P                  ! V P                  V P                  R R7      V n	        \        VP                  ,          V n        R# ©F©ÚbiasN)ÚsuperÚ__init__ÚconfigÚhidden_sizeÚintermediate_sizer   ÚLinearÚ	gate_projÚup_projÚ	down_projr   Ú
hidden_actÚact_fn©Úselfr)   Ú	__class__s   &&€Úq/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/diffllama/modeling_diffllama.pyr(   ÚDiffLlamaMLP.__init__5   s¤   ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔÜŸš 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆŒÜ˜V×.Ñ.Õ/ˆŽó    c                ó�   € V P                  V P                  V P                  V4      4      V P                  V4      ,          4      pV# ©N)r/   r1   r-   r.   )r3   Úxr/   s   && r5   ÚforwardÚDiffLlamaMLP.forward?   s6   € Ø—N‘N 4§;¡;¨t¯~©~¸aÓ/@Ó#AÀDÇLÁLÐQRÃOÕ#SÓTˆ	ØÐr7   )r1   r)   r/   r-   r*   r+   r.   ©	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r(   r;   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r4   Ú__classdict__s   @@r5   r"   r"   4   s   ù‡ € õ0÷ò r7   r"   c                   óÎ   a a€ ] tR t^Dt o]! RRR7      R
V3R lV 3R lll4       t]]! RRR7      R
V3R lR ll4       4       t]P                  ! 4       ]
R 4       4       tR	tVtV ;t# )ÚDiffLlamaRotaryEmbeddingÚdevicez5.18)Úversionc                ó    <€ V ^8„  d   QhRS[ /# ©é   r)   r   )ÚformatrF   s   "€r5   Ú__annotate__Ú%DiffLlamaRotaryEmbedding.__annotate__F   s   ø€ ÷ Oñ O™ñ Or7   c                óä  <€ \         SV `  4        VP                  V n        VP                  V n        Wn        V P
                  P                  R ,          V n        V P                  pV P                  R8w  d   \        V P                  ,          pV! V P
                  V4      w  q@n
        \        P                  ! VRR7      V n        \        P                  ! VP                  4       RR7      V n        R# )Ú	rope_typeÚdefaultF)Ú
persistentN)r'   r(   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr)   Úrope_parametersrR   Úcompute_default_rope_parametersr   Úattention_scalingr   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)r3   r)   rI   Úrope_init_fnr\   r4   s   &&&  €r5   r(   Ú!DiffLlamaRotaryEmbedding.__init__E   s¬   ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àŒàŸ™×4Ñ4°[ÕAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÔ&Ü.¨t¯~©~Õ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÔ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÔ!NˆÖr7   c                óP   <€ V ^8„  d   QhRS[ RS[S[P                  S[3,          /# )rM   r)   Úreturn)r    ÚtupleÚtorchÚTensorÚfloat)rN   rF   s   "€r5   rO   rP   X   s.   ø€ ÷ 5ñ 5±ð 5Ñ[`Ñaf×amÑamÑotÐatÕ[uñ 5r7   c           	     ó0  € V P                   R,          p\        V RR4      ;'       g    V P                  V P                  ,          pRpRV\        P
                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚhead_dimNg      ð?©Údtype)rX   Úgetattrr*   Únum_attention_headsrd   Úarangerf   Úto)r)   rI   ÚkwargsÚbaseÚdimÚattention_factorr\   s   &&,    r5   rY   Ú8DiffLlamaRotaryEmbedding.compute_default_rope_parametersV   s~   € ð ×%Ñ% lÕ3ˆÜ�f˜j¨$Ó/×cÐc°6×3EÑ3EÈ×IcÑIcÕ3cˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r7   c                ó^  € V P                   R,          P                  VP                  ^ ,          R	^4      P                  \        P
                  VP                  R7      pVR
,          P                  4       p\        VP                  P                  \        4      '       d2   VP                  P                  R8w  d   VP                  P                  MRp\        VRR7      ;_uu_ 4        W4,          P                  ^^4      p\        P                  ! Wf3R	R7      pVP                  4       V P                  ,          pVP                  4       V P                  ,          p	R R R 4       XP                  VP                   R7      X	P                  VP                   R7      3#   + '       g   i     LH; i)N)rk   rI   ÚmpsÚcpuF)Údevice_typeÚenabled©rr   rj   )NºNNNNéÿÿÿÿ)r{   Nr{   )r\   ÚexpandÚshapero   rd   rf   rI   Ú
isinstanceÚtypeÚstrr   Ú	transposeÚcatÚcosrZ   Úsinrk   )
r3   r:   Úposition_idsÚinv_freq_expandedÚposition_ids_expandedrx   ÚfreqsÚembr„   r…   s
   &&&       r5   r;   Ú DiffLlamaRotaryEmbedding.forwardj   s2  € ð �M‰M˜-Õ(×/Ñ/°×0BÑ0BÀ1Õ0EÀrÈ1ÓM×PÑPÔW\×WbÑWbÐkl×ksÑksÐPÓtð 	ð !-¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆä¨¸U×CÖCØ&Õ>×IÑIÈ!ÈQÓOˆEÜ—)’)˜U˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆC÷	 Dð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷ D×Cús   Ã(A4FÆF,	)rZ   r)   r\   rV   r^   rW   rR   r9   )r>   r?   r@   rA   r   r(   ÚstaticmethodrY   rd   Úno_gradr   r;   rB   rC   rD   rE   s   @@r5   rH   rH   D   su   ù‡ € Ù�X vÔ.÷Oô Oó /ðOð  Ù�X vÔ.÷5ñ 5ó /ó ð5ð$ ‡]‚]ƒ_Øñ<ó ó ÷<ð <r7   rH   c                óÌ   € V RRV P                   R,          ^,          13,          pV RV P                   R,          ^,          R13,          p\        P                  ! V) V3RR7      # )z*Rotates half the hidden dims of the input..Nrz   r|   )r~   rd   rƒ   )r:   Úx1Úx2s   &  r5   Úrotate_halfr‘   }   sZ   € à	
ˆ3Ð"�!—'‘'˜"•+ Õ"Ð"Ð"Õ	#€BØ	
ˆ3�—‘˜•˜qÕ Ñ"Ð"Õ	#€BÜ�9Š9�r�c˜2�Y BÔ'Ð'r7   Úrotary_pos_embc                óÈ   € VP                  V4      pVP                  V4      pW,          \        V 4      V,          ,           pW,          \        V4      V,          ,           pWV3# )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)Ú	unsqueezer‘   )ÚqÚkr„   r…   Úunsqueeze_dimÚq_embedÚk_embeds   &&&&&  r5   Úapply_rotary_pos_embrš   „   sS   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�wœ; q›>¨CÕ/Õ0€GØ�wœ; q›>¨CÕ/Õ0€GØÐÐr7   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )rM   Úhidden_statesÚn_reprb   )rd   re   Úint)rN   s   "r5   rO   rO   ž   s.   € ÷ 	Uñ 	UœUŸ\™\ð 	U´#ð 	U¼%¿,¹,ñ 	Ur7   c                ó˜   € V P                   w  r#rEV^8X  d   V # V R,          P                  W#WV4      p V P                  W#V,          WE4      # )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
)r{   r{   Nr{   r{   )r~   r}   Úreshape)rœ   r�   ÚbatchÚnum_key_value_headsÚslenri   s   &&    r5   Ú	repeat_kvr¤   ž   sU   € ð
 2?×1DÑ1DÑ.€E Ø�„zØÐØ!Ð"2Õ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÕ(CÀTÓTÐTr7   c                ó   € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R\        R	\
        \        ,          /# )
rM   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNÚscalingÚdropoutrp   )r   ÚModulerd   re   rf   r   r   )rN   s   "r5   rO   rO   ª   sw   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ð%ô ð%ô Ô'Õ(ñ%r7   c                 ó"  € \        W P                  4      p\        W0P                  4      p	\        P                  ! WP	                  ^^4      4      V,          p
Ve	   W¤,           p
\
        P                  P                  V
R\        P                  R7      P                  VP                  4      p
\
        P                  P                  W¦V P                  R7      p
\        P                  ! W©4      pVP	                  ^^4      P                  4       pWº3# )rM   ©rr   rk   )ÚpÚtrainingr|   )r¤   Únum_key_value_groupsrd   Úmatmulr‚   r   Ú
functionalÚsoftmaxÚfloat32ro   rk   r¬   r±   Ú
contiguous)r¦   r§   r¨   r©   rª   r«   r¬   rp   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs   &&&&&&&,    r5   Úeager_attention_forwardr¼   ª   sÒ   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÕL€LØÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r7   c                 óX   € R R\         P                  ! RV ,          4      ,          ,
          # )gš™™™™™é?g333333ã?g333333Ó¿)ÚmathÚexp)Ú	layer_idxs   &r5   Úlambda_init_fnrÁ   Ã   s    € Ø�”t—x’x  yÕ 0Ó1Õ1Õ1Ð1r7   c                   ó\   a a€ ] tR t^Çt oRtRV3R lV 3R llltRV3R lR lltRtVtV ;t	# )	ÚDiffLlamaAttentionuç  Multi-headed differential attention (https://huggingface.co/papers/2410.05258).

Computes ``(softmax(Q1 K1áµ€) - Î» Â· softmax(Q2 K2áµ€)) Â· V`` as two standard attention calls
sharing Q and K over the two halves of V. The two-call structure is ~30% faster than the
V-doubling shortcut at production shapes, since asymmetric V (``head_dim_v != head_dim_q``)
forces SDPA off Flash/cuDNN onto the memory-efficient/math kernel; Flash Attention 2 also
requires ``head_dim_v == head_dim_q``.
c                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# )rM   r)   rÀ   N©r    rž   )rN   rF   s   "€r5   rO   ÚDiffLlamaAttention.__annotate__Ò   s#   ø€ ÷ +jñ +j™ð +j¹3À½:ñ +jr7   c                ó"  <€ \         SV `  4        Wn        W n        \	        VR VP
                  VP                  ,          4      V n        VP                  VP                  ,          V n	        V P                  R,          V n
        VP                  V n        RV n        \        P                  ! VP
                  VP                  V P                  ,          VP                  R7      V n        \        P                  ! VP
                  VP                  V P                  ,          VP                  R7      V n        \        P                  ! VP
                  VP                  V P                  ,          VP                  R7      V n        \        P                  ! VP                  V P                  ,          VP
                  VP                  R7      V n        VP                  R8”  d   \)        R4      hVP                  e   VP                  ^,          ^ 8w  d   \)        RVP                   R24      h\+        V4      V n        \        P.                  ! \0        P2                  ! ^ VP4                  V P                  3R7      4      V n        \        P.                  ! \0        P2                  ! ^ VP4                  V P                  3R7      4      V n        \        P.                  ! \0        P2                  ! ^ VP4                  V P                  3R7      4      V n        \        P.                  ! \0        P2                  ! ^ VP4                  V P                  3R7      4      V n        \        P>                  ! ^V P                  ,          VP@                  R	R
7      V n!        R# )ri   Tr%   ç        z€DiffLlama does not support `attention_dropout > 0`: the differential attention mechanism has no paper-defined dropout semantics.Nz–DiffLlama requires `num_key_value_heads` to be even (and at least 2): the two-call differential attention splits the value tensor along KV heads, got Ú.)ÚsizeF)ÚepsÚelementwise_affineg      à¿)"r'   r(   r)   rÀ   rl   r*   rm   ri   r¢   r²   r«   Úattention_dropoutÚ	is_causalr   r,   Úattention_biasÚq_projÚk_projÚv_projÚo_projÚ
ValueErrorrÁ   Úlambda_initÚ	Parameterrd   ÚnormalÚlambda_std_devÚ	lambda_q1Ú	lambda_k1Ú	lambda_q2Ú	lambda_k2ÚRMSNormÚrms_norm_epsÚ	groupnorm©r3   r)   rÀ   r4   s   &&&€r5   r(   ÚDiffLlamaAttention.__init__Ò   sœ  ø€ Ü‰ÑÔØŒØ"ŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÕ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Õ$\ˆÔ!Ø—}‘} dÕ*ˆŒØ!'×!9Ñ!9ˆÔØˆŒä—i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×&Ñ&¨¯©Õ6¸×8JÑ8JÐQW×QfÑQfô
ˆŒð ×#Ñ# cÔ)ÜðDóð ð ×%Ñ%Ò-°×1KÑ1KÈaÕ1OÐSTÔ1TÜðVØV\×VpÑVpÐUqÐqrðtóð ô *¨)Ó4ˆÔÜŸš¤e§l¢l°1°f×6KÑ6KÐSW×S`ÑS`ÐRbÔ&cÓdˆŒÜŸš¤e§l¢l°1°f×6KÑ6KÐSW×S`ÑS`ÐRbÔ&cÓdˆŒÜŸš¤e§l¢l°1°f×6KÑ6KÐSW×S`ÑS`ÐRbÔ&cÓdˆŒÜŸš¤e§l¢l°1°f×6KÑ6KÐSW×S`ÑS`ÐRbÔ&cÓdˆŒÜŸš A¨¯©Õ$5¸6×;NÑ;NÐchÔiˆŽr7   c                óø   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          RS[ P                  R,          RS[R,          RS[S[ P                  S[ P                  3,          /# )rM   rœ   Úposition_embeddingsrª   NÚpast_key_valuesrb   )rd   re   rc   r   )rN   rF   s   "€r5   rO   rÆ   ÿ   sy   ø€ ÷ C)ñ C)á—|‘|ðC)ñ #¡5§<¡<±·±Ð#=Õ>ðC)ñ Ÿ™ tÕ+ð	C)ñ
  �ðC)ñ 
‰u�|‰|™UŸ\™\Ð)Õ	*ñC)r7   c                ó
  € VP                   R R p. VORNV P                  N5pV P                  V4      P                  V4      P	                  ^^4      pV P                  V4      P                  V4      P	                  ^^4      p	V P                  V4      P                  V4      P	                  ^^4      p
Vw  r¼\        W‰W¼4      w  r‰Ve   VP                  WšV P                  4      w  ršR \        P                  ! V
^^R7       4       w  rÞ\        P                  ! V P                  P                  \         4      pV! V VV	VV3RRRV P"                  /VB w  ppV! V VV	VV3RRRV P"                  /VB w  pp\        P$                  ! VV.RR7      p\        P                  ! V^^R7      w  pp\        P&                  ! \        P(                  ! V P*                  V P,                  ,          R\        P.                  R7      4      P1                  VP2                  4      p\        P&                  ! \        P(                  ! V P4                  V P6                  ,          R\        P.                  R7      4      P1                  VP2                  4      pVV,
          V P8                  ,           pVVV,          ,
          p^V P8                  ,
          V P;                  V4      ,          pVP<                  ! . VORN5!  pV P?                  V4      pVV3# )Nc              3   óH   "  € T F  qP                  ^^^^4      x € K  	  R# 5i)é   N)Úrepeat)Ú.0Úvs   & r5   Ú	<genexpr>Ú-DiffLlamaAttention.forward.<locals>.<genexpr>  s"   é € Ð'jÑFiÀ¯©°°A°q¸!×(<Ð(<ÓFiùs   ‚ "rz   r¬   rÈ   r«   r¯   r|   ) r~   ri   rÐ   Úviewr‚   rÑ   rÒ   rš   ÚupdaterÀ   rd   Úchunkr   Úget_interfacer)   Ú_attn_implementationr¼   r«   rƒ   r¿   ÚsumrÙ   rÚ   r¶   ro   rk   rÛ   rÜ   rÕ   rß   r    rÓ   )r3   rœ   rã   rª   rä   rp   Úinput_shapeÚhidden_shapeÚquery_statesr¸   r¹   r„   r…   Úvalue_states1Úvalue_states2Úattention_interfaceÚattn_output1rº   Úattn_output2Ú_r»   Úlambda_1Úlambda_2Úlambda_fulls   &&&&&,                  r5   r;   ÚDiffLlamaAttention.forwardÿ   sÍ  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJñ (kÄeÇkÂkÐR^Ð`aÐghÕFiÓ'jÑ$ˆä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ &9ØØØØØñ	&
ð ð	&
ð —L‘Lð	&
ð ñ	&
Ñ"ˆ�lñ .ØØØØØñ	
ð ð	
ð —L‘Lð	
ð ñ	
‰ˆ�aô —i’i ¨|Ð <À"ÔEˆô &+§[¢[°¸aÀQÔ%GÑ"ˆ�lÜ—9’9œUŸYšY t§~¡~¸¿¹Õ'FÈBÔV[×VcÑVcÔdÓe×hÑhØ×Ñó
ˆô —9’9œUŸYšY t§~¡~¸¿¹Õ'FÈBÔV[×VcÑVcÔdÓe×hÑhØ×Ñó
ˆð  Õ)¨D×,<Ñ,<Õ<ˆØ" [°<Õ%?Õ?ˆØ˜4×+Ñ+Õ+¨t¯~©~¸kÓ/JÕJˆØ!×)Ò)Ð;¨;Ð;¸Ó;ˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r7   )rÍ   r)   rß   ri   rÎ   rÑ   rÕ   rÚ   rÜ   rÙ   rÛ   rÀ   r²   rÓ   rÐ   r«   rÒ   r9   )NN)
r>   r?   r@   rA   Ú__doc__r(   r;   rB   rC   rD   rE   s   @@r5   rÃ   rÃ   Ç   s)   ù‡ € ñ÷+jõ +j÷ZC)÷ C)ò C)r7   rÃ   rÝ   c                   óZ   a a€ ] tR tRt oRV3R lV 3R llltV3R lR ltR tRtVtV ;t	# )	ÚDiffLlamaRMSNormiE  c                ó$   <€ V ^8„  d   QhRS[ RR/# )rM   rË   rb   N)rf   )rN   rF   s   "€r5   rO   ÚDiffLlamaRMSNorm.__annotate__G  s   ø€ ÷ $ñ $©ð $¸$ñ $r7   c                óŽ   <€ \         SV `  4        \        P                  ! \        P
                  ! V4      4      V n        W n        R# )z/
DiffLlamaRMSNorm is equivalent to T5LayerNorm
N)r'   r(   r   rÖ   rd   ÚonesÚweightÚvariance_epsilon)r3   r*   rË   r4   s   &&&€r5   r(   ÚDiffLlamaRMSNorm.__init__G  s/   ø€ ô 	‰ÑÔÜ—l’l¤5§:¢:¨kÓ#:Ó;ˆŒØ #Ör7   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )rM   rœ   rb   )rd   re   )rN   rF   s   "€r5   rO   r  O  s#   ø€ ÷ ;ñ ;¡U§\¡\ð ;±e·l±lñ ;r7   c                ó:  € VP                   pVP                  \        P                  4      pVP	                  ^4      P                  RRR7      pV\        P                  ! W0P                  ,           4      ,          pV P                  VP                  V4      ,          # )rM   T)Úkeepdimr|   )	rk   ro   rd   r¶   ÚpowÚmeanÚrsqrtr  r  )r3   rœ   Úinput_dtypeÚvariances   &&  r5   r;   ÚDiffLlamaRMSNorm.forwardO  sw   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯ª°H×?TÑ?TÕ4TÓ(UÕUˆØ�{‰{˜]×-Ñ-¨kÓ:Õ:Ð:r7   c                ó^   € \        V P                  P                  4       R V P                   2# )z, eps=)rc   r  r~   r  )r3   s   &r5   Ú
extra_reprÚDiffLlamaRMSNorm.extra_reprV  s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr7   )r  r  )g�íµ ÷Æ°>)
r>   r?   r@   rA   r(   r;   r  rB   rC   rD   rE   s   @@r5   r  r  E  s%   ù‡ € ÷$õ $÷;ð ;÷Jò Jr7   r  c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚDiffLlamaDecoderLayeriZ  c                ó&   <€ V ^8„  d   QhRS[ RS[/# )rM   r)   rÀ   rÅ   )rN   rF   s   "€r5   rO   Ú"DiffLlamaDecoderLayer.__annotate__[  s   ø€ ÷ fñ f™ð f¹3ñ fr7   c                ó   <€ \         SV `  4        VP                  V n        \        WR 7      V n        \        V4      V n        \        VP                  VP                  R7      V n	        \        VP                  VP                  R7      V n
        R# ))r)   rÀ   ©rË   N)r'   r(   r*   rÃ   Ú	self_attnr"   Úmlpr  rÞ   Úinput_layernormÚpost_attention_layernormrà   s   &&&€r5   r(   ÚDiffLlamaDecoderLayer.__init__[  sk   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔä+°6ÔOˆŒä Ó'ˆŒÜ/°×0BÑ0BÈ×H[ÑH[Ô\ˆÔÜ(8¸×9KÑ9KÐQW×QdÑQdÔ(eˆÖ%r7   c                ó.  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[ P                  S[ P                  3,          R,          RS[S[,          R	S[ P                  /# )
rM   rœ   rª   Nr†   rä   Ú	use_cacherã   rp   rb   )rd   re   Ú
LongTensorr   Úboolrc   r   r   )rN   rF   s   "€r5   rO   r  e  sœ   ø€ ÷ ñ á—|‘|ðñ Ÿ™ tÕ+ðñ ×&Ñ&¨Õ-ð	ñ
  �ðñ ˜$•;ðñ #¡5§<¡<±·±Ð#=Õ>ÀÕEðñ Ñ+Õ,ðñ 
�‰ñr7   c                óÔ   € TpV P                  V4      pV P                  ! RR VRVRVRVRVRV/VB w  rW�,           pTpV P                  V4      pV P                  V4      pW�,           pV# )rœ   rª   r†   rä   r"  rã   © )r  r  r  r  )
r3   rœ   rª   r†   rä   r"  rã   rp   Úresidualrû   s
   &&&&&&&,  r5   r;   ÚDiffLlamaDecoderLayer.forwarde  s©   € ð !ˆØ×,Ñ,¨]Ó;ˆàŸ>š>ñ 
Ø'ð
à)ð
ð &ð
ð ,ð	
ð
  ð
ð !4ð
ð ñ
Ñˆð !Õ0ˆð !ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ Õ0ˆØÐr7   )r*   r  r  r  r  )NNNFNr=   rE   s   @@r5   r  r  Z  s    ù‡ € ÷fó f÷÷ ò r7   r  c                   óš   a a€ ] tR tRt oRtRtR.tR.tRtRt	Rt
RtRtR]R]/t]P"                  ! 4       V 3R l4       tV3R	 ltR
tVtV ;t# )ÚDiffLlamaPreTrainedModeli…  ÚmodelTr  rä   rœ   Ú
attentionsc                ó  <€ \         SV `  V4       \        V\        4      '       dÛ   \        P
                  ! VP                  ^ V P                  P                  4       \        P
                  ! VP                  ^ V P                  P                  4       \        P
                  ! VP                  ^ V P                  P                  4       \        P
                  ! VP                  ^ V P                  P                  4       R# R# )r   N)r'   Ú_init_weightsr   rÃ   ÚinitÚnormal_rÙ   r)   rØ   rÚ   rÛ   rÜ   )r3   r¦   r4   s   &&€r5   r.  Ú&DiffLlamaPreTrainedModel._init_weights—  s£   ø€ ä‰Ñ˜fÔ%Ü�fÔ0×1Ò1Ü�LŠL˜×)Ñ)¨1¨d¯k©k×.HÑ.HÔIÜ�LŠL˜×)Ñ)¨1¨d¯k©k×.HÑ.HÔIÜ�LŠL˜×)Ñ)¨1¨d¯k©k×.HÑ.HÔIÜ�LŠL˜×)Ñ)¨1¨d¯k©k×.HÑ.HÖIñ	 2r7   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # rL   r   )rN   rF   s   "€r5   rO   Ú%DiffLlamaPreTrainedModel.__annotate__…  s   ø‡ ‚ áÑò r7   r&  )r>   r?   r@   rA   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendr  rÃ   Ú_can_record_outputsrd   r�   r.  Ú__annotate_func__rB   rC   rD   rE   s   @@r5   r*  r*  …  sy   ù‡ € ð  ÐØ&*Ð#Ø0Ð1ÐØ#4Ð"5ÐØÐØ€NØÐà!ÐØ"&ÐàÐ.ØÐ(ðÐð
 ‡]‚]ƒ_ôJó ðJ÷' … r7   r*  c                   ór   a a€ ] tR tRt oV3R lV 3R llt]]]RV3R lR ll4       4       4       tRt	Vt
V ;t# )ÚDiffLlamaModeli¡  c                ó    <€ V ^8„  d   QhRS[ /# rL   r   )rN   rF   s   "€r5   rO   ÚDiffLlamaModel.__annotate__£  s   ø€ ÷ ñ ™ñ r7   c           	     ó  <€ \         SV `  V4       VP                  V n        VP                  V n        \
        P                  ! VP                  VP                  V P                  4      V n        \
        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n        \        VP                  VP                  R 7      V n        \#        VR7      V n        RV n        V P)                  4        R# u upi )r  ©r)   FN)r'   r(   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingr*   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersr  Úlayersr  rÞ   ÚnormrH   Ú
rotary_embÚgradient_checkpointingÚ	post_initrà   s   && €r5   r(   ÚDiffLlamaModel.__init__£  sÊ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸLšL¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m’mÜGLÈV×MeÑMeÔGfÓgÑGf¸)Ô" 6Ö5ÑGfÑgó
ˆŒô % V×%7Ñ%7¸V×=PÑ=PÔQˆŒ	Ü2¸&ÔAˆŒØ&+ˆÔ#ð 	�‰Öùò hs   ÂDc                óþ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[S[,          R	S[/# )
rM   Ú	input_idsNrª   r†   rä   Úinputs_embedsr"  rp   rb   )	rd   r#  re   r   ÚFloatTensorr$  r   r   r   )rN   rF   s   "€r5   rO   rB  ¶  s“   ø€ ÷ 2
ñ 2
á×#Ñ# dÕ*ð2
ñ Ÿ™ tÕ+ð2
ñ ×&Ñ&¨Õ-ð	2
ñ
  �ð2
ñ ×(Ñ(¨4Õ/ð2
ñ ˜$•;ð2
ñ Ñ+Õ,ð2
ñ 
!ñ2
r7   c                óˆ  € VR J VR J,          '       d   \        R4      hVf   V P                  V4      pV'       d   Vf   \        V P                  R7      pVfb   Ve   VP	                  4       M^ p\
        P                  ! VP                  ^,          VP                  R7      V,           pVP                  ^ 4      p\        V P                  VVVVR7      p	Tp
V P                  W£R7      pV P                  R V P                  P                    F  pV! V
3RV	RVRVR	VR
V/VB p
K  	  V P                  V
4      p
\        V
VR7      # )Nz:You must specify exactly one of input_ids or inputs_embedsrD  )rI   )r)   rU  rª   rä   r†   )r†   rª   rã   r†   rä   r"  )Úlast_hidden_staterä   )rÔ   rI  r   r)   Úget_seq_lengthrd   rn   r~   rI   r”   r   rO  rM  rL  rN  r   )r3   rT  rª   r†   rä   rU  r"  rp   Úpast_seen_tokensÚcausal_maskrœ   rã   Údecoder_layers   &&&&&&&,     r5   r;   ÚDiffLlamaModel.forward³  se  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ Ø*.×*;Ñ*;¸IÓ*FˆMç˜Ò0Ü*°$·+±+Ô>ˆOàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLä(Ø—;‘;Ø'Ø)Ø+Ø%ô
ˆð &ˆØ"Ÿo™o¨m˜oÓWÐà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÓIˆMÙ)Øñà*ðð %8ðð *ð	ð
 !0ðð $ðð ñŠMñ Jð Ÿ	™	 -Ó0ˆÜ&Ø+Ø+ô
ð 	
r7   )rI  rP  rM  rN  rF  rO  rG  )NNNNNN)r>   r?   r@   rA   r(   r   r   r   r;   rB   rC   rD   rE   s   @@r5   r@  r@  ¡  s=   ù‡ € ÷ó ð   ØØ÷2
ñ 2
ó ó ó  ÷2
ð 2
r7   r@  c                   ó„   a a€ ] tR tRt oRR/tRR/tRR.R.3/tRR/tV 3R	 lt]	]
RV3R
 lR ll4       4       tRtVtV ;t# )ÚDiffLlamaForCausalLMië  zlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputrœ   ÚlogitsÚkeep_full_weightc                óì   <€ \         SV `  V4       \        V4      V n        VP                  V n        \
        P                  ! VP                  VP                  R R7      V n        V P                  4        R# r$   )
r'   r(   r@  r+  rG  r   r,   r*   r`  rQ  r2   s   &&€r5   r(   ÚDiffLlamaForCausalLM.__init__ò  sU   ø€ Ü‰Ñ˜Ô Ü# FÓ+ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒð 	�‰Ör7   c                óP  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[S[ P                  ,          R
S[S[,          RS[	/
# )rM   rT  Nrª   r†   rä   rU  Úlabelsr"  Úlogits_to_keeprp   rb   )
rd   r#  re   r   rV  r$  rž   r   r   r   )rN   rF   s   "€r5   rO   Ú!DiffLlamaForCausalLM.__annotate__ý  s»   ø€ ÷ 6
ñ 6
á×#Ñ# dÕ*ð6
ñ Ÿ™ tÕ+ð6
ñ ×&Ñ&¨Õ-ð	6
ñ
  �ð6
ñ ×(Ñ(¨4Õ/ð6
ñ × Ñ  4Õ'ð6
ñ ˜$•;ð6
ñ ™eŸl™lÕ*ð6
ñ Ñ+Õ,ð6
ñ 
 ñ6
r7   c	                ó–  € V P                   ! RRVRVRVRVRVRV/V	B p
V
P                  p\        V\        4      '       d   \	        V) R4      MTpV P                  VRVR3,          4      pRpVe-   V P                  ! RR	VR
VRV P                  P                  /V	B p\        VVV
P                  V
P                  V
P                  R7      # )a,  
Example:

```python
>>> from transformers import AutoTokenizer, DiffLlamaForCausalLM

>>> model = DiffLlamaForCausalLM.from_pretrained("google/diffllama-7b")
>>> tokenizer = AutoTokenizer.from_pretrained("google/diffllama-7b")

>>> prompt = "What is your favorite condiment?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"What is your favorite condiment?"
```rT  rª   r†   rä   rU  r"  Nr{   rb  rg  rG  )Úlossrb  rä   rœ   r,  r&  )r+  rX  r   rž   Úslicer`  Úloss_functionr)   rG  r   rä   rœ   r,  )r3   rT  rª   r†   rä   rU  rg  r"  rh  rp   Úoutputsrœ   Úslice_indicesrb  rk  s   &&&&&&&&&,     r5   r;   ÚDiffLlamaForCausalLM.forwardû  sÿ   € ð> ,0¯:ª:ñ ,
Øð,
à)ð,
ð &ð,
ð ,ð	,
ð
 (ð,
ð  ð,
ð ñ,
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒØ×%Ò%Ñp¨VÐp¸FÐpÈtÏ{É{×OeÑOeÐpÐioÑpˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r7   )r`  r+  rG  )NNNNNNNr   )r>   r?   r@   rA   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr(   r   r   r;   rB   rC   rD   rE   s   @@r5   r_  r_  ë  sf   ù‡ € à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÐ/Ð0€Jõð Ø÷6
ñ 6
ó ó ÷6
ð 6
r7   r_  c                   ó   € ] tR tRtRtR# )Ú"DiffLlamaForSequenceClassificationi6  r&  N©r>   r?   r@   rA   rB   r&  r7   r5   rv  rv  6  ó   † Ûr7   rv  c                   ó   € ] tR tRtRtRtR# )ÚDiffLlamaForQuestionAnsweringi:  Útransformerr&  N)r>   r?   r@   rA   r4  rB   r&  r7   r5   rz  rz  :  s   † Ø%Ôr7   rz  c                   ó   € ] tR tRtRtR# )ÚDiffLlamaForTokenClassificationi>  r&  Nrw  r&  r7   r5   r}  r}  >  rx  r7   r}  )rç   )rÈ   )r*  r@  r_  rv  rz  r}  )Cr¾   Úcollections.abcr   rd   r   Ú r   r/  Úactivationsr   Úcache_utilsr   r   Ú
generationr	   Úintegrationsr
   r   Úmasking_utilsr   Úmodeling_layersr   r   r   r   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   Úconfiguration_diffllamar    r­   r"   rH   r‘   rš   r¤   r¼   rÁ   rÃ   r  r  r*  r@  r_  rv  rz  r}  Ú__all__r&  r7   r5   Ú<module>r�     s•  ðó. Ý $ã Ý å &Ý !ß .Ý )ß LÝ /÷ó ÷ Pß Kß FÝ &ß IÑ IÝ 0ß GÝ 5Ý 4ô�2—9‘9ô ô 6<˜rŸy™yô 6<òr(ñ Ð-Ó.óó /ðõ2	U÷%ò22ñ Ð)Ó*ôz)˜Ÿ™ó z)ó +ðz)ñz ˜YÓ'ôJ�r—y‘yó Jó (ðJô((Ð6ô (ðV ôJ˜ó Jó ðJð6 ôF
Ð-ó F
ó ðF
ðR ôG
Ð3°_ó G
ó ðG
ôT	Ð)IÐKcô 	ô&Ð$?ÐAYô &ô	Ð&CÐE]ô 	ò‚r7   