+
    œ°³jè]  ã                   óø  € ^ RI Ht ^ RIt^ RIHt ^RIHt ^RIH	t	 ^RI
HtHt ^RIHt ^RIHtHt ^RIHtHt ^R	IHt ^R
IHtHtHt ^RIHtHt ^RIHtH t  ^RI!H"t"H#t# ^RI$H%t% ^RI&H't'H(t(H)t) ^RI*H+t+ ^RI,H-t-H.t. ^RI/H0t0 ^RI1H2t2  ! R R]Pf                  4      t4 ! R R]Pf                  4      t5 ! R R]Pf                  4      t6R t7]! R4      R1R l4       t8R R lt9R2R R  llt:]! ]84       ! R! R"]Pf                  4      4       t; ! R# R$]4      t< ! R% R&]Pz                  4      t>]( ! R' R(]#4      4       t?]( ! R) R*]?4      4       t@]( ! R+ R,]?]4      4       tA ! R- R.]]?4      tB ! R/ R0]]?4      tC. R3OtDR# )4é    )ÚCallableN)Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_kernel_forward_from_hubÚuse_kernelized_func)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚFlashAttentionKwargs)Ú GenericForSequenceClassificationÚGenericForTokenClassificationÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputs©ÚGemma2Configc                   óT   a a€ ] tR t^1t oRV3R lV 3R llltR tR tR tRtVt	V ;t
# )ÚGemma2RMSNormc                ó&   <€ V ^8„  d   QhRS[ RS[/# )é   ÚdimÚeps©ÚintÚfloat)ÚformatÚ__classdict__s   "€Úk/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/gemma2/modeling_gemma2.pyÚ__annotate__ÚGemma2RMSNorm.__annotate__2   s   ø€ ÷ 5ñ 5™Cð 5¡eñ 5ó    c                óŽ   <€ \         SV `  4        W n        \        P                  ! \
        P                  ! V4      4      V n        R # ©N)ÚsuperÚ__init__r&   ÚnnÚ	ParameterÚtorchÚzerosÚweight)Úselfr%   r&   Ú	__class__s   &&&€r,   r3   ÚGemma2RMSNorm.__init__2   s,   ø€ Ü‰ÑÔØŒÜ—l’l¤5§;¢;¨sÓ#3Ó4ˆŽr/   c                óž   € V\         P                  ! VP                  ^4      P                  RRR7      V P                  ,           4      ,          # )r$   T)Úkeepdiméÿÿÿÿ)r6   ÚrsqrtÚpowÚmeanr&   )r9   Úxs   &&r,   Ú_normÚGemma2RMSNorm._norm7   s4   € Ø”5—;’;˜qŸu™u Q›xŸ}™}¨R¸˜}Ó>ÀÇÁÕIÓJÕJÐJr/   c                ó²   € V P                  VP                  4       4      pVR V P                  P                  4       ,           ,          pVP                  V4      # ©ç      ð?)rC   r)   r8   Útype_as)r9   rB   Úoutputs   && r,   ÚforwardÚGemma2RMSNorm.forward:   sC   € Ø—‘˜AŸG™G›IÓ&ˆð ˜3 §¡×!2Ñ!2Ó!4Õ4Õ5ˆØ�~‰~˜aÓ Ð r/   c                ó^   € \        V P                  P                  4       R V P                   2# )z, eps=)Útupler8   Úshaper&   )r9   s   &r,   Ú
extra_reprÚGemma2RMSNorm.extra_reprA   s'   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$·(±(°Ð<Ð<r/   )r&   r8   )g�íµ ÷Æ°>)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r3   rC   rJ   rO   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r:   r+   s   @@r,   r"   r"   1   s$   ù‡ € ÷5õ 5ò
Kò!÷=ò =r/   r"   c                   ó8   a a€ ] tR t^Et oV 3R ltR tRtVtV ;t# )Ú	Gemma2MLPc                óØ  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        \
        P                  ! V P                  V P                  R R7      V n        \
        P                  ! V P                  V P                  R R7      V n        \
        P                  ! V P                  V P                  R R7      V n	        \        VP                  ,          V n        R# ©F©ÚbiasN)r2   r3   ÚconfigÚhidden_sizeÚintermediate_sizer4   ÚLinearÚ	gate_projÚup_projÚ	down_projr   Úhidden_activationÚact_fn©r9   r_   r:   s   &&€r,   r3   ÚGemma2MLP.__init__F   s¤   ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔÜŸš 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆŒÜ˜V×5Ñ5Õ6ˆŽr/   c                ó�   € V P                  V P                  V P                  V4      4      V P                  V4      ,          4      pV# r1   )re   rg   rc   rd   )r9   rB   re   s   && r,   rJ   ÚGemma2MLP.forwardP   s6   € Ø—N‘N 4§;¡;¨t¯~©~¸aÓ/@Ó#AÀDÇLÁLÐQRÃOÕ#SÓTˆ	ØÐr/   )rg   r_   re   rc   r`   ra   rd   ©	rQ   rR   rS   rT   r3   rJ   rU   rV   rW   rX   s   @@r,   rZ   rZ   E   s   ù‡ € õ7÷ò r/   rZ   c                   óÎ   a a€ ] tR t^Ut o]! RRR7      R
V3R lV 3R lll4       t]]! RRR7      R
V3R lR ll4       4       t]P                  ! 4       ]
R 4       4       tR	tVtV ;t# )ÚGemma2RotaryEmbeddingÚdevicez5.18)Úversionc                ó    <€ V ^8„  d   QhRS[ /# ©r$   r_   r   )r*   r+   s   "€r,   r-   Ú"Gemma2RotaryEmbedding.__annotate__W   s   ø€ ÷ Oñ O™|ñ Or/   c                óä  <€ \         SV `  4        VP                  V n        VP                  V n        Wn        V P
                  P                  R ,          V n        V P                  pV P                  R8w  d   \        V P                  ,          pV! V P
                  V4      w  q@n
        \        P                  ! VRR7      V n        \        P                  ! VP                  4       RR7      V n        R# )Ú	rope_typeÚdefaultF©Ú
persistentN)r2   r3   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr_   Úrope_parametersru   Úcompute_default_rope_parametersr   Úattention_scalingr4   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)r9   r_   ro   Úrope_init_fnr€   r:   s   &&&  €r,   r3   ÚGemma2RotaryEmbedding.__init__V   s¬   ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àŒàŸ™×4Ñ4°[ÕAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÔ&Ü.¨t¯~©~Õ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÔ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÔ!NˆÖr/   c                óP   <€ V ^8„  d   QhRS[ RS[S[P                  S[3,          /# )r$   r_   Úreturn)r    rM   r6   ÚTensorr)   )r*   r+   s   "€r,   r-   rs   i   s.   ø€ ÷ 5ñ 5±ð 5ÑX]Ñ^c×^jÑ^jÑlqÐ^qÕXrñ 5r/   c           	     ó0  € V P                   R,          p\        V RR4      ;'       g    V P                  V P                  ,          pRpRV\        P
                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚhead_dimNrG   ©Údtype)r|   Úgetattrr`   Únum_attention_headsr6   Úaranger)   Úto)r_   ro   ÚkwargsÚbaser%   Úattention_factorr€   s   &&,    r,   r}   Ú5Gemma2RotaryEmbedding.compute_default_rope_parametersg   s~   € ð ×%Ñ% lÕ3ˆÜ�f˜j¨$Ó/×cÐc°6×3EÑ3EÈ×IcÑIcÕ3cˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r/   c                ó”  € V P                   R,          P                  4       P                  VP                  ^ ,          R^4      P	                  VP
                  4      pVR	,          P                  4       p\        VP
                  P                  \        4      '       d2   VP
                  P                  R8w  d   VP
                  P                  MRp\        VRR7      ;_uu_ 4        VP                  4       VP                  4       ,          P                  ^^4      p\        P                  ! Wf3RR7      pVP                  4       V P                  ,          pVP                  4       V P                  ,          p	R R R 4       XP	                  VP                   R7      X	P	                  VP                   R7      3#   + '       g   i     LH; i)
NÚmpsÚcpuF)Údevice_typeÚenabled©r%   r‹   )NºNNNNr>   )r›   Nr›   )r€   r)   ÚexpandrN   r�   ro   Ú
isinstanceÚtypeÚstrr   Ú	transposer6   ÚcatÚcosr~   ÚsinrŒ   )
r9   rB   Úposition_idsÚinv_freq_expandedÚposition_ids_expandedr˜   ÚfreqsÚembr¢   r£   s
   &&&       r,   rJ   ÚGemma2RotaryEmbedding.forward{   sE  € ð !ŸM™M¨-Õ8×>Ñ>Ó@×GÑGÈ×HZÑHZÐ[\ÕH]Ð_aÐcdÓe×hÑhÐij×iqÑiqÓrÐØ ,¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆÜ¨¸U×CÖCØ&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÕN×YÑYÐZ[Ð]^Ó_ˆEÜ—)’)˜U˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆC÷	 Dð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷ D×Cús   Ã&BF7Æ7G	)r~   r_   r€   rz   r‚   r{   ru   r1   )rQ   rR   rS   rT   r   r3   Ústaticmethodr}   r6   Úno_gradr   rJ   rU   rV   rW   rX   s   @@r,   rn   rn   U   su   ù‡ € Ù�X vÔ.÷Oô Oó /ðOð  Ù�X vÔ.÷5ñ 5ó /ó ð5ð$ ‡]‚]ƒ_Øñ<ó ó ÷<ð <r/   rn   c                óÌ   € V RRV P                   R,          ^,          13,          pV RV P                   R,          ^,          R13,          p\        P                  ! V) V3RR7      # )z*Rotates half the hidden dims of the input..Nrš   r>   )rN   r6   r¡   )rB   Úx1Úx2s   &  r,   Úrotate_halfr¯   ‹   sZ   € à	
ˆ3Ð"�!—'‘'˜"•+ Õ"Ð"Ð"Õ	#€BØ	
ˆ3�—‘˜•˜qÕ Ñ"Ð"Õ	#€BÜ�9Š9�r�c˜2�Y BÔ'Ð'r/   Úrotary_pos_embc                óÈ   € VP                  V4      pVP                  V4      pW,          \        V 4      V,          ,           pW,          \        V4      V,          ,           pWV3# )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)Ú	unsqueezer¯   )ÚqÚkr¢   r£   Úunsqueeze_dimÚq_embedÚk_embeds   &&&&&  r,   Úapply_rotary_pos_embr¸   ’   sS   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�wœ; q›>¨CÕ/Õ0€GØ�wœ; q›>¨CÕ/Õ0€GØÐÐr/   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r$   Úhidden_statesÚn_repr†   )r6   r‡   r(   )r*   s   "r,   r-   r-   ¬   s.   € ÷ 	Uñ 	UœUŸ\™\ð 	U´#ð 	U¼%¿,¹,ñ 	Ur/   c                ó˜   € V P                   w  r#rEV^8X  d   V # V R,          P                  W#WV4      p V P                  W#V,          WE4      # )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
)r›   r›   Nr›   r›   )rN   rœ   Úreshape)rº   r»   ÚbatchÚnum_key_value_headsÚslenrŠ   s   &&    r,   Ú	repeat_kvrÁ   ¬   sU   € ð
 2?×1DÑ1DÑ.€E Ø�„zØÐØ!Ð"2Õ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÕ(CÀTÓTÐTr/   c                ór  € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        \
        ,          R\        R,          R	\        R,          R
\        \        P                  \        P                  3,          /	# )r$   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNÚdropoutÚscalingÚsoftcapr†   )r4   ÚModuler6   r‡   r)   r(   rM   )r*   s   "r,   r-   r-   ¸   s™   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ”S�[ð%ô �T�\ð%ô �T�\ð%ô Œ5�<‰<œŸ™Ð%Õ&ñ%r/   c                 ó¤  € Vf   V P                   R,          p\        W P                  4      p	\        W0P                  4      p
\        P                  ! WP                  ^^4      4      V,          pVe'   W·,          p\        P                  ! V4      pW·,          pVe	   W´,           p\        P                  P                  VR\        P                  R7      P                  VP                  4      p\        P                  P                  WµV P                  R7      p\        P                  ! Wº4      pVP                  ^^4      P                  4       pWË3# )N)r%   rŒ   )ÚpÚtrainingç      à¿r>   )rŠ   rÁ   Únum_key_value_groupsr6   Úmatmulr    Útanhr4   Ú
functionalÚsoftmaxÚfloat32r�   rŒ   rÈ   rÎ   Ú
contiguous)rÃ   rÄ   rÅ   rÆ   rÇ   rÈ   rÉ   rÊ   r‘   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs   &&&&&&&&,    r,   Úeager_attention_forwardrÛ   ¸   s  € ð ‚Ø—/‘/ 4Õ'ˆä˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÕL€LàÒØ#Õ-ˆÜ—z’z ,Ó/ˆØ#Õ-ˆØÒ!Ø#Õ4ˆô —=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KØÐ$Ð$r/   c                   óX   a a€ ] tR t^Út oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )ÚGemma2Attentionz=Multi-headed attention from 'Attention Is All You Need' paperc                ó&   <€ V ^8„  d   QhRS[ RS[/# ©r$   r_   Ú	layer_idx©r    r(   )r*   r+   s   "€r,   r-   ÚGemma2Attention.__annotate__Þ   s   ø€ ÷ hñ h™|ð h¹ñ hr/   c                óº  <€ \         SV `  4        \        VR 4      '       d   VP                  V,          MRV n        Wn        W n        \        VRVP                  VP                  ,          4      V n
        VP                  VP                  ,          V n        VP                  R,          V n        V P
                  P                  V n        \        VRR4      '       * V n        \"        P$                  ! VP                  VP                  V P                  ,          VP&                  R7      V n        \"        P$                  ! VP                  VP                  V P                  ,          VP&                  R7      V n        \"        P$                  ! VP                  VP                  V P                  ,          VP&                  R7      V n        \"        P$                  ! VP                  V P                  ,          VP                  VP&                  R7      V n        V P
                  P0                  V n        V P                  R8X  d   VP2                  V n        R# RV n        R# )Úlayer_typesNrŠ   Úuse_bidirectional_attentionFr]   Úsliding_attentionrÏ   )r2   r3   Úhasattrrä   Ú
layer_typer_   rà   r�   r`   rŽ   rŠ   r¿   rÐ   Úquery_pre_attn_scalarrÉ   Úattention_dropoutÚ	is_causalr4   rb   Úattention_biasÚq_projÚk_projÚv_projÚo_projÚattn_logit_softcappingÚsliding_window©r9   r_   rà   r:   s   &&&€r,   r3   ÚGemma2Attention.__init__Þ   s³  ø€ Ü‰ÑÔÜ;BÀ6È=×;YÒ;Y˜&×,Ñ,¨YÖ7Ð_cˆŒØŒØ"ŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÕ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Õ$\ˆÔ!Ø×3Ñ3°TÕ9ˆŒØ!%§¡×!>Ñ!>ˆÔÜ$ VÐ-JÈEÓRÔRˆŒä—i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×&Ñ&¨¯©Õ6¸×8JÑ8JÐQW×QfÑQfô
ˆŒð '+§k¡k×&HÑ&HˆÔ#Ø7;·±ÐJ]Ô7]˜f×3Ñ3ˆÖÐcgˆÖr/   c                ó`  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          R,          RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  S[ P                  R,          S[S[ P                  ,          R,          3,          /# )r$   rº   Úposition_embeddingsNrÇ   Úpast_key_valuesr‘   r†   )r6   r‡   rM   r   r   r   )r*   r+   s   "€r,   r-   râ   ø   sœ   ø€ ÷ ()ñ ()á—|‘|ð()ñ #¡5§<¡<±·±Ð#=Õ>ÀÕEð()ñ Ÿ™ tÕ+ð	()ñ
  �ð()ñ Ñ-Õ.ð()ñ 
‰u�|‰|™UŸ\™\¨DÕ0±%¹¿¹Õ2EÈÕ2LÐLÕ	Mñ()r/   c                ó@  € VP                   R R p. VORNV P                  N5pV P                  V4      P                  V4      P	                  ^^4      pV P                  V4      P                  V4      P	                  ^^4      p	V P                  V4      P                  V4      P	                  ^^4      p
Vw  r¼\        W‰W¼4      w  r‰Ve   VP                  WšV P                  4      w  rš\        P                  ! V P                  P                  \        4      pT! V VV	V
V3RV P                  '       d   V P                   MRRV P"                  RV P$                  RV P&                  /VB w  rïVP(                  ! . VORN5!  P+                  4       pV P-                  V4      pWï3# )NrÈ   ç        rÉ   rò   rÊ   r>   )rN   rŠ   rí   Úviewr    rî   rï   r¸   Úupdaterà   r   Úget_interfacer_   Ú_attn_implementationrÛ   rÎ   rê   rÉ   rò   rñ   r½   rÖ   rð   )r9   rº   rö   rÇ   r÷   r‘   Úinput_shapeÚhidden_shapeÚquery_statesr×   rØ   r¢   r£   Úattention_interfacerÚ   rÙ   s   &&&&&,          r,   rJ   ÚGemma2Attention.forwardø   s§  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ%
ð /3¯m¯m¨m�D×*Ò*Àð%
ð —L‘Lð%
ð  ×.Ñ.ð%
ð ×/Ñ/ð%
ð ñ%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—k‘k +Ó.ˆØÐ(Ð(r/   )rê   rñ   r_   rŠ   rë   rî   rà   rè   rÐ   rð   rí   rÉ   rò   rï   )NNN©
rQ   rR   rS   rT   Ú__doc__r3   rJ   rU   rV   rW   rX   s   @@r,   rÝ   rÝ   Ú   s#   ù‡ € áG÷hó h÷4()÷ ()ò ()r/   rÝ   c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚGemma2DecoderLayeri#  c                ó&   <€ V ^8„  d   QhRS[ RS[/# rß   rá   )r*   r+   s   "€r,   r-   ÚGemma2DecoderLayer.__annotate__$  s   ø€ ÷ 
eñ 
e™|ð 
e¹ñ 
er/   c                óÄ  <€ \         SV `  4        VP                  V n        Wn        \	        WR 7      V n        \        V4      V n        \        VP                  VP                  R7      V n
        \        VP                  VP                  R7      V n        \        VP                  VP                  R7      V n        \        VP                  VP                  R7      V n        R# ))r_   rà   ©r&   N)r2   r3   r`   r_   rÝ   Ú	self_attnrZ   Úmlpr"   Úrms_norm_epsÚinput_layernormÚpost_attention_layernormÚpre_feedforward_layernormÚpost_feedforward_layernormró   s   &&&€r,   r3   ÚGemma2DecoderLayer.__init__$  s¦   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØŒÜ(°ÔLˆŒÜ˜VÓ$ˆŒÜ,¨V×-?Ñ-?ÀV×EXÑEXÔYˆÔÜ(5°f×6HÑ6HÈf×NaÑNaÔ(bˆÔ%ä)6°v×7IÑ7IÈv×ObÑObÔ)cˆÔ&Ü*7¸×8JÑ8JÐPV×PcÑPcÔ*dˆÖ'r/   c                ó  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  /# )r$   rº   rö   NrÇ   r¤   r÷   r†   )r6   r‡   rM   Ú
LongTensorr   )r*   r+   s   "€r,   r-   r  0  s~   ø€ ÷ ñ á—|‘|ðñ #¡5§<¡<±·±Ð#=Õ>ÀÕEðñ Ÿ™ tÕ+ð	ñ
 ×&Ñ&¨Õ-ðñ  �ðñ 
�‰ñr/   c                ó  € TpV P                  V4      pV P                  ! RR VRVRVRVRV/VB w  rV P                  V4      pWq,           pTpV P                  V4      pV P	                  V4      pV P                  V4      pWq,           pV# )rº   rö   rÇ   r¤   r÷   © )r  r  r  r  r  r  )	r9   rº   rö   rÇ   r¤   r÷   r‘   ÚresidualÚ_s	   &&&&&&,  r,   rJ   ÚGemma2DecoderLayer.forward0  s¿   € ð !ˆà×,Ñ,¨]Ó;ˆð  Ÿ>š>ñ 
Ø'ð
à 3ð
ð *ð
ð &ð	
ð
 ,ð
ð ñ
Ñˆð ×5Ñ5°mÓDˆØ Õ0ˆà ˆØ×6Ñ6°}ÓEˆØŸ™ Ó/ˆØ×7Ñ7¸ÓFˆØ Õ0ˆàÐr/   )r_   r`   r  r  r  r  r  r  )NNNNrl   rX   s   @@r,   r  r  #  s    ù‡ € ÷
eó 
e÷÷ ò r/   r  c                   ó^   a a€ ] tR tRt oRtRV3R lV 3R llltV3R lV 3R lltRtVtV ;t	# )	ÚGemma2TextScaledWordEmbeddingiR  zT
This module overrides nn.Embeddings' forward by multiplying with embeddings scale.
c                ó2   <€ V ^8„  d   QhRS[ RS[ RS[ RS[/# )r$   Únum_embeddingsÚembedding_dimÚpadding_idxÚembed_scaler'   )r*   r+   s   "€r,   r-   Ú*Gemma2TextScaledWordEmbedding.__annotate__W  s0   ø€ ÷ Rñ R¡sð R¹3ð RÉSð RÑ_dñ Rr/   c                ó–   <€ \         SV `  WV4       W@n        \        P                  ! \
        P                  ! V4      R R7      V n        R# )Frw   N)r2   r3   Úscalar_embed_scaler4   r   r6   Útensorr   )r9   r  r  r  r   r:   s   &&&&&€r,   r3   Ú&Gemma2TextScaledWordEmbedding.__init__W  s4   ø€ Ü‰Ñ˜¸ÔDØ"-ÔÜŸ9š9¤U§\¢\°+Ó%>È5ÔQˆÖr/   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r$   Ú	input_ids)r6   r‡   )r*   r+   s   "€r,   r-   r!  \  s   ø€ ÷ Sñ S¡§¡ñ Sr/   c                óŠ   <€ \         SV `  V4      V P                  P                  V P                  P
                  4      ,          # r1   )r2   rJ   r   r�   r8   rŒ   )r9   r'  r:   s   &&€r,   rJ   Ú%Gemma2TextScaledWordEmbedding.forward\  s2   ø€ Ü‰w‰˜yÓ)¨D×,<Ñ,<×,?Ñ,?ÀÇÁ×@QÑ@QÓ,RÕRÐRr/   )r   r#  rF   r  rX   s   @@r,   r  r  R  s(   ù‡ € ñ÷Rõ R÷
S÷ Só Sr/   r  c                   óš   a a€ ] tR tRt oRtRtR.tR.tRtRt	Rt
RtRtR]R]/t]P"                  ! 4       V 3R l4       tV3R	 ltR
tVtV ;t# )ÚGemma2PreTrainedModeli`  ÚmodelTr  r÷   rº   Ú
attentionsc                ó&  <€ \         SV `  V4       R VP                  P                  9   d#   \        P
                  ! VP                  4       R# \        V\        4      '       d.   \        P                  ! VP                  VP                  4       R# R# )ÚRMSNormN)r2   Ú_init_weightsr:   rQ   ÚinitÚzeros_r8   r�   r  Ú	constant_r   r#  )r9   rÃ   r:   s   &&€r,   r0  Ú#Gemma2PreTrainedModel._init_weightsr  sa   ø€ ä‰Ñ˜fÔ%à˜×(Ñ(×1Ñ1Ô1Ü�KŠK˜Ÿ™Ö&Ü˜Ô =×>Ò>Ü�NŠN˜6×-Ñ-¨v×/HÑ/HÖIñ ?r/   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # rr   r   )r*   r+   s   "€r,   r-   Ú"Gemma2PreTrainedModel.__annotate__`  s   ø‡ ‚ áÑò r/   r  )rQ   rR   rS   rT   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendr  rÝ   Ú_can_record_outputsr6   r«   r0  Ú__annotate_func__rU   rV   rW   rX   s   @@r,   r+  r+  `  sx   ù‡ € ð  ÐØ&*Ð#Ø-Ð.ÐØ#4Ð"5ÐØÐØ€NØÐà!ÐØ"&ÐàÐ+Ø�oðÐð
 ‡]‚]ƒ_ôJó ðJ÷' … r/   r+  c                   ór   a a€ ] tR tRt oV3R lV 3R llt]]]RV3R lR ll4       4       4       tRt	Vt
V ;t# )ÚGemma2Modeli|  c                ó    <€ V ^8„  d   QhRS[ /# rr   r   )r*   r+   s   "€r,   r-   ÚGemma2Model.__annotate__~  s   ø€ ÷ ñ ™|ñ r/   c           	     ó6  <€ \         SV `  V4       VP                  V n        VP                  V n        \        VP                  VP                  V P                  V P                  P                  R ,          R7      V n        \        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n        \        VP                  VP                   R7      V n        \%        V4      V n        RV n        V P+                  4        R# u upi )g      à?)r   r
  FN)r2   r3   Úpad_token_idr  Ú
vocab_sizer  r`   r_   Úembed_tokensr4   Ú
ModuleListÚrangeÚnum_hidden_layersr  Úlayersr"   r  Únormrn   Ú
rotary_embÚgradient_checkpointingÚ	post_initró   s   && €r,   r3   ÚGemma2Model.__init__~  sß   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒä9Ø×Ñ˜v×1Ñ1°4×3CÑ3CÐQU×Q\ÑQ\×QhÑQhÐjmÕQmô
ˆÔô —m’mÜDIÈ&×JbÑJbÔDcÓdÑDc°yÔ Ö2ÑDcÑdó
ˆŒô " &×"4Ñ"4¸&×:MÑ:MÔNˆŒ	Ü/°Ó7ˆŒØ&+ˆÔ#ð 	�‰Öùò es   Â'Dc                óþ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[S[,          R	S[/# )
r$   r'  NrÇ   r¤   r÷   Úinputs_embedsÚ	use_cacher‘   r†   )	r6   r  r‡   r   ÚFloatTensorÚboolr   r   r   )r*   r+   s   "€r,   r-   rE  “  s“   ø€ ÷ ;
ñ ;
á×#Ñ# dÕ*ð;
ñ Ÿ™ tÕ+ð;
ñ ×&Ñ&¨Õ-ð	;
ñ
  �ð;
ñ ×(Ñ(¨4Õ/ð;
ñ ˜$•;ð;
ñ Ñ+Õ,ð;
ñ 
!ñ;
r/   c                ó2  € VR J VR J,          '       d   \        R4      hVf   V P                  V4      pV'       d   Vf   \        V P                  R7      pVfb   Ve   VP	                  4       M^ p\
        P                  ! VP                  ^,          VP                  R7      V,           pVP                  ^ 4      p\        T;p	\        4      '       g/   RV P                  RVRVRVRV/p
R	\        R/ V
B R
\        R/ V
B /p	TpV P                  W³4      p\        V P                   R V P                  P"                   4       F7  w  rÞV! V3RW�P                  P$                  V,          ,          RVRVRV/VB pK9  	  V P'                  V4      p\)        VVR7      # )Nz:You must specify exactly one of input_ids or inputs_embeds)r_   )ro   r_   rT  rÇ   r÷   r¤   Úfull_attentionræ   rö   )Úlast_hidden_stater÷   r  )Ú
ValueErrorrI  r   r_   Úget_seq_lengthr6   r�   rN   ro   r²   r�   Údictr   r   rO  Ú	enumeraterM  rL  rä   rN  r   )r9   r'  rÇ   r¤   r÷   rT  rU  r‘   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsrº   rö   ÚiÚdecoder_layers   &&&&&&&,       r,   rJ   ÚGemma2Model.forward�  s·  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ Ø*.×*;Ñ*;¸IÓ*FˆMç˜Ò0Ü*°$·+±+Ô>ˆOàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLô °Ð?Ð-Ä×FÒFð ˜$Ÿ+™+Ø Ø  .Ø! ?Ø ðˆKð !Ô"4Ñ"C°{Ñ"CØ#Ô%FÑ%UÈÑ%Uð#Ðð &ˆØ"Ÿo™o¨mÓJÐä )¨$¯+©+Ð6U¸¿¹×8UÑ8UÐ*VÖ WÑˆAÙ)Øñà2·;±;×3JÑ3JÈ1Õ3MÕNðð %8ðð *ð	ð
 !0ðð ñŠMñ !Xð Ÿ	™	 -Ó0ˆä&Ø+Ø+ô
ð 	
r/   )rI  rP  rM  rN  r  rO  rH  )NNNNNN)rQ   rR   rS   rT   r3   r   r   r   rJ   rU   rV   rW   rX   s   @@r,   rC  rC  |  s=   ù‡ € ÷ó ð$  ØØ÷;
ñ ;
ó ó ó  ÷;
ð ;
r/   rC  c                   ó„   a a€ ] tR tRt oRR/tRR/tRR.R.3/tRR/tV 3R	 lt]	]
RV3R
 lR ll4       4       tRtVtV ;t# )ÚGemma2ForCausalLMiÑ  zlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputrº   ÚlogitsÚkeep_full_weightc                óì   <€ \         SV `  V4       \        V4      V n        VP                  V n        \
        P                  ! VP                  VP                  R R7      V n        V P                  4        R# r\   )
r2   r3   rC  r,  rH  r4   rb   r`   rg  rQ  rh   s   &&€r,   r3   ÚGemma2ForCausalLM.__init__Ø  sU   ø€ Ü‰Ñ˜Ô Ü  Ó(ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒð 	�‰Ör/   c                óP  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[S[ P                  ,          R
S[S[,          RS[	/
# )r$   r'  NrÇ   r¤   r÷   rT  ÚlabelsrU  Úlogits_to_keepr‘   r†   )
r6   r  r‡   r   rV  rW  r(   r   r   r   )r*   r+   s   "€r,   r-   ÚGemma2ForCausalLM.__annotate__ã  s»   ø€ ÷ ;
ñ ;
á×#Ñ# dÕ*ð;
ñ Ÿ™ tÕ+ð;
ñ ×&Ñ&¨Õ-ð	;
ñ
  �ð;
ñ ×(Ñ(¨4Õ/ð;
ñ × Ñ  4Õ'ð;
ñ ˜$•;ð;
ñ ™eŸl™lÕ*ð;
ñ Ñ+Õ,ð;
ñ 
 ñ;
r/   c	                óF  € V P                   ! R
RVRVRVRVRVRV/V	B p
V
P                  p\        V\        4      '       d   \	        V) R4      MTpV P                  VRVR3,          4      pV P                  P                  eO   WÐP                  P                  ,          p\        P                  ! V4      pWÐP                  P                  ,          pRpVe   V P                  ! WÖV P                  3/ V	B p\        VVV
P                  V
P                  V
P                  R	7      # )a"  
Example:

```python
>>> from transformers import AutoTokenizer, Gemma2ForCausalLM

>>> model = Gemma2ForCausalLM.from_pretrained("google/gemma-2-9b")
>>> tokenizer = AutoTokenizer.from_pretrained("google/gemma-2-9b")

>>> prompt = "What is your favorite condiment?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"What is your favorite condiment?"
```r'  rÇ   r¤   r÷   rT  rU  Nr›   )Úlossri  r÷   rº   r-  r  )r,  rZ  r�   r(   Úslicerg  r_   Úfinal_logit_softcappingr6   rÒ   Úloss_functionrH  r   r÷   rº   r-  )r9   r'  rÇ   r¤   r÷   rT  rn  rU  ro  r‘   Úoutputsrº   Úslice_indicesri  rr  s   &&&&&&&&&,     r,   rJ   ÚGemma2ForCausalLM.forwardá  s0  € ð@ ,0¯:ª:ñ ,
Øð,
à)ð,
ð &ð,
ð ,ð	,
ð
 (ð,
ð  ð,
ð ñ,
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆØ�;‰;×.Ñ.Ò:ØŸk™k×AÑAÕAˆFÜ—Z’Z Ó'ˆFØŸk™k×AÑAÕAˆFàˆØÒØ×%Ò% f°d·o±oÑPÈÑPˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r/   )rg  r,  rH  )NNNNNNNr   )rQ   rR   rS   rT   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr3   r   r   rJ   rU   rV   rW   rX   s   @@r,   rf  rf  Ñ  sf   ù‡ € à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÐ/Ð0€Jõð Ø÷;
ñ ;
ó ó ÷;
ð ;
r/   rf  c                   ó   € ] tR tRtRtR# )ÚGemma2ForSequenceClassificationi!  r  N©rQ   rR   rS   rT   rU   r  r/   r,   r~  r~  !  ó   † Ûr/   r~  c                   ó   € ] tR tRtRtR# )ÚGemma2ForTokenClassificationi%  r  Nr  r  r/   r,   r‚  r‚  %  r€  r/   r‚  )é   )rù   NN)rf  rC  r+  r~  r‚  )EÚcollections.abcr   r6   Útorch.nnr4   Ú r   r1  Úactivationsr   Úcache_utilsr   r   Ú
generationr   Úintegrationsr	   r
   Úmasking_utilsr   r   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   r   r   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   Úconfiguration_gemma2r    rË   r"   rZ   rn   r¯   r¸   rÁ   rÛ   rÝ   r  Ú	Embeddingr  r+  rC  rf  r~  r‚  Ú__all__r  r/   r,   Ú<module>r™     s|  ðõ* %ã Ý å &Ý !ß .Ý )ß Lß RÝ B÷ñ ÷
 Pß Kß FÝ &ß IÑ IÝ 0ß GÝ 5Ý .ô=�B—I‘Iô =ô(�—	‘	ô ô 3<˜BŸI™Iô 3<òl(ñ Ð-Ó.óó /ðõ2	U÷%ñD Ð)Ó*ôE)�b—i‘ió E)ó +ðE)ôP,Ð3ô ,ô^S B§L¡Lô Sð ôJ˜Oó Jó ðJð6 ôQ
Ð'ó Q
ó ðQ
ðh ôL
Ð-¨ó L
ó ðL
ô^	Ð&FÐH]ô 	ô	Ð#@ÐBWô 	ò‚r/   