+
    œ°³jñV  ã                   óè  € ^ RI Ht ^ RIt^ RIHt ^RIHt ^RIHt ^RI	H
t
Ht ^RIHt ^RIHtHt ^R	IHt ^R
IHtHtHt ^RIHtHt ^RIHtHt ^RIHtHt ^RI H!t! ^RI"H#t#H$t$H%t% ^RI&H't' ^RI(H)t)H*t* ^RI+H,t, ^RI-H.t.  ! R R]P^                  4      t0 ! R R]Pb                  4      t2 ! R R]Pb                  4      t3 ! R R]Pb                  4      t4R t5]! R4      R1R l4       t6R R  lt7R2R! R" llt8]! ]64       ! R# R$]Pb                  4      4       t9 ! R% R&]4      t:]$ ! R' R(]4      4       t;]$ ! R) R*];4      4       t<]$ ! R+ R,];]4      4       t= ! R- R.]];4      t> ! R/ R0]];4      t?. R3Ot@R# )4é    )ÚCallableN)Únn)Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_kernel_forward_from_hubÚuse_kernelized_func)Úcreate_causal_mask)Ú GenericForSequenceClassificationÚGenericForTokenClassificationÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputs©ÚGemmaConfigc                   ó^   a a€ ] tR t^2t oRtRV3R lV 3R llltV3R lV 3R lltRtVtV ;t	# )ÚGemmaTextScaledWordEmbeddingzT
This module overrides nn.Embeddings' forward by multiplying with embeddings scale.
c                ó2   <€ V ^8„  d   QhRS[ RS[ RS[ RS[/# )é   Únum_embeddingsÚembedding_dimÚpadding_idxÚembed_scale©ÚintÚfloat)ÚformatÚ__classdict__s   "€Úi/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/gemma/modeling_gemma.pyÚ__annotate__Ú)GemmaTextScaledWordEmbedding.__annotate__7   s0   ø€ ÷ Rñ R¡sð R¹3ð RÉSð RÑ_dñ Ró    c                ó–   <€ \         SV `  WV4       W@n        \        P                  ! \
        P                  ! V4      R R7      V n        R# )F©Ú
persistentN)ÚsuperÚ__init__Úscalar_embed_scaler   ÚBufferÚtorchÚtensorr'   )Úselfr$   r%   r&   r'   Ú	__class__s   &&&&&€r-   r5   Ú%GemmaTextScaledWordEmbedding.__init__7   s4   ø€ Ü‰Ñ˜¸ÔDØ"-ÔÜŸ9š9¤U§\¢\°+Ó%>È5ÔQˆÖr0   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r#   Ú	input_ids)r8   ÚTensor)r+   r,   s   "€r-   r.   r/   <   s   ø€ ÷ Sñ S¡§¡ñ Sr0   c                óŠ   <€ \         SV `  V4      V P                  P                  V P                  P
                  4      ,          # ©N)r4   Úforwardr'   ÚtoÚweightÚdtype)r:   r>   r;   s   &&€r-   rB   Ú$GemmaTextScaledWordEmbedding.forward<   s2   ø€ Ü‰w‰˜yÓ)¨D×,<Ñ,<×,?Ñ,?ÀÇÁ×@QÑ@QÓ,RÕRÐRr0   )r'   r6   ©ç      ð?©
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r5   rB   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r;   r,   s   @@r-   r!   r!   2   s(   ù‡ € ñ÷Rõ R÷
S÷ Só Sr0   r!   c                   óT   a a€ ] tR t^@t oRV3R lV 3R llltR tR tR tRtVt	V ;t
# )ÚGemmaRMSNormc                ó&   <€ V ^8„  d   QhRS[ RS[/# )r#   ÚdimÚepsr(   )r+   r,   s   "€r-   r.   ÚGemmaRMSNorm.__annotate__A   s   ø€ ÷ 5ñ 5™Cð 5¡eñ 5r0   c                óŽ   <€ \         SV `  4        W n        \        P                  ! \
        P                  ! V4      4      V n        R # rA   )r4   r5   rW   r   Ú	Parameterr8   ÚzerosrD   )r:   rV   rW   r;   s   &&&€r-   r5   ÚGemmaRMSNorm.__init__A   s,   ø€ Ü‰ÑÔØŒÜ—l’l¤5§;¢;¨sÓ#3Ó4ˆŽr0   c                óž   € V\         P                  ! VP                  ^4      P                  RRR7      V P                  ,           4      ,          # )r#   T)Úkeepdiméÿÿÿÿ)r8   ÚrsqrtÚpowÚmeanrW   )r:   Úxs   &&r-   Ú_normÚGemmaRMSNorm._normF   s4   € Ø”5—;’;˜qŸu™u Q›xŸ}™}¨R¸˜}Ó>ÀÇÁÕIÓJÕJÐJr0   c                ó²   € V P                  VP                  4       4      pVR V P                  P                  4       ,           ,          pVP                  V4      # rG   )rd   r*   rD   Útype_as)r:   rc   Úoutputs   && r-   rB   ÚGemmaRMSNorm.forwardI   sC   € Ø—‘˜AŸG™G›IÓ&ˆð ˜3 §¡×!2Ñ!2Ó!4Õ4Õ5ˆØ�~‰~˜aÓ Ð r0   c                ó^   € \        V P                  P                  4       R V P                   2# )z, eps=)ÚtuplerD   ÚshaperW   )r:   s   &r-   Ú
extra_reprÚGemmaRMSNorm.extra_reprP   s'   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$·(±(°Ð<Ð<r0   )rW   rD   )g�íµ ÷Æ°>)rJ   rK   rL   rM   r5   rd   rB   rm   rO   rP   rQ   rR   s   @@r-   rT   rT   @   s$   ù‡ € ÷5õ 5ò
Kò!÷=ò =r0   rT   c                   ó8   a a€ ] tR t^Tt oV 3R ltR tRtVtV ;t# )ÚGemmaMLPc                óØ  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        \
        P                  ! V P                  V P                  R R7      V n        \
        P                  ! V P                  V P                  R R7      V n        \
        P                  ! V P                  V P                  R R7      V n	        \        VP                  ,          V n        R# ©F©ÚbiasN)r4   r5   ÚconfigÚhidden_sizeÚintermediate_sizer   ÚLinearÚ	gate_projÚup_projÚ	down_projr   Ú
hidden_actÚact_fn©r:   ru   r;   s   &&€r-   r5   ÚGemmaMLP.__init__U   s¤   ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔÜŸš 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆŒÜ˜V×.Ñ.Õ/ˆŽr0   c                ó�   € V P                  V P                  V P                  V4      4      V P                  V4      ,          4      pV# rA   )r{   r}   ry   rz   )r:   rc   r{   s   && r-   rB   ÚGemmaMLP.forward_   s6   € Ø—N‘N 4§;¡;¨t¯~©~¸aÓ/@Ó#AÀDÇLÁLÐQRÃOÕ#SÓTˆ	ØÐr0   )r}   ru   r{   ry   rv   rw   rz   ©	rJ   rK   rL   rM   r5   rB   rO   rP   rQ   rR   s   @@r-   rp   rp   T   s   ù‡ € õ0÷ò r0   rp   c                   óÎ   a a€ ] tR t^dt o]! RRR7      R
V3R lV 3R lll4       t]]! RRR7      R
V3R lR ll4       4       t]P                  ! 4       ]
R 4       4       tR	tVtV ;t# )ÚGemmaRotaryEmbeddingÚdevicez5.18)Úversionc                ó    <€ V ^8„  d   QhRS[ /# ©r#   ru   r   )r+   r,   s   "€r-   r.   Ú!GemmaRotaryEmbedding.__annotate__f   s   ø€ ÷ Oñ O™{ñ Or0   c                óä  <€ \         SV `  4        VP                  V n        VP                  V n        Wn        V P
                  P                  R ,          V n        V P                  pV P                  R8w  d   \        V P                  ,          pV! V P
                  V4      w  q@n
        \        P                  ! VRR7      V n        \        P                  ! VP                  4       RR7      V n        R# )Ú	rope_typeÚdefaultFr2   N)r4   r5   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenru   Úrope_parametersr‹   Úcompute_default_rope_parametersr   Úattention_scalingr   r7   Úinv_freqÚcloneÚoriginal_inv_freq)r:   ru   r…   Úrope_init_fnr“   r;   s   &&&  €r-   r5   ÚGemmaRotaryEmbedding.__init__e   s¬   ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àŒàŸ™×4Ñ4°[ÕAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÔ&Ü.¨t¯~©~Õ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÔ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÔ!NˆÖr0   c                óP   <€ V ^8„  d   QhRS[ RS[S[P                  S[3,          /# )r#   ru   Úreturn)r   rk   r8   r?   r*   )r+   r,   s   "€r-   r.   r‰   x   s.   ø€ ÷ 5ñ 5±ð 5ÑW\Ñ]b×]iÑ]iÑkpÐ]pÕWqñ 5r0   c           	     ó0  € V P                   R,          p\        V RR4      ;'       g    V P                  V P                  ,          pRpRV\        P
                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚhead_dimNrH   ©rE   )r�   Úgetattrrv   Únum_attention_headsr8   Úaranger*   rC   )ru   r…   ÚkwargsÚbaserV   Úattention_factorr“   s   &&,    r-   r‘   Ú4GemmaRotaryEmbedding.compute_default_rope_parametersv   s~   € ð ×%Ñ% lÕ3ˆÜ�f˜j¨$Ó/×cÐc°6×3EÑ3EÈ×IcÑIcÕ3cˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r0   c                ó^  € V P                   R,          P                  VP                  ^ ,          R	^4      P                  \        P
                  VP                  R7      pVR
,          P                  4       p\        VP                  P                  \        4      '       d2   VP                  P                  R8w  d   VP                  P                  MRp\        VRR7      ;_uu_ 4        W4,          P                  ^^4      p\        P                  ! Wf3R	R7      pVP                  4       V P                  ,          pVP                  4       V P                  ,          p	R R R 4       XP                  VP                   R7      X	P                  VP                   R7      3#   + '       g   i     LH; i)N)rE   r…   ÚmpsÚcpuF)Údevice_typeÚenabled©rV   r�   )NºNNNNr_   )r«   Nr«   )r“   Úexpandrl   rC   r8   r*   r…   Ú
isinstanceÚtypeÚstrr   Ú	transposeÚcatÚcosr’   ÚsinrE   )
r:   rc   Úposition_idsÚinv_freq_expandedÚposition_ids_expandedr¨   ÚfreqsÚembr²   r³   s
   &&&       r-   rB   ÚGemmaRotaryEmbedding.forwardŠ   s2  € ð �M‰M˜-Õ(×/Ñ/°×0BÑ0BÀ1Õ0EÀrÈ1ÓM×PÑPÔW\×WbÑWbÐkl×ksÑksÐPÓtð 	ð !-¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆä¨¸U×CÖCØ&Õ>×IÑIÈ!ÈQÓOˆEÜ—)’)˜U˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆC÷	 Dð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷ D×Cús   Ã(A4FÆF,	)r’   ru   r“   rŽ   r•   r�   r‹   rA   )rJ   rK   rL   rM   r   r5   Ústaticmethodr‘   r8   Úno_gradr   rB   rO   rP   rQ   rR   s   @@r-   r„   r„   d   su   ù‡ € Ù�X vÔ.÷Oô Oó /ðOð  Ù�X vÔ.÷5ñ 5ó /ó ð5ð$ ‡]‚]ƒ_Øñ<ó ó ÷<ð <r0   r„   c                óÌ   € V RRV P                   R,          ^,          13,          pV RV P                   R,          ^,          R13,          p\        P                  ! V) V3RR7      # )z*Rotates half the hidden dims of the input..Nrª   r_   )rl   r8   r±   )rc   Úx1Úx2s   &  r-   Úrotate_halfr¿   �   sZ   € à	
ˆ3Ð"�!—'‘'˜"•+ Õ"Ð"Ð"Õ	#€BØ	
ˆ3�—‘˜•˜qÕ Ñ"Ð"Õ	#€BÜ�9Š9�r�c˜2�Y BÔ'Ð'r0   Úrotary_pos_embc                óÈ   € VP                  V4      pVP                  V4      pW,          \        V 4      V,          ,           pW,          \        V4      V,          ,           pWV3# )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)Ú	unsqueezer¿   )ÚqÚkr²   r³   Úunsqueeze_dimÚq_embedÚk_embeds   &&&&&  r-   Úapply_rotary_pos_embrÈ   ¤   sS   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�wœ; q›>¨CÕ/Õ0€GØ�wœ; q›>¨CÕ/Õ0€GØÐÐr0   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r#   Úhidden_statesÚn_repr™   )r8   r?   r)   )r+   s   "r-   r.   r.   ¾   s.   € ÷ 	Uñ 	UœUŸ\™\ð 	U´#ð 	U¼%¿,¹,ñ 	Ur0   c                ó˜   € V P                   w  r#rEV^8X  d   V # V R,          P                  W#WV4      p V P                  W#V,          WE4      # )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
)r«   r«   Nr«   r«   )rl   r¬   Úreshape)rÊ   rË   ÚbatchÚnum_key_value_headsÚslenrœ   s   &&    r-   Ú	repeat_kvrÑ   ¾   sU   € ð
 2?×1DÑ1DÑ.€E Ø�„zØÐØ!Ð"2Õ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÕ(CÀTÓTÐTr0   c                ó   € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R\        R	\
        \        ,          /# )
r#   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNÚscalingÚdropoutr¡   )r   ÚModuler8   r?   r*   r   r   )r+   s   "r-   r.   r.   Ê   sw   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ð%ô ð%ô Ô'Õ(ñ%r0   c                 ó"  € \        W P                  4      p\        W0P                  4      p	\        P                  ! WP	                  ^^4      4      V,          p
Ve	   W¤,           p
\
        P                  P                  V
R\        P                  R7      P                  VP                  4      p
\
        P                  P                  W¦V P                  R7      p
\        P                  ! W©4      pVP	                  ^^4      P                  4       pWº3# )r#   )rV   rE   )ÚpÚtrainingr_   )rÑ   Únum_key_value_groupsr8   Úmatmulr°   r   Ú
functionalÚsoftmaxÚfloat32rC   rE   rÙ   rÝ   Ú
contiguous)rÓ   rÔ   rÕ   rÖ   r×   rØ   rÙ   r¡   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs   &&&&&&&,    r-   Úeager_attention_forwardrè   Ê   sÒ   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÕL€LØÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r0   c                   óX   a a€ ] tR t^ãt oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )ÚGemmaAttentionz=Multi-headed attention from 'Attention Is All You Need' paperc                ó&   <€ V ^8„  d   QhRS[ RS[/# ©r#   ru   Ú	layer_idx©r   r)   )r+   r,   s   "€r-   r.   ÚGemmaAttention.__annotate__ç   s   ø€ ÷ 
ñ 
™{ð 
±sñ 
r0   c                óÂ  <€ \         SV `  4        Wn        W n        \	        VR VP
                  VP                  ,          4      V n        VP                  VP                  ,          V n	        V P                  R,          V n
        VP                  V n        \	        VRR4      '       * V n        \        P                  ! VP
                  VP                  V P                  ,          VP                  R7      V n        \        P                  ! VP
                  VP                  V P                  ,          VP                  R7      V n        \        P                  ! VP
                  VP                  V P                  ,          VP                  R7      V n        \        P                  ! VP                  V P                  ,          VP
                  VP                  R7      V n        R# )rœ   Úuse_bidirectional_attentionFrs   Ng      à¿)r4   r5   ru   rí   rž   rv   rŸ   rœ   rÏ   rÞ   rØ   Úattention_dropoutÚ	is_causalr   rx   Úattention_biasÚq_projÚk_projÚv_projÚo_proj©r:   ru   rí   r;   s   &&&€r-   r5   ÚGemmaAttention.__init__ç   sV  ø€ Ü‰ÑÔØŒØ"ŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÕ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Õ$\ˆÔ!Ø—}‘} dÕ*ˆŒØ!'×!9Ñ!9ˆÔÜ$ VÐ-JÈEÓRÔRˆŒä—i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×&Ñ&¨¯©Õ6¸×8JÑ8JÐQW×QfÑQfô
ˆŽr0   c                ó  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          R,          RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  S[ P                  3,          /# )r#   rÊ   Úposition_embeddingsNr×   Úpast_key_valuesr¡   r™   )r8   r?   rk   r   r   r   )r+   r,   s   "€r-   r.   rï   þ   s‡   ø€ ÷ &)ñ &)á—|‘|ð&)ñ #¡5§<¡<±·±Ð#=Õ>ÀÕEð&)ñ Ÿ™ tÕ+ð	&)ñ
  �ð&)ñ Ñ+Õ,ð&)ñ 
‰u�|‰|™UŸ\™\Ð)Õ	*ñ&)r0   c                ó  € VP                   R R p. VORNV P                  N5pV P                  V4      P                  V4      P	                  ^^4      pV P                  V4      P                  V4      P	                  ^^4      p	V P                  V4      P                  V4      P	                  ^^4      p
Vw  r¼\        W‰W¼4      w  r‰Ve   VP                  WšV P                  4      w  rš\        P                  ! V P                  P                  \        4      pT! V VV	V
V3RV P                  '       g   RMV P                   RV P"                  /VB w  rïVP$                  ! . VORN5!  P'                  4       pV P)                  V4      pWï3# )NrÙ   ç        rØ   r_   )rl   rœ   rõ   Úviewr°   rö   r÷   rÈ   Úupdaterí   r   Úget_interfaceru   Ú_attn_implementationrè   rÝ   rò   rØ   rÍ   rã   rø   )r:   rÊ   rü   r×   rý   r¡   Úinput_shapeÚhidden_shapeÚquery_statesrä   rå   r²   r³   Úattention_interfacerç   ræ   s   &&&&&,          r-   rB   ÚGemmaAttention.forwardþ   s‡  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—k‘k +Ó.ˆØÐ(Ð(r0   )rò   ru   rœ   ró   rö   rí   rÞ   rø   rõ   rØ   r÷   )NNNrI   rR   s   @@r-   rê   rê   ã   s!   ù‡ € áG÷
ó 
÷.&)÷ &)ò &)r0   rê   c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚGemmaDecoderLayeri'  c                ó&   <€ V ^8„  d   QhRS[ RS[/# rì   rî   )r+   r,   s   "€r-   r.   ÚGemmaDecoderLayer.__annotate__(  s   ø€ ÷ bñ b™{ð b±sñ br0   c                ó   <€ \         SV `  4        VP                  V n        \        WR 7      V n        \        V4      V n        \        VP                  VP                  R7      V n	        \        VP                  VP                  R7      V n
        R# ))ru   rí   ©rW   N)r4   r5   rv   rê   Ú	self_attnrp   ÚmlprT   Úrms_norm_epsÚinput_layernormÚpost_attention_layernormrù   s   &&&€r-   r5   ÚGemmaDecoderLayer.__init__(  sj   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔä'¨vÔKˆŒä˜FÓ#ˆŒÜ+¨F×,>Ñ,>ÀF×DWÑDWÔXˆÔÜ(4°V×5GÑ5GÈV×M`ÑM`Ô(aˆÖ%r0   c                ó.  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[ P                  S[ P                  3,          R,          RS[S[,          R	S[ P                  /# )
r#   rÊ   r×   Nr´   rý   Ú	use_cacherü   r¡   r™   )r8   r?   Ú
LongTensorr   Úboolrk   r   r   )r+   r,   s   "€r-   r.   r  2  sœ   ø€ ÷ ñ á—|‘|ðñ Ÿ™ tÕ+ðñ ×&Ñ&¨Õ-ð	ñ
  �ðñ ˜$•;ðñ #¡5§<¡<±·±Ð#=Õ>ÀÕEðñ Ñ+Õ,ðñ 
�‰ñr0   c                óÔ   € TpV P                  V4      pV P                  ! RR VRVRVRVRVRV/VB w  rW�,           pTpV P                  V4      pV P                  V4      pW�,           pV# )rÊ   r×   r´   rý   r  rü   © )r  r  r  r  )
r:   rÊ   r×   r´   rý   r  rü   r¡   ÚresidualÚ_s
   &&&&&&&,  r-   rB   ÚGemmaDecoderLayer.forward2  s©   € ð !ˆØ×,Ñ,¨]Ó;ˆàŸ>š>ñ 
Ø'ð
à)ð
ð &ð
ð ,ð	
ð
  ð
ð !4ð
ð ñ
Ñˆð !Õ0ˆð !ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ Õ0ˆØÐr0   )rv   r  r  r  r  )NNNFNr‚   rR   s   @@r-   r
  r
  '  s    ù‡ € ÷bó b÷÷ ò r0   r
  c                   óš   a a€ ] tR tRt oRtRtR.tR.tRtRt	Rt
RtRtR]R]/t]P"                  ! 4       V 3R l4       tV3R	 ltR
tVtV ;t# )ÚGemmaPreTrainedModeliR  ÚmodelTr
  rý   rÊ   Ú
attentionsc                ó&  <€ \         SV `  V4       R VP                  P                  9   d#   \        P
                  ! VP                  4       R# \        V\        4      '       d.   \        P                  ! VP                  VP                  4       R# R# )ÚRMSNormN)r4   Ú_init_weightsr;   rJ   ÚinitÚzeros_rD   r­   r!   Ú	constant_r'   r6   )r:   rÓ   r;   s   &&€r-   r$  Ú"GemmaPreTrainedModel._init_weightsd  sa   ø€ ä‰Ñ˜fÔ%à˜×(Ñ(×1Ñ1Ô1Ü�KŠK˜Ÿ™Ö&Ü˜Ô <×=Ò=Ü�NŠN˜6×-Ñ-¨v×/HÑ/HÖIñ >r0   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # rˆ   r   )r+   r,   s   "€r-   r.   Ú!GemmaPreTrainedModel.__annotate__R  s   ø‡ ‚ áÑò r0   r  )rJ   rK   rL   rM   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendr
  rê   Ú_can_record_outputsr8   r»   r$  Ú__annotate_func__rO   rP   rQ   rR   s   @@r-   r  r  R  sx   ù‡ € ð  ÐØ&*Ð#Ø,Ð-ÐØ#4Ð"5ÐØÐØ€NØÐà!ÐØ"&ÐàÐ*Ø�nðÐð
 ‡]‚]ƒ_ôJó ðJ÷' … r0   r  c                   ór   a a€ ] tR tRt oV3R lV 3R llt]]]RV3R lR ll4       4       4       tRt	Vt
V ;t# )Ú
GemmaModelin  c                ó    <€ V ^8„  d   QhRS[ /# rˆ   r   )r+   r,   s   "€r-   r.   ÚGemmaModel.__annotate__p  s   ø€ ÷ ñ ™{ñ r0   c           	     ó8  <€ \         SV `  V4       VP                  V n        VP                  V n        \        VP                  VP                  V P                  V P                  P                  R ,          R7      V n        \        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n        \        VP                  VP                   R7      V n        \%        VR7      V n        RV n        V P+                  4        R# u upi )g      à?)r'   r  ©ru   FN)r4   r5   Úpad_token_idr&   Ú
vocab_sizer!   rv   ru   Úembed_tokensr   Ú
ModuleListÚrangeÚnum_hidden_layersr
  ÚlayersrT   r  Únormr„   Ú
rotary_embÚgradient_checkpointingÚ	post_initrù   s   && €r-   r5   ÚGemmaModel.__init__p  sß   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒä8Ø×Ñ˜v×1Ñ1°4×3CÑ3CÐQU×Q\ÑQ\×QhÑQhÐjmÕQmô
ˆÔô —m’mÜCHÈ×IaÑIaÔCbÓcÑCb°iÔ˜vÖ1ÑCbÑcó
ˆŒô ! ×!3Ñ!3¸×9LÑ9LÔMˆŒ	Ü.°fÔ=ˆŒØ&+ˆÔ#ð 	�‰Öùò ds   Â'Dc                óþ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[S[,          R	S[/# )
r#   r>   Nr×   r´   rý   Úinputs_embedsr  r¡   r™   )	r8   r  r?   r   ÚFloatTensorr  r   r   r   )r+   r,   s   "€r-   r.   r9  …  s“   ø€ ÷ 2
ñ 2
á×#Ñ# dÕ*ð2
ñ Ÿ™ tÕ+ð2
ñ ×&Ñ&¨Õ-ð	2
ñ
  �ð2
ñ ×(Ñ(¨4Õ/ð2
ñ ˜$•;ð2
ñ Ñ+Õ,ð2
ñ 
!ñ2
r0   c                ó¦  € VR J VR J,          '       d   \        R4      hVf   V P                  V4      pV'       d   Vf   \        V P                  R7      pVfb   Ve   VP	                  4       M^ p\
        P                  ! VP                  ^,          VP                  R7      V,           pVP                  ^ 4      p\        V P                  VVVVR7      p	Tp
V P                  W£R7      pV P                  R V P                  P                    F  pV! V
3RV	RVRVR	VR
V/VB p
K  	  V P                  V
4      p
\        T
V'       d   VR7      # R R7      # )Nz:You must specify exactly one of input_ids or inputs_embedsr;  )r…   )ru   rI  r×   rý   r´   )r´   r×   r´   rý   r  rü   )Úlast_hidden_staterý   )Ú
ValueErrorr>  r   ru   Úget_seq_lengthr8   r    rl   r…   rÂ   r   rD  rB  rA  rC  r   )r:   r>   r×   r´   rý   rI  r  r¡   Úpast_seen_tokensÚcausal_maskrÊ   rü   Údecoder_layers   &&&&&&&,     r-   rB   ÚGemmaModel.forward‚  st  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ Ø ×-Ñ-¨iÓ8ˆMç˜Ò0Ü*°$·+±+Ô>ˆOàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLä(Ø—;‘;Ø'Ø)Ø+Ø%ô
ˆð &ˆØ"Ÿo™o¨m˜oÓWÐà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÓIˆMÙ)Øñà*ðð *ðð !0ð	ð
 $ðð %8ðð ñŠMñ Jð Ÿ	™	 -Ó0ˆÜ&Ø+ß/8˜Oô
ð 	
à>Bô
ð 	
r0   )r>  rE  rB  rC  r&   rD  r=  )NNNNNN)rJ   rK   rL   rM   r5   r   r   r   rB   rO   rP   rQ   rR   s   @@r-   r7  r7  n  s=   ù‡ € ÷ó ð$  ØØ÷2
ñ 2
ó ó ó  ÷2
ð 2
r0   r7  c                   ó„   a a€ ] tR tRt oRR/tRR/tRR.R.3/tRR/tV 3R	 lt]	]
RV3R
 lR ll4       4       tRtVtV ;t# )ÚGemmaForCausalLMiº  zlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputrÊ   ÚlogitsÚkeep_full_weightc                óì   <€ \         SV `  V4       \        V4      V n        VP                  V n        \
        P                  ! VP                  VP                  R R7      V n        V P                  4        R# rr   )
r4   r5   r7  r   r=  r   rx   rv   rU  rF  r~   s   &&€r-   r5   ÚGemmaForCausalLM.__init__Á  sU   ø€ Ü‰Ñ˜Ô Ü Ó'ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒð 	�‰Ör0   c                óP  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[S[ P                  ,          R
S[S[,          RS[	/
# )r#   r>   Nr×   r´   rý   rI  Úlabelsr  Úlogits_to_keepr¡   r™   )
r8   r  r?   r   rJ  r  r)   r   r   r   )r+   r,   s   "€r-   r.   ÚGemmaForCausalLM.__annotate__Ì  s»   ø€ ÷ 6
ñ 6
á×#Ñ# dÕ*ð6
ñ Ÿ™ tÕ+ð6
ñ ×&Ñ&¨Õ-ð	6
ñ
  �ð6
ñ ×(Ñ(¨4Õ/ð6
ñ × Ñ  4Õ'ð6
ñ ˜$•;ð6
ñ ™eŸl™lÕ*ð6
ñ Ñ+Õ,ð6
ñ 
 ñ6
r0   c	                ó–  € V P                   ! RRVRVRVRVRVRV/V	B p
V
P                  p\        V\        4      '       d   \	        V) R4      MTpV P                  VRVR3,          4      pRpVe-   V P                  ! RR	VR
VRV P                  P                  /V	B p\        VVV
P                  V
P                  V
P                  R7      # )a  
Example:

```python
>>> from transformers import AutoTokenizer, GemmaForCausalLM

>>> model = GemmaForCausalLM.from_pretrained("google/gemma-7b")
>>> tokenizer = AutoTokenizer.from_pretrained("google/gemma-7b")

>>> prompt = "What is your favorite condiment?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"What is your favorite condiment?"
```r>   r×   r´   rý   rI  r  Nr«   rW  r\  r=  )ÚlossrW  rý   rÊ   r!  r  )r   rL  r­   r)   ÚslicerU  Úloss_functionru   r=  r   rý   rÊ   r!  )r:   r>   r×   r´   rý   rI  r\  r  r]  r¡   ÚoutputsrÊ   Úslice_indicesrW  r`  s   &&&&&&&&&,     r-   rB   ÚGemmaForCausalLM.forwardÊ  sÿ   € ð> ,0¯:ª:ñ ,
Øð,
à)ð,
ð &ð,
ð ,ð	,
ð
 (ð,
ð  ð,
ð ñ,
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒØ×%Ò%Ñp¨VÐp¸FÐpÈtÏ{É{×OeÑOeÐpÐioÑpˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r0   )rU  r   r=  )NNNNNNNr   )rJ   rK   rL   rM   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr5   r   r   rB   rO   rP   rQ   rR   s   @@r-   rT  rT  º  sf   ù‡ € à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÐ/Ð0€Jõð Ø÷6
ñ 6
ó ó ÷6
ð 6
r0   rT  c                   ó   € ] tR tRtRtR# )ÚGemmaForSequenceClassificationi  r  N©rJ   rK   rL   rM   rO   r  r0   r-   rk  rk    ó   † Ûr0   rk  c                   ó   € ] tR tRtRtR# )ÚGemmaForTokenClassificationi	  r  Nrl  r  r0   r-   ro  ro  	  rm  r0   ro  )é   )rÿ   )r7  rT  rk  ro  r  )AÚcollections.abcr   r8   r   Ú r   r%  Úactivationsr   Úcache_utilsr   r   Ú
generationr	   Úintegrationsr
   r   Úmasking_utilsr   Úmodeling_layersr   r   r   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   Úconfiguration_gemmar   Ú	Embeddingr!   rÚ   rT   rp   r„   r¿   rÈ   rÑ   rè   rê   r
  r  r7  rT  rk  ro  Ú__all__r  r0   r-   Ú<module>r„     sx  ðõ. %ã Ý å &Ý !ß .Ý )ß LÝ /÷ñ ÷
 Pß Kß FÝ &ß IÑ IÝ 0ß GÝ 5Ý ,ôS 2§<¡<ô Sô=�2—9‘9ô =ô(ˆr�y‰yô ô 6<˜2Ÿ9™9ô 6<òr(ñ Ð-Ó.óó /ðõ2	U÷%ñ2 Ð)Ó*ô@)�R—Y‘Yó @)ó +ð@)ôF(Ð2ô (ðV ôJ˜?ó Jó ðJð6 ôH
Ð%ó H
ó ðH
ðV ôG
Ð+¨_ó G
ó ðG
ôT	Ð%EÐG[ô 	ô	Ð"?ÐAUô 	ò‚r0   