+
    œ°³jH  ã                   ón  € ^ RI Ht ^ RIt^ RIHt ^ RIHt ^RIHt ^RI	H
t
Ht ^RIHt ^RIHtHt ^RIHt ^R	IHt ^R
IHtHt ^RIHtHtHt ^RIHt ^RIHt ^RI H!t!H"t"H#t# ^RI$H%t% ^RI&H't' ^RI(H)t)H*t*H+t+H,t,H-t-H.t.H/t/H0t0H1t1H2t2H3t3 ]#Ph                  ! ]54      t6]"! RR7      ] ! R R]4      4       4       t7 ! R R]04      t8 ! R R]-4      t9 ! R R]14      t:R,R R llt; ! R R])4      t< ! R  R!]4      t= ! R" R#]/4      t> ! R$ R%].4      t? ! R& R']*4      t@ ! R( R)]+4      tA ! R* R+],4      tB. R-OtCR# ).é    )ÚCallableN)Ústrict)ÚACT2FN)ÚCacheÚDynamicCache)ÚPreTrainedConfig)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚROPE_INIT_FUNCTIONSÚRopeParametersÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONS)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚlogging)Údeprecate_kwarg)Úmaybe_autocast)ÚGemmaAttentionÚGemmaForCausalLMÚGemmaForSequenceClassificationÚGemmaForTokenClassificationÚGemmaMLPÚ
GemmaModelÚGemmaPreTrainedModelÚGemmaRMSNormÚGemmaRotaryEmbeddingÚapply_rotary_pos_embÚ	repeat_kvzgoogle/gemma2-7b)Ú
checkpointc                   ó   a a€ ] tR t^8t oRtRtR.tRRRRRRRR	R
RRRRR	/tRR.R.3RRR.R.3RR.R.3/tRt	Rt
Rt^t^t^tRtRtRtRtRtRt^ t^t^tRtRtRtRtRtR tRtR!tR"t Rt!V 3R# lt"R$ t#V3R% lt$R&t%Vt&V ;t'# )'ÚGemma2ConfigaX  
query_pre_attn_scalar (`float`, *optional*, defaults to 256):
    scaling factor used on the attention scores
final_logit_softcapping (`float`, *optional*, defaults to 30.0):
    scaling factor when applying tanh softcapping on the logits.
attn_logit_softcapping (`float`, *optional*, defaults to 50.0):
    scaling factor when applying tanh softcapping on the attention scores.
use_bidirectional_attention (`bool`, *optional*):
    If True, the model will attend to all text tokens instead of using a causal mask.

```python
>>> from transformers import Gemma2Model, Gemma2Config
>>> # Initializing a Gemma2 gemma2-7b style configuration
>>> configuration = Gemma2Config()
>>> # Initializing a model from the gemma2-7b style configuration
>>> model = Gemma2Model(configuration)
>>> # Accessing the model configuration
>>> configuration = model.config
```Úgemma2Úpast_key_valueszlayers.*.self_attn.q_projÚcolwisezlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_projÚrowwisezlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projÚembed_tokensÚ	input_idsÚinputs_embedsÚlayersÚhidden_statesÚattention_maskÚnormi è i 	  i $  é   Úgelu_pytorch_tanhi    g{®Gáz”?g�íµ ÷Æ°>TNFç        i   g      >@g      I@c                óä   <€ V P                   fL   \        V P                  4       Uu. uF&  p\        V^,           ^,          4      '       d   RMRNK(  	  upV n         \        SV `  ! R/ VB  R # u upi )NÚsliding_attentionÚfull_attention© )Úlayer_typesÚrangeÚnum_hidden_layersÚboolÚsuperÚ__post_init__)ÚselfÚkwargsÚiÚ	__class__s   &, €Új/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/gemma2/modular_gemma2.pyr>   ÚGemma2Config.__post_init__z   si   ø€ Ø×ÑÒ#äX]Ð^b×^tÑ^tÔXuó ÙXuÐST¤t¨Q°­U°a­K×'8Ò'8Ñ#Ð>NÒNÑXuñ ˆDÔô 	‰ÒÑ' Ô'ùò	 s   §,A-c                ó–   € V P                   V P                  ,          ^ 8w  d'   \        RV P                    RV P                   R24      hR# )zOPart of `@strict`-powered validation. Validates the architecture of the config.zThe hidden size (z6) is not a multiple of the number of attention heads (z).N)Úhidden_sizeÚnum_attention_headsÚ
ValueError)r?   s   &rC   Úvalidate_architectureÚ"Gemma2Config.validate_architecture‚   sS   € à×Ñ˜d×6Ñ6Õ6¸!Ô;ÜØ# D×$4Ñ$4Ð#5ð 6Ø×2Ñ2Ð3°2ð7óð ñ <ó    c                ó"  <€ V ^8„  d   Qh/ S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[;R&   S[ ;R	&   S[;R
&   S[;R&   S[;R&   S[ R,          ;R&   S[ S[S[ ,          ,          R,          ;R&   S[ R,          ;R&   S[;R&   S[S[,          R,          ;R&   S[;R&   S[ S[,          R,          ;R&   S[ ;R&   S[ R,          ;R&   S[S[,          R,          ;R&   S[R,          ;R&   S[R,          ;R&   S[R,          ;R&   # )é   Ú
vocab_sizerF   Úintermediate_sizer;   rG   Únum_key_value_headsÚhead_dimÚhidden_activationÚmax_position_embeddingsÚinitializer_rangeÚrms_norm_epsÚ	use_cacheNÚpad_token_idÚeos_token_idÚbos_token_idÚtie_word_embeddingsÚrope_parametersÚattention_biasÚattention_dropoutÚquery_pre_attn_scalarÚsliding_windowr9   Úfinal_logit_softcappingÚattn_logit_softcappingÚuse_bidirectional_attention)ÚintÚstrÚfloatr<   Úlistr   Údict)ÚformatÚ__classdict__s   "€rC   Ú__annotate__ÚGemma2Config.__annotate__8   s½  ø‡ ‚ ñP ÑñQ ñR ÑñS ñT Ñ!ñU ñV ÑñW ñX Ñ ñY ñZ Ñ ñ[ ñ\ Ññ] ñ^ Ñ0ñ_ ñ` !Ñ'ña ñb Ñ#ñc ñd Ññe ñf Ññg ñh ˜•*Ñ ñi ñj ™™S�	•/ DÕ(Ñ,ñk ñl ˜•*Ñ ñm ñn Ñ$ño ñp $¡dÕ*¨TÕ1Ñ8ñq ñr Ñ ñs ñt ™U•{ TÕ)Ñ/ñu ñv Ñ$ñw ñx ˜$•JÑ%ñy ñz ‘c•˜TÕ!Ñ(ñ{ ñ| # T�\Ñ0ñ} ñ~ " D�LÑ/ñ ñ@ "&¨¥Ñ3òA rK   )r9   )(Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferenceÚbase_model_tp_planÚbase_model_pp_planrN   rF   rO   r;   rG   rP   rQ   rR   rS   rT   rU   rV   rW   rX   rY   rZ   r[   r\   r]   r^   r_   r9   r`   ra   rb   r>   rI   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__Ú__classcell__©rB   ri   s   @@rC   r&   r&   8   s  ù‡ € ñð( €JØ#4Ð"5Ðà# YØ# YØ# YØ# YØ  )Ø 	Ø  )ðÐð 	˜+˜¨Ð(9Ð:Ø�OÐ%5Ð6¸Ð8IÐJØ�/Ð" _Ð$5Ð6ðÐð €JØ€KØ!ÐØÐØ ÐØ ÐØ€HØ0ÐØ#'ÐØ#ÐØ€LØ€IØ €LØ+,€LØ €LØ $ÐØ48€OØ €NØ,/ÐØ!$ÐØ!%€NØ$(€KØ,0ÐØ+/ÐØ/3Ðõ(ò÷U … rK   r&   c                   ó   € ] tR t^‹tRtR# )ÚGemma2RMSNormr8   N©rl   rm   rn   ro   rv   r8   rK   rC   r{   r{   ‹   ó   † ÛrK   r{   c                   ó2   a a€ ] tR t^�t oV 3R ltRtVtV ;t# )Ú	Gemma2MLPc                ó^   <€ \         SV `  V4       \        VP                  ,          V n        R # ©N)r=   Ú__init__r   rR   Úact_fn©r?   ÚconfigrB   s   &&€rC   r‚   ÚGemma2MLP.__init__�   s"   ø€ Ü‰Ñ˜Ô Ü˜V×5Ñ5Õ6ˆŽrK   )rƒ   )rl   rm   rn   ro   r‚   rv   rw   rx   ry   s   @@rC   r   r   �   s   ù‡ € ÷7õ 7rK   r   c                   óˆ   a € ] tR t^•t o ]! RRR7      R	V 3R lR ll4       t]P                  ! 4       ]R 4       4       t	Rt
V tR# )
ÚGemma2RotaryEmbeddingÚdevicez5.18)ÚversionNc                ó    <€ V ^8„  d   QhRS[ /# ©rM   r…   ©r&   )rh   ri   s   "€rC   rj   Ú"Gemma2RotaryEmbedding.__annotate__—   s   ø€ ÷ Oñ O™|ñ OrK   c                ó  € \         P                  P                  4        VP                  V n        VP                  V n        Wn        V P                  P                  R ,          V n        V P                  pV P                  R8w  d   \        V P                  ,          pV! V P                  V4      w  q@n        \         P                  ! VRR7      V n        \         P                  ! VP                  4       RR7      V n        R# )Ú	rope_typeÚdefaultF)Ú
persistentN)ÚnnÚModuler‚   rS   Úmax_seq_len_cachedÚoriginal_max_seq_lenr…   r[   r�   Úcompute_default_rope_parametersr   Úattention_scalingÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)r?   r…   r‰   Úrope_init_fnrš   s   &&&  rC   r‚   ÚGemma2RotaryEmbedding.__init__–   s°   € ä
�	‰	×ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àŒàŸ™×4Ñ4°[ÕAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÔ&Ü.¨t¯~©~Õ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÔ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÔ!NˆÖrK   c                ó”  € V P                   R,          P                  4       P                  VP                  ^ ,          R^4      P	                  VP
                  4      pVR	,          P                  4       p\        VP
                  P                  \        4      '       d2   VP
                  P                  R8w  d   VP
                  P                  MRp\        VRR7      ;_uu_ 4        VP                  4       VP                  4       ,          P                  ^^4      p\        P                  ! Wf3RR7      pVP                  4       V P                  ,          pVP                  4       V P                  ,          p	R R R 4       XP	                  VP                   R7      X	P	                  VP                   R7      3#   + '       g   i     LH; i)
NÚmpsÚcpuF)Údevice_typeÚenabled)Údim)Údtype)NºNNNNéÿÿÿÿ)r¦   Nr¦   )rš   re   ÚexpandÚshapeÚtor‰   Ú
isinstanceÚtyperd   r   Ú	transposeÚtorchÚcatÚcosr˜   Úsinr¥   )
r?   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedr¢   ÚfreqsÚembr°   r±   s
   &&&       rC   ÚforwardÚGemma2RotaryEmbedding.forward§   sE  € ð !ŸM™M¨-Õ8×>Ñ>Ó@×GÑGÈ×HZÑHZÐ[\ÕH]Ð_aÐcdÓe×hÑhÐij×iqÑiqÓrÐØ ,¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆÜ¨¸U×CÖCØ&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÕN×YÑYÐZ[Ð]^Ó_ˆEÜ—)’)˜U˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆC÷	 Dð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷ D×Cús   Ã&BF7Æ7G	)r˜   r…   rš   r•   rœ   r–   r�   r�   )rl   rm   rn   ro   r   r‚   r®   Úno_gradr   r¸   rv   rw   )ri   s   @rC   rˆ   rˆ   •   sH   ø‡ € Ù�X vÔ.÷Oñ Oó /ðOð  ‡]‚]ƒ_Øñ<ó ó ö<rK   rˆ   c                ór  € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        \
        ,          R\        R,          R	\        R,          R
\        \        P                  \        P                  3,          /	# )rM   ÚmoduleÚqueryÚkeyÚvaluer0   NÚdropoutÚscalingÚsoftcapÚreturn)r“   r”   r®   ÚTensorre   rc   Útuple)rh   s   "rC   rj   rj   ·   s™   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ”S�[ð%ô �T�\ð%ô �T�\ð%ô Œ5�<‰<œŸ™Ð%Õ&ñ%rK   c                 ó¤  € Vf   V P                   R,          p\        W P                  4      p	\        W0P                  4      p
\        P                  ! WP                  ^^4      4      V,          pVe'   W·,          p\        P                  ! V4      pW·,          pVe	   W´,           p\        P                  P                  VR\        P                  R7      P                  VP                  4      p\        P                  P                  WµV P                  R7      p\        P                  ! Wº4      pVP                  ^^4      P                  4       pWË3# )N)r¤   r¥   )ÚpÚtrainingç      à¿r§   )rQ   r#   Únum_key_value_groupsr®   Úmatmulr­   Útanhr“   Ú
functionalÚsoftmaxÚfloat32rª   r¥   rÀ   rÈ   Ú
contiguous)r¼   r½   r¾   r¿   r0   rÀ   rÁ   rÂ   r@   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs   &&&&&&&&,    rC   Úeager_attention_forwardrÕ   ·   s  € ð ‚Ø—/‘/ 4Õ'ˆä˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÕL€LàÒØ#Õ-ˆÜ—z’z ,Ó/ˆØ#Õ-ˆØÒ!Ø#Õ4ˆô —=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KØÐ$Ð$rK   c                   óT   a a€ ] tR t^Ùt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚGemma2Attentionc                ó&   <€ V ^8„  d   QhRS[ RS[/# ©rM   r…   Ú	layer_idx©r&   rc   )rh   ri   s   "€rC   rj   ÚGemma2Attention.__annotate__Ú   s   ø€ ÷ hñ h™|ð h¹ñ hrK   c                óž  <€ \        VR 4      '       d   VP                  V,          MRV n        \        SV `  W4       V P
                  P                  V n        V P
                  P                  V n        \        VRR4      '       * V n	        VP                  R,          V n        V P                  R8X  d   VP                  V n        R# RV n        R# )r9   Nrb   Fr6   rÉ   )Úhasattrr9   Ú
layer_typer=   r‚   r…   ra   r]   ÚgetattrÚ	is_causalr^   rÁ   r_   ©r?   r…   rÚ   rB   s   &&&€rC   r‚   ÚGemma2Attention.__init__Ú   s�   ø€ Ü;BÀ6È=×;YÒ;Y˜&×,Ñ,¨YÖ7Ð_cˆŒä‰Ñ˜Ô+Ø&*§k¡k×&HÑ&HˆÔ#Ø!%§¡×!>Ñ!>ˆÔÜ$ VÐ-JÈEÓRÔRˆŒØ×3Ñ3°TÕ9ˆŒØ7;·±ÐJ]Ô7]˜f×3Ñ3ˆÖÐcgˆÖrK   c                ó`  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          R,          RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  S[ P                  R,          S[S[ P                  ,          R,          3,          /# )rM   r/   Úposition_embeddingsNr0   r(   r@   rÃ   )r®   rÄ   rÅ   r   r   r   )rh   ri   s   "€rC   rj   rÜ   ä   sœ   ø€ ÷ ()ñ ()á—|‘|ð()ñ #¡5§<¡<±·±Ð#=Õ>ÀÕEð()ñ Ÿ™ tÕ+ð	()ñ
  �ð()ñ Ñ-Õ.ð()ñ 
‰u�|‰|™UŸ\™\¨DÕ0±%¹¿¹Õ2EÈÕ2LÐLÕ	Mñ()rK   c                ó@  € VP                   R R p. VORNV P                  N5pV P                  V4      P                  V4      P	                  ^^4      pV P                  V4      P                  V4      P	                  ^^4      p	V P                  V4      P                  V4      P	                  ^^4      p
Vw  r¼\        W‰W¼4      w  r‰Ve   VP                  WšV P                  4      w  rš\        P                  ! V P                  P                  \        4      pT! V VV	V
V3RV P                  '       d   V P                   MRRV P"                  RV P$                  RV P&                  /VB w  rïVP(                  ! . VORN5!  P+                  4       pV P-                  V4      pWï3# )NrÀ   r4   rÁ   r_   rÂ   r§   )r©   rQ   Úq_projÚviewr­   Úk_projÚv_projr"   ÚupdaterÚ   r   Úget_interfacer…   Ú_attn_implementationrÕ   rÈ   r]   rÁ   r_   ra   ÚreshaperÐ   Úo_proj)r?   r/   rå   r0   r(   r@   Úinput_shapeÚhidden_shapeÚquery_statesrÑ   rÒ   r°   r±   Úattention_interfacerÔ   rÓ   s   &&&&&,          rC   r¸   ÚGemma2Attention.forwardä   s§  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ%
ð /3¯m¯m¨m�D×*Ò*Àð%
ð —L‘Lð%
ð  ×.Ñ.ð%
ð ×/Ñ/ð%
ð ñ%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—k‘k +Ó.ˆØÐ(Ð(rK   )r]   ra   rá   rß   rÁ   r_   )NNN©	rl   rm   rn   ro   r‚   r¸   rv   rw   rx   ry   s   @@rC   r×   r×   Ù   s    ù‡ € ÷hó h÷()÷ ()ò ()rK   r×   c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚGemma2DecoderLayeri  c                ó&   <€ V ^8„  d   QhRS[ RS[/# rÙ   rÛ   )rh   ri   s   "€rC   rj   ÚGemma2DecoderLayer.__annotate__  s   ø€ ÷ 
eñ 
e™|ð 
e¹ñ 
erK   c                óÄ  <€ \         SV `  4        VP                  V n        Wn        \	        WR 7      V n        \        V4      V n        \        VP                  VP                  R7      V n
        \        VP                  VP                  R7      V n        \        VP                  VP                  R7      V n        \        VP                  VP                  R7      V n        R# ))r…   rÚ   )ÚepsN)r=   r‚   rF   r…   r×   Ú	self_attnr   Úmlpr{   rU   Úinput_layernormÚpost_attention_layernormÚpre_feedforward_layernormÚpost_feedforward_layernormrâ   s   &&&€rC   r‚   ÚGemma2DecoderLayer.__init__  s¦   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØŒÜ(°ÔLˆŒÜ˜VÓ$ˆŒÜ,¨V×-?Ñ-?ÀV×EXÑEXÔYˆÔÜ(5°f×6HÑ6HÈf×NaÑNaÔ(bˆÔ%ä)6°v×7IÑ7IÈv×ObÑObÔ)cˆÔ&Ü*7¸×8JÑ8JÐPV×PcÑPcÔ*dˆÖ'rK   c                ó  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  /# )rM   r/   rå   Nr0   r³   r(   rÃ   )r®   rÄ   rÅ   Ú
LongTensorr   )rh   ri   s   "€rC   rj   rù     s~   ø€ ÷ ñ á—|‘|ðñ #¡5§<¡<±·±Ð#=Õ>ÀÕEðñ Ÿ™ tÕ+ð	ñ
 ×&Ñ&¨Õ-ðñ  �ðñ 
�‰ñrK   c                ó  € TpV P                  V4      pV P                  ! RR VRVRVRVRV/VB w  rV P                  V4      pWq,           pTpV P                  V4      pV P	                  V4      pV P                  V4      pWq,           pV# )r/   rå   r0   r³   r(   r8   )rþ   rü   rÿ   r   rý   r  )	r?   r/   rå   r0   r³   r(   r@   ÚresidualÚ_s	   &&&&&&,  rC   r¸   ÚGemma2DecoderLayer.forward  s¿   € ð !ˆà×,Ñ,¨]Ó;ˆð  Ÿ>š>ñ 
Ø'ð
à 3ð
ð *ð
ð &ð	
ð
 ,ð
ð ñ
Ñˆð ×5Ñ5°mÓDˆØ Õ0ˆà ˆØ×6Ñ6°}ÓEˆØŸ™ Ó/ˆØ×7Ñ7¸ÓFˆØ Õ0ˆàÐrK   )r…   rF   rþ   rý   rÿ   r  r   rü   )NNNNrõ   ry   s   @@rC   r÷   r÷     s    ù‡ € ÷
eó 
e÷÷ ò rK   r÷   c                   ó   € ] tR tRtRtR# )ÚGemma2PreTrainedModeli>  r8   Nr|   r8   rK   rC   r
  r
  >  r}   rK   r
  c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚGemma2ModeliB  c                ó    <€ V ^8„  d   QhRS[ /# rŒ   r�   )rh   ri   s   "€rC   rj   ÚGemma2Model.__annotate__C  s   ø€ ÷ 8ñ 8™|ñ 8rK   c           	     óÞ   <€ \         SV `  V4       \        P                  ! \	        VP
                  4       Uu. uF  p\        W4      NK  	  up4      V n        \        V4      V n	        R # u upi r�   )
r=   r‚   r“   Ú
ModuleListr:   r;   r÷   r.   rˆ   Ú
rotary_embrâ   s   && €rC   r‚   ÚGemma2Model.__init__C  sY   ø€ Ü‰Ñ˜Ô Ü—m’mÜDIÈ&×JbÑJbÔDcÓdÑDc°yÔ Ö2ÑDcÑdó
ˆŒô 0°Ó7ˆŽùò es   ¸A*c                óþ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[S[,          R	S[/# )
rM   r,   Nr0   r³   r(   r-   rV   r@   rÃ   )	r®   r  rÄ   r   ÚFloatTensorr<   r   r   r   )rh   ri   s   "€rC   rj   r  J  s“   ø€ ÷ ;
ñ ;
á×#Ñ# dÕ*ð;
ñ Ÿ™ tÕ+ð;
ñ ×&Ñ&¨Õ-ð	;
ñ
  �ð;
ñ ×(Ñ(¨4Õ/ð;
ñ ˜$•;ð;
ñ Ñ+Õ,ð;
ñ 
!ñ;
rK   c                ó2  € VR J VR J,          '       d   \        R4      hVf   V P                  V4      pV'       d   Vf   \        V P                  R7      pVfb   Ve   VP	                  4       M^ p\
        P                  ! VP                  ^,          VP                  R7      V,           pVP                  ^ 4      p\        T;p	\        4      '       g/   RV P                  RVRVRVRV/p
R	\        R/ V
B R
\        R/ V
B /p	TpV P                  W³4      p\        V P                   R V P                  P"                   4       F7  w  rÞV! V3RW�P                  P$                  V,          ,          RVRVRV/VB pK9  	  V P'                  V4      p\)        VVR7      # )Nz:You must specify exactly one of input_ids or inputs_embeds)r…   )r‰   r…   r-   r0   r(   r³   r7   r6   rå   )Úlast_hidden_stater(   r8   )rH   r+   r   r…   Úget_seq_lengthr®   Úaranger©   r‰   Ú	unsqueezer«   rg   r	   r
   r  Ú	enumerater.   r;   r9   r1   r   )r?   r,   r0   r³   r(   r-   rV   r@   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsr/   rå   rA   Údecoder_layers   &&&&&&&,       rC   r¸   ÚGemma2Model.forwardJ  s·  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ Ø*.×*;Ñ*;¸IÓ*FˆMç˜Ò0Ü*°$·+±+Ô>ˆOàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLô °Ð?Ð-Ä×FÒFð ˜$Ÿ+™+Ø Ø  .Ø! ?Ø ðˆKð !Ô"4Ñ"C°{Ñ"CØ#Ô%FÑ%UÈÑ%Uð#Ðð &ˆØ"Ÿo™o¨mÓJÐä )¨$¯+©+Ð6U¸¿¹×8UÑ8UÐ*VÖ WÑˆAÙ)Øñà2·;±;×3JÑ3JÈ1Õ3MÕNðð %8ðð *ð	ð
 !0ðð ñŠMñ !Xð Ÿ	™	 -Ó0ˆä&Ø+Ø+ô
ð 	
rK   )r.   r  )NNNNNNrõ   ry   s   @@rC   r  r  B  s   ù‡ € ÷8ó 8÷;
÷ ;
ò ;
rK   r  c                   óH   a a€ ] tR tRt oV 3R ltRV3R lR lltRtVtV ;t# )ÚGemma2ForCausalLMiˆ  c                óf   <€ \         SV `  V4       \        V4      V n        V P	                  4        R # r�   )r=   r‚   r  ÚmodelÚ	post_initr„   s   &&€rC   r‚   ÚGemma2ForCausalLM.__init__‰  s&   ø€ Ü‰Ñ˜Ô Ü  Ó(ˆŒ
Ø�‰ÖrK   c                óP  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[S[ P                  ,          R
S[S[,          RS[	/
# )rM   r,   Nr0   r³   r(   r-   ÚlabelsrV   Úlogits_to_keepr@   rÃ   )
r®   r  rÄ   r   r  r<   rc   r   r   r   )rh   ri   s   "€rC   rj   ÚGemma2ForCausalLM.__annotate__Ž  s»   ø€ ÷ ;
ñ ;
á×#Ñ# dÕ*ð;
ñ Ÿ™ tÕ+ð;
ñ ×&Ñ&¨Õ-ð	;
ñ
  �ð;
ñ ×(Ñ(¨4Õ/ð;
ñ × Ñ  4Õ'ð;
ñ ˜$•;ð;
ñ ™eŸl™lÕ*ð;
ñ Ñ+Õ,ð;
ñ 
 ñ;
rK   c	                óF  € V P                   ! R
RVRVRVRVRVRV/V	B p
V
P                  p\        V\        4      '       d   \	        V) R4      MTpV P                  VRVR3,          4      pV P                  P                  eO   WÐP                  P                  ,          p\        P                  ! V4      pWÐP                  P                  ,          pRpVe   V P                  ! WÖV P                  3/ V	B p\        VVV
P                  V
P                  V
P                  R	7      # )a"  
Example:

```python
>>> from transformers import AutoTokenizer, Gemma2ForCausalLM

>>> model = Gemma2ForCausalLM.from_pretrained("google/gemma-2-9b")
>>> tokenizer = AutoTokenizer.from_pretrained("google/gemma-2-9b")

>>> prompt = "What is your favorite condiment?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"What is your favorite condiment?"
```r,   r0   r³   r(   r-   rV   Nr¦   )ÚlossÚlogitsr(   r/   Ú
attentionsr8   )r#  r  r«   rc   ÚsliceÚlm_headr…   r`   r®   rÌ   Úloss_functionrN   r   r(   r/   r-  )r?   r,   r0   r³   r(   r-   r'  rV   r(  r@   Úoutputsr/   Úslice_indicesr,  r+  s   &&&&&&&&&,     rC   r¸   ÚGemma2ForCausalLM.forwardŽ  s/  € ð< ,0¯:ª:ñ ,
Øð,
à)ð,
ð &ð,
ð ,ð	,
ð
 (ð,
ð  ð,
ð ñ,
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆØ�;‰;×.Ñ.Ò:ØŸk™k×AÑAÕAˆFÜ—Z’Z Ó'ˆFØŸk™k×AÑAÕAˆFàˆØÒØ×%Ò% f°d·o±oÑPÈÑPˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
rK   )r#  )NNNNNNNr   rõ   ry   s   @@rC   r!  r!  ˆ  s   ù‡ € õ÷
;
÷ ;
ò ;
rK   r!  c                   ó   € ] tR tRtRtR# )ÚGemma2ForSequenceClassificationiÌ  r8   Nr|   r8   rK   rC   r5  r5  Ì  r}   rK   r5  c                   ó   € ] tR tRtRtR# )ÚGemma2ForTokenClassificationiÐ  r8   Nr|   r8   rK   rC   r7  r7  Ð  r}   rK   r7  )r4   NN)r&   r!  r  r
  r5  r7  )DÚcollections.abcr   r®   Útorch.nnr“   Úhuggingface_hub.dataclassesr   Úactivationsr   Úcache_utilsr   r   Úconfiguration_utilsr   Úmasking_utilsr	   r
   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   r   Úmodeling_utilsr   Úprocessing_utilsr   Úutilsr   r   r   Úutils.deprecationr   Úutils.genericr   Úgemma.modeling_gemmar   r   r   r   r   r   r   r    r!   r"   r#   Ú
get_loggerrl   Úloggerr&   r{   r   rˆ   rÕ   r×   r÷   r
  r  r!  r5  r7  Ú__all__r8   rK   rC   Ú<module>rL     s.  ðõ %ã Ý Ý .å !ß .Ý 3ß RÝ BÝ 9ß O÷ñ õ
 6Ý &ß @Ñ @Ý 0Ý +÷÷ ÷ ñ ð 
×	Ò	˜HÓ	%€ñ Ð-Ô.ØôNÐ#ó Nó ó /ðNôb	�Lô 	ô7�ô 7ô<Ð0ô <÷D%ôD3)�nô 3)ôl,Ð3ô ,ô^	Ð0ô 	ôC
�*ô C
ôLA
Ð(ô A
ôH	Ð&Dô 	ô	Ð#>ô 	ò‚rK   