+
    œ°³jN ã                   ól  € ^ RI t ^ RIHt ^ RIt^ RIHt ^RIHt ^RI	H
t
 ^RIHtHtHtHt ^RIHtHtHt ^RIHtHt ^RIHtHtHt ^R	IHt ^R
IHt ^RIH t H!t!H"t"H#t#H$t$H%t%H&t& ^RI'H(t(H)t) ^RI*H+t+H,t, ^RI-H.t. ^RI/H0t0H1t1H2t2H3t3 ^RI4H5t5 ^RI6H7t7H8t8 ^RI9H:t:H;t; ^RI<H=t= ^RI>H?t?H@t@HAtAHBtB  ! R R]P†                  4      tD ! R R]P†                  4      tE ! R R]P†                  4      tFR tG]! R4      RDR l4       tHR R ltIRER  R! lltJ]! ]H4       ! R" R#]P†                  4      4       tK]! ]H4       ! R$ R%]P†                  4      4       tL ! R& R']4      tM ! R( R)]4      tN ! R* R+]P†                  4      tO ! R, R-]P†                  4      tP ! R. R/]P†                  4      tQ ! R0 R1]P¤                  4      tS]1 ! R2 R3],4      4       tTRFR4 R5 lltU ! R6 R7]T4      tV ! R8 R9]T4      tW ! R: R;]T4      tX]1 ! R< R=]T4      4       tY ! R> R?]T]4      tZ]1 ! R@ RA]T4      4       t[]1 ! RB RC]T4      4       t\. RGOt]R# )Hé    N)ÚCallable)Úinitialization)ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCacheÚStaticCache)ÚGenerationConfigÚGenerationMixinÚGenerationMode)Úuse_kernel_forward_from_hubÚuse_kernelized_func)Úcreate_bidirectional_maskÚcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚ)BaseModelOutputWithPastAndCrossAttentionsÚBaseModelOutputWithPoolingÚSeq2SeqLMOutputÚSeq2SeqModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚtorch_compilable_check)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputs)Ú	AutoModel)ÚT5Gemma2ConfigÚT5Gemma2DecoderConfigÚT5Gemma2EncoderConfigÚT5Gemma2TextConfigc                   óT   a a€ ] tR t^7t oRV3R lV 3R llltR tR tR tRtVt	V ;t
# )ÚT5Gemma2RMSNormc                ó&   <€ V ^8„  d   QhRS[ RS[/# )é   ÚdimÚeps©ÚintÚfloat)ÚformatÚ__classdict__s   "€Úo/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/t5gemma2/modeling_t5gemma2.pyÚ__annotate__ÚT5Gemma2RMSNorm.__annotate__8   s   ø€ ÷ 5ñ 5™Cð 5¡eñ 5ó    c                óŽ   <€ \         SV `  4        W n        \        P                  ! \
        P                  ! V4      4      V n        R # ©N)ÚsuperÚ__init__r3   ÚnnÚ	ParameterÚtorchÚzerosÚweight)Úselfr2   r3   Ú	__class__s   &&&€r9   r@   ÚT5Gemma2RMSNorm.__init__8   s,   ø€ Ü‰ÑÔØŒÜ—l’l¤5§;¢;¨sÓ#3Ó4ˆŽr<   c                óž   € V\         P                  ! VP                  ^4      P                  RRR7      V P                  ,           4      ,          # )r1   T)Úkeepdiméÿÿÿÿ)rC   ÚrsqrtÚpowÚmeanr3   )rF   Úxs   &&r9   Ú_normÚT5Gemma2RMSNorm._norm=   s4   € Ø”5—;’;˜qŸu™u Q›xŸ}™}¨R¸˜}Ó>ÀÇÁÕIÓJÕJÐJr<   c                ó²   € V P                  VP                  4       4      pVR V P                  P                  4       ,           ,          pVP                  V4      # )ç      ð?)rP   r6   rE   Útype_as)rF   rO   Úoutputs   && r9   ÚforwardÚT5Gemma2RMSNorm.forward@   sC   € Ø—‘˜AŸG™G›IÓ&ˆð ˜3 §¡×!2Ñ!2Ó!4Õ4Õ5ˆØ�~‰~˜aÓ Ð r<   c                ó^   € \        V P                  P                  4       R V P                   2# )z, eps=)ÚtuplerE   Úshaper3   ©rF   s   &r9   Ú
extra_reprÚT5Gemma2RMSNorm.extra_reprG   s'   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$·(±(°Ð<Ð<r<   )r3   rE   )g�íµ ÷Æ°>)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r@   rP   rV   r\   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©rG   r8   s   @@r9   r/   r/   7   s$   ù‡ € ÷5õ 5ò
Kò!÷=ò =r<   r/   c                   óD   a a€ ] tR t^Kt oV3R lV 3R lltR tRtVtV ;t# )ÚT5Gemma2MLPc                ó    <€ V ^8„  d   QhRS[ /# ©r1   Úconfig©r-   )r7   r8   s   "€r9   r:   ÚT5Gemma2MLP.__annotate__L   s   ø€ ÷ 	7ñ 	7Ñ1ñ 	7r<   c                ó"  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        \
        P                  ! V P                  V P                  R R7      V n        \
        P                  ! V P                  V P                  R R7      V n        \
        P                  ! V P                  V P                  R R7      V n	        \        VP                  ,          V n        \
        P                  ! VP                  4      V n        R# )F©ÚbiasN)r?   r@   rj   Úhidden_sizeÚintermediate_sizerA   ÚLinearÚ	gate_projÚup_projÚ	down_projr   Úhidden_activationÚact_fnÚDropoutÚdropout_rateÚdropout©rF   rj   rG   s   &&€r9   r@   ÚT5Gemma2MLP.__init__L   sº   ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔÜŸš 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆŒÜ˜V×5Ñ5Õ6ˆŒÜ—z’z &×"5Ñ"5Ó6ˆŽr<   c                ó¶   € V P                  V P                  V4      4      V P                  V4      ,          pV P                  V4      pV P	                  V4      pV# r>   )rw   rs   rt   rz   ru   )rF   rO   Úhidden_statesru   s   &&  r9   rV   ÚT5Gemma2MLP.forwardW   sH   € ØŸ™ D§N¡N°1Ó$5Ó6¸¿¹Àa»ÕHˆØŸ™ ]Ó3ˆØ—N‘N =Ó1ˆ	ØÐr<   )rw   rj   ru   rz   rs   rp   rq   rt   ©	r^   r_   r`   ra   r@   rV   rb   rc   rd   re   s   @@r9   rg   rg   K   s   ù‡ € ÷	7ó 	7÷ò r<   rg   c                   óÎ   a a€ ] tR t^^t o]! RRR7      R
V3R lV 3R lll4       t]]! RRR7      RV3R lR ll4       4       t]P                  ! 4       ]
R 4       4       tR	tVtV ;t# )ÚT5Gemma2RotaryEmbeddingÚdevicez5.18)Úversionc                ó    <€ V ^8„  d   QhRS[ /# ri   rk   )r7   r8   s   "€r9   r:   Ú$T5Gemma2RotaryEmbedding.__annotate__`   s   ø€ ÷ Uñ UÑ1ñ Ur<   c           
     óÞ  <€ \         SV `  4        VP                  V n        VP                  V n        Wn        \        \        VP                  4      4      V n        / V n	        V P                   Fû  pV P
                  P                  V,          pVf   K&  VR,          V P                  V&   V P                  pV P                  V,          R8w  d   \        V P                  V,          ,          pV! V P
                  W#R7      w  rg\        W R2\        P                  ! VRR7      4       \        W R2\        P                  ! VP!                  4       RR7      4       \        W R2V4       Ký  	  R # )	NÚ	rope_typeÚdefault©Ú
layer_typeÚ	_inv_freqF©Ú
persistentÚ_original_inv_freqÚ_attention_scaling)r?   r@   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenrj   ÚsortedÚsetÚlayer_typesrˆ   Úrope_parametersÚcompute_default_rope_parametersr   ÚsetattrrA   ÚBufferÚclone)	rF   rj   rƒ   r‹   Úrope_paramsÚrope_init_fnÚcurr_inv_freqÚcurr_attention_scalingrG   s	   &&&     €r9   r@   Ú T5Gemma2RotaryEmbedding.__init___   s'  ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!ØŒÜ!¤# f×&8Ñ&8Ó"9Ó:ˆÔØˆŒØ×*Ô*ˆJØŸ+™+×5Ñ5°jÕAˆKØÒ"Ùà)4°[Õ)AˆD�N‰N˜:Ñ&Ø%)×%IÑ%IˆLØ�~‰~˜jÕ)¨YÔ6Ü2°4·>±>À*Õ3MÕN�Ù4@ÀÇÁÈfÔ4lÑ1ˆMÜ�D˜L¨	Ð2´B·I²I¸mÐX]Ô4^Ô_Ü�D˜LÐ(:Ð;¼R¿YºYÀ}×GZÑGZÓG\ÐinÔ=oÔpÜ�D˜LÐ(:Ð;Ð=SÖTó +r<   c                ód   <€ V ^8„  d   QhRS[ RS[R,          RS[S[P                  S[3,          /# )r1   rj   r‹   NÚreturn)r-   ÚstrrY   rC   ÚTensorr6   )r7   r8   s   "€r9   r:   r†   w   s7   ø€ ÷ 5ñ 5Ù"ð5Ù=@À4½Zð5á	‰u�|‰|™UÐ"Õ	#ñ5r<   c           	     ó>  € V P                   V,          R,          p\        V RR4      ;'       g    V P                  V P                  ,          pRpRV\        P
                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a.  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
    layer_type (`str`):
        The current layer type if the model has different RoPE parameters per type.
        Should not be used unless `config.layer_types is not None`

Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚhead_dimNrS   ©Údtype)r—   Úgetattrrp   Únum_attention_headsrC   Úaranger6   Úto)rj   rƒ   r‹   ÚkwargsÚbaser2   Úattention_factorÚinv_freqs   &&&,    r9   r˜   Ú7T5Gemma2RotaryEmbedding.compute_default_rope_parametersu   sƒ   € ð& ×%Ñ% jÕ1°,Õ?ˆÜ�f˜j¨$Ó/×cÐc°6×3EÑ3EÈ×IcÑIcÕ3cˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r<   c                óZ  € \        W R 24      p\        W R24      pVR
,          P                  VP                  ^ ,          R^4      P                  \        P
                  VP                  R7      pVR,          P                  4       p\        VP                  P                  \        4      '       d2   VP                  P                  R8w  d   VP                  P                  MRp\        VRR7      ;_uu_ 4        Wg,          P                  ^^4      p	\        P                  ! W™3RR7      p
V
P                  4       V,          pV
P                  4       V,          pRRR4       XP                  VP                  R	7      XP                  VP                  R	7      3#   + '       g   i     LH; i)rŒ   r�   N©r©   rƒ   ÚmpsÚcpuF)Údevice_typeÚenabled©r2   r¨   )NºNNNNrK   )rº   Nrº   )rª   ÚexpandrZ   r­   rC   r6   rƒ   Ú
isinstanceÚtyper£   r%   Ú	transposeÚcatÚcosÚsinr©   )rF   rO   Úposition_idsr‹   r±   Úattention_scalingÚinv_freq_expandedÚposition_ids_expandedr·   ÚfreqsÚembrÀ   rÁ   s   &&&&         r9   rV   ÚT5Gemma2RotaryEmbedding.forward�   sJ  € ô ˜4 <¨yÐ!9Ó:ˆÜ# D¨LÐ8JÐ*KÓLÐð �]Õ#×*Ñ*¨<×+=Ñ+=¸aÕ+@À"ÀaÓH×KÑKÔRW×R]ÑR]Ðfg×fnÑfnÐKÓoð 	ð !-¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆä¨¸U×CÖCØ&Õ>×IÑIÈ!ÈQÓOˆEÜ—)’)˜U˜N°Ô3ˆCØ—'‘'“)Ð/Õ/ˆCØ—'‘'“)Ð/Õ/ˆC÷	 Dð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷ D×Cús   Ã:A FÆF*	)rj   r–   r’   r“   rˆ   r>   )NN)r^   r_   r`   ra   r$   r@   Ústaticmethodr˜   rC   Úno_gradr   rV   rb   rc   rd   re   s   @@r9   r‚   r‚   ^   su   ù‡ € Ù�X vÔ.÷Uô Uó /ðUð* Ù�X vÔ.÷5ñ 5ó /ó ð5ð2 ‡]‚]ƒ_Øñ<ó ó ÷<ð <r<   r‚   c                óÌ   € V RRV P                   R,          ^,          13,          pV RV P                   R,          ^,          R13,          p\        P                  ! V) V3RR7      # )z*Rotates half the hidden dims of the input..Nr¹   rK   )rZ   rC   r¿   )rO   Úx1Úx2s   &  r9   Úrotate_halfrÎ   ¦   sZ   € à	
ˆ3Ð"�!—'‘'˜"•+ Õ"Ð"Ð"Õ	#€BØ	
ˆ3�—‘˜•˜qÕ Ñ"Ð"Õ	#€BÜ�9Š9�r�c˜2�Y BÔ'Ð'r<   Úrotary_pos_embc                óÈ   € VP                  V4      pVP                  V4      pW,          \        V 4      V,          ,           pW,          \        V4      V,          ,           pWV3# )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)Ú	unsqueezerÎ   )ÚqÚkrÀ   rÁ   Úunsqueeze_dimÚq_embedÚk_embeds   &&&&&  r9   Úapply_rotary_pos_embr×   ­   sS   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�wœ; q›>¨CÕ/Õ0€GØ�wœ; q›>¨CÕ/Õ0€GØÐÐr<   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r1   r~   Ún_repr¢   )rC   r¤   r5   )r7   s   "r9   r:   r:   Ç   s.   € ÷ 	Uñ 	UœUŸ\™\ð 	U´#ð 	U¼%¿,¹,ñ 	Ur<   c                ó˜   € V P                   w  r#rEV^8X  d   V # V R,          P                  W#WV4      p V P                  W#V,          WE4      # )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
)rº   rº   Nrº   rº   )rZ   r»   Úreshape)r~   rÙ   ÚbatchÚnum_key_value_headsÚslenr§   s   &&    r9   Ú	repeat_kvrß   Ç   sU   € ð
 2?×1DÑ1DÑ.€E Ø�„zØÐØ!Ð"2Õ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÕ(CÀTÓTÐTr<   c                ór  € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        \
        ,          R\        R,          R	\        R,          R
\        \        P                  \        P                  3,          /	# )r1   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNrz   ÚscalingÚsoftcapr¢   )rA   ÚModulerC   r¤   r6   r5   rY   )r7   s   "r9   r:   r:   Ó   s™   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ”S�[ð%ô �T�\ð%ô �T�\ð%ô Œ5�<‰<œŸ™Ð%Õ&ñ%r<   c                 ó¤  € Vf   V P                   R,          p\        W P                  4      p	\        W0P                  4      p
\        P                  ! WP                  ^^4      4      V,          pVe'   W·,          p\        P                  ! V4      pW·,          pVe	   W´,           p\        P                  P                  VR\        P                  R7      P                  VP                  4      p\        P                  P                  WµV P                  R7      p\        P                  ! Wº4      pVP                  ^^4      P                  4       pWË3# )N)r2   r©   )ÚpÚtrainingç      à¿rK   )r§   rß   Únum_key_value_groupsrC   Úmatmulr¾   ÚtanhrA   Ú
functionalÚsoftmaxÚfloat32r­   r©   rz   rë   Ú
contiguous)rá   râ   rã   rä   rå   rz   ræ   rç   r®   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs   &&&&&&&&,    r9   Úeager_attention_forwardrø   Ó   s  € ð ‚Ø—/‘/ 4Õ'ˆä˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÕL€LàÒØ#Õ-ˆÜ—z’z ,Ó/ˆØ#Õ-ˆØÒ!Ø#Õ4ˆô —=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KØÐ$Ð$r<   c                   óX   a a€ ] tR t^õt oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )ÚT5Gemma2SelfAttentionz=Multi-headed attention from 'Attention Is All You Need' paperc                ó&   <€ V ^8„  d   QhRS[ RS[/# ©r1   rj   Ú	layer_idx©r-   r5   )r7   r8   s   "€r9   r:   Ú"T5Gemma2SelfAttention.__annotate__ù   ó    ø€ ÷ Tñ TÑ1ð T¹cñ Tr<   c                óL  <€ \         SV `  4        \        VR 4      '       d   VP                  V,          MRV n        Wn        W n        \        VRVP                  VP                  ,          4      V n
        VP                  VP                  ,          V n        VP                  R,          V n        V P
                  P                  V n        RV n        \"        P$                  ! VP                  VP                  V P                  ,          VP&                  R7      V n        \"        P$                  ! VP                  VP                  V P                  ,          VP&                  R7      V n        \"        P$                  ! VP                  VP                  V P                  ,          VP&                  R7      V n        \"        P$                  ! VP                  V P                  ,          VP                  VP&                  R7      V n        V P
                  P0                  V n        V P                  R8X  d   VP2                  MRV n        V P                  R8H  V n        \7        VP                  VP8                  R7      V n        \7        VP                  VP8                  R7      V n        R# ©r–   Nr§   Frn   Úsliding_attention)r2   r3   rì   ©r?   r@   Úhasattrr–   r‹   rj   rý   rª   rp   r«   r§   rÝ   rí   Úquery_pre_attn_scalarræ   Úattention_dropoutÚ	is_causalrA   rr   Úattention_biasÚq_projÚk_projÚv_projÚo_projÚattn_logit_softcappingÚsliding_windowÚ
is_slidingr/   Úrms_norm_epsÚq_normÚk_norm©rF   rj   rý   rG   s   &&&€r9   r@   ÚT5Gemma2SelfAttention.__init__ù   óâ  ø€ Ü‰ÑÔÜ;BÀ6È=×;YÒ;Y˜&×,Ñ,¨YÖ7Ð_cˆŒØŒØ"ŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÕ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Õ$\ˆÔ!Ø×3Ñ3°TÕ9ˆŒØ!%§¡×!>Ñ!>ˆÔØˆŒä—i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×&Ñ&¨¯©Õ6¸×8JÑ8JÐQW×QfÑQfô
ˆŒð '+§k¡k×&HÑ&HˆÔ#Ø7;·±ÐJ]Ô7]˜f×3Ò3ÐcgˆÔØŸ/™/Ð-@Ñ@ˆŒä%¨&¯/©/¸v×?RÑ?RÔSˆŒÜ%¨&¯/©/¸v×?RÑ?RÔSˆŽr<   c                ó(  <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  S[ P                  R,          S[S[ P                  ,          R,          3,          /# )r1   r~   Úposition_embeddingsrå   NÚpast_key_valuesr®   r¢   )rC   r¤   r   r   r    rY   )r7   r8   s   "€r9   r:   rÿ     s‰   ø€ ÷ *)ñ *)á—|‘|ð*)ñ #Ÿ\™\ð*)ñ Ÿ™ tÕ+ð	*)ñ
  �ð*)ñ Ñ+Õ,ð*)ñ 
‰u�|‰|™UŸ\™\¨DÕ0±%¹¿¹Õ2EÈÕ2LÐLÕ	Mñ*)r<   c           	     ól  € VP                   R R p. VORNV P                  N5pV P                  V4      P                  V4      P	                  ^^4      pV P                  V4      P                  V4      P	                  ^^4      p	V P                  V4      P                  V4      P	                  ^^4      p
V P                  V4      pV P                  V	4      p	Vw  r¼\        W‰W¼4      w  r‰Ve   VP                  WšV P                  4      w  rš\        P                  ! V P                  P                  \         4      pT! V VV	V
V3RV P"                  '       d   V P$                  MRRV P&                  RV P(                  /VB w  rïVP*                  ! . VORN5!  P-                  4       pV P/                  V4      pWï3# )Nrz   ç        ræ   r  rK   )rZ   r§   r
  Úviewr¾   r  r  r  r  r×   Úupdaterý   r   Úget_interfacerj   Ú_attn_implementationrø   rë   r  ræ   r  rÛ   ró   r  )rF   r~   r  rå   r  r®   Úinput_shapeÚhidden_shapeÚquery_statesrô   rõ   rÀ   rÁ   Úattention_interfacer÷   rö   s   &&&&&,          r9   rV   ÚT5Gemma2SelfAttention.forward  s³  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà—{‘{ <Ó0ˆØ—[‘[ Ó,ˆ
à&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ
%
ð /3¯m¯m¨m�D×*Ò*Àð
%
ð —L‘Lð
%
ð  ×.Ñ.ð
%
ð ñ
%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—k‘k +Ó.ˆØÐ(Ð(r<   ©r  r  rj   r§   r  r  r  r  rý   r‹   rí   r  r  r
  ræ   r  r  ©NNN©
r^   r_   r`   ra   Ú__doc__r@   rV   rb   rc   rd   re   s   @@r9   rú   rú   õ   s#   ù‡ € áG÷Tó T÷<*)÷ *)ò *)r<   rú   c                   óX   a a€ ] tR tRt oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )	ÚT5Gemma2MergedAttentioniD  z6Merged self-attention and cross-attention for decoder.c                ó&   <€ V ^8„  d   QhRS[ RS[/# rü   rþ   )r7   r8   s   "€r9   r:   Ú$T5Gemma2MergedAttention.__annotate__H  r   r<   c                óL  <€ \         SV `  4        \        VR 4      '       d   VP                  V,          MRV n        Wn        W n        \        VRVP                  VP                  ,          4      V n
        VP                  VP                  ,          V n        VP                  R,          V n        V P
                  P                  V n        RV n        \"        P$                  ! VP                  VP                  V P                  ,          VP&                  R7      V n        \"        P$                  ! VP                  VP                  V P                  ,          VP&                  R7      V n        \"        P$                  ! VP                  VP                  V P                  ,          VP&                  R7      V n        \"        P$                  ! VP                  V P                  ,          VP                  VP&                  R7      V n        V P
                  P0                  V n        V P                  R8X  d   VP2                  MRV n        V P                  R8H  V n        \7        VP                  VP8                  R7      V n        \7        VP                  VP8                  R7      V n        R# r  r  r  s   &&&€r9   r@   Ú T5Gemma2MergedAttention.__init__H  r  r<   c                ól  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          RS[ P                  R,          RS[ P                  RS[R,          RS[S[,          RS[S[ P                  S[ P                  R,          S[S[ P                  ,          R,          3,          /# )	r1   r~   r  Úmerged_attention_maskNÚencoder_hidden_statesr  r®   r¢   )rC   r¤   rY   r   r   r   )r7   r8   s   "€r9   r:   r,  f  sº   ø€ ÷ TBñ TBñ —|‘|ðTBñ #¡5§<¡<±·±Ð#=Õ>ð	TBñ
  %Ÿ|™|¨dÕ2ðTBñ  %Ÿ|™|ðTBñ -¨tÕ3ðTBñ Ñ-Õ.ðTBñ 
‰u�|‰|™UŸ\™\¨DÕ0±%¹¿¹Õ2EÈÕ2LÐLÕ	MñTBr<   c                óò  € VP                   R R p. VORNV P                  N5pVP                   R R p	. V	ORNV P                  N5p
V P                  V4      P                  V4      P	                  ^^4      pV P                  V4      P                  V4      P	                  ^^4      pV P                  V4      P                  V4      P	                  ^^4      pV P                  V4      pV P                  V4      pVw  rï\        W¼Wï4      w  r¼Ve\   VP                  pVP                  WÍV P                  4      w  rÍVP                  P                  V P                  4      pVP                  pVe	   X'       g°   V P                  V4      P                  V
4      P	                  ^^4      pV P                  V4      P                  V
4      P	                  ^^4      pV P                  V4      pVe:   XP                  VVV P                  4      w  ppRVP                  V P                  &   MNXP                   V P                  ,          P"                  pVP                   V P                  ,          P$                  pTpV	^,          p\&        P(                  ! VV.^R7      p\&        P(                  ! VV.^R7      p\*        P,                  ! V P.                  P0                  \2        4      pT! V VVVV3RV P4                  '       d   V P6                  MRRV P8                  /VB w  ppVP:                  ! . VORN5!  P=                  4       pV P?                  V4      pVe   VRR V) 13,          pVRV) R 13,          pMR R ppVVV3# )NTr¹   rz   r  ræ   .rK   ) rZ   r§   r
  r  r¾   r  r  r  r  r×   Úself_attention_cacher  rý   Ú
is_updatedÚgetÚcross_attention_cacheÚlayersÚkeysÚvaluesrC   r¿   r   r  rj   r  rø   rë   r  ræ   rÛ   ró   r  )rF   r~   r  r0  r1  r  r®   r   r!  Úcross_input_shapeÚcross_hidden_shaper"  rô   rõ   rÀ   rÁ   r3  r4  r6  Úcross_key_statesÚcross_value_statesÚcross_key_sizer#  r÷   rö   Úself_attn_weightsÚcross_attn_weightss   &&&&&&,                    r9   rV   ÚT5Gemma2MergedAttention.forwardf  sp  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆØ1×7Ñ7¸¸Ð<ÐØDÐ0ÐD°"ÐD°d·m±mÑDÐð —{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà—{‘{ <Ó0ˆØ—[‘[ Ó,ˆ
à&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÒ&à#2×#GÑ#GÐ Ø';×'BÑ'BÀ:Ð]a×]kÑ]kÓ'lÑ$ˆJð )×3Ñ3×7Ñ7¸¿¹ÓGˆJØ$3×$IÑ$IÐ!àÒ"¯*Ø#Ÿ{™{Ð+@ÓA×FÑFÐGYÓZ×dÑdÐefÐhiÓjÐØ!%§¡Ð-BÓ!C×!HÑ!HÐI[Ó!\×!fÑ!fÐghÐjkÓ!lÐà#Ÿ{™{Ð+;Ó<ÐàÒ*Ø7L×7SÑ7SØ$Ð&8¸$¿.¹.ó8Ñ4Ð Ð"4ð >B�×*Ñ*¨4¯>©>Ñ:øà4×;Ñ;¸D¿N¹NÕK×PÑPÐØ!6×!=Ñ!=¸d¿n¹nÕ!M×!TÑ!TÐð $ˆØ*¨1Õ-ˆÜ—Y’Y 
Ð,<Ð=À1ÔEˆ
Ü—y’y ,Ð0BÐ!CÈÔKˆä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØ!ñ	%
ð /3¯m¯m¨m�D×*Ò*Àð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆ�\ð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—k‘k +Ó.ˆð Ò#Ø ,¨SÐ2B°N°?Ð2BÐ-BÕ CÐØ!-¨c°N°?Ñ3CÐ.CÕ!DÑà48¸$Ð1ÐØÐ-Ð/AÐAÐAr<   r%  r>   r'  re   s   @@r9   r*  r*  D  s)   ù‡ € á@÷Tó T÷<TB÷ TBò TBr<   r*  c                   óX   a a€ ] tR tRt oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )	ÚT5Gemma2EncoderLayeri½  zEncoder sub-layer.c                ó    <€ V ^8„  d   QhRS[ /# ©r1   rý   ©r5   )r7   r8   s   "€r9   r:   Ú!T5Gemma2EncoderLayer.__annotate__À  s   ø€ ÷ 7ñ 7©#ñ 7r<   c                óL  <€ \         SV `  4        VP                  V n        Wn        W n        VP
                  V,          V n        \        VVR 7      V n        \        VP                  VP                  R7      V n        \        VP                  VP                  R7      V n        \        V4      V n        \        VP                  VP                  R7      V n        \        VP                  VP                  R7      V n        \"        P$                  ! VP&                  4      V n        R# ©)rj   rý   ©r3   N)r?   r@   rp   rj   rý   r–   Úattention_typerú   Ú	self_attnr/   r  Úpre_self_attn_layernormÚpost_self_attn_layernormrg   ÚmlpÚpre_feedforward_layernormÚpost_feedforward_layernormrA   rx   ry   rz   r  s   &&&€r9   r@   ÚT5Gemma2EncoderLayer.__init__À  sÞ   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØŒØ"ŒØ$×0Ñ0°Õ;ˆÔä.ØØô
ˆŒô (7°v×7IÑ7IÈv×ObÑObÔ'cˆÔ$Ü(7¸×8JÑ8JÐPV×PcÑPcÔ(dˆÔ%ä˜vÓ&ˆŒÜ)8¸×9KÑ9KÐQW×QdÑQdÔ)eˆÔ&Ü*9¸&×:LÑ:LÐRX×ReÑReÔ*fˆÔ'ä—z’z &×"5Ñ"5Ó6ˆŽr<   c                ó  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          R,          RS[ P                  R,          RS[ P                  R,          RS[S[ P                  3,          /# )r1   r~   r  Nrå   rÂ   r¢   )rC   r¤   rY   Ú
LongTensorÚFloatTensor)r7   r8   s   "€r9   r:   rG  Ô  sz   ø€ ÷ ñ á—|‘|ðñ #¡5§<¡<±·±Ð#=Õ>ÀÕEðñ Ÿ™ tÕ+ð	ñ
 ×&Ñ&¨Õ-ðñ 
‰u× Ñ Ð!Õ	"ñr<   c                óP  € TpV P                  V4      pV P                  ! RR VRVRVRVRR/VB w  rV P                  V4      pW`P                  V4      ,           pTpV P	                  V4      pV P                  V4      pV P                  V4      pW`P                  V4      ,           pV# )r~   r  rå   rÂ   r  N© ©rM  rL  rN  rz   rP  rO  rQ  )rF   r~   r  rå   rÂ   r®   ÚresidualÚ_s   &&&&&,  r9   rV   ÚT5Gemma2EncoderLayer.forwardÔ  sÏ   € ð !ˆØ×4Ñ4°]ÓCˆØŸ>š>ñ 
Ø'ð
à 3ð
ð *ð
ð &ð	
ð
 !ð
ð ñ
Ñˆð ×5Ñ5°mÓDˆØ §<¡<°Ó#>Õ>ˆà ˆØ×6Ñ6°}ÓEˆØŸ™ Ó/ˆØ×7Ñ7¸ÓFˆØ §<¡<°Ó#>Õ>ˆØÐr<   ©rK  rj   rz   rp   rý   rO  rQ  rN  rP  rM  rL  r&  r'  re   s   @@r9   rC  rC  ½  s!   ù‡ € Ù÷7ó 7÷(÷ ò r<   rC  c                   óX   a a€ ] tR tRt oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )	ÚT5Gemma2DecoderLayeriñ  zFDecoder sub-layer: merged attention instead of vanilla self-attention.c                ó    <€ V ^8„  d   QhRS[ /# rE  rF  )r7   r8   s   "€r9   r:   Ú!T5Gemma2DecoderLayer.__annotate__ô  s   ø€ ÷ 7ñ 7©#ñ 7r<   c                óL  <€ \         SV `  4        VP                  V n        Wn        W n        VP
                  V,          V n        \        VVR 7      V n        \        VP                  VP                  R7      V n        \        VP                  VP                  R7      V n        \        V4      V n        \        VP                  VP                  R7      V n        \        VP                  VP                  R7      V n        \"        P$                  ! VP&                  4      V n        R# rI  )r?   r@   rp   rj   rý   r–   rK  r*  rL  r/   r  rM  rN  rg   rO  rP  rQ  rA   rx   ry   rz   r  s   &&&€r9   r@   ÚT5Gemma2DecoderLayer.__init__ô  sà   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØŒØ"ŒØ$×0Ñ0°Õ;ˆÔô 1ØØô
ˆŒô (7°v×7IÑ7IÈv×ObÑObÔ'cˆÔ$Ü(7¸×8JÑ8JÐPV×PcÑPcÔ(dˆÔ%ä˜vÓ&ˆŒÜ)8¸×9KÑ9KÐQW×QdÑQdÔ)eˆÔ&Ü*9¸&×:LÑ:LÐRX×ReÑReÔ*fˆÔ'ä—z’z &×"5Ñ"5Ó6ˆŽr<   c                ó2  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[ P                  R,          R	S[ P                  /# )
r1   r~   r  r0  NrÂ   r  Ú	use_cacher1  r¢   )rC   r¤   rY   rT  r   ÚboolrU  )r7   r8   s   "€r9   r:   r`  	  s�   ø€ ÷  ñ  á—|‘|ð ñ #¡5§<¡<±·±Ð#=Õ>ð ñ  %Ÿ|™|¨dÕ2ð	 ñ
 ×&Ñ&¨Õ-ð ñ -¨tÕ3ð ñ ˜$•;ð ñ  %Ÿ|™|¨dÕ2ð ñ 
×	Ñ	ñ r<   c                ó\  € Tp	V P                  V4      pV P                  ! RR VRVRVRVRVRVRV/VB w  p p
V P                  V4      pW�P                  V4      ,           pTp	V P	                  V4      pV P                  V4      pV P                  V4      pW�P                  V4      ,           pV# )r~   r  r0  rÂ   r  rd  r1  rW  rX  )rF   r~   r  r0  rÂ   r  rd  r1  r®   rY  rZ  s   &&&&&&&&,  r9   rV   ÚT5Gemma2DecoderLayer.forward	  sç   € ð !ˆØ×4Ñ4°]ÓCˆà"Ÿnšnñ 	
Ø'ð	
à 3ð	
ð #8ð	
ð &ð		
ð
 ,ð	
ð  ð	
ð #8ð	
ð ñ	
Ñˆ�q˜!ð ×5Ñ5°mÓDˆØ §<¡<°Ó#>Õ>ˆà ˆØ×6Ñ6°}ÓEˆØŸ™ Ó/ˆØ×7Ñ7¸ÓFˆØ §<¡<°Ó#>Õ>ˆØÐr<   r\  )NNNFNr'  re   s   @@r9   r^  r^  ñ  s!   ù‡ € ÙP÷7ó 7÷* ÷  ò  r<   r^  c                   óX   a a€ ] tR tRt oRtRV3R lV 3R llltV3R lR ltRtVtV ;t	# )	ÚT5Gemma2LMHeadi,  z.Head for language modeling (generation) tasks.c                ó,   <€ V ^8„  d   QhRS[ RS[ RS[/# )r1   rp   Ú
vocab_sizero   ©r5   re  )r7   r8   s   "€r9   r:   ÚT5Gemma2LMHead.__annotate__/  s'   ø€ ÷ Fñ F¡Cð F±Sð FÁñ Fr<   c                ó^   <€ \         SV `  4        \        P                  ! WVR 7      V n        R# )rn   N)r?   r@   rA   rr   Úout_proj)rF   rp   rk  ro   rG   s   &&&&€r9   r@   ÚT5Gemma2LMHead.__init__/  s    ø€ Ü‰ÑÔÜŸ	š	 +ÀÔEˆŽr<   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r1   r~   r¢   ©rC   r¤   )r7   r8   s   "€r9   r:   rm  3  s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r<   c                ó(   € V P                  V4      pV# r>   ©ro  )rF   r~   Úlogitss   && r9   rV   ÚT5Gemma2LMHead.forward3  s   € Ø—‘˜}Ó-ˆØˆr<   ru  )Fr'  re   s   @@r9   ri  ri  ,  s#   ù‡ € Ù8÷Fõ F÷÷ ð r<   ri  c                   óX   a a€ ] tR tRt oRtRV3R lV 3R llltV3R lR ltRtVtV ;t	# )	ÚT5Gemma2ClassificationHeadi8  z-Head for sentence-level classification tasks.c                ó,   <€ V ^8„  d   QhRS[ RS[ RS[/# )r1   rp   Ú
num_labelsÚclassifier_dropout_rater4   )r7   r8   s   "€r9   r:   Ú'T5Gemma2ClassificationHead.__annotate__;  s#   ø€ ÷ ;ñ ;¡Cð ;±Sð ;ÑSXñ ;r<   c                ó’   <€ \         SV `  4        \        P                  ! VR 7      V n        \        P
                  ! W4      V n        R# ))rê   N)r?   r@   rA   rx   rz   rr   ro  )rF   rp   r{  r|  rG   s   &&&&€r9   r@   Ú#T5Gemma2ClassificationHead.__init__;  s/   ø€ Ü‰ÑÔÜ—z’zÐ$;Ô<ˆŒÜŸ	š	 +Ó:ˆŽr<   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rr  rs  )r7   r8   s   "€r9   r:   r}  @  s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r<   c                óJ   € V P                  V4      pV P                  V4      pV# r>   ©rz   ro  )rF   r~   s   &&r9   rV   Ú"T5Gemma2ClassificationHead.forward@  s$   € ØŸ™ ]Ó3ˆØŸ™ mÓ4ˆØÐr<   r‚  )r  r'  re   s   @@r9   ry  ry  8  s!   ù‡ € Ù7÷;õ ;÷
÷ ð r<   ry  c                   óP   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚT5Gemma2MultiModalProjectoriF  c                ó    <€ V ^8„  d   QhRS[ /# ri   ©r,   )r7   r8   s   "€r9   r:   Ú(T5Gemma2MultiModalProjector.__annotate__G  s   ø€ ÷ \ñ \Ñ4ñ \r<   c                ó®  <€ \         SV `  4        \        P                  ! \        P
                  ! VP                  P                  VP                  P                  4      4      V n	        \        VP                  P                  VP                  P                  R 7      V n        \        VP                  P                  VP                  P                  ,          4      V n        \        VP"                  R,          4      V n        V P                   V P$                  ,          V n        \        P(                  ! V P&                  V P&                  R7      V n        R# )rJ  ç      à?)Úkernel_sizeÚstrideN)r?   r@   rA   rB   rC   rD   Úvision_configrp   Útext_configÚmm_input_projection_weightr/   Úlayer_norm_epsÚmm_soft_emb_normr5   Ú
image_sizeÚ
patch_sizeÚpatches_per_imageÚmm_tokens_per_imageÚtokens_per_sider‹  Ú	AvgPool2dÚavg_poolr{   s   &&€r9   r@   Ú$T5Gemma2MultiModalProjector.__init__G  sì   ø€ Ü‰ÑÔä*,¯,ª,Ü�KŠK˜×,Ñ,×8Ñ8¸&×:LÑ:L×:XÑ:XÓYó+
ˆÔ'ô !0Ø× Ñ ×,Ñ,°&×2FÑ2F×2UÑ2Uô!
ˆÔô "% V×%9Ñ%9×%DÑ%DÈ×H\ÑH\×HgÑHgÕ%gÓ!hˆÔÜ" 6×#=Ñ#=¸sÕ#BÓCˆÔØ×1Ñ1°T×5IÑ5IÕIˆÔÜŸš°×1AÑ1AÈ$×JZÑJZÔ[ˆŽr<   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r1   Úvision_outputsrs  )r7   r8   s   "€r9   r:   rˆ  W  s   ø€ ÷ @ñ @¡e§l¡lñ @r<   c                óž  € VP                   w  r#pVP                  ^^4      pVP                  W$V P                  V P                  4      pVP	                  4       pV P                  V4      pVP                  ^4      pVP                  ^^4      pV P                  V4      p\        P                  ! WpP                  4      pVP                  V4      # ©é   )rZ   r¾   rÛ   r”  ró   r˜  Úflattenr‘  rC   rî   r�  rT   )	rF   r›  Ú
batch_sizerZ  rp   Úreshaped_vision_outputsÚpooled_vision_outputsÚnormed_vision_outputsÚprojected_vision_outputss	   &&       r9   rV   Ú#T5Gemma2MultiModalProjector.forwardW  sÇ   € Ø%3×%9Ñ%9Ñ"ˆ
�{à"0×":Ñ":¸1¸aÓ"@ÐØ"9×"AÑ"AØ T×%;Ñ%;¸T×=SÑ=Só#
Ðð #:×"DÑ"DÓ"FÐà $§¡Ð.EÓ FÐØ 5× =Ñ =¸aÓ @ÐØ 5× ?Ñ ?ÀÀ1Ó EÐà $× 5Ñ 5Ð6KÓ LÐä#(§<¢<Ð0E×GfÑGfÓ#gÐ Ø'×/Ñ/°Ó?Ð?r<   )r˜  r‹  r�  r‘  r”  r–  r€   re   s   @@r9   r…  r…  F  s#   ù‡ € ÷\ó \÷ @÷ @ð @r<   r…  c                   ó^   a a€ ] tR tRt oRtRV3R lV 3R llltV3R lV 3R lltRtVtV ;t	# )	ÚT5Gemma2TextScaledWordEmbeddingij  zCT5Gemma2 Embedding: override to add eoi token embedding separately.c          
      ó8   <€ V ^8„  d   QhRS[ RS[ RS[ RS[RS[ /# )r1   Únum_embeddingsÚembedding_dimÚpadding_idxÚembed_scaleÚeoi_token_indexr4   )r7   r8   s   "€r9   r:   Ú,T5Gemma2TextScaledWordEmbedding.__annotate__m  sD   ø€ ÷ Kñ KáðKñ ðKñ ð	Kñ
 ðKñ ñKr<   c                ó  <€ \         SV `  WV4       W@n        \        P                  ! \
        P                  ! V4      R R7      V n        WPn        \        P                  ! \
        P                  ! V P                  4      4      V n        R# )Fr�   N)r?   r@   Úscalar_embed_scalerA   rš   rC   Útensorr¬  r­  rB   rD   rª  Úeoi_embedding)rF   r©  rª  r«  r¬  r­  rG   s   &&&&&&€r9   r@   Ú(T5Gemma2TextScaledWordEmbedding.__init__m  s\   ø€ ô 	‰Ñ˜¸ÔDØ"-ÔÜŸ9š9¤U§\¢\°+Ó%>È5ÔQˆÔØ.ÔÜŸ\š\¬%¯+ª+°d×6HÑ6HÓ*IÓJˆÖr<   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r1   Ú	input_idsrs  )r7   r8   s   "€r9   r:   r®  {  s   ø€ ÷  ñ  ¡§¡ñ  r<   c                óö   <€ \         SV `  V4      V P                  P                  V P                  P
                  4      ,          pV P                  P                  VP
                  4      W!V P                  8H  &   V# r>   )r?   rV   r¬  r­   rE   r©   r²  r­  )rF   rµ  Úinput_embeddingsrG   s   && €r9   rV   Ú'T5Gemma2TextScaledWordEmbedding.forward{  sd   ø€ Ü ™7™?¨9Ó5¸×8HÑ8H×8KÑ8KÈDÏKÉK×L]ÑL]Ó8^Õ^ÐØ>B×>PÑ>P×>SÑ>SÐTd×TjÑTjÓ>kÐ d×&:Ñ&:Ñ:Ñ;ØÐr<   )r¬  r²  r­  r°  )rS   é è r'  re   s   @@r9   r§  r§  j  s#   ù‡ € ÙM÷Kõ K÷ ÷  ó  r<   r§  c                   óª   a a€ ] tR tRt oRtRtRR.tR.tRtRt	Rt
RtRtRtRt]P                   ! 4       V 3R	 l4       tV3R
 lR ltV3R ltRtVtV ;t# )ÚT5Gemma2PreTrainedModeli�  ÚmodelTrC  r^  r  FNc                óH  <€ \         SV `  V4       \        V\        4      '       d#   \        P
                  ! VP                  4       R# \        V\        4      '       dN   \        P
                  ! VP                  4       \        P                  ! VP                  VP                  4       R# \        V\        4      '       dÛ   VP                  P                  P                  ^ ,          R
,          p\        P                   ! VP                  P                  RV P"                  P$                  V,          R7       \'        VP                  R4      '       dG   VP                  P(                  e-   \        P
                  ! VP                  P(                  4       R# R# R# RVP*                  P,                  9   d#   \        P
                  ! VP                  4       R# \        V\.        4      '       d´   VP0                   F¡  pVP2                  pVP4                  V,          R8w  d   \6        VP4                  V,          ,          pV! VP"                  VR7      w  rV\        P8                  ! \;        W R24      V4       \        P8                  ! \;        W R	24      V4       K£  	  R# R# )r   r  )rN   Ústdro   NÚRMSNormr‰   rŠ   rŒ   r�   rì   )r?   Ú_init_weightsr¼   r…  ÚinitÚzeros_r�  r§  r²  Ú	constant_r¬  r°  ry  ro  rE   rZ   Únormal_rj   Úinitializer_ranger  ro   rG   r^   r‚   r–   r˜   rˆ   r   Úcopy_rª   )rF   rá   Úscaler‹   r�   rž   rZ  rG   s   &&     €r9   rÀ  Ú%T5Gemma2PreTrainedModel._init_weightsš  sÄ  ø€ ä‰Ñ˜fÔ%Ü�fÔ9×:Ò:Ü�KŠK˜×9Ñ9Ö:Ü˜Ô ?×@Ò@Ü�KŠK˜×,Ñ,Ô-Ü�NŠN˜6×-Ñ-¨v×/HÑ/HÖIÜ˜Ô :×;Ò;Ø—O‘O×*Ñ*×0Ñ0°Õ3°tÕ;ˆEÜ�LŠL˜Ÿ™×/Ñ/°c¸t¿{¹{×?\Ñ?\Ð_dÕ?dÕeÜ�v—‘¨×/Ò/°F·O±O×4HÑ4HÒ4TÜ—’˜FŸO™O×0Ñ0Ö1ñ 5UÑ/ð ˜&×*Ñ*×3Ñ3Ô3Ü�KŠK˜Ÿ™Ö&Ü˜Ô 7×8Ò8Ø$×0Ô0�
Ø%×EÑE�Ø×#Ñ# JÕ/°9Ô<Ü#6°v×7GÑ7GÈ
Õ7SÕ#T�LÙ#/°·±È*Ô#UÑ �Ü—
’
œ7 6¨\¸Ð+CÓDÀmÔTÜ—
’
œ7 6¨\Ð9KÐ+LÓMÈ}Ö]ó 1ñ 9r<   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r1   Úlabelsrs  )r7   r8   s   "€r9   r:   Ú$T5Gemma2PreTrainedModel.__annotate__³  s   ø€ ÷ !ñ !¹E¿L¹Lñ !r<   c                óB  € V P                   P                  pVP                  pVP                  pVf   \	        R4      hVP                  VP                  4      pVRRR13,          P                  4       VR&   W5R&   Vf   \	        R4      hVP                  VR8H  V4       V# )	zÚ
Shifts input_ids to the right, prepends the decoder_start_token_id, and handles
pad_token_id replacement for labels that were -100.
This is a common preparation step for decoder inputs in sequence-to-sequence models.
Nz:self.model.config.decoder.bos_token_id has to be defined. .z9self.model.config.decoder.pad_token_id has to be defined.rK   ).:rž  NN).r   iœÿÿÿ)	rj   ÚdecoderÚbos_token_idÚpad_token_idÚ
ValueErrorÚ	new_zerosrZ   r›   Úmasked_fill_)rF   rÊ  Údecoder_configÚdecoder_start_token_idrÏ  Úshifted_input_idss   &&    r9   Ú%prepare_decoder_input_ids_from_labelsÚ=T5Gemma2PreTrainedModel.prepare_decoder_input_ids_from_labels³  s¬   € ð Ÿ™×,Ñ,ˆØ!/×!<Ñ!<ÐØ%×2Ñ2ˆà!Ò)ÜÐYÓZÐZð #×,Ñ,¨V¯\©\Ó:ÐØ%+¨C°°"°¨HÕ%5×%;Ñ%;Ó%=Ð˜'Ñ"Ø$:˜&Ñ!àÒÜÐXÓYÐYð 	×&Ñ&Ð'8¸DÑ'@À,ÔOà Ð r<   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # ri   ©r*   )r7   r8   s   "€r9   r:   rË  �  s   ø‡ ‚ áÑò r<   rW  )ÚimageÚtext)r^   r_   r`   ra   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendÚ_can_record_outputsÚinput_modalitiesrC   rÊ   rÀ  rÖ  Ú__annotate_func__rb   rc   rd   re   s   @@r9   r»  r»  �  s…   ù‡ € ð  ÐØ&*Ð#ð 	ØðÐð $5Ð"5Ðð !ÐØ€NàÐà!ÐØ"&ÐàÐØ(Ðà
‡]‚]ƒ_ô^ó ð^÷0!ð !÷e … r<   r»  c                ó0   € V ^8„  d   QhR\         R\        /# )r1   r  r¢   )r5   r   )r7   s   "r9   r:   r:   Ï  s   € ÷ ñ ´ð Ìñ r<   c                ó   a a€ R VV 3R llpV# )zD
This creates uni/bidirectional attention mask with sliding window.
c          
      óT   € V ^8„  d   QhR\         R\         R\         R\         R\        /# )r1   Ú	batch_idxÚhead_idxÚq_idxÚkv_idxr¢   rl  )r7   s   "r9   r:   Ú2sliding_window_mask_function.<locals>.__annotate__Ô  s/   € ÷ 	&ñ 	&œcð 	&¬Sð 	&¼ð 	&Äcð 	&Ìdñ 	&r<   c                 ó¶   <€ S	'       d   S
^ rTMS
^,           ^,          S
^,          ^,           rTW#,
          pV^ 8¬  Wd8  ,          pV^ 8  V) V8  ,          pWx,          # )r   rW  )rë  rì  rí  rî  Úleft_window_sizeÚright_window_sizeÚdistÚ	left_maskÚ
right_maskr  r  s   &&&&     €€r9   Ú
inner_maskÚ0sliding_window_mask_function.<locals>.inner_maskÔ  sc   ø€ ßØ2@À!Ñ/à4BÀQÕ4FÈ1Õ3LÈ~ÐbcÕNcÐfgÕNgÐ/à�~ˆØ˜Q‘Y 4Ñ#:Õ;ˆ	Ø˜Q‘h D 5Ð+<Ñ#<Õ=ˆ
ØÕ%Ð%r<   rW  )r  r  rö  s   ff r9   Úsliding_window_mask_functionrø  Ï  s   ù€ ÷
	&ñ 	&ð Ðr<   c                   óŽ   a a€ ] tR tRt oR]R]/tR
V3R lV 3R lllt]]	]
RV3R lR ll4       4       4       tV3R ltR	tVtV ;t# )ÚT5Gemma2TextEncoderiâ  Ú
attentionsr~   c                ó&   <€ V ^8„  d   QhRS[ RS[/# ©r1   rj   r­  rþ   )r7   r8   s   "€r9   r:   Ú T5Gemma2TextEncoder.__annotate__é  s   ø€ ÷ ñ á"ðñ ñr<   c           	     ón  <€ \         SV `  V4       VP                  V n        VP                  V n        \        VP                  VP                  V P                  VP                  R ,          VR7      V n        \        VP                  VP                  R7      V n
        RV n        \        P                  ! \        VP                  4       Uu. uF  p\!        W4      NK  	  up4      V n        \        P$                  ! VP&                  4      V n        \+        V4      V n        V P/                  4        R# u upi ©rŠ  )r¬  r­  rJ  FN)r?   r@   rÏ  r«  rk  r§  rp   Úembed_tokensr/   r  ÚnormÚgradient_checkpointingrA   Ú
ModuleListÚrangeÚnum_hidden_layersrC  r7  rx   ry   rz   r‚   Ú
rotary_embÚ	post_init©rF   rj   r­  rý   rG   s   &&& €r9   r@   ÚT5Gemma2TextEncoder.__init__é  sõ   ø€ ô
 	‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒä;Ø×ÑØ×ÑØ×ÑØ×*Ñ*¨CÕ/Ø+ô
ˆÔô $ F×$6Ñ$6¸F×<OÑ<OÔPˆŒ	Ø&+ˆÔ#ä—m’mÜFKÈF×LdÑLdÔFeÓfÑFe¸Ô! &Ö4ÑFeÑfó
ˆŒô —z’z &×"5Ñ"5Ó6ˆŒÜ1°&Ó9ˆŒð 	�‰Öùò gó   ÃD2c                óþ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          RS[/# )	r1   rµ  Nrå   rÂ   Úinputs_embedsÚtoken_type_idsr®   r¢   ©rC   rT  r¤   rU  r   r    r   )r7   r8   s   "€r9   r:   rþ    sŠ   ø€ ÷ <
ñ <
á×#Ñ# dÕ*ð<
ñ Ÿ™ tÕ+ð<
ñ ×&Ñ&¨Õ-ð	<
ñ
 ×(Ñ(¨4Õ/ð<
ñ Ÿ™ tÕ+ð<
ñ Ñ+Õ,ð<
ñ 
ñ<
r<   c                óä  € VR J VR J,          '       d   \        R4      hVP                  RR 4       Vf   V P                  V4      pVfD   \        P                  ! ^ VP
                  ^,          VP                  R7      P                  ^ 4      p\        T;p\        4      '       gN   RV P                  RVRV/pR\        R/ VB R\        R/ VBR	\        V P                  P                  R
R7      /B /pTp	/ p
\        V P                  P                  4       F  pV P!                  W“V4      W«&   K  	  V P#                  V	4      p	\%        V P&                  R V P                  P(                   4       FS  w  rÍV! V	W P                  P                  V,          ,          WpP                  P                  V,          ,          V3/ VB p	KU  	  V P+                  V	4      p	V P#                  V	4      p	\-        V	R7      # )Nú:You must specify exactly one of input_ids or inputs_embedsr  ©rƒ   rj   r  rå   Úfull_attentionr  Úand_mask_functionF)r  )Úlast_hidden_staterW  )rÐ  Úpopr  rC   r¬   rZ   rƒ   rÑ   r¼   Údictrj   r   rø  r  r•   r–   r  rz   Ú	enumerater7  r  r  r   )rF   rµ  rå   rÂ   r  r  r®   Úself_attn_mask_mappingÚmask_kwargsr~   r  r‹   ÚiÚlayer_modules   &&&&&&,       r9   rV   ÚT5Gemma2TextEncoder.forward  sÚ  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZð 	�
‰
Ð$ dÔ+àÒ Ø ×-Ñ-¨iÓ8ˆMàÒÜ Ÿ<š<¨¨=×+>Ñ+>¸qÕ+AÈ-×J^ÑJ^Ô_×iÑiÐjkÓlˆLä°NÐBÐ0ÄD×IÒIà˜$Ÿ+™+Ø Ø  .ðˆKð !Ô";Ñ"J¸kÑ"JØ#Ô%>ñ &Ø!ñ&ä&BÀ4Ç;Á;×C]ÑC]ÐinÔ&oò&ð&Ð"ð &ˆð !ÐÜ˜dŸk™k×5Ñ5Ö6ˆJØ.2¯o©o¸mÐ[eÓ.fÐÓ+ñ 7ð Ÿ™ ]Ó3ˆä(¨¯©Ð5T°t·{±{×7TÑ7TÐ)UÖV‰OˆAÙ(ØØ#§K¡K×$;Ñ$;¸AÕ$>Õ?Ø&§{¡{×'>Ñ'>¸qÕ'AÕBØñ	ð
 ñŠMñ  Wð Ÿ	™	 -Ó0ˆØŸ™ ]Ó3ˆÜØ+ô
ð 	
r<   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # ri   rk   )r7   r8   s   "€r9   r:   rþ  â  s   ø‡ ‚ ÙÑò r<   ©rz   r  r  r7  r  r«  r  rk  ©r¹  )NNNNN)r^   r_   r`   ra   rú   rC  rå  r@   r&   r(   r!   rV   rç  rb   rc   rd   re   s   @@r9   rú  rú  â  sV   ù‡ € ð 	Ð+ØÐ-ðÐ÷
õ ð8  ØØ÷<
ñ <
ó ó ó  ð<
÷M … r<   rú  c                   ó²   a a€ ] tR tRt oRV3R lV 3R llltR tR t]]V3R lR l4       4       t	V3R lR	 lt
]RV3R
 lR ll4       tV3R ltRtVtV ;t# )ÚT5Gemma2EncoderiG  c                ó&   <€ V ^8„  d   QhRS[ RS[/# rý  )r,   r5   )r7   r8   s   "€r9   r:   ÚT5Gemma2Encoder.__annotate__J  s   ø€ ÷ ñ á%ðñ ñr<   c                óþ   <€ \         SV `  V4       \        P                  VP                  VR 7      V n        \        P                  ! VP                  R7      V n	        \        V4      V n        V P                  4        R# ))r­  ©rj   N)r?   r@   rú  Ú_from_configrŽ  Ú
text_modelr)   Úfrom_configr�  Úvision_towerr…  Úmulti_modal_projectorr  )rF   rj   r­  rG   s   &&&€r9   r@   ÚT5Gemma2Encoder.__init__J  sb   ø€ ô
 	‰Ñ˜Ô ä-×:Ñ:¸6×;MÑ;MÐ_nÐ:ÓoˆŒÜ%×1Ò1¸×9MÑ9MÔNˆÔÜ%@ÀÓ%HˆÔ"ð 	�‰Ör<   c                ó6   € V P                   P                  4       # r>   )r(  Úget_input_embeddingsr[   s   &r9   r.  Ú$T5Gemma2Encoder.get_input_embeddingsX  s   € Ø�‰×3Ñ3Ó5Ð5r<   c                ó8   € V P                   P                  V4      # r>   )r(  Úset_input_embeddings©rF   Únew_embeddingss   &&r9   r1  Ú$T5Gemma2Encoder.set_input_embeddings[  s   € Ø�‰×3Ñ3°NÓCÐCr<   c                ó`   <€ V ^8„  d   QhRS[ P                  RS[S[,          RS[S[,          /# ©r1   Úpixel_valuesr®   r¢   ©rC   r¤   r   r    rY   r   )r7   r8   s   "€r9   r:   r$  `  s5   ø€ ÷ 
ñ 
Ù!ŸL™Lð
Ù4:Ñ;MÕ4Nð
á	Ñ+Õ	+ñ
r<   c                óx   € V P                   ! RR VRR/VB pVP                  pV P                  V4      pWSn        V# )r7  Úreturn_dictTrW  )r*  r  r+  Úpooler_output)rF   r7  r®   r›  r  Úimage_featuress   &&,   r9   Úget_image_featuresÚ"T5Gemma2Encoder.get_image_features^  sL   € ð ×*Ò*Ña¸ÐaÐRVÐaÐZ`ÑaˆØ*×<Ñ<ÐØ×3Ñ3Ð4EÓFˆØ'5Ô$àÐr<   c                ó„   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  /# )r1   rµ  Nr  r<  )rC   rT  rU  )r7   r8   s   "€r9   r:   r$  l  sE   ø€ ÷ "ñ "á×#Ñ# dÕ*ð"ñ ×(Ñ(¨4Õ/ð"ñ ×)Ñ)ñ	"r<   c           
     óR  € V P                   P                  pVfj   Vf   \        R4      hW P                  4       ! \        P
                  ! RV\        P                  VP                  R7      4      8H  pVP                  R4      pMW8H  pVP                  4       pVP                  R4      P                  VP                  4      pVP                  ^ ,          VP                  ^,          ,          p\        WbP                  R,          ,          VP                  4       8H  RV RV 24       V# )z×
Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
equal to the length of multimodal features. If the lengths are different, an error is raised.
z9Either `input_ids` or `inputs_embeds` has to be provided.r´   z6Image features and image tokens do not match: tokens: z, features rW  rK   )rj   Úimage_token_idrÐ  r.  rC   ÚfullÚlongrƒ   ÚallÚsumrÑ   r­   rZ   r#   Únumel)rF   rµ  r  r<  rA  Úspecial_image_maskÚn_image_tokensÚn_image_featuress   &&&&    r9   Úget_image_placeholder_maskÚ*T5Gemma2Encoder.get_image_placeholder_maskl  s  € ð Ÿ™×3Ñ3ˆØÒØÒ$Ü Ð!\Ó]Ð]Ø!.×2KÑ2KÔ2MÜ—
’
˜2˜~´U·Z±ZÈ×H\ÑH\Ô]ó3ñ "Ðð "4×!7Ñ!7¸Ó!;Ñà!*Ñ!<Ðà+×/Ñ/Ó1ˆØ/×9Ñ9¸"Ó=×@Ñ@À×AUÑAUÓVÐØ)×/Ñ/°Õ2°^×5IÑ5IÈ!Õ5LÕLÐÜØ×0Ñ0°Õ4Õ4¸×8LÑ8LÓ8NÑNØDÀ^ÐDTÐT_Ð`pÐ_qÐrô	
ð "Ð!r<   c                ó&  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          R	S[/# )
r1   rµ  Nrå   rÂ   r  r7  r  r®   r¢   r  )r7   r8   s   "€r9   r:   r$  ‹  sŸ   ø€ ÷ !ñ !á×#Ñ# dÕ*ð!ñ Ÿ™ tÕ+ð!ñ ×&Ñ&¨Õ-ð	!ñ
 ×(Ñ(¨4Õ/ð!ñ ×'Ñ'¨$Õ.ð!ñ Ÿ™ tÕ+ð!ñ Ñ+Õ,ð!ñ 
ñ!r<   c           	     ó†  € VR J VR J,          '       d   \        R4      hVf   V P                  P                  V4      pVeh   V P                  VRR7      P                  pVP                  VP                  VP                  4      pV P                  WVR7      p	VP                  W˜4      pV P                  ! RRVRVRV/VB p
V
# )	Nr  T)r:  )r  r<  r  rå   rÂ   rW  )
rÐ  r(  r  r=  r;  r­   rƒ   r©   rJ  Úmasked_scatter)rF   rµ  rå   rÂ   r  r7  r  r®   r<  Ú
image_maskÚoutputss   &&&&&&&,   r9   rV   ÚT5Gemma2Encoder.forwardŠ  sã   € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ Ø ŸO™O×8Ñ8¸ÓCˆMàÒ#Ø!×4Ñ4°\ÈtÐ4ÓT×bÑbˆNØ+×.Ñ.¨}×/CÑ/CÀ]×EXÑEXÓYˆNà×8Ñ8ØÀ~ð 9ó ˆJð *×8Ñ8¸ÓTˆMà—/’/ñ 
Ø'ð
à)ð
ð &ð
ð ñ	
ˆð ˆr<   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # ri   r‡  )r7   r8   s   "€r9   r:   r$  G  ó   ø‡ ‚ Ù!Ñ!ò r<   )r+  r(  r*  r   )NNNNNN)r^   r_   r`   ra   r@   r.  r1  r"   r!   r=  rJ  rV   rç  rb   rc   rd   re   s   @@r9   r"  r"  G  sa   ù‡ € ÷õ ò6òDð Ø÷
ó ó ð
÷"ð "ð< ÷!ñ !ó ð!÷I … r<   r"  c                   ó²   a a€ ] tR tRt oR]! ]^R7      R]! ]^R7      R]/tRV3R lV 3R lllt]	]
]RV3R lR	 ll4       4       4       tV3R
 ltRtVtV ;t# )ÚT5Gemma2Decoderi¯  rû  )ÚindexÚcross_attentionsr~   c                ó&   <€ V ^8„  d   QhRS[ RS[/# rý  )r+   r5   )r7   r8   s   "€r9   r:   ÚT5Gemma2Decoder.__annotate__·  s   ø€ ÷ ñ Ñ4ð Ásñ r<   c           	     ón  <€ \         SV `  V4       VP                  V n        VP                  V n        \        VP                  VP                  VP                  VP                  R ,          VR7      V n        \        VP                  VP                  R7      V n
        RV n        \        P                  ! \        VP                  4       Uu. uF  p\!        W4      NK  	  up4      V n        \        P$                  ! VP&                  4      V n        \+        V4      V n        V P/                  4        R# u upi r   )r?   r@   rÏ  r«  rk  r§  rp   r  r/   r  r  r  rA   r  r  r  r^  r7  rx   ry   rz   r‚   r  r  r	  s   &&& €r9   r@   ÚT5Gemma2Decoder.__init__·  sñ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒä;Ø×ÑØ×ÑØ×ÑØ×*Ñ*¨CÕ/Ø+ô
ˆÔô $ F×$6Ñ$6¸F×<OÑ<OÔPˆŒ	Ø&+ˆÔ#ä—m’mÜFKÈF×LdÑLdÔFeÓfÑFe¸Ô! &Ö4ÑFeÑfó
ˆŒô —z’z &×"5Ñ"5Ó6ˆŒÜ1°&Ó9ˆŒØ�‰Öùò	 gr  c                óN  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[ P                  R,          R	S[ P                  R,          R
S[S[,          RS[/
# )r1   rµ  Nrå   rÂ   r  r  rd  r1  Úencoder_attention_maskr®   r¢   )	rC   rT  r¤   r   rU  re  r   r    r   )r7   r8   s   "€r9   r:   rY  Ð  sÆ   ø€ ÷ \
ñ \
á×#Ñ# dÕ*ð\
ñ Ÿ™ tÕ+ð\
ñ ×&Ñ&¨Õ-ð	\
ñ
 -¨tÕ3ð\
ñ ×(Ñ(¨4Õ/ð\
ñ ˜$•;ð\
ñ  %Ÿ|™|¨dÕ2ð\
ñ !&§¡¨tÕ 3ð\
ñ Ñ+Õ,ð\
ñ 
3ñ\
r<   c	                ó¦  € VR J VR J,          '       d   \        R4      hVf   \        R4      hVf   V P                  V4      pV P                  '       g5   V'       d-   Vf)   \        \	        V P
                  R7      \	        4       4      pVfb   Ve   VP                  4       M^ p
\        P                  ! VP                  ^,          VP                  R7      V
,           pVP                  ^ 4      p\        T;p\        4      '       gA   RV P
                  RTRTRVe   VP                  MR R	VR
R/pR\        R/ VB R\!        R/ VB /p\        T;p\        4      '       g   R\#        V P
                  VVVRR7      /pR\        P$                  ! VR,          VR,          .RR7      R\        P$                  ! VR,          VR,          .RR7      /pTp/ p\'        V P
                  P(                  4       F  pV P+                  WóV4      VV&   K  	  V P-                  V4      p\/        V P0                  R V P
                  P2                   4       FX  w  ppV! VVV P
                  P(                  V,          ,          WàP
                  P(                  V,          ,          VVVV3/ V	B pKZ  	  V P5                  V4      pV P-                  V4      p\7        VVR7      # )Nr  z0`encoder_hidden_states` must be given in decoderr&  r  rj   r  rå   r  rÂ   Úallow_is_causal_skipFr  r  )rj   r  rå   r1  Úallow_is_bidirectional_skipr¹   )r  r  rW  rK   )rÐ  r  rë   r   r   rj   Úget_seq_lengthrC   r¬   rZ   rƒ   rÑ   r¼   r  r3  r   r   r   r¿   r•   r–   r  rz   r  r7  r  r  r   )rF   rµ  rå   rÂ   r  r  rd  r1  r]  r®   Úpast_seen_tokensr  r  Úcross_attn_mask_mappingÚmerged_attn_mask_mappingr~   r  r‹   r  r  s   &&&&&&&&&,          r9   rV   ÚT5Gemma2Decoder.forwardÍ  sÉ  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZØ Ò(ÜÐOÓPÐPàÒ Ø ×-Ñ-¨iÓ8ˆMà�}�}ˆ}§¨Ò/FÜ1´,ÀdÇkÁkÔ2RÔT`ÓTbÓcˆOàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLä°NÐBÐ0ÄD×IÒIð ˜$Ÿ+™+Ø Ø  .Ø!È?ÒKf ?×#GÒ#GÐlpØ Ø&¨ðˆKð !Ô"4Ñ"C°{Ñ"CØ#Ô%FÑ%UÈÑ%Uð&Ð"ô
 Ð5KÐKÐ1ÌT×RÒRà Ô";ØŸ;™;Ø"/Ø#9Ø*?Ø05ô#ð'Ð#ð œeŸišiØ'Ð(8Õ9Ð;RÐScÕ;dÐeÐkmôð  ¤§¢Ø'Ð(;Õ<Ð>UÐVfÕ>gÐhÐnpô"ð	$
Ð ð &ˆð !ÐÜ˜dŸk™k×5Ñ5Ö6ˆJØ.2¯o©o¸mÐ[eÓ.fÐ 
Ó+ñ 7ð Ÿ™ ]Ó3ˆä(¨¯©Ð5T°t·{±{×7TÑ7TÐ)UÖV‰OˆAˆ|Ù(ØØ# D§K¡K×$;Ñ$;¸AÕ$>Õ?Ø(¯©×)@Ñ)@ÀÕ)CÕDØØØØ%ñ	ð ñ	ŠMñ  Wð Ÿ	™	 -Ó0ˆØŸ™ ]Ó3ˆÜ8Ø+Ø+ô
ð 	
r<   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # ri   )r+   )r7   r8   s   "€r9   r:   rY  ¯  rS  r<   r  r   )NNNNNNNN)r^   r_   r`   ra   r'   r*  r^  rå  r@   r&   r(   r!   rV   rç  rb   rc   rd   re   s   @@r9   rU  rU  ¯  sm   ù‡ € ð 	‘nÐ%<ÀAÔFØ™NÐ+BÈ!ÔLØÐ-ðÐ÷õ ð,  ØØ÷\
ñ \
ó ó ó  ð\
÷C … r<   rU  c                   óœ   a a€ ] tR tRt oRRRR/tV3R lV 3R lltRV3R lR	 lltR
 tR tR t	]
]RV3R lR ll4       4       tRtVtV ;t# )ÚT5Gemma2Modeli/  zdecoder.embed_tokens.weightz&encoder.text_model.embed_tokens.weightz"decoder.embed_tokens.eoi_embeddingz-encoder.text_model.embed_tokens.eoi_embeddingc                ó    <€ V ^8„  d   QhRS[ /# ri   rÙ  )r7   r8   s   "€r9   r:   ÚT5Gemma2Model.__annotate__6  s   ø€ ÷ ñ ™~ñ r<   c                óÚ   <€ \         SV `  V4       \        VP                  VP                  4      V n        \        VP                  VP                  4      V n        V P                  4        R # r>   )r?   r@   r"  Úencoderr­  rU  rÍ  r  r{   s   &&€r9   r@   ÚT5Gemma2Model.__init__6  sL   ø€ Ü‰Ñ˜Ô ô ' v§~¡~°v×7MÑ7MÓNˆŒÜ& v§~¡~°v×7MÑ7MÓNˆŒà�‰Ör<   c                ó.   <€ V ^8„  d   QhRS[ R,          /# ©r1   ÚmodalityN©r£   )r7   r8   s   "€r9   r:   rj  ?  s   ø€ ÷ ñ ¡C¨$¥Jñ r<   c                ó   € V P                   # r>   )rl  ©rF   rp  s   &&r9   Úget_encoderÚT5Gemma2Model.get_encoder?  ó   € Ø�|‰|Ðr<   c                ó   € V P                   # r>   ©rÍ  r[   s   &r9   Úget_decoderÚT5Gemma2Model.get_decoderB  rv  r<   c                ó6   € V P                   P                  4       # r>   )rl  r.  r[   s   &r9   r.  Ú"T5Gemma2Model.get_input_embeddingsE  s   € Ø�|‰|×0Ñ0Ó2Ð2r<   c                ó8   € V P                   P                  V4      # r>   )rl  r1  r2  s   &&r9   r1  Ú"T5Gemma2Model.set_input_embeddingsH  s   € Ø�|‰|×0Ñ0°Ó@Ð@r<   c                óÚ  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[R,          R
S[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[S[	,          RS[
/# )r1   rµ  Nr7  rå   rÂ   Údecoder_input_idsÚdecoder_attention_maskÚdecoder_position_idsÚencoder_outputsr  r  Údecoder_inputs_embedsrd  r®   r¢   )rC   rT  rU  Ú
BoolTensorr   r   r¤   re  r   r    r   )r7   r8   s   "€r9   r:   rj  M  s  ø€ ÷ =
ñ =
ñ ×#Ñ# dÕ*ð=
ñ ×'Ñ'¨$Õ.ð	=
ñ
 ×)Ñ)¨DÕ0ð=
ñ ×&Ñ&¨Õ-ð=
ñ !×+Ñ+¨dÕ2ð=
ñ !&× 0Ñ 0°4Õ 7ð=
ñ $×.Ñ.°Õ5ð=
ñ )¨4Õ/ð=
ñ -¨tÕ3ð=
ñ —|‘| dÕ*ð=
ñ  %Ÿ|™|¨dÕ2ð=
ñ  ˜$•;ð!=
ñ" Ñ+Õ,ð#=
ñ$ 
ñ%=
r<   c                ól  € Vf   V P                   ! RRVRVRVRV
RVRR/VB pVP                  pV P                  ! RRVRVRVRVRV	R	VR
VRVRR/	VB p\        VP                  VP                  VP
                  VP                  VP                  VP                  VP
                  VP                  R7      # )a8  
decoder_position_ids (`torch.LongTensor` of shape `(batch_size, decoder_sequence_length)`, *optional*):
    Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the range `[0,
    config.decoder.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)
rµ  rå   rÂ   r  r7  r:  Tr  r1  r]  rd  )r  r  Údecoder_hidden_statesÚdecoder_attentionsrW  Úencoder_last_hidden_stater1  Úencoder_attentionsrW  )rl  r  rÍ  r   r  r~   rû  rW  )rF   rµ  r7  rå   rÂ   r€  r�  r‚  rƒ  r  r  r„  rd  r®   r1  Údecoder_outputss   &&&&&&&&&&&&&,  r9   rV   ÚT5Gemma2Model.forwardK  s/  € ð6 Ò"Ø"Ÿlšlñ Ø#ðà-ðð *ðð ,ð	ð
 *ðð !ðð ñˆOð !0× AÑ AÐð Ÿ,š,ñ 
Ø'ð
à1ð
ð .ð
ð 0ð	
ð
 ,ð
ð #8ð
ð $2ð
ð  ð
ð ð
ð ñ
ˆô "Ø-×?Ñ?Ø+×;Ñ;Ø"1×"?Ñ"?Ø.×9Ñ9Ø,×=Ñ=Ø&5×&GÑ&GØ"1×"?Ñ"?Ø.×9Ñ9ô	
ð 		
r<   )rÍ  rl  r>   )NNNNNNNNNNNN)r^   r_   r`   ra   Ú_tied_weights_keysr@   rt  ry  r.  r1  r"   r!   rV   rb   rc   rd   re   s   @@r9   rh  rh  /  se   ù‡ € ð 	&Ð'OØ,Ð.]ðÐ÷
ó ÷ò òò3òAð Ø÷=
ñ =
ó ó ÷=
ð =
r<   rh  c                   ó
  a a€ ] tR tRt oRR/tRR/tRR.R.3/tV3R lV 3R	 lltR
 tR t	R t
R tRV3R lR lltR t]]V3R lR l4       4       t]R 4       t]]RV3R lR ll4       4       tV3R lV 3R lltRtVtV ;t# )Ú T5Gemma2ForConditionalGenerationi�  zlm_head.out_proj.weightz,model.encoder.text_model.embed_tokens.weightzlm_head.out_projÚcolwise_gather_outputr~   rv  c                ó    <€ V ^8„  d   QhRS[ /# ri   rÙ  )r7   r8   s   "€r9   r:   Ú-T5Gemma2ForConditionalGeneration.__annotate__”  s   ø€ ÷ ñ ™~ñ r<   c                ó  <€ \         SV `  V4       \        V4      V n        VP                  P
                  V n        \        VP                  P                  V P
                  4      V n        R V n	        V P                  4        R# )ÚForMaskedLMN)r?   r@   rh  r¼  rÍ  rk  ri  rp   Úlm_headÚ	loss_typer  r{   s   &&€r9   r@   Ú)T5Gemma2ForConditionalGeneration.__init__”  sZ   ø€ Ü‰Ñ˜Ô ä" 6Ó*ˆŒ
Ø Ÿ.™.×3Ñ3ˆŒÜ% f§n¡n×&@Ñ&@À$Ç/Á/ÓRˆŒØ&ˆŒà�‰Ör<   c                ó&   € WP                   n        R # r>   ©r•  ro  r2  s   &&r9   Úset_output_embeddingsÚ6T5Gemma2ForConditionalGeneration.set_output_embeddingsž  s   € Ø .�‰Ör<   c                ó.   € V P                   P                  # r>   r™  r[   s   &r9   Úget_output_embeddingsÚ6T5Gemma2ForConditionalGeneration.get_output_embeddings¡  s   € Ø�|‰|×$Ñ$Ð$r<   c                ó6   € V P                   P                  4       # r>   ©r¼  r.  r[   s   &r9   r.  Ú5T5Gemma2ForConditionalGeneration.get_input_embeddings¤  ó   € Ø�z‰z×.Ñ.Ó0Ð0r<   c                ó<   € V P                   P                  V4       R # r>   ©r¼  r1  ©rF   rä   s   &&r9   r1  Ú5T5Gemma2ForConditionalGeneration.set_input_embeddings§  ó   € Ø�
‰
×'Ñ'¨Ö.r<   c                ó.   <€ V ^8„  d   QhRS[ R,          /# ro  rq  )r7   r8   s   "€r9   r:   r’  ª  s   ø€ ÷ 9ñ 9¡C¨$¥Jñ 9r<   c                ó:   € V P                   P                  VR 7      # ))rp  )r¼  rt  rs  s   &&r9   rt  Ú,T5Gemma2ForConditionalGeneration.get_encoderª  s   € Ø�z‰z×%Ñ%¨xÐ%Ó8Ð8r<   c                ó6   € V P                   P                  4       # r>   )r¼  ry  r[   s   &r9   ry  Ú,T5Gemma2ForConditionalGeneration.get_decoder­  s   € Ø�z‰z×%Ñ%Ó'Ð'r<   c                ó`   <€ V ^8„  d   QhRS[ P                  RS[S[,          RS[S[,          /# r6  r8  )r7   r8   s   "€r9   r:   r’  ²  s:   ø€ ÷ Mñ MÙ!ŸL™LðMÙ4:Ñ;MÕ4NðMá	Ñ+Õ	+ñMr<   c                óD   € V P                  4       P                  ! V3/ VB # r>   )rt  r=  )rF   r7  r®   s   &&,r9   r=  Ú3T5Gemma2ForConditionalGeneration.get_image_features°  s#   € ð
 ×ÑÓ!×4Ò4°\ÑLÀVÑLÐLr<   c                ó6   € V P                  4       P                  # r>   )rt  r*  r[   s   &r9   r*  Ú-T5Gemma2ForConditionalGeneration.vision_tower·  s   € à×ÑÓ!×.Ñ.Ð.r<   c          !      ó`  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[R,          R
S[R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[S[ P                  ,          RS[	S[
,          RS[S[ P                  ,          S[,          /# )r1   rµ  Nr7  rå   rÂ   r€  r�  r‚  rƒ  r  r  r„  rÊ  rd  Úlogits_to_keepr®   r¢   )rC   rT  rU  r…  r   r   re  r5   r¤   r   r    rY   r   )r7   r8   s   "€r9   r:   r’  ½  sZ  ø€ ÷ M
ñ M
ñ ×#Ñ# dÕ*ðM
ñ ×'Ñ'¨$Õ.ð	M
ñ
 ×)Ñ)¨DÕ0ðM
ñ ×&Ñ&¨Õ-ðM
ñ !×+Ñ+¨dÕ2ðM
ñ !&× 0Ñ 0°4Õ 7ðM
ñ $×.Ñ.°Õ5ðM
ñ )¨4Õ/ðM
ñ -¨tÕ3ðM
ñ ×(Ñ(¨4Õ/ðM
ñ  %×0Ñ0°4Õ7ðM
ñ  × Ñ  4Õ'ð!M
ñ" ˜$•;ð#M
ñ$ ™eŸl™lÕ*ð%M
ñ& Ñ+Õ,ð'M
ñ( 
‰u× Ñ Õ	!¡OÕ	3ñ)M
r<   c                óä  € Ve   Vf   Vf   V P                  V4      pV P                  ! RRVRVRVRVRVRVRVR	VR
V	RV
RVRV/VB pVP                  p\        V\        4      '       d   \        V) R4      MTpV P                  VRVR3,          4      pV P                  P                  pVP                  e=   VVP                  ,          p\        P                  ! V4      pVVP                  ,          pRpVe   V P                  ! VWÀP                  3/ VB p\        VVVP                  VP                   VP"                  VP$                  VP&                  VP(                  VP*                  R7	      # )a¼  
decoder_position_ids (`torch.LongTensor` of shape `(batch_size, decoder_sequence_length)`, *optional*):
    Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the range `[0,
    config.decoder.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
Nrµ  r7  rå   rÂ   r€  r�  r‚  rƒ  r  r  r„  rd  rº   )	Úlossrv  r  r‡  rˆ  rW  r‰  r1  rŠ  rW  )rÖ  r¼  r  r¼   r5   Úslicer•  rj   rÍ  Úfinal_logit_softcappingrC   rï   Úloss_functionrk  r   r  r‡  rˆ  rW  r‰  r1  rŠ  )rF   rµ  r7  rå   rÂ   r€  r�  r‚  rƒ  r  r  r„  rÊ  rd  r³  r®   r‹  r~   Úslice_indicesrv  rÓ  rµ  s   &&&&&&&&&&&&&&&,      r9   rV   Ú(T5Gemma2ForConditionalGeneration.forward»  sº  € ðB ÒÐ"3Ò";Ð@UÒ@]à $× JÑ JÈ6Ó RÐà.2¯jªjñ /
Øð/
à%ð/
ð *ð/
ð &ð	/
ð
 0ð/
ð $:ð/
ð "6ð/
ð ,ð/
ð ,ð/
ð (ð/
ð #8ð/
ð  ð/
ð ñ/
ˆð  (×9Ñ9ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàŸ™×,Ñ,ˆØ×1Ñ1Ò=Ø˜n×DÑDÕDˆFÜ—Z’Z Ó'ˆFØ˜n×DÑDÕDˆFàˆØÒà×%Ò% f¨f·o±oÑPÈÑPˆDäØØØ+×;Ñ;Ø"1×"GÑ"GØ.×AÑAØ,×=Ñ=Ø&5×&OÑ&OØ"1×"GÑ"GØ.×AÑAô

ð 
	
r<   c                ó>   <€ V ^8„  d   QhRS[ RS[RS[RS[RS[RS[/# )r1   Úgeneration_configÚmodel_kwargsÚgeneration_moder   Úmax_cache_lengthr¢   )r
   r  r   r5   re  )r7   r8   s   "€r9   r:   r’    sO   ø€ ÷ I:ñ I:á+ðI:ñ ðI:ñ (ð	I:ñ
 ðI:ñ ðI:ñ 
ñI:r<   c                ó  <€ \         SV `  VVVVV4       VP                  RJ d   R# VP                  pVf   RpMRVP                  9   p\        P
                  ! V P                  P                  RR7      4      pRVn        R.VP                  ,          Vn
        RVRV/p	VP                  R	4      p
V
e±   \        V
\        4      '       g   \        R
4      h\        V
P                   4      ^ 8”  d$   V
P                   P                  ^ 4      '       d   R# \#        V
P$                  4      pV\&        8X  d%   VR,          ^ ,          P(                  ^,          V	R&   V! R/ V	B V
n        M>\        \+        R/ RV P                  P                  RR7      RV/B \+        4       4      VR	&   \-        V R4      '       dL   V P.                  e<   \        V P.                  \        4      '       g   \        R4      hVR	,          V n        R# R# R# )zMOverride cache preparation to support T5Gemma2-specific EncoderDecoder Cache.FNÚ	offloadedTrx  r  rj   Ú
offloadingr  zaThe `past_key_values` in `model_kwargs` must be of type `EncoderDecoderCache` for T5Gemma2 model.rƒ  Úmax_cache_lenÚ_cachezLThe internal cache must be of type `EncoderDecoderCache` for T5Gemma2 model.rW  )r?   Ú_prepare_cache_for_generationrd  Úcache_implementationÚcopyÚdeepcopyrj   Úget_text_configr  r  r–   r5  r¼   r   rÐ  Úlenr4  r½   r6  r	   rZ   r   r  rÄ  )rF   r¼  r½  r¾  r   r¿  rÆ  Úoffload_cacheÚcross_attn_configÚcross_attn_cache_kwargsr  Úcross_attn_clsrG   s   &&&&&&      €r9   rÅ  Ú>T5Gemma2ForConditionalGeneration._prepare_cache_for_generation  sü  ø€ ô 	‰Ñ-ØØØØØô	
ð ×&Ñ&¨%Ó/Ùà0×EÑEÐØÒ'Ø!‰Mà'Ð+<×+QÑ+QÑQˆMô !ŸMšM¨$¯+©+×*EÑ*EÈdÐ*EÓ*SÓTÐð ,0ÐÔ(Ø)9Ð(:Ð=N×=`Ñ=`Õ(`ÐÔ%ð Ð'Ø˜-ð#
Ðð
 '×*Ñ*Ð+<Ó=ˆØÒ&Ü˜oÔ/B×CÒCÜ Øwóð ô
 �?×-Ñ-Ó.°Ô2°×7QÑ7Q×7UÑ7UÐVW×7XÒ7XÙä! /×"GÑ"GÓHˆNØ¤Ô,Ø;GÐHYÕ;ZÐ[\Õ;]×;cÑ;cÐdeÕ;fÐ'¨Ñ8á4BÑ4]ÐE\Ñ4]ˆOÕ1ô /BÜñ à  $§+¡+×"=Ñ"=ÀdÐ"=Ó"KØ$ mðñô “ó/ˆLÐ*Ñ+ô �4˜×"Ò" t§{¡{Ò'>Ü˜dŸk™kÔ+>×?Ò?Ü Ð!oÓpÐpà&Ð'8Õ9ˆDŽKñ	 (?Ñ"r<   )rÄ  r•  r–  r¼  rk  r>   )NNNNNNNNNNNNNr   )r^   r_   r`   ra   r�  Ú_tp_planÚ_pp_planr@   rš  r�  r.  r1  rt  ry  r"   r!   r=  Úpropertyr*  rV   rÅ  rb   rc   rd   re   s   @@r9   r�  r�  �  sÊ   ù‡ € à!Ð#QðÐð #Ð$;Ð<€HØ" oÐ%6¸¸
Ð$CÐD€H÷ó ò/ò%ò1ò/÷9ò 9ò(ð Ø÷Mó ó ðMð
 ñ/ó ð/ð Ø÷M
ñ M
ó ó ðM
÷^I:÷ I:ó I:r<   r�  c                   ót   a a€ ] tR tRt oV3R lV 3R lltR tR t]]R	V3R lR ll4       4       t	Rt
VtV ;t# )
Ú!T5Gemma2ForSequenceClassificationiX  c                ó    <€ V ^8„  d   QhRS[ /# ri   rÙ  )r7   r8   s   "€r9   r:   Ú.T5Gemma2ForSequenceClassification.__annotate__Z  s   ø€ ÷ 	ñ 	™~ñ 	r<   c                ó$  <€ \         SV `  V4       VP                  V n        VP                  P                  V n        \        V4      V n        \        VR R4      p\        V P                  V P                  V4      V n	        V P                  4        R# ©r|  gš™™™™™¹?N©r?   r@   r{  rÍ  rp   rh  r¼  rª   ry  Úscorer  ©rF   rj   Úclassifier_dropoutrG   s   && €r9   r@   Ú*T5Gemma2ForSequenceClassification.__init__Z  sp   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒØ!Ÿ>™>×5Ñ5ˆÔä" 6Ó*ˆŒ
ä$ VÐ-FÈÓLÐÜ/°×0@Ñ0@À$Ç/Á/ÐSeÓfˆŒ
Ø�‰Ör<   c                ó6   € V P                   P                  4       # r>   r   r[   s   &r9   r.  Ú6T5Gemma2ForSequenceClassification.get_input_embeddingse  r¢  r<   c                ó<   € V P                   P                  V4       R # r>   r¤  r¥  s   &&r9   r1  Ú6T5Gemma2ForSequenceClassification.set_input_embeddingsh  r§  r<   c                óÚ  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[R,          R
S[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          RS[/# ©r1   rµ  Nr7  rå   rÂ   r€  r�  r‚  rƒ  r  r„  rÊ  r®   r¢   )rC   rT  rU  r¤   r   r   r    r   )r7   r8   s   "€r9   r:   rÖ  m  s  ø€ ÷ J
ñ J
á×#Ñ# dÕ*ðJ
ñ ×'Ñ'¨$Õ.ðJ
ñ Ÿ™ tÕ+ð	J
ñ
 ×&Ñ&¨Õ-ðJ
ñ !×+Ñ+¨dÕ2ðJ
ñ !&§¡¨tÕ 3ðJ
ñ $×.Ñ.°Õ5ðJ
ñ )¨4Õ/ðJ
ñ ×(Ñ(¨4Õ/ðJ
ñ  %×0Ñ0°4Õ7ðJ
ñ × Ñ  4Õ'ðJ
ñ Ñ+Õ,ðJ
ñ 
"ñJ
r<   c                óÐ  € V	f   V
e$   \        RV P                  P                   R24      hVf   \        R4      hVf   V P	                  V4      pV P
                  ! V3RVRVRVRVR	VR
VRVRV	RV
RR/
VB pVP                  pVP                  pVP                  pV P                  V4      pVP                  ^ ,          pWPP                  P                  8g  P                  VP                  \        P                   4      p\        P"                  ! VP                  R,          VP                  \        P                   R7      pVV,          P%                  R4      p\        P&                  ! VVP                  R,          ^,
          R7      pV\        P"                  ! VVP                  R7      V3,          pRpVe    V P)                  VVVV P                  R7      p\+        VVVVR7      # )áœ  
decoder_position_ids (`torch.LongTensor` of shape `(batch_size, decoder_sequence_length)`, *optional*):
    Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the range `[0,
    config.decoder.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
    `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
Nú8Passing input embeddings is currently not supported for Ú.úYou have to specify input_idsr7  rå   rÂ   r€  r�  r‚  rƒ  r  r„  rd  F)rƒ   r©   )Úmaxr  )rv  rÊ  Úpooled_logitsrj   ©rµ  rv  r~   rû  rK   )ÚNotImplementedErrorrG   r^   rÐ  rÖ  r¼  r  r‡  rˆ  rÚ  rZ   rj   rÏ  r­   rƒ   rC   Úint32r¬   ÚargmaxÚclampr¸  r   )rF   rµ  r7  rå   rÂ   r€  r�  r‚  rƒ  r  r„  rÊ  r®   rP  r  r~   rû  rv  r   Únon_pad_maskÚtoken_indicesÚlast_non_pad_tokenrê  rµ  s   &&&&&&&&&&&&,           r9   rV   Ú)T5Gemma2ForSequenceClassification.forwardk  s  € ð4 Ò$Ð(=Ò(IÜ%ØJÈ4Ï>É>×KbÑKbÐJcÐcdÐeóð ð ÒÜÐ<Ó=Ð=àÒ$Ø $× JÑ JÈ9Ó UÐà&*§j¢jØñ'
à%ð'
ð *ð'
ð &ð	'
ð
 0ð'
ð $:ð'
ð "6ð'
ð ,ð'
ð (ð'
ð #8ð'
ð ð'
ð ñ'
ˆð $×5Ñ5ÐØ×5Ñ5ˆØ×/Ñ/ˆ
à—‘Ð-Ó.ˆà—_‘_ QÕ'ˆ
à)¯[©[×-EÑ-EÑE×IÑIÈ&Ï-É-ÔY^×YdÑYdÓeˆÜŸšÐ%6×%<Ñ%<¸RÕ%@ÈÏÉÔ^c×^iÑ^iÔjˆØ+¨lÕ:×BÑBÀ2ÓFÐÜ"Ÿ[š[Ð);ÐAR×AXÑAXÐY[ÕA\Ð_`ÕA`ÔaÐàœuŸ|š|¨J¸v¿}¹}ÔMÐOaÐaÕbˆàˆØÒØ×%Ñ%¨V¸FÐR_Ðhl×hsÑhsÐ%ÓtˆDä'ØØ Ø'Ø!ô	
ð 	
r<   ©rp   r¼  r{  rÚ  ©NNNNNNNNNNN©r^   r_   r`   ra   r@   r.  r1  r"   r!   rV   rb   rc   rd   re   s   @@r9   rÔ  rÔ  X  sC   ù‡ € ÷	ó 	ò1ò/ð Ø÷J
ñ J
ó ó ÷J
ð J
r<   rÔ  c                   ót   a a€ ] tR tRt oV3R lV 3R lltR tR t]]R	V3R lR ll4       4       t	Rt
VtV ;t# )
ÚT5Gemma2ForTokenClassificationiº  c                ó    <€ V ^8„  d   QhRS[ /# ri   rÙ  )r7   r8   s   "€r9   r:   Ú+T5Gemma2ForTokenClassification.__annotate__¼  s   ø€ ÷ 
ñ 
™~ñ 
r<   c                ó$  <€ \         SV `  V4       VP                  V n        VP                  P                  V n        \        V4      V n        \        VR R4      p\        V P                  V P                  V4      V n	        V P                  4        R# rØ  rÙ  rÛ  s   && €r9   r@   Ú'T5Gemma2ForTokenClassification.__init__¼  sp   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒØ!Ÿ>™>×5Ñ5ˆÔä" 6Ó*ˆŒ
ä$ VÐ-FÈÓLÐÜ/°×0@Ñ0@À$Ç/Á/ÐSeÓfˆŒ
à�‰Ör<   c                ó6   € V P                   P                  4       # r>   r   r[   s   &r9   r.  Ú3T5Gemma2ForTokenClassification.get_input_embeddingsÈ  r¢  r<   c                ó<   € V P                   P                  V4       R # r>   r¤  r¥  s   &&r9   r1  Ú3T5Gemma2ForTokenClassification.set_input_embeddingsË  r§  r<   c                óÚ  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[R,          R
S[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          RS[/# rã  )rC   rT  rU  r¤   r   r   r    r   )r7   r8   s   "€r9   r:   rú  Ð  s  ø€ ÷ @
ñ @
á×#Ñ# dÕ*ð@
ñ ×'Ñ'¨$Õ.ð@
ñ Ÿ™ tÕ+ð	@
ñ
 ×&Ñ&¨Õ-ð@
ñ !×+Ñ+¨dÕ2ð@
ñ !&§¡¨tÕ 3ð@
ñ $×.Ñ.°Õ5ð@
ñ )¨4Õ/ð@
ñ ×(Ñ(¨4Õ/ð@
ñ  %×0Ñ0°4Õ7ð@
ñ × Ñ  4Õ'ð@
ñ Ñ+Õ,ð@
ñ 
ñ@
r<   c                óº  € V	f   V
e$   \        RV P                  P                   R24      hVf   \        R4      hVf   V P	                  V4      pV P
                  ! V3RVRVRVRVR	VR
VRVRV	RV
RR/
VB pVP                  pVP                  pVP                  pV P                  V4      pRpVe   V P                  VW°P                  4      p\        VVVVR7      # )rå  Nræ  rç  rè  r7  rå   rÂ   r€  r�  r‚  rƒ  r  r„  rd  Frë  )rì  rG   r^   rÐ  rÖ  r¼  r  r‡  rˆ  rÚ  r¸  rj   r   )rF   rµ  r7  rå   rÂ   r€  r�  r‚  rƒ  r  r„  rÊ  r®   rP  r  r~   rû  rv  rµ  s   &&&&&&&&&&&&,      r9   rV   Ú&T5Gemma2ForTokenClassification.forwardÎ  sJ  € ð4 Ò$Ð(=Ò(IÜ%ØJÈ4Ï>É>×KbÑKbÐJcÐcdÐeóð ð ÒÜÐ<Ó=Ð=àÒ$Ø $× JÑ JÈ9Ó UÐà&*§j¢jØñ'
à%ð'
ð *ð'
ð &ð	'
ð
 0ð'
ð $:ð'
ð "6ð'
ð ,ð'
ð (ð'
ð #8ð'
ð ð'
ð ñ'
ˆð $×5Ñ5ÐØ×5Ñ5ˆØ×/Ñ/ˆ
à—‘Ð-Ó.ˆàˆØÒØ×%Ñ% f¨f·k±kÓBˆDä$ØØØ'Ø!ô	
ð 	
r<   rô  rõ  rö  re   s   @@r9   rø  rø  º  sC   ù‡ € ÷
ó 
ò1ò/ð Ø÷@
ñ @
ó ó ÷@
ð @
r<   rø  r�  )r  NN)T)r�  rh  r"  r»  rÔ  rø  )^rÇ  Úcollections.abcr   rC   Útorch.nnrA   Ú r   rÁ  Úactivationsr   Úcache_utilsr   r   r   r	   Ú
generationr
   r   r   Úintegrationsr   r   Úmasking_utilsr   r   r   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr    r!   r"   r#   Úutils.deprecationr$   Úutils.genericr%   r&   Úutils.output_capturingr'   r(   Úautor)   Úconfiguration_t5gemma2r*   r+   r,   r-   rè   r/   rg   r‚   rÎ   r×   rß   rø   rú   r*  rC  r^  ri  ry  r…  Ú	Embeddingr§  r»  rø  rú  r"  rU  rh  r�  rÔ  rø  Ú__all__rW  r<   r9   Ú<module>r     sF  ðó* Ý $ã Ý å &Ý !ß PÓ Pß KÑ Kß Lß mÑ mÝ BÝ 9÷÷ ñ ÷ Lß FÝ &ß aÓ aÝ 0ß Gß EÝ ß tÓ tô=�b—i‘iô =ô(�"—)‘)ô ô&E<˜bŸi™iô E<òP(ñ Ð-Ó.óó /ðõ2	U÷%ñD Ð)Ó*ôK)˜BŸI™Ió K)ó +ðK)ñ\ Ð)Ó*ôuB˜bŸi™ió uBó +ðuBôp1Ð5ô 1ôh8Ð5ô 8ôv	�R—Y‘Yô 	ô §¡ô ô!@ "§)¡)ô !@ôH  b§l¡lô  ð. ôJ!˜oó J!ó ðJ!÷Zô&b
Ð1ô b
ôJeÐ-ô eôP}
Ð-ô }
ð@ ôZ
Ð+ó Z
ó ðZ
ôzH:Ð'>Àô H:ðV ô^
Ð(?ó ^
ó ð^
ðB ôU
Ð%<ó U
ó ðU
òp‚r<   