+
    œ°³j%ì  ã                   ó.  € ^ RI t ^ RIHt ^ RIt^ RIHt ^RIHt ^RI	H
t
 ^RIHtHtHtHt ^RIHtHtHt ^RIHtHt ^RIHtHtHtHt ^R	IHt ^R
IHt ^RI H!t!H"t"H#t#H$t$H%t%H&t& ^RI'H(t(H)t) ^RI*H+t+H,t, ^RI-H.t. ^RI/H0t0H1t1H2t2H3t3 ^RI4H5t5 ^RI6H7t7H8t8 ^RI9H:t:H;t; ^RI<H=t=H>t> ]3P~                  ! ]@4      tA ! R R]P„                  4      tC ! R R]P„                  4      tD ! R R]P„                  4      tER tF]! R4      R?R l4       tGR R ltHR@R R  lltI]! ]G4       ! R! R"]P„                  4      4       tJ]! ]G4       ! R# R$]P„                  4      4       tK ! R% R&]4      tL ! R' R(]4      tM ! R) R*]P„                  4      tN ! R+ R,]P„                  4      tO]1 ! R- R.],4      4       tPR/ R0 ltQ ! R1 R2]P4      tR ! R3 R4]P4      tS]1 ! R5 R6]P4      4       tT]1 ! R7 R8]P4      4       tU ! R9 R:]P]4      tV]1 ! R; R<]P4      4       tW]1 ! R= R>]P4      4       tX. RAOtYR# )Bé    N)ÚCallable)Úinitialization)ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCacheÚStaticCache)ÚGenerationConfigÚGenerationMixinÚGenerationMode)Úuse_kernel_forward_from_hubÚuse_kernelized_func)Úcreate_bidirectional_maskÚ(create_bidirectional_sliding_window_maskÚcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚ)BaseModelOutputWithPastAndCrossAttentionsÚSeq2SeqLMOutputÚSeq2SeqModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputs)ÚT5GemmaConfigÚT5GemmaModuleConfigc                   óT   a a€ ] tR t^=t oRV3R lV 3R llltR tR tR tRtVt	V ;t
# )ÚT5GemmaRMSNormc                ó&   <€ V ^8„  d   QhRS[ RS[/# )é   ÚdimÚeps©ÚintÚfloat)ÚformatÚ__classdict__s   "€Úm/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/t5gemma/modeling_t5gemma.pyÚ__annotate__ÚT5GemmaRMSNorm.__annotate__>   s   ø€ ÷ 5ñ 5™Cð 5¡eñ 5ó    c                óŽ   <€ \         SV `  4        W n        \        P                  ! \
        P                  ! V4      4      V n        R # ©N)ÚsuperÚ__init__r0   ÚnnÚ	ParameterÚtorchÚzerosÚweight)Úselfr/   r0   Ú	__class__s   &&&€r6   r=   ÚT5GemmaRMSNorm.__init__>   s,   ø€ Ü‰ÑÔØŒÜ—l’l¤5§;¢;¨sÓ#3Ó4ˆŽr9   c                óž   € V\         P                  ! VP                  ^4      P                  RRR7      V P                  ,           4      ,          # )r.   T)Úkeepdiméÿÿÿÿ)r@   ÚrsqrtÚpowÚmeanr0   )rC   Úxs   &&r6   Ú_normÚT5GemmaRMSNorm._normC   s4   € Ø”5—;’;˜qŸu™u Q›xŸ}™}¨R¸˜}Ó>ÀÇÁÕIÓJÕJÐJr9   c                ó²   € V P                  VP                  4       4      pVR V P                  P                  4       ,           ,          pVP                  V4      # )ç      ð?)rM   r3   rB   Útype_as)rC   rL   Úoutputs   && r6   ÚforwardÚT5GemmaRMSNorm.forwardF   sC   € Ø—‘˜AŸG™G›IÓ&ˆð ˜3 §¡×!2Ñ!2Ó!4Õ4Õ5ˆØ�~‰~˜aÓ Ð r9   c                ó^   € \        V P                  P                  4       R V P                   2# )z, eps=)ÚtuplerB   Úshaper0   ©rC   s   &r6   Ú
extra_reprÚT5GemmaRMSNorm.extra_reprM   s'   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$·(±(°Ð<Ð<r9   )r0   rB   )g�íµ ÷Æ°>)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r=   rM   rS   rY   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©rD   r5   s   @@r6   r,   r,   =   s$   ù‡ € ÷5õ 5ò
Kò!÷=ò =r9   r,   c                   ó8   a a€ ] tR t^Qt oV 3R ltR tRtVtV ;t# )Ú
T5GemmaMLPc                ó"  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        \
        P                  ! V P                  V P                  R R7      V n        \
        P                  ! V P                  V P                  R R7      V n        \
        P                  ! V P                  V P                  R R7      V n	        \        VP                  ,          V n        \
        P                  ! VP                  4      V n        R# )F©ÚbiasN)r<   r=   ÚconfigÚhidden_sizeÚintermediate_sizer>   ÚLinearÚ	gate_projÚup_projÚ	down_projr   Úhidden_activationÚact_fnÚDropoutÚdropout_rateÚdropout©rC   rh   rD   s   &&€r6   r=   ÚT5GemmaMLP.__init__R   sº   ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔÜŸš 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆŒÜ˜V×5Ñ5Õ6ˆŒÜ—z’z &×"5Ñ"5Ó6ˆŽr9   c                ó¶   € V P                  V P                  V4      4      V P                  V4      ,          pV P                  V4      pV P	                  V4      pV# r;   )rp   rl   rm   rs   rn   )rC   rL   Úhidden_statesrn   s   &&  r6   rS   ÚT5GemmaMLP.forward]   sH   € ØŸ™ D§N¡N°1Ó$5Ó6¸¿¹Àa»ÕHˆØŸ™ ]Ó3ˆØ—N‘N =Ó1ˆ	ØÐr9   )rp   rh   rn   rs   rl   ri   rj   rm   )	r[   r\   r]   r^   r=   rS   r_   r`   ra   rb   s   @@r6   rd   rd   Q   s   ù‡ € õ	7÷ò r9   rd   c                   óÎ   a a€ ] tR t^dt o]! RRR7      R
V3R lV 3R lll4       t]]! RRR7      R
V3R lR ll4       4       t]P                  ! 4       ]
R 4       4       tR	tVtV ;t# )ÚT5GemmaRotaryEmbeddingÚdevicez5.18)Úversionc                ó    <€ V ^8„  d   QhRS[ /# ©r.   rh   ©r)   )r4   r5   s   "€r6   r7   Ú#T5GemmaRotaryEmbedding.__annotate__f   s   ø€ ÷ Oñ O™}ñ Or9   c                óä  <€ \         SV `  4        VP                  V n        VP                  V n        Wn        V P
                  P                  R ,          V n        V P                  pV P                  R8w  d   \        V P                  ,          pV! V P
                  V4      w  q@n
        \        P                  ! VRR7      V n        \        P                  ! VP                  4       RR7      V n        R# )Ú	rope_typeÚdefaultF)Ú
persistentN)r<   r=   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenrh   Úrope_parametersr‚   Úcompute_default_rope_parametersr   Úattention_scalingr>   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)rC   rh   r{   Úrope_init_fnrŒ   rD   s   &&&  €r6   r=   ÚT5GemmaRotaryEmbedding.__init__e   s¬   ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àŒàŸ™×4Ñ4°[ÕAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÔ&Ü.¨t¯~©~Õ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÔ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÔ!NˆÖr9   c                óP   <€ V ^8„  d   QhRS[ RS[S[P                  S[3,          /# )r.   rh   Úreturn)r)   rV   r@   ÚTensorr3   )r4   r5   s   "€r6   r7   r€   x   s.   ø€ ÷ 5ñ 5±ð 5ÑY^Ñ_d×_kÑ_kÑmrÐ_rÕYsñ 5r9   c           	     ó0  € V P                   R,          p\        V RR4      ;'       g    V P                  V P                  ,          pRpRV\        P
                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚhead_dimNrP   ©Údtype)rˆ   Úgetattrri   Únum_attention_headsr@   Úaranger3   Úto)rh   r{   ÚkwargsÚbaser/   Úattention_factorrŒ   s   &&,    r6   r‰   Ú6T5GemmaRotaryEmbedding.compute_default_rope_parametersv   s~   € ð ×%Ñ% lÕ3ˆÜ�f˜j¨$Ó/×cÐc°6×3EÑ3EÈ×IcÑIcÕ3cˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r9   c                ó”  € V P                   R,          P                  4       P                  VP                  ^ ,          R^4      P	                  VP
                  4      pVR	,          P                  4       p\        VP
                  P                  \        4      '       d2   VP
                  P                  R8w  d   VP
                  P                  MRp\        VRR7      ;_uu_ 4        VP                  4       VP                  4       ,          P                  ^^4      p\        P                  ! Wf3RR7      pVP                  4       V P                  ,          pVP                  4       V P                  ,          p	R R R 4       XP	                  VP                   R7      X	P	                  VP                   R7      3#   + '       g   i     LH; i)
NÚmpsÚcpuF)Údevice_typeÚenabled©r/   r—   )NºNNNNrH   )r§   Nr§   )rŒ   r3   ÚexpandrW   rœ   r{   Ú
isinstanceÚtypeÚstrr%   Ú	transposer@   ÚcatÚcosrŠ   Úsinr˜   )
rC   rL   Úposition_idsÚinv_freq_expandedÚposition_ids_expandedr¤   ÚfreqsÚembr®   r¯   s
   &&&       r6   rS   ÚT5GemmaRotaryEmbedding.forwardŠ   sE  € ð !ŸM™M¨-Õ8×>Ñ>Ó@×GÑGÈ×HZÑHZÐ[\ÕH]Ð_aÐcdÓe×hÑhÐij×iqÑiqÓrÐØ ,¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆÜ¨¸U×CÖCØ&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÕN×YÑYÐZ[Ð]^Ó_ˆEÜ—)’)˜U˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆC÷	 Dð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷ D×Cús   Ã&BF7Æ7G	)rŠ   rh   rŒ   r†   rŽ   r‡   r‚   r;   )r[   r\   r]   r^   r$   r=   Ústaticmethodr‰   r@   Úno_gradr   rS   r_   r`   ra   rb   s   @@r6   rz   rz   d   su   ù‡ € Ù�X vÔ.÷Oô Oó /ðOð  Ù�X vÔ.÷5ñ 5ó /ó ð5ð$ ‡]‚]ƒ_Øñ<ó ó ÷<ð <r9   rz   c                óÌ   € V RRV P                   R,          ^,          13,          pV RV P                   R,          ^,          R13,          p\        P                  ! V) V3RR7      # )z*Rotates half the hidden dims of the input..Nr¦   rH   )rW   r@   r­   )rL   Úx1Úx2s   &  r6   Úrotate_halfr»   š   sZ   € à	
ˆ3Ð"�!—'‘'˜"•+ Õ"Ð"Ð"Õ	#€BØ	
ˆ3�—‘˜•˜qÕ Ñ"Ð"Õ	#€BÜ�9Š9�r�c˜2�Y BÔ'Ð'r9   Úrotary_pos_embc                óÈ   € VP                  V4      pVP                  V4      pW,          \        V 4      V,          ,           pW,          \        V4      V,          ,           pWV3# )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)Ú	unsqueezer»   )ÚqÚkr®   r¯   Úunsqueeze_dimÚq_embedÚk_embeds   &&&&&  r6   Úapply_rotary_pos_embrÄ   ¡   sS   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�wœ; q›>¨CÕ/Õ0€GØ�wœ; q›>¨CÕ/Õ0€GØÐÐr9   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r.   rw   Ún_repr’   )r@   r“   r2   )r4   s   "r6   r7   r7   »   s.   € ÷ 	Uñ 	UœUŸ\™\ð 	U´#ð 	U¼%¿,¹,ñ 	Ur9   c                ó˜   € V P                   w  r#rEV^8X  d   V # V R,          P                  W#WV4      p V P                  W#V,          WE4      # )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
)r§   r§   Nr§   r§   )rW   r¨   Úreshape)rw   rÆ   ÚbatchÚnum_key_value_headsÚslenr–   s   &&    r6   Ú	repeat_kvrÌ   »   sU   € ð
 2?×1DÑ1DÑ.€E Ø�„zØÐØ!Ð"2Õ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÕ(CÀTÓTÐTr9   c                ór  € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        \
        ,          R\        R,          R	\        R,          R
\        \        P                  \        P                  3,          /	# )r.   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNrs   ÚscalingÚsoftcapr’   )r>   ÚModuler@   r“   r3   r2   rV   )r4   s   "r6   r7   r7   Ç   s™   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ”S�[ð%ô �T�\ð%ô �T�\ð%ô Œ5�<‰<œŸ™Ð%Õ&ñ%r9   c                 ó¤  € Vf   V P                   R,          p\        W P                  4      p	\        W0P                  4      p
\        P                  ! WP                  ^^4      4      V,          pVe'   W·,          p\        P                  ! V4      pW·,          pVe	   W´,           p\        P                  P                  VR\        P                  R7      P                  VP                  4      p\        P                  P                  WµV P                  R7      p\        P                  ! Wº4      pVP                  ^^4      P                  4       pWË3# )N)r/   r˜   )ÚpÚtrainingç      à¿rH   )r–   rÌ   Únum_key_value_groupsr@   Úmatmulr¬   Útanhr>   Ú
functionalÚsoftmaxÚfloat32rœ   r˜   rs   rØ   Ú
contiguous)rÎ   rÏ   rÐ   rÑ   rÒ   rs   rÓ   rÔ   r�   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs   &&&&&&&&,    r6   Úeager_attention_forwardrå   Ç   s  € ð ‚Ø—/‘/ 4Õ'ˆä˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÕL€LàÒØ#Õ-ˆÜ—z’z ,Ó/ˆØ#Õ-ˆØÒ!Ø#Õ4ˆô —=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KØÐ$Ð$r9   c                   óX   a a€ ] tR t^ét oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )ÚT5GemmaSelfAttentionú=Multi-headed attention from 'Attention Is All You Need' paperc                ó&   <€ V ^8„  d   QhRS[ RS[/# ©r.   rh   Ú	layer_idx©r*   r2   )r4   r5   s   "€r6   r7   Ú!T5GemmaSelfAttention.__annotate__í   s    ø€ ÷ hñ hÑ2ð h¹sñ hr9   c                ó®  <€ \         SV `  4        \        VR 4      '       d   VP                  V,          MRV n        Wn        W n        \        VRVP                  VP                  ,          4      V n
        VP                  VP                  ,          V n        VP                  R,          V n        V P
                  P                  V n        VP                   V n        \$        P&                  ! VP                  VP                  V P                  ,          VP(                  R7      V n        \$        P&                  ! VP                  VP                  V P                  ,          VP(                  R7      V n        \$        P&                  ! VP                  VP                  V P                  ,          VP(                  R7      V n        \$        P&                  ! VP                  V P                  ,          VP                  VP(                  R7      V n        V P
                  P2                  V n        V P                  R8X  d   VP4                  V n        R# RV n        R# )Úlayer_typesNr–   rf   Úsliding_attentionrÙ   )r<   r=   Úhasattrrï   Ú
layer_typerh   rë   r™   ri   rš   r–   rÊ   rÚ   Úquery_pre_attn_scalarrÓ   Úattention_dropoutÚ
is_decoderÚ	is_causalr>   rk   Úattention_biasÚq_projÚk_projÚv_projÚo_projÚattn_logit_softcappingÚsliding_window©rC   rh   rë   rD   s   &&&€r6   r=   ÚT5GemmaSelfAttention.__init__í   s¬  ø€ Ü‰ÑÔÜ;BÀ6È=×;YÒ;Y˜&×,Ñ,¨YÖ7Ð_cˆŒØŒØ"ŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÕ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Õ$\ˆÔ!Ø×3Ñ3°TÕ9ˆŒØ!%§¡×!>Ñ!>ˆÔà×*Ñ*ˆŒä—i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×&Ñ&¨¯©Õ6¸×8JÑ8JÐQW×QfÑQfô
ˆŒð '+§k¡k×&HÑ&HˆÔ#Ø7;·±ÐJ]Ô7]˜f×3Ñ3ˆÖÐcgˆÖr9   c                ó`  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          R,          RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  S[ P                  R,          S[S[ P                  ,          R,          3,          /# )r.   rw   Úposition_embeddingsNrÒ   Úpast_key_valuesr�   r’   )r@   r“   rV   r   r   r   )r4   r5   s   "€r6   r7   rí     sœ   ø€ ÷ ()ñ ()á—|‘|ð()ñ #¡5§<¡<±·±Ð#=Õ>ÀÕEð()ñ Ÿ™ tÕ+ð	()ñ
  �ð()ñ Ñ-Õ.ð()ñ 
‰u�|‰|™UŸ\™\¨DÕ0±%¹¿¹Õ2EÈÕ2LÐLÕ	Mñ()r9   c                ó@  € VP                   R R p. VORNV P                  N5pV P                  V4      P                  V4      P	                  ^^4      pV P                  V4      P                  V4      P	                  ^^4      p	V P                  V4      P                  V4      P	                  ^^4      p
Vw  r¼\        W‰W¼4      w  r‰Ve   VP                  WšV P                  4      w  rš\        P                  ! V P                  P                  \        4      pT! V VV	V
V3RV P                  '       d   V P                   MRRV P"                  RV P$                  RV P&                  /VB w  rïVP(                  ! . VORN5!  P+                  4       pV P-                  V4      pWï3# )Nrs   ç        rÓ   rý   rÔ   rH   )rW   r–   rø   Úviewr¬   rù   rú   rÄ   Úupdaterë   r   Úget_interfacerh   Ú_attn_implementationrå   rØ   rô   rÓ   rý   rü   rÈ   rà   rû   )rC   rw   r  rÒ   r  r�   Úinput_shapeÚhidden_shapeÚquery_statesrá   râ   r®   r¯   Úattention_interfacerä   rã   s   &&&&&,          r6   rS   ÚT5GemmaSelfAttention.forward  s§  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ%
ð /3¯m¯m¨m�D×*Ò*Àð%
ð —L‘Lð%
ð  ×.Ñ.ð%
ð ×/Ñ/ð%
ð ñ%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—k‘k +Ó.ˆØÐ(Ð(r9   )rô   rü   rh   r–   rö   rù   rë   rò   rÚ   rû   rø   rÓ   rý   rú   ©NNN©
r[   r\   r]   r^   Ú__doc__r=   rS   r_   r`   ra   rb   s   @@r6   rç   rç   é   s#   ù‡ € áG÷hó h÷6()÷ ()ò ()r9   rç   c                   óX   a a€ ] tR tRt oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )	ÚT5GemmaCrossAttentioni3  rè   c                ó&   <€ V ^8„  d   QhRS[ RS[/# rê   rì   )r4   r5   s   "€r6   r7   Ú"T5GemmaCrossAttention.__annotate__7  s    ø€ ÷ cñ cÑ2ð c¹sñ cr9   c                ó  <€ \         SV `  4        Wn        W n        \	        VR VP
                  VP                  ,          4      V n        VP                  VP                  ,          V n	        VP                  R,          V n        V P                  P                  V n        RV n        \        P                  ! VP
                  VP                  V P                  ,          VP                   R7      V n        \        P                  ! VP$                  VP                  V P                  ,          VP                   R7      V n        \        P                  ! VP$                  VP                  V P                  ,          VP                   R7      V n        \        P                  ! VP                  V P                  ,          VP
                  VP                   R7      V n        V P                  P,                  V n        VP$                  f   \/        R4      hR# )r–   Frf   NzBCross-attention needs cross_attention_hidden_size to be specified.rÙ   )r<   r=   rh   rë   r™   ri   rš   r–   rÊ   rÚ   ró   rÓ   rô   rö   r>   rk   r÷   rø   Úcross_attention_hidden_sizerù   rú   rû   rü   Ú
ValueErrorrþ   s   &&&€r6   r=   ÚT5GemmaCrossAttention.__init__7  s€  ø€ Ü‰ÑÔØŒØ"ŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÕ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Õ$\ˆÔ!Ø×3Ñ3°TÕ9ˆŒØ!%§¡×!>Ñ!>ˆÔØˆŒä—i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×.Ñ.°×0JÑ0JÈTÏ]É]Õ0ZÐag×avÑavô
ˆŒô —i’iØ×.Ñ.°×0JÑ0JÈTÏ]É]Õ0ZÐag×avÑavô
ˆŒô —i’iØ×&Ñ&¨¯©Õ6¸×8JÑ8JÐQW×QfÑQfô
ˆŒð '+§k¡k×&HÑ&HˆÔ#à×-Ñ-Ò5ÜÐaÓbÐbñ 6r9   c                ó6  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  S[ P                  R,          S[S[ P                  ,          R,          3,          /# )r.   rw   rÒ   NÚencoder_hidden_statesr  r�   r’   )r@   r“   r   r   r   rV   )r4   r5   s   "€r6   r7   r  S  sŽ   ø€ ÷ 3)ñ 3)á—|‘|ð3)ñ Ÿ™ tÕ+ð3)ñ  %Ÿ|™|¨dÕ2ð	3)ñ
  �ð3)ñ Ñ-Õ.ð3)ñ 
‰u�|‰|™UŸ\™\¨DÕ0±%¹¿¹Õ2EÈÕ2LÐLÕ	Mñ3)r9   c                óÀ  € Vf   \        R4      hVP                  R R p. VORNV P                  N5pV P                  V4      P	                  V4      P                  ^^4      pVe2   VP                  P                  V P                  4      p	VP                  p
Ve	   X	'       g¿   VP                  R R p. VORNV P                  N5pV P                  V4      P	                  V4      P                  ^^4      pV P                  V4      P	                  V4      P                  ^^4      pVe8   X
P                  WÞV P                  4      w  rÞRVP                  V P                  &   MNX
P                  V P                  ,          P                  pV
P                  V P                  ,          P                  p\         P"                  ! V P$                  P&                  \(        4      pT! V VVVV3RV P*                  '       d   V P,                  MRRV P.                  RR RV P0                  /VB w  ppVP2                  ! . VORN5!  P5                  4       pV P7                  V4      pVV3# )	Nz5Encoder hidden state is required for cross attention.Trs   r  rÓ   rý   rÔ   rH   )r  rW   r–   rø   r  r¬   Ú
is_updatedÚgetrë   Úcross_attention_cacherù   rú   r  ÚlayersÚkeysÚvaluesr   r  rh   r  rå   rØ   rô   rÓ   rü   rÈ   rà   rû   )rC   rw   rÒ   r  r  r�   r	  r
  r  r  Úcurr_past_key_valuesÚencoder_input_shapeÚencoder_hidden_shaperá   râ   r  rä   rã   s   &&&&&,            r6   rS   ÚT5GemmaCrossAttention.forwardS  sP  € ð !Ò(ÜÐTÓUÐUà#×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆØ—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆàÒ&Ø(×3Ñ3×7Ñ7¸¿¹ÓGˆJØ#2×#HÑ#HÐ àÒ"¯*Ø"7×"=Ñ"=¸c¸rÐ"BÐØ#LÐ%8Ð#L¸"Ð#L¸d¿m¹mÑ#LÐ ØŸ™Ð%:Ó;×@Ñ@ÐAUÓV×`Ñ`ÐabÐdeÓfˆJØŸ;™;Ð'<Ó=×BÑBÐCWÓX×bÑbÐcdÐfgÓhˆLàÒ*Ø+?×+FÑ+FÀzÐae×aoÑaoÓ+pÑ(�
Ø=A�×*Ñ*¨4¯>©>Ñ:øà-×4Ñ4°T·^±^ÕD×IÑIˆJØ/×6Ñ6°t·~±~ÕF×MÑMˆLä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ%
ð /3¯m¯m¨m�D×*Ò*Àð%
ð —L‘Lð%
ð  ð%
ð ×/Ñ/ð%
ð ñ%
Ñ!ˆ�\ð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r9   )rô   rü   rh   r–   rö   rù   rë   rÚ   rû   rø   rÓ   rú   r;   r  rb   s   @@r6   r  r  3  s#   ù‡ € áG÷có c÷83)÷ 3)ò 3)r9   r  c                   óX   a a€ ] tR tRt oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )	ÚT5GemmaEncoderLayeri‰  zEncoder sub-layer.c                ó    <€ V ^8„  d   QhRS[ /# ©r.   rë   ©r2   )r4   r5   s   "€r6   r7   Ú T5GemmaEncoderLayer.__annotate__Œ  s   ø€ ÷ 7ñ 7©#ñ 7r9   c                óL  <€ \         SV `  4        VP                  V n        Wn        W n        VP
                  V,          V n        \        VVR 7      V n        \        VP                  VP                  R7      V n        \        VP                  VP                  R7      V n        \        V4      V n        \        VP                  VP                  R7      V n        \        VP                  VP                  R7      V n        \"        P$                  ! VP&                  4      V n        R# ©)rh   rë   ©r0   N)r<   r=   ri   rh   rë   rï   Úattention_typerç   Ú	self_attnr,   Úrms_norm_epsÚpre_self_attn_layernormÚpost_self_attn_layernormrd   ÚmlpÚpre_feedforward_layernormÚpost_feedforward_layernormr>   rq   rr   rs   rþ   s   &&&€r6   r=   ÚT5GemmaEncoderLayer.__init__Œ  sÝ   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØŒØ"ŒØ$×0Ñ0°Õ;ˆÔä-ØØô
ˆŒô (6°f×6HÑ6HÈf×NaÑNaÔ'bˆÔ$Ü(6°v×7IÑ7IÈv×ObÑObÔ(cˆÔ%ä˜fÓ%ˆŒÜ)7¸×8JÑ8JÐPV×PcÑPcÔ)dˆÔ&Ü*8¸×9KÑ9KÐQW×QdÑQdÔ*eˆÔ'ä—z’z &×"5Ñ"5Ó6ˆŽr9   c                ó  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          R,          RS[ P                  R,          RS[ P                  R,          RS[S[ P                  3,          /# )r.   rw   r  NrÒ   r°   r’   )r@   r“   rV   Ú
LongTensorÚFloatTensor)r4   r5   s   "€r6   r7   r+     sz   ø€ ÷ ñ á—|‘|ðñ #¡5§<¡<±·±Ð#=Õ>ÀÕEðñ Ÿ™ tÕ+ð	ñ
 ×&Ñ&¨Õ-ðñ 
‰u× Ñ Ð!Õ	"ñr9   c                óP  € TpV P                  V4      pV P                  ! RR VRVRVRVRR/VB w  rV P                  V4      pW`P                  V4      ,           pTpV P	                  V4      pV P                  V4      pV P                  V4      pW`P                  V4      ,           pV# )rw   r  rÒ   r°   r  N© )r2  r0  r3  rs   r5  r4  r6  )rC   rw   r  rÒ   r°   r�   ÚresidualÚ_s   &&&&&,  r6   rS   ÚT5GemmaEncoderLayer.forward   sÏ   € ð !ˆØ×4Ñ4°]ÓCˆØŸ>š>ñ 
Ø'ð
à 3ð
ð *ð
ð &ð	
ð
 !ð
ð ñ
Ñˆð ×5Ñ5°mÓDˆØ §<¡<°Ó#>Õ>ˆà ˆØ×6Ñ6°}ÓEˆØŸ™ Ó/ˆØ×7Ñ7¸ÓFˆØ §<¡<°Ó#>Õ>ˆØÐr9   )r/  rh   rs   ri   rë   r4  r6  r3  r5  r2  r0  r  r  rb   s   @@r6   r'  r'  ‰  s!   ù‡ € Ù÷7ó 7÷(÷ ò r9   r'  c                   óX   a a€ ] tR tRt oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )	ÚT5GemmaDecoderLayeri½  z2Decoder sub-layer: an extra cross-attention layer.c                ó    <€ V ^8„  d   QhRS[ /# r)  r*  )r4   r5   s   "€r6   r7   Ú T5GemmaDecoderLayer.__annotate__À  s   ø€ ÷ eñ e©#ñ er9   c                ó  <€ \         SV `  4        VP                  V n        Wn        W n        VP
                  V,          V n        \        VVR 7      V n        \        VP                  VP                  R7      V n        \        VP                  VP                  R7      V n        \        V4      V n        \        VP                  VP                  R7      V n        \        VP                  VP                  R7      V n        \"        P$                  ! VP&                  4      V n        \+        WR 7      V n        \        VP                  VP                  R7      V n        \        VP                  VP                  R7      V n        R# r-  )r<   r=   ri   rh   rë   rï   r/  rç   r0  r,   r1  r2  r3  rd   r4  r5  r6  r>   rq   rr   rs   r  Ú
cross_attnÚpre_cross_attn_layernormÚpost_cross_attn_layernormrþ   s   &&&€r6   r=   ÚT5GemmaDecoderLayer.__init__À  s   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØŒØ"ŒØ$×0Ñ0°Õ;ˆÔä-ØØô
ˆŒô (6°f×6HÑ6HÈf×NaÑNaÔ'bˆÔ$Ü(6°v×7IÑ7IÈv×ObÑObÔ(cˆÔ%ä˜fÓ%ˆŒÜ)7¸×8JÑ8JÐPV×PcÑPcÔ)dˆÔ&Ü*8¸×9KÑ9KÐQW×QdÑQdÔ*eˆÔ'ä—z’z &×"5Ñ"5Ó6ˆŒÜ/°vÔSˆŒÜ(6°v×7IÑ7IÈv×ObÑObÔ(cˆÔ%Ü)7¸×8JÑ8JÐPV×PcÑPcÔ)dˆÖ&r9   c                óh  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[ P                  R,          R	S[ P                  R,          R
S[ P                  /	# )r.   rw   r  NrÒ   r°   r  Ú	use_cacher  Úencoder_attention_maskr’   )r@   r“   rV   r9  r   Úboolr:  )r4   r5   s   "€r6   r7   rC  ×  sµ   ø€ ÷ ,ñ ,á—|‘|ð,ñ #¡5§<¡<±·±Ð#=Õ>ÀÕEð,ñ Ÿ™ tÕ+ð	,ñ
 ×&Ñ&¨Õ-ð,ñ -¨tÕ3ð,ñ ˜$•;ð,ñ  %Ÿ|™|¨dÕ2ð,ñ !&§¡¨tÕ 3ð,ñ 
×	Ñ	ñ,r9   c	                ó&  € Tp
V P                  V4      pV P                  ! RR TRTRTRTRVe   VP                  MRRV/V	B w  rV P                  V4      pW P	                  V4      ,           pTp
V P                  V4      pV P                  ! RR VRVRVRVRV/V	B w  rV P                  V4      pW P	                  V4      ,           pTp
V P                  V4      pV P                  V4      pV P                  V4      pW P	                  V4      ,           pV# )	rw   r  rÒ   r°   r  NrJ  r  r<  )r2  r0  Úself_attention_cacher3  rs   rF  rE  rG  r5  r4  r6  )rC   rw   r  rÒ   r°   r  rJ  r  rK  r�   r=  r>  s   &&&&&&&&&,  r6   rS   ÚT5GemmaDecoderLayer.forward×  sj  € ð !ˆØ×4Ñ4°]ÓCˆØŸ>š>ñ 
Ø'ð
à 3ð
ð *ð
ð &ð	
ð
 ETÒD_˜O×@Ò@Ðeið
ð  ð
ð ñ
Ñˆð ×5Ñ5°mÓDˆØ §<¡<°Ó#>Õ>ˆà ˆØ×5Ñ5°mÓDˆØŸ?š?ñ 
Ø'ð
à"7ð
ð 2ð
ð ,ð	
ð
  ð
ð ñ
Ñˆð ×6Ñ6°}ÓEˆØ §<¡<°Ó#>Õ>ˆà ˆØ×6Ñ6°}ÓEˆØŸ™ Ó/ˆØ×7Ñ7¸ÓFˆØ §<¡<°Ó#>Õ>ˆØÐr9   )r/  rh   rE  rs   ri   rë   r4  rG  r6  r3  rF  r5  r2  r0  )NNNNFNNr  rb   s   @@r6   rA  rA  ½  s#   ù‡ € Ù<÷eó e÷.,÷ ,ò ,r9   rA  c                   óX   a a€ ] tR tRt oRtRV3R lV 3R llltV3R lR ltRtVtV ;t	# )	ÚT5GemmaClassificationHeadi  z-Head for sentence-level classification tasks.c                ó,   <€ V ^8„  d   QhRS[ RS[ RS[/# )r.   ri   Ú
num_labelsÚclassifier_dropout_rater1   )r4   r5   s   "€r6   r7   Ú&T5GemmaClassificationHead.__annotate__	  s#   ø€ ÷ ;ñ ;¡Cð ;±Sð ;ÑSXñ ;r9   c                ó’   <€ \         SV `  4        \        P                  ! VR 7      V n        \        P
                  ! W4      V n        R# ))r×   N)r<   r=   r>   rq   rs   rk   Úout_proj)rC   ri   rS  rT  rD   s   &&&&€r6   r=   Ú"T5GemmaClassificationHead.__init__	  s/   ø€ Ü‰ÑÔÜ—z’zÐ$;Ô<ˆŒÜŸ	š	 +Ó:ˆŽr9   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r.   rw   r’   ©r@   r“   )r4   r5   s   "€r6   r7   rU    s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r9   c                óJ   € V P                  V4      pV P                  V4      pV# r;   ©rs   rW  )rC   rw   s   &&r6   rS   Ú!T5GemmaClassificationHead.forward  s$   € ØŸ™ ]Ó3ˆØŸ™ mÓ4ˆØÐr9   r]  )r  r  rb   s   @@r6   rQ  rQ    s!   ù‡ € Ù7÷;õ ;÷
÷ ð r9   rQ  c                   óX   a a€ ] tR tRt oRtRV3R lV 3R llltV3R lR ltRtVtV ;t	# )	ÚT5GemmaLMHeadi  z.Head for language modeling (generation) tasks.c                ó,   <€ V ^8„  d   QhRS[ RS[ RS[/# )r.   ri   Ú
vocab_sizerg   )r2   rL  )r4   r5   s   "€r6   r7   ÚT5GemmaLMHead.__annotate__  s'   ø€ ÷ Fñ F¡Cð F±Sð FÁñ Fr9   c                ó^   <€ \         SV `  4        \        P                  ! WVR 7      V n        R# )rf   N)r<   r=   r>   rk   rW  )rC   ri   rb  rg   rD   s   &&&&€r6   r=   ÚT5GemmaLMHead.__init__  s    ø€ Ü‰ÑÔÜŸ	š	 +ÀÔEˆŽr9   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rZ  r[  )r4   r5   s   "€r6   r7   rc    s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r9   c                ó(   € V P                  V4      pV# r;   ©rW  )rC   rw   Úlogitss   && r6   rS   ÚT5GemmaLMHead.forward  s   € Ø—‘˜}Ó-ˆØˆr9   rh  )Fr  rb   s   @@r6   r`  r`    s#   ù‡ € Ù8÷Fõ F÷÷ ð r9   r`  c                   óš   a a€ ] tR tRt oRtRtRR.tR.tRtRt	Rt
RtRtRt]P                  ! 4       V 3R l4       tR	 tV3R
 ltRtVtV ;t# )ÚT5GemmaPreTrainedModeli   ÚmodelTr'  rA  r  Nc                ó°  <€ \         SV `  V4       V P                  P                  p\	        V\
        4      '       dÆ   VP                  P                  P                  ^ ,          R,          p\        P                  ! VP                  P                  RW#,          R7       \        VP                  R4      '       dG   VP                  P                  e-   \        P                  ! VP                  P                  4       R# R# R# \	        V\        4      '       d‚   V P                  P                  '       gd   VP                  P                  P                  ^ ,          R,          p\        P                  ! VP                  P                  RW#,          R7       R# R# RVP                   P"                  9   d#   \        P                  ! VP                  4       R# R# )r   r  )rK   Ústdrg   NÚRMSNormrÙ   )r<   Ú_init_weightsrh   Úinitializer_ranger©   rQ  rW  rB   rW   ÚinitÚnormal_rñ   rg   Úzeros_r`  Útie_word_embeddingsrD   r[   )rC   rÎ   ro  ÚscalerD   s   &&  €r6   rq  Ú$T5GemmaPreTrainedModel._init_weights0  s1  ø€ ô 	‰Ñ˜fÔ%Ø�k‰k×+Ñ+ˆÜ�fÔ7×8Ò8Ø—O‘O×*Ñ*×0Ñ0°Õ3°tÕ;ˆEÜ�LŠL˜Ÿ™×/Ñ/°c¸s½{ÕKÜ�v—‘¨×/Ò/°F·O±O×4HÑ4HÒ4TÜ—’˜FŸO™O×0Ñ0Ö1ñ 5UÑ/ä˜¤×.Ò.Ø—;‘;×2×2Ð2ØŸ™×.Ñ.×4Ñ4°QÕ7¸4Õ?�Ü—’˜VŸ_™_×3Ñ3¸#À3Å;×Oñ 3ð ˜&×*Ñ*×3Ñ3Ô3Ü�KŠK˜Ÿ™Ö&ñ 4r9   c                óf  € V P                   P                  P                  pV P                   P                  P                  pVf   \	        R4      hVP                  VP                  4      pVRRR13,          P                  4       VR&   W$R&   Vf   \	        R4      hVP                  VR8H  V4       V# )	zÚ
Shifts input_ids to the right, prepends the decoder_start_token_id, and handles
pad_token_id replacement for labels that were -100.
This is a common preparation step for decoder inputs in sequence-to-sequence models.
Nz:self.model.config.decoder.bos_token_id has to be defined. .z9self.model.config.decoder.pad_token_id has to be defined.rH   ).:é   NN).r   iœÿÿÿ)	rh   ÚdecoderÚbos_token_idÚpad_token_idr  Ú	new_zerosrW   r�   Úmasked_fill_)rC   Ú	input_idsÚdecoder_start_token_idr}  Úshifted_input_idss   &&   r6   Ú_shift_rightÚ#T5GemmaPreTrainedModel._shift_rightB  s±   € ð "&§¡×!4Ñ!4×!AÑ!AÐØ—{‘{×*Ñ*×7Ñ7ˆà!Ò)ÜÐYÓZÐZð &×/Ñ/°	·±Ó@ÐØ%.¨s°C°R°C¨xÕ%8×%>Ñ%>Ó%@Ð˜'Ñ"Ø$:˜&Ñ!àÒÜÐXÓYÐYð 	×&Ñ&Ð'8¸DÑ'@À,ÔOà Ð r9   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r~   r   )r4   r5   s   "€r6   r7   Ú#T5GemmaPreTrainedModel.__annotate__   s   ø‡ ‚ áÑò r9   r<  )r[   r\   r]   r^   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendÚ_can_record_outputsr@   r·   rq  rƒ  Ú__annotate_func__r_   r`   ra   rb   s   @@r6   rl  rl     sp   ù‡ € ð  ÐØ&*Ð#Ø.Ð0EÐFÐØ#4Ð"5ÐØÐØ€NØÐà!ÐØ"&ÐàÐà
‡]‚]ƒ_ô'ó ð'ò"!÷E … r9   rl  c                ó    € V ^8„  d   QhR\         P                  R,          R\         P                  R\        R,          R\         P                  /# )r.   Ú	token_idsNrw   r}  r’   )r@   r9  r“   r2   )r4   s   "r6   r7   r7   ]  sG   € ÷ ñ Ü×Ñ $Õ&ðä—<‘<ðô ˜•*ðô ‡\�\ñ	r9   c                ó2  € V e>   Vf   \        R4      hW8g  P                  VP                  \        P                  4      pV# \        P
                  ! VP                  ^ ,          VP                  ^,          3VP                  \        P                  R7      pV# )z%Construct the default attention mask.z3`pad_token_id` is required for padding information.©r{   r˜   )r  rœ   r{   r@   ÚlongÚonesrW   )r“  rw   r}  rÒ   s   &&& r6   Úmake_default_2d_attention_maskr˜  ]  sŠ   € ð ÒØÒÜÐRÓSÐSØ#Ñ3×7Ñ7¸×8LÑ8LÌeÏjÉjÓYˆð
 Ðô ŸšØ× Ñ  Õ# ]×%8Ñ%8¸Õ%;Ð<À]×EYÑEYÔaf×akÑakô
ˆð Ðr9   c                   óh   a a€ ] tR tRt oR]R]/tV 3R lt]]	RV3R lR ll4       4       t
RtVtV ;t# )	ÚT5GemmaEncoderin  Ú
attentionsrw   c           	     ó^  <€ \         SV `  V4       VP                  V n        VP                  V n        \
        P                  ! VP                  VP                  V P                  4      V n        \        VP                  VP                  R 7      V n        RV n        \
        P                  ! \        VP                  4       Uu. uF  p\!        W4      NK  	  up4      V n        \
        P$                  ! VP&                  4      V n        \+        VR7      V n        V P/                  4        R# u upi ©r.  F©rh   N)r<   r=   r}  Úpadding_idxrb  r>   Ú	Embeddingri   Úembed_tokensr,   r1  ÚnormÚgradient_checkpointingÚ
ModuleListÚrangeÚnum_hidden_layersr'  r  rq   rr   rs   rz   Ú
rotary_embÚ	post_initrþ   s   && €r6   r=   ÚT5GemmaEncoder.__init__t  óà   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸLšL¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ" 6×#5Ñ#5¸6×;NÑ;NÔOˆŒ	Ø&+ˆÔ#ä—m’mÜEJÈ6×KcÑKcÔEdÓeÑEd¸	Ô  Ö3ÑEdÑeó
ˆŒô —z’z &×"5Ñ"5Ó6ˆŒÜ0¸Ô?ˆŒð 	�‰Öùò fó   ÃD*c                óæ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          RS[S[,          /# ©r.   r€  NrÒ   r°   Úinputs_embedsr�   r’   )r@   r9  r“   r:  r   r    rV   r   )r4   r5   s   "€r6   r7   ÚT5GemmaEncoder.__annotate__ˆ  s|   ø€ ÷ 6
ñ 6
á×#Ñ# dÕ*ð6
ñ Ÿ™ tÕ+ð6
ñ ×&Ñ&¨Õ-ð	6
ñ
 ×(Ñ(¨4Õ/ð6
ñ Ñ+Õ,ð6
ñ 
‘Õ	 ñ6
r9   c                óÚ  € VR J VR J,          '       d   \        R4      hVP                  RR 4       Vf   V P                  V4      pVfE   \        P                  ! VP
                  ^,          VP                  R7      pVP                  ^ 4      pVf!   \        WV P                  P                  4      p\        T;p\        4      '       g+   RV P                  RVRV/pR\        R/ VB R\        R/ VB /pTp\        P                  ! V P                  P                   R	,          VP"                  R
7      p	W‰,          pV P%                  V4      pV P'                  Wƒ4      p
\)        V P*                  R V P                  P,                   4       F2  w  r¼V! VV
W`P                  P.                  V,          ,          V3/ VB pK4  	  V P1                  V4      pV P%                  V4      p\3        VR7      # )Nú:You must specify exactly one of input_ids or inputs_embedsr  ©r{   rh   r®  rÒ   Úfull_attentionrð   ç      à?r—   )Úlast_hidden_stater<  )r  Úpopr¡  r@   r›   rW   r{   r¾   r˜  rh   r}  r©   Údictr   r   Útensorri   r˜   rs   r§  Ú	enumerater  r¦  rï   r¢  r   )rC   r€  rÒ   r°   r®  r�   Úself_attn_mask_mappingÚmask_kwargsrw   Ú
normalizerr  ÚiÚlayer_modules   &&&&&,       r6   rS   ÚT5GemmaEncoder.forward†  sÏ  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZð 	�
‰
Ð$ dÔ+àÒ Ø ×-Ñ-¨iÓ8ˆMàÒÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\ˆLØ'×1Ñ1°!Ó4ˆLàÒ!Ü;¸IÐVZ×VaÑVa×VnÑVnÓoˆNä°NÐBÐ0ÄD×IÒIà˜$Ÿ+™+Ø Ø  .ðˆKð !Ô";Ñ"J¸kÑ"JØ#Ô%MÑ%\ÐP[Ñ%\ð&Ð"ð
 &ˆÜ—\’\ $§+¡+×"9Ñ"9¸3Õ">Àm×FYÑFYÔZˆ
Ø%Õ2ˆØŸ™ ]Ó3ˆà"Ÿo™o¨mÓJÐä(¨¯©Ð5T°t·{±{×7TÑ7TÐ)UÖV‰OˆAÙ(ØØ#Ø&§{¡{×'>Ñ'>¸qÕ'AÕBØñ	ð
 ñŠMñ  Wð Ÿ	™	 -Ó0ˆØŸ™ ]Ó3ˆÜØ+ô
ð 	
r9   ©rs   r¡  r£  r  r¢  rŸ  r§  rb  ©NNNN)r[   r\   r]   r^   rç   r'  r�  r=   r&   r(   rS   r_   r`   ra   rb   s   @@r6   rš  rš  n  sD   ù‡ € àÐ*ØÐ,ðÐõ
ð$  Ø÷6
ñ 6
ó ó  ÷6
ð 6
r9   rš  c                   óŒ   a a€ ] tR tRt oR]! ]^R7      R]! ]^R7      R]/tV 3R lt	]
]R
V3R lR ll4       4       tR	tVtV ;t# )ÚT5GemmaDecoderiÁ  r›  )ÚindexÚcross_attentionsrw   c           	     ó^  <€ \         SV `  V4       VP                  V n        VP                  V n        \
        P                  ! VP                  VP                  V P                  4      V n        \        VP                  VP                  R 7      V n        RV n        \
        P                  ! \        VP                  4       Uu. uF  p\!        W4      NK  	  up4      V n        \
        P$                  ! VP&                  4      V n        \+        VR7      V n        V P/                  4        R# u upi r�  )r<   r=   r}  rŸ  rb  r>   r   ri   r¡  r,   r1  r¢  r£  r¤  r¥  r¦  rA  r  rq   rr   rs   rz   r§  r¨  rþ   s   && €r6   r=   ÚT5GemmaDecoder.__init__È  rª  r«  c                ó^  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[ P                  R,          R	S[ P                  R,          R
S[S[,          RS[S[	,          /
# )r.   r€  NrÒ   r°   r  r®  rJ  r  rK  r�   r’   )
r@   r9  r“   r   r:  rL  r   r    rV   r   )r4   r5   s   "€r6   r7   ÚT5GemmaDecoder.__annotate__Ü  sÌ   ø€ ÷ P
ñ P
á×#Ñ# dÕ*ðP
ñ Ÿ™ tÕ+ðP
ñ ×&Ñ&¨Õ-ð	P
ñ
 -¨tÕ3ðP
ñ ×(Ñ(¨4Õ/ðP
ñ ˜$•;ðP
ñ  %Ÿ|™|¨dÕ2ðP
ñ !&§¡¨tÕ 3ðP
ñ Ñ+Õ,ðP
ñ 
Ñ:Õ	:ñP
r9   c	                óJ  € VR J VR J,          '       d   \        R4      hVf   \        R4      hVf   V P                  V4      pV P                  '       g5   V'       d-   Vf)   \        \	        V P
                  R7      \	        4       4      pVfb   Ve   VP                  4       M^ p
\        P                  ! VP                  ^,          VP                  R7      V
,           pVP                  ^ 4      pVf%   Vf!   \        WV P
                  P                  4      p\        T;p\        4      '       g?   RV P
                  RTRTRVe   VP                   MR R	V/pR
\#        R/ VB R\%        R/ VB /p\        T;p\        4      '       g   R
\'        V P
                  VVVR7      /pTp\        P(                  ! V P
                  P*                  R,          VP,                  R7      pWï,          pV P/                  V4      pV P1                  Wã4      p\3        V P4                  R V P
                  P6                   4       F>  w  ppV! VVW°P
                  P8                  V,          ,          VVVVVR
,          3/ V	B pK@  	  V P;                  V4      pV P/                  V4      p\=        VVR7      # )Nr±  z0`encoder_hidden_states` must be given in decoderrž  r²  rh   r®  rÒ   r  r°   r³  rð   )rh   r®  rÒ   r  r´  r—   )rµ  r  r<  )r  r¡  rØ   r   r   rh   Úget_seq_lengthr@   r›   rW   r{   r¾   r˜  r}  r©   r·  rN  r   r   r   r¸  ri   r˜   rs   r§  r¹  r  r¦  rï   r¢  r   )rC   r€  rÒ   r°   r  r®  rJ  r  rK  r�   Úpast_seen_tokensrº  r»  Úcross_attn_mask_mappingrw   r¼  r  r½  r¾  s   &&&&&&&&&,         r6   rS   ÚT5GemmaDecoder.forwardÚ  s„  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZØ Ò(ÜÐOÓPÐPàÒ Ø ×-Ñ-¨iÓ8ˆMà�}�}ˆ}§¨Ò/Fô 2´,ÀdÇkÁkÔ2RÔT`ÓTbÓcˆOàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLàÒ! oÒ&=Ü;¸IÐVZ×VaÑVa×VnÑVnÓoˆNä°NÐBÐ0ÄD×IÒIà˜$Ÿ+™+Ø Ø  .Ø!È?ÒKf ?×#GÒ#GÐlpØ ðˆKð !Ô"4Ñ"C°{Ñ"CØ#Ô%FÑ%UÈÑ%Uð&Ð"ô
 Ð5KÐKÐ1ÌT×RÒRà Ô";ØŸ;™;Ø"/Ø#9Ø*?ô	#ð'Ð#ð &ˆÜ—\’\ $§+¡+×"9Ñ"9¸3Õ">Àm×FYÑFYÔZˆ
Ø%Õ2ˆØŸ™ ]Ó3ˆà"Ÿo™o¨mÓJÐä(¨¯©Ð5T°t·{±{×7TÑ7TÐ)UÖV‰OˆAˆ|Ù(ØØ#Ø&§{¡{×'>Ñ'>¸qÕ'AÕBØØØØ%Ø'Ð(8Õ9ñ
ð ñ
ŠMñ  Wð Ÿ	™	 -Ó0ˆØŸ™ ]Ó3ˆÜ8Ø+Ø+ô
ð 	
r9   rÀ  )NNNNNNNN)r[   r\   r]   r^   r'   rç   r  rA  r�  r=   r&   r(   rS   r_   r`   ra   rb   s   @@r6   rÃ  rÃ  Á  s\   ù‡ € à‘nÐ%9ÀÔCØ™NÐ+@ÈÔJØÐ,ðÐõð$  Ø÷P
ñ P
ó ó  ÷P
ð P
r9   rÃ  c                   ót   a a€ ] tR tRt oV3R lV 3R lltR tR t]]R	V3R lR ll4       4       t	Rt
VtV ;t# )
ÚT5GemmaModeli/  c                ó    <€ V ^8„  d   QhRS[ /# r~   r   )r4   r5   s   "€r6   r7   ÚT5GemmaModel.__annotate__1  ó   ø€ ÷ 	ñ 	™}ñ 	r9   c                óè   <€ \         SV `  V4       VP                  '       g   \        R 4      h\	        VP
                  4      V n        \        VP                  4      V n        V P                  4        R# )zVT5GemmaModel only support encoder-decoder modeling. Use `T5GemmaEncoderModel` instead.N)	r<   r=   Úis_encoder_decoderr  rš  ÚencoderrÃ  r{  r¨  rt   s   &&€r6   r=   ÚT5GemmaModel.__init__1  sR   ø€ Ü‰Ñ˜Ô à×(×(Ð(ÜÐuÓvÐvä% f§n¡nÓ5ˆŒÜ% f§n¡nÓ5ˆŒà�‰Ör9   c                ó6   € V P                   P                  4       # r;   ©rÖ  Úget_input_embeddingsrX   s   &r6   rÚ  Ú!T5GemmaModel.get_input_embeddings<  ó   € Ø�|‰|×0Ñ0Ó2Ð2r9   c                ó8   € V P                   P                  V4      # r;   ©rÖ  Úset_input_embeddings©rC   Únew_embeddingss   &&r6   rß  Ú!T5GemmaModel.set_input_embeddings?  ó   € Ø�|‰|×0Ñ0°Ó@Ð@r9   c                ó²  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[R,          R
S[ P                  R,          RS[ P                  R,          RS[R,          RS[S[	,          RS[
/# )r.   r€  NrÒ   r°   Údecoder_input_idsÚdecoder_attention_maskÚdecoder_position_idsÚencoder_outputsr  r®  Údecoder_inputs_embedsrJ  r�   r’   )r@   r9  r:  Ú
BoolTensorr   r   r“   rL  r   r    r   )r4   r5   s   "€r6   r7   rÒ  D  sõ   ø€ ÷ 6
ñ 6
á×#Ñ# dÕ*ð6
ñ ×)Ñ)¨DÕ0ð6
ñ ×&Ñ&¨Õ-ð	6
ñ
 !×+Ñ+¨dÕ2ð6
ñ !&× 0Ñ 0°4Õ 7ð6
ñ $×.Ñ.°Õ5ð6
ñ )¨4Õ/ð6
ñ -¨tÕ3ð6
ñ —|‘| dÕ*ð6
ñ  %Ÿ|™|¨dÕ2ð6
ñ ˜$•;ð6
ñ Ñ+Õ,ð6
ñ 
ñ6
r9   c                óª  € Vf   V P                   ! RRVRVRVRV	/VB pVP                  pV P                  ! RRVRVRVRV
RVRVRVRV/VB p\        VP                  VP                  VP                  R	R
4      '       d   VP                  MVP                  3VP                  VP                  VP                  VP                  VP                  R7      # )a8  
decoder_position_ids (`torch.LongTensor` of shape `(batch_size, decoder_sequence_length)`, *optional*):
    Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the range `[0,
    config.decoder.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)
r€  rÒ   r°   r®  r  r  rK  rJ  Úoutput_hidden_statesF)rµ  r  Údecoder_hidden_statesÚdecoder_attentionsrÅ  Úencoder_last_hidden_stater  Úencoder_attentionsr<  )	rÖ  rµ  r{  r   r  r  rw   r›  rÅ  )rC   r€  rÒ   r°   rå  ræ  rç  rè  r  r®  ré  rJ  r�   r  Údecoder_outputss   &&&&&&&&&&&&,  r6   rS   ÚT5GemmaModel.forwardB  s/  € ð, Ò"Ø"Ÿlšlñ Ø#ðà-ðð *ðð ,ð	ð
 ñˆOð !0× AÑ AÐàŸ,š,ñ 

Ø'ð

à1ð

ð .ð

ð 0ð	

ð
 ,ð

ð #8ð

ð $2ð

ð  ð

ð ñ

ˆô "Ø-×?Ñ?Ø+×;Ñ;à�z‰zÐ0°%×8Ò8ð #2×"?Ò"?à!×3Ñ3Ð5Ø.×9Ñ9Ø,×=Ñ=Ø&5×&GÑ&GØ"1×"?Ñ"?Ø.×9Ñ9ô
ð 	
r9   )r{  rÖ  )NNNNNNNNNNN©r[   r\   r]   r^   r=   rÚ  rß  r"   r!   rS   r_   r`   ra   rb   s   @@r6   rÐ  rÐ  /  s@   ù‡ € ÷	ó 	ò3òAð Ø÷6
ñ 6
ó ó ÷6
ð 6
r9   rÐ  c                   ót   a a€ ] tR tRt oV3R lV 3R lltR tR t]]R	V3R lR ll4       4       t	Rt
VtV ;t# )
ÚT5GemmaEncoderModeli}  c                ó    <€ V ^8„  d   QhRS[ /# r~   r   )r4   r5   s   "€r6   r7   Ú T5GemmaEncoderModel.__annotate__  s   ø€ ÷ ñ ™}ñ r9   c                ó´   <€ \         SV `  V4       VP                  '       d   \        R 4      h\	        VP
                  4      V n        V P                  4        R# )zQT5GemmaEncoderModel only supports encoder-only model. Use `T5GemmaModel` instead.N)r<   r=   rÕ  r  rš  rÖ  r¨  rt   s   &&€r6   r=   ÚT5GemmaEncoderModel.__init__  sB   ø€ Ü‰Ñ˜Ô à×$×$Ð$ÜÐpÓqÐqä% f§n¡nÓ5ˆŒØ�‰Ör9   c                ó6   € V P                   P                  4       # r;   rÙ  rX   s   &r6   rÚ  Ú(T5GemmaEncoderModel.get_input_embeddingsˆ  rÜ  r9   c                ó8   € V P                   P                  V4      # r;   rÞ  rà  s   &&r6   rß  Ú(T5GemmaEncoderModel.set_input_embeddings‹  rã  r9   c                óÖ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          RS[/# r­  )r@   r9  r:  r“   r   r    r   )r4   r5   s   "€r6   r7   r÷  �  sw   ø€ ÷ ñ á×#Ñ# dÕ*ðñ ×)Ñ)¨DÕ0ðñ ×&Ñ&¨Õ-ð	ñ
 —|‘| dÕ*ðñ Ñ+Õ,ðñ 
ñr9   c                ó:   € V P                   ! RR VRVRVRV/VB pV# )r€  rÒ   r°   r®  r<  ©rÖ  )rC   r€  rÒ   r°   r®  r�   rè  s   &&&&&, r6   rS   ÚT5GemmaEncoderModel.forwardŽ  sH   € ð Ÿ,š,ñ 
Øð
à)ð
ð &ð
ð (ð	
ð
 ñ
ˆð Ðr9   r   rÁ  ró  rb   s   @@r6   rõ  rõ  }  s@   ù‡ € ÷ó ò3òAð Ø÷ñ ó ó ÷ð r9   rõ  c                   ó¾   a a€ ] tR tRt oRR/tRR/tRR.R.3/tV3R lV 3R	 lltR
 tR t	]
]RV3R lR ll4       4       tV3R lR ltV3R lV 3R lltRtVtV ;t# )ÚT5GemmaForConditionalGenerationi¢  zlm_head.out_proj.weightz!model.decoder.embed_tokens.weightzlm_head.out_projÚcolwise_gather_outputrw   ri  c                ó    <€ V ^8„  d   QhRS[ /# r~   r   )r4   r5   s   "€r6   r7   Ú,T5GemmaForConditionalGeneration.__annotate__§  rÓ  r9   c                ó  <€ R Vn         \        SV `	  V4       \        V4      V n        VP
                  P                  V n        \        VP
                  P                  V P                  4      V n	        RV n
        V P                  4        R# )TÚForMaskedLMN)rÕ  r<   r=   rÐ  rm  r{  rb  r`  ri   Úlm_headÚ	loss_typer¨  rt   s   &&€r6   r=   Ú(T5GemmaForConditionalGeneration.__init__§  sb   ø€ Ø$(ˆÔ!Ü‰Ñ˜Ô ä! &Ó)ˆŒ
Ø Ÿ.™.×3Ñ3ˆŒÜ$ V§^¡^×%?Ñ%?ÀÇÁÓQˆŒØ&ˆŒà�‰Ör9   c                ó@  € WP                   n        V P                  P                  '       dr   VP                  V P
                  P                  P                  n        VP                  P                  ^ ,          V P
                  P                  P                  n	        R# R# )r   N)
r	  rW  rh   rv  rB   rm  r{  r¡  rW   Únum_embeddingsrà  s   &&r6   Úset_output_embeddingsÚ5T5GemmaForConditionalGeneration.set_output_embeddings²  sk   € Ø .�‰Ôð �;‰;×*×*Ð*Ø5C×5JÑ5JˆD�J‰J×Ñ×+Ñ+Ô2Ø=K×=RÑ=R×=XÑ=XÐYZÕ=[ˆD�J‰J×Ñ×+Ñ+Ö:ñ +r9   c                ó.   € V P                   P                  # r;   )r	  rW  rX   s   &r6   Úget_output_embeddingsÚ5T5GemmaForConditionalGeneration.get_output_embeddings»  s   € Ø�|‰|×$Ñ$Ð$r9   c                ó8  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[R,          R
S[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[S[ P                  ,          RS[	S[
,          RS[S[ P                  ,          S[,          /# )r.   r€  NrÒ   r°   rå  ræ  rç  rè  r  r®  ré  ÚlabelsrJ  Úlogits_to_keepr�   r’   )r@   r9  r:  rê  r   r   rL  r2   r“   r   r    rV   r   )r4   r5   s   "€r6   r7   r  À  sB  ø€ ÷ G
ñ G
á×#Ñ# dÕ*ðG
ñ ×)Ñ)¨DÕ0ðG
ñ ×&Ñ&¨Õ-ð	G
ñ
 !×+Ñ+¨dÕ2ðG
ñ !&× 0Ñ 0°4Õ 7ðG
ñ $×.Ñ.°Õ5ðG
ñ )¨4Õ/ðG
ñ -¨tÕ3ðG
ñ ×(Ñ(¨4Õ/ðG
ñ  %×0Ñ0°4Õ7ðG
ñ × Ñ  4Õ'ðG
ñ ˜$•;ðG
ñ ™eŸl™lÕ*ðG
ñ Ñ+Õ,ðG
ñ  
‰u× Ñ Õ	!¡OÕ	3ñ!G
r9   c                óè  € Ve   Vf   V
f   V P                  V4      pV P                  ! RRVRVRVRVRVRVRVR	VR
V	RV
RV/VB pVP                  p\        V\        4      '       d   \        V) R4      MTpV P                  VRVR3,          4      pV P                  4       P                  pVP                  e=   VVP                  ,          p\        P                  ! V4      pVVP                  ,          pRpVe   V P                  ! VW°P                  3/ VB p\        VVVP                  VP                   VP"                  VP$                  VP&                  VP(                  VP*                  R7	      # )a¼  
decoder_position_ids (`torch.LongTensor` of shape `(batch_size, decoder_sequence_length)`, *optional*):
    Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the range `[0,
    config.decoder.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
Nr€  rÒ   r°   rå  ræ  rç  rè  r  r®  ré  rJ  r§   )	Úlossri  r  rí  rî  rÅ  rï  r  rð  r<  )rƒ  rm  rµ  r©   r2   Úslicer	  Úget_decoderrh   Úfinal_logit_softcappingr@   rÜ   Úloss_functionrb  r   r  rí  rî  rÅ  rï  r  rð  )rC   r€  rÒ   r°   rå  ræ  rç  rè  r  r®  ré  r  rJ  r  r�   rñ  rw   Úslice_indicesri  Údecoder_configr  s   &&&&&&&&&&&&&&,      r6   rS   Ú'T5GemmaForConditionalGeneration.forward¾  s´  € ð: ÒÐ"3Ò";Ð@UÒ@]à $× 1Ñ 1°&Ó 9Ðà.2¯jªjñ /
Øð/
à)ð/
ð &ð/
ð 0ð	/
ð
 $:ð/
ð "6ð/
ð ,ð/
ð ,ð/
ð (ð/
ð #8ð/
ð  ð/
ð ñ/
ˆð (×9Ñ9ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆØ×)Ñ)Ó+×2Ñ2ˆØ×1Ñ1Ò=Ø˜n×DÑDÕDˆFÜ—Z’Z Ó'ˆFØ˜n×DÑDÕDˆFàˆØÒà×%Ò% f¨f·o±oÑPÈÑPˆDäØØØ+×;Ñ;Ø"1×"GÑ"GØ.×AÑAØ,×=Ñ=Ø&5×&OÑ&OØ"1×"GÑ"GØ.×AÑAô

ð 
	
r9   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r.   r  r[  )r4   r5   s   "€r6   r7   r  	  s   ø€ ÷ )ñ )¹E¿L¹Lñ )r9   c                ó$   € V P                  V4      # r;   )rƒ  )rC   r  s   &&r6   Ú%prepare_decoder_input_ids_from_labelsÚET5GemmaForConditionalGeneration.prepare_decoder_input_ids_from_labels	  s   € Ø× Ñ  Ó(Ð(r9   c                ó>   <€ V ^8„  d   QhRS[ RS[RS[RS[RS[RS[/# )r.   Úgeneration_configÚmodel_kwargsÚgeneration_modeÚ
batch_sizeÚmax_cache_lengthr’   )r
   r·  r   r2   rL  )r4   r5   s   "€r6   r7   r    sO   ø€ ÷ I:ñ I:á+ðI:ñ ðI:ñ (ð	I:ñ
 ðI:ñ ðI:ñ 
ñI:r9   c                ó  <€ \         SV `  VVVVV4       VP                  RJ d   R# VP                  pVf   RpMRVP                  9   p\        P
                  ! V P                  P                  RR7      4      pRVn        R.VP                  ,          Vn
        RVRV/p	VP                  R	4      p
V
e±   \        V
\        4      '       g   \        R
4      h\        V
P                   4      ^ 8”  d$   V
P                   P                  ^ 4      '       d   R# \#        V
P$                  4      pV\&        8X  d%   VR,          ^ ,          P(                  ^,          V	R&   V! R/ V	B V
n        M>\        \+        R/ RV P                  P                  RR7      RV/B \+        4       4      VR	&   \-        V R4      '       dL   V P.                  e<   \        V P.                  \        4      '       g   \        R4      hVR	,          V n        R# R# R# )a  Override cache preparation to force full attention on the cross-attention cache.

The decoder config may declare sliding-window layers, but cross-attention must always use full attention.
The default `_prepare_cache_for_generation` would otherwise build a sliding cross-attention cache.
FNÚ	offloadedT)r{  r³  rh   Ú
offloadingr  z`The `past_key_values` in `model_kwargs` must be of type `EncoderDecoderCache` for T5Gemma model.rè  Úmax_cache_lenÚ_cachezKThe internal cache must be of type `EncoderDecoderCache` for T5Gemma model.r<  )r<   Ú_prepare_cache_for_generationrJ  Úcache_implementationÚcopyÚdeepcopyrh   Úget_text_configrý   r¦  rï   r  r©   r   r  Úlenr  rª   r  r	   rW   r   rñ   r-  )rC   r$  r%  r&  r'  r(  r/  Úoffload_cacheÚcross_attn_configÚcross_attn_cache_kwargsr  Úcross_attn_clsrD   s   &&&&&&      €r6   r.  Ú=T5GemmaForConditionalGeneration._prepare_cache_for_generation  sú  ø€ ô 	‰Ñ-ØØØØØô	
ð ×&Ñ&¨%Ó/Ùà0×EÑEÐØÒ'Ø!‰Mà'Ð+<×+QÑ+QÑQˆMô !ŸMšM¨$¯+©+×*EÑ*EÈdÐ*EÓ*SÓTÐØ+/ÐÔ(Ø)9Ð(:Ð=N×=`Ñ=`Õ(`ÐÔ%ð Ð'Ø˜-ð#
Ðð
 '×*Ñ*Ð+<Ó=ˆØÒ&Ü˜oÔ/B×CÒCÜ Øvóð ô
 �?×-Ñ-Ó.°Ô2°×7QÑ7Q×7UÑ7UÐVW×7XÒ7XÙä! /×"GÑ"GÓHˆNØ¤Ô,Ø;GÐHYÕ;ZÐ[\Õ;]×;cÑ;cÐdeÕ;fÐ'¨Ñ8á4BÑ4]ÐE\Ñ4]ˆOÕ1ô /BÜñ à  $§+¡+×"=Ñ"=ÀdÐ"=Ó"KØ$ mðñô “ó/ˆLÐ*Ñ+ô �4˜×"Ò" t§{¡{Ò'>Ü˜dŸk™kÔ+>×?Ò?Ü Ð!nÓoÐoà&Ð'8Õ9ˆDŽKñ	 (?Ñ"r9   )r-  r	  r
  rm  rb  )NNNNNNNNNNNNr   )r[   r\   r]   r^   Ú_tied_weights_keysÚ_tp_planÚ_pp_planr=   r  r  r"   r!   rS   r!  r.  r_   r`   ra   rb   s   @@r6   r  r  ¢  s†   ù‡ € Ø3Ð5XÐYÐØ"Ð$;Ð<€HØ" oÐ%6¸¸
Ð$CÐD€H÷	ó 	ò\ò%ð Ø÷G
ñ G
ó ó ðG
÷R)ð )÷I:÷ I:ó I:r9   r  c                   óx   a a€ ] tR tRt oR	V3R lV 3R llltR tR t]]R
V3R lR ll4       4       t	Rt
VtV ;t# )Ú T5GemmaForSequenceClassificationiX  c                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# ©r.   rh   rÕ  N©r)   rL  )r4   r5   s   "€r6   r7   Ú-T5GemmaForSequenceClassification.__annotate__Z  s   ø€ ÷ ñ ™}ð Á$ÈÅ+ñ r9   c                ó®  <€ Ve   W!n         \        SV `	  V4       VP                  V n        VP                   '       d   \	        V4      V n        M\        V4      V n        VP                  P                  pVP                   '       d   VP                  P                  p\        VRR4      p\        W0P                  V4      V n        V P                  4        R# )z”
is_encoder_decoder (`Optional`, *optional*):
    Whether use encoder_decoder for sequence classification. When set to False, only encoder is used.
NrT  çš™™™™™¹?©rÕ  r<   r=   rS  rÐ  rm  rõ  rÖ  ri   r{  r™   rQ  Úscorer¨  ©rC   rh   rÕ  ri   Úclassifier_dropoutrD   s   &&&  €r6   r=   Ú)T5GemmaForSequenceClassification.__init__Z  s¦   ø€ ð
 Ò)Ø(:Ô%Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒà×$×$Ð$Ü% fÓ-ˆD�Jä,¨VÓ4ˆDŒJà—n‘n×0Ñ0ˆØ×$×$Ð$Ø Ÿ.™.×4Ñ4ˆKä$ VÐ-FÈÓLÐÜ.¨{¿O¹OÐM_Ó`ˆŒ
Ø�‰Ör9   c                ó6   € V P                   P                  4       # r;   ©rm  rÚ  rX   s   &r6   rÚ  Ú5T5GemmaForSequenceClassification.get_input_embeddingsq  ó   € Ø�z‰z×.Ñ.Ó0Ð0r9   c                ó<   € V P                   P                  V4       R # r;   ©rm  rß  ©rC   rÑ   s   &&r6   rß  Ú5T5GemmaForSequenceClassification.set_input_embeddingst  ó   € Ø�
‰
×'Ñ'¨Ö.r9   c                ó²  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[ P                  R,          R
S[ P                  R,          RS[ P                  R,          RS[S[,          RS[/# ©r.   r€  NrÒ   r°   rå  ræ  rç  rè  r®  ré  r  r�   r’   )r@   r9  r“   r   r:  r   r    r   )r4   r5   s   "€r6   r7   rA  y  sû   ø€ ÷ i
ñ i
á×#Ñ# dÕ*ði
ñ Ÿ™ tÕ+ði
ñ ×&Ñ&¨Õ-ð	i
ñ
 !×+Ñ+¨dÕ2ði
ñ !&§¡¨tÕ 3ði
ñ $×.Ñ.°Õ5ði
ñ )¨4Õ/ði
ñ ×(Ñ(¨4Õ/ði
ñ  %×0Ñ0°4Õ7ði
ñ × Ñ  4Õ'ði
ñ Ñ+Õ,ði
ñ 
"ñi
r9   c                ód  € V P                   P                  '       d,   Vf(   Ve$   \        RV P                  P                   R24      hV P                   P                  '       d)   Vf%   V	f!   Vf   \        R4      hV P                  V4      pV P                   P                  '       dK   V P                  ! V3RVRVRVRVR	VR
VRVRV	RR/	VB pVP                  pVP                  pVP                  pM=V P                  ! V3RVRVRV/VB pVP                  pVP                  pVP                  pV P                  V4      pVe   VP                  ^ ,          pMVP                  ^ ,          pV P                   P                  f   V^8w  d   \        R4      hV P                   P                  f   RpEM"Veñ   WP                   P                  8g  P!                  VP"                  \$        P&                  4      p\$        P(                  ! VP                  R,          VP"                  \$        P&                  R7      pVV,          P+                  R4      pV P                   P                  '       d:   V^,          p\$        P,                  ! VVP                  R,          ^,
          R7      pM.Rp\.        P1                  V P                  P                   R24       V\$        P(                  ! VVP"                  R7      V3,          pRpV
e    V P3                  VV
VV P                   R7      p\5        VVVVR7      # )áœ  
decoder_position_ids (`torch.LongTensor` of shape `(batch_size, decoder_sequence_length)`, *optional*):
    Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the range `[0,
    config.decoder.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
    `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
Nú8Passing input embeddings is currently not supported for ú in encoder-decoder mode.ú°If no `decoder_input_ids` or `decoder_inputs_embeds` are passed, `input_ids` cannot be `None`. Please pass either `input_ids` or `decoder_input_ids` or `decoder_inputs_embeds`.rÒ   r°   rå  ræ  rç  rè  r®  ré  rJ  Fz=Cannot handle batch sizes > 1 if no padding token is defined.r•  )ÚmaxzŠ will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`r²  )ri  r  Úpooled_logitsrh   ©r  ri  rw   r›  rH   )rh   rÕ  ÚNotImplementedErrorrD   r[   r  rƒ  rm  rµ  rí  rî  rw   r›  rE  rW   r}  rœ   r{   r@   Úint32r›   ÚargmaxÚclampÚloggerÚwarning_oncer  r   )rC   r€  rÒ   r°   rå  ræ  rç  rè  r®  ré  r  r�   Úoutputsrµ  rw   r›  ri  r'  Úlast_non_pad_tokenÚnon_pad_maskÚtoken_indicesrZ  r  s   &&&&&&&&&&&,           r6   rS   Ú(T5GemmaForSequenceClassification.forwardw  s=  € ð2 �;‰;×)×)Ð)¨yÒ/@À]ÒE^Ü%ØJÈ4Ï>É>×KbÑKbÐJcÐc|Ð}óð ð
 �;‰;×)×)Ð)Ð/@Ò/HÐMbÒMjØÒ Ü ðUóð ð
 !%× 1Ñ 1°)Ó <Ðà�;‰;×)×)Ð)Ø*.¯*ª*Øñ+à-ð+ð *ð+ð #4ð	+ð
 (>ð+ð &:ð+ð !0ð+ð ,ð+ð '<ð+ð  ð+ð ñ+ˆGð !(× 9Ñ 9ÐØ#×9Ñ9ˆMØ ×3Ñ3‰Jà'+§z¢zØñ(à-ð(ð *ð(ð ,ð	(ð
 ñ(ˆGð !(× 9Ñ 9ÐØ#×1Ñ1ˆMØ ×+Ñ+ˆJà—‘Ð-Ó.ˆàÒ Ø"Ÿ™¨Õ+‰Jà&×,Ñ,¨QÕ/ˆJà�;‰;×#Ñ#Ò+°
¸a´ÜÐ\Ó]Ð]Ø�;‰;×#Ñ#Ò+Ø!#ÒØÒ"à%¯©×)AÑ)AÑA×EÑEÀfÇmÁmÔUZ×U`ÑU`ÓaˆLÜ!ŸLšL¨¯©¸Õ)<ÀVÇ]Á]ÔZ_×ZeÑZeÔfˆMØ"/°,Õ">×!FÑ!FÀrÓ!JÐà�{‰{×-×-Ð-Ø" aÕ'Ð"Ü%*§[¢[Ð1CÐIZ×I`ÑI`ÐacÕIdÐghÕIhÔ%iÐ"øà!#ÐÜ×ÑØ—>‘>×*Ñ*Ð+ð ,Zð Zôð
 œuŸ|š|¨J¸v¿}¹}ÔMÐOaÐaÕbˆàˆØÒØ×%Ñ%¨V¸FÐR_Ðhl×hsÑhsÐ%ÓtˆDä'ØØ Ø'Ø!ô	
ð 	
r9   ©rm  rS  rE  r;   ©
NNNNNNNNNNró  rb   s   @@r6   r=  r=  X  sC   ù‡ € ÷õ ò.1ò/ð Ø÷i
ñ i
ó ó ÷i
ð i
r9   r=  c                   óx   a a€ ] tR tRt oR	V3R lV 3R llltR tR t]]R
V3R lR ll4       4       t	Rt
VtV ;t# )ÚT5GemmaForTokenClassificationiå  c                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# r?  r@  )r4   r5   s   "€r6   r7   Ú*T5GemmaForTokenClassification.__annotate__ç  s   ø€ ÷ ñ ™}ð Á$ÈÅ+ñ r9   c                ó®  <€ Ve   W!n         \        SV `	  V4       VP                  V n        VP                   '       d   \	        V4      V n        M\        V4      V n        VP                  P                  pVP                   '       d   VP                  P                  p\        VRR4      p\        W0P                  V4      V n        V P                  4        R# )z‘
is_encoder_decoder (`Optional`, *optional*):
    Whether use encoder_decoder for token classification. When set to False, only encoder is used.
NrT  rC  rD  rF  s   &&&  €r6   r=   Ú&T5GemmaForTokenClassification.__init__ç  s¦   ø€ ð
 Ò)Ø(:Ô%Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒà×$×$Ð$Ü% fÓ-ˆD�Jä,¨VÓ4ˆDŒJà—n‘n×0Ñ0ˆØ×$×$Ð$Ø Ÿ.™.×4Ñ4ˆKä$ VÐ-FÈÓLÐÜ.¨{¿O¹OÐM_Ó`ˆŒ
à�‰Ör9   c                ó6   € V P                   P                  4       # r;   rJ  rX   s   &r6   rÚ  Ú2T5GemmaForTokenClassification.get_input_embeddingsÿ  rL  r9   c                ó<   € V P                   P                  V4       R # r;   rN  rO  s   &&r6   rß  Ú2T5GemmaForTokenClassification.set_input_embeddings  rQ  r9   c                ó²  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[ P                  R,          R
S[ P                  R,          RS[ P                  R,          RS[S[,          RS[/# rS  )r@   r9  r“   r   r:  r   r    r   )r4   r5   s   "€r6   r7   rl    sû   ø€ ÷ N
ñ N
á×#Ñ# dÕ*ðN
ñ Ÿ™ tÕ+ðN
ñ ×&Ñ&¨Õ-ð	N
ñ
 !×+Ñ+¨dÕ2ðN
ñ !&§¡¨tÕ 3ðN
ñ $×.Ñ.°Õ5ðN
ñ )¨4Õ/ðN
ñ ×(Ñ(¨4Õ/ðN
ñ  %×0Ñ0°4Õ7ðN
ñ × Ñ  4Õ'ðN
ñ Ñ+Õ,ðN
ñ 
ñN
r9   c                óâ  € V P                   P                  '       d,   Vf(   Ve$   \        RV P                  P                   R24      hV P                   P                  '       d)   Vf%   V	f!   Vf   \        R4      hV P                  V4      pV P                   P                  '       dK   V P                  ! V3RVRVRVRVR	VR
VRVRV	RR/	VB pVP                  pVP                  pVP                  pM=V P                  ! V3RVRVRV/VB pVP                  pVP                  pVP                  pV P                  V4      pRpV
e   V P                  VW P                   4      p\        VVVVR7      # )rU  NrV  rW  rX  rÒ   r°   rå  ræ  rç  rè  r®  ré  rJ  Fr[  )rh   rÕ  r\  rD   r[   r  rƒ  rm  rµ  rí  rî  rw   r›  rE  r  r   )rC   r€  rÒ   r°   rå  ræ  rç  rè  r®  ré  r  r�   rb  rµ  rw   r›  ri  r  s   &&&&&&&&&&&,      r6   rS   Ú%T5GemmaForTokenClassification.forward  sØ  € ð4 �;‰;×)×)Ð)¨yÒ/@À]ÒE^Ü%ØJÈ4Ï>É>×KbÑKbÐJcÐc|Ð}óð ð �;‰;×)×)Ð)Ð/@Ò/HÐMbÒMjØÒ Ü ðUóð ð
 !%× 1Ñ 1°)Ó <Ðà�;‰;×)×)Ð)Ø*.¯*ª*Øñ+à-ð+ð *ð+ð #4ð	+ð
 (>ð+ð &:ð+ð !0ð+ð ,ð+ð '<ð+ð  ð+ð ñ+ˆGð !(× 9Ñ 9ÐØ#×9Ñ9ˆMØ ×3Ñ3‰Jà'+§z¢zØñ(à-ð(ð *ð(ð ,ð	(ð
 ñ(ˆGð !(× 9Ñ 9ÐØ#×1Ñ1ˆMØ ×+Ñ+ˆJà—‘Ð-Ó.ˆàˆØÒØ×%Ñ% f¨f·k±kÓBˆDä$ØØØ'Ø!ô	
ð 	
r9   rg  r;   rh  ró  rb   s   @@r6   rj  rj  å  sC   ù‡ € ÷õ ò01ò/ð Ø÷N
ñ N
ó ó ÷N
ð N
r9   rj  )rz  )r  NN)r  rÐ  rõ  rl  r=  rj  )Zr0  Úcollections.abcr   r@   Útorch.nnr>   Ú r   rs  Úactivationsr   Úcache_utilsr   r   r   r	   Ú
generationr
   r   r   Úintegrationsr   r   Úmasking_utilsr   r   r   r   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr    r!   r"   r#   Úutils.deprecationr$   Úutils.genericr%   r&   Úutils.output_capturingr'   r(   Úconfiguration_t5gemmar)   r*   Ú
get_loggerr[   r`  rÕ   r,   rd   rz   r»   rÄ   rÌ   rå   rç   r  r'  rA  rQ  r`  rl  r˜  rš  rÃ  rÐ  rõ  r  r=  rj  Ú__all__r<  r9   r6   Ú<module>r‹     s8  ðó* Ý $ã Ý å &Ý !ß PÓ Pß KÑ Kß L÷ó õ CÝ 9÷÷ ÷ Lß FÝ &ß RÓ RÝ 0ß Gß Eß Eð 
×	Ò	˜HÓ	%€ô=�R—Y‘Yô =ô(�—‘ô ô&3<˜RŸY™Yô 3<òl(ñ Ð-Ó.óó /ðõ2	U÷%ñD Ð)Ó*ôF)˜2Ÿ9™9ó F)ó +ðF)ñR Ð)Ó*ôR)˜BŸI™Ió R)ó +ðR)ôj1Ð4ô 1ôhFÐ4ô FôR §	¡	ô ô	�B—I‘Iô 	ð ô9!˜_ó 9!ó ð9!õxô"P
Ð+ô P
ôfk
Ð+ô k
ð\ ôJ
Ð)ó J
ó ðJ
ðZ ô!Ð0ó !ó ð!ôHs:Ð&<¸oô s:ðl ôI
Ð'=ó I
ó ðI
ðX ôo
Ð$:ó o
ó ðo
òd‚r9   