+
    œ°³j¿L ã                   ó@  € R t ^ RIt^ RIt^ RIHt ^ RIHt ^ RIt^ RIHt ^ RI	H
t
 ^RIHt ^RIHt ^RIHtHtHt ^R	IHt ^R
IHtHt ^RIHt ^RIHtHtHtHt ^RI H!t!H"t" ^RI#H$t$ ^RI%H&t&H't'H(t(H)t)H*t*H+t+ ^RI,H-t- ]+P\                  ! ]/4      t0RIR R llt1RJR R llt2R R lt3RJR R llt4R R lt5R R lt6R R lt7R R  lt8R! R" lt9R# R$ lt: ! R% R&]Pv                  4      t< ! R' R(]Pv                  4      t= ! R) R*]Pv                  4      t> ! R+ R,]Pv                  4      t? ! R- R.]Pv                  4      t@ ! R/ R0]Pv                  4      tA ! R1 R2]Pv                  4      tB ! R3 R4]Pv                  4      tC ! R5 R6]Pv                  4      tD ! R7 R8]Pv                  4      tE ! R9 R:]Pv                  4      tF ! R; R<]4      tG]) ! R= R>]"4      4       tH ! R? R@]H4      tI]) ! RA RB]H4      4       tJ])! RCRD7       ! RE RF]H]4      4       tK]) ! RG RH]H4      4       tL. RKOtMR# )LzPyTorch LongT5 model.N)ÚCallable©ÚAny)Únn)ÚCrossEntropyLoss)Úinitialization)ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚ)BaseModelOutputWithPastAndCrossAttentionsÚSeq2SeqLMOutputÚSeq2SeqModelOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚDUMMY_INPUTSÚ
DUMMY_MASKÚTransformersKwargsÚauto_docstringÚis_torchdynamo_compilingÚlogging©ÚLongT5Configc                ó<  € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R,          R\        R	\        P                  R,          R
\
        \        ,          /	# )é   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNÚscalingÚdropoutÚposition_biasÚkwargs)r   ÚModuleÚtorchÚTensorÚfloatr   r   )Úformats   "Úk/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/longt5/modeling_longt5.pyÚ__annotate__r0   6   sŽ   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô �T�\ð%ô ð%ô —<‘< $Õ&ð%ô Ô'Õ(ñ%ó    c                 óÎ  € Vf   VP                  R4      R,          p\        P                  ! WP                  ^^4      4      V,          p	Ve	   W—,           p	Ve	   W”,           p	\        P
                  P                  V	RR7      p	\        P
                  P                  W–V P                  R7      p	\        P                  ! W“4      p
V
P                  ^^4      P                  4       p
W©3# )N©Údim©ÚpÚtrainingéÿÿÿÿç      à¿)
Úsizer+   ÚmatmulÚ	transposer   Ú
functionalÚsoftmaxr'   r7   Ú
contiguous)r!   r"   r#   r$   r%   r&   r'   r(   r)   Úattn_weightsÚattn_outputs   &&&&&&&&,  r/   Úeager_attention_forwardrB   6   s¿   € ð ‚Ø—*‘*˜R“. DÕ(ˆô —<’< §}¡}°Q¸Ó':Ó;¸gÕE€LàÒ Ø#Õ3ˆàÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2Ð(Ó>€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€Lä—,’,˜|Ó3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r1   c          
      ó|   € V ^8„  d   QhR\         P                  R\        R\        R\        R\         P                  /# )r    ÚxÚ	block_lenr4   Ú	pad_valueÚreturn©r+   r,   Úint)r.   s   "r/   r0   r0   Y   s:   € ÷ ñ œŸ™ð ´ð ¼3ð Ì3ð ÔW\×WcÑWcñ r1   c                óª  € V P                   V,          ) V,          p\        V P                   4      '       gK   \        V P                   4      pWR;;,          V,          uu&   \        P                  ! WPP
                  R7      # R.V P                  ,          p^ V3Wb&   \        VRRR1,          R4      p\        P                  P                  WRVR7      p V # )zHPad a tensor so that a sequence length will be a multiple of `block_len`©ÚdtypeNÚconstant©ÚpadÚmoder$   ©é    rR   r8   © )ÚshapeÚallÚlistr+   ÚzerosrL   ÚndimÚsumr   r=   rO   )rD   rE   r4   rF   Úpad_lenÚ	new_shaperO   s   &&&&   r/   Ú_pad_to_multipler\   Y   sž   € à�w‰w�s�|ˆm˜iÕ'€Gäˆq�w‰w�<Š<Ü˜Ÿ™“Mˆ	Ø�˜'Õ!‹Ü�{Š{˜9¯G©GÔ4Ð4àˆ(�Q—V‘VÕ
€CØ�7ˆ|€C�HÜ
ˆc‘$�B�$�i˜Ó
€CÜ
�‰×Ñ˜!¨:¸YÐÓG€AØ€Hr1   c                óp   € V ^8„  d   QhR\         P                  R\        R\        R\         P                  /# )r    rD   rE   r4   rG   rH   )r.   s   "r/   r0   r0   i   s0   € ÷ #ñ #œ%Ÿ,™,ð #´3ð #¼Sð #ÄUÇ\Á\ñ #r1   c                ó|  € V P                   V,          V,          ^ 8w  d   \        WV^ R7      p V P                   V,          V,          pV P                   RV W13,           V P                   V^,           R ,           p^ V9   d-   \        P                  ! W@P                  V P
                  R7      # V P                  V4      # )z½Split an input tensor into blocks of a given `block_len` along the given `dim`. If the dimension length
is not a multiple of `block_len`, it will be padded first with selected `pad_value`.
)rF   N©rL   Údevice)rT   r\   r+   ÚemptyrL   r`   Úreshape)rD   rE   r4   Ú
num_blocksÚoutput_shapes   &&&  r/   Ú_split_into_blocksre   i   s”   € ð
 	‡w�wˆs…|�iÕ 1Ô$Ü˜Q¨3¸!Ô<ˆØ—‘˜• Õ*€JØ—7‘7˜4˜C�= JÐ#:Õ:¸Q¿W¹WÀcÈAÅgÀ[Ð=QÕQ€LàˆLÔÜ�{Š{˜<¯w©w¸q¿x¹xÔHÐHØ�9‰9�\Ó"Ð"r1   c          
      ó|   € V ^8„  d   QhR\         P                  R\        R\        R\        R\         P                  /# )r    rD   Ú	block_dimÚsequence_dimrF   rG   rH   )r.   s   "r/   r0   r0   x   s;   € ÷ 4ñ 4œUŸ\™\ð 4´cð 4Ìð 4ÔY\ð 4Ôej×eqÑeqñ 4r1   c                óÆ  € V P                   V,          pR.V P                  ,          pRWQ&   \        VRRR1,          R4      p\        P                  P                  WRVR7      p . p\        ^4       FW  p\        ^ R4      .V P                  ,          p\        WwV,           4      W�&   \        V4      pVP                  W,          4       KY  	  \        P                  ! WbR7      # )	z•Concatenate three consecutive blocks for each input block for local attention.

For more information, see: https://huggingface.co/papers/2112.07916.
NrM   rN   r3   rQ   )é   rj   r8   rS   )rT   rX   rY   r   r=   rO   ÚrangeÚsliceÚtupleÚappendr+   Úcat)	rD   rg   rh   rF   rc   rO   Úblocks_listÚiÚindicess	   &&&&     r/   Ú_concatenate_3_blocksrs   x   s¿   € ð
 —‘˜Õ#€Jàˆ(�Q—V‘VÕ
€CØ€C�NÜ
ˆc‘$�B�$�i˜Ó
€Cä
�‰×Ñ˜!¨:¸YÐÓG€Aà&(€KÜ�1ŽXˆô ˜˜D“>Ð" Q§V¡VÕ+ˆÜ" 1¨*¥nÓ5ˆÑÜ˜“.ˆØ×Ñ˜1�:Ö&ñ ô �9Š9�[Ô3Ð3r1   c                óD   € V ^8„  d   QhR\         R\        P                  /# )r    rE   rG   )rI   r+   r,   )r.   s   "r/   r0   r0   ‘   s   € ÷ !ñ !´#ð !¼%¿,¹,ñ !r1   c                ó¸   € \         P                  ! ^V ,          \         P                  R7      pWV )  pVP                  ^ 4      VP                  ^4      ,
          pV# )z:Makes 3-blocked relative position ids for local attention.rK   )r+   ÚarangeÚint32Ú	unsqueeze)rE   Úposition_idsÚcenter_position_idsÚrelative_position_idss   &   r/   Ú"_make_3block_relative_position_idsr|   ‘   sP   € ä—<’<  I¥´U·[±[ÔA€LØ&°)°Ð<Ðà(×2Ñ2°1Ó5Ð8K×8UÑ8UÐVWÓ8XÕXÐØ Ð r1   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r    Úlocal_attention_maskrE   rG   rH   )r.   s   "r/   r0   r0   š   s1   € ÷ Bñ B´U·\±\ð BÌcð BÔV[×VbÑVbñ Br1   c                ó¾   € \        V4      p\        P                  ! V4      V8  pVR,          pVP                  V P                  4      p\        P
                  ! W4      # )znMask local attention mask to enforce that tokens are not allowed to attend tokens farther than ``local_radius.)NNºNNNr€   )r|   r+   ÚabsÚtor`   Úlogical_and)r~   rE   r{   Úlocality_masks   &&  r/   Ú_mask_local_attention_maskr…   š   sU   € ä>¸yÓIÐÜ—I’IÐ3Ó4°yÑ@€MØ!Ð"2Õ3€MØ!×$Ñ$Ð%9×%@Ñ%@ÓA€MÜ×ÒÐ1ÓAÐAr1   c                ó„   € V ^8„  d   QhR\         P                  R\        R\         P                  R\         P                  /# )r    r%   rE   r`   rG   )r+   r,   rI   r`   )r.   s   "r/   r0   r0   £   s:   € ÷ 8ñ 8¬e¯l©lð 8Äsð 8ÔTY×T`ÑT`ð 8Ôej×eqÑeqñ 8r1   c                óþ   € \        W^R7      p\        V^^R7      pVP                  R4      pVP                  R4      p\        P                  ! W44      p\        WQ4      pVP                  ^4      P                  V4      # )z;Prepare attention mask to be applied for a local attention.r3   ©rg   rh   r8   éþÿÿÿ)re   rs   rx   r+   rƒ   r…   r‚   )r%   rE   r`   Ú_blocked_attention_maskÚ_3blocked_attention_maskr~   s   &&&   r/   Ú_get_local_attention_maskrŒ   £   s€   € ô 1°ÐPQÔRÐä4Ð5LÐXYÐhiÔjÐà5×?Ñ?ÀÓCÐØ7×AÑAÀ"ÓEÐä ×,Ò,Ð-DÓ_ÐÜ5Ð6JÓVÐà×)Ñ)¨!Ó,×/Ñ/°Ó7Ð7r1   c                óš   € V ^8„  d   QhR\         P                  R\        R\        \         P                  \         P                  3,          /# ©r    r%   Úglobal_block_sizerG   )r+   r,   rI   rm   )r.   s   "r/   r0   r0   ³   s?   € ÷ .Pñ .PÜ—L‘Lð.PÜ58ð.Pä
Œ5�<‰<œŸ™Ð%Õ&ñ.Pr1   c                óÌ  aa€ V P                   R,          w  poR VV3R llp\        P                  ! W P                  R7      S,          p\        P                  ! V^R7      V,
          p\        P
                  ! V R8g  RR
4      P                  V P                  4      p\        P                  ! WT,           R,
          4      P                  V P                  4      p\        P                  ! RRVP                  VP                  R7      p\        P
                  ! Wg8„  Wg4      pW`,          V ^,
          ,           pV! V4      pSS,          pV^ 8”  dD   \        P                  ! VRR	7      P                  P                  V^4      P                  ^ ^4      p	M.\        P                  ! V^ VP                  VP                  R7      p	\        P                  ! \        P                  ! W(4      RR	7      ^,
          p
V
P!                  V P                  4      p
\        P
                  ! W©8*  ^^ 4      p
VP                  \        P"                  4      V
P                  \        P"                  4      3# )aÿ  Obtain the "fixed block" global id corresponding to each input token.

This implementation is a simplified version of the original Flaxformr implementation adopted from:
https://github.com/google/flaxformer/blob/main/flaxformer/architectures/longt5/long_attention.py.

In our scenario, as we use this strategy only for a decoder, orphan tokens, i.e. those tokens which do not make for
the whole fixed block, are assigned to the preceding block.

Padding tokens from the original sequence are represented by -1.
ºNr    Nc                óX   € V ^8„  d   QhR\         P                  R\         P                  /# )r    Ú	block_idsrG   ©r+   r,   )r.   s   "r/   r0   Ú2_make_global_fixed_block_ids.<locals>.__annotate__Â   s"   € ÷ ñ ¬¯©ð ¼¿¹ñ r1   c                 óp  <€ \         P                  ! S4      S,          S^,
          8H  pVP                  V P                  4      p\         P                  ! W^ 8¬  4      pVP                  R4      P                  R4      P                  V P                  4      ^,
          p\         P                  ! W8  W4      p V # )rj   r8   )
r+   rv   r‚   r`   rƒ   rY   rx   ÚtyperL   Úwhere)r“   Ú
block_endsÚtrue_block_endsÚfull_blocksr�   Úseq_lens   &   €€r/   Úhandle_orphan_tokensÚ:_make_global_fixed_block_ids.<locals>.handle_orphan_tokensÂ   s”   ø€ Ü—l’l 7Ó+Ð.?Õ?ÐDUÐXYÕDYÑYˆ
Ø—]‘] 9×#3Ñ#3Ó4ˆ
Ü×+Ò+¨JÀQ¹ÓGˆØ%×)Ñ)¨"Ó-×7Ñ7¸Ó;×@Ñ@ÀÇÁÓQÐTUÕUˆÜ—K’K 	Ñ 7¸ÓPˆ	ØÐr1   ©r`   )Úaxisç        ç      ð?r_   r3   g     @�ÀrS   r8   )rT   r+   Ú	ones_liker`   Úcumsumr˜   r—   rL   ÚfloorÚfullÚmaxÚvaluesÚrepeatr<   rW   Úonesr‚   rI   )r%   r�   Ú
batch_sizer�   Úfixed_block_maskÚmaskÚglobal_block_idsÚ_global_block_ids_lower_boundÚnum_globalsÚ_sequence_block_ids_maxÚglobal_segment_idsrœ   s   &f         @r/   Ú_make_global_fixed_block_idsr³   ³   sï  ù€ ð )×.Ñ.¨rÕ2Ñ€J�÷ñ ô —’ ~×>SÑ>SÔTÐWhÕhÐÜ—|’|Ð$4¸1Ô=Ð@PÕPÐÜ�;Š;�~¨Ñ,¨c°7Ó;×@Ñ@À×AUÑAUÓV€DÜ—{’{ 4Õ#:¸SÕ#@ÓA×FÑFÀ~×G[ÑG[Ó\ÐÜ$)§J¢J¨r°2Ð=M×=SÑ=SÐ\l×\sÑ\sÔ$tÐ!Ü—{’{ØÑ8Ð:JóÐð )Õ9¸nÈqÕ>PÕQÐá+Ð,<Ó=ÐØÐ.Õ.€Kà�Q„Ü"'§)¢)Ð,<À"Ô"E×"LÑ"L×"SÑ"SÐT_ÐabÓ"c×"mÑ"mÐnoÐqrÓ"sÑä"'§+¢+Ø˜Ð!1×!7Ñ!7Ð@P×@WÑ@Wô#
Ðô Ÿš¤e§j¢j°Ó&IÈrÔRÐUVÕVÐØ+×.Ñ.¨~×/DÑ/DÓEÐÜŸšÐ%7Ñ%RÐTUÐWXÓYÐØ× Ñ ¤§¡Ó+Ð-?×-DÑ-DÄUÇYÁYÓ-OÐOÐOr1   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# rŽ   rH   )r.   s   "r/   r0   r0   ä   s-   € ÷ 4ñ 4´U·\±\ð 4ÔVYð 4Ô^c×^jÑ^jñ 4r1   c                óà   € \        W4      w  r#VP                  R,          p\        P                  ! WBP                  R7      pWRR,          ,
          pVP                  \        P                  4      # )zBCreate the relative position tensor for local -> global attention.rŸ   r8   ©.N)r³   rT   r+   rv   r`   r—   Úint64)r%   r�   r“   r²   Úglobal_seq_lenÚglobal_positionsÚside_relative_positions   &&     r/   Ú _make_side_relative_position_idsr»   ä   sW   € ä$@ÀÓ$cÑ!€IØ'×-Ñ-¨bÕ1€NÜ—|’| N×;KÑ;KÔLÐØ-¸)Õ0DÕDÐØ!×&Ñ&¤u§{¡{Ó3Ð3r1   c                ó„   € V ^8„  d   QhR\         P                  R\         P                  R\        R\         P                  /# )r    Úhidden_statesr“   r¸   rG   rH   )r.   s   "r/   r0   r0   í   s>   € ÷ 	jñ 	jÜ—<‘<ð	jÜ,1¯L©Lð	jÜJMð	jä
‡\�\ñ	jr1   c           	     ó€  € VP                  V^ 8¬  \        P                  ! W!P                  VP                  R7      4      p\
        P                  P                  VP                  \        P                  4      V^,           4      RRRR13,          p\        P                  ! RWP                  V P                  4      4      # )zFCompute individual block aggregates by summing over individual blocks.r_   r€   Nz...nd,...ng->...gdr8   )r˜   r+   ÚtensorrL   r`   r   r=   Úone_hotr—   r·   Úeinsum)r½   r“   r¸   Úone_hot_block_idss   &&& r/   Ú_create_global_aggregatesrÃ   í   s–   € ð
 —‘Ø�Q‰œŸš ^¿?¹?ÐS\×ScÑScÔdó€Iô Ÿ™×-Ñ-¨i¯n©n¼U¿[¹[Ó.IÈ>Ð\]ÕK]Ó^Ð_`ÐbcÐehÐfhÐehÐ_hÕiÐÜ�<Š<Ð,¨m×=SÑ=SÐTa×TgÑTgÓ=hÓiÐir1   c                   ó<   a a€ ] tR t^út oRV 3R lltR tRtVtV ;t# )ÚLongT5LayerNormc                óŽ   <€ \         SV `  4        \        P                  ! \        P
                  ! V4      4      V n        W n        R# )zW
Construct a layernorm module in the LongT5 style. No bias and no subtraction of mean.
N)ÚsuperÚ__init__r   Ú	Parameterr+   rª   ÚweightÚvariance_epsilon)ÚselfÚhidden_sizeÚepsÚ	__class__s   &&&€r/   rÈ   ÚLongT5LayerNorm.__init__û   s/   ø€ ô 	‰ÑÔÜ—l’l¤5§:¢:¨kÓ#:Ó;ˆŒØ #Ör1   c                ó¼  € VP                  \        P                  4      P                  ^4      P	                  RRR7      pV\        P
                  ! W P                  ,           4      ,          pV P                  P                  \        P                  \        P                  39   d&   VP                  V P                  P                  4      pV P                  V,          # )r    T)Úkeepdimr8   )r‚   r+   Úfloat32ÚpowÚmeanÚrsqrtrË   rÊ   rL   Úfloat16Úbfloat16)rÌ   r½   Úvariances   && r/   ÚforwardÚLongT5LayerNorm.forward  s™   € ð !×#Ñ#¤E§M¡MÓ2×6Ñ6°qÓ9×>Ñ>¸rÈ4Ð>ÓPˆØ%¬¯ª°H×?TÑ?TÕ4TÓ(UÕUˆð �;‰;×Ñ¤§¡´·±Ð ?Ô?Ø)×,Ñ,¨T¯[©[×->Ñ->Ó?ˆMà�{‰{˜]Õ*Ð*r1   )rË   rÊ   )g�íµ ÷Æ°>©	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__rÈ   rÚ   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©rÏ   Ú__classdict__s   @@r/   rÅ   rÅ   ú   s   ù‡ € ÷$÷+ò +r1   rÅ   c                   óD   a a€ ] tR tRt oV3R lV 3R lltR tRtVtV ;t# )ÚLongT5DenseActDensei  c                ó    <€ V ^8„  d   QhRS[ /# ©r    Úconfigr   )r.   rå   s   "€r/   r0   Ú LongT5DenseActDense.__annotate__  s   ø€ ÷ /ñ /™|ñ /r1   c                ón  <€ \         SV `  4        \        P                  ! VP                  VP
                  R R7      V n        \        P                  ! VP
                  VP                  R R7      V n        \        P                  ! VP                  4      V n
        \        VP                  ,          V n        R# ©F©ÚbiasN)rÇ   rÈ   r   ÚLinearÚd_modelÚd_ffÚwiÚwoÚDropoutÚdropout_rater'   r   Údense_act_fnÚact©rÌ   rê   rÏ   s   &&€r/   rÈ   ÚLongT5DenseActDense.__init__  sn   ø€ Ü‰ÑÔÜ—)’)˜FŸN™N¨F¯K©K¸eÔDˆŒÜ—)’)˜FŸK™K¨¯©¸eÔDˆŒÜ—z’z &×"5Ñ"5Ó6ˆŒÜ˜&×-Ñ-Õ.ˆŽr1   c                ó  € V P                  V4      pV P                  V4      pV P                  V4      p\        V P                  P
                  \        P                  4      '       d’   VP                  V P                  P
                  P                  8w  dc   V P                  P
                  P                  \        P                  8w  d0   VP                  V P                  P
                  P                  4      pV P	                  V4      pV# ©N)ró   rø   r'   Ú
isinstancerô   rÊ   r+   r,   rL   Úint8r‚   )rÌ   r½   s   &&r/   rÚ   ÚLongT5DenseActDense.forward  s­   € ØŸ™ Ó.ˆØŸ™ Ó/ˆØŸ™ ]Ó3ˆä�t—w‘w—~‘~¤u§|¡|×4Ò4Ø×#Ñ# t§w¡w§~¡~×';Ñ';Ô;Ø—‘—‘×$Ñ$¬¯
©
Ô2à)×,Ñ,¨T¯W©W¯^©^×-AÑ-AÓBˆMØŸ™ Ó.ˆØÐr1   )rø   r'   ró   rô   rÜ   rä   s   @@r/   rç   rç     s   ù‡ € ÷/ó /÷ò r1   rç   c                   óD   a a€ ] tR tRt oV3R lV 3R lltR tRtVtV ;t# )ÚLongT5DenseGatedActDensei*  c                ó    <€ V ^8„  d   QhRS[ /# ré   r   )r.   rå   s   "€r/   r0   Ú%LongT5DenseGatedActDense.__annotate__+  s   ø€ ÷ /ñ /™|ñ /r1   c                óÒ  <€ \         SV `  4        \        P                  ! VP                  VP
                  R R7      V n        \        P                  ! VP                  VP
                  R R7      V n        \        P                  ! VP
                  VP                  R R7      V n        \        P                  ! VP                  4      V n        \        VP                  ,          V n        R# rí   )rÇ   rÈ   r   rð   rñ   rò   Úwi_0Úwi_1rô   rõ   rö   r'   r   r÷   rø   rù   s   &&€r/   rÈ   Ú!LongT5DenseGatedActDense.__init__+  sŠ   ø€ Ü‰ÑÔÜ—I’I˜fŸn™n¨f¯k©kÀÔFˆŒ	Ü—I’I˜fŸn™n¨f¯k©kÀÔFˆŒ	Ü—)’)˜FŸK™K¨¯©¸eÔDˆŒÜ—z’z &×"5Ñ"5Ó6ˆŒÜ˜&×-Ñ-Õ.ˆŽr1   c                ó¼   € V P                  V P                  V4      4      pV P                  V4      pW#,          pV P                  V4      pV P	                  V4      pV# rü   )rø   r  r  r'   rô   )rÌ   r½   Úhidden_geluÚhidden_linears   &&  r/   rÚ   Ú LongT5DenseGatedActDense.forward3  sQ   € Ø—h‘h˜tŸy™y¨Ó7Ó8ˆØŸ	™	 -Ó0ˆØ#Õ3ˆØŸ™ ]Ó3ˆØŸ™ Ó.ˆØÐr1   )rø   r'   r  r  rô   rÜ   rä   s   @@r/   r  r  *  s   ù‡ € ÷/ó /÷ò r1   r  c                   óD   a a€ ] tR tRt oV3R lV 3R lltR tRtVtV ;t# )ÚLongT5LayerFFi=  c                ó    <€ V ^8„  d   QhRS[ /# ré   r   )r.   rå   s   "€r/   r0   ÚLongT5LayerFF.__annotate__>  s   ø€ ÷ 7ñ 7™|ñ 7r1   c                ó   <€ \         SV `  4        VP                  '       d   \        V4      V n        M\        V4      V n        \        VP                  VP                  R 7      V n	        \        P                  ! VP                  4      V n        R# )©rÎ   N)rÇ   rÈ   Úis_gated_actr  ÚDenseReluDenserç   rÅ   rñ   Úlayer_norm_epsilonÚ
layer_normr   rõ   rö   r'   rù   s   &&€r/   rÈ   ÚLongT5LayerFF.__init__>  sb   ø€ Ü‰ÑÔØ××ÐÜ":¸6Ó"BˆDÕä"5°fÓ"=ˆDÔä)¨&¯.©.¸f×>WÑ>WÔXˆŒÜ—z’z &×"5Ñ"5Ó6ˆŽr1   c                óx   € V P                  V4      pV P                  V4      pWP                  V4      ,           pV# rü   )r  r  r'   )rÌ   r½   Úforwarded_statess   && r/   rÚ   ÚLongT5LayerFF.forwardH  s;   € ØŸ?™?¨=Ó9ÐØ×.Ñ.Ð/?Ó@ÐØ%¯©Ð5EÓ(FÕFˆØÐr1   )r  r'   r  rÜ   rä   s   @@r/   r  r  =  s   ù‡ € ÷7ó 7÷ò r1   r  c                   óv   a a€ ] tR tRt oR	V3R lV 3R lllt]R
R l4       tRR ltRV3R lR lltRt	Vt
V ;t# )ÚLongT5AttentioniP  c                ó:   <€ V ^8„  d   QhRS[ RS[R,          RS[/# )r    rê   Ú	layer_idxNÚ	is_causal)r   rI   Úbool)r.   rå   s   "€r/   r0   ÚLongT5Attention.__annotate__Q  s-   ø€ ÷ %,ñ %,áð%,ñ ˜•:ð	%,ñ
 ñ%,r1   c                ó<  <€ \         SV `  4        Wn        VP                  V n        W@n        W n        VP                  V n        VP                  V n        VP                  V n        VP                  V n
        VP                  V n        VP                  V n        V P                  V P                  ,          V n        R V n        W0n        Vf@   V P                  '       d.   \$        P'                  RV P(                  P*                   R24       \,        P.                  ! V P                  V P                  RR7      V n        \,        P.                  ! V P                  V P                  RR7      V n        \,        P.                  ! V P                  V P                  RR7      V n        \,        P.                  ! V P                  V P                  RR7      V n        V P
                  '       d1   \,        P8                  ! V P                  V P                  4      V n        RV n        R# )r¢   NzInstantiating a decoder z³ without passing `layer_idx` is not recommended and will to errors during the forward call, if caching is used. Please make sure to provide a `layer_idx` when creating this class.Frî   )rÇ   rÈ   rê   Ú
is_decoderr  Úhas_relative_attention_biasÚrelative_attention_num_bucketsÚrelative_attention_max_distancerñ   Úd_kvÚkey_value_proj_dimÚ	num_headsÚn_headsrö   r'   Ú	inner_dimr&   r  ÚloggerÚwarning_oncerÏ   rÝ   r   rð   ÚqÚkÚvÚoÚ	EmbeddingÚrelative_attention_biasÚgradient_checkpointing)rÌ   rê   r#  r  r  rÏ   s   &&&&&€r/   rÈ   ÚLongT5Attention.__init__Q  s{  ø€ ô 	‰ÑÔØŒØ ×+Ñ+ˆŒØ"ŒØ+FÔ(Ø.4×.SÑ.SˆÔ+Ø/5×/UÑ/UˆÔ,Ø—~‘~ˆŒØ"(§+¡+ˆÔØ×'Ñ'ˆŒØ×*Ñ*ˆŒØŸ™¨×(?Ñ(?Õ?ˆŒàˆŒØ"ŒØÒ §§ Ü×ÑØ*¨4¯>©>×+BÑ+BÐ*Cð D,ð ,ôô —’˜4Ÿ<™<¨¯©¸eÔDˆŒÜ—’˜4Ÿ<™<¨¯©¸eÔDˆŒÜ—’˜4Ÿ<™<¨¯©¸eÔDˆŒÜ—’˜4Ÿ>™>¨4¯<©<¸eÔDˆŒà×+×+Ð+Ü+-¯<ª<¸×8[Ñ8[Ð]a×]iÑ]iÓ+jˆDÔ(à&+ˆÖ#r1   c                óÄ  € ^ pV'       dP   V^,          pW@^ 8„  P                  \        P                  4      V,          ,          p\        P                  ! V 4      p M,\        P                  ! V \        P
                  ! V 4      4      ) p V^,          pW8  pV\        P                  ! V P                  4       V,          4      \        P                  ! W5,          4      ,          W%,
          ,          P                  \        P                  4      ,           p\        P                  ! V\        P                  ! Wr^,
          4      4      pV\        P                  ! W`V4      ,          pV# ©aR  
Adapted from Mesh Tensorflow:
https://github.com/tensorflow/mesh/blob/0cb87fe07da627bf0b7e60475d59f95ed6b5be3d/mesh_tensorflow/transformer/transformer_layers.py#L593

Translate relative position to a bucket number for relative attention. The relative position is defined as
memory_position - query_position, i.e. the distance in tokens from the attending position to the attended-to
position. If bidirectional=False, then positive relative positions are invalid. We use smaller buckets for
small absolute relative_position and larger buckets for larger absolute relative_positions. All relative
positions >=max_distance map to the same bucket. All relative positions <=-max_distance map to the same bucket.
This should allow for more graceful generalization to longer sequences than the model has been trained on

Args:
    relative_position: an int32 Tensor
    bidirectional: a boolean - whether the attention is bidirectional
    num_buckets: an integer
    max_distance: an integer

Returns:
    a Tensor with the same shape as relative_position, containing int32 values in the range [0, num_buckets)
©r‚   r+   Úlongr�   ÚminÚ
zeros_likeÚlogr-   ÚmathÚ	full_liker˜   ©Úrelative_positionÚbidirectionalÚnum_bucketsÚmax_distanceÚrelative_bucketsÚ	max_exactÚis_smallÚrelative_position_if_larges   &&&&    r/   Ú_relative_position_bucketÚ)LongT5Attention._relative_position_bucketx  s  € ð, ÐßØ˜AÕˆKØ°QÑ!6× :Ñ :¼5¿:¹:Ó FÈÕ TÕTÐÜ %§	¢	Ð*;Ó <Ñä!&§¢Ð+<¼e×>NÒ>NÐO`Ó>aÓ!bÐ bÐð   1Õ$ˆ	Ø$Ñ0ˆð &/Ü�IŠIÐ'×-Ñ-Ó/°)Õ;Ó<Ü�hŠh�|Õ/Ó0õ1àÕ&õ(÷ ‰"ŒU�Z‰Z‹.õ	&Ð"ô
 &+§Y¢YØ&¬¯ªÐ8RÐbcÕTcÓ(dó&
Ð"ð 	œEŸKšK¨ÐE_Ó`Õ`ÐØÐr1   c                óú  € Vf!   V P                   P                  P                  p\        P                  ! V\        P
                  VR7      R,          V,           p\        P                  ! V\        P
                  VR7      R,          pWe,
          pV P                  VV P                  '       * V P                  V P                  R7      pV P                  V4      p	V	P                  . RO4      P                  ^ 4      p	V	# )ú%Compute binned relative position biasr_   ©r@  rA  rB  ©r€   N©Nr€   ©r    rR   rj   )r2  rÊ   r`   r+   rv   r8  rG  r"  r$  r%  Úpermuterx   )
rÌ   Úquery_lengthÚ
key_lengthr`   Úpast_seen_tokensÚcontext_positionÚmemory_positionr?  Úrelative_position_bucketr¨   s
   &&&&&     r/   Úcompute_biasÚLongT5Attention.compute_bias¨  sÍ   € àŠ>Ø×1Ñ1×8Ñ8×?Ñ?ˆFÜ Ÿ<š<¨¼E¿J¹JÈvÔVÐW^Õ_ÐbrÕrÐÜŸ,š, z¼¿¹ÈFÔSÐT[Õ\ˆØ+Õ>ÐØ#'×#AÑ#AØØ#Ÿ™Ô.Ø×;Ñ;Ø×=Ñ=ð	 $Bó $
Ð ð ×-Ñ-Ð.FÓGˆØ—‘¢	Ó*×4Ñ4°QÓ7ˆØˆr1   c                ó0   <€ V ^8„  d   QhRS[ S[,          /# ©r    r)   ©r   r   )r.   rå   s   "€r/   r0   r   ¹  s    ø€ ÷ X8ñ X8ñ Ñ+Õ,ñX8r1   c           	     óp  € VP                   RR
 p. VOR
NV P                  N5pVe   VP                  V P                  4      M^ p	\	        V	\
        P                  4      '       d   V	P                  4       MT	p	VRJp
V P                  V4      P                  V4      P                  ^^4      pRp\	        V\        4      '       dH   VP                  P                  V P                  4      pV
'       d   VP                  pMVP                  pMTpV
'       d   TMTpV
'       d\   VeX   V'       dP   VP                   V P                  ,          P"                  pVP                   V P                  ,          P$                  pMÛ. VP                   RR
 OR
NV P                  N5pV P'                  V4      P                  V4      P                  ^^4      pV P)                  V4      P                  V4      P                  ^^4      pVeX   VP+                  VVV P                  4      w  ppV
'       d0   \	        V\        4      '       d   RVP                  V P                  &   Vfº   VP                   R,          pV P,                  '       go   \
        P.                  ! ^V P0                  V^,          V3VP2                  VP4                  R7      pV P6                  '       d   V P8                  '       d   RVn        M&V P=                  V^,          VVP2                  V	R7      p\>        P@                  ! V PB                  PD                  \F        4      pT! V VVVV3RV P8                  '       g   RMV PH                  RV PJ                  R	V/VB w  ppVPL                  ! . VOR
N5!  PO                  4       pV PQ                  V4      pVVV3# )zp
Self-attention (if key_value_states is None) or attention over source sentence (provided by key_value_states).
NFT©r`   rL   )r`   rR  r'   r¡   r&   r(   r8   r‰   ))rT   r'  Úget_seq_lengthr  rý   r+   r,   Úcloner-  Úviewr<   r   Ú
is_updatedÚgetÚcross_attention_cacheÚself_attention_cacheÚlayersÚkeysr¨   r.  r/  Úupdater#  rW   r)  r`   rL   r3  r7   Úrequires_gradrV  r   Úget_interfacerê   Ú_attn_implementationrB   r'   r&   rb   r?   r0  )rÌ   r½   r­   Úkey_value_statesr(   Úpast_key_valuesr)   Úinput_shapeÚhidden_shaperR  Úis_cross_attentionÚquery_statesr`  Úcurr_past_key_valuesÚcurrent_statesÚ
key_statesÚvalue_statesÚkv_shaperQ  Úattention_interfacerA   r@   s   &&&&&&,               r/   rÚ   ÚLongT5Attention.forward¹  sY  € ð $×)Ñ)¨#¨2Ð.ˆØB˜ÐB bÐB¨$×*AÑ*AÑBˆØM\ÒMh˜?×9Ñ9¸$¿.¹.ÔIÐnoÐä7AÐBRÔTY×T`ÑT`×7aÒ7aÐ+×1Ñ1Ô3ÐgwÐð .°TÐ9Ðà—v‘v˜mÓ,×1Ñ1°,Ó?×IÑIÈ!ÈQÓOˆð ˆ
Ü�oÔ':×;Ò;Ø(×3Ñ3×7Ñ7¸¿¹ÓGˆJß!à'6×'LÑ'LÑ$à'6×'KÑ'KÑ$à#2Ð ç-?Ñ)À]ˆß /Ò"=Ç*à-×4Ñ4°T·^±^ÕD×IÑIˆJØ/×6Ñ6°t·~±~ÕF×MÑM‰LàP˜×-Ñ-¨c¨rÐ2ÐP°BÐP¸×8OÑ8OÑPˆHØŸ™ Ó/×4Ñ4°XÓ>×HÑHÈÈAÓNˆJØŸ6™6 .Ó1×6Ñ6°xÓ@×JÑJÈ1ÈaÓPˆLàÒ*Ø+?×+FÑ+FÀzÐS_Ðae×aoÑaoÓ+pÑ(�
˜Lç%¬*°_ÔFY×*ZÒ*ZØAE�O×.Ñ.¨t¯~©~Ñ>ð Ò Ø#×)Ñ)¨"Õ-ˆJØ×3×3Ð3Ü %§¢Ø˜Ÿ™ k°!¥n°jÐAØ'×.Ñ.Ø&×,Ñ,ô!�ð
 ×.×.Ð.°4·=·=°=Ø26�MÔ/øà $× 1Ñ 1Ø •N J°|×7JÑ7JÐ]mð !2ó !�ô )@×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ
%
ð  $Ÿ}Ÿ}˜}‘C°$·,±,ð
%
ð —L‘Lð
%
ð (ð
%
ð ñ
%
Ñ!ˆ�\ð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—f‘f˜[Ó)ˆà˜M¨<Ð7Ð7r1   )rê   rñ   r'   r3  r#  r*  r  r"  r.  r'  r  r)  r0  r-  r2  r%  r$  r&   r/  )FNF©Té    é€   )NrR   )NNNN©rÝ   rÞ   rß   rà   rÈ   ÚstaticmethodrG  rV  rÚ   rá   râ   rã   rä   s   @@r/   r  r  P  s<   ù‡ € ÷%,õ %,ðN ó- ó ð- ô^÷"X8÷ X8ò X8r1   r  c                   ór   a a€ ] tR tRt oR	V3R lV 3R lllt]R
R l4       tV3R lR ltRR ltRt	Vt
V ;t# )ÚLongT5LocalAttentioni  c                ó*   <€ V ^8„  d   QhRS[ RS[RR/# ©r    rê   r#  rG   N©r   r  )r.   rå   s   "€r/   r0   Ú!LongT5LocalAttention.__annotate__  s#   ø€ ÷ ,ñ ,™|ð ,É$ð ,Ð[_ñ ,r1   c                óÖ  <€ \         SV `  4        VP                  V n        W n        VP                  V n        VP
                  V n        VP                  V n        VP                  V n        VP                  V n
        VP                  V n        V P                  ^,           V n        VP                  V n        V P                  V P                  ,          V n        \         P"                  ! V P                  V P                  RR7      V n        \         P"                  ! V P                  V P                  RR7      V n        \         P"                  ! V P                  V P                  RR7      V n        \         P"                  ! V P                  V P                  RR7      V n        V P                  '       d1   \         P,                  ! V P                  V P                  4      V n        RV n        R# )rj   Frî   N)rÇ   rÈ   r"  r#  r$  r%  rñ   r&  r'  r(  r)  Úlocal_radiusrE   rö   r'   r*  r   rð   r-  r.  r/  r0  r1  r2  r3  ©rÌ   rê   r#  rÏ   s   &&&€r/   rÈ   ÚLongT5LocalAttention.__init__  sH  ø€ Ü‰ÑÔØ ×+Ñ+ˆŒØ+FÔ(Ø.4×.SÑ.SˆÔ+Ø/5×/UÑ/UˆÔ,Ø—~‘~ˆŒØ"(§+¡+ˆÔØ×'Ñ'ˆŒØ"×/Ñ/ˆÔØ×*Ñ*¨QÕ.ˆŒØ×*Ñ*ˆŒØŸ™¨×(?Ñ(?Õ?ˆŒä—’˜4Ÿ<™<¨¯©¸eÔDˆŒÜ—’˜4Ÿ<™<¨¯©¸eÔDˆŒÜ—’˜4Ÿ<™<¨¯©¸eÔDˆŒÜ—’˜4Ÿ>™>¨4¯<©<¸eÔDˆŒà×+×+Ð+Ü+-¯<ª<¸×8[Ñ8[Ð]a×]iÑ]iÓ+jˆDÔ(à&+ˆÖ#r1   c                óÄ  € ^ pV'       dP   V^,          pW@^ 8„  P                  \        P                  4      V,          ,          p\        P                  ! V 4      p M,\        P                  ! V \        P
                  ! V 4      4      ) p V^,          pW8  pV\        P                  ! V P                  4       V,          4      \        P                  ! W5,          4      ,          W%,
          ,          P                  \        P                  4      ,           p\        P                  ! V\        P                  ! Wr^,
          4      4      pV\        P                  ! W`V4      ,          pV# r6  r7  r>  s   &&&&    r/   rG  Ú.LongT5LocalAttention._relative_position_bucket-  ó  € ð. ÐßØ˜AÕˆKØ°QÑ!6× :Ñ :¼5¿:¹:Ó FÈÕ TÕTÐÜ %§	¢	Ð*;Ó <Ñä!&§¢Ð+<¼e×>NÒ>NÐO`Ó>aÓ!bÐ bÐð   1Õ$ˆ	Ø$Ñ0ˆð &/Ü�IŠIÐ'×-Ñ-Ó/°)Õ;Ó<Ü�hŠh�|Õ/Ó0õ1àÕ&õ(÷ ‰"ŒU�Z‰Z‹.õ	&Ð"ô
 &+§Y¢YØ&¬¯ªÐ8RÐbcÕTcÓ(dó&
Ð"ð 	œEŸKšK¨ÐE_Ó`Õ`ÐØÐr1   c                ó    <€ V ^8„  d   QhRS[ /# ©r    Úblock_length©rI   )r.   rå   s   "€r/   r0   r�  ^  ó   ø€ ÷ ñ ©ñ r1   c                ó0  € V P                   P                  P                  P                  R8w  d!   V P                   P                  P                  MRp\        P
                  ! ^V,          \        P                  VR7      pW1V)  pVR,          VR,          ,
          pV P                  VV P                  '       * V P                  V P                  R7      pV P                  V4      pVP                  . RO4      P                  ^ 4      P                  ^ 4      pV# ©rJ  ÚmetaNr_   rK  rM  rL  rN  ©r2  rÊ   r`   r—   r+   rv   r8  rG  r"  r$  r%  rO  rx   ©rÌ   r‹  Útarget_devicerT  rS  r?  rU  r¨   s   &&      r/   rV  Ú!LongT5LocalAttention.compute_bias^  óø   € ð ×+Ñ+×2Ñ2×9Ñ9×>Ñ>À&ÔHð ×(Ñ(×/Ñ/×6Ò6àð 	ô
  Ÿ,š, q¨<Õ'7¼u¿z¹zÐR_Ô`ˆØ*¸¸ÐFÐð ,¨GÕ4Ð7GÈÕ7PÕPÐØ#'×#AÑ#AØØ#Ÿ™Ô.Ø×;Ñ;Ø×=Ñ=ð	 $Bó $
Ð ð ×-Ñ-Ð.FÓGˆà—‘¢	Ó*×4Ñ4°QÓ7×AÑAÀ!ÓDˆØˆr1   c                óx  a a€ VP                   R ,          w  opVV 3R lpVV 3R lpV! S P                  V4      4      pV! S P                  V4      4      p	V! S P                  V4      4      p
\	        VS P
                  ^R7      p\	        V	S P
                  ^R7      p	\	        V
S P
                  ^R7      p
\        V	^^R7      p	\        V
^^R7      p
\        P                  ! RW‰4      pVfè   S P                  '       g„   \        P                  ! ^^S P                  S P
                  ^S P
                  ,          3VP                  VP                  R7      pS P                  '       d   S P                  '       d   RVn        MS P#                  S P
                  4      pVe4   \        P$                  ! V^ 8„  R	R4      pW2P'                  ^^4      ,           pW³,          p\(        P*                  P-                  VP/                  4       RR7      P1                  V4      p\(        P*                  P3                  VS P2                  S P                  R
7      pVP5                  V
P                  4      pV! \        P                  ! RWÊ4      4      pVRRV1R3,          pS P7                  V4      pVV3pV'       d
   Wì3,           pV# )r‘   c                óT   <€ V P                  SRSP                  SP                  4      # ©Ú
projectionr8   ©r_  r)  r'  ©Ústatesr«   rÌ   s   &€€r/   rT   Ú+LongT5LocalAttention.forward.<locals>.shape  ó"   ø€ à—;‘;˜z¨2¨t¯|©|¸T×=TÑ=TÓUÐUr1   c                óZ   <€ V P                  4       P                  SRSP                  4      # ©rb   r8   ©r?   r_  r*  r›  s   &€€r/   ÚunshapeÚ-LongT5LocalAttention.forward.<locals>.unshapeƒ  ó%   ø€ à×$Ñ$Ó&×+Ñ+¨J¸¸D¿N¹NÓKÐKr1   r3   rˆ   ú...qhd,...khd->...hqkNr\  Tr¡   r5   ú...hqk,...khd->...qhdr€   ç    _ Âr8   )rT   r-  r.  r/  re   rE   rs   r+   rÁ   r#  rW   r)  r`   rL   r3  r7   rg  rV  r˜   r<   r   r=   r>   r-   Útype_asr'   r—   r0  )rÌ   r½   r­   r(   Úoutput_attentionsÚ
seq_lengthrT   r¢  ro  rr  rs  Úscoresr@   rA   Úoutputsr«   s   f&&&&          @r/   rÚ   ÚLongT5LocalAttention.forwardv  sM  ù€ ð "/×!4Ñ!4°RÕ!8Ñˆ
�Jö	Vö	Lñ
 ˜TŸV™V MÓ2Ó3ˆÙ˜4Ÿ6™6 -Ó0Ó1ˆ
Ù˜TŸV™V MÓ2Ó3ˆô *¨,¸¿¹ÈAÔNˆÜ'¨
°D·N±NÈÔJˆ
Ü)¨,¸¿¹ÈAÔNˆô +¨:ÀÐQRÔSˆ
Ü,¨\ÀQÐUVÔWˆô —’Ø# \ó
ˆð Ò à×3×3Ð3Ü %§¢Ø˜˜4Ÿ<™<¨¯©¸¸T¿^¹^Õ9KÐLÐU[×UbÑUbÐjp×jvÑjvô!�ð ×.×.Ð.°4·=·=°=Ø26�MÔ/øà $× 1Ñ 1°$·.±.Ó A�àÒä—{’{ 4¨!¡8¨S°%Ó8�à -·±¸qÀ!Ó0DÕ D�àÕˆä—}‘}×,Ñ,¨V¯\©\«^ÀÐ,ÓD×LÑLÈVÓTˆä—}‘}×,Ñ,¨\¸T¿\¹\ÐTX×TaÑTaÐ,Óbˆà#×(Ñ(¨×);Ñ);Ó<ˆÙœeŸlšlÐ+BÀLÓ_Ó`ˆØ! ! [ j [°!Ð"3Õ4ˆØ—f‘f˜[Ó)ˆð Øð
ˆ÷
 Ø Õ/ˆGØˆr1   )rE   rñ   r'   r3  r#  r*  r"  r.  r'  rƒ  r)  r0  r-  r2  r%  r$  r/  ©Frw  ©NNFrz  rä   s   @@r/   r}  r}    s:   ù‡ € ÷,õ ,ð0 ó- ó ð- ÷^ð ÷0Gô Gr1   r}  c                   ó„   a a€ ] tR tRt oRV3R lV 3R lllt]RR l4       tV3R lR ltV3R lR ltRR	 lt	R
t
VtV ;t# )ÚLongT5TransientGlobalAttentioniÀ  c                ó*   <€ V ^8„  d   QhRS[ RS[RR/# r  r€  )r.   rå   s   "€r/   r0   Ú+LongT5TransientGlobalAttention.__annotate__Á  s(   ø€ ÷ fñ f™|ð fÉ$ð fÐ[_ñ fr1   c                óº  <€ \         SV `  4        VP                  V n        W n        VP                  V n        VP
                  V n        VP                  V n        VP                  V n        VP                  V n
        VP                  V n        V P                  ^,           V n        VP                  V n        VP                  V n        V P                  V P                  ,          V n        \"        P$                  ! V P                  V P                   RR7      V n        \"        P$                  ! V P                  V P                   RR7      V n        \"        P$                  ! V P                  V P                   RR7      V n        \"        P$                  ! V P                   V P                  RR7      V n        V P                  '       d1   \"        P.                  ! V P                  V P                  4      V n        V P                  '       d1   \"        P.                  ! V P                  V P                  4      V n        \5        VP                  VP6                  R7      V n        R# )rj   Frî   r  N)rÇ   rÈ   r"  r#  r$  r%  rñ   r&  r'  r(  r)  rƒ  rE   r�   rö   r'   r*  r   rð   r-  r.  r/  r0  r1  r2  Úglobal_relative_attention_biasrÅ   r  Úglobal_input_layer_normr„  s   &&&€r/   rÈ   Ú'LongT5TransientGlobalAttention.__init__Á  s•  ø€ Ü‰ÑÔØ ×+Ñ+ˆŒØ+FÔ(Ø.4×.SÑ.SˆÔ+Ø/5×/UÑ/UˆÔ,Ø—~‘~ˆŒØ"(§+¡+ˆÔØ×'Ñ'ˆŒØ"×/Ñ/ˆÔØ×*Ñ*¨QÕ.ˆŒØ!'×!9Ñ!9ˆÔØ×*Ñ*ˆŒØŸ™¨×(?Ñ(?Õ?ˆŒä—’˜4Ÿ<™<¨¯©¸eÔDˆŒÜ—’˜4Ÿ<™<¨¯©¸eÔDˆŒÜ—’˜4Ÿ<™<¨¯©¸eÔDˆŒÜ—’˜4Ÿ>™>¨4¯<©<¸eÔDˆŒà×+×+Ð+Ü+-¯<ª<¸×8[Ñ8[Ð]a×]iÑ]iÓ+jˆDÔ(ð ×+×+Ð+Ü24·,²,¸t×?bÑ?bÐdh×dpÑdpÓ2qˆDÔ/Ü'6°v·~±~È6×KdÑKdÔ'eˆÖ$r1   c                óÄ  € ^ pV'       dP   V^,          pW@^ 8„  P                  \        P                  4      V,          ,          p\        P                  ! V 4      p M,\        P                  ! V \        P
                  ! V 4      4      ) p V^,          pW8  pV\        P                  ! V P                  4       V,          4      \        P                  ! W5,          4      ,          W%,
          ,          P                  \        P                  4      ,           p\        P                  ! V\        P                  ! Wr^,
          4      4      pV\        P                  ! W`V4      ,          pV# r6  r7  r>  s   &&&&    r/   rG  Ú8LongT5TransientGlobalAttention._relative_position_bucketÝ  rˆ  r1   c                ó    <€ V ^8„  d   QhRS[ /# rŠ  rŒ  )r.   rå   s   "€r/   r0   r³    r�  r1   c                ó0  € V P                   P                  P                  P                  R8w  d!   V P                   P                  P                  MRp\        P
                  ! ^V,          \        P                  VR7      pW1V)  pVR,          VR,          ,
          pV P                  VV P                  '       * V P                  V P                  R7      pV P                  V4      pVP                  . RO4      P                  ^ 4      P                  ^ 4      pV# r�  r‘  r’  s   &&      r/   rV  Ú+LongT5TransientGlobalAttention.compute_bias  r•  r1   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# )r    r­   r²   rG   r”   )r.   rå   s   "€r/   r0   r³  &  s1   ø€ ÷ #ñ #¡e§l¡lð #ÉÏÉð #ÑY^×YeÑYeñ #r1   c                ó†  € \         P                  ! VR,          VR,          4      R,          p\         P                  ! V^ 8„  RR4      p\        WP                  4      pV P                  VV P                  '       * V P                  V P                  R7      pV P                  V4      pVP                  . RO4      pWG,           pV# ).r¡   rK  r¶   )r€   Nr€   )r€   N.r§  )rR   é   rj   r    )r+   Úeqr˜   r»   r�   rG  r"  r$  r%  rµ  rO  )rÌ   r­   r²   Úside_attention_maskÚattention_side_biasrº   Úside_relative_position_bucketÚ	side_biass   &&&     r/   Úcompute_side_biasÚ0LongT5TransientGlobalAttention.compute_side_bias&  s·   € ä#Ÿhšh t¨I¥Ð8JÈ:Õ8VÓWÐXdÕeÐÜ#ŸkškÐ*=ÀÑ*AÀ3ÈÓNÐä!AÀ$×H^ÑH^Ó!_ÐØ(,×(FÑ(FØ"Ø#Ÿ™Ô.Ø×;Ñ;Ø×=Ñ=ð	 )Gó )
Ð%ð ×7Ñ7Ð8UÓVˆ	ð ×%Ñ%¢lÓ3ˆ	à1Õ=ÐØ"Ð"r1   c                óÎ	  a a€ VP                   R ,          w  opVV 3R lpVV 3R lp\        Ve   TM"\        P                  ! VP                   RR 4      S P                  4      w  r‰V	P                   R,          p
\        WV
4      pS P                  V4      pV! S P                  V4      4      pV! S P                  V4      4      pV! S P                  V4      4      pV! S P                  V4      4      pV! S P                  V4      4      p\        VS P                  ^R7      p\        VS P                  ^R7      p\        VS P                  ^R7      p\        V^^R7      p\        V^^R7      p^.VP                  ^,           ,          pVP                   ^,          V^&   VP                  ^4      P                  V4      pVP                  ^4      P                  V4      p\        P                   ! Wß.^R7      p\        P                   ! VV.^R7      p\        P"                  ! RWÍ4      pVe>   \%        VS P                  VP&                  4      p\        P(                  ! V^ 8„  RR4      pMRpVEf‹   S P*                  '       g„   \        P,                  ! ^^S P.                  S P                  ^S P                  ,          3VP&                  VP0                  R7      pS P2                  '       d   S P4                  '       d   R	Vn        MS P9                  S P                  4      pVe   VVP;                  ^^4      ,           pVP=                  VP0                  4      pVf   \        P                  ! SV4      pS P?                  W)4      p\        VS P                  RR7      P;                  ^^4      pVP=                  VP0                  4      PA                  VP&                  4      p\        P                   ! VV.RR7      pVV,          p\B        PD                  PG                  VPI                  4       RR7      PK                  V4      p\B        PD                  PM                  VS PL                  S P4                  R
7      pVP=                  VP0                  4      pV! \        P"                  ! RVV4      4      pVRRV1R3,          pS PO                  V4      pVV3pV'       d   VV3,           pV# )r‘   c                óT   <€ V P                  SRSP                  SP                  4      # r˜  rš  r›  s   &€€r/   rT   Ú5LongT5TransientGlobalAttention.forward.<locals>.shapeD  rž  r1   c                óZ   <€ V P                  4       P                  SRSP                  4      # r   r¡  r›  s   &€€r/   r¢  Ú7LongT5TransientGlobalAttention.forward.<locals>.unshapeH  r¤  r1   Nr3   rˆ   r¥  r¡   r\  Tr5   r¦  r€   r8   r§  r‰   )(rT   r³   r+   rª   r�   rÃ   r¶  r-  r.  r/  re   rE   rs   rX   rx   r©   ro   rÁ   rŒ   r`   r˜   r#  rW   r)  rL   r3  r7   rg  rV  r<   r—   rÅ  r‚   r   r=   r>   r-   r¨  r'   r0  )rÌ   r½   r­   r(   r©  rª  rT   r¢  r“   r²   Ú_global_seq_lenÚglobal_inputsro  rr  rs  Úside_key_statesÚside_value_statesÚrepsr«  r~   Úside_position_biasr@   rA   r¬  r«   s   f&&&&                   @r/   rÚ   Ú&LongT5TransientGlobalAttention.forward;  s+  ù€ ð "/×!4Ñ!4°RÕ!8Ñˆ
�Jö	Vö	Lô )EØÒ$‰D¬%¯*ª*°]×5HÑ5HÈÈ"Ð5MÓ*NØ×"Ñ"ó)
Ñ%ˆ	ð
 -×2Ñ2°2Õ6ˆÜ1°-ÈOÓ\ˆØ×4Ñ4°]ÓCˆñ ˜TŸV™V MÓ2Ó3ˆÙ˜4Ÿ6™6 -Ó0Ó1ˆ
Ù˜TŸV™V MÓ2Ó3ˆá §¡ }Ó 5Ó6ˆÙ! $§&¡&¨Ó"7Ó8Ðô *¨,¸¿¹ÈAÔNˆÜ'¨
°D·N±NÈÔJˆ
Ü)¨,¸¿¹ÈAÔNˆô +¨:ÀÐQRÔSˆ
Ü,¨\ÀQÐUVÔWˆð ˆs�o×*Ñ*¨QÕ.Õ/ˆØ×"Ñ" 1Õ%ˆˆQ‰Ø)×3Ñ3°AÓ6×=Ñ=¸dÓCˆØ-×7Ñ7¸Ó:×AÑAÀ$ÓGÐô —Y’Y 
Ð<À!ÔDˆ
Ü—y’y ,Ð0AÐ!BÈÔJˆô —’Ð5°|ÓPˆàÒä#<¸TÀ4Ç>Á>ÐS`×SgÑSgÓ#hÐ ä#(§;¢;Ð/CÀaÑ/GÈÈeÓ#TÑ à#'Ð àÓ à×3×3Ð3Ü %§¢Ø˜˜4Ÿ<™<¨¯©¸¸T¿^¹^Õ9KÐLØ!Ÿ=™=Ø Ÿ,™,ô!�ð
 ×.×.Ð.°4·=·=°=Ø26�MÔ/øà $× 1Ñ 1°$·.±.Ó A�à#Ò/à -Ð0D×0NÑ0NÈqÐRSÓ0TÕ T�Ø)×.Ñ.¨v¯|©|Ó<ˆMð Š|Ü—z’z *¨jÓ9�à!%×!7Ñ!7¸Ó!QÐä!3Ð4FÈÏÉÐ\^Ô!_×!iÑ!iÐjkÐmnÓ!oÐØ!3×!8Ñ!8¸¿¹Ó!F×!IÑ!IÈ&Ï-É-Ó!XÐä!ŸIšI }Ð6HÐ&IÈrÔRˆMà�-Õˆä—}‘}×,Ñ,¨V¯\©\«^ÀÐ,ÓD×LÑLÈVÓTˆÜ—}‘}×,Ñ,¨\¸T¿\¹\ÐTX×TaÑTaÐ,Óbˆà#×(Ñ(¨×);Ñ);Ó<ˆÙœeŸlšlÐ+BÀLÐR^Ó_Ó`ˆØ! ! [ j [°!Ð"3Õ4ˆØ—f‘f˜[Ó)ˆà Ð.ˆçØ  Õ/ˆGØˆr1   )rE   rñ   r'   r�   r¶  rµ  r#  r*  r"  r.  r'  rƒ  r)  r0  r-  r2  r%  r$  r/  r®  rw  r¯  )rÝ   rÞ   rß   rà   rÈ   r{  rG  rV  rÅ  rÚ   rá   râ   rã   rä   s   @@r/   r±  r±  À  sF   ù‡ € ÷fõ fð8 ó- ó ð- ÷^ð ÷0#ð #÷*qô qr1   r±  c                   óX   a a€ ] tR tRt oRV3R lV 3R llltRV3R lR lltRtVtV ;t# )	ÚLongT5LayerSelfAttentioni°  c                ó.   <€ V ^8„  d   QhRS[ R,          /# ©r    r  NrŒ  )r.   rå   s   "€r/   r0   Ú%LongT5LayerSelfAttention.__annotate__±  s   ø€ ÷ 	7ñ 	7ÉSÐSWÍZñ 	7r1   c                óö   <€ \         SV `  4        \        VVVVP                  R 7      V n        \        VP                  VP                  R7      V n        \        P                  ! VP                  4      V n        R# )©r#  r  r  r  N)rÇ   rÈ   r  r"  ÚSelfAttentionrÅ   rñ   r  r  r   rõ   rö   r'   ©rÌ   rê   r#  r  rÏ   s   &&&&€r/   rÈ   Ú!LongT5LayerSelfAttention.__init__±  s]   ø€ Ü‰ÑÔÜ,ØØ(CØØ×'Ñ'ô	
ˆÔô *¨&¯.©.¸f×>WÑ>WÔXˆŒÜ—z’z &×"5Ñ"5Ó6ˆŽr1   c                ó0   <€ V ^8„  d   QhRS[ S[,          /# rY  rZ  )r.   rå   s   "€r/   r0   r×  ¼  s   ø€ ÷ =ñ =ñ Ñ+Õ,ñ=r1   c           	     ó’   € V P                  V4      pV P                  ! V3R VRVRV/VB w  rspWP                  V4      ,           pWsV3# )r­   r(   rk  )r  rÚ  r'   )	rÌ   r½   r%   r(   rk  r)   Únormed_hidden_statesÚattention_outputr@   s	   &&&&&,   r/   rÚ   Ú LongT5LayerSelfAttention.forward¼  sr   € ð  $Ÿ™¨}Ó=ÐØ8<×8JÒ8JØ ñ9
àð9
ð (ð9
ð ,ð	9
ð
 ñ9
Ñ5Ð¨ð )¯<©<Ð8HÓ+IÕIÐØ°Ð<Ð<r1   )rÚ  r'   r  ©FN©NNNrÜ   rä   s   @@r/   rÔ  rÔ  °  s   ù‡ € ÷	7õ 	7÷=÷ =ò =r1   rÔ  c                   ó\   a a€ ] tR tRt oRtRV3R lV 3R llltR	V3R lR lltRtVtV ;t	# )
ÚLongT5LayerLocalSelfAttentioniÐ  z$Local self attention used in encoderc                ó.   <€ V ^8„  d   QhRS[ R,          /# rÖ  rŒ  )r.   rå   s   "€r/   r0   Ú*LongT5LayerLocalSelfAttention.__annotate__Ó  s   ø€ ÷ 7ñ 7ÉSÐSWÍZñ 7r1   c                óÜ   <€ \         SV `  4        \        WR 7      V n        \	        VP
                  VP                  R7      V n        \        P                  ! VP                  4      V n        R# ©)r#  r  N)rÇ   rÈ   r}  ÚLocalSelfAttentionrÅ   rñ   r  r  r   rõ   rö   r'   rÛ  s   &&&&€r/   rÈ   Ú&LongT5LayerLocalSelfAttention.__init__Ó  sI   ø€ Ü‰ÑÔÜ"6°vÔ"wˆÔÜ)¨&¯.©.¸f×>WÑ>WÔXˆŒÜ—z’z &×"5Ñ"5Ó6ˆŽr1   c                ó    <€ V ^8„  d   QhRS[ /# rY  r   )r.   rå   s   "€r/   r0   rç  Ù  ó   ø€ ÷ ñ ñ ñr1   c                ó°   € V P                  V4      pV P                  VVVVR 7      pWP                  V^ ,          4      ,           pV3VR,          ,           pV# ©)r­   r(   r©  ºrj   NN)r  rê  r'   ©	rÌ   r½   r%   r(   r©  r)   rß  rà  r¬  s	   &&&&&,   r/   rÚ   Ú%LongT5LayerLocalSelfAttention.forwardÙ  sf   € ð  $Ÿ™¨}Ó=ÐØ×2Ñ2Ø ØØ'Ø/ð	 3ó 
Ðð &¯©Ð5EÀaÕ5HÓ(IÕIˆØ Ð"Ð%5°bÕ%9Õ9ˆØˆr1   )rê  r'   r  râ  r¯  ©
rÝ   rÞ   rß   rà   Ú__doc__rÈ   rÚ   rá   râ   rã   rä   s   @@r/   rå  rå  Ð  s!   ù‡ € Ù.÷7õ 7÷÷ ò r1   rå  c                   ó\   a a€ ] tR tRt oRtRV3R lV 3R llltR	V3R lR lltRtVtV ;t	# )
Ú'LongT5LayerTransientGlobalSelfAttentionií  z/Transient-Global self attention used in encoderc                ó.   <€ V ^8„  d   QhRS[ R,          /# rÖ  rŒ  )r.   rå   s   "€r/   r0   Ú4LongT5LayerTransientGlobalSelfAttention.__annotate__ð  s   ø€ ÷ 7ñ 7ÉSÐSWÍZñ 7r1   c                óÜ   <€ \         SV `  4        \        WR 7      V n        \	        VP
                  VP                  R7      V n        \        P                  ! VP                  4      V n        R# ré  )rÇ   rÈ   r±  ÚTransientGlobalSelfAttentionrÅ   rñ   r  r  r   rõ   rö   r'   rÛ  s   &&&&€r/   rÈ   Ú0LongT5LayerTransientGlobalSelfAttention.__init__ð  sN   ø€ Ü‰ÑÔÜ,JØô-
ˆÔ)ô *¨&¯.©.¸f×>WÑ>WÔXˆŒÜ—z’z &×"5Ñ"5Ó6ˆŽr1   c                ó    <€ V ^8„  d   QhRS[ /# rY  r   )r.   rå   s   "€r/   r0   rø  ø  rí  r1   c                ó°   € V P                  V4      pV P                  VVVVR 7      pWP                  V^ ,          4      ,           pV3VR,          ,           pV# rï  )r  rú  r'   rñ  s	   &&&&&,   r/   rÚ   Ú/LongT5LayerTransientGlobalSelfAttention.forwardø  sf   € ð  $Ÿ™¨}Ó=ÐØ×<Ñ<Ø ØØ'Ø/ð	 =ó 
Ðð &¯©Ð5EÀaÕ5HÓ(IÕIˆØ Ð"Ð%5°bÕ%9Õ9ˆØˆr1   )rú  r'   r  râ  r¯  ró  rä   s   @@r/   rö  rö  í  s!   ù‡ € Ù9÷7õ 7÷÷ ò r1   rö  c                   óX   a a€ ] tR tRt oRV3R lV 3R llltRV3R lR lltRtVtV ;t# )	ÚLongT5LayerCrossAttentioni  c                ó.   <€ V ^8„  d   QhRS[ R,          /# rÖ  rŒ  )r.   rå   s   "€r/   r0   Ú&LongT5LayerCrossAttention.__annotate__  s   ø€ ÷ 7ñ 7©#°­*ñ 7r1   c                óâ   <€ \         SV `  4        \        VR VR R7      V n        \	        VP
                  VP                  R7      V n        \        P                  ! VP                  4      V n        R# )FrÙ  r  N)rÇ   rÈ   r  ÚEncDecAttentionrÅ   rñ   r  r  r   rõ   rö   r'   )rÌ   rê   r  rÏ   s   &&&€r/   rÈ   Ú"LongT5LayerCrossAttention.__init__  sU   ø€ Ü‰ÑÔÜ.Ø°ÀÐV[ô 
ˆÔô *¨&¯.©.¸f×>WÑ>WÔXˆŒÜ—z’z &×"5Ñ"5Ó6ˆŽr1   c                ó0   <€ V ^8„  d   QhRS[ S[,          /# rY  rZ  )r.   rå   s   "€r/   r0   r    s   ø€ ÷ 9ñ 9ñ Ñ+Õ,ñ9r1   c                ó–   € V P                  V4      pV P                  ! V3R VRVRVRV/VB w  r„p	WP                  V4      ,           p
W¤V	3# )r­   rj  r(   rk  )r  r  r'   )rÌ   r½   rj  r%   r(   rk  r)   rß  rà  r@   Úlayer_outputs   &&&&&&,    r/   rÚ   Ú!LongT5LayerCrossAttention.forward  s{   € ð  $Ÿ™¨}Ó=ÐØ8<×8LÒ8LØ ñ9
àð9
ð .ð9
ð (ð	9
ð
 ,ð9
ð ñ9
Ñ5Ð¨ð %§|¡|Ð4DÓ'EÕEˆØ¨LÐ8Ð8r1   )r  r'   r  rü   rã  rÜ   rä   s   @@r/   r   r     s   ù‡ € ÷7õ 7÷9÷ 9ò 9r1   r   c                   óL   a a€ ] tR tRt oRV3R lV 3R llltRR ltRtVtV ;t# )ÚLongT5Blocki,  c                ó.   <€ V ^8„  d   QhRS[ R,          /# rÖ  rŒ  )r.   rå   s   "€r/   r0   ÚLongT5Block.__annotate__-  s   ø€ ÷ 1ñ 1ÉSÐSWÍZñ 1r1   c                ó:  <€ \         SV `  4        VP                  V n        VP                  '       d   \        pMIVP                  R 8X  d   \
        pM1VP                  R8X  d   \        pM\        RVP                   R24      h\        P                  ! 4       V n
        V P                  P                  V! WVR7      4       V P                  '       d&   V P                  P                  \        WR7      4       V P                  P                  \        V4      4       R# )Úlocalztransient-globalzjFor encoder attention mechanism, either `local` or `transient-global` attention type is expected, but got Ú.©r#  r  )r  N)rÇ   rÈ   r"  rÔ  Úencoder_attention_typerå  rö  Ú
ValueErrorr   Ú
ModuleListÚlayerrn   r   r  )rÌ   rê   r#  r  Úattention_layerrÏ   s   &&&& €r/   rÈ   ÚLongT5Block.__init__-  sÞ   ø€ Ü‰ÑÔØ ×+Ñ+ˆŒØ××ÐÜ6‰OØ×*Ñ*¨gÔ5Ü;‰OØ×*Ñ*Ð.@Ô@ÜE‰OäðØ!×8Ñ8Ð9¸ð<óð ô —]’]“_ˆŒ
Ø�
‰
×ÑÙ˜FÐgpÔqô	
ð �?�?ˆ?Ø�J‰J×ÑÔ7¸ÔTÔUà�
‰
×Ñœ-¨Ó/Ö0r1   c           	     óÌ  € V P                   ^ ,          ! VVVVVV	R7      pV^ ,          pVR,          pVP                  \        P                  8X  du   \        P                  ! V4      P                  4       '       dK   \        P                  ! VP                  4      P                  R,
          p\        P                  ! W) VR7      pV P                  ;'       d    VRJpV'       dÌ   V P                   ^,          ! VVVVVV	R7      pV^ ,          pVP                  \        P                  8X  du   \        P                  ! V4      P                  4       '       dK   \        P                  ! VP                  4      P                  R,
          p\        P                  ! W) VR7      pVVR,          ,           pV P                   R,          ! V4      pVP                  \        P                  8X  du   \        P                  ! V4      P                  4       '       dK   \        P                  ! VP                  4      P                  R,
          p\        P                  ! W) VR7      pV3V,           # )rR   )r%   r(   rk  Ú	use_cacher©  rð  iè  )r9  r§   N)rj  r%   r(   rk  r©  r8   )
r  rL   r+   r×   ÚisinfÚanyÚfinfor§   Úclampr"  )rÌ   r½   r%   r(   Úencoder_hidden_statesÚencoder_attention_maskÚencoder_decoder_position_biasrk  r  r©  Úreturn_dictr)   Úself_attention_outputsÚattention_outputsÚclamp_valueÚdo_cross_attentionÚcross_attention_outputss   &&&&&&&&&&&,     r/   rÚ   ÚLongT5Block.forwardD  sÛ  € ð "&§¡¨A¦ØØ)Ø'Ø+ØØ/ô"
Ðð /¨qÕ1ˆØ2°2Õ6Ðð ×Ñ¤%§-¡-Ô/´E·K²KÀÓ4N×4RÑ4R×4TÒ4TÜŸ+š+ m×&9Ñ&9Ó:×>Ñ>ÀÕEˆKÜ!ŸKšK¨¸<È[ÔYˆMà!Ÿ_™_×RÐRÐ1FÈdÐ1RÐßØ&*§j¡j°¦mØØ!6Ø5Ø;Ø /Ø"3ô'Ð#ð 4°AÕ6ˆMð ×"Ñ"¤e§m¡mÔ3¼¿ºÀMÓ8R×8VÑ8V×8XÒ8XÜ#Ÿkšk¨-×*=Ñ*=Ó>×BÑBÀTÕI�Ü %§¢¨M¸|ÐQ\Ô ]�ð !2Ð4KÈBÕ4OÕ OÐð Ÿ
™
 2ž }Ó5ˆð ×Ñ¤%§-¡-Ô/´E·K²KÀÓ4N×4RÑ4R×4TÒ4TÜŸ+š+ m×&9Ñ&9Ó:×>Ñ>ÀÕEˆKÜ!ŸKšK¨¸<È[ÔYˆMð ÐÐ0Õ0ð	
r1   )r"  r  râ  )	NNNNNNFFTrÜ   rä   s   @@r/   r  r  ,  s   ù‡ € ÷1õ 1÷.<
ô <
r1   r  c                   óŽ   a a€ ] tR tRt oRtRtR.tRt]R 4       t	]
P                  ! 4       V 3R l4       tR tV3R	 ltR
tVtV ;t# )ÚLongT5PreTrainedModeliƒ  ÚtransformerTr  Fc                ó~   € \         P                  ! \        4      p\         P                  ! \        4      pR VRVRV/pV# )Údecoder_input_idsÚ	input_idsÚdecoder_attention_mask)r+   r¿   r   r   )rÌ   r-  Ú
input_maskÚdummy_inputss   &   r/   r0  Ú"LongT5PreTrainedModel.dummy_inputsŒ  s>   € ô —L’L¤Ó.ˆ	Ü—\’\¤*Ó-ˆ
à Ø˜Ø$ jð
ˆð
 Ðr1   c                óp  <€ \         SV `  V4       V P                  P                  p\	        V\
        4      '       d+   \        P                  ! VP                  VR,          4       R# \	        V\        \        \        34      '       d�   \        P                  ! VP                  P                  RVR,          R7       \        VR4      '       dU   V P                  P                  '       g7   \        P                  ! VP                   P                  RVR,          R7       R# R# R# \	        V\"        4      '       Ed_   \        P                  ! VP$                  P                  RW P                  P&                  R,          ,          R7       \        VP$                  R4      '       dC   VP$                  P(                  e+   \        P*                  ! VP$                  P(                  4       \        P                  ! VP,                  P                  RW P                  P.                  R,          ,          R7       \        VP,                  R4      '       dG   VP,                  P(                  e-   \        P*                  ! VP,                  P(                  4       R# R# R# \	        V\0        4      '       Ed   \        P                  ! VP2                  P                  RW P                  P&                  R,          ,          R7       \        VP2                  R4      '       dC   VP2                  P(                  e+   \        P*                  ! VP2                  P(                  4       \        P                  ! VP4                  P                  RW P                  P&                  R,          ,          R7       \        VP4                  R4      '       dC   VP4                  P(                  e+   \        P*                  ! VP4                  P(                  4       \        P                  ! VP,                  P                  RW P                  P.                  R,          ,          R7       \        VP,                  R4      '       dG   VP,                  P(                  e-   \        P*                  ! VP,                  P(                  4       R# R# R# \	        V\6        \8        \:        34      '       EdÛ   V P                  P&                  pV P                  P<                  pV P                  P>                  p\        P                  ! VP@                  P                  RW#V,          R,          ,          R7       \        P                  ! VPB                  P                  RW#R,          ,          R7       \        P                  ! VPD                  P                  RW#R,          ,          R7       \        P                  ! VPF                  P                  RW%V,          R,          ,          R7       VPH                  '       d�   \        P                  ! VPJ                  P                  RW#R,          ,          R7       \	        V\:        4      '       d=   \        P                  ! VPL                  P                  RW#R,          ,          R7       R# R# R# R# )zInitialize the weightsr¢   r¡   )rÕ   ÚstdÚlm_headrï   Nr9   )'rÇ   Ú_init_weightsrê   Úinitializer_factorrý   rÅ   ÚinitÚ	constant_rÊ   ÚLongT5ModelÚLongT5ForConditionalGenerationÚLongT5EncoderModelÚnormal_ÚsharedÚhasattrÚtie_word_embeddingsr4  rç   ró   rñ   rï   Úzeros_rô   rò   r  r  r  r  r}  r±  r&  r(  r-  r.  r/  r0  r#  r2  rµ  )rÌ   r!   Úfactorrñ   r'  r)  rÏ   s   &&    €r/   r5  Ú#LongT5PreTrainedModel._init_weights˜  s  ø€ ô 	‰Ñ˜fÔ%Ø—‘×/Ñ/ˆÜ�fœo×.Ò.Ü�NŠN˜6Ÿ=™=¨&°3­,Ö7Ü˜¤Ô.LÔN`Ð a×bÒbÜ�LŠL˜Ÿ™×-Ñ-°C¸VÀc½\ÕJÜ�v˜y×)Ò)°$·+±+×2Q×2QÐ2QÜ—’˜VŸ^™^×2Ñ2¸À&È3Å,×Oñ 3RÑ)ä˜Ô 3×4Ó4Ü�LŠL˜Ÿ™×)Ñ)°¸ÇKÁK×DWÑDWÐ\`ÕC`Õ9aÕbÜ�v—y‘y &×)Ò)¨f¯i©i¯n©nÒ.HÜ—’˜FŸI™IŸN™NÔ+Ü�LŠL˜Ÿ™×)Ñ)°¸ÇKÁK×DTÑDTÐY]ÕC]Õ9^Õ_Ü�v—y‘y &×)Ò)¨f¯i©i¯n©nÒ.HÜ—’˜FŸI™IŸN™NÖ+ñ /IÑ)ä˜Ô 8×9Ó9Ü�LŠL˜Ÿ™×+Ñ+°#¸6ÇkÁk×FYÑFYÐ^bÕEbÕ;cÕdÜ�v—{‘{ F×+Ò+°·±×0@Ñ0@Ò0LÜ—’˜FŸK™K×,Ñ,Ô-Ü�LŠL˜Ÿ™×+Ñ+°#¸6ÇkÁk×FYÑFYÐ^bÕEbÕ;cÕdÜ�v—{‘{ F×+Ò+°·±×0@Ñ0@Ò0LÜ—’˜FŸK™K×,Ñ,Ô-Ü�LŠL˜Ÿ™×)Ñ)°¸ÇKÁK×DTÑDTÐY]ÕC]Õ9^Õ_Ü�v—y‘y &×)Ò)¨f¯i©i¯n©nÒ.HÜ—’˜FŸI™IŸN™NÖ+ñ /IÑ)ä˜¤Ô2FÔHfÐ g×hÓhØ—k‘k×)Ñ)ˆGØ!%§¡×!1Ñ!1ÐØ—k‘k×+Ñ+ˆGÜ�LŠL˜Ÿ™Ÿ™¨s¸ÐM_ÕC_ÐdhÕBhÕ8iÕjÜ�LŠL˜Ÿ™Ÿ™¨s¸È4Å-Õ8PÕQÜ�LŠL˜Ÿ™Ÿ™¨s¸È4Å-Õ8PÕQÜ�LŠL˜Ÿ™Ÿ™¨s¸ÐM_ÕC_ÐdhÕBhÕ8iÕjØ×1×1Ð1Ü—’˜V×;Ñ;×BÑBÈÐRXÐimÕ\mÕRnÕoÜ˜fÔ&D×EÒEÜ—L’LØ×=Ñ=×DÑDÈ3ÐTZÐkoÕ^oÕTp÷ñ Fñ 2ñ ir1   c                ó>  € V P                   P                  pV P                   P                  pVf   \        R4      hVP	                  VP
                  4      pVRR R13,          P                  4       VR&   W$R&   Vf   \        R4      hVP                  VR8H  V4       V# )Nz’self.model.config.decoder_start_token_id has to be defined. In LongT5 it is usually set to the pad_token_id. See LongT5 docs for more information..z1self.model.config.pad_token_id has to be defined.r8   ).rð  ).rR   éœÿÿÿ)rê   Údecoder_start_token_idÚpad_token_idr  Ú	new_zerosrT   r^  Úmasked_fill_)rÌ   r-  rE  rF  Úshifted_input_idss   &&   r/   Ú_shift_rightÚ"LongT5PreTrainedModel._shift_rightÄ  s§   € Ø!%§¡×!CÑ!CÐØ—{‘{×/Ñ/ˆà!Ò)Üð8óð ð
 &×/Ñ/°	·±Ó@ÐØ%.¨s°C°R°C¨xÕ%8×%>Ñ%>Ó%@Ð˜'Ñ"Ø$:˜&Ñ!àÒÜÐPÓQÐQà×&Ñ&Ð'8¸DÑ'@À,ÔOà Ð r1   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # ré   r   )r.   rå   s   "€r/   r0   Ú"LongT5PreTrainedModel.__annotate__ƒ  s   ø‡ ‚ áÑò r1   rS   )rÝ   rÞ   rß   rà   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_can_compile_fullgraphÚpropertyr0  r+   Úno_gradr5  rJ  Ú__annotate_func__rá   râ   rã   rä   s   @@r/   r)  r)  ƒ  s[   ù‡ € ð &ÐØ&*Ð#Ø&˜Ðà"Ðàñó ðð ‡]‚]ƒ_ô(ó ð(òV!÷C … r1   r)  c                   óB   a a€ ] tR tRt oV 3R ltR tRR ltRtVtV ;t	# )ÚLongT5StackiÚ  c                ót  <€ \         SV `  V4       \        P                  ! VP                  VP
                  4      V n        VP                  V n        VP                  V n        V P                  ^,           V n	        \        P                  ! \        VP                  4       Uu. uF  p\        V\        V^ 8H  4      VR7      NK  	  up4      V n        \!        VP
                  VP"                  R7      V n        \        P&                  ! VP(                  4      V n        RV n        V P/                  4        R# u upi )rj   r  r  FN)rÇ   rÈ   r   r1  Ú
vocab_sizerñ   Úembed_tokensr"  rƒ  rE   r  rk   Ú
num_layersr  r  ÚblockrÅ   r  Úfinal_layer_normrõ   rö   r'   r3  Ú	post_init)rÌ   rê   rq   rÏ   s   && €r/   rÈ   ÚLongT5Stack.__init__Û  sî   ø€ Ü‰Ñ˜Ô äŸLšL¨×):Ñ):¸F¿N¹NÓKˆÔØ ×+Ñ+ˆŒà"×/Ñ/ˆÔØ×*Ñ*¨QÕ.ˆŒä—]’]ô ˜v×0Ñ0Ô1óá1�Aô ˜FÄÀQÈ!ÁVÃÐXY×ZÙ1ñó
ˆŒ
ô !0°·±ÀF×D]ÑD]Ô ^ˆÔÜ—z’z &×"5Ñ"5Ó6ˆŒà&+ˆÔ#ð 	�‰Öùòs   Â"#D5c                ó   € Wn         R # rü   )rY  ©rÌ   Únew_embeddingss   &&r/   Úset_input_embeddingsÚ LongT5Stack.set_input_embeddingsó  s   € Ø*Ör1   c                óð  € Ve   TMV P                   P                  pVe   TMV P                   P                  pV	e   T	MV P                   P                  p	V
e   T
MV P                   P                  p
Ve-   Ve)   V P
                  '       d   RMRp\        RV RV R24      hVe+   VP                  4       pVP                  RVR,          4      pM@Ve   VP                  4       R R pM(V P
                  '       d   RMRp\        RV RV R24      hV P                  '       d2   V P                  '       d    V'       d   \        P                  R	4       R
pVf'   V P                  f   Q R4       hV P                  V4      pVw  rïV P
                  '       du   V'       dl   Vfh   V P                   P                  '       d6   \        \!        V P                   R7      \!        V P                   R7      4      pM+\!        V P                   R7      pMV P
                  '       g   R pVe   VP#                  4       M^ pVf=   \%        4       '       g-   VV,           p\&        P(                  ! VVVP*                  R7      pV P
                  '       d   \-        V P                   VVVR7      pM>V P                   P.                  R8X  d"   \1        W P2                  VP*                  4      pMTpVe   \5        V P                   VVVR7      pV	'       d   RMR pV'       d   RMR pV'       d   V P
                  '       d   RMR pR pR pV P7                  V4      p\9        V P:                  4       F›  w  ppV	'       d   VV3,           pV! VVVVVVVVVV
R7
      pV^ ,          pV^,          pV P
                  '       d   Ve
   V^,          pV'       g   Ke  VV^,          3,           pV P
                  '       g   KŠ  VV^,          3,           pK�  	  V P=                  V4      pV P7                  V4      pV	'       d   VV3,           pV
'       g7   \>        ;QJ d    . R VVVVV3 4       F  NK  	  5# ! R VVVVV3 4       4      # \A        VVVVVR7      # )NÚdecoder_Ú zYou cannot specify both zinput_ids and zinputs_embeds at the same timezYou have to specify either zinput_ids or Úinputs_embedszZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Fz<You have to initialize the model with valid token embeddings)rê   rŸ   )rê   rg  r%   rk  r  )rê   rg  r%   r  )rk  r  r©  r!  c              3   ó2   "  € T F  pVf   K	  Vx € K  	  R # 5irü   rS   )Ú.0r/  s   & r/   Ú	<genexpr>Ú&LongT5Stack.forward.<locals>.<genexpr>�  s"   é € ð 
ñ�Að ÷ ’óùs   ‚�
)Úlast_hidden_staterk  r½   Ú
attentionsÚcross_attentionsr8   rS   )!rê   r  r©  Úoutput_hidden_statesr!  r"  r  r:   r_  r3  r7   r+  r,  rY  Úis_encoder_decoderr   r
   r]  r   r+   rª   r`   r   r  rŒ   rE   r   r'   Ú	enumerater[  r\  rm   r   )rÌ   r-  r%   r  r  rg  rk  r  r©  ro  r!  r)   Úerr_msg_prefixrl  r«   rª  Úpast_key_values_lengthÚmask_seq_lengthÚcausal_maskÚall_hidden_statesÚall_attentionsÚall_cross_attentionsr(   r   r½   rq   Úlayer_moduleÚlayer_outputss   &&&&&&&&&&&,                r/   rÚ   ÚLongT5Stack.forwardö  s>  € ð "+Ò!6‘I¸D¿K¹K×<QÑ<Qˆ	Ø1BÒ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ò$DÑ È$Ï+É+×JjÑJjð 	ð &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆàÒ  ]Ò%>Ø+/¯?¯?¨?™ZÀˆNÜØ*¨>Ð*:¸.ÈÐHXÐXvÐwóð ð Ò"Ø#Ÿ.™.Ó*ˆKØ!Ÿ™ r¨;°r­?Ó;‰IØÒ&Ø'×,Ñ,Ó.¨s°Ð3‰Kà+/¯?¯?¨?™ZÀˆNÜÐ:¸>Ð:JÈ-ÐXfÐWgÐgtÐuÓvÐvà×&×&Ð&¨4¯=¯=¨=ßÜ×#Ñ#Øpôð "�	àÒ Ø×$Ñ$Ò0ÐpÐ2pÓpÐ0Ø ×-Ñ-¨iÓ8ˆMà!,Ñˆ
à�?�?ˆ?ß˜_Ò4Ø—;‘;×1×1Ð1Ü&9Ü$¨D¯K©KÔ8¼,ÈdÏkÉkÔ:Zó'‘Oô '3¸$¿+¹+Ô&F�OøØ——�ð #ˆOàETÒE` ×!?Ñ!?Ô!AÐfgÐØÒ!Ô*B×*DÒ*Dà4°zÕAˆOÜ"ŸZšZ¨
°OÈM×L`ÑL`ÔaˆNà�?�?ˆ?Ü,Ø—{‘{Ø+Ø-Ø /ô	‰Kð �[‰[×/Ñ/°7Ô:Ü3°NÇNÁNÐTa×ThÑThÓi‰Kà(ˆKà!Ò-Ü%>Ø—{‘{Ø+Ø5Ø&;ô	&Ð"÷ #7™B¸DÐß0™°dˆß&7¸D¿O¿O¸O™rÐRVÐØˆØ(,Ð%àŸ™ ]Ó3ˆä(¨¯©Ö4‰OˆAˆ|ß#Ø$5¸Ð8HÕ$HÐ!á(ØØØØ%Ø&Ø-Ø /Ø#Ø"3Ø'ôˆMð  *¨!Õ,ˆMð *¨!Õ,ˆMØ��ˆÐ#8Ò#DØ0=¸aÕ0@Ð-ç Ñ Ø!/°=ÀÕ3CÐ2EÕ!E�Ø—?—?’?Ø+?À=ÐQRÕCSÐBUÕ+UÒ(ñC  5ðF ×-Ñ-¨mÓ<ˆØŸ™ ]Ó3ˆ÷  Ø 1°]Ð4DÕ DÐçß”5ñ 
ð "Ø#Ø%Ø"Ø(ñó
—5ð 
�5ñ 
ð "Ø#Ø%Ø"Ø(ñó
ó 
ð 
ô 9Ø+Ø+Ø+Ø%Ø1ô
ð 	
r1   )r[  rE   r'   rY  r\  r3  r"  rƒ  )
NNNNNNNNNN)
rÝ   rÞ   rß   rà   rÈ   rb  rÚ   rá   râ   rã   rä   s   @@r/   rV  rV  Ú  s   ù‡ € õò0+÷\
ô \
r1   rV  c                   ó|   a a€ ] tR tRt oR.tRRRR/tV3R lV 3R lltR tR	 t]	RV3R
 lR ll4       t
RtVtV ;t# )r9  i•  úFdecoder.block.0.layer.1.EncDecAttention.relative_attention_bias.weightúencoder.embed_tokens.weightúshared.weightúdecoder.embed_tokens.weightc                ó    <€ V ^8„  d   QhRS[ /# ré   r   )r.   rå   s   "€r/   r0   ÚLongT5Model.__annotate__Ÿ  s   ø€ ÷ ñ ™|ñ r1   c                óŠ  <€ \         SV `  V4       \        P                  ! VP                  VP
                  4      V n        \        P                  ! V4      pR Vn	        R Vn
        \        V4      V n        \        P                  ! V4      pRVn	        VP                  Vn        \        V4      V n        V P!                  4        R# )FTN)rÇ   rÈ   r   r1  rX  rñ   r=  ÚcopyÚdeepcopyr"  r  rV  ÚencoderÚnum_decoder_layersrZ  Údecoderr]  ©rÌ   rê   Úencoder_configÚdecoder_configrÏ   s   &&  €r/   rÈ   ÚLongT5Model.__init__Ÿ  s’   ø€ Ü‰Ñ˜Ô Ü—l’l 6×#4Ñ#4°f·n±nÓEˆŒäŸš vÓ.ˆØ$)ˆÔ!Ø#(ˆÔ Ü" >Ó2ˆŒäŸš vÓ.ˆØ$(ˆÔ!Ø$*×$=Ñ$=ˆÔ!Ü" >Ó2ˆŒð 	�‰Ör1   c                ó   € V P                   # rü   ©r=  ©rÌ   s   &r/   Úget_input_embeddingsÚ LongT5Model.get_input_embeddings°  ó   € Ø�{‰{Ðr1   c                ó~   € Wn         V P                  P                  V4       V P                  P                  V4       R # rü   ©r=  r†  rb  rˆ  r`  s   &&r/   rb  Ú LongT5Model.set_input_embeddings³  ó+   € Ø$ŒØ�‰×)Ñ)¨.Ô9Ø�‰×)Ñ)¨.Ö9r1   c                óð  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[S[ P                  ,          ,          R,          RS[R,          RS[ P                  R,          R	S[ P                  R,          R
S[R,          RS[R,          RS[R,          RS[R,          RS[S[ P                  ,          S[,          /# )r    r-  Nr%   r,  r.  Úencoder_outputsrk  rg  Údecoder_inputs_embedsr  r©  ro  r!  rG   )	r+   Ú
LongTensorÚFloatTensorÚ
BoolTensorrm   r	   r,   r  r   )r.   rå   s   "€r/   r0   r‚  ¹  s  ø€ ÷ q
ñ q
á×#Ñ# dÕ*ðq
ñ ×)Ñ)¨DÕ0ðq
ñ !×+Ñ+¨dÕ2ð	q
ñ
 !&× 0Ñ 0°4Õ 7ðq
ñ ™u¡U×%6Ñ%6Õ7Õ8¸4Õ?ðq
ñ  �ðq
ñ —|‘| dÕ*ðq
ñ  %Ÿ|™|¨dÕ2ðq
ñ ˜$•;ðq
ñ   $�;ðq
ñ # T�kðq
ñ ˜D•[ðq
ñ 
‰u× Ñ Õ	!Ñ$6Õ	6ñq
r1   c                ó¢  € V	e   T	MV P                   P                  p	Ve   TMV P                   P                  pVf   V P                  VVVV
VVR7      pMeV'       d^   \	        V\
        4      '       gH   \        V^ ,          \        V4      ^8”  d
   V^,          MR\        V4      ^8”  d
   V^,          MRR7      pV^ ,          pV P                  VVVVVVV	V
VVR7
      pV'       g	   Wõ,           # \        VP                  VP                  VP                  VP                  VP                  VP                  VP                  VP                  R7      # )a�  
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
    Indices of input sequence tokens in the vocabulary. LongT5 is a model with relative position embeddings so
    you should be able to pad the inputs on both the right and the left.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for detail.

    [What are input IDs?](../glossary#input-ids)

    To know more on how to prepare `input_ids` for pretraining take a look a [LONGT5
    Training](./longt5#training).
decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Indices of decoder input sequence tokens in the vocabulary.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are decoder input IDs?](../glossary#decoder-input-ids)

    LONGT5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If
    `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
    `past_key_values`).

    To know more on how to prepare `decoder_input_ids` for pretraining take a look at [LONGT5
    Training](./longt5#training).
decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
    be used by default.

Example:

```python
>>> from transformers import AutoTokenizer, LongT5Model

>>> tokenizer = AutoTokenizer.from_pretrained("google/long-t5-local-base")
>>> model = LongT5Model.from_pretrained("google/long-t5-local-base")

>>> # Let's try a very long encoder input.
>>> input_ids = tokenizer(
...     100 * "Studies have been shown that owning a dog is good for you", return_tensors="pt"
... ).input_ids  # Batch size 1

>>> decoder_input_ids = tokenizer("Studies show that", return_tensors="pt").input_ids  # Batch size 1

>>> # forward pass
>>> outputs = model(input_ids=input_ids, decoder_input_ids=decoder_input_ids)
>>> last_hidden_states = outputs.last_hidden_state
```N©r-  r%   rg  r©  ro  r!  ©rl  r½   rm  ©
r-  r%   rg  rk  r  r  r  r©  ro  r!  )rl  rk  Údecoder_hidden_statesÚdecoder_attentionsrn  Úencoder_last_hidden_stater  Úencoder_attentions)rê   r  r!  r†  rý   r   Úlenrˆ  r   rl  rk  r½   rm  rn  )rÌ   r-  r%   r,  r.  r˜  rk  rg  r™  r  r©  ro  r!  r)   r½   Údecoder_outputss   &&&&&&&&&&&&&,  r/   rÚ   ÚLongT5Model.forward¸  sP  € ðD "+Ò!6‘I¸D¿K¹K×<QÑ<Qˆ	Ø%0Ò%<‘kÀ$Ç+Á+×BYÑBYˆð Ò"Ø"Ÿl™lØ#Ø-Ø+Ø"3Ø%9Ø'ð +ó ‰O÷ ¤¨O¼_×!MÒ!MÜ-Ø"1°!Õ"4Ü47¸Ó4HÈ1Ô4L˜o¨aÖ0ÐRVÜ14°_Ó1EÈÔ1I˜?¨1Ö-ÈtôˆOð (¨Õ*ˆð Ÿ,™,Ø'Ø1Ø/Ø+Ø"/Ø#1ØØ/Ø!5Ø#ð 'ó 
ˆ÷ Ø"Õ4Ð4ä!Ø-×?Ñ?Ø+×;Ñ;Ø"1×"?Ñ"?Ø.×9Ñ9Ø,×=Ñ=Ø&5×&GÑ&GØ"1×"?Ñ"?Ø.×9Ñ9ô	
ð 		
r1   )rˆ  r†  r=  )NNNNNNNNNNNN)rÝ   rÞ   rß   rà   Ú"_keys_to_ignore_on_load_unexpectedÚ_tied_weights_keysrÈ   r�  rb  r   rÚ   rá   râ   rã   rä   s   @@r/   r9  r9  •  s]   ù‡ € ð 	Rð*Ð&ð 	& Ø% ðÐ÷
ó ò"ò:ð
 ÷q
ñ q
ó ÷q
ð q
r1   r9  z>
    LONGT5 Model with a `language modeling` head on top.
    )Úcustom_introc                   ó’   a a€ ] tR tRt oR.tRRRRRR/tV3R lV 3R lltR	 tR
 t]	RV3R lR ll4       t
V3R lR ltRtVtV ;t# )r:  i-  r}  r~  r  r€  zlm_head.weightc                ó    <€ V ^8„  d   QhRS[ /# ré   r   )r.   rå   s   "€r/   r0   Ú+LongT5ForConditionalGeneration.__annotate__<  s   ø€ ÷ ñ ™|ñ r1   c                ó  <€ \         SV `  V4       VP                  V n        \        P
                  ! VP                  VP                  4      V n        \        P                  ! V4      pR Vn
        R Vn        \        V4      V n        \        P                  ! V4      pRVn
        VP                  Vn        \        V4      V n        \        P"                  ! VP                  VP                  R R7      V n        V P'                  4        R# )FTrî   N)rÇ   rÈ   rñ   Ú	model_dimr   r1  rX  r=  r„  r…  r"  r  rV  r†  r‡  rZ  rˆ  rð   r4  r]  r‰  s   &&  €r/   rÈ   Ú'LongT5ForConditionalGeneration.__init__<  s»   ø€ Ü‰Ñ˜Ô ØŸ™ˆŒä—l’l 6×#4Ñ#4°f·n±nÓEˆŒäŸš vÓ.ˆØ$)ˆÔ!Ø#(ˆÔ Ü" >Ó2ˆŒäŸš vÓ.ˆØ$(ˆÔ!Ø$*×$=Ñ$=ˆÔ!Ü" >Ó2ˆŒä—y’y §¡°×1BÑ1BÈÔOˆŒð 	�‰Ör1   c                ó   € V P                   # rü   rŽ  r�  s   &r/   r�  Ú3LongT5ForConditionalGeneration.get_input_embeddingsQ  r’  r1   c                ó~   € Wn         V P                  P                  V4       V P                  P                  V4       R # rü   r”  r`  s   &&r/   rb  Ú3LongT5ForConditionalGeneration.set_input_embeddingsT  r–  r1   c                ó  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[S[ P
                  ,          ,          R,          RS[R,          RS[ P                  R,          R	S[ P                  R,          R
S[ P                  R,          RS[R,          RS[R,          RS[R,          RS[R,          RS[S[ P                  ,          S[,          /# )r    r-  Nr%   r,  r.  r˜  rk  rg  r™  Úlabelsr  r©  ro  r!  rG   )	r+   rš  r›  rœ  rm   r,   r	   r  r   )r.   rå   s   "€r/   r0   r­  Z  s+  ø€ ÷ J
ñ J
á×#Ñ# dÕ*ðJ
ñ ×)Ñ)¨DÕ0ðJ
ñ !×+Ñ+¨dÕ2ð	J
ñ
 !&× 0Ñ 0°4Õ 7ðJ
ñ ™u¡U§\¡\Õ2Õ3°dÕ:ðJ
ñ  �ðJ
ñ ×(Ñ(¨4Õ/ðJ
ñ  %×0Ñ0°4Õ7ðJ
ñ × Ñ  4Õ'ðJ
ñ ˜$•;ðJ
ñ   $�;ðJ
ñ # T�kðJ
ñ ˜D•[ðJ
ñ  
‰u× Ñ Õ	!¡OÕ	3ñ!J
r1   c                ór  € V
e   T
MV P                   P                  p
Ve   TMV P                   P                  pVf   V P                  VVVVVVR7      pMeV'       d^   \	        V\
        4      '       gH   \        V^ ,          \        V4      ^8”  d
   V^,          MR\        V4      ^8”  d
   V^,          MRR7      pV^ ,          pV	e   Vf   Vf   V P                  V	4      pV P                  VVVVVVV
VVVR7
      pV^ ,          pV P                   P                  '       d   VV P                  R,          ,          pV P                  V4      pRpV	e_   \        R	R7      pV	P                  VP                  4      p	V! VP                  R
VP!                  R
4      4      V	P                  R
4      4      pV'       g)   V3VR,          ,           V,           pVe   V3V,           # T# \#        VVVP$                  VP&                  VP(                  VP*                  VP,                  VP&                  VP(                  R7	      # )aÿ	  
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
    Indices of input sequence tokens in the vocabulary. LongT5 is a model with relative position embeddings so
    you should be able to pad the inputs on both the right and the left.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for detail.

    [What are input IDs?](../glossary#input-ids)

    To know more on how to prepare `input_ids` for pretraining take a look a [LONGT5
    Training](./longt5#training).
decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Indices of decoder input sequence tokens in the vocabulary.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are decoder input IDs?](../glossary#decoder-input-ids)

    LONGT5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If
    `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
    `past_key_values`).

    To know more on how to prepare `decoder_input_ids` for pretraining take a look at [LONGT5
    Training](./longt5#training).
decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
    be used by default.
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the sequence classification/regression loss. Indices should be in `[-100, 0, ...,
    config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for
    labels in `[0, ..., config.vocab_size]`

Examples:

```python
>>> from transformers import AutoTokenizer, LongT5ForConditionalGeneration

>>> tokenizer = AutoTokenizer.from_pretrained("Stancld/longt5-tglobal-large-16384-pubmed-3k_steps")
>>> model = LongT5ForConditionalGeneration.from_pretrained(
...     "Stancld/longt5-tglobal-large-16384-pubmed-3k_steps"
... )

>>> # Let's try a very long input.
>>> inputs = tokenizer(100 * "studies have shown that owning a dog is good for you ", return_tensors="pt")
>>> input_ids = inputs.input_ids

>>> outputs = model.generate(input_ids)
>>> print(tokenizer.decode(outputs[0], skip_special_tokens=True))
abstractthe aim of this article is to provide an overview of the literature on the role of dog
```Nrž  rŸ  r   )Úignore_indexrð  )	ÚlossÚlogitsrk  r¡  r¢  rn  r£  r  r¤  r9   rD  r8   )rê   r  r!  r†  rý   r   r¥  rJ  rˆ  Úscale_decoder_outputsr¯  r4  r   r‚   r`   r_  r:   r   rk  r½   rm  rn  rl  )rÌ   r-  r%   r,  r.  r˜  rk  rg  r™  r¶  r  r©  ro  r!  r)   r½   r¦  Úsequence_outputÚ	lm_logitsr¹  Úloss_fctÚoutputs   &&&&&&&&&&&&&&,       r/   rÚ   Ú&LongT5ForConditionalGeneration.forwardY  s!  € ðL "+Ò!6‘I¸D¿K¹K×<QÑ<Qˆ	Ø%0Ò%<‘kÀ$Ç+Á+×BYÑBYˆð Ò"à"Ÿl™lØ#Ø-Ø+Ø"3Ø%9Ø'ð +ó ‰O÷ ¤¨O¼_×!MÒ!MÜ-Ø"1°!Õ"4Ü47¸Ó4HÈ1Ô4L˜o¨aÖ0ÐRVÜ14°_Ó1EÈÔ1I˜?¨1Ö-ÈtôˆOð (¨Õ*ˆàÒÐ"3Ò";Ð@UÒ@]à $× 1Ñ 1°&Ó 9Ðð Ÿ,™,Ø'Ø1Ø/Ø+Ø"/Ø#1ØØ/Ø!5Ø#ð 'ó 
ˆð *¨!Õ,ˆà�;‰;×,×,Ð,Ø-°·±ÀÕ1EÕFˆOà—L‘L Ó1ˆ	àˆØÒÜ'°TÔ:ˆHà—Y‘Y˜y×/Ñ/Ó0ˆFÙ˜IŸN™N¨2¨y¯~©~¸bÓ/AÓBÀFÇKÁKÐPRÃOÓTˆDçØ�\ O°BÕ$7Õ7¸/ÕIˆFØ)-Ò)9�T�G˜fÕ$ÐE¸vÐEäØØØ+×;Ñ;Ø"1×"?Ñ"?Ø.×9Ñ9Ø,×=Ñ=Ø&5×&GÑ&GØ"1×"?Ñ"?Ø.×9Ñ9ô

ð 
	
r1   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r    r¶  r”   )r.   rå   s   "€r/   r0   r­  æ  s   ø€ ÷ )ñ )¹E¿L¹Lñ )r1   c                ó$   € V P                  V4      # rü   )rJ  )rÌ   r¶  s   &&r/   Ú%prepare_decoder_input_ids_from_labelsÚDLongT5ForConditionalGeneration.prepare_decoder_input_ids_from_labelsæ  s   € Ø× Ñ  Ó(Ð(r1   )rˆ  r†  r4  r¯  r=  )NNNNNNNNNNNNN)rÝ   rÞ   rß   rà   r¨  r©  rÈ   r�  rb  r   rÚ   rÃ  rá   râ   rã   rä   s   @@r/   r:  r:  -  sl   ù‡ € ð 	Rð*Ð&ð 	& Ø% Ø˜/ðÐ÷ó ò*ò:ð
 ÷J
ñ J
ó ðJ
÷X)÷ )ð )r1   r:  c                   óx   a a€ ] tR tRt oRR/tR.tV3R lV 3R lltR tR t]	RV3R	 lR
 ll4       t
RtVtV ;t# )r;  iê  r~  r  rˆ  c                ó    <€ V ^8„  d   QhRS[ /# ré   r   )r.   rå   s   "€r/   r0   ÚLongT5EncoderModel.__annotate__ñ  s   ø€ ÷ 	ñ 	™|ñ 	r1   c                ó   <€ \         SV `  V4       \        P                  ! VP                  VP
                  4      V n        \        P                  ! V4      pR Vn	        \        V4      V n        V P                  4        R# râ  )rÇ   rÈ   r   r1  rX  rñ   r=  r„  r…  r  rV  r†  r]  )rÌ   rê   rŠ  rÏ   s   && €r/   rÈ   ÚLongT5EncoderModel.__init__ñ  sZ   ø€ Ü‰Ñ˜Ô Ü—l’l 6×#4Ñ#4°f·n±nÓEˆŒäŸš vÓ.ˆØ#(ˆÔ Ü" >Ó2ˆŒð 	�‰Ör1   c                ó   € V P                   # rü   rŽ  r�  s   &r/   r�  Ú'LongT5EncoderModel.get_input_embeddingsü  r’  r1   c                óH   € Wn         V P                  P                  V4       R # rü   )r=  r†  rb  r`  s   &&r/   rb  Ú'LongT5EncoderModel.set_input_embeddingsÿ  s   € Ø$ŒØ�‰×)Ñ)¨.Ö9r1   c                ó  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[R,          RS[S[ P                  ,          S[,          /# )	r    r-  Nr%   rg  r©  ro  r!  rG   )r+   rš  r›  r  rm   r   )r.   rå   s   "€r/   r0   rÇ    sŽ   ø€ ÷ -ñ -á×#Ñ# dÕ*ð-ñ ×)Ñ)¨DÕ0ð-ñ ×(Ñ(¨4Õ/ð	-ñ
   $�;ð-ñ # T�kð-ñ ˜D•[ð-ñ 
‰u× Ñ Õ	!¡OÕ	3ñ-r1   c           	     ól   € Ve   TMV P                   P                  pV P                  VVVVVVR7      pV# )aú  
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
    Indices of input sequence tokens in the vocabulary. LongT5 is a model with relative position embeddings so
    you should be able to pad the inputs on both the right and the left.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for detail.

    To know more on how to prepare `input_ids` for pretraining take a look a [LONGT5
    Training](./longt5#training).

Example:

```python
>>> from transformers import AutoTokenizer, LongT5ForConditionalGeneration

>>> tokenizer = AutoTokenizer.from_pretrained("google/long-t5-local-base")
>>> model = LongT5EncoderModel.from_pretrained("google/long-t5-local-base")
>>> input_ids = tokenizer(
...     100 * "Studies have been shown that owning a dog is good for you ", return_tensors="pt"
... ).input_ids  # Batch size 1
>>> outputs = model(input_ids=input_ids)
>>> last_hidden_states = outputs.last_hidden_state
```rž  )rê   r!  r†  )	rÌ   r-  r%   rg  r©  ro  r!  r)   r˜  s	   &&&&&&&, r/   rÚ   ÚLongT5EncoderModel.forward  sH   € ðF &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆàŸ,™,ØØ)Ø'Ø/Ø!5Ø#ð 'ó 
ˆð Ðr1   )r†  r=  )NNNNNN)rÝ   rÞ   rß   rà   r©  r¨  rÈ   r�  rb  r   rÚ   rá   râ   rã   rä   s   @@r/   r;  r;  ê  sP   ù‡ € ð 	& ðÐð +5¨Ð&÷	ó 	òò:ð ÷-ñ -ó ÷-ð -r1   r;  )Nr¡   N)rR   )r;  r:  r9  r)  )Nrô  r„  r<  Úcollections.abcr   Útypingr   r+   r   Útorch.nnr   rf  r   r7  Úactivationsr   Úcache_utilsr	   r
   r   Ú
generationr   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   r   r   Úconfiguration_longt5r   Ú
get_loggerrÝ   r+  rB   r\   re   rs   r|   r…   rŒ   r³   r»   rÃ   r*   rÅ   rç   r  r  r  r}  r±  rÔ  rå  rö  r   r  r)  rV  r9  r:  r;  Ú__all__rS   r1   r/   Ú<module>rà     s  ðñ ã Û Ý $Ý ã Ý Ý %å &Ý !ß CÑ CÝ )ß JÝ 9÷ó ÷ GÝ &÷÷ õ /ð 
×	Ò	˜HÓ	%€÷%÷Fõ #÷4õ2!õBõ8õ .Põb4õ	jô+�b—i‘iô +ô4˜"Ÿ)™)ô ô,˜rŸy™yô ô&�B—I‘Iô ô&A8�b—i‘iô A8ôHi˜2Ÿ9™9ô iôXl R§Y¡Yô lô`=˜rŸy™yô =ô@ B§I¡Iô ô:¨b¯i©iô ô@9 §	¡	ô 9ô>T
Ð,ô T
ðn ôS!˜Oó S!ó ðS!ôlx
Ð'ô x
ðv ôT
Ð'ó T
ó ðT
ñn ðôô
u)Ð%:¸Oó u)óð
u)ðp ôFÐ.ó Fó ðFòR k‚r1   