+
    œ°³j� ã                   óä  € R t ^ RIt^ RIt^ RIHt ^ RIt^ RIHt ^ RIHtH	t	H
t
 ^RIHt ^RIHt ^RIHtHtHt ^RIHt ^R	IHtHt ^R
IHt ^RIHtHtHtHtH t H!t!H"t" ^RI#H$t$H%t% ^RI&H't' ^RI(H)t)H*t*H+t+H,t,H-t-H.t. ^RI/H0t0H1t1 ^RI2H3t3H4t4 ^RI5H6t6 ]-Pn                  ! ]84      t9 ! R R]Pt                  4      t; ! R R]Pt                  4      t< ! R R]Pt                  4      t= ! R R]Pt                  4      t>R9R R llt? ! R R]Pt                  4      t@ ! R R]Pt                  4      tA ! R  R!]Pt                  4      tB ! R" R#]4      tC ! R$ R%]Pt                  4      tD], ! R& R']%4      4       tE ! R( R)]E4      tF], ! R* R+]E4      4       tG],! R,R-7       ! R. R/]E]4      4       tH], ! R0 R1]E4      4       tI],! R2R-7       ! R3 R4]E4      4       tJ], ! R5 R6]E4      4       tK], ! R7 R8]E4      4       tL. R:OtMR# );zPyTorch UMT5 model.N)ÚCallable)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELoss)Úinitialization)ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚ)BaseModelOutputWithPastAndCrossAttentionsÚSeq2SeqLMOutputÚSeq2SeqModelOutputÚ#Seq2SeqQuestionAnsweringModelOutputÚSeq2SeqSequenceClassifierOutputÚTokenClassifierOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚDUMMY_INPUTSÚ
DUMMY_MASKÚTransformersKwargsÚauto_docstringÚloggingÚtorch_compilable_check)Úcan_return_tupleÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputs©Ú
UMT5Configc                   ó<   a a€ ] tR t^:t oRV 3R lltR tRtVtV ;t# )ÚUMT5LayerNormc                óŽ   <€ \         SV `  4        \        P                  ! \        P
                  ! V4      4      V n        W n        R# )zU
Construct a layernorm module in the UMT5 style. No bias and no subtraction of mean.
N)ÚsuperÚ__init__r   Ú	ParameterÚtorchÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizeÚepsÚ	__class__s   &&&€Úg/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/umt5/modeling_umt5.pyr*   ÚUMT5LayerNorm.__init__;   s/   ø€ ô 	‰ÑÔÜ—l’l¤5§:¢:¨kÓ#:Ó;ˆŒØ #Öó    c                ó¼  € VP                  \        P                  4      P                  ^4      P	                  RRR7      pV\        P
                  ! W P                  ,           4      ,          pV P                  P                  \        P                  \        P                  39   d&   VP                  V P                  P                  4      pV P                  V,          # )é   T)Úkeepdiméÿÿÿÿ)Útor,   Úfloat32ÚpowÚmeanÚrsqrtr/   r.   ÚdtypeÚfloat16Úbfloat16)r0   Úhidden_statesÚvariances   && r4   ÚforwardÚUMT5LayerNorm.forwardC   s™   € ð !×#Ñ#¤E§M¡MÓ2×6Ñ6°qÓ9×>Ñ>¸rÈ4Ð>ÓPˆØ%¬¯ª°H×?TÑ?TÕ4TÓ(UÕUˆð �;‰;×Ñ¤§¡´·±Ð ?Ô?Ø)×,Ñ,¨T¯[©[×->Ñ->Ó?ˆMà�{‰{˜]Õ*Ð*r6   )r/   r.   )g�íµ ÷Æ°>©	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r*   rE   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r3   Ú__classdict__s   @@r4   r'   r'   :   s   ù‡ € ÷$÷+ò +r6   r'   c                   óD   a a€ ] tR t^Tt oV3R lV 3R lltR tRtVtV ;t# )ÚUMT5DenseActDensec                ó    <€ V ^8„  d   QhRS[ /# ©r8   Úconfigr$   )ÚformatrP   s   "€r4   Ú__annotate__ÚUMT5DenseActDense.__annotate__U   s   ø€ ÷ /ñ /™zñ /r6   c                ón  <€ \         SV `  4        \        P                  ! VP                  VP
                  R R7      V n        \        P                  ! VP
                  VP                  R R7      V n        \        P                  ! VP                  4      V n
        \        VP                  ,          V n        R# ©F©ÚbiasN)r)   r*   r   ÚLinearÚd_modelÚd_ffÚwiÚwoÚDropoutÚdropout_rateÚdropoutr   Údense_act_fnÚact©r0   rU   r3   s   &&€r4   r*   ÚUMT5DenseActDense.__init__U   sn   ø€ Ü‰ÑÔÜ—)’)˜FŸN™N¨F¯K©K¸eÔDˆŒÜ—)’)˜FŸK™K¨¯©¸eÔDˆŒÜ—z’z &×"5Ñ"5Ó6ˆŒÜ˜&×-Ñ-Õ.ˆŽr6   c                ó  € V P                  V4      pV P                  V4      pV P                  V4      p\        V P                  P
                  \        P                  4      '       d’   VP                  V P                  P
                  P                  8w  dc   V P                  P
                  P                  \        P                  8w  d0   VP                  V P                  P
                  P                  4      pV P	                  V4      pV# ©N)r`   rf   rd   Ú
isinstancera   r.   r,   ÚTensorr@   Úint8r;   ©r0   rC   s   &&r4   rE   ÚUMT5DenseActDense.forward\   s­   € ØŸ™ Ó.ˆØŸ™ Ó/ˆØŸ™ ]Ó3ˆä�t—w‘w—~‘~¤u§|¡|×4Ò4Ø×#Ñ# t§w¡w§~¡~×';Ñ';Ô;Ø—‘—‘×$Ñ$¬¯
©
Ô2à)×,Ñ,¨T¯W©W¯^©^×-AÑ-AÓBˆMØŸ™ Ó.ˆØÐr6   )rf   rd   r`   ra   rG   rO   s   @@r4   rR   rR   T   s   ù‡ € ÷/ó /÷ò r6   rR   c                   óD   a a€ ] tR t^kt oV3R lV 3R lltR tRtVtV ;t# )ÚUMT5DenseGatedActDensec                ó    <€ V ^8„  d   QhRS[ /# rT   r$   )rV   rP   s   "€r4   rW   Ú#UMT5DenseGatedActDense.__annotate__l   s   ø€ ÷ /ñ /™zñ /r6   c                óÒ  <€ \         SV `  4        \        P                  ! VP                  VP
                  R R7      V n        \        P                  ! VP                  VP
                  R R7      V n        \        P                  ! VP
                  VP                  R R7      V n        \        P                  ! VP                  4      V n        \        VP                  ,          V n        R# rZ   )r)   r*   r   r]   r^   r_   Úwi_0Úwi_1ra   rb   rc   rd   r   re   rf   rg   s   &&€r4   r*   ÚUMT5DenseGatedActDense.__init__l   sŠ   ø€ Ü‰ÑÔÜ—I’I˜fŸn™n¨f¯k©kÀÔFˆŒ	Ü—I’I˜fŸn™n¨f¯k©kÀÔFˆŒ	Ü—)’)˜FŸK™K¨¯©¸eÔDˆŒÜ—z’z &×"5Ñ"5Ó6ˆŒÜ˜&×-Ñ-Õ.ˆŽr6   c                óF  € V P                  V P                  V4      4      pV P                  V4      pW#,          pV P                  V4      p\	        V P
                  P                  \        P                  4      '       d’   VP                  V P
                  P                  P                  8w  dc   V P
                  P                  P                  \        P                  8w  d0   VP                  V P
                  P                  P                  4      pV P                  V4      pV# rj   )rf   ru   rv   rd   rk   ra   r.   r,   rl   r@   rm   r;   )r0   rC   Úhidden_geluÚhidden_linears   &&  r4   rE   ÚUMT5DenseGatedActDense.forwardt   sÀ   € Ø—h‘h˜tŸy™y¨Ó7Ó8ˆØŸ	™	 -Ó0ˆØ#Õ3ˆØŸ™ ]Ó3ˆô �t—w‘w—~‘~¤u§|¡|×4Ò4Ø×#Ñ# t§w¡w§~¡~×';Ñ';Ô;Ø—‘—‘×$Ñ$¬¯
©
Ô2à)×,Ñ,¨T¯W©W¯^©^×-AÑ-AÓBˆMàŸ™ Ó.ˆØÐr6   )rf   rd   ru   rv   ra   rG   rO   s   @@r4   rq   rq   k   s   ù‡ € ÷/ó /÷ò r6   rq   c                   óD   a a€ ] tR t^‰t oV3R lV 3R lltR tRtVtV ;t# )ÚUMT5LayerFFc                ó    <€ V ^8„  d   QhRS[ /# rT   r$   )rV   rP   s   "€r4   rW   ÚUMT5LayerFF.__annotate__Š   s   ø€ ÷ 7ñ 7™zñ 7r6   c                ó   <€ \         SV `  4        VP                  '       d   \        V4      V n        M\        V4      V n        \        VP                  VP                  R 7      V n	        \        P                  ! VP                  4      V n        R# )©r2   N)r)   r*   Úis_gated_actrq   ÚDenseReluDenserR   r'   r^   Úlayer_norm_epsilonÚ
layer_normr   rb   rc   rd   rg   s   &&€r4   r*   ÚUMT5LayerFF.__init__Š   sb   ø€ Ü‰ÑÔØ××ÐÜ"8¸Ó"@ˆDÕä"3°FÓ";ˆDÔä'¨¯©¸F×<UÑ<UÔVˆŒÜ—z’z &×"5Ñ"5Ó6ˆŽr6   c                óx   € V P                  V4      pV P                  V4      pWP                  V4      ,           pV# rj   )r…   rƒ   rd   )r0   rC   Úforwarded_statess   && r4   rE   ÚUMT5LayerFF.forward”   s;   € ØŸ?™?¨=Ó9ÐØ×.Ñ.Ð/?Ó@ÐØ%¯©Ð5EÓ(FÕFˆØÐr6   )rƒ   rd   r…   rG   rO   s   @@r4   r}   r}   ‰   s   ù‡ € ÷7ó 7÷ò r6   r}   c                ó<  € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R,          R\        R	\        P                  R,          R
\
        \        ,          /	# )r8   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNÚscalingrd   Úposition_biasÚkwargs)r   ÚModuler,   rl   Úfloatr   r   )rV   s   "r4   rW   rW   œ   sŽ   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô �T�\ð%ô ð%ô —<‘< $Õ&ð%ô Ô'Õ(ñ%r6   c                 óÎ  € Vf   VP                  R4      R,          p\        P                  ! WP                  ^^4      4      V,          p	Ve	   W—,           p	Ve	   W”,           p	\        P
                  P                  V	RR7      p	\        P
                  P                  W–V P                  R7      p	\        P                  ! W“4      p
V
P                  ^^4      P                  4       p
W©3# )N©Údim)ÚpÚtrainingr:   ç      à¿)
Úsizer,   ÚmatmulÚ	transposer   Ú
functionalÚsoftmaxrd   r™   Ú
contiguous)r‹   rŒ   r�   rŽ   r�   r�   rd   r‘   r’   Úattn_weightsÚattn_outputs   &&&&&&&&,  r4   Úeager_attention_forwardr£   œ   s¿   € ð ‚Ø—*‘*˜R“. DÕ(ˆô —<’< §}¡}°Q¸Ó':Ó;¸gÕE€LàÒ Ø#Õ3ˆàÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2Ð(Ó>€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€Lä—,’,˜|Ó3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r6   c                   ó~   a a€ ] tR t^¼t oRtRV3R lV 3R llltV3R lR ltR tRR ltRV3R lR	 llt	R
t
VtV ;t# )ÚUMT5Attentionz/
T5's attention using relative_attention_bias.
c                ó4   <€ V ^8„  d   QhRS[ R,          RS[/# )r8   Ú	layer_idxNÚ	is_causal)ÚintÚbool)rV   rP   s   "€r4   rW   ÚUMT5Attention.__annotate__Á   s%   ø€ ÷ kñ kÙDGÈ$ÅJðkÙbfñkr6   c                ó2  <€ \         SV `  4        Wn        VP                  V n        W@n        W n        VP                  V n        VP                  V n        VP                  V n        VP                  V n
        VP                  V n        VP                  V n        V P                  V P                  ,          V n        R V n        W0n        Vf@   V P                  '       d.   \$        P'                  RV P(                  P*                   R24       \,        P.                  ! V P                  V P                  RR7      V n        \,        P.                  ! V P                  V P                  RR7      V n        \,        P.                  ! V P                  V P                  RR7      V n        \,        P.                  ! V P                  V P                  RR7      V n        V P
                  '       d3   \,        P8                  ! V P                  V P                  4      V n        R# R# )ç      ð?NzInstantiating a decoder z³ without passing `layer_idx` is not recommended and will to errors during the forward call, if caching is used. Please make sure to provide a `layer_idx` when creating this class.Fr[   )r)   r*   rU   Ú
is_decoderr¨   Úhas_relative_attention_biasÚrelative_attention_num_bucketsÚrelative_attention_max_distancer^   Úd_kvÚkey_value_proj_dimÚ	num_headsÚn_headsrc   rd   Ú	inner_dimr�   r§   ÚloggerÚwarning_oncer3   rH   r   r]   ÚqÚkÚvÚoÚ	EmbeddingÚrelative_attention_bias)r0   rU   r¯   r§   r¨   r3   s   &&&&&€r4   r*   ÚUMT5Attention.__init__Á   sx  ø€ ô 	‰ÑÔØŒØ ×+Ñ+ˆŒØ"ŒØ+FÔ(Ø.4×.SÑ.SˆÔ+Ø/5×/UÑ/UˆÔ,Ø—~‘~ˆŒØ"(§+¡+ˆÔØ×'Ñ'ˆŒØ×*Ñ*ˆŒØŸ™¨×(?Ñ(?Õ?ˆŒàˆŒØ"ŒØÒ §§ Ü×ÑØ*¨4¯>©>×+BÑ+BÐ*Cð D,ð ,ôô —’˜4Ÿ<™<¨¯©¸eÔDˆŒÜ—’˜4Ÿ<™<¨¯©¸eÔDˆŒÜ—’˜4Ÿ<™<¨¯©¸eÔDˆŒÜ—’˜4Ÿ>™>¨4¯<©<¸eÔDˆŒà×+×+Ð+Ü+-¯<ª<¸×8[Ñ8[Ð]a×]iÑ]iÓ+jˆDÖ(ñ ,r6   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r8   Ú
projectionÚreturn©r,   rl   )rV   rP   s   "€r4   rW   r«   â   s#   ø€ ÷ ñ ¡§¡ð ±%·,±,ñ r6   c                ó¬   € VP                  4       R R V P                  V P                  3,           pVP                  V4      P	                  ^ ^^^4      pV# )Nr:   )r›   rµ   r³   ÚviewÚpermute)r0   rÁ   Únew_projection_shapeÚnew_projections   &&  r4   Ú_shapeÚUMT5Attention._shapeâ   sQ   € Ø)Ÿ™Ó0°°"Ð5¸¿¹Àt×G^ÑG^Ð8_Õ_Ðà#Ÿ™Ð)=Ó>×FÑFÀqÈ!ÈQÐPQÓRˆØÐr6   c                ó  € ^ pV P                   pV P                  pV P                  '       gP   V^,          pW!^ 8„  P                  \        P
                  4      V,          ,          p\        P                  ! V4      pM,\        P                  ! V\        P                  ! V4      4      ) pV^,          pW8  p\        P                  ! VP                  4       V,          4      \        P                  ! WE,          4      ,          pWsV,
          ,          pWWP                  \        P
                  4      ,           p\        P                  ! V\        P                  ! Wƒ^,
          4      4      pV\        P                  ! WaV4      ,          pV# )aR  
Adapted from Mesh Tensorflow:
https://github.com/tensorflow/mesh/blob/0cb87fe07da627bf0b7e60475d59f95ed6b5be3d/mesh_tensorflow/transformer/transformer_layers.py#L593

Translate relative position to a bucket number for relative attention. The relative position is defined as
memory_position - query_position, i.e. the distance in tokens from the attending position to the attended-to
position. If bidirectional=False, then positive relative positions are invalid. We use smaller buckets for
small absolute relative_position and larger buckets for larger absolute relative_positions. All relative
positions >=max_distance map to the same bucket. All relative positions <=-max_distance map to the same bucket.
This should allow for more graceful generalization to longer sequences than the model has been trained on

Args:
    relative_position: an int32 Tensor
    bidirectional: a boolean - whether the attention is bidirectional
    num_buckets: an integer
    max_distance: an integer

Returns:
    a Tensor with the same shape as relative_position, containing int32 values in the range [0, num_buckets)
)r°   r±   r®   r;   r,   ÚlongÚabsÚminÚ
zeros_likeÚlogr”   ÚmathÚ	full_likeÚwhere)	r0   Úrelative_positionÚrelative_bucketsÚnum_bucketsÚmax_distanceÚ	max_exactÚis_smallÚ	log_ratioÚrelative_position_if_larges	   &&       r4   Ú_relative_position_bucketÚ'UMT5Attention._relative_position_bucketè   s1  € ð* ÐØ×9Ñ9ˆØ×;Ñ;ˆØ��ˆØ˜AÕˆKØ°QÑ!6× :Ñ :¼5¿:¹:Ó FÈÕ TÕTÐÜ %§	¢	Ð*;Ó <Ñä!&§¢Ð+<¼e×>NÒ>NÐO`Ó>aÓ!bÐ bÐð   1Õ$ˆ	Ø$Ñ0ˆô —I’IÐ/×5Ñ5Ó7¸)ÕCÓDÄtÇxÂxÐP\ÕPhÓGiÕiˆ	Ø¨yÕ!8Õ9ˆ	Ø%.·±¼e¿j¹jÓ1IÕ%IÐ"Ü%*§Y¢YØ&¬¯ªÐ8RÐbcÕTcÓ(dó&
Ð"ð 	œEŸKšK¨ÐE_Ó`Õ`ÐØÐr6   c                ó¬  € Vf!   V P                   P                  P                  p\        P                  ! V\        P
                  VR7      R,          V,           p\        P                  ! V\        P
                  VR7      R,          pWe,
          pV P                  V4      pV P                  V4      p	V	P                  . RO4      P                  ^ 4      p	V	# )z%Compute binned relative position bias)r@   Údevice)ºNNNN)Nrà   )r8   é    é   )	r¾   r.   rß   r,   ÚarangerÌ   rÜ   rÆ   Ú	unsqueeze)
r0   Úquery_lengthÚ
key_lengthrß   Úpast_seen_tokensÚcontext_positionÚmemory_positionrÔ   Úrelative_position_bucketÚvaluess
   &&&&&     r4   Úcompute_biasÚUMT5Attention.compute_bias  s§   € àŠ>Ø×1Ñ1×8Ñ8×?Ñ?ˆFÜ Ÿ<š<¨¼E¿J¹JÈvÔVÐW^Õ_ÐbrÕrÐÜŸ,š, z¼¿¹ÈFÔSÐT[Õ\ˆØ+Õ>ÐØ#'×#AÑ#AÐBSÓ#TÐ Ø×-Ñ-Ð.FÓGˆØ—‘¢	Ó*×4Ñ4°QÓ7ˆØˆr6   c          	      ó˜   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[R,          RS[ P                  R,          /# )r8   rC   Úencoder_hidden_statesNÚpast_key_valuesr�   )r,   rl   r	   )rV   rP   s   "€r4   rW   r«   #  sS   ø€ ÷ N)ñ N)á—|‘|ðN)ñ  %Ÿ|™|¨dÕ2ðN)ñ  �ð	N)ñ
 Ÿ™ tÕ+ñN)r6   c           	     ó.  € VP                   R ,          w  rgVe   VP                  V P                  4      M^ p\        V\        P
                  4      '       d   VP                  4       MTpVRJp	V P                  V4      p
V
P                  VR
V P                  V P                  4      P                  ^^4      p
RpVe^   \        V\        4      '       dH   VP                  P                  V P                  4      pV	'       d   VP                  pMVP                   pMTpV	'       d   TMTpV	'       d\   VeX   V'       dP   VP"                  V P                  ,          P$                  pVP"                  V P                  ,          P&                  pMëV P)                  V4      pV P+                  V4      pVP                  VR
V P                  V P                  4      P                  ^^4      pVP                  VR
V P                  V P                  4      P                  ^^4      pVeV   VP-                  WïV P                  4      w  rïV	'       d0   \        V\        4      '       d   RVP                  V P                  &   VP                   R,          pV P.                  '       g=   \        P0                  ! ^V P                  VV3V
P2                  V
P4                  R7      pMV P7                  VVV
P2                  VR7      p\8        P:                  ! V P<                  P>                  \@        4      pT! V V
VVV3RV PB                  '       g   RMV PD                  RV PF                  R	V/VB w  ppVPI                  WgR
4      PK                  4       pV PM                  V4      pVV3# ):Nr8   NNFT)rß   r@   )rß   rç   rd   ç        r�   r‘   r:   éþÿÿÿ)'ÚshapeÚget_seq_lengthr§   rk   r,   rl   Úcloner¹   rÅ   rµ   r³   r�   r   Ú
is_updatedÚgetÚcross_attention_cacheÚself_attention_cacheÚlayersÚkeysrë   rº   r»   Úupdater¯   Úzerosrß   r@   rì   r   Úget_interfacerU   Ú_attn_implementationr£   r™   rd   r�   Úreshaper    r¼   )r0   rC   rï   rð   r�   r’   Ú
batch_sizeÚ
seq_lengthrç   Úis_cross_attentionÚquery_statesr÷   Úcurr_past_key_valuesÚcurrent_statesÚ
key_statesÚvalue_statesræ   r‘   Úattention_interfacer¢   r¡   s   &&&&&,               r4   rE   ÚUMT5Attention.forward#  s(  € ð "/×!4Ñ!4°RÕ!8Ñˆ
ØM\ÒMh˜?×9Ñ9¸$¿.¹.ÔIÐnoÐä7AÐBRÔTY×T`ÑT`×7aÒ7aÐ+×1Ñ1Ô3ÐgwÐð 3¸$Ð>Ðà—v‘v˜mÓ,ˆØ#×(Ñ(¨°R¸¿¹Àt×G^ÑG^Ó_×iÑiÐjkÐmnÓoˆð ˆ
ØÒ&¬:°oÔGZ×+[Ò+[Ø(×3Ñ3×7Ñ7¸¿¹ÓGˆJß!à'6×'LÑ'LÑ$à'6×'KÑ'KÑ$à#2Ð ç2DÑ.È-ˆß /Ò"=Ç*à-×4Ñ4°T·^±^ÕD×IÑIˆJØ/×6Ñ6°t·~±~ÕF×MÑM‰LàŸ™ Ó/ˆJØŸ6™6 .Ó1ˆLØ#Ÿ™¨°R¸¿¹Àt×G^ÑG^Ó_×iÑiÐjkÐmnÓoˆJØ'×,Ñ,¨Z¸¸T¿\¹\È4×KbÑKbÓc×mÑmÐnoÐqrÓsˆLàÒ*à+?×+FÑ+FÀzÐae×aoÑaoÓ+pÑ(�
ç%¬*°_ÔFY×*ZÒ*ZØAE�O×.Ñ.¨t¯~©~Ñ>ð  ×%Ñ% bÕ)ˆ
Ø×/×/Ð/Ü!ŸKšKØ�D—L‘L *¨jÐ9À,×BUÑBUÐ]i×]oÑ]oô‰Mð !×-Ñ-Ø˜J¨|×/BÑ/BÐUeð .ó ˆMô )@×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ
%
ð  $Ÿ}Ÿ}˜}‘C°$·,±,ð
%
ð —L‘Lð
%
ð (ð
%
ð ñ
%
Ñ!ˆ�\ð "×)Ñ)¨*À"ÓE×PÑPÓRˆØ—f‘f˜[Ó)ˆØ˜LÐ(Ð(r6   )rU   r^   rd   r¯   r¶   r¨   r®   rº   r³   r§   rµ   r¼   r¹   r¾   r±   r°   r�   r»   )FNF)Nrá   ©NNN)rH   rI   rJ   rK   Ú__doc__r*   rÉ   rÜ   rì   rE   rL   rM   rN   rO   s   @@r4   r¥   r¥   ¼   s>   ù‡ € ñ÷kõ k÷Bð ò- ô^
÷N)÷ N)ò N)r6   r¥   c                   óL   a a€ ] tR tRt oRV3R lV 3R llltRR ltRtVtV ;t# )ÚUMT5LayerSelfAttentionit  c                ó.   <€ V ^8„  d   QhRS[ R,          /# ©r8   r§   N©r©   )rV   rP   s   "€r4   rW   Ú#UMT5LayerSelfAttention.__annotate__u  ó   ø€ ÷ 7ñ 7©#°­*ñ 7r6   c                óà   <€ \         SV `  4        \        VR VR7      V n        \	        VP
                  VP                  R7      V n        \        P                  ! VP                  4      V n        R# )T©r¯   r§   r�   N)r)   r*   r¥   ÚSelfAttentionr'   r^   r„   r…   r   rb   rc   rd   ©r0   rU   r§   r3   s   &&&€r4   r*   ÚUMT5LayerSelfAttention.__init__u  sN   ø€ Ü‰ÑÔÜ*¨6ÈtÐ_hÔiˆÔÜ'¨¯©¸F×<UÑ<UÔVˆŒÜ—z’z &×"5Ñ"5Ó6ˆŽr6   c                ó´   € V P                  V4      pV P                  ! V3R VRV/VB pWP                  V^ ,          4      ,           pV3VR,          ,           pV# )r�   rð   ºrâ   NN)r…   r  rd   )r0   rC   r�   rð   r’   Únormed_hidden_statesÚattention_outputÚoutputss   &&&&,   r4   rE   ÚUMT5LayerSelfAttention.forward{  st   € ð  $Ÿ™¨}Ó=ÐØ×-Ò-Ø ñ
à)ð
ð ,ð
ð ñ	
Ðð &¯©Ð5EÀaÕ5HÓ(IÕIˆØ Ð"Ð%5°bÕ%9Õ9ˆØˆr6   )r  rd   r…   rj   )NNrG   rO   s   @@r4   r  r  t  s   ù‡ € ÷7õ 7÷ô r6   r  c                   óL   a a€ ] tR tRt oRV3R lV 3R llltRR ltRtVtV ;t# )ÚUMT5LayerCrossAttentioniŽ  c                ó.   <€ V ^8„  d   QhRS[ R,          /# r  r  )rV   rP   s   "€r4   rW   Ú$UMT5LayerCrossAttention.__annotate__�  r  r6   c                óà   <€ \         SV `  4        \        VR VR7      V n        \	        VP
                  VP                  R7      V n        \        P                  ! VP                  4      V n        R# )Fr  r�   N)r)   r*   r¥   ÚEncDecAttentionr'   r^   r„   r…   r   rb   rc   rd   r  s   &&&€r4   r*   Ú UMT5LayerCrossAttention.__init__�  sO   ø€ Ü‰ÑÔÜ,¨VÐQVÐbkÔlˆÔÜ'¨¯©¸F×<UÑ<UÔVˆŒÜ—z’z &×"5Ñ"5Ó6ˆŽr6   c           	     ó¸   € V P                  V4      pV P                  ! V3R VRVRV/VB pWP                  V^ ,          4      ,           pV3VR,          ,           p	V	# )rï   r�   rð   r  )r…   r%  rd   )
r0   rC   rï   r�   rð   r’   r  r  Úlayer_outputr  s
   &&&&&,    r4   rE   ÚUMT5LayerCrossAttention.forward•  s}   € ð  $Ÿ™¨}Ó=ÐØ×/Ò/Ø ñ
à"7ð
ð *ð
ð ,ð	
ð
 ñ
Ðð %§|¡|Ð4DÀQÕ4GÓ'HÕHˆØ�/Ð$4°RÕ$8Õ8ˆØˆr6   )r%  rd   r…   rj   r  rG   rO   s   @@r4   r!  r!  Ž  s   ù‡ € ÷7õ 7÷ô r6   r!  c                   óL   a a€ ] tR tRt oRV3R lV 3R llltRR ltRtVtV ;t# )Ú	UMT5Blockiª  c                ó.   <€ V ^8„  d   QhRS[ R,          /# r  r  )rV   rP   s   "€r4   rW   ÚUMT5Block.__annotate__«  s   ø€ ÷ /ñ /©#°­*ñ /r6   c                óz  <€ \         SV `  4        VP                  V n        \        P                  ! 4       V n        V P
                  P                  \        WR 7      4       V P                  '       d&   V P
                  P                  \        WR 7      4       V P
                  P                  \        V4      4       R# )©r§   N)
r)   r*   r®   r   Ú
ModuleListÚlayerÚappendr  r!  r}   r  s   &&&€r4   r*   ÚUMT5Block.__init__«  sx   ø€ Ü‰ÑÔØ ×+Ñ+ˆŒÜ—]’]“_ˆŒ
Ø�
‰
×ÑÔ0°ÔMÔNØ�?�?ˆ?Ø�J‰J×ÑÔ5°fÔRÔSà�
‰
×Ñœ+ fÓ-Ö.r6   c           	     óÎ  € V P                   ^ ,          ! V3RVRV/VB w  rVP                  \        P                  8X  d…   \        P                  ! VP                  4      P
                  p\        P                  ! \        P                  ! V4      P                  4       VR,
          V4      p	\        P                  ! W) V	R7      pV P                  ;'       d    VRJp
V
'       dÆ   V P                   ^,          ! V3RVRVRV/VB w  rVP                  \        P                  8X  d…   \        P                  ! VP                  4      P
                  p\        P                  ! \        P                  ! V4      P                  4       VR,
          V4      p	\        P                  ! W) V	R7      pV P                   R,          ! V4      pVP                  \        P                  8X  d…   \        P                  ! VP                  4      P
                  p\        P                  ! \        P                  ! V4      P                  4       VR,
          V4      p	\        P                  ! W) V	R7      pV# )rá   r�   rð   iè  )rÎ   ÚmaxNrï   r:   )r1  r@   r,   rA   Úfinfor5  rÓ   ÚisinfÚanyÚclampr®   )r0   rC   r�   rï   Úencoder_attention_maskrð   r’   Ú_Ú	max_dtypeÚclamp_valueÚdo_cross_attentions   &&&&&&,    r4   rE   ÚUMT5Block.forwardµ  sï  € ð  Ÿ:™: až=Øñ
à)ð
ð ,ð
ð ñ	
Ñˆð ×Ñ¤%§-¡-Ô/ÜŸš M×$7Ñ$7Ó8×<Ñ<ˆIÜŸ+š+¤e§k¢k°-Ó&@×&DÑ&DÓ&FÈ	ÐTXÕHXÐZcÓdˆKÜ!ŸKšK¨¸<È[ÔYˆMð "Ÿ_™_×RÐRÐ1FÈdÐ1RÐßØ#Ÿz™z¨!ž}Øñ à&;ð ð  6ð ð !0ð	 ð
 ñ ÑˆMð ×"Ñ"¤e§m¡mÔ3Ü!ŸKšK¨×(;Ñ(;Ó<×@Ñ@�	Ü#Ÿkšk¬%¯+ª+°mÓ*D×*HÑ*HÓ*JÈIÐX\ÕL\Ð^gÓh�Ü %§¢¨M¸|ÐQ\Ô ]�ð Ÿ
™
 2ž }Ó5ˆð ×Ñ¤%§-¡-Ô/ÜŸš M×$7Ñ$7Ó8×<Ñ<ˆIÜŸ+š+¤e§k¢k°-Ó&@×&DÑ&DÓ&FÈ	ÐTXÕHXÐZcÓdˆKÜ!ŸKšK¨¸<È[ÔYˆMàÐr6   )r®   r1  rj   ©NNNNrG   rO   s   @@r4   r+  r+  ª  s   ù‡ € ÷/õ /÷/ô /r6   r+  c                   óT   a a€ ] tR tRt oRtV3R lV 3R lltV3R lR ltRtVtV ;t	# )ÚUMT5ClassificationHeadiè  z-Head for sentence-level classification tasks.c                ó    <€ V ^8„  d   QhRS[ /# rT   r$   )rV   rP   s   "€r4   rW   Ú#UMT5ClassificationHead.__annotate__ë  s   ø€ ÷ Eñ E™zñ Er6   c                ó0  <€ \         SV `  4        \        P                  ! VP                  VP                  4      V n        \        P                  ! VP                  R 7      V n        \        P                  ! VP                  VP                  4      V n
        R# ))r˜   N)r)   r*   r   r]   r^   Údenserb   Úclassifier_dropoutrd   Ú
num_labelsÚout_projrg   s   &&€r4   r*   ÚUMT5ClassificationHead.__init__ë  sZ   ø€ Ü‰ÑÔÜ—Y’Y˜vŸ~™~¨v¯~©~Ó>ˆŒ
Ü—z’z F×$=Ñ$=Ô>ˆŒÜŸ	š	 &§.¡.°&×2CÑ2CÓDˆŽr6   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r8   rC   rÂ   rÃ   )rV   rP   s   "€r4   rW   rD  ñ  s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r6   c                óº   € V P                  V4      pV P                  V4      p\        P                  ! V4      pV P                  V4      pV P	                  V4      pV# rj   )rd   rF  r,   ÚtanhrI  rn   s   &&r4   rE   ÚUMT5ClassificationHead.forwardñ  sN   € ØŸ™ ]Ó3ˆØŸ
™
 =Ó1ˆÜŸ
š
 =Ó1ˆØŸ™ ]Ó3ˆØŸ™ mÓ4ˆØÐr6   )rF  rd   rI  )
rH   rI   rJ   rK   r  r*   rE   rL   rM   rN   rO   s   @@r4   rB  rB  è  s#   ù‡ € Ù7÷Eó E÷÷ ð r6   rB  c            
       óà   a a€ ] tR tRt oRtRtR.tR.tRtRt	Rt
RtR]! ]^ R7      R	]! ]RR7      R
]! ]RR7      /t]R 4       t]P(                  ! 4       V 3R l4       tR tV3R ltRtVtV ;t# )ÚUMT5PreTrainedModeliú  ÚtransformerTr+  ra   FrC   )ÚindexÚ
attentionsÚcross_attentionsc                ó~   € \         P                  ! \        4      p\         P                  ! \        4      pR VRVRV/pV# )Údecoder_input_idsÚ	input_idsÚdecoder_attention_mask)r,   Útensorr   r   )r0   rW  Ú
input_maskÚdummy_inputss   &   r4   r[  Ú UMT5PreTrainedModel.dummy_inputs  s<   € ä—L’L¤Ó.ˆ	Ü—\’\¤*Ó-ˆ
à Ø˜Ø$ jð
ˆð
 Ðr6   c                óÐ  <€ \         SV `  V4       V P                  P                  p\	        V\
        4      '       d+   \        P                  ! VP                  VR,          4       R# \	        V\        \        \        \        34      '       Ed%   \        P                  ! VP                  P                  RVR,          R7       \        VR4      '       dQ   V P                  P                   '       g5   \        P                  ! VP"                  P                  RVR,          R7       \        VR4      '       d{   \        P                  ! VP$                  P                  RW P                  P&                  R	,          ,          R7       \        P(                  ! VP$                  P*                  4       R# R# \	        V\,        4      '       du   \        VR4      '       da   \        P                  ! VP.                  P                  RVR,          R7       \        P(                  ! VP.                  P*                  4       R# R# \	        V\0        4      '       Ed_   \        P                  ! VP2                  P                  RW P                  P&                  R	,          ,          R7       \        VP2                  R4      '       dC   VP2                  P*                  e+   \        P(                  ! VP2                  P*                  4       \        P                  ! VP4                  P                  RW P                  P&                  R	,          ,          R7       \        VP4                  R4      '       dG   VP4                  P*                  e-   \        P(                  ! VP4                  P*                  4       R# R# R# \	        V\6        4      '       Ed_   \        P                  ! VP8                  P                  RW P                  P&                  R	,          ,          R7       \        VP8                  R4      '       dC   VP8                  P*                  e+   \        P(                  ! VP8                  P*                  4       \        P                  ! VP:                  P                  RW P                  P<                  R	,          ,          R7       \        VP:                  R4      '       dG   VP:                  P*                  e-   \        P(                  ! VP:                  P*                  4       R# R# R# \	        V\>        4      '       Ed   \        P                  ! VP@                  P                  RW P                  P&                  R	,          ,          R7       \        VP@                  R4      '       dC   VP@                  P*                  e+   \        P(                  ! VP@                  P*                  4       \        P                  ! VPB                  P                  RW P                  P&                  R	,          ,          R7       \        VPB                  R4      '       dC   VPB                  P*                  e+   \        P(                  ! VPB                  P*                  4       \        P                  ! VP:                  P                  RW P                  P<                  R	,          ,          R7       \        VP:                  R4      '       dG   VP:                  P*                  e-   \        P(                  ! VP:                  P*                  4       R# R# R# \	        V\D        4      '       Ed‰   V P                  P&                  pV P                  PF                  pV P                  PH                  p\        P                  ! VPJ                  P                  RW#V,          R	,          ,          R7       \        P                  ! VPL                  P                  RW#R	,          ,          R7       \        P                  ! VPN                  P                  RW#R	,          ,          R7       \        P                  ! VPP                  P                  RW%V,          R	,          ,          R7       VPR                  '       d=   \        P                  ! VPT                  P                  RW#R	,          ,          R7       R# R# R# )
zInitialize the weightsr­   rò   )r>   ÚstdÚlm_headÚ
qa_outputsÚ
classifierr\   Nrš   )+r)   Ú_init_weightsrU   Úinitializer_factorrk   r'   ÚinitÚ	constant_r.   Ú	UMT5ModelÚUMT5ForConditionalGenerationÚUMT5EncoderModelÚUMT5ForQuestionAnsweringÚnormal_ÚsharedÚhasattrÚtie_word_embeddingsr_  r`  r^   Úzeros_r\   ÚUMT5ForTokenClassificationra  rB  rF  rI  rR   r`   ra   r_   rq   ru   rv   r¥   r²   r´   r¹   rº   r»   r¼   r¯   r¾   )r0   r‹   Úfactorr^   r³   rµ   r3   s   &&    €r4   rb  Ú!UMT5PreTrainedModel._init_weights  sk  ø€ ô 	‰Ñ˜fÔ%Ø—‘×/Ñ/ˆÜ�fœm×,Ò,Ü�NŠN˜6Ÿ=™=¨&°3­,Ö7ÜØäÜ,Ü Ü(ð	÷
ó 
ô �LŠL˜Ÿ™×-Ñ-°C¸VÀc½\ÕJÜ�v˜y×)Ò)°$·+±+×2Q×2QÐ2QÜ—’˜VŸ^™^×2Ñ2¸À&È3Å,ÕOÜ�v˜|×,Ò,Ü—’˜V×.Ñ.×5Ñ5¸CÀV×P[ÑP[×PcÑPcÐhlÕOlÕEmÕnÜ—’˜F×-Ñ-×2Ñ2Ö3ñ -ô ˜Ô :×;Ò;Ü�v˜|×,Ò,Ü—’˜V×.Ñ.×5Ñ5¸CÀVÈcÅ\ÕRÜ—’˜F×-Ñ-×2Ñ2Ö3ñ -ô ˜Ô 6×7Ó7Ü�LŠL˜Ÿ™×,Ñ,°3¸FÇ{Á{×GZÑGZÐ_cÕFcÕ<dÕeÜ�v—|‘| V×,Ò,°·±×1BÑ1BÒ1NÜ—’˜FŸL™L×-Ñ-Ô.Ü�LŠL˜Ÿ™×/Ñ/°c¸vÏ+É+×J]ÑJ]ÐbfÕIfÕ?gÕhÜ�v—‘¨×/Ò/°F·O±O×4HÑ4HÒ4TÜ—’˜FŸO™O×0Ñ0Ö1ñ 5UÑ/ä˜Ô 1×2Ó2ô �LŠL˜Ÿ™×)Ñ)°¸ÇKÁK×DWÑDWÐ\`ÕC`Õ9aÕbÜ�v—y‘y &×)Ò)¨f¯i©i¯n©nÒ.HÜ—’˜FŸI™IŸN™NÔ+Ü�LŠL˜Ÿ™×)Ñ)°¸ÇKÁK×DTÑDTÐY]ÕC]Õ9^Õ_Ü�v—y‘y &×)Ò)¨f¯i©i¯n©nÒ.HÜ—’˜FŸI™IŸN™NÖ+ñ /IÑ)ä˜Ô 6×7Ó7Ü�LŠL˜Ÿ™×+Ñ+°#¸6ÇkÁk×FYÑFYÐ^bÕEbÕ;cÕdÜ�v—{‘{ F×+Ò+°·±×0@Ñ0@Ò0LÜ—’˜FŸK™K×,Ñ,Ô-Ü�LŠL˜Ÿ™×+Ñ+°#¸6ÇkÁk×FYÑFYÐ^bÕEbÕ;cÕdÜ�v—{‘{ F×+Ò+°·±×0@Ñ0@Ò0LÜ—’˜FŸK™K×,Ñ,Ô-Ü�LŠL˜Ÿ™×)Ñ)°¸ÇKÁK×DTÑDTÐY]ÕC]Õ9^Õ_Ü�v—y‘y &×)Ò)¨f¯i©i¯n©nÒ.HÜ—’˜FŸI™IŸN™NÖ+ñ /IÑ)ä˜¤×.Ó.ð —k‘k×)Ñ)ˆGØ!%§¡×!1Ñ!1ÐØ—k‘k×+Ñ+ˆGÜ�LŠL˜Ÿ™Ÿ™¨s¸ÐM_ÕC_ÐdhÕBhÕ8iÕjÜ�LŠL˜Ÿ™Ÿ™¨s¸È4Å-Õ8PÕQÜ�LŠL˜Ÿ™Ÿ™¨s¸È4Å-Õ8PÕQÜ�LŠL˜Ÿ™Ÿ™¨s¸ÐM_ÕC_ÐdhÕBhÕ8iÕjØ×1×1Ð1Ü—’˜V×;Ñ;×BÑBÈÐRXÐimÕ\mÕRn×oñ 2ñ /r6   c                ó>  € V P                   P                  pV P                   P                  pVf   \        R4      hVP	                  VP
                  4      pVRR R13,          P                  4       VR&   W$R&   Vf   \        R4      hVP                  VR8H  V4       V# )NzŽself.model.config.decoder_start_token_id has to be defined. In UMT5 it is usually set to the pad_token_id. See UMT5 docs for more information..z1self.model.config.pad_token_id has to be defined.r:   ).r  ).rá   éœÿÿÿ)rU   Údecoder_start_token_idÚpad_token_idÚ
ValueErrorÚ	new_zerosrô   rö   Úmasked_fill_)r0   rW  rt  ru  Úshifted_input_idss   &&   r4   Ú_shift_rightÚ UMT5PreTrainedModel._shift_right]  s§   € Ø!%§¡×!CÑ!CÐØ—{‘{×/Ñ/ˆà!Ò)Üð6óð ð
 &×/Ñ/°	·±Ó@ÐØ%.¨s°C°R°C¨xÕ%8×%>Ñ%>Ó%@Ð˜'Ñ"Ø$:˜&Ñ!àÒÜÐPÓQÐQà×&Ñ&Ð'8¸DÑ'@À,ÔOà Ð r6   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # rT   r$   )rV   rP   s   "€r4   rW   Ú UMT5PreTrainedModel.__annotate__ú  s   ø‡ ‚ áÑò r6   © r:   )rH   rI   rJ   rK   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_keep_in_fp32_modulesÚ_supports_flash_attnÚ_supports_flex_attnÚ_supports_sdpaÚ_can_compile_fullgraphr"   r+  r  r!  Ú_can_record_outputsÚpropertyr[  r,   Úno_gradrb  rz  Ú__annotate_func__rL   rM   rN   rO   s   @@r4   rP  rP  ú  sª   ù‡ € ð &ÐØ&*Ð#à$˜ÐØ!˜FÐà ÐØÐØ€NØ!Ðð 	™¨	¸Ô;Ø‘nÐ%;À2ÔFØ™NÐ+BÈ"ÔMðÐð ñó ðð ‡]‚]ƒ_ôApó ðApòF!÷G … r6   rP  c                   ól   a a€ ] tR tRt oV 3R ltR t]]]RV3R lR ll4       4       4       t	Rt
VtV ;t# )Ú	UMT5Stackis  c           
     ó  <€ \         SV `  V4       \        P                  ! VP                  VP
                  4      V n        VP                  V n        \        P                  ! \        VP                  4       Uu. uF  p\        WR 7      NK  	  up4      V n        \        VP
                  VP                  R7      V n        \        P                   ! VP"                  4      V n        RV n        V P)                  4        R# u upi )r/  r�   FN)r)   r*   r   r½   Ú
vocab_sizer^   Úembed_tokensr®   r0  ÚrangeÚ
num_layersr+  Úblockr'   r„   Úfinal_layer_normrb   rc   rd   Úgradient_checkpointingÚ	post_init)r0   rU   Úir3   s   && €r4   r*   ÚUMT5Stack.__init__t  sµ   ø€ Ü‰Ñ˜Ô ÜŸLšL¨×):Ñ):¸F¿N¹NÓKˆÔØ ×+Ñ+ˆŒÜ—]’]ÌEÐRX×RcÑRcÔLdÓ#eÑLdÀq¤I¨f×$BÑLdÑ#eÓfˆŒ
Ü -¨f¯n©nÀ&×B[ÑB[Ô \ˆÔÜ—z’z &×"5Ñ"5Ó6ˆŒð ',ˆÔ#Ø�‰Öùò $fs   Á9C>c                ó   € Wn         R # rj   )r�  ©r0   Únew_embeddingss   &&r4   Úset_input_embeddingsÚUMT5Stack.set_input_embeddings€  s   € Ø*Ör6   c                ó6   <€ V ^8„  d   QhRS[ S[,          RS[/# )r8   r’   rÂ   )r   r   r   )rV   rP   s   "€r4   rW   ÚUMT5Stack.__annotate__†  s+   ø€ ÷ O
ñ O
ñ Ñ+Õ,ðO
ñ 
3ñO
r6   c           	     óÞ  € VR J VR J,          '       d   \        R4      hVf   V P                  V4      pVRJ d"   V P                  '       g   \        RV  R24      hV P                  '       du   V'       dl   Vfh   V P                  P                  '       d6   \        \        V P                  R7      \        V P                  R7      4      pM+\        V P                  R7      pMV P                  '       g   R pV P                  '       d=   \        V P                  TT\        V\
        4      '       d   VP                  MTR7      p	M\        V P                  VVR7      p	V P                  '       d   Ve   \        V P                  VVVR7      p
MR p
V P                  V4      pV P                   F  pV! VV	V3R	V
R
V/VB pK  	  V P                  V4      pV P                  V4      p\        VVR7      # )Nz:You must specify exactly one of input_ids or inputs_embedsTz)`use_cache` can only be set to `True` if z is used as a decoder)rU   )rU   Úinputs_embedsr�   rð   )rU   r   r�   )rU   r   r�   rï   r:  rð   )Úlast_hidden_staterð   )rv  r�  r®   rU   Úis_encoder_decoderr   r
   r   rk   rú   r   rd   r’  r“  r   )r0   rW  r�   rï   r:  r   rð   Ú	use_cacher’   Úcausal_maskÚencoder_extended_attention_maskrC   Úlayer_modules   &&&&&&&&,    r4   rE   ÚUMT5Stack.forwardƒ  sÑ  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ Ø ×-Ñ-¨iÓ8ˆMà˜ÓØ—?—?�?Ü Ð#LÈTÈFÐRgÐ!hÓiÐið �?�?ˆ?ß˜_Ò4Ø—;‘;×1×1Ð1Ü&9Ü$¨D¯K©KÔ8¼,ÈdÏkÉkÔ:Zó'‘Oô '3¸$¿+¹+Ô&F�OøØ——�ð #ˆOà�?�?ˆ?Ü,Ø—{‘{Ø+Ø-ä˜oÔ/B×CÒCð !0× DÒ Dà$ô‰Kô 4Ø—{‘{Ø+Ø-ôˆKð �?�?ˆ?Ð4Ò@Ü.GØ—{‘{Ø+Ø5Ø&;ô	/Ñ+ð /3Ð+àŸ™ ]Ó3ˆà ŸJœJˆLÙ(ØØØ%ñð (Gð	ð
 !0ðð ñŠMñ 'ð ×-Ñ-¨mÓ<ˆØŸ™ ]Ó3ˆä8Ø+Ø+ô
ð 	
r6   )r’  rd   r�  r“  r”  r®   )NNNNNNN)rH   rI   rJ   rK   r*   r›  r!   r#   r   rE   rL   rM   rN   rO   s   @@r4   rŒ  rŒ  s  sA   ù‡ € õ
ò+ð  ØØ÷O
ñ O
ó ó ó  ÷O
ð O
r6   rŒ  c                   óˆ   a a€ ] tR tRt oRtRtRRRR/tV 3R ltR tR	 t	]
]RV3R
 lR ll4       4       tV3R ltRtVtV ;t# )rf  iØ  a?  
Examples:

```python
>>> from transformers import UMT5Model, AutoTokenizer

>>> model = UMT5Model.from_pretrained("google/umt5-small")
>>> tokenizer = AutoTokenizer.from_pretrained("google/umt5-small")
>>> noisy_text = "UN Offizier sagt, dass weiter <extra_id_0> werden muss in Syrien."
>>> label = "<extra_id_0> verhandelt"
>>> inputs = tokenizer(inputs, return_tensors="pt")
>>> labels = tokenizer(label=label, return_tensors="pt")

>>> outputs = model(input_ids=inputs["input_ids"], decoder_input_ids=labels["input_ids"])
>>> hidden_states = outputs.last_hidden_state
```Úumt5úencoder.embed_tokens.weightúshared.weightúdecoder.embed_tokens.weightc                óŠ  <€ \         SV `  V4       \        P                  ! VP                  VP
                  4      V n        \        P                  ! V4      pR Vn	        R Vn
        \        V4      V n        \        P                  ! V4      pRVn	        VP                  Vn        \        V4      V n        V P!                  4        R# ©FTN)r)   r*   r   r½   rŽ  r^   rk  ÚcopyÚdeepcopyr®   r£  rŒ  ÚencoderÚnum_decoder_layersr‘  Údecoderr•  ©r0   rU   Úencoder_configÚdecoder_configr3   s   &&  €r4   r*   ÚUMT5Model.__init__ò  s’   ø€ Ü‰Ñ˜Ô Ü—l’l 6×#4Ñ#4°f·n±nÓEˆŒäŸš vÓ.ˆØ$)ˆÔ!Ø#(ˆÔ Ü  Ó0ˆŒäŸš vÓ.ˆØ$(ˆÔ!Ø$*×$=Ñ$=ˆÔ!Ü  Ó0ˆŒð 	�‰Ör6   c                ó   € V P                   # rj   ©rk  ©r0   s   &r4   Úget_input_embeddingsÚUMT5Model.get_input_embeddings  ó   € Ø�{‰{Ðr6   c                ó~   € Wn         V P                  P                  V4       V P                  P                  V4       R # rj   ©rk  r±  r›  r³  r™  s   &&r4   r›  ÚUMT5Model.set_input_embeddings  ó+   € Ø$ŒØ�‰×)Ñ)¨.Ô9Ø�‰×)Ñ)¨.Ö9r6   c                ó–  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[S[ P                  ,          ,          R,          RS[R,          RS[ P                  R,          R	S[ P                  R,          R
S[R,          RS[S[	,          RS[
/# )r8   rW  Nr�   rV  rX  Úencoder_outputsrð   r   Údecoder_inputs_embedsr£  r’   rÂ   )r,   Ú
LongTensorÚFloatTensorÚ
BoolTensorÚtupler	   rl   rª   r   r   r   )rV   rP   s   "€r4   rW   ÚUMT5Model.__annotate__  sç   ø€ ÷ e
ñ e
á×#Ñ# dÕ*ðe
ñ ×)Ñ)¨DÕ0ðe
ñ !×+Ñ+¨dÕ2ð	e
ñ
 !&× 0Ñ 0°4Õ 7ðe
ñ ™u¡U×%6Ñ%6Õ7Õ8¸4Õ?ðe
ñ  �ðe
ñ —|‘| dÕ*ðe
ñ  %Ÿ|™|¨dÕ2ðe
ñ ˜$•;ðe
ñ Ñ+Õ,ðe
ñ 
ñe
r6   c
                ó  € Vf   V P                   ! RRVRVRV/V
B pM]\        V\        4      '       gH   \        V^ ,          \        V4      ^8”  d
   V^,          MR\        V4      ^8”  d
   V^,          MRR7      pV^ ,          pV P                  ! RRVRVRVRVRVRVR	V	/V
B p\        VP                  VP                  VP                  VP                  VP                  VP                  VP                  VP                  R
7      # )aP	  
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
    Indices of input sequence tokens in the vocabulary. UMT5 is a model with relative position embeddings so
    you should be able to pad the inputs on both the right and the left.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for detail.

    [What are input IDs?](../glossary#input-ids)

    To know more on how to prepare `input_ids` for pretraining take a look a [UMT5 Training](./umt5#training).
decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Indices of decoder input sequence tokens in the vocabulary.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are decoder input IDs?](../glossary#decoder-input-ids)

    UMT5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
    is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).

    To know more on how to prepare `decoder_input_ids` for pretraining take a look at [UMT5
    Training](./umt5#training).
decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
    be used by default.

Example:

```python
>>> from transformers import AutoTokenizer, UMT5Model

>>> tokenizer = AutoTokenizer.from_pretrained("google/umt5-small")
>>> model = UMT5Model.from_pretrained("google/umt5-small")

>>> input_ids = tokenizer(
...     "Studies have been shown that owning a dog is good for you", return_tensors="pt"
... ).input_ids  # Batch size 1
>>> decoder_input_ids = tokenizer("Studies show that", return_tensors="pt").input_ids  # Batch size 1

>>> # preprocess: Prepend decoder_input_ids with start token which is pad token for UMT5Model.
>>> # This is not needed for torch's UMT5ForConditionalGeneration as it does this internally using labels arg.
>>> decoder_input_ids = model._shift_right(decoder_input_ids)

>>> # forward pass
>>> outputs = model(input_ids=input_ids, decoder_input_ids=decoder_input_ids)
>>> last_hidden_states = outputs.last_hidden_state
```NrW  r�   r   ©r¡  rC   rS  rð   rï   r:  r£  )r¡  rð   Údecoder_hidden_statesÚdecoder_attentionsrT  Úencoder_last_hidden_staterï   Úencoder_attentionsr~  )r±  rk   r   Úlenr³  r   r¡  rð   rC   rS  rT  )r0   rW  r�   rV  rX  rÃ  rð   r   rÄ  r£  r’   rC   Údecoder_outputss   &&&&&&&&&&,  r4   rE   ÚUMT5Model.forward  sJ  € ðD Ò"Ø"Ÿlšlñ Ø#ðà-ðð ,ðð ñ	‰Oô ˜O¬_×=Ò=Ü-Ø"1°!Õ"4Ü47¸Ó4HÈ1Ô4L˜o¨aÖ0ÐRVÜ14°_Ó1EÈÔ1I˜?¨1Ö-ÈtôˆOð (¨Õ*ˆð Ÿ,š,ñ 	
Ø'ð	
à1ð	
ð 0ð	
ð ,ð		
ð
 #0ð	
ð $2ð	
ð  ð	
ð ñ	
ˆô "Ø-×?Ñ?Ø+×;Ñ;Ø"1×"?Ñ"?Ø.×9Ñ9Ø,×=Ñ=Ø&5×&GÑ&GØ"1×"?Ñ"?Ø.×9Ñ9ô	
ð 		
r6   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # rT   r$   )rV   rP   s   "€r4   rW   rÉ  Ø  s   ø‡ ‚ ñ( Ñò) r6   )r³  r±  rk  ©	NNNNNNNNN)rH   rI   rJ   rK   r  Ú
model_typeÚ_tied_weights_keysr*   r»  r›  r    r   rE   rŠ  rL   rM   rN   rO   s   @@r4   rf  rf  Ø  s`   ù‡ € ñð" €Jð 	& Ø% ðÐõ
ò$ò:ð
 Ø÷e
ñ e
ó ó ðe
÷o … r6   rf  z<
    UMT5 Model with a `language modeling` head on top.
    )Úcustom_introc                   ó’   a a€ ] tR tRt oRtRtRRRRRR/tV 3R ltR	 tR
 t	]
]RV3R lR ll4       4       tV3R lR ltRtVtV ;t# )rg  iw  aó  
Examples:

```python
>>> from transformers import UMT5ForConditionalGeneration, AutoTokenizer

>>> model = UMT5ForConditionalGeneration.from_pretrained("google/umt5-small")
>>> tokenizer = AutoTokenizer.from_pretrained("google/umt5-small")
>>> article = "UN Offizier sagt, dass weiter verhandelt werden muss in Syrien."
>>> summary = "Weiter Verhandlung in Syrien."
>>> inputs = tokenizer(article, text_target=summary, return_tensors="pt")

>>> outputs = model(**inputs)
>>> loss = outputs.loss
```r©  rª  r«  r¬  zlm_head.weightc                ó  <€ \         SV `  V4       VP                  V n        \        P
                  ! VP                  VP                  4      V n        \        P                  ! V4      pR Vn
        R Vn        \        V4      V n        \        P                  ! V4      pRVn
        VP                  Vn        \        V4      V n        \        P"                  ! VP                  VP                  R R7      V n        V P'                  4        R# )FTr[   N)r)   r*   r^   Ú	model_dimr   r½   rŽ  rk  r¯  r°  r®   r£  rŒ  r±  r²  r‘  r³  r]   r_  r•  r´  s   &&  €r4   r*   Ú%UMT5ForConditionalGeneration.__init__”  s»   ø€ Ü‰Ñ˜Ô ØŸ™ˆŒä—l’l 6×#4Ñ#4°f·n±nÓEˆŒäŸš vÓ.ˆØ$)ˆÔ!Ø#(ˆÔ Ü  Ó0ˆŒäŸš vÓ.ˆØ$(ˆÔ!Ø$*×$=Ñ$=ˆÔ!Ü  Ó0ˆŒä—y’y §¡°×1BÑ1BÈÔOˆŒð 	�‰Ör6   c                ó   € V P                   # rj   r¹  rº  s   &r4   r»  Ú1UMT5ForConditionalGeneration.get_input_embeddingsª  r½  r6   c                ó~   € Wn         V P                  P                  V4       V P                  P                  V4       R # rj   r¿  r™  s   &&r4   r›  Ú1UMT5ForConditionalGeneration.set_input_embeddings®  rÁ  r6   c                ó¾  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[S[ P
                  ,          ,          R,          RS[R,          RS[ P                  R,          R	S[ P                  R,          R
S[ P                  R,          RS[R,          RS[S[	,          RS[
/# )r8   rW  Nr�   rV  rX  rÃ  rð   r   rÄ  Úlabelsr£  r’   rÂ   )r,   rÅ  rÆ  rÇ  rÈ  rl   r	   rª   r   r   r   )rV   rP   s   "€r4   rW   Ú)UMT5ForConditionalGeneration.__annotate__µ  sÿ   ø€ ÷ 
ñ 
á×#Ñ# dÕ*ð
ñ ×)Ñ)¨DÕ0ð
ñ !×+Ñ+¨dÕ2ð	
ñ
 !&× 0Ñ 0°4Õ 7ð
ñ ™u¡U§\¡\Õ2Õ3°dÕ:ð
ñ  �ð
ñ ×(Ñ(¨4Õ/ð
ñ  %×0Ñ0°4Õ7ð
ñ × Ñ  4Õ'ð
ñ ˜$•;ð
ñ Ñ+Õ,ð
ñ 
ñ
r6   c                óœ  € Vf   V P                   ! RRVRVRV/VB pM]\        V\        4      '       gH   \        V^ ,          \        V4      ^8”  d
   V^,          MR\        V4      ^8”  d
   V^,          MRR7      pV^ ,          pV	e   Vf   Vf   V P	                  V	4      pV P
                  ! RRVRVRVRVRVRVR	V
/VB pV^ ,          pV P                  P                  '       d   WàP                  R,          ,          pV P                  V4      pRpV	e_   \        RR
7      pV	P                  VP                  4      p	V! VP                  RVP                  R4      4      V	P                  R4      4      p\        VVVP                   VP"                  VP$                  VP&                  VP(                  VP"                  VP$                  R7	      # )aP
  
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
    Indices of input sequence tokens in the vocabulary. UMT5 is a model with relative position embeddings so
    you should be able to pad the inputs on both the right and the left.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for detail.

    [What are input IDs?](../glossary#input-ids)

    To know more on how to prepare `input_ids` for pretraining take a look a [UMT5 Training](./umt5#training).
decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Indices of decoder input sequence tokens in the vocabulary.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are decoder input IDs?](../glossary#decoder-input-ids)

    UMT5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
    is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).

    To know more on how to prepare `decoder_input_ids` for pretraining take a look at [UMT5
    Training](./umt5#training).
decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
    be used by default.
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the sequence classification/regression loss. Indices should be in `[-100, 0, ...,
    config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for
    labels in `[0, ..., config.vocab_size]`

Examples:

```python
>>> from transformers import AutoTokenizer, UMT5ForConditionalGeneration

>>> tokenizer = AutoTokenizer.from_pretrained("google/umt5-small")
>>> model = UMT5ForConditionalGeneration.from_pretrained("google/umt5-small")

>>> # training
>>> input_ids = tokenizer("The <extra_id_0> walks in <extra_id_1> park", return_tensors="pt").input_ids
>>> labels = tokenizer("<extra_id_0> cute dog <extra_id_1> the <extra_id_2>", return_tensors="pt").input_ids
>>> outputs = model(input_ids=input_ids, labels=labels)
>>> loss = outputs.loss
>>> logits = outputs.logits

>>> # inference
>>> input_ids = tokenizer("Studies have shown that <extra_id_0> good for you", return_tensors="pt").input_ids
>>> outputs = model.generate(input_ids)
>>> tokenizer.decode(outputs[0], skip_special_tokens=True)
```NrW  r�   r   rË  rð   rï   r:  r£  ©Úignore_index©	ÚlossÚlogitsrð   rÌ  rÍ  rT  rÎ  rï   rÏ  r~  rš   rs  r:   )r±  rk   r   rÐ  rz  r³  rU   rm  rÚ  r_  r   r;   rß   rÅ   r›   r   rð   rC   rS  rT  r¡  )r0   rW  r�   rV  rX  rÃ  rð   r   rÄ  rá  r£  r’   rC   rÑ  Úsequence_outputÚ	lm_logitsrç  Úloss_fcts   &&&&&&&&&&&,      r4   rE   Ú$UMT5ForConditionalGeneration.forward³  s÷  € ðL Ò"à"Ÿlšlñ Ø#ðà-ðð ,ðð ñ	‰Oô ˜O¬_×=Ò=Ü-Ø"1°!Õ"4Ü47¸Ó4HÈ1Ô4L˜o¨aÖ0ÐRVÜ14°_Ó1EÈÔ1I˜?¨1Ö-ÈtôˆOð (¨Õ*ˆàÒÐ"3Ò";Ð@UÒ@]à $× 1Ñ 1°&Ó 9Ðð Ÿ,š,ñ 	
Ø'ð	
à1ð	
ð 0ð	
ð ,ð		
ð
 #0ð	
ð $2ð	
ð  ð	
ð ñ	
ˆð *¨!Õ,ˆà�;‰;×*×*Ð*ð .·±ÀÕ1EÕFˆOà—L‘L Ó1ˆ	àˆØÒÜ'°TÔ:ˆHà—Y‘Y˜y×/Ñ/Ó0ˆFÙ˜IŸN™N¨2¨y¯~©~¸bÓ/AÓBÀFÇKÁKÐPRÃOÓTˆDäØØØ+×;Ñ;Ø"1×"?Ñ"?Ø.×9Ñ9Ø,×=Ñ=Ø&5×&GÑ&GØ"1×"?Ñ"?Ø.×9Ñ9ô

ð 
	
r6   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r8   rá  rÃ   )rV   rP   s   "€r4   rW   râ  7  s   ø€ ÷ )ñ )¹E¿L¹Lñ )r6   c                ó$   € V P                  V4      # rj   )rz  )r0   rá  s   &&r4   Ú%prepare_decoder_input_ids_from_labelsÚBUMT5ForConditionalGeneration.prepare_decoder_input_ids_from_labels7  s   € Ø× Ñ  Ó(Ð(r6   )r³  r±  r_  rÚ  rk  ©
NNNNNNNNNN)rH   rI   rJ   rK   r  rÕ  rÖ  r*   r»  r›  r    r   rE   rï  rL   rM   rN   rO   s   @@r4   rg  rg  w  sk   ù‡ € ñð  €Jà% Ø% Ø˜/ðÐõò,ò:ð
 Ø÷
ñ 
ó ó ð
÷D)÷ )ð )r6   rg  c                   óx   a a€ ] tR tRt oRtRtRR/tV 3R ltR tR t	]
]RV3R	 lR
 ll4       4       tRtVtV ;t# )rh  i;  aº  
Examples:

```python
>>> from transformers import UMT5EncoderModel, AutoTokenizer

>>> model = UMT5EncoderModel.from_pretrained("google/umt5-small")
>>> tokenizer = AutoTokenizer.from_pretrained("google/umt5-small")
>>> article = "UN Offizier sagt, dass weiter verhandelt werden muss in Syrien."
>>> input_ids = tokenizer(article, return_tensors="pt").input_ids
>>> outputs = model(input_ids)
>>> hidden_state = outputs.last_hidden_state
```r©  rª  r«  c                ó  <€ \         SV `  V4       \        P                  ! VP                  VP
                  4      V n        \        P                  ! V4      pR Vn	        R Vn
        \        V4      V n        V P                  4        R# )FN)r)   r*   r   r½   rŽ  r^   rk  r¯  r°  r£  r¢  rŒ  r±  r•  )r0   rU   rµ  r3   s   && €r4   r*   ÚUMT5EncoderModel.__init__Q  sb   ø€ Ü‰Ñ˜Ô Ü—l’l 6×#4Ñ#4°f·n±nÓEˆŒäŸš vÓ.ˆØ#(ˆÔ Ø,1ˆÔ)Ü  Ó0ˆŒð 	�‰Ör6   c                ó   € V P                   # rj   r¹  rº  s   &r4   r»  Ú%UMT5EncoderModel.get_input_embeddings^  r½  r6   c                óH   € Wn         V P                  P                  V4       R # rj   )rk  r±  r›  r™  s   &&r4   r›  Ú%UMT5EncoderModel.set_input_embeddingsb  s   € Ø$ŒØ�‰×)Ñ)¨.Ö9r6   c          
      ó®   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          RS[/# )r8   rW  Nr�   r   r’   rÂ   )r,   rÅ  rÆ  r   r   r   )rV   rP   s   "€r4   rW   ÚUMT5EncoderModel.__annotate__i  sd   ø€ ÷ %ñ %á×#Ñ# dÕ*ð%ñ ×)Ñ)¨DÕ0ð%ñ ×(Ñ(¨4Õ/ð	%ñ
 Ñ+Õ,ð%ñ 
ñ%r6   c           	     ó6   € V P                   ! RRVRVRV/VB pV# )aÉ  
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
    Indices of input sequence tokens in the vocabulary. UMT5 is a model with relative position embeddings so you
    should be able to pad the inputs on both the right and the left.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for detail.

    To know more on how to prepare `input_ids` for pretraining take a look a [UMT5 Training](./umt5#training).

Example:

```python
>>> from transformers import AutoTokenizer, UMT5EncoderModel

>>> tokenizer = AutoTokenizer.from_pretrained("google/umt5-small")
>>> model = UMT5EncoderModel.from_pretrained("google/umt5-small")
>>> input_ids = tokenizer(
...     "Studies have been shown that owning a dog is good for you", return_tensors="pt"
... ).input_ids  # Batch size 1
>>> outputs = model(input_ids=input_ids)
>>> last_hidden_states = outputs.last_hidden_state
```rW  r�   r   r~  )r±  )r0   rW  r�   r   r’   rÃ  s   &&&&, r4   rE   ÚUMT5EncoderModel.forwardf  s?   € ðB ,0¯<ª<ñ ,
Øð,
à)ð,
ð (ð,
ð ñ	,
ˆð Ðr6   )r±  rk  r  )rH   rI   rJ   rK   r  rÕ  rÖ  r*   r»  r›  r    r   rE   rL   rM   rN   rO   s   @@r4   rh  rh  ;  sU   ù‡ € ñð €Jð 	& ðÐõ
òò:ð Ø÷%ñ %ó ó ÷%ð %r6   rh  z…
    UMT5 model with a sequence classification/head on top (a linear layer on top of the pooled output) e.g. for GLUE
    tasks.
    c                   ón   a a€ ] tR tRt oR.tV3R lV 3R llt]]RV3R lR ll4       4       tRt	Vt
V ;t# )	ÚUMT5ForSequenceClassificationi‘  úFdecoder.block.0.layer.1.EncDecAttention.relative_attention_bias.weightc                ó    <€ V ^8„  d   QhRS[ /# rT   r$   )rV   rP   s   "€r4   rW   Ú*UMT5ForSequenceClassification.__annotate__›  s   ø€ ÷ ñ ™zñ r6   c                ó†   <€ \         SV `  V4       \        V4      V n        \	        V4      V n        V P                  4        R # rj   )r)   r*   rf  rQ  rB  Úclassification_headr•  rg   s   &&€r4   r*   Ú&UMT5ForSequenceClassification.__init__›  s6   ø€ Ü‰Ñ˜Ô Ü$ VÓ,ˆÔÜ#9¸&Ó#AˆÔ ð 	�‰Ör6   c                óš  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[ P                  ,          R,          RS[ P                  R,          RS[ P                  R,          R	S[ P                  R,          R
S[R,          RS[S[,          RS[/# )r8   rW  Nr�   rV  rX  rÃ  r   rÄ  rá  r£  r’   rÂ   )	r,   rÅ  rl   ÚlistrÆ  rª   r   r   r   )rV   rP   s   "€r4   rW   r  ¥  së   ø€ ÷ }
ñ }
á×#Ñ# dÕ*ð}
ñ Ÿ™ tÕ+ð}
ñ !×+Ñ+¨dÕ2ð	}
ñ
 !&× 0Ñ 0°4Õ 7ð}
ñ ™e×/Ñ/Õ0°4Õ7ð}
ñ ×(Ñ(¨4Õ/ð}
ñ  %×0Ñ0°4Õ7ð}
ñ × Ñ  4Õ'ð}
ñ ˜$•;ð}
ñ Ñ+Õ,ð}
ñ 
)ñ}
r6   c
                óR  € Ve   Rp	Vf'   Ve#   \        RV P                  P                   24      hVf%   Vf!   Vf   \        R4      hV P	                  V4      pV P
                  ! V3RVRVRVRVR	VR
VRV	/V
B pV^ ,          pVP                  V P                  P                  4      P                  VP                  4      p\        \        P                  ! VP                  ^4      4      P                  4       ^8H  R4       VP                   w  rïpWÍR3,          p\        VP                   ^ ,          V,          ^8¬  R4       VP#                  VRV4      R,          pV P%                  V4      pRpVEeÝ   VP                  VP                  4      pV P                  P&                  fª   V P                  P(                  ^8X  d   RV P                  n        M|V P                  P(                  ^8”  dQ   VP*                  \        P,                  8X  g    VP*                  \        P.                  8X  d   RV P                  n        MRV P                  n        V P                  P&                  R8X  dV   \1        4       pV P                  P(                  ^8X  d'   V! VP3                  4       VP3                  4       4      pMšV! VV4      pM�V P                  P&                  R8X  dH   \5        4       pV! VP#                  RV P                  P(                  4      VP#                  R4      4      pM.V P                  P&                  R8X  d   \7        4       pV! VV4      p\9        VVVP:                  VP<                  VP>                  VP@                  VPB                  VPD                  VPF                  R7	      # )aø  
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
    Indices of input sequence tokens in the vocabulary. UMT5 is a model with relative position embeddings so
    you should be able to pad the inputs on both the right and the left.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for detail.

    [What are input IDs?](../glossary#input-ids)

    To know more on how to prepare `input_ids` for pretraining take a look a [UMT5 Training](./umt5#training).
decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Indices of decoder input sequence tokens in the vocabulary.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are decoder input IDs?](../glossary#decoder-input-ids)

    UMT5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
    is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).

    To know more on how to prepare `decoder_input_ids` for pretraining take a look at [UMT5
    Training](./umt5#training).
decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
    be used by default.
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
NFz8Passing input embeddings is currently not supported for ú°If no `decoder_input_ids` or `decoder_inputs_embeds` are passed, `input_ids` cannot be `None`. Please pass either `input_ids` or `decoder_input_ids` or `decoder_inputs_embeds`.r�   rV  rX  rÃ  r   rÄ  r£  z7All examples must have the same number of <eos> tokens.rà   z3Each example must contain at least one <eos> token.Ú
regressionÚsingle_label_classificationÚmulti_label_classificationræ  r:   )rà   r:   rà   )$ÚNotImplementedErrorr3   rH   rv  rz  rQ  ÚeqrU   Úeos_token_idr;   rß   r   r,   Úunique_consecutiveÚsumÚnumelrô   rÅ   r  Úproblem_typerH  r@   rÌ   r©   r   Úsqueezer   r   r   rð   rÌ  rÍ  rT  rÎ  rï   rÏ  )r0   rW  r�   rV  rX  rÃ  r   rÄ  rá  r£  r’   r  ré  Úeos_maskr  r;  r1   ÚselectedÚsentence_representationrè  rç  rë  s   &&&&&&&&&&,           r4   rE   Ú%UMT5ForSequenceClassification.forward£  sM  € ð\ ÒØˆIàÒ Ò!:Ü%ØJÈ4Ï>É>×KbÑKbÐJcÐdóð ð Ò$Ð)>Ò)FØÒ Ü ðUóð ð
 !%× 1Ñ 1°)Ó <Ðà×"Ò"Øñ

à)ð

ð 0ð

ð $:ð	

ð
 ,ð

ð (ð

ð #8ð

ð  ð

ð ñ

ˆð " !�*ˆà—<‘< §¡× 8Ñ 8Ó9×<Ñ<¸_×=SÑ=SÓTˆäÜ×$Ò$ X§\¡\°!£_Ó5×;Ñ;Ó=ÀÑBØEô	
ð &5×%:Ñ%:Ñ"ˆ
�{Ø"¨Q ;Õ/ˆÜØ�N‰N˜1Õ Õ+¨qÑ0ØAô	
ð #+§-¡-°
¸BÀÓ"LÈXÕ"VÐØ×)Ñ)Ð*AÓBˆàˆØÓØ—Y‘Y˜vŸ}™}Ó-ˆFØ�{‰{×'Ñ'Ò/Ø—;‘;×)Ñ)¨QÔ.Ø/;�D—K‘KÕ,Ø—[‘[×+Ñ+¨aÔ/°V·\±\ÄUÇZÁZÔ5OÐSY×S_ÑS_Ôch×clÑclÔSlØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ô7Ü"›9�Ø—;‘;×)Ñ)¨QÔ.Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# F¨FÓ3‘DØ—‘×)Ñ)Ð-JÔJÜ+Ó-�Ù §¡¨B°·±×0FÑ0FÓ GÈÏÉÐUWËÓY‘Ø—‘×)Ñ)Ð-IÔIÜ,Ó.�Ù ¨Ó/�ä.ØØØ#×3Ñ3Ø")×"?Ñ"?Ø&×9Ñ9Ø$×5Ñ5Ø&-×&GÑ&GØ")×"?Ñ"?Ø&×9Ñ9ô

ð 
	
r6   )r  rQ  rÔ  ©rH   rI   rJ   rK   Ú"_keys_to_ignore_on_load_unexpectedr*   r    r   rE   rL   rM   rN   rO   s   @@r4   rþ  rþ  ‘  sE   ù‡ € ð +sÐ)sÐ&÷ó ð Ø÷}
ñ }
ó ó ÷}
ð }
r6   rþ  c                   ón   a a€ ] tR tRt oR.tV3R lV 3R llt]]RV3R lR ll4       4       tRt	Vt
V ;t# )	ro  i%  rÿ  c                ó    <€ V ^8„  d   QhRS[ /# rT   r$   )rV   rP   s   "€r4   rW   Ú'UMT5ForTokenClassification.__annotate__*  s   ø€ ÷ 	ñ 	™zñ 	r6   c                ó2  <€ \         SV `  V4       VP                  V n        \        V4      V n        \
        P                  ! VP                  4      V n        \
        P                  ! VP                  VP                  4      V n        V P                  4        R # rj   )r)   r*   rH  rh  rQ  r   rb   rG  rd   r]   r1   ra  r•  rg   s   &&€r4   r*   Ú#UMT5ForTokenClassification.__init__*  sj   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä+¨FÓ3ˆÔÜ—z’z &×";Ñ";Ó<ˆŒÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Ör6   c                óÖ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          RS[/# )r8   rW  Nr�   r   rá  r’   rÂ   )r,   rl   r   r   r   )rV   rP   s   "€r4   rW   r  8  sq   ø€ ÷ +
ñ +
á—<‘< $Õ&ð+
ñ Ÿ™ tÕ+ð+
ñ —|‘| dÕ*ð	+
ñ
 —‘˜tÕ#ð+
ñ Ñ+Õ,ð+
ñ 
ñ+
r6   c                óP  € V P                   ! V3RVRV/VB pV^ ,          pV P                  V4      pV P                  V4      pRp	Ve=   \        4       p
V
! VP	                  RV P
                  4      VP	                  R4      4      p	\        V	VVP                  VP                  R7      # )aò  
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
    Indices of input sequence tokens in the vocabulary. UMT5 is a model with relative position embeddings so you
    should be able to pad the inputs on both the right and the left.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for detail.

    [What are input IDs?](../glossary#input-ids)

    To know more on how to prepare `input_ids` for pretraining take a look a [UMT5 Training](./umt5#training).
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
r�   r   N)rç  rè  rC   rS  r:   )	rQ  rd   ra  r   rÅ   rH  r   rC   rS  )r0   rW  r�   r   rá  r’   r  rC   rè  rç  rë  s   &&&&&,     r4   rE   Ú"UMT5ForTokenClassification.forward5  sµ   € ð2 $(×#3Ò#3Øñ$
à)ð$
ð (ð$
ð ñ	$
ˆð   �
ˆØŸ™ ]Ó3ˆØ—‘ Ó/ˆàˆØÒÜ'Ó)ˆHÙ˜FŸK™K¨¨D¯O©OÓ<¸f¿k¹kÈ"»oÓNˆDä$ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r6   )ra  rd   rH  rQ  r@  r  rO   s   @@r4   ro  ro  %  s>   ù‡ € à*rÐ)sÐ&÷	ó 	ð Ø÷+
ñ +
ó ó ÷+
ð +
r6   ro  c                   ót   a a€ ] tR tRt oRRRR/tV 3R ltR tR t]]	RV3R lR	 ll4       4       t
R
tVtV ;t# )ri  if  rª  r«  r¬  c                ó.  <€ \         SV `  V4       VP                  V n        \        P
                  ! VP                  VP                  4      V n        \        P                  ! V4      pR Vn
        R Vn        \        V4      V n        \        P                  ! V4      pRVn
        VP                  Vn        \        V4      V n        VP"                  V n        \        P$                  ! VP                  VP"                  4      V n        V P)                  4        R# r®  )r)   r*   r^   rÚ  r   r½   rŽ  rk  r¯  r°  r®   r£  rŒ  r±  r²  r‘  r³  rH  r]   r`  r•  r´  s   &&  €r4   r*   Ú!UMT5ForQuestionAnswering.__init__m  sÆ   ø€ Ü‰Ñ˜Ô ØŸ™ˆŒä—l’l 6×#4Ñ#4°f·n±nÓEˆŒäŸš vÓ.ˆØ$)ˆÔ!Ø#(ˆÔ Ü  Ó0ˆŒäŸš vÓ.ˆØ$(ˆÔ!Ø$*×$=Ñ$=ˆÔ!Ü  Ó0ˆŒà ×+Ñ+ˆŒÜŸ)š) F§N¡N°F×4EÑ4EÓFˆŒð 	�‰Ör6   c                ó   € V P                   # rj   r¹  rº  s   &r4   r»  Ú-UMT5ForQuestionAnswering.get_input_embeddings„  r½  r6   c                ó~   € Wn         V P                  P                  V4       V P                  P                  V4       R # rj   r¿  r™  s   &&r4   r›  Ú-UMT5ForQuestionAnswering.set_input_embeddingsˆ  rÁ  r6   c                óÒ  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[S[ P
                  ,          ,          R,          RS[ P                  R,          RS[ P                  R,          R	S[ P                  R,          R
S[ P                  R,          RS[R,          RS[S[,          RS[	/# )r8   rW  Nr�   rV  rX  rÃ  Ústart_positionsÚend_positionsr   rÄ  r£  r’   rÂ   )
r,   rÅ  rÆ  rÇ  rÈ  rl   rª   r   r   r   )rV   rP   s   "€r4   rW   Ú%UMT5ForQuestionAnswering.__annotate__�  s  ø€ ÷ y
ñ y
á×#Ñ# dÕ*ðy
ñ ×)Ñ)¨DÕ0ðy
ñ !×+Ñ+¨dÕ2ð	y
ñ
 !&× 0Ñ 0°4Õ 7ðy
ñ ™u¡U§\¡\Õ2Õ3°dÕ:ðy
ñ ×)Ñ)¨DÕ0ðy
ñ ×'Ñ'¨$Õ.ðy
ñ ×(Ñ(¨4Õ/ðy
ñ  %×0Ñ0°4Õ7ðy
ñ ˜$•;ðy
ñ Ñ+Õ,ðy
ñ 
-ñy
r6   c                ó8  € Ve   Ve   Rp
Vf%   V	f!   Vf   \        R4      hV P                  V4      pVf   V P                  ! RRVRVRV/VB pM]\        V\        4      '       gH   \	        V^ ,          \        V4      ^8”  d
   V^,          MR\        V4      ^8”  d
   V^,          MRR7      pV^ ,          pV P                  ! RRVRVRV	RRR	VR
VRV
/VB pV^ ,          pV P                  V4      pVP                  ^RR7      w  ppVP                  R4      P                  4       pVP                  R4      P                  4       pRpVeø   Veô   \        VP                  4       4      ^8”  d+   VP                  R4      P                  VP                  4      p\        VP                  4       4      ^8”  d+   VP                  R4      P                  VP                  4      pVP                  ^4      pVP                  ^ V4      pVP                  ^ V4      p\        VR7      pV! VV4      pV! VV4      pVV,           ^,          p\!        VVVVP"                  VP$                  VP&                  VP(                  VP*                  VP$                  VP&                  R7
      # )aæ  
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
    Indices of input sequence tokens in the vocabulary. UMT5 is a model with relative position embeddings so
    you should be able to pad the inputs on both the right and the left.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for detail.

    [What are input IDs?](../glossary#input-ids)

    To know more on how to prepare `input_ids` for pretraining take a look a [UMT5 Training](./umt5#training).
decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Indices of decoder input sequence tokens in the vocabulary.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are decoder input IDs?](../glossary#decoder-input-ids)

    UMT5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
    is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).

    To know more on how to prepare `decoder_input_ids` for pretraining take a look at [UMT5
    Training](./umt5#training).
decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
    be used by default.
NFr  rW  r�   r   rË  rð   rï   r:  r£  r–   rä  )
rç  Ústart_logitsÚ
end_logitsrð   rÌ  rÍ  rT  rÎ  rï   rÏ  r~  r:   )rv  rz  r±  rk   r   rÐ  r³  r`  Úsplitr  r    r›   r;   rß   r9  r   r   rð   rC   rS  rT  r¡  )r0   rW  r�   rV  rX  rÃ  r*  r+  r   rÄ  r£  r’   rC   rÑ  ré  rè  r.  r/  Ú
total_lossÚignored_indexrë  Ú
start_lossÚend_losss   &&&&&&&&&&&,           r4   rE   Ú UMT5ForQuestionAnswering.forward�  sÚ  € ðX Ò&¨=Ò+DØˆIð
 Ò$Ð)>Ò)FØÒ Ü ðUóð ð
 !%× 1Ñ 1°)Ó <Ðð Ò"Ø"Ÿlšlñ Ø#ðà-ðð ,ðð ñ	‰Oô ˜O¬_×=Ò=Ü-Ø"1°!Õ"4Ü47¸Ó4HÈ1Ô4L˜o¨aÖ0ÐRVÜ14°_Ó1EÈÔ1I˜?¨1Ö-ÈtôˆOð (¨Õ*ˆð Ÿ,š,ñ 	
Ø'ð	
à1ð	
ð 0ð	
ð !ð		
ð
 #0ð	
ð $2ð	
ð  ð	
ð ñ	
ˆð *¨!Õ,ˆà—‘ Ó1ˆØ#)§<¡<°°r <Ó#:Ñ ˆ�jØ#×+Ñ+¨BÓ/×:Ñ:Ó<ˆØ×'Ñ'¨Ó+×6Ñ6Ó8ˆ
àˆ
ØÒ&¨=Ò+Dä�?×'Ñ'Ó)Ó*¨QÔ.Ø"1×"9Ñ"9¸"Ó"=×"@Ñ"@À×ATÑATÓ"U�Ü�=×%Ñ%Ó'Ó(¨1Ô,Ø -× 5Ñ 5°bÓ 9× <Ñ <¸Z×=NÑ=NÓ O�à(×-Ñ-¨aÓ0ˆMØ-×3Ñ3°A°}ÓEˆOØ)×/Ñ/°°=ÓAˆMä'°]ÔCˆHÙ! ,°Ó@ˆJÙ 
¨MÓ:ˆHØ$ xÕ/°1Õ4ˆJä2ØØ%Ø!Ø+×;Ñ;Ø"1×"?Ñ"?Ø.×9Ñ9Ø,×=Ñ=Ø&5×&GÑ&GØ"1×"?Ñ"?Ø.×9Ñ9ô
ð 	
r6   )r³  r±  rÚ  rH  r`  rk  rñ  )rH   rI   rJ   rK   rÖ  r*   r»  r›  r    r   rE   rL   rM   rN   rO   s   @@r4   ri  ri  f  sR   ù‡ € ð 	& Ø% ðÐõ
ò.ò:ð
 Ø÷y
ñ y
ó ó ÷y
ð y
r6   ri  )Nrò   N)rh  rg  ri  rþ  ro  rf  rP  )Nr  r¯  rÑ   Úcollections.abcr   r,   r   Útorch.nnr   r   r   Ú r   rd  Úactivationsr   Úcache_utilsr	   r
   r   Ú
generationr   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   r   r   Úutils.genericr    r!   Úutils.output_capturingr"   r#   Úconfiguration_umt5r%   Ú
get_loggerrH   r·   r“   r'   rR   rq   r}   r£   r¥   r  r!  r+  rB  rP  rŒ  rf  rg  rh  rþ  ro  ri  Ú__all__r~  r6   r4   Ú<module>rG     s  ðñ ã Û Ý $ã Ý ß AÑ Aå &Ý !ß CÑ CÝ )ß JÝ 9÷÷ ñ ÷ GÝ &÷÷ ÷ Jß EÝ *ð 
×	Ò	˜HÓ	%€ô+�B—I‘Iô +ô4˜Ÿ	™	ô ô.˜RŸY™Yô ô<�"—)‘)ô ÷&%ô@u)�B—I‘Iô u)ôp˜RŸY™Yô ô4˜bŸi™iô ô8:Ð*ô :ô|˜RŸY™Yô ð$ ôu!˜/ó u!ó ðu!ôpb
Ð#ô b
ðJ ô[
Ð#ó [
ó ð[
ñ| ðôô
|)Ð#6¸ó |)óð
|)ð~ ôRÐ*ó Ró ðRñj ðôôK
Ð$7ó K
óðK
ð\ ô=
Ð!4ó =
ó ð=
ð@ ôa
Ð2ó a
ó ða
òH‚r6   