+
    œ°³jß½  ã                   ó¼  € R t ^ RIt^ RIt^ RIHt ^ RIt^ RIt^ RIHt ^ RI	H
t
 ^RIHt ^RIHt ^RIHtHtHt ^RIHt ^R	IHtHt ^R
IHt ^RIHt ^RIHtHtH t H!t!H"t" ^RI#H$t$H%t% ^RI&H't' ^RI(H)t)H*t*H+t+H,t,H-t- ^RI.H/t/ ^RI0H1t1H2t2 ^RI3H4t4 ]-Pj                  ! ]64      t7R R lt8 ! R R]Pr                  4      t:R/R R llt; ! R R]Px                  4      t= ! R R]4      t> ! R R]4      t?]* ! R R ]%4      4       t@ ! R! R"]@4      tA ! R# R$]@4      tB]* ! R% R&]@4      4       tC]*! R'R(7       ! R) R*]@]4      4       tD ! R+ R,]@4      tE ! R- R.]@]4      tF. R0OtGR# )1zPyTorch PEGASUS model.N)ÚCallable)Únn)ÚCrossEntropyLoss)Úinitialization)ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚ)BaseModelOutputWithPastAndCrossAttentionsÚ!CausalLMOutputWithCrossAttentionsÚSeq2SeqLMOutputÚSeq2SeqModelOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚis_torchdynamo_compilingÚlogging)Úmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputs©ÚPegasusConfigc                óP   € V ^8„  d   QhR\         P                  R\        R\        /# )é   Ú	input_idsÚpad_token_idÚdecoder_start_token_id)ÚtorchÚTensorÚint)Úformats   "Úm/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/pegasus/modeling_pegasus.pyÚ__annotate__r+   9   s&   € ÷ ñ ¤%§,¡,ð ¼cð Ô[^ñ ó    c                óÈ   € V P                  V P                  4      pV RRR13,          P                  4       VR&   W#R&   Vf   \        R4      hVP	                  VR8H  V4       V# )z)
Shift input ids one token to the right.
ºNNNNz1self.model.config.pad_token_id has to be defined.éÿÿÿÿ)r.   :é   NN)r.   é    iœÿÿÿ)Ú	new_zerosÚshapeÚcloneÚ
ValueErrorÚmasked_fill_)r#   r$   r%   Úshifted_input_idss   &&& r*   Úshift_tokens_rightr8   9   sp   € ð "×+Ñ+¨I¯O©OÓ<ÐØ(¨¨C¨R¨C¨Õ0×6Ñ6Ó8Ð�eÑØ4�dÑàÒÜÐLÓMÐMà×"Ñ"Ð#4¸Ñ#<¸lÔKàÐr,   c                   ó�   a a€ ] tR t^Jt oRtRV3R lV 3R llltR t]P                  ! 4       R	V3R lV 3R lll4       t	Rt
VtV ;t# )
Ú$PegasusSinusoidalPositionalEmbeddingzDThis module produces sinusoidal positional embeddings of any length.c                ó>   <€ V ^8„  d   QhRS[ RS[ RS[ R,          RR/# )r"   Únum_positionsÚembedding_dimÚpadding_idxNÚreturn©r(   )r)   Ú__classdict__s   "€r*   r+   Ú1PegasusSinusoidalPositionalEmbedding.__annotate__M   s5   ø€ ÷ Eñ E¡cð E¹#ð EÉCÐRVÍJð EÐbfñ Er,   c                ó*   <€ \         SV `  WR R7       R# )T)Ú_freezeN)ÚsuperÚ__init__)Úselfr<   r=   r>   Ú	__class__s   &&&&€r*   rF   Ú-PegasusSinusoidalPositionalEmbedding.__init__M   s   ø€ Ü‰Ñ˜¸tÐÖDr,   c                óÈ  € V P                   P                  w  r\        P                  ! \	        V4       UUu. uFN  p\	        V4       Uu. uF5  qC\        P
                  ! R^V^,          ,          V,          4      ,          NK7  	  upNKP  	  upp4      p\        P                  ! WV P                   P                  RR7      pV^,          ^ 8X  d
   V^,          MV^,          ^,           p\        P                  ! \        P                  ! VR,          4      4      VR^ V13&   \        P                  ! \        P                  ! VR,          4      4      VRVR13&   V# u upi u uppi )z˜
Identical to the XLM create_sinusoidal_embeddings except features are not interleaved. The cos features are in
the 2nd half of the vector. [dim // 2:]
i'  F)ÚdtypeÚrequires_gradr.   N)r.   :r1   Nr"   )r.   :r0   Nr"   )Úweightr3   ÚnpÚarrayÚrangeÚpowerr&   ÚemptyrK   ÚFloatTensorÚsinÚcos)rG   Ún_posÚdimÚposÚjÚposition_encÚoutÚsentinels   &       r*   Úcreate_weightÚ2PegasusSinusoidalPositionalEmbedding.create_weightP   s  € ð
 —[‘[×&Ñ&‰
ˆÜ—x’xÜX]Ð^cÔXdÔeÑXdÐQTÄÀsÄÓLÁ¸A”B—H’H˜U A¨¨a­¥L°3Õ$6Ó7×7Ð7ÁÔLÑXdÒeó
ˆô �kŠk˜%¨D¯K©K×,=Ñ,=ÈUÔSˆØ" Q�w¨!œ|�3˜!–8°#¸µ(¸aµˆÜ"×.Ò.¬r¯vªv°lÀ7Õ6KÓ/LÓMˆˆAˆq�ˆzˆMÑÜ!×-Ò-¬b¯fªf°\À'Õ5JÓ.KÓLˆˆAˆx‰yˆLÑØˆ
ùò MùÓes   ·E
Á	;EÂE
ÅE
c                ó|   <€ V ^8„  d   QhRS[ P                  RS[RS[ P                  R,          RS[ P                  /# )r"   Úinput_ids_shapeÚpast_key_values_lengthÚposition_idsNr?   )r&   ÚSizer(   r'   )r)   rA   s   "€r*   r+   rB   `   sA   ø€ ÷ 	-ñ 	-Ù$Ÿz™zð	-ÙCFð	-ÙZ_×ZfÑZfÐimÕZmð	-á	�‰ñ	-r,   c                óÄ   <€ VfN   VR,          w  rE\         P                  ! W"V,           \         P                  V P                  P                  R7      p\
        SV `  V4      # )z3`input_ids_shape` is expected to be [bsz x seqlen].:Nr"   N)rK   Údevice)r&   ÚarangeÚlongrM   re   rE   Úforward)rG   r`   ra   rb   ÚbszÚseq_lenrH   s   &&&&  €r*   rh   Ú,PegasusSinusoidalPositionalEmbedding.forward_   sV   ø€ ð
 ÒØ*¨2Õ.‰LˆCÜ Ÿ<š<Ø&ÀÕ(HÔPU×PZÑPZÐcg×cnÑcn×cuÑcuôˆLô ‰w‰˜|Ó,Ð,r,   © ©N)r1   N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__rF   r]   r&   Úno_gradrh   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©rH   rA   s   @@r*   r:   r:   J   s=   ù‡ € ÙN÷Eõ Eòð ‡]‚]ƒ_÷	-ô 	-ó ÷	-ð 	-r,   r:   c                ó  € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R,          R\        R	\
        \        ,          /# )
r"   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNÚscalingÚdropoutÚkwargs)r   ÚModuler&   r'   Úfloatr   r   )r)   s   "r*   r+   r+   m   s{   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô �T�\ð%ô ð%ô Ô'Õ(ñ%r,   c                 ó¶  € Vf   VP                  R4      R,          p\        P                  ! WP                  ^^4      4      V,          pVe	   W„,           p\        P
                  P                  VRR7      p\        P
                  P                  W†V P                  R7      p\        P                  ! Wƒ4      p	V	P                  ^^4      P                  4       p	W˜3# )N©rW   ©ÚpÚtrainingr/   ç      à¿)
Úsizer&   ÚmatmulÚ	transposer   Ú
functionalÚsoftmaxr   r‡   Ú
contiguous)
ry   rz   r{   r|   r}   r~   r   r€   Úattn_weightsÚattn_outputs
   &&&&&&&,  r*   Úeager_attention_forwardr‘   m   s±   € ð ‚Ø—*‘*˜R“. DÕ(ˆô —<’< §}¡}°Q¸Ó':Ó;¸gÕE€LàÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2Ð(Ó>€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€Lä—,’,˜|Ó3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r,   c                   ó\   a a€ ] tR t^Št oRtRV3R lV 3R llltRV3R lR lltRtVtV ;t	# )	ÚPegasusAttentionz=Multi-headed attention from 'Attention Is All You Need' paperc                óf   <€ V ^8„  d   QhRS[ RS[ RS[RS[RS[RS[RS[R,          R	S[ R,          /# )
r"   Ú	embed_dimÚ	num_headsr   Ú
is_decoderÚbiasÚ	is_causalÚconfigNÚ	layer_idx)r(   r‚   Úboolr    )r)   rA   s   "€r*   r+   ÚPegasusAttention.__annotate__�   sn   ø€ ÷ %Cñ %Cáð%Cñ ð%Cñ ð	%Cñ
 ð%Cñ ð%Cñ ð%Cñ  Õ$ð%Cñ ˜•:ñ%Cr,   c	                ó¬  <€ \         S	V `  4        Wn        W n        W0n        W,          V n        Wpn        V P
                  V,          V P                  8w  d   \        R V P                   RV R24      hV P
                  R,          V n        W@n	        W`n
        W€n        Vf@   V P                  '       d.   \        P                  RV P                  P                   R24       \         P"                  ! WVR7      V n        \         P"                  ! WVR7      V n        \         P"                  ! WVR7      V n        \         P"                  ! WVR7      V n        R# )z;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).NzInstantiating a decoder z¸ without passing `layer_idx` is not recommended and will lead to errors during the forward call, if caching is used. Please make sure to provide a `layer_idx` when creating this class.©r˜   rˆ   )rE   rF   r•   r–   r   Úhead_dimrš   r5   r~   r—   r™   r›   ÚloggerÚwarning_oncerH   rn   r   ÚLinearÚk_projÚv_projÚq_projÚout_proj)
rG   r•   r–   r   r—   r˜   r™   rš   r›   rH   s
   &&&&&&&&&€r*   rF   ÚPegasusAttention.__init__�   s  ø€ ô 	‰ÑÔØ"ŒØ"ŒØŒØ!Õ.ˆŒØŒà�M‰M˜IÕ%¨$¯.©.Ô8ÜØMÈdÏnÉnÐM]Ø$ Y K¨rð3óð ð —}‘} dÕ*ˆŒØ$ŒØ"ŒØ"ŒØÒ §§ Ü×ÑØ*¨4¯>©>×+BÑ+BÐ*Cð D,ð ,ôô —i’i 	¸4Ô@ˆŒÜ—i’i 	¸4Ô@ˆŒÜ—i’i 	¸4Ô@ˆŒÜŸ	š	 )¸TÔBˆŽr,   c                ó   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[R,          RS[ P                  R,          RS[S[,          RS[S[ P                  S[ P                  R,          3,          /# )r"   Úhidden_statesÚkey_value_statesNÚpast_key_valuesr}   r€   r?   )r&   r'   r   r   r   Útuple)r)   rA   s   "€r*   r+   r�   ´   s†   ø€ ÷ H)ñ H)á—|‘|ðH)ñ  Ÿ,™,¨Õ-ðH)ñ  �ð	H)ñ
 Ÿ™ tÕ+ðH)ñ Ñ-Õ.ðH)ñ 
‰u�|‰|™UŸ\™\¨DÕ0Ð0Õ	1ñH)r,   c                óV  € VRJpVP                   RR p. VORNV P                  N5pV P                  V4      P                  V4      P	                  ^^4      p	Rp
Ve`   \        V\        4      '       dH   VP                  P                  V P                  4      p
V'       d   VP                  pMVP                  pMTpV'       d   TMTpV'       d\   VeX   V
'       dP   XP                  V P                  ,          P                  pVP                  V P                  ,          P                  pMÝV P                  V4      pV P!                  V4      p. VP                   RR ORNV P                  N5pVP                  V4      P	                  ^^4      pVP                  V4      P	                  ^^4      pVeV   XP#                  WÞV P                  4      w  rÞV'       d0   \        V\        4      '       d   RVP                  V P                  &   \$        P&                  ! V P(                  P*                  \,        4      pT! V V	VVV3RV P.                  '       g   RMV P0                  RV P2                  /VB w  ppVP4                  ! . VORN5!  P7                  4       pV P9                  V4      pVV3# )z#Input shape: Batch x Time x ChannelNFTr   ç        r~   r/   )r3   r    r¦   Úviewr‹   Ú
isinstancer	   Ú
is_updatedÚgetr›   Úcross_attention_cacheÚself_attention_cacheÚlayersÚkeysÚvaluesr¤   r¥   Úupdater   Úget_interfacerš   Ú_attn_implementationr‘   r‡   r   r~   ÚreshaperŽ   r§   )rG   rª   r«   r¬   r}   r€   Úis_cross_attentionÚinput_shapeÚhidden_shapeÚquery_statesr²   Úcurr_past_key_valuesÚcurrent_statesÚ
key_statesÚvalue_statesÚkv_shapeÚattention_interfacer�   r�   s   &&&&&,             r*   rh   ÚPegasusAttention.forward´   sl  € ð .°TÐ9Ðð $×)Ñ)¨#¨2Ð.ˆà8˜Ð8 bÐ8¨$¯-©-Ñ8ˆð —{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆàˆ
ØÒ&Ü˜/Ô+>×?Ò?Ø,×7Ñ7×;Ñ;¸D¿N¹NÓK�
ß%à+:×+PÑ+PÑ(à+:×+OÑ+OÑ(à'6Ð$ç-?Ñ)À]ˆß /Ò"=Ç*à-×4Ñ4°T·^±^ÕD×IÑIˆJØ/×6Ñ6°t·~±~ÕF×MÑM‰LàŸ™ ^Ó4ˆJØŸ;™; ~Ó6ˆLØF˜×-Ñ-¨c¨rÐ2ÐF°BÐF¸¿¹ÑFˆHØ#Ÿ™¨Ó2×<Ñ<¸QÀÓBˆJØ'×,Ñ,¨XÓ6×@Ñ@ÀÀAÓFˆLàÒ*Ø+?×+FÑ+FÀzÐae×aoÑaoÓ+pÑ(�
ç%¬*°_ÔFY×*ZÒ*ZØAE�O×.Ñ.¨t¯~©~Ñ>ä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð  $Ÿ}Ÿ}˜}‘C°$·,±,ð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆ�\ð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—m‘m KÓ0ˆà˜LÐ(Ð(r,   )rš   r   r•   r    r™   r—   r¤   r›   r–   r§   r¦   r~   r¥   )r¯   FTFNN©NNN©
rn   ro   rp   rq   rr   rF   rh   rt   ru   rv   rw   s   @@r*   r“   r“   Š   s'   ù‡ € ÙG÷%Cõ %C÷NH)÷ H)ò H)r,   r“   c                   óP   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚPegasusEncoderLayeré   c                ó    <€ V ^8„  d   QhRS[ /# ©r"   rš   r   )r)   rA   s   "€r*   r+   Ú PegasusEncoderLayer.__annotate__  s   ø€ ÷ =ñ =™}ñ =r,   c                óz  <€ \         SV `  4        VP                  V n        \	        V P                  VP
                  VP                  VR 7      V n        \        P                  ! V P                  4      V n
        VP                  V n        \        VP                  ,          V n        VP                  V n        \        P                   ! V P                  VP"                  4      V n        \        P                   ! VP"                  V P                  4      V n        \        P                  ! V P                  4      V n        R# ))r•   r–   r   rš   N)rE   rF   Úd_modelr•   r“   Úencoder_attention_headsÚattention_dropoutÚ	self_attnr   Ú	LayerNormÚself_attn_layer_normr   r   Úactivation_functionÚactivation_fnÚactivation_dropoutr£   Úencoder_ffn_dimÚfc1Úfc2Úfinal_layer_norm©rG   rš   rH   s   &&€r*   rF   ÚPegasusEncoderLayer.__init__  sÑ   ø€ Ü‰ÑÔØŸ™ˆŒä)Ø—n‘nØ×4Ñ4Ø×,Ñ,Øô	
ˆŒô %'§L¢L°·±Ó$@ˆÔ!Ø—~‘~ˆŒÜ# F×$>Ñ$>Õ?ˆÔØ"(×";Ñ";ˆÔÜ—9’9˜TŸ^™^¨V×-CÑ-CÓDˆŒÜ—9’9˜V×3Ñ3°T·^±^ÓDˆŒÜ "§¢¨T¯^©^Ó <ˆÖr,   c                ó~   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[S[,          RS[ P                  /# )r"   rª   r}   r€   r?   )r&   r'   r   r   )r)   rA   s   "€r*   r+   rÏ     sE   ø€ ÷ "ñ "á—|‘|ð"ñ Ÿ™ð"ñ Ñ+Õ,ð	"ñ
 
�‰ñ"r,   c                ó  € TpV P                  V4      pV P                  ! RRVRV/VB w  r\        P                  P	                  WP                  V P
                  R7      pWA,           pTpV P                  V4      pV P                  V P                  V4      4      p\        P                  P	                  WP                  V P
                  R7      pV P                  V4      p\        P                  P	                  WP                  V P
                  R7      pWA,           pVP                  \        P                  8X  dK   \        P                  ! VP                  4      P                  R,
          p\        P                   ! W) VR7      pV# )a  
Args:
    hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
    attention_mask (`torch.FloatTensor`): attention mask of size
        `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
rª   r}   r…   iè  )ÚminÚmaxrl   )rÖ   rÔ   r   rŒ   r   r‡   rÝ   rØ   rÛ   rÙ   rÜ   rK   r&   Úfloat16Úfinforã   Úclamp)rG   rª   r}   r€   ÚresidualÚ_Úclamp_values   &&&,   r*   rh   ÚPegasusEncoderLayer.forward  sI  € ð !ˆØ×1Ñ1°-Ó@ˆØŸ>š>ñ 
Ø'ð
à)ð
ð ñ
Ñˆô
 Ÿ™×-Ñ-¨m¿|¹|ÐVZ×VcÑVcÐ-ÓdˆØ Õ0ˆà ˆØ×-Ñ-¨mÓ<ˆØ×*Ñ*¨4¯8©8°MÓ+BÓCˆÜŸ™×-Ñ-¨m×?VÑ?VÐae×anÑanÐ-ÓoˆØŸ™ Ó/ˆÜŸ™×-Ñ-¨m¿|¹|ÐVZ×VcÑVcÐ-ÓdˆØ Õ0ˆà×Ñ¤%§-¡-Ô/ÜŸ+š+ m×&9Ñ&9Ó:×>Ñ>ÀÕEˆKÜ!ŸKšK¨¸<È[ÔYˆMàÐr,   )	rÙ   rØ   r   r•   rÛ   rÜ   rÝ   rÔ   rÖ   ©	rn   ro   rp   rq   rF   rh   rt   ru   rv   rw   s   @@r*   rË   rË      s   ù‡ € ÷=ó =÷$"÷ "ð "r,   rË   c                   óX   a a€ ] tR tRt oRV3R lV 3R llltRV3R lR lltRtVtV ;t# )	ÚPegasusDecoderLayeri9  c                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# )r"   rš   r›   N)r    r(   )r)   rA   s   "€r*   r+   Ú PegasusDecoderLayer.__annotate__:  s   ø€ ÷ =ñ =™}ð =¹¸t½ñ =r,   c           
     ó2  <€ \         SV `  4        VP                  V n        \	        V P                  VP
                  VP                  R R VVR7      V n        VP                  V n        \        VP                  ,          V n        VP                  V n        \        P                  ! V P                  4      V n        \	        V P                  VP
                  VP                  R VVR7      V n        \        P                  ! V P                  4      V n        \        P$                  ! V P                  VP&                  4      V n        \        P$                  ! VP&                  V P                  4      V n        \        P                  ! V P                  4      V n        R# )T)r•   r–   r   r—   r™   rš   r›   )r   r—   rš   r›   N)rE   rF   rÑ   r•   r“   Údecoder_attention_headsrÓ   rÔ   r   r   r×   rØ   rÙ   r   rÕ   rÖ   Úencoder_attnÚencoder_attn_layer_normr£   Údecoder_ffn_dimrÛ   rÜ   rÝ   )rG   rš   r›   rH   s   &&&€r*   rF   ÚPegasusDecoderLayer.__init__:  s   ø€ Ü‰ÑÔØŸ™ˆŒä)Ø—n‘nØ×4Ñ4Ø×,Ñ,ØØØØô
ˆŒð —~‘~ˆŒÜ# F×$>Ñ$>Õ?ˆÔØ"(×";Ñ";ˆÔä$&§L¢L°·±Ó$@ˆÔ!Ü,Ø�N‰NØ×*Ñ*Ø×,Ñ,ØØØô
ˆÔô (*§|¢|°D·N±NÓ'CˆÔ$Ü—9’9˜TŸ^™^¨V×-CÑ-CÓDˆŒÜ—9’9˜V×3Ñ3°T·^±^ÓDˆŒÜ "§¢¨T¯^©^Ó <ˆÖr,   c                ó  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[,          R	S[ P                  /# )
r"   rª   r}   NÚencoder_hidden_statesÚencoder_attention_maskr¬   Ú	use_cacher€   r?   )r&   r'   r   rœ   r   r   )r)   rA   s   "€r*   r+   rï   Y  sŒ   ø€ ÷ :ñ :á—|‘|ð:ñ Ÿ™ tÕ+ð:ñ  %Ÿ|™|¨dÕ2ð	:ñ
 !&§¡¨tÕ 3ð:ñ  �ð:ñ ˜$•;ð:ñ Ñ+Õ,ð:ñ 
�‰ñ:r,   c                ó&  € TpV P                  V4      pV P                  ! RRVRVRV/VB w  r\        P                  P	                  WP                  V P
                  R7      pW�,           pVem   TpV P                  V4      pV P                  ! RRVRVRVRV/VB w  r\        P                  P	                  WP                  V P
                  R7      pW�,           pTpV P                  V4      pV P                  V P                  V4      4      p\        P                  P	                  WP                  V P
                  R7      pV P                  V4      p\        P                  P	                  WP                  V P
                  R7      pW�,           pV# )a   
Args:
    hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
    attention_mask (`torch.FloatTensor`): attention mask of size
        `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
    encoder_hidden_states (`torch.FloatTensor`):
        cross attention input to the layer of shape `(batch, seq_len, embed_dim)`
    encoder_attention_mask (`torch.FloatTensor`): encoder attention mask of size
        `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
    past_key_values (`Cache`): cached past key and value projection states
rª   r¬   r}   r…   r«   rl   )rÖ   rÔ   r   rŒ   r   r‡   ró   rò   rÝ   rØ   rÛ   rÙ   rÜ   )
rG   rª   r}   r÷   rø   r¬   rù   r€   rç   rè   s
   &&&&&&&,  r*   rh   ÚPegasusDecoderLayer.forwardY  s   € ð* !ˆØ×1Ñ1°-Ó@ˆð  Ÿ>š>ñ 
Ø'ð
à+ð
ð *ð
ð ñ	
Ñˆô Ÿ™×-Ñ-¨m¿|¹|ÐVZ×VcÑVcÐ-ÓdˆØ Õ0ˆð !Ò,Ø$ˆHØ ×8Ñ8¸ÓGˆMà#×0Ò0ñ  Ø+ð à!6ð ð  6ð ð !0ð	 ð
 ñ ÑˆMô ŸM™M×1Ñ1°-Ç<Á<ÐZ^×ZgÑZgÐ1ÓhˆMØ$Õ4ˆMð !ˆØ×-Ñ-¨mÓ<ˆØ×*Ñ*¨4¯8©8°MÓ+BÓCˆÜŸ™×-Ñ-¨m×?VÑ?VÐae×anÑanÐ-ÓoˆØŸ™ Ó/ˆÜŸ™×-Ñ-¨m¿|¹|ÐVZ×VcÑVcÐ-ÓdˆØ Õ0ˆàÐr,   )rÙ   rØ   r   r•   rò   ró   rÛ   rÜ   rÝ   rÔ   rÖ   rm   )NNNNTrë   rw   s   @@r*   rí   rí   9  s   ù‡ € ÷=õ =÷>:÷ :ò :r,   rí   c                   ó~   a a€ ] tR tRt oRtRtRtRtRtRt	]
P                  ! 4       V 3R l4       tV3R ltRtVtV ;t# )ÚPegasusPreTrainedModeli–  ÚmodelTc                ó$  <€ \         SV `  V4       \        V\        4      '       d2   \        P
                  ! VP                  VP                  4       4       R # \        V\        4      '       d#   \        P                  ! VP                  4       R # R # rm   )rE   Ú_init_weightsr±   r:   ÚinitÚcopy_rM   r]   ÚPegasusForConditionalGenerationÚzeros_Úfinal_logits_bias)rG   ry   rH   s   &&€r*   r   Ú$PegasusPreTrainedModel._init_weights   s_   ø€ ä‰Ñ˜fÔ%Ü�fÔB×CÒCÜ�JŠJ�v—}‘} f×&:Ñ&:Ó&<Ö=Ü˜Ô ?×@Ò@Ü�KŠK˜×0Ñ0Ö1ñ Ar,   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # rÎ   r   )r)   rA   s   "€r*   r+   Ú#PegasusPreTrainedModel.__annotate__–  s   ø‡ ‚ áÑò r,   rl   )rn   ro   rp   rq   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphr&   rs   r   Ú__annotate_func__rt   ru   rv   rw   s   @@r*   rý   rý   –  sI   ù‡ € ð  ÐØ&*Ð#ØÐØ€NØÐØ!Ðà
‡]‚]ƒ_ô2ó ð2÷ … r,   rý   c                   ó¦   a a€ ] tR tRt oRtR]R]/tV3R lV 3R lltV3R lR lt	V3R	 lR
 lt
]]]RV3R lR ll4       4       4       tRtVtV ;t# )ÚPegasusEncoderi©  zÎ
Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a
[`PegasusEncoderLayer`].

Args:
    config: PegasusConfig
    embed_tokens (nn.Embedding): output embedding
rª   Ú
attentionsc                ó    <€ V ^8„  d   QhRS[ /# rÎ   r   )r)   rA   s   "€r*   r+   ÚPegasusEncoder.__annotate__¸  s   ø€ ÷ ñ ™}ñ r,   c                óà  <€ \         SV `  V4       VP                  V n        VP                  V n        VP
                  pVP                  V n        VP                  V n	        VP                  '       d   \        P                  ! V4      MR V n        \        P                  ! VP                   W P                  4      V n        \%        VP                  VV P                  4      V n        \        P(                  ! \+        VP,                  4       Uu. uF  p\/        V4      NK  	  up4      V n        \        P2                  ! VP
                  4      V n        RV n        V P9                  4        R# u upi )ç      ð?FN)rE   rF   r   Úencoder_layerdropÚ	layerdroprÑ   r$   r>   Úmax_position_embeddingsÚmax_source_positionsÚscale_embeddingÚmathÚsqrtÚembed_scaler   Ú	EmbeddingÚ
vocab_sizeÚembed_tokensr:   Úembed_positionsÚ
ModuleListrP   Úencoder_layersrË   r¶   rÕ   Ú
layer_normÚgradient_checkpointingÚ	post_init)rG   rš   r•   rè   rH   s   &&  €r*   rF   ÚPegasusEncoder.__init__¸  s  ø€ Ü‰Ñ˜Ô à—~‘~ˆŒØ×1Ñ1ˆŒà—N‘Nˆ	Ø!×.Ñ.ˆÔØ$*×$BÑ$BˆÔ!Ø39×3I×3IÐ3Iœ4Ÿ9š9 YÔ/ÈsˆÔäŸLšL¨×):Ñ):¸I×GWÑGWÓXˆÔäCØ×*Ñ*ØØ×Ñó 
ˆÔô
 —m’mÌ%ÐPV×PeÑPeÔJfÓ$gÑJfÀQÔ%8¸Ö%@ÑJfÑ$gÓhˆŒÜŸ,š, v§~¡~Ó6ˆŒà&+ˆÔ#à�‰Öùò %hs   ÄE+c                ó    <€ V ^8„  d   QhRS[ /# ©r"   Únew_num_position_embeddingsr@   )r)   rA   s   "€r*   r+   r  Ñ  ó   ø€ ÷ -ñ -Ácñ -r,   c                ó°  € \         P                  RV R24       WP                  n        \	        V P                  P                  V P                  P
                  V P                  4      V n        \        P                  ! V P                  P                  V P                  P                  4       4       V P                  P                  V P                  4       R# ©á‹  
Resizes position embeddings matrix of the model if `new_num_position_embeddings !=
config.max_position_embeddings`.

Arguments:
    new_num_position_embeddings (`int`):
        The number of new position embeddings. If position embeddings are learned, increasing the size will add
        newly initialized vectors at the end, whereas reducing the size will remove vectors from the end. If
        position embeddings are not learned (*e.g.* sinusoidal position embeddings), increasing the size will
        add correct vectors at the end following the position encoding algorithm, whereas reducing the size
        will remove vectors from the end.
z(Setting `config.max_position_embeddings=z`...N©r¡   Úinforš   r  r:   rÑ   r>   r"  r  r  rM   r]   Útore   ©rG   r+  s   &&r*   Úresize_position_embeddingsÚ)PegasusEncoder.resize_position_embeddingsÑ  óš   € ô 	�‰Ð>Ð?ZÐ>[Ð[_Ð`ÔaØ.I�‰Ô+äCØ�K‰K×/Ñ/Ø�K‰K×ÑØ×Ñó 
ˆÔô
 	�
Š
�4×'Ñ'×.Ñ.°×0DÑ0D×0RÑ0RÓ0TÔUØ×Ñ×Ñ §¡Ö,r,   c                ó4   <€ V ^8„  d   QhRS[ P                  /# ©r"   r?   ©r   r  )r)   rA   s   "€r*   r+   r  é  ó   ø€ ÷ $ñ $©¯©ñ $r,   c                ó   € V P                   # ©z(
Returns the position embeddings matrix
©r"  ©rG   s   &r*   Úget_position_embeddingsÚ&PegasusEncoder.get_position_embeddingsé  ó   € ð ×#Ñ#Ð#r,   c                ó6   <€ V ^8„  d   QhRS[ S[,          RS[/# ©r"   r€   r?   )r   r   r   )r)   rA   s   "€r*   r+   r  ò  s'   ø€ ÷ -
ñ -
ñ
 Ñ+Õ,ð-
ñ 
ñ-
r,   c                óŠ  € VR J VR J,          '       d   \        R4      hVf#   V P                  V4      V P                  ,          pVP                  R R pV P	                  V4      pW6,           p\
        P                  P                  WpP                  V P                  R7      p\        V P                  VVR7      p\        V P                  4       FU  w  r‰Rp
V P                  '       d)   \        P                  ! . 4      pW°P                  8  d   Rp
V
'       d   KK  V	! VV3/ VB pKW  	  V P!                  V4      p\#        VR7      # )Nz:You must specify exactly one of input_ids or inputs_embedsr…   )rš   Úinputs_embedsr}   FT)Úlast_hidden_stater/   )r5   r!  r  r3   r"  r   rŒ   r   r‡   r   rš   Ú	enumerater¶   r&   Úrandr  r%  r   )rG   r#   r}   rE  r€   r¾   Ú	embed_posrª   ÚidxÚencoder_layerÚto_dropÚdropout_probabilitys   &&&&,       r*   rh   ÚPegasusEncoder.forwardï  s-  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ Ø ×-Ñ-¨iÓ8¸4×;KÑ;KÕKˆMà#×)Ñ)¨#¨2Ð.ˆØ×(Ñ(¨Ó5ˆ	à%Õ1ˆäŸ™×-Ñ-¨m¿|¹|ÐVZ×VcÑVcÐ-Ódˆä2Ø—;‘;Ø'Ø)ô
ˆô #,¨D¯K©KÖ"8ÑˆCàˆGØ�}�}ˆ}Ü&+§j¢j°£nÐ#Ø&¯©Ô7Ø"�Gç‘7Ù -Ø!Ø"ñ!ð ñ!’ñ #9ð Ÿ™¨Ó6ˆäØ+ô
ð 	
r,   )
r   r"  r  r!  r&  r%  r  r¶   r  r>   rÈ   )rn   ro   rp   rq   rr   rË   r“   Ú_can_record_outputsrF   r4  r?  r   r   r   rh   rt   ru   rv   rw   s   @@r*   r  r  ©  sl   ù‡ € ñð 	Ð,ØÐ&ðÐ÷
ó ÷2-ð -÷0$ð $ð  ØØ÷-
ñ -
ó ó ó  ÷-
ð -
r,   r  c                   óÎ   a a€ ] tR tRt oRtR]R]! ]^RR7      R]! ]^RR7      /tV3R	 lV 3R
 llt	V3R lR lt
V3R lR lt]]]RV3R lR ll4       4       4       tRtVtV ;t# )ÚPegasusDecoderi"  z¾
Transformer decoder consisting of *config.decoder_layers* layers. Each layer is a [`PegasusDecoderLayer`]

Args:
    config: PegasusConfig
    embed_tokens (nn.Embedding): output embedding
rª   r  rÔ   )ÚindexÚ
layer_nameÚcross_attentionsrò   c                ó    <€ V ^8„  d   QhRS[ /# rÎ   r   )r)   rA   s   "€r*   r+   ÚPegasusDecoder.__annotate__1  s   ø€ ÷ ñ ™}ñ r,   c           
     ó  <€ \         SV `  V4       VP                  V n        VP                  V n        VP
                  V n        VP                  V n        VP                  '       d!   \        P                  ! VP                  4      MR V n        \        P                  ! VP                   VP                  V P                  4      V n        \%        VP                  VP                  V P                  4      V n        \        P(                  ! \+        VP,                  4       Uu. uF  p\/        WR7      NK  	  up4      V n        \        P2                  ! VP                  4      V n        RV n        V P9                  4        R# u upi )r  )r›   FN)rE   rF   r   Údecoder_layerdropr  r$   r>   r  Úmax_target_positionsr  r  r  rÑ   r  r   r  r   r!  r:   r"  r#  rP   Údecoder_layersrí   r¶   rÕ   r%  r&  r'  )rG   rš   ÚirH   s   && €r*   rF   ÚPegasusDecoder.__init__1  s  ø€ Ü‰Ñ˜Ô Ø—~‘~ˆŒØ×1Ñ1ˆŒØ!×.Ñ.ˆÔØ$*×$BÑ$BˆÔ!Ø8>×8N×8NÐ8Nœ4Ÿ9š9 V§^¡^Ô4ÐTWˆÔäŸLšL¨×):Ñ):¸F¿N¹NÈD×L\ÑL\Ó]ˆÔäCØ×*Ñ*Ø�N‰NØ×Ñó 
ˆÔô
 —m’mÔW\Ð]c×]rÑ]rÔWsÓ$tÑWsÐRSÔ%8¸×%MÑWsÑ$tÓuˆŒÜŸ,š, v§~¡~Ó6ˆŒà&+ˆÔ#à�‰Öùò %us   Ä E?c                ó    <€ V ^8„  d   QhRS[ /# r*  r@   )r)   rA   s   "€r*   r+   rV  G  r,  r,   c                ó°  € \         P                  RV R24       WP                  n        \	        V P                  P                  V P                  P
                  V P                  4      V n        \        P                  ! V P                  P                  V P                  P                  4       4       V P                  P                  V P                  4       R# r.  r0  r3  s   &&r*   r4  Ú)PegasusDecoder.resize_position_embeddingsG  r6  r,   c                ó4   <€ V ^8„  d   QhRS[ P                  /# r8  r9  )r)   rA   s   "€r*   r+   rV  _  r:  r,   c                ó   € V P                   # r<  r=  r>  s   &r*   r?  Ú&PegasusDecoder.get_position_embeddings_  rA  r,   c                ó6   <€ V ^8„  d   QhRS[ S[,          RS[/# rC  )r   r   r   )r)   rA   s   "€r*   r+   rV  h  s+   ø€ ÷ S
ñ S
ñ Ñ+Õ,ðS
ñ 
3ñS
r,   c           
     óþ  € VR J VR J,          '       d   \        R4      hVf   V P                  V4      pW`P                  ,          pV'       do   Vfk   Vf   V P                  P                  '       d5   \        \        V P                  R7      \        V P                  R7      4      M\        V P                  R7      pVP                  4       R R w  ršVe   VP                  4       M^ p\        P                  ! W¦P                  R7      V,           pVf;   \        4       '       g+   Wº,           p\        P                  ! W�VP                  R7      p\        V\
        4      '       d   VP                  MTp\!        V P                  VVVR7      p\#        V P                  VVVR7      pV P%                  Wš3W¼R7      pVV,           p\&        P(                  P+                  VV P*                  V P,                  R7      p\/        V P0                  4       FR  w  ppV P,                  '       d*   \        P2                  ! . 4      pVV P4                  8  d   KA  V! VVV3RVR	VR
V/VB pKT  	  V P7                  V4      p\9        VVR7      # )NzTYou cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time)rš   ©re   )rš   rE  r}   r¬   )rš   rE  r}   r÷   )rb   r…   rø   r¬   rù   )rF  r¬   r/   )r5   r!  r  rš   Úis_encoder_decoderr	   r   r‰   Úget_seq_lengthr&   rf   re   r   Úonesr±   rµ   r   r   r"  r   rŒ   r   r‡   rG  r¶   rH  r  r%  r   )rG   r#   r}   r÷   rø   r¬   rE  rù   r€   Ú
batch_sizeÚ
seq_lengthra   rb   Úmask_seq_lengthÚself_attn_cacheÚcausal_maskÚ	positionsrª   rJ  Údecoder_layerrM  s   &&&&&&&&,            r*   rh   ÚPegasusDecoder.forwarde  sY  € ð ˜Ð -°tÐ";×<Ô<ÜÐsÓtÐtàÒ Ø ×-Ñ-¨iÓ8ˆMð &×(8Ñ(8Õ8ˆ÷ ˜Ò0ð )Ò4¸¿¹×8V×8VÐ8Vô $¤L¸¿¹Ô$DÄlÐZ^×ZeÑZeÔFfÔgä!¨¯©Ô5ð ð "/×!3Ñ!3Ó!5°c°rÐ!:Ñˆ
ØETÒE` ×!?Ñ!?Ô!AÐfgÐÜ—|’| J×7KÑ7KÔLÐOeÕeˆàÒ!Ô*B×*DÒ*Dà4ÕAˆOÜ"ŸZšZ¨
ÈM×L`ÑL`ÔaˆNô ˜/Ô+>×?Ò?ð ×0Ò0à ð 	ô )Ø—;‘;Ø'Ø)Ø+ô	
ˆô ";Ø—;‘;Ø'Ø1Ø"7ô	"
Ðð ×(Ñ(¨*Ð)AÐCYÐ(Óuˆ	Ø%¨	Õ1ˆÜŸ™×-Ñ-¨m¸t¿|¹|ÐVZ×VcÑVcÐ-Ódˆä"+¨D¯K©KÖ"8ÑˆC�à�}�}ˆ}Ü&+§j¢j°£nÐ#Ø&¨¯©Ô7Ùá)ØØØ%ñð (>ð	ð
 !0ðð $ðð ñŠMñ #9ð" Ÿ™¨Ó6ˆä8Ø+Ø+ô
ð 	
r,   )
r   r"  r  r!  r&  r%  r  r¶   rY  r>   )NNNNNNN)rn   ro   rp   rq   rr   rí   r   r“   rO  rF   r4  r?  r   r   r   rh   rt   ru   rv   rw   s   @@r*   rQ  rQ  "  s‰   ù‡ € ñð 	Ð,Ø‘nÐ%5¸QÈ;ÔWØ™NÐ+;À1ÐQ_Ô`ðÐ÷ó ÷,-ð -÷0$ð $ð  ØØ÷S
ñ S
ó ó ó  ÷S
ð S
r,   rQ  c                   ó¤   a a€ ] tR tRt oRRRR/tV3R lV 3R lltR tR tV3R	 lR
 ltV3R lR lt	]
]RV3R lR ll4       4       tRtVtV ;t# )ÚPegasusModeli¾  zdecoder.embed_tokens.weightzshared.weightzencoder.embed_tokens.weightc                ó    <€ V ^8„  d   QhRS[ /# rÎ   r   )r)   rA   s   "€r*   r+   ÚPegasusModel.__annotate__Å  s   ø€ ÷ 
ñ 
™}ñ 
r,   c                ó   <€ \         SV `  V4       VP                  VP                  r2\        P
                  ! W1P                  V4      V n        \        V4      V n	        \        V4      V n        V P                  4        R # rm   )rE   rF   r$   r   r   r  rÑ   Úsharedr  ÚencoderrQ  Údecoderr'  )rG   rš   r>   r   rH   s   &&  €r*   rF   ÚPegasusModel.__init__Å  s]   ø€ Ü‰Ñ˜Ô à"(×"5Ñ"5°v×7HÑ7H�ZÜ—l’l :¯~©~¸{ÓKˆŒä% fÓ-ˆŒÜ% fÓ-ˆŒð 	�‰Ör,   c                ó   € V P                   # rm   )rv  r>  s   &r*   Úget_input_embeddingsÚ!PegasusModel.get_input_embeddingsÑ  s   € Ø�{‰{Ðr,   c                ó~   € Wn         V P                   V P                  n        V P                   V P                  n        R # rm   )rv  rw  r!  rx  ©rG   r|   s   &&r*   Úset_input_embeddingsÚ!PegasusModel.set_input_embeddingsÔ  s'   € ØŒØ$(§K¡Kˆ�‰Ô!Ø$(§K¡Kˆ�‰Ö!r,   c                ó    <€ V ^8„  d   QhRS[ /# r*  r@   )r)   rA   s   "€r*   r+   rt  Ù  s   ø€ ÷ Mñ MÁcñ Mr,   c                ó’   € WP                   n        V P                  P                  V4       V P                  P                  V4       R# ©r/  N)rš   r  rw  r4  rx  r3  s   &&r*   r4  Ú'PegasusModel.resize_position_embeddingsÙ  s5   € ð /J�‰Ô+Ø�‰×/Ñ/Ð0KÔLØ�‰×/Ñ/Ð0KÖLr,   c                óD   <€ V ^8„  d   QhRS[ S[P                  ,          /# r8  ©r­   r   r  )r)   rA   s   "€r*   r+   rt  ê  s    ø€ ÷ `ñ `©©r¯|©|Õ)<ñ `r,   c                ój   € V P                   P                  4       V P                  P                  4       3# r<  )rw  r?  rx  r>  s   &r*   r?  Ú$PegasusModel.get_position_embeddingsê  s)   € ð —‘×4Ñ4Ó6¸¿¹×8\Ñ8\Ó8^Ð_Ð_r,   c                ó–  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[ P                  ,          R,          RS[R,          RS[ P                  R,          R	S[ P                  R,          R
S[R,          RS[S[,          RS[S[,          /# )r"   r#   Nr}   Údecoder_input_idsÚdecoder_attention_maskÚencoder_outputsr¬   rE  Údecoder_inputs_embedsrù   r€   r?   )	r&   r'   r­   rS   r   rœ   r   r   r   )r)   rA   s   "€r*   r+   rt  ò  sà   ø€ ÷ R
ñ R
á—<‘< $Õ&ðR
ñ Ÿ™ tÕ+ðR
ñ !Ÿ<™<¨$Õ.ð	R
ñ
 !&§¡¨tÕ 3ðR
ñ ™u×0Ñ0Õ1°DÕ8ðR
ñ  �ðR
ñ —|‘| dÕ*ðR
ñ  %Ÿ|™|¨dÕ2ðR
ñ ˜$•;ðR
ñ Ñ+Õ,ðR
ñ 
Ñ#Õ	#ñR
r,   c
                ó
  € Vf   V P                   ! RRVRVRV/V
B pM]\        V\        4      '       gH   \        V^ ,          \        V4      ^8”  d
   V^,          MR\        V4      ^8”  d
   V^,          MRR7      pV P                  ! RRVRVRV^ ,          RVRVRVR	V	/V
B p\        VP                  VP                  VP                  VP                  VP                  VP                  VP                  VP                  R
7      # )a�  
decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Indices of decoder input sequence tokens in the vocabulary.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are decoder input IDs?](../glossary#decoder-input-ids)

    Pegasus uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If
    `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
    `past_key_values`).
decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
    be used by default.

Example:

```python
>>> from transformers import AutoTokenizer, PegasusModel

>>> tokenizer = AutoTokenizer.from_pretrained("google/pegasus-large")
>>> model = PegasusModel.from_pretrained("google/pegasus-large")

>>> inputs = tokenizer("Studies have been shown that owning a dog is good for you", return_tensors="pt")
>>> decoder_inputs = tokenizer("Studies show that", return_tensors="pt")
>>> outputs = model(input_ids=inputs.input_ids, decoder_input_ids=decoder_inputs.input_ids)

>>> last_hidden_states = outputs.last_hidden_state
>>> list(last_hidden_states.shape)
[1, 4, 1024]
```Nr#   r}   rE  )rF  rª   r  r÷   rø   r¬   rù   )rF  r¬   Údecoder_hidden_statesÚdecoder_attentionsrT  Úencoder_last_hidden_stater÷   Úencoder_attentionsrl   )rw  r±   r   Úlenrx  r   rF  r¬   rª   r  rT  )rG   r#   r}   rŠ  r‹  rŒ  r¬   rE  r�  rù   r€   Údecoder_outputss   &&&&&&&&&&, r*   rh   ÚPegasusModel.forwardð  sC  € ð` Ò"Ø"Ÿlšlñ Ø#ðà-ðð ,ðð ñ	‰Oô ˜O¬_×=Ò=Ü-Ø"1°!Õ"4Ü47¸Ó4HÈ1Ô4L˜o¨aÖ0ÐRVÜ14°_Ó1EÈÔ1I˜?¨1Ö-ÈtôˆOð Ÿ,š,ñ 	
Ø'ð	
à1ð	
ð #2°!Õ"4ð	
ð $2ð		
ð
 ,ð	
ð 0ð	
ð  ð	
ð ñ	
ˆô "Ø-×?Ñ?Ø+×;Ñ;Ø"1×"?Ñ"?Ø.×9Ñ9Ø,×=Ñ=Ø&5×&GÑ&GØ"1×"?Ñ"?Ø.×9Ñ9ô	
ð 		
r,   )rx  rw  rv  )	NNNNNNNNN)rn   ro   rp   rq   Ú_tied_weights_keysrF   r{  r  r4  r?  r   r   rh   rt   ru   rv   rw   s   @@r*   rr  rr  ¾  so   ù‡ € ð 	& Ø% ðÐ÷

ó 
òò0÷
Mð M÷"`ð `ð Ø÷R
ñ R
ó ó ÷R
ð R
r,   rr  zY
    The PEGASUS Model with a language modeling head. Can be used for summarization.
    )Úcustom_introc                   óÞ   a a€ ] tR tRt oRtR.tRR/tV3R lV 3R lltRV3R lV 3R	 llltV3R
 lR lt	V3R lR lt
V3R lR lt]]RV3R lR ll4       4       tV3R lR ltRtVtV ;t# )r  iG  rþ   r  úlm_head.weightzmodel.shared.weightc                ó    <€ V ^8„  d   QhRS[ /# rÎ   r   )r)   rA   s   "€r*   r+   Ú,PegasusForConditionalGeneration.__annotate__S  s   ø€ ÷ ñ ™}ñ r,   c                ó�  <€ \         SV `  V4       \        V4      V n        \        P
                  ! \        P                  ! ^V P                  P                  P                  34      4      V n
        \        P                  ! VP                  V P                  P                  P                  RR7      V n        V P                  4        R# )r0   FrŸ   N)rE   rF   rr  rþ   r   ÚBufferr&   Úzerosrv  Únum_embeddingsr  r£   rÑ   Úlm_headr'  rÞ   s   &&€r*   rF   Ú(PegasusForConditionalGeneration.__init__S  s€   ø€ Ü‰Ñ˜Ô Ü! &Ó)ˆŒ
Ü!#§¢¬5¯;ª;¸¸4¿:¹:×;LÑ;L×;[Ñ;[Ð7\Ó+]Ó!^ˆÔÜ—y’y §¡°·±×1BÑ1B×1QÑ1QÐX]Ô^ˆŒð 	�‰Ör,   c                óT   <€ V ^8„  d   QhRS[ RS[ R,          RS[RS[P                  /# )r"   Únew_num_tokensÚpad_to_multiple_ofNÚmean_resizingr?   )r(   rœ   r   r  )r)   rA   s   "€r*   r+   r›  \  s5   ø€ ÷ ñ Ù!ðÙ7:¸TµzðÙY]ðá	�‰ñr,   c                ó€   <€ \         SV `  WV4      pV P                  VP                  P                  ^ ,          4       V# )r1   )rE   Úresize_token_embeddingsÚ_resize_final_logits_biasrM   r3   )rG   r£  r¤  r¥  Únew_embeddingsrH   s   &&&& €r*   r§  Ú7PegasusForConditionalGeneration.resize_token_embeddings\  s<   ø€ ô ™Ñ8¸Ð]jÓkˆØ×&Ñ& ~×'<Ñ'<×'BÑ'BÀ1Õ'EÔFØÐr,   c                ó$   <€ V ^8„  d   QhRS[ RR/# )r"   r£  r?   Nr@   )r)   rA   s   "€r*   r+   r›  c  s   ø€ ÷ 5ñ 5¹ð 5Àñ 5r,   c                ób  € V P                   P                  R,          pW8:  d   V P                   RRV13,          pMX\        P                  ! ^W,
          3V P                   P                  R7      p\        P
                  ! V P                   V.^R7      p\        P                  ! V4      V n         R# )r0   r.   Nre  r„   r/   )r  r3   r&   rž  re   Úcatr   r�  )rG   r£  Úold_num_tokensÚnew_biasÚ
extra_biass   &&   r*   r¨  Ú9PegasusForConditionalGeneration._resize_final_logits_biasc  sˆ   € Ø×/Ñ/×5Ñ5°bÕ9ˆØÔ+Ø×-Ñ-¨a°°.°Ð.@ÕA‰HäŸš a¨Õ)HÐ%IÐRV×RhÑRh×RoÑRoÔpˆJÜ—y’y $×"8Ñ"8¸*Ð!EÈ1ÔMˆHÜ!#§¢¨8Ó!4ˆÖr,   c                ó    <€ V ^8„  d   QhRS[ /# r*  r@   )r)   rA   s   "€r*   r+   r›  l  s   ø€ ÷ Sñ SÁcñ Sr,   c                óº   € WP                   n        V P                  P                  P	                  V4       V P                  P
                  P	                  V4       R# rƒ  )rš   r  rþ   rw  r4  rx  r3  s   &&r*   r4  Ú:PegasusForConditionalGeneration.resize_position_embeddingsl  sA   € ð /J�‰Ô+Ø�
‰
×Ñ×5Ñ5Ð6QÔRØ�
‰
×Ñ×5Ñ5Ð6QÖRr,   c                óD   <€ V ^8„  d   QhRS[ S[P                  ,          /# r8  r†  )r)   rA   s   "€r*   r+   r›  }  s    ø€ ÷ lñ l©©r¯|©|Õ)<ñ lr,   c                ó’   € V P                   P                  P                  4       V P                   P                  P                  4       3# r<  )rþ   rw  r?  rx  r>  s   &r*   r?  Ú7PegasusForConditionalGeneration.get_position_embeddings}  s5   € ð —
‘
×"Ñ"×:Ñ:Ó<¸d¿j¹j×>PÑ>P×>hÑ>hÓ>jÐkÐkr,   c                ó¾  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[ P                  ,          R,          RS[R,          RS[ P                  R,          R	S[ P                  R,          R
S[ P                  R,          RS[R,          RS[S[,          RS[S[,          /# )r"   r#   Nr}   rŠ  r‹  rŒ  r¬   rE  r�  Úlabelsrù   r€   r?   )	r&   r'   r­   rS   r   rœ   r   r   r   )r)   rA   s   "€r*   r+   r›  …  só   ø€ ÷ ]
ñ ]
á—<‘< $Õ&ð]
ñ Ÿ™ tÕ+ð]
ñ !Ÿ<™<¨$Õ.ð	]
ñ
 !&§¡¨tÕ 3ð]
ñ ™u×0Ñ0Õ1°DÕ8ð]
ñ  �ð]
ñ —|‘| dÕ*ð]
ñ  %Ÿ|™|¨dÕ2ð]
ñ —‘˜tÕ#ð]
ñ ˜$•;ð]
ñ Ñ+Õ,ð]
ñ 
‘Õ	 ñ]
r,   c                óª  € V	e\   V
'       d   \         P                  R4       Rp
Vf9   Vf5   \        W�P                  P                  V P                  P
                  4      pV P                  ! V3RVRVRVRVRVR	VR
VRV
/VB pV P                  VP                  4      V P                  ,           pRpV	eG   \        4       pV! VP                  RV P                  P                  4      V	P                  R4      4      p\        VVVP                  VP                  VP                   VP"                  VP$                  VP&                  VP(                  R7	      # )a¡  
decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Indices of decoder input sequence tokens in the vocabulary.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are decoder input IDs?](../glossary#decoder-input-ids)

    Pegasus uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If
    `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
    `past_key_values`).
decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
    be used by default.
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

Example Summarization:

```python
>>> from transformers import AutoTokenizer, PegasusForConditionalGeneration

>>> model = PegasusForConditionalGeneration.from_pretrained("google/pegasus-xsum")
>>> tokenizer = AutoTokenizer.from_pretrained("google/pegasus-xsum")

>>> ARTICLE_TO_SUMMARIZE = (
...     "PG&E stated it scheduled the blackouts in response to forecasts for high winds "
...     "amid dry conditions. The aim is to reduce the risk of wildfires. Nearly 800 thousand customers were "
...     "scheduled to be affected by the shutoffs which were expected to last through at least midday tomorrow."
... )
>>> inputs = tokenizer(ARTICLE_TO_SUMMARIZE, max_length=1024, return_tensors="pt")

>>> # Generate Summary
>>> summary_ids = model.generate(inputs["input_ids"])
>>> tokenizer.batch_decode(summary_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"California's largest electricity provider has turned off power to hundreds of thousands of customers."
```
NzJThe `use_cache` argument is changed to `False` since `labels` is provided.Fr}   rŠ  rŒ  r‹  r¬   rE  r�  rù   )	ÚlossÚlogitsr¬   r�  r�  rT  r‘  r÷   r’  r/   )r¡   Úwarningr8   rš   r$   r%   rþ   r   rF  r  r   r°   r   r   r¬   r�  r�  rT  r‘  r÷   r’  )rG   r#   r}   rŠ  r‹  rŒ  r¬   rE  r�  r¹  rù   r€   ÚoutputsÚ	lm_logitsÚmasked_lm_lossÚloss_fcts   &&&&&&&&&&&,    r*   rh   Ú'PegasusForConditionalGeneration.forwardƒ  so  € ðr ÒßÜ—‘ÐkÔlØˆIØ Ò(Ð-BÒ-JÜ$6ØŸK™K×4Ñ4°d·k±k×6XÑ6Xó%Ð!ð '+§j¢jØñ'
à)ð'
ð 0ð'
ð ,ð	'
ð
 $:ð'
ð ,ð'
ð (ð'
ð #8ð'
ð  ð'
ð ñ'
ˆð —L‘L ×!:Ñ!:Ó;¸d×>TÑ>TÕTˆ	àˆØÒÜ'Ó)ˆHÙ% i§n¡n°R¸¿¹×9OÑ9OÓ&PÐRX×R]ÑR]Ð^`ÓRaÓbˆNäØØØ#×3Ñ3Ø")×"?Ñ"?Ø&×9Ñ9Ø$×5Ñ5Ø&-×&GÑ&GØ")×"?Ñ"?Ø&×9Ñ9ô

ð 
	
r,   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r"   r¹  )r&   r'   )r)   rA   s   "€r*   r+   r›  ä  s   ø€ ÷ hñ h¹E¿L¹Lñ hr,   c                ój   € \        WP                  P                  V P                  P                  4      # rm   )r8   rš   r$   r%   )rG   r¹  s   &&r*   Ú%prepare_decoder_input_ids_from_labelsÚEPegasusForConditionalGeneration.prepare_decoder_input_ids_from_labelsä  s#   € Ü! &¯+©+×*BÑ*BÀDÇKÁK×DfÑDfÓgÐgr,   )r  r   rþ   )NT)
NNNNNNNNNN)rn   ro   rp   rq   r	  Ú_keys_to_ignore_on_load_missingr–  rF   r§  r¨  r4  r?  r   r   rh   rÅ  rt   ru   rv   rw   s   @@r*   r  r  G  s‘   ù‡ € ð  ÐØ':Ð&;Ð#àÐ/ðÐ÷ó ÷õ ÷5ð 5÷Sð S÷"lð lð Ø÷]
ñ ]
ó ó ð]
÷~h÷ hð hr,   r  c                   ó<   a a€ ] tR tRt oRtV 3R ltR tRtVtV ;t	# )ÚPegasusDecoderWrapperié  z±
This wrapper class is a helper class to correctly load pretrained checkpoints when the causal language model is
used in combination with the [`EncoderDecoderModel`] framework.
c                óf   <€ \         SV `  V4       \        V4      V n        V P	                  4        R # rm   )rE   rF   rQ  rx  r'  rÞ   s   &&€r*   rF   ÚPegasusDecoderWrapper.__init__ï  s&   ø€ Ü‰Ñ˜Ô Ü% fÓ-ˆŒØ�‰Ör,   c                ó&   € V P                   ! V/ VB # rm   ©rx  )rG   Úargsr€   s   &*,r*   rh   ÚPegasusDecoderWrapper.forwardô  s   € Ø�|Š|˜TÐ, VÑ,Ð,r,   rÍ  rÉ   rw   s   @@r*   rÉ  rÉ  é  s   ù‡ € ñõ
÷
-ò -r,   rÉ  c                   ó”   a a€ ] tR tRt oRR/tV 3R ltR tR tV3R lR ltV3R	 lR
 lt	]
]RV3R lR ll4       4       tRtVtV ;t# )ÚPegasusForCausalLMiø  r™  z!model.decoder.embed_tokens.weightc                ó  <€ \         P                  ! V4      pR Vn        RVn        \        SV `  V4       \        V4      V n        \        P                  ! VP                  VP                  RR7      V n        V P                  4        R# )TFrŸ   N)ÚcopyÚdeepcopyr—   rf  rE   rF   rÉ  rþ   r   r£   Úhidden_sizer   r   r'  rÞ   s   &&€r*   rF   ÚPegasusForCausalLM.__init__ý  sf   ø€ Ü—’˜vÓ&ˆØ ˆÔØ$)ˆÔ!Ü‰Ñ˜Ô Ü*¨6Ó2ˆŒ
ä—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒð 	�‰Ör,   c                óB   € V P                   P                  P                  # rm   ©rþ   rx  r!  r>  s   &r*   r{  Ú'PegasusForCausalLM.get_input_embeddings	  s   € Ø�z‰z×!Ñ!×.Ñ.Ð.r,   c                ó:   € WP                   P                  n        R # rm   rØ  r~  s   &&r*   r  Ú'PegasusForCausalLM.set_input_embeddings  s   € Ø*/�
‰
×ÑÖ'r,   c                ó4   <€ V ^8„  d   QhRS[ P                  /# r8  r9  )r)   rA   s   "€r*   r+   ÚPegasusForCausalLM.__annotate__  s   ø€ ÷ <ñ <©¯©ñ <r,   c                óJ   € V P                   P                  P                  4       # r<  )rþ   rx  r?  r>  s   &r*   r?  Ú*PegasusForCausalLM.get_position_embeddings  s   € ð �z‰z×!Ñ!×9Ñ9Ó;Ð;r,   c                ó    <€ V ^8„  d   QhRS[ /# r*  r@   )r)   rA   s   "€r*   r+   rÝ    s   ø€ ÷ Sñ SÁcñ Sr,   c                óp   € WP                   n        V P                  P                  P	                  V4       R# rƒ  )rš   r  rþ   rx  r4  r3  s   &&r*   r4  Ú-PegasusForCausalLM.resize_position_embeddings  s(   € ð /J�‰Ô+Ø�
‰
×Ñ×5Ñ5Ð6QÖRr,   c                óˆ  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          R	S[R,          R
S[S[ P                  ,          RS[S[,          RS[	S[
,          /# )r"   r#   Nr}   r÷   rø   r¬   rE  r¹  rù   Úlogits_to_keepr€   r?   )r&   Ú
LongTensorr'   rS   r   rœ   r(   r   r   r­   r   )r)   rA   s   "€r*   r+   rÝ  (  sã   ø€ ÷ A
ñ A
á×#Ñ# dÕ*ðA
ñ Ÿ™ tÕ+ðA
ñ  %×0Ñ0°4Õ7ð	A
ñ
 !&× 1Ñ 1°DÕ 8ðA
ñ  �ðA
ñ ×(Ñ(¨4Õ/ðA
ñ × Ñ  4Õ'ðA
ñ ˜$•;ðA
ñ ™eŸl™lÕ*ðA
ñ Ñ+Õ,ðA
ñ 
Ñ2Õ	2ñA
r,   c
                ó(  € V P                   P                  ! RRVRVRVRVRVRVRV/V
B pV^ ,          p\        V	\        4      '       d   \	        V	) R4      MT	pV P                  VR	VR	3,          4      pRpVeb   VP                  VP                  4      p\        4       pV! VP                  RV P                  P                  4      VP                  R4      4      p\        VVVP                  VP                  VP                  VP                   R
7      # )aÂ  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

Example:

```python
>>> from transformers import AutoTokenizer, PegasusForCausalLM

>>> tokenizer = AutoTokenizer.from_pretrained("google/pegasus-large")
>>> model = PegasusForCausalLM.from_pretrained("google/pegasus-large")
>>> assert model.config.is_decoder, f"{model.__class__} has to be configured as a decoder."
>>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
>>> outputs = model(**inputs)

>>> logits = outputs.logits
>>> expected_shape = [1, inputs.input_ids.shape[-1], model.config.vocab_size]
>>> list(logits.shape) == expected_shape
True
```r#   r}   r÷   rø   r¬   rE  rù   Nr.   )r»  r¼  r¬   rª   r  rT  rl   r/   )rþ   rx  r±   r(   Úslicer   r2  re   r   r°   rš   r   r   r¬   rª   r  rT  )rG   r#   r}   r÷   rø   r¬   rE  r¹  rù   rä  r€   r¾  rª   Úslice_indicesr¼  r»  rÁ  s   &&&&&&&&&&,      r*   rh   ÚPegasusForCausalLM.forward%  s/  € ðN >B¿Z¹Z×=OÒ=Oñ 	>
Øð	>
à)ð	>
ð #8ð	>
ð $:ð		>
ð
 ,ð	>
ð (ð	>
ð  ð	>
ð ñ	>
ˆð   �
ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒØ—Y‘Y˜vŸ}™}Ó-ˆFÜ'Ó)ˆHÙ˜FŸK™K¨¨D¯K©K×,BÑ,BÓCÀVÇ[Á[ÐQSÃ_ÓUˆDä0ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø$×5Ñ5ô
ð 	
r,   )r   rþ   )	NNNNNNNNr1   )rn   ro   rp   rq   r–  rF   r{  r  r?  r4  r   r   rh   rt   ru   rv   rw   s   @@r*   rÑ  rÑ  ø  sb   ù‡ € àÐ=ðÐõ
ò/ò0÷<ð <÷Sð Sð  Ø÷A
ñ A
ó ó ÷A
ð A
r,   rÑ  )Nr¯   )rÑ  r  rr  rý   )Hrr   rÓ  r  Úcollections.abcr   ÚnumpyrN   r&   r   Útorch.nnr   Ú r   r  Úactivationsr   Úcache_utilsr   r   r	   Ú
generationr
   Úmasking_utilsr   r   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   r   Úutils.genericr   Úutils.output_capturingr   r   Úconfiguration_pegasusr    Ú
get_loggerrn   r¡   r8   r  r:   r‘   r�   r“   rË   rí   rý   r  rQ  rr  r  rÉ  rÑ  Ú__all__rl   r,   r*   Ú<module>rý     sn  ðñ ã Û Ý $ã Û Ý Ý %å &Ý !ß CÑ CÝ )ß JÝ BÝ 9÷õ ÷ GÝ &÷õ õ 8ß EÝ 0ð 
×	Ò	˜HÓ	%€õô"-¨2¯<©<ô -÷F%ô:r)�r—y‘yô r)ôl5Ð4ô 5ôrZÐ4ô Zðz ô2˜_ó 2ó ð2ô$v
Ð+ô v
ôrY
Ð+ô Y
ðx ôE
Ð)ó E
ó ðE
ñP ðôô
YhÐ&<¸oó Yhóð
Yhôz-Ð2ô -ôq
Ð/°ô q
òh n‚r,   