+
    œ°³j˜¡ ã                   óä  € R t ^ RIt^ RIHt ^ RIt^ RIHt ^ RIHt ^RIH	t
 ^RIHt ^RIHtHtHt ^RIHt ^R	IHtHt ^R
IHt ^RIHt ^RIHt ^RIHtHtHt ^RI H!t! ]PD                  ! ]#4      t$R R lt%R>R R llt& ! R R]PN                  4      t( ! R R]PR                  4      t* ! R R]PR                  4      t+ ! R R]PR                  4      t, ! R R]4      t- ! R R]4      t. ! R R ]PR                  4      t/] ! R! R"]4      4       t0]! R#R$7      ] ! R% R&]4      4       4       t1]! R'R$7      ] ! R( R)]4      4       4       t2]! R*R$7      ] ! R+ R,]4      4       4       t3]! R-R$7      ] ! R. R/]4      4       4       t4]! R0R$7      ] ! R1 R2]4      4       4       t5 ! R3 R4]04      t6 ! R5 R6]04      t7] ! R7 R8]04      4       t8]! R9R$7       ! R: R;]0]4      4       t9] ! R< R=]04      4       t:. R?Ot;R# )@zPyTorch LED model.N)Ú	dataclass)Únn)ÚCrossEntropyLoss)Úinitialization)ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚGradientCheckpointingLayer)Ú)BaseModelOutputWithPastAndCrossAttentions)ÚPreTrainedModel)ÚModelOutputÚauto_docstringÚlogging©Ú	LEDConfigc                óP   € V ^8„  d   QhR\         P                  R\        R\        /# )é   Ú	input_idsÚpad_token_idÚdecoder_start_token_id©ÚtorchÚTensorÚint)Úformats   "Úe/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/led/modeling_led.pyÚ__annotate__r    &   s&   € ÷ ñ ¤%§,¡,ð ¼cð Ô[^ñ ó    c                óÈ   € V P                  V P                  4      pV RRR13,          P                  4       VR&   W#R&   Vf   \        R4      hVP	                  VR8H  V4       V# )z)
Shift input ids one token to the right.
ºNNNNz&config.pad_token_id has to be defined.éÿÿÿÿ)r#   ºé   NN)r#   é    iœÿÿÿ)Ú	new_zerosÚshapeÚcloneÚ
ValueErrorÚmasked_fill_)r   r   r   Úshifted_input_idss   &&& r   Úshift_tokens_rightr.   &   sp   € ð "×+Ñ+¨I¯O©OÓ<ÐØ(¨¨C¨R¨C¨Õ0×6Ñ6Ó8Ð�eÑØ4�dÑàÒÜÐAÓBÐBà×"Ñ"Ð#4¸Ñ#<¸lÔKàÐr!   c                ór   € V ^8„  d   QhR\         P                  R\         P                  R\        R,          /# )r   ÚmaskÚdtypeÚtgt_lenN)r   r   r1   r   )r   s   "r   r    r    6   s0   € ÷ #ñ #¬e¯l©lð #Ä5Ç;Á;ð #ÔY\Ð_cÕYcñ #r!   c                ó*  € V P                  4       w  r4Ve   TMTpV R,          P                  V^W$4      P                  V4      pRV,
          pVP                  VP	                  4       \
        P                  ! V4      P                  4      pWv,          pV# )zW
Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`.
g      ð?©r#   NNr#   )ÚsizeÚexpandÚtoÚmasked_fillÚboolr   ÚfinfoÚmin)r0   r1   r2   ÚbszÚsrc_lenÚexpanded_maskÚinverted_maskÚexpanded_attention_masks   &&&     r   Ú#_prepare_4d_attention_mask_invertedrA   6   s‰   € ð —9‘9“;�L€CØ Ò,‰g°'€GàÐ)Õ*×1Ñ1°#°q¸'ÓK×NÑNÈuÓU€Mà˜-Õ'€MØ+×7Ñ7¸×8JÑ8JÓ8LÌeÏkÊkÐZ_ÓN`×NdÑNdÓeÐð 6ÕEÐà"Ð"r!   c                   ó^   a a€ ] tR t^Ht oRtV3R lV 3R lltRV3R lV 3R llltRtVtV ;t	# )ÚLEDLearnedPositionalEmbeddingzF
This module learns positional embeddings up to a fixed maximum size.
c                ó&   <€ V ^8„  d   QhRS[ RS[ /# )r   Únum_embeddingsÚembedding_dim©r   )r   Ú__classdict__s   "€r   r    Ú*LEDLearnedPositionalEmbedding.__annotate__M   s   ø€ ÷ 8ñ 8¡sð 8¹3ñ 8r!   c                ó&   <€ \         SV `  W4       R # ©N)ÚsuperÚ__init__)ÚselfrE   rF   Ú	__class__s   &&&€r   rM   Ú&LEDLearnedPositionalEmbedding.__init__M   s   ø€ Ü‰Ñ˜Ö7r!   c                ó:   <€ V ^8„  d   QhRS[ P                  RS[/# )r   Úinput_ids_shapeÚpast_key_values_length)r   ÚSizer   )r   rH   s   "€r   r    rI   P   s   ø€ ÷ *ñ *¡u§z¡zð *É3ñ *r!   c                ó¼   <€ VR,          w  r4\         P                  ! W"V,           \         P                  V P                  P                  R7      p\
        SV `  V4      # )z3`input_ids_shape` is expected to be [bsz x seqlen].ºNr   N)r1   Údevice)r   ÚarangeÚlongÚweightrW   rL   Úforward)rN   rR   rS   r<   Úseq_lenÚ	positionsrO   s   &&&   €r   r[   Ú%LEDLearnedPositionalEmbedding.forwardP   sK   ø€ à& rÕ*‰ˆÜ—L’LØ"¸WÕ$DÌEÏJÉJÐ_c×_jÑ_j×_qÑ_qô
ˆ	ô ‰w‰˜yÓ)Ð)r!   © ©r'   ©
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__rM   r[   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©rO   rH   s   @@r   rC   rC   H   s#   ù‡ € ñ÷8ó 8÷*÷ *õ *r!   rC   c                   óÞ   a a€ ] tR t^Zt oV 3R ltRR lt]R 4       t]R 4       t]RV3R lR ll4       t	]V3R lR l4       t
V3R	 lR
 ltV3R lR lt]R 4       tR tR tR tRtVtV ;t# )ÚLEDEncoderSelfAttentionc                ó¤  <€ \         SV `  4        VP                  VP                  ,          ^ 8w  d'   \	        RVP                   RVP                   R24      hVP                  V n        \        VP                  VP                  ,          4      V n        VP                  V n        \        P                  ! VP                  V P                  4      V n        \        P                  ! VP                  V P                  4      V n        \        P                  ! VP                  V P                  4      V n        \        P                  ! VP                  V P                  4      V n        \        P                  ! VP                  V P                  4      V n        \        P                  ! VP                  V P                  4      V n        VP"                  V n        W n        VP(                  V P&                  ,          pV^,          ^ 8X  g   Q RV P&                   RV 24       hV^ 8”  g   Q RV P&                   RV 24       hV^,          V n        Wn        R# )r'   zThe hidden size (z6) is not a multiple of the number of attention heads (Ú)z`attention_window` for layer z  has to be an even value. Given z has to be positive. Given N)rL   rM   Úhidden_sizeÚnum_attention_headsr+   Ú	num_headsr   Úhead_dimÚ	embed_dimr   ÚLinearÚqueryÚkeyÚvalueÚquery_globalÚ
key_globalÚvalue_globalÚattention_probs_dropout_probÚdropoutÚlayer_idÚattention_windowÚone_sided_attn_window_sizeÚconfig)rN   r€   r}   r~   rO   s   &&& €r   rM   Ú LEDEncoderSelfAttention.__init__[   sÙ  ø€ Ü‰ÑÔØ×Ñ × :Ñ :Õ:¸aÔ?ÜØ# F×$6Ñ$6Ð#7ð 8Ø ×4Ñ4Ð5°Qð8óð ð  ×3Ñ3ˆŒÜ˜F×.Ñ.°×1KÑ1KÕKÓLˆŒØ×+Ñ+ˆŒä—Y’Y˜v×1Ñ1°4·>±>ÓBˆŒ
Ü—9’9˜V×/Ñ/°·±Ó@ˆŒÜ—Y’Y˜v×1Ñ1°4·>±>ÓBˆŒ
ô ŸIšI f×&8Ñ&8¸$¿.¹.ÓIˆÔÜŸ)š) F×$6Ñ$6¸¿¹ÓGˆŒÜŸIšI f×&8Ñ&8¸$¿.¹.ÓIˆÔà×:Ñ:ˆŒà ŒØ!×2Ñ2°4·=±=ÕAÐØ !Õ# qÔ(ð 	
Ø+¨D¯M©M¨?Ð:ZÐ[kÐZlÐmó	
Ð(ð   !Ô#ð 	
Ø+¨D¯M©M¨?Ð:UÐVfÐUgÐhó	
Ð#ð +;¸aÕ*?ˆÔ'àŽr!   c                ó€	  € VP                  ^ ^4      pV P                  V4      pV P                  V4      pV P                  V4      p	VP	                  4       w  r«pWÀP
                  8X  g   Q RV P
                   RV 24       hV\        P                  ! V P                  4      ,          pVP                  W«V P                  V P                  4      P                  ^ ^4      pVP                  W«V P                  V P                  4      P                  ^ ^4      pV P                  WxV P                  4      pV^ 8g  R,          pVP                  V4      P                  V\        P                   ! VP"                  4      P$                  4      pV P                  VP'                  VP	                  4       R7      WðP                  4      pVV,          p\)        VP	                  4       4      VV
V P                  V P                  ^,          ^,           .8X  gG   Q RV RV
 RV P                   RV P                  ^,          ^,            RVP	                  4        2
4       hV'       dI   V P+                  V4      w  ppppV P-                  VVVVVVR	7      p\        P.                  ! VV3RR
7      p?\0        P2                  P5                  VR\        P6                  R7      p\        P                  ! VVR,          R4      pVP                  V4      p?\0        P2                  P9                  VV P8                  V P:                  R7      pV	P                  W«V P                  V P                  4      P                  ^ ^4      p	V'       d   V P=                  V	VXXXR7      pMV P?                  VW�P                  4      pVP	                  4       WºV P                  V P                  38X  g   Q R4       hVP                  ^ ^4      PA                  W«V4      PC                  4       pV'       di   V PE                  VXXXXVR7      w  ppVV^ ,          RV^,          3,          pVP                  \G        V^ ,          4      R4      VVRRR1,          &   ^ VV&   VP                  ^ ^4      3pV'       d   VV3,          pV'       d   V'       d   VX3,           # T# )av  
[`LEDEncoderSelfAttention`] expects *len(hidden_states)* to be multiple of *attention_window*. Padding to
*attention_window* happens in [`LEDEncoderModel.forward`] to avoid redoing the padding on each layer.

The *attention_mask* is changed in [`LEDEncoderModel.forward`] from 0, 1, 2 to:

    - -10000: no attention
    - 0: local attention
    - +10000: global attention
z&hidden_states should have embed_dim = z
, but has r#   N)r5   z$local_attn_probs should be of size (z, z), but is of size )Úquery_vectorsÚkey_vectorsÚmax_num_global_attn_indicesÚis_index_global_attn_nonzeroÚ"is_local_index_global_attn_nonzeroÚ%is_local_index_no_global_attn_nonzero©Údim©rŠ   r1   ç        ©ÚpÚtraining)Úvalue_vectorsÚ
attn_probsr…   r†   r‡   zUnexpected size)Úhidden_statesr…   r‡   r†   rˆ   Úis_index_masked)r#   r#   NNr$   )$Ú	transposeru   rv   rw   r5   rs   ÚmathÚsqrtrr   Úviewrq   Ú _sliding_chunks_query_key_matmulr   Útype_asr8   r   r:   r1   r;   Únew_onesÚlistÚ_get_global_attn_indicesÚ"_concat_with_global_key_attn_probsÚcatr   Ú
functionalÚsoftmaxÚfloat32r|   r�   Ú(_compute_attn_output_with_global_indicesÚ'_sliding_chunks_matmul_attn_probs_valueÚreshapeÚ
contiguousÚ'_compute_global_attn_output_from_hiddenÚlen)rN   r’   Úattention_maskr“   Úis_index_global_attnÚis_global_attnÚoutput_attentionsrƒ   r„   r�   r\   Ú
batch_sizers   Úattn_scoresÚ#remove_from_windowed_attention_maskÚ
float_maskÚdiagonal_maskr…   r†   r‡   rˆ   Úglobal_key_attn_scoresr‘   Úattn_outputÚglobal_attn_outputÚglobal_attn_probsÚnonzero_global_attn_outputÚoutputss   &&&&&&&                     r   r[   ÚLEDEncoderSelfAttention.forward~   s¹  € ð& &×/Ñ/°°1Ó5ˆð Ÿ
™
 =Ó1ˆØ—h‘h˜}Ó-ˆØŸ
™
 =Ó1ˆà)6×);Ñ);Ó)=Ñ&ˆ˜YØŸN™NÔ*ð 	
Ø4°T·^±^Ð4DÀJÈyÈkÐZó	
Ð*ð
 	œŸš 4§=¡=Ó1Õ1ˆà%×*Ñ*¨7ÀÇÁÐPT×P]ÑP]Ó^×hÑhÐijÐlmÓnˆØ!×&Ñ& w¸D¿N¹NÈDÏMÉMÓZ×dÑdÐefÐhiÓjˆà×;Ñ;Ø¨×(GÑ(Gó
ˆð
 0>ÀÑ/BÐDTÕ.UÐ+ð 9×@Ñ@ÀÓO×[Ñ[Ø/´·²¸]×=PÑ=PÓ1Q×1UÑ1Uó
ˆ
ð ×=Ñ=Ø×Ñ Z§_¡_Ó%6ÐÓ7¸×EdÑEdó
ˆð
 	�}Õ$ˆä�K×$Ñ$Ó&Ó'ØØØ�N‰NØ×+Ñ+¨aÕ/°!Õ3ð	,
ô 
ð 	
ð 3°:°,¸bÀÀ	ÈÈDÏNÉNÐK[ð \Ø×/Ñ/°!Õ3°aÕ7Ð8Ð8JÈ;×K[ÑK[ÓK]ÐJ^ð`ó	
ð 
÷ ð ×-Ñ-Ð.BÓCñØ+Ø,Ø2Ø5ð &*×%LÑ%LØ+Ø'Ø,GØ-IØ3UØ6[ð &Mó &Ð"ô  Ÿ)š)Ð%;¸[Ð$IÈrÔRˆKð 'ä—]‘]×*Ñ*Ø˜R¤u§}¡}ð +ó 
ˆ
ô
 ×&Ò& z°?ÐCSÕ3TÐVYÓZˆ
Ø×'Ñ'¨Ó4ˆ
ð ô —]‘]×*Ñ*¨:¸¿¹ÐPT×P]ÑP]Ð*Ó^ˆ
à%×*Ñ*¨7ÀÇÁÐPT×P]ÑP]Ó^×hÑhÐijÐlmÓnˆ÷ à×GÑGØ+Ø%Ø,GØ-IØ3Uð Hó ‰Kð ×FÑFØ˜M×+JÑ+JóˆKð ×ÑÓ! j¸4¿>¹>È4Ï=É=Ð%YÔYÐlÐ[lÓlÐYØ!×+Ñ+¨A¨qÓ1×9Ñ9¸'ÈyÓY×dÑdÓfˆ÷ Ø48×4`Ñ4`Ø+Ø,GØ3UØ-IØ6[Ø /ð 5aó 5Ñ1ÐÐ 1ð *<Ø2°1Õ5°qÐ:\Ð]^Õ:_Ð_õ*Ð&ð
 ?Y×>]Ñ>]ÜÐ6°qÕ9Ó:¸Bó?ˆKÐ4±T°r°TÕ:Ñ;ð 89ˆJÐ3Ñ4à×(Ñ(¨¨AÓ.Ð0ˆçØ˜
�}Õ$ˆGç2@×EVˆwÐ+Ð-Õ-ÐdÐ]dÐdr!   c                óÐ   € \         P                  P                  W4      p V P                  ! . V P	                  4       RR OV P	                  R4      NV P	                  R4      N5!  p V # )z)pads rows and then flips rows and columnsNéþÿÿÿr$   )r   rŸ   Úpadr—   r5   )Úhidden_states_paddedÚpaddings   &&r   Ú _pad_and_transpose_last_two_dimsÚ8LEDEncoderSelfAttention._pad_and_transpose_last_two_dims  sv   € ô  "Ÿ}™}×0Ñ0Ø ó 
Ðð  4×8Ò8ð  
Ø!×&Ñ&Ó(¨¨"Ð-ð 
Ø/C×/HÑ/HÈÓ/Lð 
ØNb×NgÑNgÐhjÓNkó 
Ðð $Ð#r!   c                ó  € V P                  4       w  rr4\        P                  P                  V ^ V^,           34      p V P	                  WR4      p V RRRV) 13,          p V P	                  WW3V,           4      p V RRRRR13,          p V # )a�  
shift every row 1 step right, converting columns into diagonals.

Example:

```python
chunked_hidden_states: [
    0.4983,
    2.6918,
    -0.0071,
    1.0492,
    -1.8348,
    0.7672,
    0.2986,
    0.0285,
    -0.7584,
    0.4206,
    -0.0405,
    0.1599,
    2.0514,
    -1.1600,
    0.5372,
    0.2629,
]
window_overlap = num_rows = 4
```

             (pad & diagonalize) => [ 0.4983, 2.6918, -0.0071, 1.0492, 0.0000, 0.0000, 0.0000
               0.0000, -1.8348, 0.7672, 0.2986, 0.0285, 0.0000, 0.0000 0.0000, 0.0000, -0.7584, 0.4206,
               -0.0405, 0.1599, 0.0000 0.0000, 0.0000, 0.0000, 2.0514, -1.1600, 0.5372, 0.2629 ]
r#   Nr$   )r5   r   rŸ   rº   r—   )Úchunked_hidden_statesÚtotal_num_headsÚ
num_chunksÚwindow_overlapÚ
hidden_dims   &    r   Ú_pad_and_diagonalizeÚ,LEDEncoderSelfAttention._pad_and_diagonalize)  s±   € ðB CX×B\ÑB\ÓB^Ñ?ˆ ^Ü "§¡× 1Ñ 1Ø! A ~¸Õ'9Ð#:ó!
Ðð !6× :Ñ :Ø¨ó!
Ðð !6ØˆqÐ"�N�?Ð"Ð"õ!
Ðð !6× :Ñ :Ø¨È*Õ9Tó!
Ðð !6°a¸¸A¸sÀ¸s°lÕ CÐØ$Ð$r!   c                ó    <€ V ^8„  d   QhRS[ /# )r   Úonnx_export)r9   )r   rH   s   "€r   r    Ú$LEDEncoderSelfAttention.__annotate__[  s   ø€ ÷ &"ñ &"¹4ñ &"r!   c           	     óF  € V'       gØ   V P                  V P                  ^ 4      \        P                  ! V P                  ^4      V^,          RR7      V^,          V P                  ^4      4      p \	        V P                  4       4      pV^,          ^,          ^,
          V^&   \	        V P                  4       4      pV^,          ^,          V^&   V P                  W4R7      # V P                  ^ 4      \        P                  ! V P                  ^4      VRR7      ^,
          V^,          V P                  ^4      .p\        P                  ! W0P                  R7      p\        V^,          4       F2  pV RWa,          Wa,          ^V,          ,           1R3,          VRVRR3&   K4  	  V# )zBconvert into overlapping chunks. Chunk size = 2w, overlap size = wÚtrunc©Úrounding_mode©r5   Ústride©rW   r#   )
r—   r5   r   Údivr›   rÏ   Ú
as_stridedÚemptyrW   Úrange)r’   rÃ   rÈ   Ú
chunk_sizeÚchunk_strideÚoverlapping_chunksÚchunks   &&&    r   Ú_chunkÚLEDEncoderSelfAttention._chunkZ  sy  € ÷ à)×.Ñ.Ø×"Ñ" 1Ó%Ü—	’	˜-×,Ñ,¨QÓ/°.À1Õ2DÐU\Ô]Ø Õ"Ø×"Ñ" 1Ó%ó	ˆMô ˜m×0Ñ0Ó2Ó3ˆJØ& q�M¨AÕ-°Õ1ˆJ�q‰Mä × 4Ñ 4Ó 6Ó7ˆLØ*¨1�o°Õ2ˆL˜‰OØ ×+Ñ+°Ð+ÓQÐQð ×Ñ˜qÓ!Ü�IŠI�m×(Ñ(¨Ó+¨^È7ÔSÐVWÕWØ˜QÕØ×Ñ˜qÓ!ð	
ˆ
ô #Ÿ[š[¨×<PÑ<PÔQÐÜ˜: a�=Ö)ˆEØ1>Ø�5Õ)¨EÕ,BÀQÈÕEWÕ,WÐWÐYZÐZõ2Ð˜q %¨¨A˜~Ó.ñ *ð "Ð!r!   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r   Úreturn©r   r   )r   rH   s   "€r   r    rÉ   „  s   ø€ ÷ 2ñ 2Á5Ç<Á<ñ 2r!   c                óÒ  € V P                  W^,           4      P                  4       P                  ^ .R7      pVR,          pVP                  RR7      pV RRV1RRV^,           13,          pVP                  VP	                  4       4      p\
        P                  ! V\        R4      ) 4      P                  VP                  4       V4      V RRV1RRV^,           13&   V RV) R1RV^,           ) R13,          pVP                  VP	                  4       4      p\
        P                  ! V\        R4      ) 4      P                  VP                  4       V4      V RV) R1RV^,           ) R13&   R# )r&   )ÚdimsNr#   Úinf)Nr#   Nr#   )r&   é   )
rš   ÚtrilÚflipr6   r5   r   Ú	full_likeÚfloatÚwherer9   )Úinput_tensorÚaffected_seq_lenÚbeginning_mask_2dÚbeginning_maskÚending_maskÚbeginning_inputÚending_inputs   &&     r   Ú_mask_invalid_locationsÚ/LEDEncoderSelfAttention._mask_invalid_locationsƒ  s{  € à(×1Ñ1Ð2BÐWXÕDXÓY×^Ñ^Ó`×eÑeÐlmÐknÐeÓoÐØ*Ð+;Õ<ˆØ$×)Ñ)¨vÐ)Ó6ˆØ& qÐ*;Ð+;Ð*;¸QÐ@VÐBRÐUVÕBVÐ@VÐ'VÕWˆØ'×.Ñ.¨×/CÑ/CÓ/EÓFˆÜHMÏÊØœe E›l˜]óI
ç
‰%�×#Ñ#Ó% Ó
7ð 	�QÐ)Ð)Ð)¨1Ð.DÐ0@À1Õ0DÐ.DÐDÑEð $ AÐ(8Ð'8Ñ'9¸1Ð@PÐSTÕ@TÐ>UÑ>WÐ$WÕXˆØ!×(Ñ(¨×):Ñ):Ó)<Ó=ˆÜLQÏOÊOØœ5 ›<˜-óM
ç
‰%�× Ñ Ó" LÓ
1ð 	�QÐ)Ð)Ñ*¨AÐ1AÀAÕ1EÐ/FÑ/HÐHÓIr!   c                óT   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[/# )r   ru   rv   rÃ   r   )r   rH   s   "€r   r    rÉ   “  s0   ø€ ÷ @)ñ @)±e·l±lð @)ÉÏÉð @)Ñgjñ @)r!   c           	     ó¦  € VP                  4       w  rErgWS^,          ,          ^ 8X  g   Q RV^,           RV 24       hVP                  4       VP                  4       8X  g   Q h\        P                  ! WSRR7      ^,
          pVP                  ^^4      P	                  WF,          WW4      pVP                  ^^4      P	                  WF,          WW4      pV P                  W\        V P                  RR4      4      pV P                  W#\        V P                  RR4      4      p\        P                  ! RW34      p	V P                  V	RR7      p	V	P                  WF,          V^,           W3^,          ^,           34      p
V	R	R	R
V1R
V^,           13,          V
R	R
R1R	VR
13&   V	R	RVR
1R
V^,           13,          V
R	RR	VR
13&   V	R	R	V^,           ) R1V^,           R
13,          V
R	RR	R
V13&   V	R	^ R
V^,
          1^V,
          R
13,          V
R	^ ^V1^V13&   V
P                  WFV^V,          ^,           4      P                  ^^4      p
V P                  W£4       V
# )zó
Matrix multiplication of query and key tensors using with a sliding window attention pattern. This
implementation splits the input into overlapping chunks of size 2w (e.g. 512 for pretrained LEDEncoder) with an
overlap of size window_overlap
z&Sequence length should be multiple of z. Given rË   rÌ   rÈ   Fzbcxd,bcyd->bcxy)r¼   r#   Nr%   )r'   r'   r'   r&   r$   )r5   r   rÑ   r”   r¤   rÙ   Úgetattrr€   Úeinsumr½   r(   r—   rî   )rN   ru   rv   rÃ   r¬   r\   rq   rr   Úchunks_countÚ!diagonal_chunked_attention_scoresÚdiagonal_attention_scoress   &&&&       r   r˜   Ú8LEDEncoderSelfAttention._sliding_chunks_query_key_matmul“  s©  € ð 49·:±:³<Ñ0ˆ
˜YØ¨1Õ,Õ-°Ô2ð 	
Ø4°^ÀaÕ5GÐ4HÈÐQXÐPYÐZó	
Ð2ð �z‰z‹|˜sŸx™x›zÔ)Ð)Ð)ä—y’y ÈÔPÐSTÕTˆð —‘  1Ó%×-Ñ-¨jÕ.DÀgÓXˆØ�m‰m˜A˜qÓ!×)Ñ)¨*Õ*@À'ÓTˆà—‘˜E´7¸4¿;¹;ÈÐW\Ó3]Ó^ˆØ�k‰k˜#¬w°t·{±{ÀMÐSXÓ/YÓZˆô -2¯LªLÐ9JÈUÈLÓ,YÐ)ð -1×,QÑ,QØ-°|ð -Ró -
Ð)ð %F×$OÑ$OØÕ# \°AÕ%5°~ÐXYÕGYÐ\]ÕG]Ð^ó%
Ð!ð AbØˆq�/�>�/Ð#7 ^°aÕ%7Ð#7Ð7õA
Ð! ! S b S¨!¨^©_Ð"<Ñ=ð @aØˆr�>‘?Ð$8 n°qÕ&8Ð$8Ð8õ@
Ð! ! R¨¨N©OÐ";Ñ<ð @aØˆq�N QÕ&Ð'¨"Ð,¨n¸qÕ.@Ñ.BÐBõ@
Ð! ! R¨¨O¨^¨OÐ";Ñ<ð OpØˆqÐ&�N QÕ&Ð&¨¨NÕ(:Ñ(<Ð<õO
Ð! ! Q¨¨.Ð(8¸!¸NÐ:JÐ"JÑKð
 %>×$BÑ$BØ 7¨A°Õ,>ÀÕ,Bó%
ç
‰)�A�q‹/ð 	"ð 	×$Ñ$Ð%>ÔOØ(Ð(r!   c                óT   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[/# )r   r‘   rw   rÃ   r   )r   rH   s   "€r   r    rÉ   Õ  s2   ø€ ÷ *Vñ *VÙŸ,™,ð*VÙ/4¯|©|ð*VÙMPñ*Vr!   c           	     óØ  € VP                  4       w  rErgWS^,          ,          ^ 8X  g   Q hVP                  4       R,          VP                  4       R,          8X  g   Q hVP                  ^4      ^V,          ^,           8X  g   Q h\        P                  ! WSRR7      ^,
          pVP                  ^^4      P	                  WF,          \        P                  ! WSRR7      V^V,          ^,           4      p	VP                  ^^4      P	                  WF,          WW4      p\
        P                  P                  V^ ^ W33RR7      p
WF,          V^,           ^V,          V3pV
P                  4       pV^ ,          W<^,          ,          V^,          V^,          3pV
P                  W¼R7      pV P                  V	4      p	\        P                  ! RW�34      pVP                  WFWW4      P                  ^^4      # )zŠ
Same as _sliding_chunks_query_key_matmul but for attn_probs and value tensors. Returned tensor will be of the
same shape as `attn_probs`
:Nrá   NrË   rÌ   ©rw   rÎ   zbcwd,bcdh->bcwhr$   )r5   r   rÑ   r”   r¤   r   rŸ   rº   rÏ   rÒ   rÅ   ró   r—   )rN   r‘   rw   rÃ   r¬   r\   rq   rr   rô   Úchunked_attn_probsÚpadded_valueÚchunked_value_sizeÚchunked_value_strideÚchunked_valueÚcontexts   &&&&           r   r£   Ú?LEDEncoderSelfAttention._sliding_chunks_matmul_attn_probs_valueÕ  sÇ  € ð 49·:±:³<Ñ0ˆ
˜Yà¨1Õ,Õ-°Ô2Ð2Ð2Ø�‰Ó  Õ$¨¯
©
«°RÕ(8Ô8Ð8Ð8Ø�‰˜qÓ! Q¨Õ%7¸!Õ%;Ô;Ð;Ð;Ü—y’y ÈÔPÐSTÕTˆð (×1Ñ1°!°QÓ7×?Ñ?ØÕ"Ü�IŠI�g¸WÔEØØ�Õ Õ"ó	
Ðð —‘  1Ó%×-Ñ-¨jÕ.DÀgÓXˆô —}‘}×(Ñ(¨°°A°~Ð0VÐ^`Ð(Óaˆð )Õ4°lÀQÕ6FÈÈNÕHZÐ\dÐeÐØ+×2Ñ2Ó4Ðà  Õ#Ø°!Õ4Õ4Ø  Õ#Ø  Õ#ð	 
Ðð %×/Ñ/Ð5GÐ/Óeˆà!×6Ñ6Ð7IÓJÐä—,’,Ð0Ð3EÐ2UÓVˆØ�|‰|˜J°7ÓE×OÑOÐPQÐSTÓUÐUr!   c                óH  € V P                  4       P                  ^R7      pVP                  4       pV P                  RR7      p\        P
                  ! W P                  R7      VP                  RR7      8  pVP                  RR7      pV^ 8H  P                  RR7      pVVVV3# )z<compute global attn indices required throughout forward passr‰   T)Úas_tuplerÐ   r$   )rY   ÚsumÚmaxÚnonzeror   rX   rW   Ú	unsqueeze)r©   Únum_global_attn_indicesr…   r†   Úis_local_index_global_attnr‡   rˆ   s   &      r   rœ   Ú0LEDEncoderSelfAttention._get_global_attn_indices  sÃ   € ð #7×";Ñ";Ó"=×"AÑ"AÀaÐ"AÓ"HÐð '>×&AÑ&AÓ&CÐ#ð (<×'CÑ'CÈTÐ'CÓ'RÐ$ô &+§\¢\Ø'×0KÑ0Kô&
à#×-Ñ-°"Ð-Ó5ñ&6Ð"ð
 .H×-OÑ-OÐY]Ð-OÓ-^Ð*ð 2LÈqÑ1P×0YÑ0YÐcgÐ0YÓ0hÐ-à'Ø(Ø.Ø1ð	
ð 	
r!   c                ó„  € VP                   ^ ,          pVP                  WsV P                  V P                  4      pW,          W…&   \        P
                  ! RW(34      p	V	P                  ^^4      p	\        P                  ! V	P                  4      P                  V	V^ ,          V^,          RR3&   V	P                  ^^4      p	V	# )r'   zblhd,bshd->blhsr#   )
r)   r(   rq   rr   r   ró   r”   r:   r1   r;   )
rN   r„   rƒ   r…   r†   r‡   rˆ   r¬   Úkey_vectors_only_globalÚattn_probs_from_global_keys
   &&&&&&&   r   r�   Ú:LEDEncoderSelfAttention._concat_with_global_key_attn_probs  sÎ   € ð !×&Ñ& qÕ)ˆ
ð #.×"7Ñ"7Ø°T·^±^ÀTÇ]Á]ó#
Ðð GRÕFoÐÑCô &+§\¢\Ð2CÀmÐEmÓ%nÐ"ð &@×%IÑ%IÈ!ÈQÓ%OÐ"ô �KŠKÐ2×8Ñ8Ó9×=Ñ=ð 	#Ø1°!Õ4Ð6[Ð\]Õ6^Ð`aÐcdÐdñ	
ð &@×%IÑ%IÈ!ÈQÓ%OÐ"à)Ð)r!   c                ó*  € VP                   ^ ,          pVP                  R^ V4      pVP                  WcV P                  V P                  4      pW,          W…&   \
        P                  ! VP                  ^^4      P                  4       VP                  ^^4      P                  4       4      P                  ^^4      p	VP                  RW2P                  R4      V,
          4      P                  4       p
V P                  W¡V P                  4      pW›,           # )r'   r$   )r)   Únarrowr(   rq   rr   r   Úmatmulr”   r*   r5   r¥   r£   r   )rN   r�   r‘   r…   r†   r‡   r¬   Úattn_probs_only_globalÚvalue_vectors_only_globalÚattn_output_only_globalÚattn_probs_without_globalÚattn_output_without_globals   &&&&&&      r   r¢   Ú@LEDEncoderSelfAttention._compute_attn_output_with_global_indices<  s  € ð  ×%Ñ% aÕ(ˆ
ð ",×!2Ñ!2°2°qÐ:UÓ!VÐà$1×$;Ñ$;Ø°T·^±^ÀTÇ]Á]ó%
Ð!ð IVÕHsÐ!ÑEô
 #(§,¢,Ø"×,Ñ,¨Q°Ó2×8Ñ8Ó:Ð<U×<_Ñ<_Ð`aÐcdÓ<e×<kÑ<kÓ<mó#
ç
‰)�A�q‹/ð 	 ð
 %/×$5Ñ$5ØÐ+¯_©_¸RÓ-@ÐC^Õ-^ó%
ç
‰*‹,ð 	"ð
 &*×%QÑ%QØ%°d×6UÑ6Uó&
Ð"ð 'ÕCÐCr!   c                ón  € VP                   R ,          w  rxVP                  W(V P                  4      p	VVRRR	1,          ,          W“RRR	1,          &   V P                  V	4      p
V P	                  V4      pV P                  V4      pV
\        P                  ! V P                  4      ,          p
V
P                  4       P                  W(V P                  ,          V P                  4      P                  ^ ^4      p
VP                  4       P                  R	W€P                  ,          V P                  4      P                  ^ ^4      pVP                  4       P                  R	W€P                  ,          V P                  4      P                  ^ ^4      p\        P                  ! W«P                  ^^4      4      p\        VP!                  4       4      W€P                  ,          VV.8X  g/   Q RW€P                  ,          W'3 RVP!                  4        R24       hVP                  W€P                  W'4      pVP                  ^^4      p\        P"                  ! VP$                  4      P&                  VV^ ,          V^,          RR3&   VP                  ^^4      pVP)                  VR
,          \        P"                  ! VP$                  4      P&                  4      pVP                  W€P                  ,          W'4      p\*        P,                  P/                  VR	\        P0                  R7      p\*        P,                  P3                  VP5                  V4      V P2                  V P6                  R7      p\        P                  ! Wü4      p\        VP!                  4       4      W€P                  ,          VV P                  .8X  g9   Q RW€P                  ,          W P                  3 RVP!                  4        R24       hVP                  W€P                  W'4      pVP                  W€P                  W P                  4      pVV3# )rV   Nz7global_attn_scores have the wrong size. Size should be ú	, but is Ú.r#   r‹   r�   z=global_attn_output tensor has the wrong size. Size should be r$   r4   )r)   r(   rs   rx   ry   rz   r•   r–   rr   r¥   r—   rq   r”   r   Úbmmr›   r5   r:   r1   r;   r8   r   rŸ   r    r¡   r|   r™   r�   )rN   r’   r…   r‡   r†   rˆ   r“   r\   r¬   Úglobal_attn_hidden_statesÚ global_query_vectors_only_globalÚglobal_key_vectorsÚglobal_value_vectorsÚglobal_attn_scoresÚglobal_attn_probs_floatr´   r³   s   &&&&&&&          r   r¦   Ú?LEDEncoderSelfAttention._compute_global_attn_output_from_hidden`  sð  € ð ,×1Ñ1°"Õ5Ñˆð %2×$;Ñ$;Ð<WÐei×esÑesÓ$tÐ!ØN[Ø(©¨2¨Õ.õO
Ð!ÁTÀrÀTÕ"JÑKð
 ,0×+<Ñ+<Ð=VÓ+WÐ(Ø!Ÿ_™_¨]Ó;ÐØ#×0Ñ0°Ó?Ðð 	)¬D¯IªI°d·m±mÓ,DÕDÐ(ð -×7Ñ7Ó9ß‰TÐ-¸D¿N¹NÕ/JÈDÏMÉMÓZß‰Y�q˜!‹_ð 	)ð ×)Ñ)Ó+×0Ñ0°°ZÇ.Á.Õ5PÐRV×R_ÑR_Ó`×jÑjÐklÐnoÓpð 	ð !×+Ñ+Ó-×2Ñ2°2°zÇNÁNÕ7RÐTX×TaÑTaÓb×lÑlÐmnÐpqÓrð 	ô
 #ŸYšYÐ'G×IeÑIeÐfgÐijÓIkÓlÐäÐ&×+Ñ+Ó-Ó.ØŸ™Õ'Ø'Øð3
ô 
ð 	
ð
ØŸn™nÕ,Ð.IÐSÐTð UØ"×'Ñ'Ó)Ð*¨!ð-ó	
ð 
ð 0×4Ñ4°ZÇÁÐQlÓvÐð 0×9Ñ9¸!¸QÓ?Ðô �KŠKÐ*×0Ñ0Ó1×5Ñ5ð 	Ø1°!Õ4Ð6[Ð\]Õ6^Ð`aÐcdÐdñ	
ð 0×9Ñ9¸!¸QÓ?Ðà/×;Ñ;ØÐ,Õ-Ü�KŠKÐ*×0Ñ0Ó1×5Ñ5ó
Ðð
 0×4Ñ4°ZÇ.Á.Õ5PÐRmÓwÐô #%§-¡-×"7Ñ"7Ø B¬e¯m©mð #8ó #
Ðô ŸM™M×1Ñ1Ø#×+Ñ+Ð,>Ó?À4Ç<Á<ÐZ^×ZgÑZgð 2ó 
Ðô
 #ŸYšYÐ'8ÓOÐäÐ&×+Ñ+Ó-Ó.ØŸ™Õ'Ø'Ø�M‰Mð3
ô 
ð 	
ð
ØŸn™nÕ,Ð.IÏ=É=ÐYÐZð [Ø"×'Ñ'Ó)Ð*¨!ð-ó	
ð 
ð .×2Ñ2°:¿~¹~ÐOjÓtÐØ/×4Ñ4ØŸ™Ð(CÇ]Á]ó
Ðð "Ð#4Ð4Ð4r!   )r€   r|   rs   rr   rv   ry   r}   rq   r   ru   rx   rw   rz   ©NNNNF)F)rb   rc   rd   re   rM   r[   Ústaticmethodr½   rÅ   rÙ   rî   r˜   r£   rœ   r�   r¢   r¦   rg   rh   ri   rj   s   @@r   rl   rl   Z   s°   ù‡ € õ!ôF^eð@ ñ$ó ð$ð ñ.%ó ð.%ð` ÷&"ñ &"ó ð&"ðP ÷2ó ð2÷@)ð @)÷D*Vð *VðX ñ
ó ð
ò8*ò<"D÷H]5ò ]5r!   rl   c                   óH   a a€ ] tR tRt oV 3R ltRV3R lR lltRtVtV ;t# )ÚLEDEncoderAttentioniÀ  c                ó¦   <€ \         SV `  4        \        WR 7      V n        \        P
                  ! VP                  VP                  4      V n        R# ))r}   N)rL   rM   rl   Úlongformer_self_attnr   rt   Úd_modelÚoutput©rN   r€   r}   rO   s   &&&€r   rM   ÚLEDEncoderAttention.__init__Á  s5   ø€ Ü‰ÑÔÜ$;¸FÔ$VˆÔ!Ü—i’i §¡°·±Ó?ˆŽr!   c                óN  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[RS[S[ P                  S[ P                  R,          S[S[ P                  ,          R,          3,          /# )	r   r’   r¨   Nr“   r©   rª   r«   rÜ   )r   r   r9   Útuple)r   rH   s   "€r   r    Ú LEDEncoderAttention.__annotate__Æ  s›   ø€ ÷ ñ á—|‘|ðñ Ÿ™ tÕ+ðñ Ÿ™¨Õ,ð	ñ
 $Ÿl™l¨TÕ1ðñ ˜t�ðñ  ðñ 
‰u�|‰|™UŸ\™\¨DÕ0±%¹¿¹Õ2EÈÕ2LÐLÕ	Mñr!   c           	     ó†   € V P                  VVVVVVR7      pV P                  V^ ,          4      pV3VR,          ,           p	V	# )ú#Input shape: Batch x Time x Channel©r’   r¨   r“   r©   rª   r«   r%   ©r(  r*  )
rN   r’   r¨   r“   r©   rª   r«   Úself_outputsr²   r¶   s
   &&&&&&&   r   r[   ÚLEDEncoderAttention.forwardÆ  sU   € ð ×0Ñ0Ø'Ø)Ø+Ø!5Ø)Ø/ð 1ó 
ˆð —k‘k ,¨q¥/Ó2ˆØ�. <°Õ#3Õ3ˆàˆr!   r3  r#  ©	rb   rc   rd   re   rM   r[   rg   rh   ri   rj   s   @@r   r&  r&  À  s   ù‡ € õ@÷
÷ ò r!   r&  c                   ó\   a a€ ] tR tRt oRtRV3R lV 3R llltR	V3R lR lltRtVtV ;t	# )
ÚLEDDecoderAttentionià  z=Multi-headed attention from 'Attention Is All You Need' paperc                óv   <€ V ^8„  d   QhRS[ RS[ RS[R,          RS[R,          RS[R,          RS[R,          /# )r   rs   rq   r|   NÚ
is_decoderÚbiasÚ	layer_idx)r   rå   r9   )r   rH   s   "€r   r    Ú LEDDecoderAttention.__annotate__ã  s_   ø€ ÷ Cñ CáðCñ ðCñ ˜•ð	Cñ
 ˜4•KðCñ �T�kðCñ ˜$•;ñCr!   c                ó  <€ \         SV `  4        Wn        W n        W0n        W,          V n        V P
                  V,          V P                  8w  d   \        R V P                   RV R24      hV P
                  R,          V n        W@n        W`n	        \        P                  ! WVR7      V n        \        P                  ! WVR7      V n        \        P                  ! WVR7      V n        \        P                  ! WVR7      V n        R# )z;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).©r;  Ng      à¿)rL   rM   rs   rq   r|   rr   r+   Úscalingr:  r<  r   rt   Úk_projÚv_projÚq_projÚout_proj)rN   rs   rq   r|   r:  r;  r<  rO   s   &&&&&&&€r   rM   ÚLEDDecoderAttention.__init__ã  sÍ   ø€ ô 	‰ÑÔØ"ŒØ"ŒØŒØ!Õ.ˆŒØ�=‰=˜9Õ$¨¯©Ô6ÜØMÈdÏnÉnÐM]ð ^Ø�;˜bð"óð ð —}‘} dÕ*ˆŒØ$ŒØ"Œä—i’i 	¸4Ô@ˆŒÜ—i’i 	¸4Ô@ˆŒÜ—i’i 	¸4Ô@ˆŒÜŸ	š	 )¸TÔBˆŽr!   c                ó  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[R,          RS[ P                  R,          RS[RS[S[ P                  S[ P                  R,          S[R,          3,          /# )r   r’   Úkey_value_statesNÚpast_key_valuesr¨   r«   rÜ   )r   r   r   r9   r.  )r   rH   s   "€r   r    r=  ÿ  sŽ   ø€ ÷ eCñ eCá—|‘|ðeCñ  Ÿ,™,¨Õ-ðeCñ  �ð	eCñ
 Ÿ™ tÕ+ðeCñ  ðeCñ 
‰u�|‰|™UŸ\™\¨DÕ0±%¸$µ,Ð>Õ	?ñeCr!   c                ó$
  € VRJpVP                  4       w  rxp	V P                  V4      V P                  ,          p
RpVe`   \        V\        4      '       dH   VP
                  P                  V P                  4      pV'       d   VP                  pMVP                  pMTpV'       d   TMTpV'       d\   VeX   V'       dP   XP                  V P                  ,          P                  pVP                  V P                  ,          P                  pMëV P                  V4      pV P                  V4      pVP                  VR
V P                   V P"                  4      P%                  ^^4      pVP                  VR
V P                   V P"                  4      P%                  ^^4      pVeV   XP'                  WïV P                  4      w  rïV'       d0   \        V\        4      '       d   RVP
                  V P                  &   WpP                   ,          R
V P"                  3pV
P                  WxV P                   V P"                  4      P%                  ^^4      p
V
P(                  ! V!  p
VP(                  ! V!  pVP(                  ! V!  pVP                  ^4      p\*        P,                  ! W®P%                  ^^4      4      pVP                  4       WpP                   ,          VV38w  d3   \/        RWpP                   ,          VV3 RVP                  4        24      hVe„   VP                  4       V^VV38w  d$   \/        RV^VV3 RVP                  4        24      hVP                  WpP                   VV4      V,           pVP                  WpP                   ,          VV4      p\0        P2                  P5                  VR
R7      pV'       dB   VP                  WpP                   VV4      pVP                  WpP                   ,          VV4      pMRp\0        P2                  P7                  VV P6                  V P8                  R7      p\*        P,                  ! VV4      pVP                  4       WpP                   ,          W€P"                  38w  d6   \/        R	WpP                   W€P"                  3 RVP                  4        24      hVP                  WpP                   W€P"                  4      P%                  ^^4      P)                  WxV	4      pV P;                  V4      pVVV3# )r1  NFTz$Attention weights should be of size r  z!Attention mask should be of size r‰   r�   z `attn_output` should be of size r$   )r5   rC  r@  Ú
isinstancer	   Ú
is_updatedÚgetr<  Úcross_attention_cacheÚself_attention_cacheÚlayersÚkeysÚvaluesrA  rB  r—   rq   rr   r”   Úupdater¤   r   r  r+   r   rŸ   r    r|   r�   rD  )rN   r’   rG  rH  r¨   r«   Úis_cross_attentionr<   r2   rs   Úquery_statesrK  Úcurr_past_key_valuesÚcurrent_statesÚ
key_statesÚvalue_statesÚ
proj_shaper=   Úattn_weightsÚattn_weights_reshapedr‘   r²   s   &&&&&&                r   r[   ÚLEDDecoderAttention.forwardÿ  s\  € ð .°TÐ9ÐØ"/×"4Ñ"4Ó"6Ñˆ�ið —{‘{ =Ó1°D·L±LÕ@ˆàˆ
ØÒ&Ü˜/Ô+>×?Ò?Ø,×7Ñ7×;Ñ;¸D¿N¹NÓK�
ß%à+:×+PÑ+PÑ(à+:×+OÑ+OÑ(à'6Ð$ç-?Ñ)À]ˆß /Ò"=Ç*à-×4Ñ4°T·^±^ÕD×IÑIˆJØ/×6Ñ6°t·~±~ÕF×MÑM‰LàŸ™ ^Ó4ˆJØŸ;™; ~Ó6ˆLØ#Ÿ™¨¨b°$·.±.À$Ç-Á-ÓP×ZÑZÐ[\Ð^_Ó`ˆJØ'×,Ñ,¨S°"°d·n±nÀdÇmÁmÓT×^Ñ^Ð_`ÐbcÓdˆLàÒ*à+?×+FÑ+FÀzÐae×aoÑaoÓ+pÑ(�
ç%¬*°_ÔFY×*ZÒ*ZØAE�O×.Ñ.¨t¯~©~Ñ>àŸN™NÕ*¨B°·±Ð>ˆ
Ø#×(Ñ(¨°t·~±~ÀtÇ}Á}ÓU×_Ñ_Ð`aÐcdÓeˆØ#×+Ò+¨ZÑ8ˆØ×'Ò'¨Ñ4ˆ
Ø#×+Ò+¨ZÑ8ˆà—/‘/ !Ó$ˆÜ—y’y ×/CÑ/CÀAÀqÓ/IÓJˆà×ÑÓ 3¯©Õ#7¸À'Ð"JÔJÜØ6¸¿n¹nÕ8LÈgÐW^Ð7_Ð6`ð aØ ×%Ñ%Ó'Ð(ð*óð ð
 Ò%Ø×"Ñ"Ó$¨¨a°¸'Ð(BÔBÜ Ø7¸¸aÀÈ'Ð8RÐ7SÐS\Ð]k×]pÑ]pÓ]rÐ\sÐtóð ð (×,Ñ,¨S·.±.À'È7ÓSÐVdÕdˆLØ'×,Ñ,¨S·>±>Õ-AÀ7ÈGÓTˆLä—}‘}×,Ñ,¨\¸rÐ,ÓBˆçð
 %1×$5Ñ$5°c¿>¹>È7ÐT[Ó$\Ð!Ø0×5Ñ5°c¿N¹NÕ6JÈGÐU\Ó]‰Là$(Ð!ä—]‘]×*Ñ*¨<¸4¿<¹<ÐRV×R_ÑR_Ð*Ó`ˆ
ä—i’i 
¨LÓ9ˆà×ÑÓ #¯©Õ"6¸ÇÁÐ!OÔOÜØ2°C¿¹È×R_ÑR_Ð3`Ð2að bØ×$Ñ$Ó&Ð'ð)óð ð ×Ñ˜S§.¡.°'¿=¹=ÓIß‰Y�q˜!‹_ß‰W�S 9Ó-ð 	ð —m‘m KÓ0ˆàÐ1°?ÐBÐBr!   )r|   rs   rr   r:  rA  r<  rq   rD  rC  r@  rB  )rŒ   FTN©NNNFra   rj   s   @@r   r8  r8  à  s)   ù‡ € ÙG÷Cõ C÷8eC÷ eCò eCr!   r8  c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚLEDEncoderLayerig  c                ó&   <€ V ^8„  d   QhRS[ RS[/# )r   r€   r}   )r   r   )r   rH   s   "€r   r    ÚLEDEncoderLayer.__annotate__h  s   ø€ ÷ 
=ñ 
=™yð 
=±Cñ 
=r!   c                ó6  <€ \         SV `  4        VP                  V n        \	        W4      V n        \        P                  ! V P                  4      V n        VP                  V n	        \        VP                  ,          V n        VP                  V n        \        P                  ! V P                  VP                  4      V n        \        P                  ! VP                  V P                  4      V n        \        P                  ! V P                  4      V n        R # rK   )rL   rM   r)  rs   r&  Ú	self_attnr   Ú	LayerNormÚself_attn_layer_normr|   r   Úactivation_functionÚactivation_fnÚactivation_dropoutrt   Úencoder_ffn_dimÚfc1Úfc2Úfinal_layer_normr+  s   &&&€r   rM   ÚLEDEncoderLayer.__init__h  s³   ø€ Ü‰ÑÔØŸ™ˆŒÜ,¨VÓ>ˆŒÜ$&§L¢L°·±Ó$@ˆÔ!Ø—~‘~ˆŒÜ# F×$>Ñ$>Õ?ˆÔØ"(×";Ñ";ˆÔÜ—9’9˜TŸ^™^¨V×-CÑ-CÓDˆŒÜ—9’9˜V×3Ñ3°T·^±^ÓDˆŒÜ "§¢¨T¯^©^Ó <ˆÖr!   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r   r’   r¨   rÝ   )r   rH   s   "€r   r    ra  t  s'   ø€ ÷ (3ñ (3á—|‘|ð(3ñ Ÿ™ñ(3r!   c           	     ó–  € TpV P                  VVVVVVR7      pV^ ,          p\        P                  P                  WP                  V P                  R7      pWq,           pV P                  V4      pTpV P                  V P                  V4      4      p\        P                  P                  WP                  V P                  R7      pV P                  V4      p\        P                  P                  WP                  V P                  R7      pWq,           pV P                  V4      pVP                  \        P                  8X  du   \        P                  ! V4      P                  4       '       gK   \        P                   ! VP                  4      P"                  R,
          p	\        P$                  ! W) V	R7      pV3VR,          ,           # )a  
Args:
    hidden_states (`torch.FloatTensor`): input to the layer of shape *(batch, seq_len, embed_dim)*
    attention_mask (`torch.FloatTensor`): attention mask of size
        *(batch, 1, tgt_len, src_len)* where padding elements are indicated by very large negative values.
r2  r�   iè  )r;   r  r%   )rc  r   rŸ   r|   r�   re  rg  rj  rh  rk  rl  r1   r   Úfloat16ÚisfiniteÚallr:   r  Úclamp)
rN   r’   r¨   r“   r©   rª   r«   ÚresidualÚattn_outputsÚclamp_values
   &&&&&&&   r   r[   ÚLEDEncoderLayer.forwardt  sp  € ð !ˆØ—~‘~Ø'Ø)Ø+Ø!5Ø)Ø/ð &ó 
ˆð % Q�ˆÜŸ™×-Ñ-¨m¿|¹|ÐVZ×VcÑVcÐ-ÓdˆØ Õ0ˆØ×1Ñ1°-Ó@ˆà ˆØ×*Ñ*¨4¯8©8°MÓ+BÓCˆÜŸ™×-Ñ-¨m×?VÑ?VÐae×anÑanÐ-ÓoˆØŸ™ Ó/ˆÜŸ™×-Ñ-¨m¿|¹|ÐVZ×VcÑVcÐ-ÓdˆØ Õ0ˆØ×-Ñ-¨mÓ<ˆà×Ñ¤%§-¡-Ô/¼¿ºÀ}Ó8U×8YÑ8Y×8[Ò8[ÜŸ+š+ m×&9Ñ&9Ó:×>Ñ>ÀÕEˆKÜ!ŸKšK¨¸<È[ÔYˆMØÐ ,¨rÕ"2Õ2Ð2r!   )	rh  rg  r|   rs   rj  rk  rl  rc  re  r]  r6  rj   s   @@r   r_  r_  g  s   ù‡ € ÷
=ó 
=÷(3÷ (3ò (3r!   r_  c                   óX   a a€ ] tR tRt oRV3R lV 3R llltRV3R lR lltRtVtV ;t# )	ÚLEDDecoderLayeriŸ  c                ó    <€ V ^8„  d   QhRS[ /# ©r   r€   r   )r   rH   s   "€r   r    ÚLEDDecoderLayer.__annotate__   s   ø€ ÷ =ñ =™yñ =r!   c                ó,  <€ \         SV `  4        VP                  V n        \	        V P                  VP
                  VP                  R VR7      V n        VP                  V n        \        VP                  ,          V n        VP                  V n        \        P                  ! V P                  4      V n        \	        V P                  VP
                  VP                  R VR7      V n        \        P                  ! V P                  4      V n        \        P$                  ! V P                  VP&                  4      V n        \        P$                  ! VP&                  V P                  4      V n        \        P                  ! V P                  4      V n        R# )T)rs   rq   r|   r:  r<  )r|   r:  r<  N)rL   rM   r)  rs   r8  Údecoder_attention_headsÚattention_dropoutrc  r|   r   rf  rg  rh  r   rd  re  Úencoder_attnÚencoder_attn_layer_normrt   Údecoder_ffn_dimrj  rk  rl  )rN   r€   r<  rO   s   &&&€r   rM   ÚLEDDecoderLayer.__init__   s  ø€ Ü‰ÑÔØŸ™ˆŒä,Ø—n‘nØ×4Ñ4Ø×,Ñ,ØØô
ˆŒð —~‘~ˆŒÜ# F×$>Ñ$>Õ?ˆÔØ"(×";Ñ";ˆÔä$&§L¢L°·±Ó$@ˆÔ!Ü/Ø�N‰NØ×*Ñ*Ø×,Ñ,ØØô
ˆÔô (*§|¢|°D·N±NÓ'CˆÔ$Ü—9’9˜TŸ^™^¨V×-CÑ-CÓDˆŒÜ—9’9˜V×3Ñ3°T·^±^ÓDˆŒÜ "§¢¨T¯^©^Ó <ˆÖr!   c                óè   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[R,          /# )	r   r’   r¨   NÚencoder_hidden_statesÚencoder_attention_maskrH  r«   Ú	use_cache)r   r   r   r9   )r   rH   s   "€r   r    r|  ¼  s…   ø€ ÷ Gñ Gá—|‘|ðGñ Ÿ™ tÕ+ðGñ  %Ÿ|™|¨dÕ2ð	Gñ
 !&§¡¨tÕ 3ðGñ  �ðGñ   $�;ðGñ ˜$•;ñGr!   c                óp  € Tp	V P                  VVVVR7      w  rp\        P                  P                  WP                  V P                  R7      pW‘,           pV P                  V4      pRpRpVej   Tp	V P                  VVVVVR7      w  rp\        P                  P                  WP                  V P                  R7      pW‘,           pV P                  V4      pTp	V P                  V P                  V4      4      p\        P                  P                  WP                  V P                  R7      pV P                  V4      p\        P                  P                  WP                  V P                  R7      pW‘,           pV P                  V4      pV3pV'       d   WêV3,          pV'       d
   Wå3,          pV# )a8  
Args:
    hidden_states (`torch.FloatTensor`): input to the layer of shape *(batch, seq_len, embed_dim)*
    attention_mask (`torch.FloatTensor`): attention mask of size
        *(batch, 1, tgt_len, src_len)* where padding elements are indicated by very large negative values.
    encoder_hidden_states (`torch.FloatTensor`):
        cross attention input to the layer of shape *(batch, seq_len, embed_dim)*
    encoder_attention_mask (`torch.FloatTensor`): encoder attention mask of size
        *(batch, 1, tgt_len, src_len)* where padding elements are indicated by very large negative values.
    past_key_values (`Cache`): cached past key and value projection states
    output_attentions (`bool`): Whether the base model outputs attentions.
        This requires the attentions tensor to be reshaped in this function.
)r’   rH  r¨   r«   r�   N)r’   rG  r¨   rH  r«   )rc  r   rŸ   r|   r�   re  r€  r�  rg  rj  rh  rk  rl  )rN   r’   r¨   r…  r†  rH  r«   r‡  Úkwargsrt  Úself_attn_weightsÚpresent_key_valueÚcross_attn_present_key_valueÚcross_attn_weightsr¶   s   &&&&&&&&,      r   r[   ÚLEDDecoderLayer.forward¼  s°  € ð0 !ˆð ?C¿n¹nØ'Ø+Ø)Ø/ð	 ?Mó ?
Ñ;ˆÐ*;ô Ÿ™×-Ñ-¨m¿|¹|ÐVZ×VcÑVcÐ-ÓdˆØ Õ0ˆØ×1Ñ1°-Ó@ˆð (,Ð$Ø!ÐØ Ò,Ø$ˆHàNR×N_ÑN_Ø+Ø!6Ø5Ø /Ø"3ð O`ó OÑKˆMÐ/Kô ŸM™M×1Ñ1°-Ç<Á<ÐZ^×ZgÑZgÐ1ÓhˆMØ$Õ4ˆMØ ×8Ñ8¸ÓGˆMð !ˆØ×*Ñ*¨4¯8©8°MÓ+BÓCˆÜŸ™×-Ñ-¨m×?VÑ?VÐae×anÑanÐ-ÓoˆØŸ™ Ó/ˆÜŸ™×-Ñ-¨m¿|¹|ÐVZ×VcÑVcÐ-ÓdˆØ Õ0ˆØ×-Ñ-¨mÓ<ˆà Ð"ˆçØÐ+=Ð>Õ>ˆGçØÐ)Õ)ˆGàˆr!   )rh  rg  r|   rs   r€  r�  rj  rk  rl  rc  re  rK   )NNNNFTr6  rj   s   @@r   ry  ry  Ÿ  s!   ù‡ € ÷=õ =÷8G÷ Gò Gr!   ry  c                   óT   a a€ ] tR tRt oRtV3R lV 3R lltV3R lR ltRtVtV ;t	# )ÚLEDClassificationHeadi  z-Head for sentence-level classification tasks.c                ó2   <€ V ^8„  d   QhRS[ RS[ RS[ RS[/# )r   Ú	input_dimÚ	inner_dimÚnum_classesÚpooler_dropout)r   rå   )r   rH   s   "€r   r    Ú"LEDClassificationHead.__annotate__	  s3   ø€ ÷ 
:ñ 
:áð
:ñ ð
:ñ ð	
:ñ
 ñ
:r!   c                óÈ   <€ \         SV `  4        \        P                  ! W4      V n        \        P
                  ! VR 7      V n        \        P                  ! W#4      V n        R# ))rŽ   N)rL   rM   r   rt   ÚdenseÚDropoutr|   rD  )rN   r’  r“  r”  r•  rO   s   &&&&&€r   rM   ÚLEDClassificationHead.__init__	  s@   ø€ ô 	‰ÑÔÜ—Y’Y˜yÓ4ˆŒ
Ü—z’z NÔ3ˆŒÜŸ	š	 )Ó9ˆŽr!   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r   r’   rÝ   )r   rH   s   "€r   r    r–    s   ø€ ÷ ñ ¡U§\¡\ñ r!   c                óº   € V P                  V4      pV P                  V4      p\        P                  ! V4      pV P                  V4      pV P	                  V4      pV# rK   )r|   r˜  r   ÚtanhrD  )rN   r’   s   &&r   r[   ÚLEDClassificationHead.forward  sN   € ØŸ™ ]Ó3ˆØŸ
™
 =Ó1ˆÜŸ
š
 =Ó1ˆØŸ™ ]Ó3ˆØŸ™ mÓ4ˆØÐr!   )r˜  r|   rD  ra   rj   s   @@r   r�  r�    s!   ù‡ € Ù7÷
:ó 
:÷÷ ð r!   r�  c                   óV   a a€ ] tR tRt oRtRt]R 4       tV 3R ltV3R lt	Rt
VtV ;t# )ÚLEDPreTrainedModeli  ÚledTc                ó²   € V P                   P                  p\        P                  ! . RO^ ^^^V..V P                  R7      pRVP                  V4      RV/pV# )r'   rÐ   r¨   r   )r'   é   é
   é   r   )r€   r   r   ÚtensorrW   Úne)rN   Ú	pad_tokenr   Údummy_inputss   &   r   r©  ÚLEDPreTrainedModel.dummy_inputs$  s[   € à—K‘K×,Ñ,ˆ	Ü—L’LÒ"2°Q¸¸2¸qÀ)Ð4LÐ!MÐVZ×VaÑVaÔbˆ	à˜iŸl™l¨9Ó5Ø˜ð
ˆð Ðr!   c                ó–   <€ \         SV `  V4       \        V\        4      '       d#   \        P
                  ! VP                  4       R # R # rK   )rL   Ú_init_weightsrJ  ÚLEDForConditionalGenerationÚinitÚzeros_Úfinal_logits_bias)rN   ÚmodulerO   s   &&€r   r¬  Ú LEDPreTrainedModel._init_weights.  s5   ø€ Ü‰Ñ˜fÔ%Ü�fÔ9×:Ò:Ü�KŠK˜×0Ñ0Ö1ñ ;r!   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r{  r   )r   rH   s   "€r   r    ÚLEDPreTrainedModel.__annotate__  s   ø‡ ‚ áÑò r!   r_   )rb   rc   rd   re   Úbase_model_prefixÚsupports_gradient_checkpointingÚpropertyr©  r¬  Ú__annotate_func__rg   rh   ri   rj   s   @@r   r   r     s1   ù‡ € ð ÐØ&*Ð#àñó ðõ2÷! … r!   r   zi
    Base class for LEDEncoder's outputs, with potential hidden states, local and global attentions.
    )Úcustom_introc                   ó<   a € ] tR tRt o RtRtRtRtV 3R ltRt	V t
R# )ÚLEDEncoderBaseModelOutputi4  aõ  
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
    Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x +
    attention_window + 1)`, where `x` is the number of tokens with global attention mask.

    Local attentions weights after the attention softmax, used to compute the weighted average in the
    self-attention heads. Those are the attention weights from every token in the sequence to every token with
    global attention (first `x` values) and to every token in the attention window (remaining `attention_window
    + 1` values). Note that the first `x` values refer to tokens with fixed positions in the text, but the
    remaining `attention_window + 1` values refer to tokens with relative positions: the attention weight of a
    token to itself is located at index `x + attention_window / 2` and the `attention_window / 2` preceding
    (succeeding) values are the attention weights to the `attention_window / 2` preceding (succeeding) tokens.
    If the attention window contains a token with global attention, the attention weight at the corresponding
    index is set to 0; the value should be accessed from the first `x` attention weights. If a token has global
    attention, the attention weights to all other tokens in `attentions` is set to 0, the values should be
    accessed from `global_attentions`.
global_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
    Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`,
    where `x` is the number of tokens with global attention mask.

    Global attentions weights after the attention softmax, used to compute the weighted average in the
    self-attention heads. Those are the attention weights from every token with global attention to every token
    in the sequence.
Nc                ó   <€ V ^8„  d   Qh/ S[ P                  ;R&   S[S[ P                  R3,          R,          ;R&   S[S[ P                  R3,          R,          ;R&   S[S[ P                  R3,          R,          ;R&   # )r   Úlast_hidden_state.Nr’   Ú
attentionsÚglobal_attentions)r   ÚFloatTensorr.  )r   rH   s   "€r   r    Ú&LEDEncoderBaseModelOutput.__annotate__4  s†   ø‡ ‚ ñB ×(Ñ(Ñ(ñC ñD ™×*Ñ*¨CÐ/Õ0°4Õ7Ñ>ñE ñF ‘e×'Ñ'¨Ð,Õ-°Õ4Ñ;ñG ñH ™U×.Ñ.°Ð3Õ4°tÕ;ÑBòI r!   r_   )rb   rc   rd   re   rf   r’   r¾  r¿  r¸  rg   rh   ©rH   s   @r   r»  r»  4  s$   ø‡ € ñð4 ;?€MØ7;€JØ>BÐ÷I ƒ r!   r»  z‹
    Base class for model encoder's outputs that also contains : pre-computed hidden states that can speed up sequential
    decoding.
    c                   óT   a € ] tR tRt o RtRtRtRtRtRt	Rt
RtRtRtV 3R ltRtV tR# )ÚLEDSeq2SeqModelOutputi[  a0  
last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
    Sequence of hidden-states at the output of the last layer of the decoder of the model.

    If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1,
    hidden_size)` is output.
past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
    It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

    Contains pre-computed hidden-states (key and values in the attention blocks) of the decoder that can be
    used (see `past_key_values` input) to speed up sequential decoding.
encoder_global_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
    Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`,
    where `x` is the number of tokens with global attention mask.

    Global attentions weights after the attention softmax, used to compute the weighted average in the
    self-attention heads. Those are the attention weights from every token with global attention to every token
    in the sequence.
Nc                ó  <€ V ^8„  d   Qh/ S[ P                  R,          ;R&   S[R,          ;R&   S[S[ P                  R3,          R,          ;R&   S[S[ P                  R3,          R,          ;R&   S[S[ P                  R3,          R,          ;R&   S[ P                  R,          ;R&   S[S[ P                  R3,          R,          ;R	&   S[S[ P                  R3,          R,          ;R
&   S[S[ P                  R3,          R,          ;R&   # )r   Nr½  rH  .Údecoder_hidden_statesÚdecoder_attentionsÚcross_attentionsÚencoder_last_hidden_stater…  Úencoder_attentionsÚencoder_global_attentions©r   rÀ  r   r.  )r   rH   s   "€r   r    Ú"LEDSeq2SeqModelOutput.__annotate__[  s  ø‡ ‚ ñ8 ×(Ñ(¨4Õ/Ñ6ñ9 ñ: ˜T•\Ñ(ñ; ñ< !¡×!2Ñ!2°CÐ!7Õ8¸4Õ?ÑFñ= ñ> ™e×/Ñ/°Ð4Õ5¸Õ<ÑCñ? ñ@ ™E×-Ñ-¨sÐ2Õ3°dÕ:ÑAñA ñB  %×0Ñ0°4Õ7Ñ>ñC ñD !¡×!2Ñ!2°CÐ!7Õ8¸4Õ?ÑFñE ñF ™e×/Ñ/°Ð4Õ5¸Õ<ÑCñG ñH  %¡U×%6Ñ%6¸Ð%;Õ<¸tÕCÑJòI r!   r_   )rb   rc   rd   re   rf   r½  rH  rÆ  rÇ  rÈ  rÉ  r…  rÊ  rË  r¸  rg   rh   rÂ  s   @r   rÄ  rÄ  [  sI   ø‡ € ñð( 37ÐØ$(€OØBFÐØ?CÐØ=AÐØ:>ÐØBFÐØ?CÐØFJÐ÷I ƒ r!   rÄ  zF
    Base class for sequence-to-sequence language models outputs.
    c                   óX   a € ] tR tRt o RtRtRtRtRtRt	Rt
RtRtRtRtV 3R ltRtV tR# )ÚLEDSeq2SeqLMOutputi‚  a*  
loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
    Language modeling loss.
logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
    Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
    It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

    Contains pre-computed hidden-states (key and values in the attention blocks) of the decoder that can be
    used (see `past_key_values` input) to speed up sequential decoding.
encoder_global_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
    Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`,
    where `x` is the number of tokens with global attention mask.

    Global attentions weights after the attention softmax, used to compute the weighted average in the
    self-attention heads. Those are the attention weights from every token with global attention to every token
    in the sequence.
Nc                óJ  <€ V ^8„  d   Qh/ S[ P                  R,          ;R&   S[ P                  R,          ;R&   S[R,          ;R&   S[S[ P                  R3,          R,          ;R&   S[S[ P                  R3,          R,          ;R&   S[S[ P                  R3,          R,          ;R&   S[ P                  R,          ;R	&   S[S[ P                  R3,          R,          ;R
&   S[S[ P                  R3,          R,          ;R&   S[S[ P                  R3,          R,          ;R&   # ©r   NÚlossÚlogitsrH  .rÆ  rÇ  rÈ  rÉ  r…  rÊ  rË  rÌ  )r   rH   s   "€r   r    ÚLEDSeq2SeqLMOutput.__annotate__‚  ó2  ø‡ ‚ ñ4 ×
Ñ
˜dÕ
"Ñ)ñ5 ñ6 ×Ñ Õ$Ñ+ñ7 ñ8 ˜T•\Ñ(ñ9 ñ: !¡×!2Ñ!2°CÐ!7Õ8¸4Õ?ÑFñ; ñ< ™e×/Ñ/°Ð4Õ5¸Õ<ÑCñ= ñ> ™E×-Ñ-¨sÐ2Õ3°dÕ:ÑAñ? ñ@  %×0Ñ0°4Õ7Ñ>ñA ñB !¡×!2Ñ!2°CÐ!7Õ8¸4Õ?ÑFñC ñD ™e×/Ñ/°Ð4Õ5¸Õ<ÑCñE ñF  %¡U×%6Ñ%6¸Ð%;Õ<¸tÕCÑJòG r!   r_   ©rb   rc   rd   re   rf   rÒ  rÓ  rH  rÆ  rÇ  rÈ  rÉ  r…  rÊ  rË  r¸  rg   rh   rÂ  s   @r   rÏ  rÏ  ‚  óM   ø‡ € ñð& &*€DØ'+€FØ$(€OØBFÐØ?CÐØ=AÐØ:>ÐØBFÐØ?CÐØFJÐ÷G ƒ r!   rÏ  zX
    Base class for outputs of sequence-to-sequence sentence classification models.
    c                   óX   a € ] tR tRt o RtRtRtRtRtRt	Rt
RtRtRtRtV 3R ltRtV tR# )Ú"LEDSeq2SeqSequenceClassifierOutputi¨  a*  
loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `label` is provided):
    Classification (or regression if config.num_labels==1) loss.
logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
    Classification (or regression if config.num_labels==1) scores (before SoftMax).
past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
    It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

    Contains pre-computed hidden-states (key and values in the attention blocks) of the decoder that can be
    used (see `past_key_values` input) to speed up sequential decoding.
encoder_global_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
    Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`,
    where `x` is the number of tokens with global attention mask.

    Global attentions weights after the attention softmax, used to compute the weighted average in the
    self-attention heads. Those are the attention weights from every token with global attention to every token
    in the sequence.
Nc                óJ  <€ V ^8„  d   Qh/ S[ P                  R,          ;R&   S[ P                  R,          ;R&   S[R,          ;R&   S[S[ P                  R3,          R,          ;R&   S[S[ P                  R3,          R,          ;R&   S[S[ P                  R3,          R,          ;R&   S[ P                  R,          ;R	&   S[S[ P                  R3,          R,          ;R
&   S[S[ P                  R3,          R,          ;R&   S[S[ P                  R3,          R,          ;R&   # rÑ  rÌ  )r   rH   s   "€r   r    Ú/LEDSeq2SeqSequenceClassifierOutput.__annotate__¨  rÕ  r!   r_   rÖ  rÂ  s   @r   rÙ  rÙ  ¨  r×  r!   rÙ  zS
    Base class for outputs of sequence-to-sequence question answering models.
    c                   ó\   a € ] tR tRt o RtRtRtRtRtRt	Rt
RtRtRtRtRtV 3R ltRtV tR# )Ú&LEDSeq2SeqQuestionAnsweringModelOutputiÎ  a«  
loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
    Total span extraction loss is the sum of a Cross-Entropy for the start and end positions.
past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
    It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

    Contains pre-computed hidden-states (key and values in the attention blocks) of the decoder that can be
    used (see `past_key_values` input) to speed up sequential decoding.
encoder_global_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
    Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`,
    where `x` is the number of tokens with global attention mask.

    Global attentions weights after the attention softmax, used to compute the weighted average in the
    self-attention heads. Those are the attention weights from every token with global attention to every token
    in the sequence.
Nc                óx  <€ V ^8„  d   Qh/ S[ P                  R,          ;R&   S[ P                  R,          ;R&   S[ P                  R,          ;R&   S[R,          ;R&   S[S[ P                  R3,          R,          ;R&   S[S[ P                  R3,          R,          ;R&   S[S[ P                  R3,          R,          ;R	&   S[ P                  R,          ;R
&   S[S[ P                  R3,          R,          ;R&   S[S[ P                  R3,          R,          ;R&   S[S[ P                  R3,          R,          ;R&   # )r   NrÒ  Ústart_logitsÚ
end_logitsrH  .rÆ  rÇ  rÈ  rÉ  r…  rÊ  rË  rÌ  )r   rH   s   "€r   r    Ú3LEDSeq2SeqQuestionAnsweringModelOutput.__annotate__Î  sH  ø‡ ‚ ñ0 ×
Ñ
˜dÕ
"Ñ)ñ1 ñ2 ×#Ñ# dÕ*Ñ1ñ3 ñ4 ×!Ñ! DÕ(Ñ/ñ5 ñ6 ˜T•\Ñ(ñ7 ñ8 !¡×!2Ñ!2°CÐ!7Õ8¸4Õ?ÑFñ9 ñ: ™e×/Ñ/°Ð4Õ5¸Õ<ÑCñ; ñ< ™E×-Ñ-¨sÐ2Õ3°dÕ:ÑAñ= ñ>  %×0Ñ0°4Õ7Ñ>ñ? ñ@ !¡×!2Ñ!2°CÐ!7Õ8¸4Õ?ÑFñA ñB ™e×/Ñ/°Ð4Õ5¸Õ<ÑCñC ñD  %¡U×%6Ñ%6¸Ð%;Õ<¸tÕCÑJòE r!   r_   )rb   rc   rd   re   rf   rÒ  rß  rà  rH  rÆ  rÇ  rÈ  rÉ  r…  rÊ  rË  r¸  rg   rh   rÂ  s   @r   rÝ  rÝ  Î  sR   ø‡ € ñð" &*€DØ-1€LØ+/€JØ$(€OØBFÐØ?CÐØ=AÐØ:>ÐØBFÐØ?CÐØFJÐ÷E ƒ r!   rÝ  c                   óp   a a€ ] tR tRt oRtV3R lV 3R lltV3R lR ltV3R lR ltRR	 ltR
t	Vt
V ;t# )Ú
LEDEncoderió  zÆ
Transformer encoder consisting of *config.encoder_layers* self-attention layers. Each layer is a
[`LEDEncoderLayer`].

Args:
    config: LEDConfig
    embed_tokens (nn.Embedding): output embedding
c                ó    <€ V ^8„  d   QhRS[ /# r{  r   )r   rH   s   "€r   r    ÚLEDEncoder.__annotate__ý  s   ø€ ÷ "ñ "™yñ "r!   c           	     ó  <€ \         SV `  V4       VP                  V n        VP                  V n        VP
                  pVP                  V n        VP                  V n	        \        VP                  \        4      '       dd   VP                  ^,          ^ 8w  d   \        R4      hVP                  ^ 8:  d   \        R4      hVP                  .VP                  ,          Vn        MR\        VP                  4      VP                  8w  d/   \        RVP                   R\        VP                  4       24      h\         P"                  ! VP$                  W P                  4      V n        \)        V P                  V4      V n        \         P,                  ! \/        VP0                  4       Uu. uF  p\3        W4      NK  	  up4      V n        \         P6                  ! V4      V n        RV n        V P=                  4        R# u upi )r   z1`config.attention_window` has to be an even valuez,`config.attention_window` has to be positivezQ`len(config.attention_window)` should equal `config.num_hidden_layers`. Expected z, given FN)rL   rM   r|   Úencoder_layerdropÚ	layerdropr)  r   Úpadding_idxÚmax_encoder_position_embeddingsÚmax_source_positionsrJ  r~   r   r+   Únum_hidden_layersr§   r   Ú	EmbeddingÚ
vocab_sizeÚembed_tokensrC   Úembed_positionsÚ
ModuleListrÔ   Úencoder_layersr_  rO  rd  Úlayernorm_embeddingÚgradient_checkpointingÚ	post_init)rN   r€   rs   ÚirO   s   &&  €r   rM   ÚLEDEncoder.__init__ý  sš  ø€ Ü‰Ñ˜Ô à—~‘~ˆŒØ×1Ñ1ˆŒà—N‘Nˆ	Ø!×.Ñ.ˆÔØ$*×$JÑ$JˆÔ!ä�f×-Ñ-¬s×3Ò3Ø×&Ñ&¨Õ*¨aÔ/Ü Ð!TÓUÐUØ×&Ñ&¨!Ô+Ü Ð!OÓPÐPØ'-×'>Ñ'>Ð&?À&×BZÑBZÕ&ZˆFÕ#ä�6×*Ñ*Ó+¨v×/GÑ/GÔGÜ ð Ø &× 8Ñ 8Ð9¸Ä#Àf×F]ÑF]ÓB^ÐA_ðaóð ô
 ŸLšL¨×):Ñ):¸I×GWÑGWÓXˆÔä<Ø×%Ñ%Øó 
ˆÔô —m’mÌÈv×OdÑOdÔIeÓ$fÑIeÀA¤_°VÖ%?ÑIeÑ$fÓgˆŒÜ#%§<¢<°	Ó#:ˆÔ à&+ˆÔ#à�‰Öùò %gs   Æ(G<c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r   r¨   Úglobal_attention_maskrÝ   )r   rH   s   "€r   r    rå  !  s&   ø€ ÷ 
ñ 
±u·|±|ð 
Ñ\a×\hÑ\hñ 
r!   c                óB   € Ve   W^,           ,          pV# V^,           pV# rK   r_   )rN   r¨   rù  s   &&&r   Ú_merge_to_attention_maskÚ#LEDEncoder._merge_to_attention_mask!  s1   € ð Ò%Ø+ÀqÕ/HÕIˆNð
 Ðð 3°QÕ6ˆNØÐr!   c                ón   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[/# )r   r   r¨   Úinputs_embedsr   r   )r   rH   s   "€r   r    rå  -  sE   ø€ ÷ )Eñ )Eá—<‘<ð)Eñ Ÿ™ð)Eñ —|‘|ð	)Eñ
 ñ)Er!   c                ó2  € \        V P                  P                  \        4      '       d   V P                  P                  M\	        V P                  P                  4      pV^,          ^ 8w  d   \        RV 24      hVe   VP                  MVP                  pVR,          w  rxWXV,          ,
          V,          p	V	^ 8”  dÖ   \        P                  RV RW‰,            RV 24       Ve%   \        P                  P                  V^ V	3VR7      pVeb   VP                  Wy3V P                  P                  \        P                  R7      p
V P!                  V
4      p\        P"                  ! W;.R
R7      p\        P                  P                  V^ V	3R	R7      pW‘W#3# )zbA helper function to pad tokens and mask to work with implementation of Longformer self-attention.z2`attention_window` should be an even value. Given rV   z(Input ids are automatically padded from z to z0 to be a multiple of `config.attention_window`: rú   )r1   r‰   Fr¹   )rJ  r€   r~   r   r  r+   r)   ÚloggerÚwarning_oncer   rŸ   rº   Únew_fullr   r   rY   rï  rž   )rN   r   r¨   rþ  r   r~   Úinput_shaper¬   r\   Úpadding_lenÚinput_ids_paddingÚinputs_embeds_paddings   &&&&&       r   Ú_pad_to_window_sizeÚLEDEncoder._pad_to_window_size-  sŒ  € ô ˜$Ÿ+™+×6Ñ6¼×<Ò<ð �K‰K×(Ò(ä�T—[‘[×1Ñ1Ó2ð 	ð ˜aÕ 1Ô$ÜÐQÐRbÐQcÐdÓeÐeØ)2Ò)>�i—o’oÀM×DWÑDWˆØ)¨"�oÑˆ
à'Ð4DÕ*DÕDÐHXÕXˆØ˜Œ?Ü×ÑØ:¸7¸)À4ÈÕH]ÐG^ð _.Ø.>Ð-?ðAôð Ò$ÜŸM™M×-Ñ-¨i¸!¸[Ð9IÐQ]Ð-Ó^�	ØÒ(Ø$1×$:Ñ$:ØÐ-Ø—K‘K×,Ñ,ÜŸ*™*ð %;ó %Ð!ð
 )-×(9Ñ(9Ð:KÓ(LÐ%Ü %§	¢	¨=Ð*PÐVXÔ Y�äŸ]™]×.Ñ.Ø  KÐ 0¸ð /ó ˆNð  ~ÐDÐDr!   c           
     óö  a€ Ve   TMV P                   P                  pVe   TMV P                   P                  pVe   TMV P                   P                  pVe   Ve   \	        R4      hVf   Vf   \	        R4      hVf   V P                  V4      pVfC   \        P                  ! VP                  4       RR VP                  \        P                  R7      pVe   V P                  W#4      pV P                  VVVV P                   P                  R7      w  orpVe+   VP                  4       p	VP                  RV	R,          4      pMVe   VP                  4       RR p	Ve   \        W$P                   4      R,          pV^ 8  p
V^ 8„  pVP#                  4       P%                  4       P'                  4       pV P)                  X	4      pWM,           pV P+                  V4      p\,        P.                  P1                  WàP0                  V P2                  R7      pV'       d   RMRpV'       d   RMRpV'       d   V'       d   RMRp\5        V P6                  4       FÀ  w  ppV'       d
   Wþ3,           p\        P8                  ! . 4      pV P2                  '       d   VV P:                  8  d   RpMV! VVV
VVVR7      pV^ ,          pV'       g   Kt  VV^,          P=                  ^^4      3,           pV'       g   KŸ  VV^,          P=                  ^^4      3,           pKÂ  	  V'       d
   Wþ3,           pS^ 8”  dƒ   VRRS) 13,          pV'       d3   \>        ;QJ d    . V3R	 lV 4       F  NK  	  5M! V3R	 lV 4       4      pV'       d3   \>        ;QJ d    . V3R
 lV 4       F  NK  	  5M! V3R
 lV 4       4      pV'       g3   \>        ;QJ d    . R WïVV3 4       F  NK  	  5# ! R WïVV3 4       4      # \A        VVVVR7      # )a
  
Args:
    input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
        Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
        provide it.

        Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
        [`PreTrainedTokenizer.__call__`] for details.

        [What are input IDs?](../glossary#input-ids)
    attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
        Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

        - 1 for tokens that are **not masked**,
        - 0 for tokens that are **masked**.

        [What are attention masks?](../glossary#attention-mask)
    global_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
        Mask to decide the attention given on each token, local attention or global attention for the encoder.
        Tokens with global attention attends to all other tokens, and all other tokens attend to them. This is
        important for task-specific finetuning because it makes the model more flexible at representing the
        task. For example, for classification, the <s> token should be given global attention. For QA, all
        question tokens should also have global attention. Please refer to the [Longformer
        paper](https://huggingface.co/papers/2004.05150) for more details. Mask values selected in `[0, 1]`:

        - 0 for local attention (a sliding window attention),
        - 1 for global attention (tokens that attend to all other tokens, and all other tokens attend to them).
    inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
        Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
        This is useful if you want more control over how to convert `input_ids` indices into associated vectors
        than the model's internal embedding lookup matrix.
    output_attentions (`bool`, *optional*):
        Whether or not to return the attentions tensors of all attention layers. See `attentions` under
        returned tensors for more detail.
    output_hidden_states (`bool`, *optional*):
        Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
        for more detail.
    return_dict (`bool`, *optional*):
        Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
NzDYou cannot specify both input_ids and inputs_embeds at the same timez5You have to specify either input_ids or inputs_embeds)rW   r1   )r   r¨   rþ  r   r#   r�   )r¨   r“   r©   rª   r«   c              3   ó>   <"  € T F  qR RS) 13,          x € K  	  R# 5i©r#   Nr_   ©Ú.0Ústater  s   & €r   Ú	<genexpr>Ú%LEDEncoder.forward.<locals>.<genexpr>è  s"   øé € Ð&[ÉNÀ5¨Q°°+°°Ð-=×'>Ò'>ËNùs   ƒc              3   óB   <"  € T F  qR R RS) 1R 3,          x € K  	  R# 5ir  r_   r  s   & €r   r  r  ë  s(   øé € Ð&aÑR`È¨Q°°=°[°L°=À!Ð-C×'DÒ'DÓR`ùs   ƒc              3   ó0   "  € T F  qf   K  Vx € K  	  R # 5irK   r_   ©r  Úvs   & r   r  r  î  s   é € ð Ùa�a—’Óaùs   ‚Œ
©r½  r’   r¾  r¿  r$   )r#   r'   r'   r#   r_   )NNN)!r€   r«   Úoutput_hidden_statesÚreturn_dictr+   rï  r   Úonesr5   rW   rY   rû  r  r   r—   rA   r1   ÚflattenÚanyÚitemrð  ró  r   rŸ   r|   r�   Ú	enumeraterO  Úrandrè  r”   r.  r»  )rN   r   r¨   rù  rþ  r«   r  r  r‰  r  r“   r©   rª   Ú	embed_posr’   Úencoder_statesÚall_attentionsÚall_global_attentionsÚidxÚencoder_layerÚdropout_probabilityÚlayer_outputsr  s   &&&&&&&&,             @r   r[   ÚLEDEncoder.forwardX  s»  ø€ ðf 2CÒ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ò$DÑ È$Ï+É+×JjÑJjð 	ð &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆð Ò  ]Ò%>ÜÐcÓdÐdØÒ =Ò#8ÜÐTÓUÐUàÒ Ø ×-Ñ-¨iÓ8ˆMð Ò!Ü"ŸZšZ¨×(:Ñ(:Ó(<¸S¸bÐ(AÈ-×J^ÑJ^Ôfk×fpÑfpÔqˆNð !Ò,Ø!×:Ñ:¸>ÓaˆNð AE×@XÑ@XØØ)Ø'ØŸ™×1Ñ1ð	 AYó A
Ñ=ˆ�Y°ð Ò Ø#Ÿ.™.Ó*ˆKØ!Ÿ™ r¨;°r­?Ó;‰IØÒ&Ø'×,Ñ,Ó.¨s°Ð3ˆKð Ò%ä@À×QdÑQdÓeÐfpÕqˆNð )¨1Ñ,ˆØ-°Ñ1ÐØ-×5Ñ5Ó7×;Ñ;Ó=×BÑBÓDˆà×(Ñ(¨Ó5ˆ	à%Õ1ˆØ×0Ñ0°Ó?ˆÜŸ™×-Ñ-¨m¿|¹|ÐVZ×VcÑVcÐ-Ódˆç3™¸ˆß0™°dˆß'8¿^¡ÐRVÐä"+¨D¯K©KÖ"8ÑˆC�ß#Ø!/Ð2BÕ!B�ä"'§*¢*¨R£.Ðà�}�}ˆ}Ð"5¸¿¹Ô"FØ 2‘á -Ø!Ø#1Ø$3Ø)=Ø#1Ø&7ô!�ð !.¨aÕ 0�ç Ñ à!/°=ÀÕ3C×3MÑ3MÈaÐQRÓ3SÐ2UÕ!U�ç!‘>à,AÀ]ÐSTÕEU×E_ÑE_Ð`aÐcdÓEeÐDgÕ,gÒ)ñ3 #9÷6  Ø+Ð.>Õ>ˆNð ˜Œ?à)¨!¨]¨{¨l¨]Ð*:Õ;ˆMß#ß!&¤Ô&[ÉNÓ&[§¡Ô&[ÉNÓ&[Ó![�ç ß!&¤Ô&aÑR`Ó&a§¡Ô&aÑR`Ó&aÓ!a�çß”5ñ Ø)¸>ÐK`Ñaó—5ð �5ñ Ø)¸>ÐK`Ñaóó ð ô )Ø+Ø(Ø%Ø3ô	
ð 	
r!   )	r|   rð  rï  rô  rè  ró  rO  rë  ré  )NNNNNNN)rb   rc   rd   re   rf   rM   rû  r  r[   rg   rh   ri   rj   s   @@r   rã  rã  ó  s8   ù‡ € ñ÷"ó "÷H
ð 
÷)Eð )E÷V^
ô ^
r!   rã  c                   óL   a a€ ] tR tRt oRtV3R lV 3R lltRR ltRtVtV ;t	# )Ú
LEDDecoderiù  z¶
Transformer decoder consisting of *config.decoder_layers* layers. Each layer is a [`LEDDecoderLayer`]

Args:
    config: LEDConfig
    embed_tokens (nn.Embedding): output embedding
c                ó    <€ V ^8„  d   QhRS[ /# r{  r   )r   rH   s   "€r   r    ÚLEDDecoder.__annotate__  s   ø€ ÷ ñ ™yñ r!   c           
     ó€  <€ \         SV `  V4       VP                  V n        VP                  V n        VP
                  V n        VP                  V n        \        P                  ! VP                  VP                  V P                  4      V n        \        V P                  VP                  4      V n        \        P                   ! \#        VP$                  4       Uu. uF  p\'        WR 7      NK  	  up4      V n        \        P*                  ! VP                  4      V n        RV n        V P1                  4        R# u upi ))r<  FN)rL   rM   r|   Údecoder_layerdroprè  r   ré  Úmax_decoder_position_embeddingsÚmax_target_positionsr   rí  rî  r)  rï  rC   rð  rñ  rÔ   Údecoder_layersry  rO  rd  ró  rô  rõ  )rN   r€   rö  rO   s   && €r   rM   ÚLEDDecoder.__init__  sè   ø€ Ü‰Ñ˜Ô Ø—~‘~ˆŒØ×1Ñ1ˆŒØ!×.Ñ.ˆÔØ$*×$JÑ$JˆÔ!äŸLšL¨×):Ñ):¸F¿N¹NÈD×L\ÑL\Ó]ˆÔä<Ø×%Ñ%Ø�N‰Nó 
ˆÔô —m’mÔSXÐY_×YnÑYnÔSoÓ$pÑSoÈa¤_°V×%IÑSoÑ$pÓqˆŒÜ#%§<¢<°·±Ó#?ˆÔ à&+ˆÔ#à�‰Öùò %qs   ÃD;c                ó´  € V	e   T	MV P                   P                  p	V
e   T
MV P                   P                  p
Ve   TMV P                   P                  pVe   TMV P                   P                  pVe   Ve   \        R4      hVe+   VP                  4       pVP                  RVR,          4      pM#Ve   VP                  4       RR pM\        R4      hVf   V P                  V4      pV P                  '       d2   V P                  '       d    V'       d   \        P                  R4       RpV'       d9   Vf5   \        \        V P                   R7      \        V P                   R7      4      pVe   VP                  4       M^ pRpVR,          ^8”  d   \!        V P                   VVVR7      p\#        V P                   VVVR7      pV P%                  WÞ4      pVV,           pV P'                  V4      p\(        P*                  P-                  VV P,                  V P                  R	7      pV
'       d   RMRpV	'       d   RMRpV	'       d   RMRp\/        V P0                  4       F—  w  ppV
'       d   VV3,          pV P                  '       d*   \2        P4                  ! . 4      pVV P6                  8  d   KS  V! VVVVVV	VR
7      pV^ ,          pV	'       g   Ku  VV^,          3,          pVV^,          3,          pK™  	  V
'       d   VV3,          pV'       g7   \8        ;QJ d    . R VVVVV3 4       F  NK  	  5# ! R VVVVV3 4       4      # \;        VVVVVR7      # )a�  
Args:
    input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
        Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
        provide it.

        Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
        [`PreTrainedTokenizer.__call__`] for details.

        [What are input IDs?](../glossary#input-ids)
    attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
        Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

        - 1 for tokens that are **not masked**,
        - 0 for tokens that are **masked**.

        [What are attention masks?](../glossary#attention-mask)
    global_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
        Mask to decide the attention given on each token, local attention or global attention. Tokens with
        global attention attends to all other tokens, and all other tokens attend to them. This is important
        for task-specific finetuning because it makes the model more flexible at representing the task. For
        example, for classification, the <s> token should be given global attention. For QA, all question
        tokens should also have global attention. Please refer to the [Longformer
        paper](https://huggingface.co/papers/2004.05150) for more details. Mask values selected in `[0, 1]`:

        - 0 for local attention (a sliding window attention),
        - 1 for global attention (tokens that attend to all other tokens, and all other tokens attend to them).
    encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*):
        Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention
        of the decoder.
    encoder_attention_mask (`torch.LongTensor` of shape `(batch_size, encoder_sequence_length)`, *optional*):
        Mask to avoid performing cross-attention on padding tokens indices of encoder input_ids. Mask values
        selected in `[0, 1]`:

        - 1 for tokens that are **not masked**,
        - 0 for tokens that are **masked**.

        [What are attention masks?](../glossary#attention-mask)
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks and in the
        cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.

        If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those
        that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of
        all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
    inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
        Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
        This is useful if you want more control over how to convert `input_ids` indices into associated vectors
        than the model's internal embedding lookup matrix.
    output_attentions (`bool`, *optional*):
        Whether or not to return the attentions tensors of all attention layers. See `attentions` under
        returned tensors for more detail.
    output_hidden_states (`bool`, *optional*):
        Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
        for more detail.
    return_dict (`bool`, *optional*):
        Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
NzTYou cannot specify both decoder_input_ids and decoder_inputs_embeds at the same timezEYou have to specify either decoder_input_ids or decoder_inputs_embedszZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...F)r€   )r€   rþ  r¨   rH  )r€   rþ  r¨   r…  r�   )r†  rH  r«   r‡  c              3   ó2   "  € T F  pVf   K	  Vx € K  	  R # 5irK   r_   r  s   & r   r  Ú%LEDDecoder.forward.<locals>.<genexpr>¼  s   é € ð ár�AØ÷ ’Ûrùs   ‚�
)r½  rH  r’   r¾  rÈ  r$   r_   )r€   r«   r  r‡  r  r+   r5   r—   rï  rô  r�   r   r  r	   r   Úget_seq_lengthr   r   rð  ró  r   rŸ   r|   r  rO  r   r  rè  r.  r   )rN   r   r¨   rù  r…  r†  rH  rþ  r‡  r«   r  r  r‰  r  rS   Úcombined_attention_maskr]   r’   Úall_hidden_statesÚall_self_attnsÚall_cross_attentionsr"  Údecoder_layerr$  r%  s   &&&&&&&&&&&&,            r   r[   ÚLEDDecoder.forward  s5  € ðV 2CÒ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ò$DÑ È$Ï+É+×JjÑJjð 	ð "+Ò!6‘I¸D¿K¹K×<QÑ<Qˆ	Ø%0Ò%<‘kÀ$Ç+Á+×BYÑBYˆð Ò  ]Ò%>ÜÐsÓtÐtØÒ"Ø#Ÿ.™.Ó*ˆKØ!Ÿ™ r¨;°r­?Ó;‰IØÒ&Ø'×,Ñ,Ó.¨s°Ð3‰KäÐdÓeÐeàÒ Ø ×-Ñ-¨iÓ8ˆMà×&×&Ð&¨4¯=¯=¨=ßÜ×#Ñ#Øpôð "�	ç˜Ò0Ü1´,ÀdÇkÁkÔ2RÔT`Ðhl×hsÑhsÔTtÓuˆOàETÒE` ×!?Ñ!?Ô!AÐfgÐà"&ÐØ�r�?˜QÔÜ&8Ø—{‘{Ø+Ø-Ø /ô	'Ð#ô ";Ø—;‘;Ø'Ø1Ø"7ô	"
Ðð ×(Ñ(¨ÓMˆ	à%¨	Õ1ˆØ×0Ñ0°Ó?ˆäŸ™×-Ñ-¨m¸t¿|¹|ÐVZ×VcÑVcÐ-Ódˆ÷ #7™B¸DÐß0™°dˆß%6™r¸DÐä"+¨D¯K©KÖ"8ÑˆC�ç#Ø! mÐ%5Õ5Ð!Ø�}�}ˆ}Ü&+§j¢j°£nÐ#Ø&¨¯©Ô7Ùá)ØØ'Ø%Ø'=Ø /Ø"3Ø#ôˆMð *¨!Õ,ˆMß Ñ Ø =°Õ#3Ð"5Õ5�Ø$¨°qÕ)9Ð(;Õ;Ò$ñ- #9÷2  Ø -Ð!1Õ1Ðçß”5ñ à'¨Ð:KÈ^Ð]qÑró—5ð �5ñ à'¨Ð:KÈ^Ð]qÑróó ð ô
 9Ø+Ø+Ø+Ø%Ø1ô
ð 	
r!   )	r|   rð  rï  rô  rè  ró  rO  r.  ré  )NNNNNNNNNNNra   rj   s   @@r   r(  r(  ù  s    ù‡ € ñ÷ó ÷(q
ô q
r!   r(  c                   óv   a a€ ] tR tRt oRRRR/tV3R lV 3R lltR tR t]RV3R	 lR
 ll4       t	Rt
VtV ;t# )ÚLEDModeliÊ  zencoder.embed_tokens.weightzshared.weightzdecoder.embed_tokens.weightc                ó    <€ V ^8„  d   QhRS[ /# r{  r   )r   rH   s   "€r   r    ÚLEDModel.__annotate__Ñ  s   ø€ ÷ 
ñ 
™yñ 
r!   c                ó   <€ \         SV `  V4       VP                  VP                  r2\        P
                  ! W1P                  V4      V n        \        V4      V n	        \        V4      V n        V P                  4        R # rK   )rL   rM   r   rî  r   rí  r)  Úsharedrã  Úencoderr(  Údecoderrõ  )rN   r€   ré  rî  rO   s   &&  €r   rM   ÚLEDModel.__init__Ñ  s]   ø€ Ü‰Ñ˜Ô à"(×"5Ñ"5°v×7HÑ7H�ZÜ—l’l :¯~©~¸{ÓKˆŒä! &Ó)ˆŒÜ! &Ó)ˆŒð 	�‰Ör!   c                ó   € V P                   # rK   )r@  )rN   s   &r   Úget_input_embeddingsÚLEDModel.get_input_embeddingsÝ  s   € Ø�{‰{Ðr!   c                ó~   € Wn         V P                   V P                  n        V P                   V P                  n        R # rK   )r@  rA  rï  rB  )rN   rw   s   &&r   Úset_input_embeddingsÚLEDModel.set_input_embeddingsà  s'   € ØŒØ$(§K¡Kˆ�‰Ô!Ø$(§K¡Kˆ�‰Ö!r!   c                ó  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[S[ P                  ,          ,          R,          RS[ P                  R,          RS[R,          R	S[ P                  R,          R
S[ P                  R,          RS[R,          RS[R,          RS[R,          RS[R,          RS[S[ P                  ,          S[,          /# )r   r   Nr¨   Údecoder_input_idsÚdecoder_attention_maskÚencoder_outputsrù  rH  rþ  Údecoder_inputs_embedsr‡  r«   r  r  rÜ   )r   Ú
LongTensorr   r.  rÀ  r   r9   rÄ  )r   rH   s   "€r   r    r>  æ  s*  ø€ ÷ k
ñ k
á×#Ñ# dÕ*ðk
ñ Ÿ™ tÕ+ðk
ñ !×+Ñ+¨dÕ2ð	k
ñ
 !&× 0Ñ 0°4Õ 7ðk
ñ ™u¡U×%6Ñ%6Õ7Õ8¸4Õ?ðk
ñ  %×0Ñ0°4Õ7ðk
ñ  �ðk
ñ ×(Ñ(¨4Õ/ðk
ñ  %×0Ñ0°4Õ7ðk
ñ ˜$•;ðk
ñ   $�;ðk
ñ # T�kðk
ñ ˜D•[ðk
ñ  
‰u�|‰|Õ	Ñ4Õ	4ñ!k
r!   c                óÒ  € Ve   TMV P                   P                  pVe   TMV P                   P                  pV
e   T
MV P                   P                  p
Ve   TMV P                   P                  pVf9   V	f5   \        WP                   P                  V P                   P                  4      pVf   V P                  VVVVVVVR7      pMV'       dx   \        V\        4      '       gb   \        V^ ,          \        V4      ^8”  d
   V^,          MR\        V4      ^8”  d
   V^,          MR\        V4      ^8”  d
   V^,          MRR7      pV P                  VVV^ ,          VVV	V
VVVR7
      pV'       g	   Wõ,           # \        VP                  VP                  VP                   VP"                  VP$                  VP                  VP                   VP"                  VP&                  R7	      # )á“  
decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Indices of decoder input sequence tokens in the vocabulary.

    Indices can be obtained using [`LedTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are input IDs?](../glossary#input-ids)

    LED uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
    is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).
decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
    be used by default.

    If you want to change padding behavior, you should read [`modeling_led._prepare_decoder_inputs`] and modify
    to your needs. See diagram 1 in [the paper](https://huggingface.co/papers/1910.13461) for more information on the
    default strategy.
global_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Mask to decide the attention given on each token, local attention or global attention for the encoder.
    Tokens with global attention attends to all other tokens, and all other tokens attend to them. This is
    important for task-specific finetuning because it makes the model more flexible at representing the task.
    For example, for classification, the <s> token should be given global attention. For QA, all question
    tokens should also have global attention. Please refer to the [Longformer
    paper](https://huggingface.co/papers/2004.05150) for more details. Mask values selected in `[0, 1]`:

    - 0 for local attention (a sliding window attention),
    - 1 for global attention (tokens that attend to all other tokens, and all other tokens attend to them).
N)r   r¨   rù  rþ  r«   r  r  r  )
r   r¨   r…  r†  rH  rþ  r‡  r«   r  r  )	r½  rH  rÆ  rÇ  rÈ  rÉ  r…  rÊ  rË  )r€   r«   r  r‡  r  r.   r   r   rA  rJ  r»  r§   rB  rÄ  r½  rH  r’   r¾  rÈ  r¿  )rN   r   r¨   rK  rL  rM  rù  rH  rþ  rN  r‡  r«   r  r  r‰  Údecoder_outputss   &&&&&&&&&&&&&&, r   r[   ÚLEDModel.forwardå  s×  € ð^ 2CÒ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ò$DÑ È$Ï+É+×JjÑJjð 	ð "+Ò!6‘I¸D¿K¹K×<QÑ<Qˆ	Ø%0Ò%<‘kÀ$Ç+Á+×BYÑBYˆð
 Ò$Ð)>Ò)FÜ 2ØŸ;™;×3Ñ3°T·[±[×5WÑ5Wó!Ðð Ò"Ø"Ÿl™lØ#Ø-Ø&;Ø+Ø"3Ø%9Ø'ð +ó ‰O÷ ¤¨OÔ=V×!WÒ!WÜ7Ø"1°!Õ"4Ü47¸Ó4HÈ1Ô4L˜o¨aÖ0ÐRVÜ14°_Ó1EÈÔ1I˜?¨1Ö-ÈtÜ8;¸OÓ8LÈqÔ8P /°!Ö"4ÐVZô	ˆOð Ÿ,™,Ø'Ø1Ø"1°!Õ"4Ø#1Ø+Ø/ØØ/Ø!5Ø#ð 'ó 
ˆ÷ Ø"Õ4Ð4ä$Ø-×?Ñ?Ø+×;Ñ;Ø"1×"?Ñ"?Ø.×9Ñ9Ø,×=Ñ=Ø&5×&GÑ&GØ"1×"?Ñ"?Ø.×9Ñ9Ø&5×&GÑ&Gô

ð 
	
r!   )rB  rA  r@  )NNNNNNNNNNNNN)rb   rc   rd   re   Ú_tied_weights_keysrM   rE  rH  r   r[   rg   rh   ri   rj   s   @@r   r<  r<  Ê  sO   ù‡ € ð 	& Ø% ðÐ÷

ó 
òò0ð
 ÷k
ñ k
ó ÷k
ð k
r!   r<  zU
    The LED Model with a language modeling head. Can be used for summarization.
    c                   ó°   a a€ ] tR tRt oRtR.tRR/tV3R lV 3R lltRV3R lV 3R	 llltV3R
 lR lt	]
RV3R lR ll4       tV3R lR ltRtVtV ;t# )r­  iT  r¡  r°  zlm_head.weightzled.shared.weightc                ó    <€ V ^8„  d   QhRS[ /# r{  r   )r   rH   s   "€r   r    Ú(LEDForConditionalGeneration.__annotate__`  s   ø€ ÷ ñ ™yñ r!   c                ó�  <€ \         SV `  V4       \        V4      V n        \        P
                  ! \        P                  ! ^V P                  P                  P                  34      4      V n
        \        P                  ! VP                  V P                  P                  P                  RR7      V n        V P                  4        R# )r&   Fr?  N)rL   rM   r<  r¡  r   ÚBufferr   Úzerosr@  rE   r°  rt   r)  Úlm_headrõ  ©rN   r€   rO   s   &&€r   rM   Ú$LEDForConditionalGeneration.__init__`  s|   ø€ Ü‰Ñ˜Ô Ü˜FÓ#ˆŒÜ!#§¢¬5¯;ª;¸¸4¿8¹8¿?¹?×;YÑ;YÐ7ZÓ+[Ó!\ˆÔÜ—y’y §¡°·±·±×1OÑ1OÐV[Ô\ˆŒð 	�‰Ör!   c                óT   <€ V ^8„  d   QhRS[ RS[ R,          RS[RS[P                  /# )r   Únew_num_tokensÚpad_to_multiple_ofNÚmean_resizingrÜ   )r   r9   r   rí  )r   rH   s   "€r   r    rW  i  s5   ø€ ÷ ñ Ù!ðÙ7:¸TµzðÙY]ðá	�‰ñr!   c                ó€   <€ \         SV `  WV4      pV P                  VP                  P                  ^ ,          4       V# r`   )rL   Úresize_token_embeddingsÚ_resize_final_logits_biasrZ   r)   )rN   r_  r`  ra  Únew_embeddingsrO   s   &&&& €r   rc  Ú3LEDForConditionalGeneration.resize_token_embeddingsi  s<   ø€ ô ™Ñ8¸Ð]jÓkˆØ×&Ñ& ~×'<Ñ'<×'BÑ'BÀ1Õ'EÔFØÐr!   c                ó$   <€ V ^8„  d   QhRS[ RR/# )r   r_  rÜ   NrG   )r   rH   s   "€r   r    rW  p  s   ø€ ÷ 5ñ 5¹ð 5Àñ 5r!   c                ób  € V P                   P                  R,          pW8:  d   V P                   RRV13,          pMX\        P                  ! ^W,
          3V P                   P                  R7      p\        P
                  ! V P                   V.^R7      p\        P                  ! V4      V n         R# )r&   r#   NrÐ   r‰   r$   )r°  r)   r   rZ  rW   rž   r   rY  )rN   r_  Úold_num_tokensÚnew_biasÚ
extra_biass   &&   r   rd  Ú5LEDForConditionalGeneration._resize_final_logits_biasp  sˆ   € Ø×/Ñ/×5Ñ5°bÕ9ˆØÔ+Ø×-Ñ-¨a°°.°Ð.@ÕA‰HäŸš a¨Õ)HÐ%IÐRV×RhÑRh×RoÑRoÔpˆJÜ—y’y $×"8Ñ"8¸*Ð!EÈ1ÔMˆHÜ!#§¢¨8Ó!4ˆÖr!   c                ó@  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[S[ P                  ,          ,          R,          RS[ P                  R,          RS[R,          R	S[ P                  R,          R
S[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[R,          RS[R,          RS[S[ P                  ,          S[,          /# )r   r   Nr¨   rK  rL  rM  rù  rH  rþ  rN  Úlabelsr‡  r«   r  r  rÜ   )r   rO  r   r.  rÀ  r   r9   rÏ  )r   rH   s   "€r   r    rW  z  s@  ø€ ÷ V
ñ V
á×#Ñ# dÕ*ðV
ñ Ÿ™ tÕ+ðV
ñ !×+Ñ+¨dÕ2ð	V
ñ
 !&× 0Ñ 0°4Õ 7ðV
ñ ™u¡U×%6Ñ%6Õ7Õ8¸4Õ?ðV
ñ  %×0Ñ0°4Õ7ðV
ñ  �ðV
ñ ×(Ñ(¨4Õ/ðV
ñ  %×0Ñ0°4Õ7ðV
ñ × Ñ  4Õ'ðV
ñ ˜$•;ðV
ñ   $�;ðV
ñ # T�kðV
ñ ˜D•[ðV
ñ" 
‰u�|‰|Õ	Ñ1Õ	1ñ#V
r!   c                ó:  € Ve   TMV P                   P                  pV
e\   V'       d   \        P                  R4       RpVf9   V	f5   \	        W P                   P
                  V P                   P                  4      pV P                  VVVVVVVVV	VVVVR7      pV P                  V^ ,          4      V P                  ,           pRpV
eG   \        4       pV! VP                  RV P                   P                  4      V
P                  R4      4      pV'       g"   V3VR,          ,           pVe   V3V,           # T# \        VVVP                  VP                  VP                   VP"                  VP$                  VP&                  VP(                  VP*                  R7
      # )a³  
decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Indices of decoder input sequence tokens in the vocabulary.

    Indices can be obtained using [`LedTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are input IDs?](../glossary#input-ids)

    LED uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
    is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).
decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
    be used by default.

    If you want to change padding behavior, you should read [`modeling_led._prepare_decoder_inputs`] and modify
    to your needs. See diagram 1 in [the paper](https://huggingface.co/papers/1910.13461) for more information on the
    default strategy.
global_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Mask to decide the attention given on each token, local attention or global attention for the encoder.
    Tokens with global attention attends to all other tokens, and all other tokens attend to them. This is
    important for task-specific finetuning because it makes the model more flexible at representing the task.
    For example, for classification, the <s> token should be given global attention. For QA, all question
    tokens should also have global attention. Please refer to the [Longformer
    paper](https://huggingface.co/papers/2004.05150) for more details. Mask values selected in `[0, 1]`:

    - 0 for local attention (a sliding window attention),
    - 1 for global attention (tokens that attend to all other tokens, and all other tokens attend to them).
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

Example Summarization:

```python
>>> import torch
>>> from transformers import AutoTokenizer, LEDForConditionalGeneration

>>> model = LEDForConditionalGeneration.from_pretrained("allenai/led-large-16384-arxiv")
>>> tokenizer = AutoTokenizer.from_pretrained("allenai/led-large-16384-arxiv")

>>> ARTICLE_TO_SUMMARIZE = '''Transformers (Vaswani et al., 2017) have achieved state-of-the-art
...     results in a wide range of natural language tasks including generative language modeling
...     (Dai et al., 2019; Radford et al., 2019) and discriminative ... language understanding (Devlin et al., 2019).
...     This success is partly due to the self-attention component which enables the network to capture contextual
...     information from the entire sequence. While powerful, the memory and computational requirements of
...     self-attention grow quadratically with sequence length, making it infeasible (or very expensive) to
...     process long sequences. To address this limitation, we present Longformer, a modified Transformer
...     architecture with a self-attention operation that scales linearly with the sequence length, making it
...     versatile for processing long documents (Fig 1). This is an advantage for natural language tasks such as
...     long document classification, question answering (QA), and coreference resolution, where existing approaches
...     partition or shorten the long context into smaller sequences that fall within the typical 512 token limit
...     of BERT-style pretrained models. Such partitioning could potentially result in loss of important
...     cross-partition information, and to mitigate this problem, existing methods often rely on complex
...     architectures to address such interactions. On the other hand, our proposed Longformer is able to build
...     contextual representations of the entire context using multiple layers of attention, reducing the need for
...     task-specific architectures.'''
>>> inputs = tokenizer.encode(ARTICLE_TO_SUMMARIZE, return_tensors="pt")

>>> # Global attention on the first token (cf. Beltagy et al. 2020)
>>> global_attention_mask = torch.zeros_like(inputs)
>>> global_attention_mask[:, 0] = 1

>>> # Generate Summary
>>> summary_ids = model.generate(inputs, global_attention_mask=global_attention_mask, num_beams=3, max_length=32)
>>> print(tokenizer.decode(summary_ids[0], skip_special_tokens=True, clean_up_tokenization_spaces=True))
```

Example Conditional generation :

```python
>>> from transformers import AutoTokenizer, LEDForConditionalGeneration

>>> tokenizer = AutoTokenizer.from_pretrained("allenai/led-base-16384")
>>> TXT = "My friends are <mask> but they eat too many carbs."

>>> model = LEDForConditionalGeneration.from_pretrained("allenai/led-base-16384")
>>> input_ids = tokenizer([TXT], return_tensors="pt")["input_ids"]

>>> prediction = model.generate(input_ids)[0]
>>> print(tokenizer.decode(prediction, skip_special_tokens=True))
```
NzJThe `use_cache` argument is changed to `False` since `labels` is provided.F)r¨   rK  rL  rM  rù  rH  rþ  rN  r‡  r«   r  r  r%   )
rÒ  rÓ  rH  rÆ  rÇ  rÈ  rÉ  r…  rÊ  rË  r$   )r€   r  r   Úwarningr.   r   r   r¡  r[  r°  r   r—   rî  rÏ  rH  rÆ  rÇ  rÈ  rÉ  r…  rÊ  rË  )rN   r   r¨   rK  rL  rM  rù  rH  rþ  rN  rn  r‡  r«   r  r  r‰  r¶   Ú	lm_logitsÚmasked_lm_lossÚloss_fctr*  s   &&&&&&&&&&&&&&&,     r   r[   Ú#LEDForConditionalGeneration.forwardy  s‰  € ðN &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆàÒßÜ—‘ÐkÔlØˆIØ Ò(Ð-BÒ-JÜ$6ØŸK™K×4Ñ4°d·k±k×6XÑ6Xó%Ð!ð —(‘(ØØ)Ø/Ø#9Ø+Ø"7Ø+Ø'Ø"7ØØ/Ø!5Ø#ð ó 
ˆð —L‘L ¨¥Ó,¨t×/EÑ/EÕEˆ	àˆØÒÜ'Ó)ˆHÙ% i§n¡n°R¸¿¹×9OÑ9OÓ&PÐRX×R]ÑR]Ð^`ÓRaÓbˆNçØ�\ G¨B¥KÕ/ˆFØ3AÒ3M�^Ð%¨Õ.ÐYÐSYÐYä!ØØØ#×3Ñ3Ø")×"?Ñ"?Ø&×9Ñ9Ø$×5Ñ5Ø&-×&GÑ&GØ")×"?Ñ"?Ø&×9Ñ9Ø&-×&GÑ&Gô
ð 	
r!   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r   rn  rÝ   )r   rH   s   "€r   r    rW    s   ø€ ÷ hñ h¹E¿L¹Lñ hr!   c                ój   € \        WP                  P                  V P                  P                  4      # rK   )r.   r€   r   r   )rN   rn  s   &&r   Ú%prepare_decoder_input_ids_from_labelsÚALEDForConditionalGeneration.prepare_decoder_input_ids_from_labels  s#   € Ü! &¯+©+×*BÑ*BÀDÇKÁK×DfÑDfÓgÐgr!   )r°  r¡  r[  )NT©NNNNNNNNNNNNNN)rb   rc   rd   re   rµ  Ú_keys_to_ignore_on_load_missingrT  rM   rc  rd  r   r[   rw  rg   rh   ri   rj   s   @@r   r­  r­  T  sq   ù‡ € ð ÐØ':Ð&;Ð#àÐ-ðÐ÷ó ÷õ ÷5ð 5ð ÷V
ñ V
ó ðV
÷ph÷ hð hr!   r­  c                   óR   a a€ ] tR tRt oV 3R lt]RV3R lR ll4       tRtVtV ;t	# )ÚLEDForQuestionAnsweringi  c                óö   <€ \         SV `  V4       ^Vn        VP                  V n        \        V4      V n        \
        P                  ! VP                  VP                  4      V n        V P                  4        R# )r   N)
rL   rM   Ú
num_labelsr<  r¡  r   rt   ro   Ú
qa_outputsrõ  r\  s   &&€r   rM   Ú LEDForQuestionAnswering.__init__  s[   ø€ Ü‰Ñ˜Ô àˆÔØ ×+Ñ+ˆŒä˜FÓ#ˆŒÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Ör!   c                óT  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[S[ P                  ,          ,          R,          RS[ P                  R,          RS[ P                  R,          R	S[ P                  R,          R
S[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[R,          RS[R,          RS[S[ P                  ,          S[,          /# )r   r   Nr¨   rK  rL  rM  rù  Ústart_positionsÚend_positionsrþ  rN  r‡  r«   r  r  rÜ   )r   rO  r   r.  rÀ  r9   rÝ  )r   rH   s   "€r   r    Ú$LEDForQuestionAnswering.__annotate__%  sG  ø€ ÷ m
ñ m
á×#Ñ# dÕ*ðm
ñ Ÿ™ tÕ+ðm
ñ !×+Ñ+¨dÕ2ð	m
ñ
 !&× 0Ñ 0°4Õ 7ðm
ñ ™u¡U×%6Ñ%6Õ7Õ8¸4Õ?ðm
ñ  %×0Ñ0°4Õ7ðm
ñ ×)Ñ)¨DÕ0ðm
ñ ×'Ñ'¨$Õ.ðm
ñ ×(Ñ(¨4Õ/ðm
ñ  %×0Ñ0°4Õ7ðm
ñ ˜$•;ðm
ñ   $�;ðm
ñ # T�kðm
ñ ˜D•[ðm
ñ" 
‰u�|‰|Õ	ÑEÕ	Eñ#m
r!   c                ó  € Ve   TMV P                   P                  pVe   Ve   RpV P                  VVVVVVV	V
VVVVR7      pV^ ,          pV P                  V4      pVP	                  ^RR7      w  ppVP                  R4      P                  4       pVP                  R4      P                  4       pRpVeÆ   VeÂ   \        VP                  4       4      ^8”  d   VP                  R4      p\        VP                  4       4      ^8”  d   VP                  R4      pVP                  ^4      pVP                  ^ V4      pVP                  ^ V4      p\        VR7      pV! VV4      pV! VV4      pVV,           ^,          pV'       g#   VV3VR,          ,           pVe   V3V,           # T# \        VVVVP                  VP                  VP                  VP                  VP                   VP"                  VP$                  VP&                  R7      # )	rQ  NF)r¨   rK  rL  rù  rM  rþ  rN  r‡  r«   r  r  r‰   )Úignore_indexr%   )rÒ  rß  rà  rH  rÆ  rÇ  rÈ  rÉ  r…  rÊ  rË  r$   )r€   r  r¡  r  ÚsplitÚsqueezer¥   r§   r5   rs  r   rÝ  rH  rÆ  rÇ  rÈ  rÉ  r…  rÊ  rË  )rN   r   r¨   rK  rL  rM  rù  r‚  rƒ  rþ  rN  r‡  r«   r  r  r‰  r¶   Úsequence_outputrÓ  rß  rà  Ú
total_lossÚignored_indexrs  Ú
start_lossÚend_lossr*  s   &&&&&&&&&&&&&&&,           r   r[   ÚLEDForQuestionAnswering.forward$  s  € ð` &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆØÒ&¨=Ò+DØˆIà—(‘(ØØ)Ø/Ø#9Ø"7Ø+Ø'Ø"7ØØ/Ø!5Ø#ð ó 
ˆð " !�*ˆà—‘ Ó1ˆØ#)§<¡<°°r <Ó#:Ñ ˆ�jØ#×+Ñ+¨BÓ/×:Ñ:Ó<ˆØ×'Ñ'¨Ó+×6Ñ6Ó8ˆ
àˆ
ØÒ&¨=Ò+Dä�?×'Ñ'Ó)Ó*¨QÔ.Ø"1×"9Ñ"9¸"Ó"=�Ü�=×%Ñ%Ó'Ó(¨1Ô,Ø -× 5Ñ 5°bÓ 9�à(×-Ñ-¨aÓ0ˆMØ-×3Ñ3°A°}ÓEˆOØ)×/Ñ/°°=ÓAˆMä'°]ÔCˆHÙ! ,°Ó@ˆJÙ 
¨MÓ:ˆHØ$ xÕ/°1Õ4ˆJçàØðð ˜•õˆFð 0:Ò/E�Z�M FÕ*ÐQÈ6ÐQä5ØØ%Ø!Ø#×3Ñ3Ø")×"?Ñ"?Ø&×9Ñ9Ø$×5Ñ5Ø&-×&GÑ&GØ")×"?Ñ"?Ø&×9Ñ9Ø&-×&GÑ&Gô
ð 	
r!   )r¡  r~  r  ry  )
rb   rc   rd   re   rM   r   r[   rg   rh   ri   rj   s   @@r   r|  r|    s,   ù‡ € õ
ð ÷m
ñ m
ó ÷m
ð m
r!   r|  rK   )r­  r|  r<  r   )<rf   r•   Údataclassesr   r   r   Útorch.nnr   Ú r   r®  Úactivationsr   Úcache_utilsr   r   r	   Ú
generationr
   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   Úmodeling_utilsr   Úutilsr   r   r   Úconfiguration_ledr   Ú
get_loggerrb   r   r.   rA   rí  rC   ÚModulerl   r&  r8  r_  ry  r�  r   r»  rÄ  rÏ  rÙ  rÝ  rã  r(  r<  r­  r|  Ú__all__r_   r!   r   Ú<module>rž     s  ðñ ã Ý !ã Ý Ý %å &Ý !ß CÑ CÝ )ß JÝ 9Ý IÝ -ß 9Ñ 9Ý (ð 
×	Ò	˜HÓ	%€õ÷ #ô$* B§L¡Lô *ô$c	5˜bŸi™iô c	5ôL˜"Ÿ)™)ô ô@DC˜"Ÿ)™)ô DCôN53Ð0ô 53ôpdÐ0ô dôN˜BŸI™Iô ð0 ô2˜ó 2ó ð2ñ* ðôð
 ôC ó Có óðCñ@ ðôð ôK˜Kó Kó óðKñ@ ðôð
 ôK˜ó Kó óðKñ@ ðôð
 ôK¨ó Kó óðKñ@ ðôð
 ôK¨[ó Kó óðKô>C
Ð#ô C
ôLN
Ð#ô N
ðb ôF
Ð!ó F
ó ðF
ñR ðôô
zhÐ"4°oó zhóð
zhðz ô{
Ð0ó {
ó ð{
ò|‚r!   