+
    œ°³jâ  ã                   óò  € R t ^ RIHt ^ RIt^ RIt^ RIHt ^ RIHtH	t	H
t
 ^RIHt ^RIHt ^RIHtHtHtHtHtHt ^RIHt ^R	IHtHtHt ^R
IHt ]P>                  ! ] 4      t!Rt" ! R R]PF                  4      t$ ! R R]PF                  4      t%R R lt& ! R R]PF                  4      t' ! R R]PF                  4      t( ! R R]PF                  4      t) ! R R]PF                  4      t*R;R R llt+ ! R R]PF                  4      t, ! R R]PF                  4      t-] ! R  R!]4      4       t. ! R" R#]PF                  4      t/]! R$R%7      ] ! R& R']4      4       4       t0]! R(R%7       ! R) R*].4      4       t1] ! R+ R,].4      4       t2]! R-R%7       ! R. R/].4      4       t3] ! R0 R1].4      4       t4]! R2R%7       ! R3 R4].4      4       t5] ! R5 R6].4      4       t6] ! R7 R8].4      4       t7] ! R9 R:].4      4       t8. R<Ot9R# )=z!PyTorch Funnel Transformer model.)Ú	dataclassN)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELoss)Úinitialization)ÚACT2FN)ÚBaseModelOutputÚMaskedLMOutputÚMultipleChoiceModelOutputÚQuestionAnsweringModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚPreTrainedModel)ÚModelOutputÚauto_docstringÚlogging©ÚFunnelConfigg    €„.Ac                   óT   a a€ ] tR t^,t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚFunnelEmbeddingsc                ó$   <€ V ^8„  d   QhRS[ RR/# ©é   ÚconfigÚreturnNr   )ÚformatÚ__classdict__s   "€Úk/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/funnel/modeling_funnel.pyÚ__annotate__ÚFunnelEmbeddings.__annotate__-   s   ø€ ÷ 9ñ 9™|ð 9°ñ 9ó    c                óH  <€ \         SV `  4        \        P                  ! VP                  VP
                  VP                  R 7      V n        \        P                  ! VP                  VP                  R7      V n        \        P                  ! VP                  4      V n        R# ))Úpadding_idx©ÚepsN)ÚsuperÚ__init__r   Ú	EmbeddingÚ
vocab_sizeÚhidden_sizeÚpad_token_idÚword_embeddingsÚ	LayerNormÚd_modelÚlayer_norm_epsÚ
layer_normÚDropoutÚhidden_dropoutÚdropout©Úselfr   Ú	__class__s   &&€r   r'   ÚFunnelEmbeddings.__init__-   sh   ø€ Ü‰ÑÔÜ!Ÿ|š|¨F×,=Ñ,=¸v×?QÑ?QÐ_e×_rÑ_rÔsˆÔÜŸ,š, v§~¡~¸6×;PÑ;PÔQˆŒÜ—z’z &×"7Ñ"7Ó8ˆŽr!   c                ó„   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  /# )r   Ú	input_idsNÚinputs_embedsr   ©ÚtorchÚTensor)r   r   s   "€r   r   r    3   s<   ø€ ÷ ñ ÙŸ™¨Õ,ðÙDIÇLÁLÐSWÕDWðá	�‰ñr!   c                ót   € Vf   V P                  V4      pV P                  V4      pV P                  V4      pV# ©N)r,   r0   r3   )r5   r9   r:   Ú
embeddingss   &&& r   ÚforwardÚFunnelEmbeddings.forward3   s<   € ð Ò Ø ×0Ñ0°Ó;ˆMØ—_‘_ ]Ó3ˆ
Ø—\‘\ *Ó-ˆ
ØÐr!   )r3   r0   r,   ©NN©	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r'   rA   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r6   r   s   @@r   r   r   ,   s   ù‡ € ÷9ó 9÷÷ ò r!   r   c                   ó   a a€ ] tR t^=t oRt^tV3R lV 3R lltRV3R lR lltV3R lR ltV3R lR	 lt	V3R
 lR lt
RV3R lR lltV3R lR ltRV3R lR lltV3R lR ltV3R lR ltV3R ltRtVtV ;t# )ÚFunnelAttentionStructurez6
Contains helpers for `FunnelRelMultiheadAttention `.
c                ó$   <€ V ^8„  d   QhRS[ RR/# r   r   )r   r   s   "€r   r   Ú%FunnelAttentionStructure.__annotate__D   s   ø€ ÷ !ñ !™|ð !°ñ !r!   c                óÒ   <€ \         SV `  4        Wn        \        P                  ! VP
                  4      V n        \        P                  ! VP
                  4      V n        R V n        R # r?   )	r&   r'   r   r   r1   r2   Úsin_dropoutÚcos_dropoutÚpooling_multr4   s   &&€r   r'   Ú!FunnelAttentionStructure.__init__D   sK   ø€ Ü‰ÑÔØŒÜŸ:š: f×&;Ñ&;Ó<ˆÔÜŸ:š: f×&;Ñ&;Ó<ˆÔð !ˆÖr!   c          	      ó®   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[S[ P                  ,          /# )r   r:   Úattention_maskNÚtoken_type_idsr   )r<   r=   Útuple)r   r   s   "€r   r   rP   M   sX   ø€ ÷ Kñ Ká—|‘|ðKñ Ÿ™ tÕ+ðKñ Ÿ™ tÕ+ð	Kñ
 
‰u�|‰|Õ	ñKr!   c                ó|  € ^V n         VP                  ^4      ;V n        pV P                  WAP                  VP
                  4      pVe   V P                  V4      MRpV P                  P                  '       d@   \        P                  P                  VP                  V^,
          V^,
          .4      R4      MRpWVW'3# )zCReturns the attention inputs associated to the inputs of the model.N)é   é    r[   r\   )rT   ÚsizeÚseq_lenÚget_position_embedsÚdtypeÚdeviceÚtoken_type_ids_to_matr   Úseparate_clsr   Ú
functionalÚpadÚnew_ones)r5   r:   rW   rX   r^   Úposition_embedsÚtoken_type_matÚcls_masks   &&&&    r   Úinit_attention_inputsÚ.FunnelAttentionStructure.init_attention_inputsM   s°   € ð ˆÔØ!.×!3Ñ!3°AÓ!6Ð6ˆŒ�wØ×2Ñ2°7×<OÑ<OÐQ^×QeÑQeÓfˆØGUÒGa˜×3Ñ3°NÔCÐgkˆð �{‰{×'×'Ð'ô �M‰M×Ñ˜m×4Ñ4°gÀµkÀ7ÈQÅ;Ð5OÓPÐR^Ô_àð 	ð
  °ÐJÐJr!   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r   rX   r   r;   )r   r   s   "€r   r   rP   a   s#   ø€ ÷ (ñ (±E·L±Lð (ÁUÇ\Á\ñ (r!   c                ó‚   € VR,          VR,          8H  pWP                   8H  pVR,          VR,          ,          pWB,          # )z-Convert `token_type_ids` to `token_type_mat`.)ºNNNrn   N©rn   N)Úcls_token_type_id)r5   rX   rh   Úcls_idsÚcls_mats   &&   r   rb   Ú.FunnelAttentionStructure.token_type_ids_to_mata   sA   € à'¨
Õ3°~ÀgÕ7NÑNˆà ×$:Ñ$:Ñ:ˆØ˜*Õ%¨°Õ(8Õ8ˆØÕ'Ð'r!   c                óÂ   <€ V ^8„  d   QhRS[ RS[P                  RS[P                  RS[S[P
                  ,          S[S[S[P
                  ,          ,          ,          /# )r   r^   r`   ra   r   )Úintr<   r`   ra   rY   r=   Úlist)r   r   s   "€r   r   rP   i   sW   ø€ ÷ N(ñ N(ÙðN(Ù#(§;¡;ðN(Ù8=¿¹ðN(á	‰u�|‰|Õ	™t¡D©¯©Õ$6Õ7Õ	7ñN(r!   c                óL  € V P                   P                  pV P                   P                  R8X  Ede   \        P                  ! ^ VR\        P
                  VR7      P                  V4      p\        P                  ! ^ V^,          R\        P
                  VR7      P                  V4      p^RWd^,          ,          ,          ,          pVR,          VR,          ,          p\        P                  ! V4      p	V P                  V	4      p
\        P                  ! V4      pV P                  V4      p\        P                  ! Wª.R	R7      p\        P                  ! W¹.R	R7      p\        P                  ! WÌ.R	R7      p\        P                  ! V	) V.R	R7      pWßVV3# \        P                  ! ^ V^,          R\        P
                  VR7      P                  V4      p^RWd^,          ,          ,          ,          p\        P                  ! V) ^,          V^,          R\        P
                  VR7      P                  V4      pV^,          pVR,          VR,          ,          pV P                  \        P                  ! V4      4      p	V P                  \        P                  ! V4      4      p\        P                  ! W›.R	R7      p\        P                  ! ^ V\        P
                  VR7      P                  V4      pTp. p\        ^ V P                   P                  4       EF  pV^ 8X  d   RpM€V P                  VV4      p^V^,
          ,          pV P                  VVV^R7      pVR,          V,           pVP!                  VP#                  ^ 4      V4      p\        P$                  ! V^ V4      pTp^V,          pV P                  VV4      pVR,          V,           pVP!                  VP#                  ^ 4      V4      p\        P$                  ! V^ V4      pVP'                  VV.4       EK
  	  V# )
a×  
Create and cache inputs related to relative position encoding. Those are very different depending on whether we
are using the factorized or the relative shift attention:

For the factorized attention, it returns the matrices (phi, pi, psi, omega) used in the paper, appendix A.2.2,
final formula.

For the relative shift attention, it returns all possible vectors R used in the paper, appendix A.2.1, final
formula.

Paper link: https://huggingface.co/papers/2006.03236
Ú
factorizedç      ð?©r`   ra   i'  N©Údim)Úshiftro   éÿÿÿÿ)r   r.   Úattention_typer<   ÚarangeÚint64ÚtoÚsinrR   ÚcosrS   ÚcatÚrangeÚ
num_blocksÚstride_pool_posÚrelative_posÚexpandr]   ÚgatherÚappend)r5   r^   r`   ra   r.   Úpos_seqÚfreq_seqÚinv_freqÚsinusoidÚ	sin_embedÚsin_embed_dÚ	cos_embedÚcos_embed_dÚphiÚpsiÚpiÚomegaÚ
rel_pos_idÚzero_offsetÚ	pos_embedÚposÚ
pooled_posÚposition_embeds_listÚblock_indexÚposition_embeds_poolingÚstrideÚrel_posÚposition_embeds_no_poolings   &&&&                        r   r_   Ú,FunnelAttentionStructure.get_position_embedsi   sX  € ð —+‘+×%Ñ%ˆØ�;‰;×%Ñ%¨Õ5ô —l’l 1 g¨s¼%¿+¹+ÈfÔU×XÑXÐY^Ó_ˆGÜ—|’| A w°!¥|°SÄÇÁÐTZÔ[×^Ñ^Ð_dÓeˆHØ˜E h¸Qµ,Õ&?Õ@ÕAˆHØ˜wÕ'¨(°4­.Õ8ˆHÜŸ	š	 (Ó+ˆIØ×*Ñ*¨9Ó5ˆKÜŸ	š	 (Ó+ˆIØ×*Ñ*¨9Ó5ˆKä—)’)˜[Ð6¸BÔ?ˆCÜ—)’)˜YÐ2¸Ô;ˆCÜ—’˜KÐ5¸2Ô>ˆBÜ—I’I 	˜z¨9Ð5¸2Ô>ˆEØ˜S %Ð(Ð(ô —|’| A w°!¥|°SÄÇÁÐTZÔ[×^Ñ^Ð_dÓeˆHØ˜E h¸Qµ,Õ&?Õ@ÕAˆHäŸš w h°¥l°G¸aµKÀÌEÏKÉKÐ`fÔg×jÑjÐkpÓqˆJØ! A�+ˆKØ! 'Õ*¨X°d­^Õ;ˆHØ×(Ñ(¬¯ª°8Ó)<Ó=ˆIØ×(Ñ(¬¯ª°8Ó)<Ó=ˆIÜŸ	š	 9Ð"8¸bÔAˆIä—,’,˜q '´·±ÀVÔL×OÑOÐPUÓVˆCØˆJØ#%Ð Ü$ Q¨¯©×(>Ñ(>×?�ð  !Ô#Ø.2Ñ+à!%×!5Ñ!5°c¸;Ó!G�Jð  ;°¥?Õ3�FØ"×/Ñ/°°V¸ZÈqÐ/ÓQ�GØ% gÕ.°Õ<�GØ%Ÿn™n¨W¯\©\¸!«_¸gÓF�GÜ.3¯lªl¸9ÀaÈÓ.QÐ+ð !�Ø˜K��Ø×+Ñ+¨C°Ó8�à! 'Õ*¨[Õ8�Ø!Ÿ.™.¨¯©°a«¸'ÓB�Ü-2¯\ª\¸)ÀQÈÓ-PÐ*à$×+Ñ+Ð-GÐI`Ð,a×bñ9  @ð: (Ð'r!   c                ó:   <€ V ^8„  d   QhRS[ P                  RS[/# )r   Úpos_idrŸ   ©r<   r=   ru   )r   r   s   "€r   r   rP   ¹   s   ø€ ÷ ñ ¡e§l¡lð Áñ r!   c                ó   € V P                   P                  '       dk   VP                  ^V,          ) ^,           .4      pV P                   P                  '       d   V^R MVR,          p\        P
                  ! W4R,          .^ 4      # VR,          # )zU
Pool `pos_id` while keeping the cls token separate (if `config.separate_cls=True`).
ºr[   NN:NNr   r~   )r   rc   Ú
new_tensorÚtruncate_seqr<   r…   )r5   r¦   rŸ   Úcls_posÚpooled_pos_ids   &&&  r   rˆ   Ú(FunnelAttentionStructure.stride_pool_pos¹   sy   € ð �;‰;×#×#Ð#ð
 ×'Ñ'¨1¨k­>Ð):¸QÕ)>Ð(?Ó@ˆGØ,0¯K©K×,D×,DÐ,D˜F 1 R™LÈ&ÐQSÍ*ˆMÜ—9’9˜g°SÕ'9Ð:¸AÓ>Ð>à˜#•;Ðr!   c                óZ   <€ V ^8„  d   QhRS[ P                  RS[RS[RS[ P                  /# )r   rœ   r¡   r}   r   r§   )r   r   s   "€r   r   rP   È   s;   ø€ ÷ bñ b¡§¡ð b±cð bÑSVð bÑ_d×_kÑ_kñ br!   c                ó.  € Vf   TpV^ ,          V^ ,          ,
          pWCP                   ^ ,          ,          pWVV,          ,           pV^ ,          VR,          ,
          p\        P                  ! Wx^,
          V) \        P                  VP                  R7      # )zF
Build the relative positional vector between `pos` and `pooled_pos`.
rz   r~   )Úshaper<   r€   Úlongra   )	r5   rœ   r¡   r�   r}   Ú	ref_pointÚ
num_removeÚmax_distÚmin_dists	   &&&&&    r   r‰   Ú%FunnelAttentionStructure.relative_posÈ   sy   € ð ÒØˆJà˜q•M C¨¥FÕ*ˆ	Ø×-Ñ-¨aÕ0Õ0ˆ
Ø¨FÕ2Õ2ˆØ˜a•= 3 r¥7Õ*ˆä�|Š|˜H°¥l°V°GÄ5Ç:Á:ÐVY×V`ÑV`ÔaÐar!   c                óü   <€ V ^8„  d   QhRS[ P                  S[S[ P                  ,          ,          S[S[ P                  ,          ,          RS[S[S[,          ,          S[S[,          ,          RS[ P                  /# )r   ÚtensorÚaxisr   )r<   r=   rY   rv   ru   )r   r   s   "€r   r   rP   Ö   s_   ø€ ÷ !ñ !á—‘™u¡U§\¡\Õ2Õ2±T¹%¿,¹,Õ5GÕGð!ñ ‘E™#•JÕ¡¡c¥Õ*ð!ñ 
�‰ñ	!r!   c                óè  a a€ Vf   R# \        S\        \        34      '       d   S F  pS P                  W4      pK  	  V# \        V\        \        34      '       d   \	        V4      ! VV 3R lV 4       4      # SVP
                  ,          oS P                  P                  '       d*   S P                  P                  '       d   \        RR^4      M\        RR^4      p\        \        R4      .S,          V.,           4      pS P                  P                  '       dO   \        \        R4      .S,          \        R^4      .,           4      p\        P                  ! W,          V.SR7      pW,          # )zD
Perform pooling by stride slicing the tensor along the given axis.
Nc              3   óH   <"  € T F  pSP                  VS4      x € K  	  R # 5ir?   )Ústride_pool)Ú.0Úxrº   r5   s   & €€r   Ú	<genexpr>Ú7FunnelAttentionStructure.stride_pool.<locals>.<genexpr>é   s!   øé € ÐJÁ6¸a × 0Ñ 0°°D× 9Ð 9Ã6ùs   ƒ")rº   r~   )Ú
isinstancerv   rY   r½   ÚtypeÚndimr   rc   r«   Úslicer<   r…   )r5   r¹   rº   ÚaxÚ
axis_sliceÚ	enc_sliceÚ	cls_slices   f&f    r   r½   Ú$FunnelAttentionStructure.stride_poolÖ   s)  ù€ ð Š>Ùô �dœT¤5˜M×*Ò*Û�Ø×)Ñ)¨&Ó5’ñ àˆMô �fœu¤d˜m×,Ò,Ü˜”<ÕJÁ6ÓJÓJÐJð 	�—‘Õˆð #'§+¡+×":×":Ð":¸t¿{¹{×?W×?WÐ?WŒE�$˜˜AÔÔ]bÐcgÐimÐopÓ]qð 	ô œ5 ›;˜-¨$Õ.°*°Õ=Ó>ˆ	Ø�;‰;×#×#Ð#Üœu T›{˜m¨dÕ2´e¸DÀ!³nÐ5EÕEÓFˆIÜ—Y’Y Õ 1°6Ð:ÀÔFˆFØÕ Ð r!   c                óÂ   <€ V ^8„  d   QhRS[ P                  S[S[ P                  ,          ,          S[S[ P                  ,          ,          RS[RS[RS[ P                  /# )r   r¹   Úmoder¡   r   )r<   r=   rY   rv   Ústrru   )r   r   s   "€r   r   rP   ÷   sQ   ø€ ÷ $ñ $Ù—l‘l¡U©5¯<©<Õ%8Õ8¹4ÁÇÁÕ;MÕMð$ÙUXð$Ùknð$á	�‰ñ$r!   c                ó  a aaa€ Sf   R# \        S\        \        34      '       d   \        S4      ! VV VV3R lS 4       4      # S P                  P
                  '       dM   S P                  P                  '       d   SRRR13,          MSp\        P                  ! SR,          V.^R7      oSP                  pV^8X  d   SR,          oMV^8X  d
   SR,          oS^3oSR8X  d%   \        P                  P                  SSSRR7      oMcSR8X  d%   \        P                  P                  SSSRR7      oM8SR	8X  d'   \        P                  P                  S) SSRR7      ) oM\        R
4      hV^8X  d
   SR,          # V^8X  d
   SR,          # S# )z3Apply 1D pooling to a tensor of size [B x T (x H)].Nc              3   óL   <"  € T F  pSP                  SSSR 7      x € K  	  R# 5i))rÌ   r¡   N)Úpool_tensor)r¾   r¿   rÌ   r5   r¡   r¹   s   & €€€€r   rÀ   Ú7FunnelAttentionStructure.pool_tensor.<locals>.<genexpr>   s)   øé € ÐcÑ\bÐWX × 0Ñ 0°¸dÈ6Ð 0× RÐ RÓ\bùs   ƒ!$rn   r{   ÚmeanT)r¡   Ú	ceil_modeÚmaxÚminz0The supported modes are 'mean', 'max' and 'min'.r~   ©rn   :Nr[   N)rn   Nrn   N)rn   Nrn   rn   )rn   r\   rn   r\   ©rn   r\   )rÂ   rY   rv   rÃ   r   rc   r«   r<   r…   rÄ   r   rd   Ú
avg_pool2dÚ
max_pool2dÚNotImplementedError)r5   r¹   rÌ   r¡   ÚsuffixrÄ   s   ffff  r   rÐ   Ú$FunnelAttentionStructure.pool_tensor÷   s^  û€ ð Š>Ùô �fœu¤d˜m×,Ò,Ü˜”<×cÑ\bÓcÓcÐcà�;‰;×#×#Ð#Ø'+§{¡{×'?×'?Ð'?�V˜A˜s ˜s˜F–^ÀVˆFÜ—Y’Y  u¥¨vÐ6¸AÔ>ˆFà�{‰{ˆØ�1Œ9ØÐ,Õ-‰FØ�QŒYØ˜MÕ*ˆFà˜!�ˆà�6Œ>Ü—]‘]×-Ñ-¨f°fÀVÐW[Ð-Ó\‰FØ�UŒ]Ü—]‘]×-Ñ-¨f°fÀVÐW[Ð-Ó\‰FØ�UŒ]Ü—m‘m×.Ñ.°¨w¸ÀvÐY]Ð.Ó^Ð^‰Fä%Ð&XÓYÐYà�1Œ9Ø˜*Õ%Ð%Ø�QŒYØ˜$•<ÐØˆr!   c                ó˜   <€ V ^8„  d   QhRS[ S[P                  ,          RS[ S[P                  S[ S[P                  ,          3,          /# ©r   Úattention_inputsr   ©rY   r<   r=   )r   r   s   "€r   r   rP     s=   ø€ ÷ (ñ (Ù(-©e¯l©lÕ(;ð(á	‰u�|‰|™U¡5§<¡<Õ0Ð0Õ	1ñ(r!   c                óº  € Vw  r4rVV P                   P                  '       dŽ   V P                   P                  R8X  d(   V P                  VR,          ^ 4      VR,          ,           pV P                  V^4      pV P                  V^ 4      pV P	                  WP                   P
                  R7      pM§V ;P                  ^,          un        V P                   P                  R8X  d   V P                  V^ 4      pV P                  V^^.4      pV P                  V^^.4      pV P	                  VRR7      pV P	                  WP                   P
                  R7      pW4WV3pW3# )zTPool `output` and the proper parts of `attention_inputs` before the attention layer.rx   ºNr   Nºr   NN©rÌ   rÕ   )r   Úpool_q_onlyr   r½   rÐ   Úpooling_typerT   )r5   Úoutputrß   rg   rh   rW   ri   s   &&&    r   Úpre_attention_poolingÚ.FunnelAttentionStructure.pre_attention_pooling  s<  € ð EUÑAˆ¨Ø�;‰;×"×"Ð"Ø�{‰{×)Ñ)¨\Ô9Ø"&×"2Ñ"2°?À2Õ3FÈÓ"JÈ_Ð]_ÕM`Õ"`�Ø!×-Ñ-¨n¸aÓ@ˆNØ×'Ñ'¨°!Ó4ˆHØ×%Ñ% f·;±;×3KÑ3KÐ%ÓL‰Fà×Ò Õ"ÕØ�{‰{×)Ñ)¨\Ô9Ø"&×"2Ñ"2°?ÀAÓ"F�Ø!×-Ñ-¨n¸qÀ!¸fÓEˆNØ×'Ñ'¨°1°a°&Ó9ˆHØ!×-Ñ-¨nÀ5Ð-ÓIˆNØ×%Ñ% f·;±;×3KÑ3KÐ%ÓLˆFØ+¸^ÐVÐØÐ'Ð'r!   c                ón   <€ V ^8„  d   QhRS[ S[P                  ,          RS[ S[P                  ,          /# rÞ   rà   )r   r   s   "€r   r   rP   3  s0   ø€ ÷  ñ  ±u¹U¿\¹\Õ7Jð  ÉuÑUZ×UaÑUaÕObñ  r!   c                ót  € Vw  r#rEV P                   P                  '       d“   V ;P                  ^,          un        V P                   P                  R8X  d(   VR,          V P	                  VR,          ^ 4      ,           pV P	                  V^4      pV P	                  V^4      pV P                  VRR7      pW#WE3pV# )zFPool the proper parts of `attention_inputs` after the attention layer.rx   râ   rã   rÕ   rä   )r   rå   rT   r   r½   rÐ   )r5   rß   rg   rh   rW   ri   s   &&    r   Úpost_attention_poolingÚ/FunnelAttentionStructure.post_attention_pooling3  s¬   € àDTÑAˆ¨Ø�;‰;×"×"Ð"Ø×Ò Õ"ÕØ�{‰{×)Ñ)¨\Ô9Ø"1°"Õ"5¸×8HÑ8HÈÐY[ÕI\Ð^_Ó8`Õ"`�Ø!×-Ñ-¨n¸aÓ@ˆNØ×'Ñ'¨°!Ó4ˆHØ!×-Ñ-¨nÀ5Ð-ÓIˆNØ+¸^ÐVÐØÐr!   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # )r   rp   )ru   )r   r   s   "€r   r   rP   =   s   ø‡ ‚ ñ
 Ñò r!   )r   rS   rT   r^   rR   rC   )Nr[   )rÒ   r   )rE   rF   rG   rH   Ú__doc__rp   r'   rj   rb   r_   rˆ   r‰   r½   rÐ   rè   rì   Ú__annotate_func__rI   rJ   rK   rL   s   @@r   rN   rN   =   s‡   ù‡ € ñð Ð÷!ó !÷Kò K÷((ð (÷N(ð N(÷`ð ÷bò b÷!ð !÷B$ò $÷L(ð (÷, ð  ÷m … r!   rN   c                óp   € V ^8„  d   QhR\         P                  R\        R\        R\         P                  /# )r   Úpositional_attnÚcontext_lenr}   r   r§   )r   s   "r   r   r   A  s4   € ÷ ñ ¬E¯L©Lð Äsð ÔSVð Ô[`×[gÑ[gñ r!   c                 óÌ   € V P                   w  r4rV\        P                  ! WWFV.4      p V R R VR1R 3,          p \        P                  ! WWEWb,
          .4      p V RRV13,          p V # )rn   N.)r±   r<   Úreshape)rò   ró   r}   Ú
batch_sizeÚn_headr^   Úmax_rel_lens   &&&    r   Ú_relative_shift_gatherrù   A  sp   € Ø/>×/DÑ/DÑ,€J˜ô —m’m OÀ&ÐW^Ð5_Ó`€OØ% a¨¨E©F°A oÕ6€OÜ—m’m OÀ&ÐS^ÕSfÐ5gÓh€OØ% c¨<¨K¨<Ð&7Õ8€OØÐr!   c                   óh   a a€ ] tR tRt oV3R lV 3R lltR	R ltR	R ltR
V3R lR lltRtVt	V ;t
# )ÚFunnelRelMultiheadAttentioniQ  c                ó*   <€ V ^8„  d   QhRS[ RS[RR/# ©r   r   rŸ   r   N©r   ru   )r   r   s   "€r   r   Ú(FunnelRelMultiheadAttention.__annotate__R  s"   ø€ ÷ )ñ )™|ð )¹#ð )À$ñ )r!   c                ó„  <€ \         SV `  4        Wn        W n        VP                  VP
                  VP                  rTp\        P                  ! VP                  4      V n	        \        P                  ! VP                  4      V n
        \        P                  ! W4V,          R R7      V n        \        P                  ! W4V,          4      V n        \        P                  ! W4V,          4      V n        \        P                  ! \         P"                  ! WE.4      4      V n        \        P                  ! \         P"                  ! WE.4      4      V n        \        P                  ! \         P"                  ! W4V.4      4      V n        \        P                  ! \         P"                  ! WE.4      4      V n        \        P                  ! \         P"                  ! ^WE.4      4      V n        \        P                  ! WE,          V4      V n        \        P0                  ! W1P2                  R7      V n        RVR,          ,          V n        R# )F)Úbiasr$   ry   g      à?N)r&   r'   r   rŸ   r.   r÷   Úd_headr   r1   r2   Úattention_dropoutÚLinearÚq_headÚk_headÚv_headÚ	Parameterr<   ÚzerosÚr_w_biasÚr_r_biasÚr_kernelÚr_s_biasÚ	seg_embedÚ	post_projr-   r/   r0   Úscale)r5   r   rŸ   r.   r÷   r  r6   s   &&&   €r   r'   Ú$FunnelRelMultiheadAttention.__init__R  sf  ø€ Ü‰ÑÔØŒØ&ÔØ"(§.¡.°&·-±-ÀÇÁ˜ˆä Ÿjšj¨×)>Ñ)>Ó?ˆÔÜ!#§¢¨F×,DÑ,DÓ!EˆÔä—i’i °&­¸uÔEˆŒÜ—i’i °&­Ó9ˆŒÜ—i’i °&­Ó9ˆŒäŸš¤U§[¢[°&Ð1AÓ%BÓCˆŒÜŸš¤U§[¢[°&Ð1AÓ%BÓCˆŒÜŸš¤U§[¢[°'À6Ð1JÓ%KÓLˆŒÜŸš¤U§[¢[°&Ð1AÓ%BÓCˆŒÜŸš¤e§k¢k°1°fÐ2EÓ&FÓGˆŒäŸš 6¥?°GÓ<ˆŒÜŸ,š, w×4IÑ4IÔJˆŒØ˜F C�KÕ(ˆŽ
r!   c                óà  € V P                   P                  R8X  dŸ   Vw  rVrxV P                  V P                  ,          p	V P                  p
\
        P                  ! RW),           V
4      pWµR,          ,          pW¶R,          ,          p\
        P                  ! RWÇ4      \
        P                  ! RWØ4      ,           pM¨VP                  ^,          V8w  d   ^M^pWP                  ,          V^,
          ,          pV P                  V P                  ,          pV P                  p
\
        P                  ! RVV
4      p\
        P                  ! RVV,           V4      p\        WãV4      pVe	   Wä,          pV# )z5Relative attention score for the positional encodingsrx   zbinh,dnh->bindzbind,jd->bnijztd,dnh->tnhzbinh,tnh->bnitro   )
r   r   r  r  r  r<   Úeinsumr±   rŸ   rù   )r5   rg   r  ró   ri   r•   r—   r–   r˜   ÚuÚw_rÚq_r_attentionÚq_r_attention_1Úq_r_attention_2rò   r}   ÚrÚvÚr_heads   &&&&&              r   Úrelative_positional_attentionÚ9FunnelRelMultiheadAttention.relative_positional_attentioni  s/  € ð �;‰;×%Ñ%¨Ô5ð #2ÑˆC�Sà—‘ §
¡
Õ*ˆAà—-‘-ˆCô "ŸLšLÐ)9¸6½:ÀsÓKˆMØ+°'­lÕ:ˆOØ+°­kÕ9ˆOô $Ÿlšl¨?¸OÓQÔTY×T`ÒT`Ø óUõ ‰Oð  Ÿ™ a�¨KÔ7‘A¸QˆEð  × 0Ñ 0Õ1°%¸!µ)Õ<ˆAà—‘ §
¡
Õ*ˆAà—-‘-ˆCô —\’\ -°°CÓ8ˆFä#ŸlšlÐ+;¸VÀa½ZÈÓPˆOä4°_ÐSXÓYˆOàÒØÕ'ˆOØÐr!   c                óð  € Vf   ^ # VP                   w  rEpV P                  V P                  ,          p\        P                  ! RW',           V P
                  4      pVR,          P                  WBP                   ^,          WV.4      p\        P                  ! V^RR7      w  rš\        P                  ! WP                  VP                   4      V	P                  VP                   4      4      pVe	   W³,          pV# )z/Relative attention score for the token_type_idszbind,snd->bnisr{   ro   r~   )	r±   r  r  r<   r  r  rŠ   ÚsplitÚwhere)r5   rh   r  ri   rö   r^   ró   r  Útoken_type_biasÚdiff_token_typeÚsame_token_typeÚtoken_type_attns   &&&&        r   Úrelative_token_type_attentionÚ9FunnelRelMultiheadAttention.relative_token_type_attention“  sÐ   € àÒ!ÙØ+9×+?Ñ+?Ñ(ˆ
˜[ð —=‘= 4§:¡:Õ-ˆô  Ÿ,š,Ð'7¸Õ9JÈDÏNÉNÓ[ˆà'¨Õ0×7Ñ7¸Ç\Á\ÐRSÅ_ÐV]Ð8kÓlˆä+0¯;ª;°ÈÈrÔ+RÑ(ˆäŸ+š+Ø×2Ñ2°>×3GÑ3GÓHÈ/×J`ÑJ`Ðao×auÑauÓJvó
ˆð ÒØÕ'ˆOØÐr!   c                óÆ   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[S[ P                  ,          RS[RS[S[ P                  R3,          /# )r   ÚqueryÚkeyÚvaluerß   Úoutput_attentionsr   .)r<   r=   rY   Úbool)r   r   s   "€r   r   rÿ   «  sr   ø€ ÷ 3Gñ 3Gá�|‰|ð3Gñ �\‰\ð3Gñ �|‰|ð	3Gñ
  ¡§¡Õ-ð3Gñ  ð3Gñ 
‰u�|‰|˜SÐ Õ	!ñ3Gr!   c                óX  € Vw  rgr‰VP                   w  r«pVP                   ^,          pV P                  P                  V P                  P                  rþV P	                  V4      P                  W«Wï4      pV P                  V4      P                  W­Wï4      pV P                  V4      P                  W­Wï4      pVV P                  ,          pV P                  V P                  ,          p\        P                  ! RVV,           V4      pV P                  VVWÙ4      pV P                  VVV	4      pVV,           V,           pVP                  pVP                  4       pVe1   V\         ^VR,          P                  4       ,
          ,          ,
          p\        P"                  ! VRVR7      pV P%                  V4      p\        P                  ! RVV4      pV P'                  VP)                  W«Wï,          4      4      pV P+                  V4      pV P-                  VV,           4      pV'       d   VV3# V3# )r[   zbind,bjnd->bnij)r|   r`   zbnij,bjnd->bind)rn   NNr~   )r±   r   r÷   r  r  Úviewr  r  r  r
  r<   r  r  r%  r`   ÚfloatÚINFÚsoftmaxr  r  rõ   r2   r0   )r5   r(  r)  r*  rß   r+  rg   rh   rW   ri   rö   r^   Ú_ró   r÷   r  r  r  r  r
  Úcontent_scorerò   r$  Ú
attn_scorer`   Ú	attn_probÚattn_vecÚattn_outrç   s   &&&&&&                       r   rA   Ú#FunnelRelMultiheadAttention.forward«  sÜ  € ð EUÑAˆ¨à!&§¡Ñˆ
˜QØ—i‘i •lˆØŸ™×+Ñ+¨T¯[©[×-?Ñ-?�ð —‘˜UÓ#×(Ñ(¨¸fÓMˆà—‘˜SÓ!×&Ñ& zÀÓOˆØ—‘˜UÓ#×(Ñ(¨À&ÓQˆà˜$Ÿ*™*Õ$ˆà—=‘= 4§:¡:Õ-ˆäŸšÐ%6¸ÀÕ8IÈ6ÓRˆØ×<Ñ<¸_ÈfÐVaÓlˆØ×<Ñ<¸^ÈVÐU]Ó^ˆð # _Õ4°ÕFˆ
ð × Ñ ˆØ×%Ñ%Ó'ˆ
àÒ%Ø#¤c¨Q°ÀÕ1N×1TÑ1TÓ1VÕ-VÕ&WÕWˆJä—M’M *°"¸EÔBˆ	Ø×*Ñ*¨9Ó5ˆ	ô —<’<Ð 1°9¸fÓEˆð —>‘> (×"2Ñ"2°:ÈÍÓ"XÓYˆØ×&Ñ& xÓ0ˆà—‘ ¨Õ!1Ó2ˆß&7�˜	Ð"ÐF¸f¸YÐFr!   )r  rŸ   r   r2   r  r0   r  r  r  r  r  r
  r  r  r  r?   ©F)rE   rF   rG   rH   r'   r  r%  rA   rI   rJ   rK   rL   s   @@r   rû   rû   Q  s,   ù‡ € ÷)ó )ô.(ôT÷03G÷ 3Gò 3Gr!   rû   c                   óP   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚFunnelPositionwiseFFNiá  c                ó$   <€ V ^8„  d   QhRS[ RR/# r   r   )r   r   s   "€r   r   Ú"FunnelPositionwiseFFN.__annotate__â  s   ø€ ÷ Nñ N™|ð N°ñ Nr!   c                ó  <€ \         SV `  4        \        P                  ! VP                  VP
                  4      V n        \        VP                  ,          V n	        \        P                  ! VP                  4      V n        \        P                  ! VP
                  VP                  4      V n        \        P                  ! VP                  4      V n        \        P                  ! VP                  VP                   4      V n        R # r?   )r&   r'   r   r  r.   Úd_innerÚlinear_1r   Ú
hidden_actÚactivation_functionr1   Úactivation_dropoutÚlinear_2r2   r3   r-   r/   r0   r4   s   &&€r   r'   ÚFunnelPositionwiseFFN.__init__â  sž   ø€ Ü‰ÑÔÜŸ	š	 &§.¡.°&·.±.ÓAˆŒÜ#)¨&×*;Ñ*;Õ#<ˆÔ Ü"$§*¢*¨V×-FÑ-FÓ"GˆÔÜŸ	š	 &§.¡.°&·.±.ÓAˆŒÜ—z’z &×"7Ñ"7Ó8ˆŒÜŸ,š, v§~¡~°v×7LÑ7LÓMˆŽr!   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r   Úhiddenr   r;   )r   r   s   "€r   r   r=  ë  s#   ø€ ÷ +ñ +™eŸl™lð +©u¯|©|ñ +r!   c                óÚ   € V P                  V4      pV P                  V4      pV P                  V4      pV P                  V4      pV P	                  V4      pV P                  W,           4      # r?   )r@  rB  rC  rD  r3   r0   )r5   rH  Úhs   && r   rA   ÚFunnelPositionwiseFFN.forwardë  s\   € Ø�M‰M˜&Ó!ˆØ×$Ñ$ QÓ'ˆØ×#Ñ# AÓ&ˆØ�M‰M˜!ÓˆØ�L‰L˜‹OˆØ�‰˜v�zÓ*Ð*r!   )rC  rB  r3   r0   r@  rD  rD   rL   s   @@r   r;  r;  á  s    ù‡ € ÷Nó N÷+÷ +ð +r!   r;  c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚFunnelLayeriô  c                ó*   <€ V ^8„  d   QhRS[ RS[RR/# rý   rþ   )r   r   s   "€r   r   ÚFunnelLayer.__annotate__õ  s"   ø€ ÷ 1ñ 1™|ð 1¹#ð 1À$ñ 1r!   c                ód   <€ \         SV `  4        \        W4      V n        \	        V4      V n        R # r?   )r&   r'   rû   Ú	attentionr;  Úffn)r5   r   rŸ   r6   s   &&&€r   r'   ÚFunnelLayer.__init__õ  s&   ø€ Ü‰ÑÔÜ4°VÓIˆŒÜ(¨Ó0ˆŽr!   c          
      ót   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[RS[/# )r   r(  r)  r*  r+  r   )r<   r=   r,  rY   )r   r   s   "€r   r   rO  ú  sP   ø€ ÷ 
Eñ 
Eá�|‰|ð
Eñ �\‰\ð
Eñ �|‰|ð	
Eñ  ð
Eñ 
ñ
Er!   c                ó„   € V P                  WW4VR 7      pV P                  V^ ,          4      pV'       d   Wv^,          3# V3# )©r+  ©rQ  rR  )r5   r(  r)  r*  rß   r+  Úattnrç   s   &&&&&&  r   rA   ÚFunnelLayer.forwardú  sA   € ð �~‰~˜e¨%ÐUfˆ~ÓgˆØ—‘˜$˜q�'Ó"ˆß$5�˜Q�Ð ÐD¸F¸9ÐDr!   rW  r9  rD   rL   s   @@r   rM  rM  ô  s!   ù‡ € ÷1ó 1÷

E÷ 
Eò 
Er!   rM  c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚFunnelEncoderi  c                ó$   <€ V ^8„  d   QhRS[ RR/# r   r   )r   r   s   "€r   r   ÚFunnelEncoder.__annotate__  s   ø€ ÷ 	
ñ 	
™|ð 	
°ñ 	
r!   c                ó\  <€ \         SV `  4        Wn        \        V4      V n        \
        P                  ! \        VP                  4       UUUu. uF=  w  r#\
        P                  ! \        V4       Uu. uF  p\        W4      NK  	  up4      NK?  	  uppp4      V n        R # u upi u upppi r?   )r&   r'   r   rN   Úattention_structurer   Ú
ModuleListÚ	enumerateÚblock_sizesr†   rM  Úblocks)r5   r   rŸ   Ú
block_sizer2  r6   s   &&   €r   r'   ÚFunnelEncoder.__init__  s‡   ø€ Ü‰ÑÔØŒÜ#;¸FÓ#CˆÔ Ü—m’mô 09¸×9KÑ9KÔ/Lõá/LÑ+�Kô —’ÌÈzÔIZÓ[ÑIZÀAœ{¨6Ö?ÑIZÑ[Ö\Ù/Lóó
ˆŽùâ[ùôs   Á$B'Á3B"ÂB'Â"B'c                ó¬   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[RS[RS[RS[S[,          /# )	r   r:   rW   NrX   r+  Úoutput_hidden_statesÚreturn_dictr   ©r<   r=   r,  rY   r	   )r   r   s   "€r   r   r]    su   ø€ ÷ 0uñ 0uá—|‘|ð0uñ Ÿ™ tÕ+ð0uñ Ÿ™ tÕ+ð	0uñ
  ð0uñ #ð0uñ ð0uñ 
‘Õ	 ñ0ur!   c                ó  € VP                  V4      pV P                  P                  VVVR 7      pTpV'       d   V3MRp	V'       d   RMRp
\        V P                  4       EF`  w  r¼VP                  ^4      V P                  P                  '       d   ^M^8„  pT;'       d    V^ 8„  pV'       d   V P                  P                  W‡4      w  rç\        V4       Få  w  pp\        V P                  P                  V,          4       Fµ  pV^ 8H  ;'       d    V^ 8H  ;'       d    TpV'       d&   XpV P                  P                  '       d   TMT;ppMT;p;ppV! VVVWtR7      pV^ ,          pV'       d   V P                  P                  V4      pV'       d   V
VR,          ,           p
V'       g   K¬  W˜3,           p	K·  	  Kç  	  EKc  	  V'       g1   \        ;QJ d    . R W‰V
3 4       F  NK  	  5# ! R W‰V
3 4       4      # \        W‰V
R7      # )©rW   rX   NrV  r©   c              3   ó0   "  € T F  qf   K  Vx € K  	  R # 5ir?   © ©r¾   r  s   & r   rÀ   Ú(FunnelEncoder.forward.<locals>.<genexpr>B  ó   é € ÐaÑ$O˜qŸšÓ$Oùó   ‚Œ
©Úlast_hidden_stateÚhidden_statesÚ
attentionsrm  )Útype_asr_  rj   ra  rc  r]   r   rc   rè   r†   Úblock_repeatsrå   rì   rY   r	   )r5   r:   rW   rX   r+  rg  rh  rß   rH  Úall_hidden_statesÚall_attentionsrŸ   ÚblockÚpooling_flagÚpooled_hiddenÚlayer_indexÚlayerÚrepeat_indexÚ
do_poolingr(  r)  r*  Úlayer_outputs   &&&&&&&                r   rA   ÚFunnelEncoder.forward  sß  € ð (×/Ñ/°Ó>ˆØ×3Ñ3×IÑIØØ)Ø)ð Jó 
Ðð
 ˆç0D˜]Ñ,È$Ðß0™°dˆä"+¨D¯K©K×"8ÑˆKØ!Ÿ;™; q›>°$·+±+×2J×2JÐ2J©QÐPQÑRˆLØ'×;Ð;¨K¸!©OˆLßØ26×2JÑ2J×2`Ñ2`Øó3Ñ/�ô '0°Ö&6Ñ"�˜UÜ$)¨$¯+©+×*CÑ*CÀKÕ*PÖ$Q�LØ".°!Ñ"3×!\Ð!\¸+ÈÑ:J×!\Ð!\ÐP\�Jß!Ø -˜Ø04·±×0G×0GÐ0G¡fÈ]ÐZ˜™eà.4Ð4˜Ð4  eÙ#(¨°°UÐ<LÔ#r�LØ)¨!�_�Fß!Ø+/×+CÑ+C×+ZÑ+ZÐ[kÓ+lÐ(ç(Ø)7¸,ÀrÕ:JÕ)J˜ß+Ñ+Ø,=À	Õ,IÒ)ó %Rô '7ñ #9÷2 ß”5Ña VÀÑ$OÓa—5Ða�5Ña VÀÑ$OÓaÓaÐaÜ°ÐesÔtÐtr!   )r_  rc  r   ©NNFFTrD   rL   s   @@r   r[  r[    s!   ù‡ € ÷	
ó 	
÷0u÷ 0uò 0ur!   r[  c                óˆ   € V ^8„  d   QhR\         P                  R\        R\        R\        R\        R\         P                  /# )r   r¿   r¡   Ú
target_lenrc   r«   r   )r<   r=   ru   r,  )r   s   "r   r   r   F  sD   € ÷ ñ Ü‡|�|ðÜ ðÜ.1ðÜAEðÜ\`ðä
‡\�\ñr!   c           	     óp  € V^8X  d   V # V'       d   V R,          pV R,          p \         P                  ! W^R7      pV'       df   V'       d.   \        P                  P	                  V^ ^ ^ V^,
          ^ ^ 34      pVRRV^,
          13,          p\         P
                  ! XV.^R7      pV# VRRV13,          pV# )zs
Upsample tensor `x` to match `target_len` by repeating the tokens `stride` time on the sequence length dimension.
rn   )Úrepeatsr|   Nr{   rÖ   )rn   r©   )r<   Úrepeat_interleaver   rd   re   r…   )r¿   r¡   r…  rc   r«   Úclsrç   s   &&&&&  r   ÚupsamplerŠ  F  s±   € ð �„{ØˆßØ��hˆØˆe�HˆÜ×$Ò$ Q¸AÔ>€FßßÜ—]‘]×&Ñ& v°°1°a¸À!½ÀQÈÐ/JÓKˆFØ˜Ð+˜Z¨!�^Ð+Ð+Õ,ˆÜ—’˜C ˜=¨aÔ0ˆð €Mð ˜˜;˜J˜;˜Õ'ˆØ€Mr!   c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚFunnelDecoderi\  c                ó$   <€ V ^8„  d   QhRS[ RR/# r   r   )r   r   s   "€r   r   ÚFunnelDecoder.__annotate__]  s   ø€ ÷ hñ h™|ð h°ñ hr!   c           	     óê   <€ \         SV `  4        Wn        \        V4      V n        \
        P                  ! \        VP                  4       Uu. uF  p\        V^ 4      NK  	  up4      V n
        R# u upi )r\   N)r&   r'   r   rN   r_  r   r`  r†   Únum_decoder_layersrM  Úlayers)r5   r   r2  r6   s   && €r   r'   ÚFunnelDecoder.__init__]  sV   ø€ Ü‰ÑÔØŒÜ#;¸FÓ#CˆÔ Ü—m’mÄUÈ6×KdÑKdÔEeÓ$fÑEeÀ¤[°¸Ö%;ÑEeÑ$fÓgˆŽùÒ$fs   ÁA0c                óÆ   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  R,          RS[ P                  R,          RS[RS[RS[R	S[S[,          /# )
r   Úfinal_hiddenÚfirst_block_hiddenrW   NrX   r+  rg  rh  r   ri  )r   r   s   "€r   r   rŽ  c  s„   ø€ ÷ 'uñ 'uá—l‘lð'uñ "ŸL™Lð'uñ Ÿ™ tÕ+ð	'uñ
 Ÿ™ tÕ+ð'uñ  ð'uñ #ð'uñ ð'uñ 
‘Õ	 ñ'ur!   c           	     ó”  € \        V^\        V P                  P                  4      ^,
          ,          VP                  ^,          V P                  P
                  V P                  P                  R7      pW‚,           p	V'       d   V	3MRp
V'       d   RMRpV P                  P                  V	VVR7      pV P                   FA  pV! W™WœVR7      pV^ ,          p	V'       d   W¾R,          ,           pV'       g   K8  W©3,           p
KC  	  V'       g1   \        ;QJ d    . R WšV3 4       F  NK  	  5# ! R WšV3 4       4      # \        WšVR7      # )	r   )r¡   r…  rc   r«   Nrk  rV  r©   c              3   ó0   "  € T F  qf   K  Vx € K  	  R # 5ir?   rm  rn  s   & r   rÀ   Ú(FunnelDecoder.forward.<locals>.<genexpr>‰  rp  rq  rr  rm  )rŠ  Úlenr   rb  r±   rc   r«   r_  rj   r‘  rY   r	   )r5   r”  r•  rW   rX   r+  rg  rh  Úupsampled_hiddenrH  rx  ry  rß   r~  r�  s   &&&&&&&&       r   rA   ÚFunnelDecoder.forwardc  s#  € ô $ØØœ˜TŸ[™[×4Ñ4Ó5¸Õ9Õ:Ø)×/Ñ/°Õ2ØŸ™×1Ñ1ØŸ™×1Ñ1ô
Ðð "Õ6ˆß)=˜V™IÀ4Ðß0™°dˆà×3Ñ3×IÑIØØ)Ø)ð Jó 
Ðð —[”[ˆEÙ  °Ð]nÔoˆLØ! !•_ˆFç Ø!/¸rÕ2BÕ!B�ß#Ñ#Ø$5¸	Õ$AÒ!ñ !÷ ß”5Ña VÀÑ$OÓa—5Ða�5Ña VÀÑ$OÓaÓaÐaÜ°ÐesÔtÐtr!   )r_  r   r‘  rƒ  rD   rL   s   @@r   rŒ  rŒ  \  s#   ù‡ € ÷hó h÷'u÷ 'uò 'ur!   rŒ  c                   óT   a a€ ] tR tRt oRtV3R lV 3R lltV3R lR ltRtVtV ;t	# )ÚFunnelDiscriminatorPredictionsi�  zEPrediction module for the discriminator, made up of two dense layers.c                ó$   <€ V ^8„  d   QhRS[ RR/# r   r   )r   r   s   "€r   r   Ú+FunnelDiscriminatorPredictions.__annotate__�  s   ø€ ÷ =ñ =™|ð =°ñ =r!   c                óÜ   <€ \         SV `  4        Wn        \        P                  ! VP
                  VP
                  4      V n        \        P                  ! VP
                  ^4      V n        R# ©r[   N)r&   r'   r   r   r  r.   ÚdenseÚdense_predictionr4   s   &&€r   r'   Ú'FunnelDiscriminatorPredictions.__init__�  sD   ø€ Ü‰ÑÔØŒÜ—Y’Y˜vŸ~™~¨v¯~©~Ó>ˆŒ
Ü "§	¢	¨&¯.©.¸!Ó <ˆÖr!   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r   Údiscriminator_hidden_statesr   r;   )r   r   s   "€r   r   rŸ  –  s#   ø€ ÷ ñ ±5·<±<ð ÁEÇLÁLñ r!   c                ó¶   € V P                  V4      p\        V P                  P                  ,          ! V4      pV P	                  V4      P                  R4      pV# )r[   r~   )r¢  r   r   rA  r£  Úsqueeze)r5   r¦  rt  Úlogitss   &&  r   rA   Ú&FunnelDiscriminatorPredictions.forward–  sJ   € ØŸ
™
Ð#>Ó?ˆÜ˜tŸ{™{×5Ñ5Ö6°}ÓEˆØ×&Ñ& }Ó5×=Ñ=¸bÓAˆØˆr!   )r   r¢  r£  )
rE   rF   rG   rH   rï   r'   rA   rI   rJ   rK   rL   s   @@r   r�  r�  �  s!   ù‡ € ÙO÷=ó =÷÷ ð r!   r�  c                   ój   a a€ ] tR tRt oRt]P                  ! 4       V 3R l4       tV3R ltRt	Vt
V ;t# )ÚFunnelPreTrainedModeli�  Úfunnelc                óæ  <€ \         SV `  V4       VP                  P                  pVP	                  R 4      R
8w  dÙ   \        VRR4      e–   V P                  P                  fF   VP                  P                  w  r4\        P                  ! R\        WC,           4      ,          4      pMV P                  P                  p\        P                  ! VP                  VR7       \        VRR4      e$   \        P                  ! VP                   R4       R# R# VR8X  Ed   \        P"                  ! VP$                  V P                  P&                  R7       \        P"                  ! VP(                  V P                  P&                  R7       \        P"                  ! VP*                  V P                  P&                  R7       \        P"                  ! VP,                  V P                  P&                  R7       \        P"                  ! VP.                  V P                  P&                  R7       R# VR	8X  d¾   V P                  P                  f   RMV P                  P                  p\        P                  ! VP0                  P                  VR7       VP0                  P2                  eH   \        P4                  ! VP0                  P                  VP0                  P2                  ,          4       R# R# R# )r  ÚweightNry   )Ústdr  g        rû   )Úbr   r~   )r&   Ú_init_weightsr6   rE   ÚfindÚgetattrr   Úinitializer_stdr¯  r±   ÚnpÚsqrtr/  ÚinitÚnormal_Ú	constant_r  Úuniform_r
  Úinitializer_ranger  r  r  r  r,   r#   Úzeros_)r5   ÚmoduleÚ	classnameÚfan_outÚfan_inr°  r6   s   &&    €r   r²  Ú#FunnelPreTrainedModel._init_weights¢  sã  ø€ ä‰Ñ˜fÔ%Ø×$Ñ$×-Ñ-ˆ	Ø�>‰>˜(Ó# rÔ)Ü�v˜x¨Ó.Ò:Ø—;‘;×.Ñ.Ò6Ø&,§m¡m×&9Ñ&9‘O�GÜŸ'š' #¬¨fÕ.>Ó(?Õ"?Ó@‘CàŸ+™+×5Ñ5�CÜ—’˜VŸ]™]°Õ4Ü�v˜v tÓ,Ò8Ü—’˜vŸ{™{¨CÖ0ñ 9àÐ7Õ7Ü�MŠM˜&Ÿ/™/¨T¯[©[×-JÑ-JÕKÜ�MŠM˜&Ÿ/™/¨T¯[©[×-JÑ-JÕKÜ�MŠM˜&Ÿ/™/¨T¯[©[×-JÑ-JÕKÜ�MŠM˜&Ÿ/™/¨T¯[©[×-JÑ-JÕKÜ�MŠM˜&×*Ñ*¨d¯k©k×.KÑ.K×LØÐ,Ô,ØŸ™×4Ñ4Ò<‘#À$Ç+Á+×B]ÑB]ˆCÜ�LŠL˜×/Ñ/×6Ñ6¸CÕ@Ø×%Ñ%×1Ñ1Ò=Ü—’˜F×2Ñ2×9Ñ9¸&×:PÑ:P×:\Ñ:\Õ]Ö^ñ >ñ -r!   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # )r   r   r   )r   r   s   "€r   r   Ú"FunnelPreTrainedModel.__annotate__�  s   ø‡ ‚ áÑò r!   rm  )rE   rF   rG   rH   Úbase_model_prefixr<   Úno_gradr²  rð   rI   rJ   rK   rL   s   @@r   r¬  r¬  �  s.   ù‡ € ð !Ðà
‡]‚]ƒ_ô_ó ð_÷ … r!   r¬  c                   óP   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚFunnelClassificationHeadi½  c                ó*   <€ V ^8„  d   QhRS[ RS[RR/# )r   r   Ún_labelsr   Nrþ   )r   r   s   "€r   r   Ú%FunnelClassificationHead.__annotate__¾  s"   ø€ ÷ >ñ >™|ð >±sð >¸tñ >r!   c                ó  <€ \         SV `  4        \        P                  ! VP                  VP                  4      V n        \        P                  ! VP                  4      V n        \        P                  ! VP                  V4      V n	        R # r?   )
r&   r'   r   r  r.   Úlinear_hiddenr1   r2   r3   Ú
linear_out)r5   r   rÊ  r6   s   &&&€r   r'   Ú!FunnelClassificationHead.__init__¾  sU   ø€ Ü‰ÑÔÜŸYšY v§~¡~°v·~±~ÓFˆÔÜ—z’z &×"7Ñ"7Ó8ˆŒÜŸ)š) F§N¡N°HÓ=ˆŽr!   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rG  r;   )r   r   s   "€r   r   rË  Ä  s#   ø€ ÷ 'ñ '™eŸl™lð '©u¯|©|ñ 'r!   c                ó”   € V P                  V4      p\        P                  ! V4      pV P                  V4      pV P	                  V4      # r?   )rÍ  r<   Útanhr3   rÎ  )r5   rH  s   &&r   rA   Ú FunnelClassificationHead.forwardÄ  s=   € Ø×#Ñ# FÓ+ˆÜ—’˜FÓ#ˆØ—‘˜fÓ%ˆØ�‰˜vÓ&Ð&r!   )r3   rÍ  rÎ  rD   rL   s   @@r   rÈ  rÈ  ½  s   ù‡ € ÷>ó >÷'÷ 'ð 'r!   rÈ  z2
    Output type of [`FunnelForPreTraining`].
    )Úcustom_introc                   ó@   a € ] tR tRt o RtRtRtRtRtV 3R lt	Rt
V tR# )ÚFunnelForPreTrainingOutputiË  a  
loss (*optional*, returned when `labels` is provided, `torch.FloatTensor` of shape `(1,)`):
    Total loss of the ELECTRA-style objective.
logits (`torch.FloatTensor` of shape `(batch_size, sequence_length)`):
    Prediction scores of the head (scores for each token before SoftMax).
Nc                óò   <€ V ^8„  d   Qh/ S[ P                  R,          ;R&   S[ P                  R,          ;R&   S[S[ P                  ,          R,          ;R&   S[S[ P                  ,          R,          ;R&   # )r   NÚlossr©  rt  ru  )r<   ÚFloatTensorrY   )r   r   s   "€r   r   Ú'FunnelForPreTrainingOutput.__annotate__Ë  so   ø‡ ‚ ñ ×
Ñ
˜dÕ
"Ñ)ñ ñ ×Ñ Õ$Ñ+ñ ñ  ™×*Ñ*Õ+¨dÕ2Ñ9ñ! ñ" ‘e×'Ñ'Õ(¨4Õ/Ñ6ò# r!   rm  )rE   rF   rG   rH   rï   rØ  r©  rt  ru  rð   rI   rJ   )r   s   @r   rÖ  rÖ  Ë  s'   ø‡ € ñð &*€DØ'+€FØ59€MØ26€J÷# ƒ r!   rÖ  z¨
    The base Funnel Transformer Model transformer outputting raw hidden-states without upsampling head (also called
    decoder) or any task-specific head on top.
    c                   ó‚   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltV3R lR lt]RV3R lR	 ll4       tR
t	Vt
V ;t# )ÚFunnelBaseModeliß  c                ó$   <€ V ^8„  d   QhRS[ RR/# r   r   )r   r   s   "€r   r   ÚFunnelBaseModel.__annotate__æ  ó   ø€ ÷ ñ ™|ð °ñ r!   c                ó†   <€ \         SV `  V4       \        V4      V n        \	        V4      V n        V P                  4        R # r?   )r&   r'   r   r@   r[  ÚencoderÚ	post_initr4   s   &&€r   r'   ÚFunnelBaseModel.__init__æ  s4   ø€ Ü‰Ñ˜Ô ä*¨6Ó2ˆŒÜ$ VÓ,ˆŒð 	�‰Ör!   c                ó4   <€ V ^8„  d   QhRS[ P                  /# ©r   r   ©r   r(   )r   r   s   "€r   r   rÞ  ï  ó   ø€ ÷ /ñ /¡b§l¡lñ /r!   c                ó.   € V P                   P                  # r?   ©r@   r,   ©r5   s   &r   Úget_input_embeddingsÚ$FunnelBaseModel.get_input_embeddingsï  ó   € Ø�‰×.Ñ.Ð.r!   c                ó8   <€ V ^8„  d   QhRS[ P                  RR/# ©r   Únew_embeddingsr   Nræ  )r   r   s   "€r   r   rÞ  ò  ó   ø€ ÷ 9ñ 9±2·<±<ð 9ÀDñ 9r!   c                ó&   € WP                   n        R # r?   ré  ©r5   rð  s   &&r   Úset_input_embeddingsÚ$FunnelBaseModel.set_input_embeddingsò  ó   € Ø*8�‰Ö'r!   c                ó4  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          R	S[R,          R
S[S[,          /	# )r   r9   NrW   rX   Úposition_idsr:   r+  rg  rh  r   ri  )r   r   s   "€r   r   rÞ  ö  s£   ø€ ÷ .ñ .á—<‘< $Õ&ð.ñ Ÿ™ tÕ+ð.ñ Ÿ™ tÕ+ð	.ñ
 —l‘l TÕ)ð.ñ —|‘| dÕ*ð.ñ   $�;ð.ñ # T�kð.ñ ˜D•[ð.ñ 
‘Õ	 ñ.r!   c	           	     ó|  € Ve   TMV P                   P                  pVe   TMV P                   P                  pVe   TMV P                   P                  pVe   Ve   \	        R4      hVe#   V P                  W4       VP                  4       p
M#Ve   VP                  4       R R p
M\	        R4      hVe   VP                  MVP                  pVf   \        P                  ! W«R7      pVf(   \        P                  ! V
\        P                  VR7      pV P                  WR7      pV P                  VVVVVVR7      pV# )NúDYou cannot specify both input_ids and inputs_embeds at the same timeú5You have to specify either input_ids or inputs_embeds©ra   rz   ©r:   ©rW   rX   r+  rg  rh  r~   )r   r+  rg  rh  Ú
ValueErrorÚ%warn_if_padding_and_no_attention_maskr]   ra   r<   Úonesr	  r²   r@   rá  )r5   r9   rW   rX   rø  r:   r+  rg  rh  ÚkwargsÚinput_shapera   Úencoder_outputss   &&&&&&&&&,   r   rA   ÚFunnelBaseModel.forwardõ  s5  € ð 2CÒ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ò$DÑ È$Ï+É+×JjÑJjð 	ð &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆàÒ  ]Ò%>ÜÐcÓdÐdØÒ"Ø×6Ñ6°yÔQØ#Ÿ.™.Ó*‰KØÒ&Ø'×,Ñ,Ó.¨s°Ð3‰KäÐTÓUÐUà%.Ò%:�×!Ò!À×@TÑ@TˆàÒ!Ü"ŸZšZ¨ÔCˆNØÒ!Ü"Ÿ[š[¨¼E¿J¹JÈvÔVˆNàŸ™¨	˜ÓOˆàŸ,™,ØØ)Ø)Ø/Ø!5Ø#ð 'ó 
ˆð Ðr!   )r@   rá  ©NNNNNNNN©rE   rF   rG   rH   r'   rë  rô  r   rA   rI   rJ   rK   rL   s   @@r   rÜ  rÜ  ß  sA   ù‡ € ÷ó ÷/ð /÷9ð 9ð ÷.ñ .ó ÷.ð .r!   rÜ  c                   ó‚   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltV3R lR lt]RV3R lR	 ll4       tR
t	Vt
V ;t# )ÚFunnelModeli'  c                ó$   <€ V ^8„  d   QhRS[ RR/# r   r   )r   r   s   "€r   r   ÚFunnelModel.__annotate__)  ó   ø€ ÷ ñ ™|ð °ñ r!   c                ó²   <€ \         SV `  V4       Wn        \        V4      V n        \        V4      V n        \        V4      V n        V P                  4        R # r?   )
r&   r'   r   r   r@   r[  rá  rŒ  Údecoderrâ  r4   s   &&€r   r'   ÚFunnelModel.__init__)  sE   ø€ Ü‰Ñ˜Ô ØŒÜ*¨6Ó2ˆŒÜ$ VÓ,ˆŒÜ$ VÓ,ˆŒð 	�‰Ör!   c                ó4   <€ V ^8„  d   QhRS[ P                  /# rå  ræ  )r   r   s   "€r   r   r  3  rç  r!   c                ó.   € V P                   P                  # r?   ré  rê  s   &r   rë  Ú FunnelModel.get_input_embeddings3  rí  r!   c                ó8   <€ V ^8„  d   QhRS[ P                  RR/# rï  ræ  )r   r   s   "€r   r   r  6  rñ  r!   c                ó&   € WP                   n        R # r?   ré  ró  s   &&r   rô  Ú FunnelModel.set_input_embeddings6  rö  r!   c                ó  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[R,          R	S[S[,          /# )
r   r9   NrW   rX   r:   r+  rg  rh  r   ri  )r   r   s   "€r   r   r  :  sš   ø€ ÷ H
ñ H
á—<‘< $Õ&ðH
ñ Ÿ™ tÕ+ðH
ñ Ÿ™ tÕ+ð	H
ñ
 —|‘| dÕ*ðH
ñ   $�;ðH
ñ # T�kðH
ñ ˜D•[ðH
ñ 
‘Õ	 ñH
r!   c           
     ó´  € Ve   TMV P                   P                  pVe   TMV P                   P                  pVe   TMV P                   P                  pVe   Ve   \	        R4      hVe#   V P                  W4       VP                  4       p	M#Ve   VP                  4       R R
 p	M\	        R4      hVe   VP                  MVP                  p
Vf   \        P                  ! WšR7      pVf(   \        P                  ! V	\        P                  V
R7      pV P                  WR7      pV P                  VVVVRVR7      pV P                  V^ ,          V^,          V P                   P                  ^ ,          ,          VVVVVR7      pV'       gk   ^ pV^ ,          3pV'       d'   V^,          pWë^,          WÍ,          ,           3,           pV'       d'   V^,          pWë^,          WÍ,          ,           3,           pV# \!        V^ ,          V'       d   VP"                  VP"                  ,           MR V'       d#   VP$                  VP$                  ,           R	7      # R R	7      # )Nrú  rû  rü  rz   rý  Trþ  )r”  r•  rW   rX   r+  rg  rh  rr  r~   )r   r+  rg  rh  rÿ  r   r]   ra   r<   r  r	  r²   r@   rá  r  rb  r	   rt  ru  )r5   r9   rW   rX   r:   r+  rg  rh  r  r  ra   r  Údecoder_outputsÚidxÚoutputss   &&&&&&&&,      r   rA   ÚFunnelModel.forward9  s1  € ð 2CÒ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ò$DÑ È$Ï+É+×JjÑJjð 	ð &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆàÒ  ]Ò%>ÜÐcÓdÐdØÒ"Ø×6Ñ6°yÔQØ#Ÿ.™.Ó*‰KØÒ&Ø'×,Ñ,Ó.¨s°Ð3‰KäÐTÓUÐUà%.Ò%:�×!Ò!À×@TÑ@TˆàÒ!Ü"ŸZšZ¨ÔCˆNØÒ!Ü"Ÿ[š[¨¼E¿J¹JÈvÔVˆNàŸ™¨	˜ÓOˆàŸ,™,ØØ)Ø)Ø/Ø!%Ø#ð 'ó 
ˆð Ÿ,™,Ø(¨Õ+Ø.¨qÕ1°$·+±+×2IÑ2IÈ!Õ2LÕMØ)Ø)Ø/Ø!5Ø#ð 'ó 
ˆ÷ ØˆCØ& qÕ)Ð+ˆGß#Ø�q•�Ø!°QÕ%7¸/Õ:NÕ%NÐ$PÕP�ß Ø�q•�Ø!°QÕ%7¸/Õ:NÕ%NÐ$PÕP�ØˆNäØ-¨aÕ0ç#ð +×8Ñ8¸?×;XÑ;XÖXàßTe˜×2Ñ2°_×5OÑ5OÕOô
ð 	
ð
 lpô
ð 	
r!   )r   r  r@   rá  )NNNNNNNr  rL   s   @@r   r	  r	  '  sE   ù‡ € ÷ó ÷/ð /÷9ð 9ð ÷H
ñ H
ó ÷H
ð H
r!   r	  zŒ
    Funnel Transformer model with a binary classification head on top as used during pretraining for identifying
    generated tokens.
    c                   ó^   a a€ ] tR tRt oV3R lV 3R llt]RV3R lR ll4       tRtVtV ;t	# )ÚFunnelForPreTrainingi…  c                ó$   <€ V ^8„  d   QhRS[ RR/# r   r   )r   r   s   "€r   r   Ú!FunnelForPreTraining.__annotate__Œ  ó   ø€ ÷ ñ ™|ð °ñ r!   c                ó†   <€ \         SV `  V4       \        V4      V n        \	        V4      V n        V P                  4        R # r?   )r&   r'   r	  r­  r�  Údiscriminator_predictionsrâ  r4   s   &&€r   r'   ÚFunnelForPreTraining.__init__Œ  s3   ø€ Ü‰Ñ˜Ô ä! &Ó)ˆŒÜ)GÈÓ)OˆÔ&à�‰Ör!   c                ó4  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          R	S[R,          R
S[S[,          /	# ©r   r9   NrW   rX   r:   Úlabelsr+  rg  rh  r   )r<   r=   r,  rY   rÖ  )r   r   s   "€r   r   r  •  s¯   ø€ ÷ C
ñ C
á—<‘< $Õ&ðC
ñ Ÿ™ tÕ+ðC
ñ Ÿ™ tÕ+ð	C
ñ
 —|‘| dÕ*ðC
ñ —‘˜tÕ#ðC
ñ   $�;ðC
ñ # T�kðC
ñ ˜D•[ðC
ñ 
Ñ+Õ	+ñC
r!   c	           
     óÄ  € Ve   TMV P                   P                  pV P                  VVVVVVVR7      p
V
^ ,          pV P                  V4      pRpVeÂ   \        P
                  ! 4       pVeq   VP                  RVP                  ^,          4      ^8H  pVP                  RVP                  ^,          4      V,          pW_,          pV! VVP                  4       4      pM8V! VP                  RVP                  ^,          4      VP                  4       4      pV'       g"   V3V
R,          ,           pVe   V3V,           # T# \        VVV
P                  V
P                  R7      # )a²  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the ELECTRA-style loss. Input should be a sequence of tokens (see `input_ids`
    docstring) Indices should be in `[0, 1]`:

    - 0 indicates the token is an original token,
    - 1 indicates the token was replaced.

Examples:

```python
>>> from transformers import AutoTokenizer, FunnelForPreTraining
>>> import torch

>>> tokenizer = AutoTokenizer.from_pretrained("funnel-transformer/small")
>>> model = FunnelForPreTraining.from_pretrained("funnel-transformer/small")

>>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
>>> logits = model(**inputs).logits
```N©rW   rX   r:   r+  rg  rh  r©   ©rØ  r©  rt  ru  r~   )r   rh  r­  r"  r   r   r.  r±   r/  rÖ  rt  ru  )r5   r9   rW   rX   r:   r&  r+  rg  rh  r  r¦  Údiscriminator_sequence_outputr©  rØ  Úloss_fctÚactive_lossÚactive_logitsÚactive_labelsrç   s   &&&&&&&&&,         r   rA   ÚFunnelForPreTraining.forward”  sf  € ðB &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆà&*§k¡kØØ)Ø)Ø'Ø/Ø!5Ø#ð '2ó '
Ð#ð )DÀAÕ(FÐ%à×/Ñ/Ð0MÓNˆàˆØÒÜ×+Ò+Ó-ˆHØÒ)Ø,×1Ñ1°"Ð6S×6YÑ6YÐZ[Õ6\Ó]ÐabÑb�Ø &§¡¨BÐ0M×0SÑ0SÐTUÕ0VÓ WÐXcÕ d�Ø &Õ 3�Ù ¨}×/BÑ/BÓ/DÓE‘á §¡¨BÐ0M×0SÑ0SÐTUÕ0VÓ WÐY_×YeÑYeÓYgÓh�çØ�YÐ!<¸RÕ!@Õ@ˆFØ)-Ò)9�T�G˜fÕ$ÐE¸vÐEä)ØØØ5×CÑCØ2×=Ñ=ô	
ð 	
r!   )r"  r­  r  ©
rE   rF   rG   rH   r'   r   rA   rI   rJ   rK   rL   s   @@r   r  r  …  s1   ù‡ € ÷ó ð ÷C
ñ C
ó ÷C
ð C
r!   r  c                   óŠ   a a€ ] tR tRt oRR/tV3R lV 3R lltV3R lR ltV3R lR	 lt]RV3R
 lR ll4       t	Rt
VtV ;t# )ÚFunnelForMaskedLMiÛ  zlm_head.weightz(funnel.embeddings.word_embeddings.weightc                ó$   <€ V ^8„  d   QhRS[ RR/# r   r   )r   r   s   "€r   r   ÚFunnelForMaskedLM.__annotate__ß  rß  r!   c                óÆ   <€ \         SV `  V4       \        V4      V n        \        P
                  ! VP                  VP                  4      V n        V P                  4        R # r?   )
r&   r'   r	  r­  r   r  r.   r)   Úlm_headrâ  r4   s   &&€r   r'   ÚFunnelForMaskedLM.__init__ß  sD   ø€ Ü‰Ñ˜Ô ä! &Ó)ˆŒÜ—y’y §¡°×1BÑ1BÓCˆŒð 	�‰Ör!   c                ó4   <€ V ^8„  d   QhRS[ P                  /# rå  )r   r  )r   r   s   "€r   r   r4  è  s   ø€ ÷ ñ ¡r§y¡yñ r!   c                ó   € V P                   # r?   ©r6  rê  s   &r   Úget_output_embeddingsÚ'FunnelForMaskedLM.get_output_embeddingsè  s   € Ø�|‰|Ðr!   c                ó8   <€ V ^8„  d   QhRS[ P                  RR/# rï  ræ  )r   r   s   "€r   r   r4  ë  s   ø€ ÷ &ñ &±B·L±Lð &ÀTñ &r!   c                ó   € Wn         R # r?   r:  ró  s   &&r   Úset_output_embeddingsÚ'FunnelForMaskedLM.set_output_embeddingsë  s   € Ø%Žr!   c                ó4  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          R	S[R,          R
S[S[,          /	# r%  )r<   r=   r,  rY   r
   )r   r   s   "€r   r   r4  ï  s£   ø€ ÷ /
ñ /
á—<‘< $Õ&ð/
ñ Ÿ™ tÕ+ð/
ñ Ÿ™ tÕ+ð	/
ñ
 —|‘| dÕ*ð/
ñ —‘˜tÕ#ð/
ñ   $�;ð/
ñ # T�kð/
ñ ˜D•[ð/
ñ 
‘Õ	ñ/
r!   c	           
     óÎ  € Ve   TMV P                   P                  pV P                  VVVVVVVR7      p
V
^ ,          pV P                  V4      pRpVeG   \	        4       pV! VP                  RV P                   P                  4      VP                  R4      4      pV'       g"   V3V
R,          ,           pVe   V3V,           # T# \        VVV
P                  V
P                  R7      # )az  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
    config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
    loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
Nr(  r©   r)  r~   )
r   rh  r­  r6  r   r.  r)   r
   rt  ru  )r5   r9   rW   rX   r:   r&  r+  rg  rh  r  r  rs  Úprediction_logitsÚmasked_lm_lossr+  rç   s   &&&&&&&&&,      r   rA   ÚFunnelForMaskedLM.forwardî  sô   € ð& &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆà—+‘+ØØ)Ø)Ø'Ø/Ø!5Ø#ð ó 
ˆð $ A�JÐØ ŸL™LÐ):Ó;ÐàˆØÒÜ'Ó)ˆHÙ%Ð&7×&<Ñ&<¸RÀÇÁ×AWÑAWÓ&XÐZ`×ZeÑZeÐfhÓZiÓjˆNçØ'Ð)¨G°B­KÕ7ˆFØ3AÒ3M�^Ð%¨Õ.ÐYÐSYÐYäØØ$Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r!   )r­  r6  r  )rE   rF   rG   rH   Ú_tied_weights_keysr'   r;  r?  r   rA   rI   rJ   rK   rL   s   @@r   r2  r2  Û  sM   ù‡ € à*Ð,VÐWÐ÷ó ÷ð ÷&ð &ð ÷/
ñ /
ó ÷/
ð /
r!   r2  zº
    Funnel Transformer Model with a sequence classification/regression head on top (two linear layer on top of the
    first timestep of the last hidden state) e.g. for GLUE tasks.
    c                   ó^   a a€ ] tR tRt oV3R lV 3R llt]RV3R lR ll4       tRtVtV ;t	# )ÚFunnelForSequenceClassificationi!  c                ó$   <€ V ^8„  d   QhRS[ RR/# r   r   )r   r   s   "€r   r   Ú,FunnelForSequenceClassification.__annotate__(  r  r!   c                óÈ   <€ \         SV `  V4       VP                  V n        Wn        \	        V4      V n        \        WP                  4      V n        V P                  4        R # r?   )	r&   r'   Ú
num_labelsr   rÜ  r­  rÈ  Ú
classifierrâ  r4   s   &&€r   r'   Ú(FunnelForSequenceClassification.__init__(  sJ   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒØŒä% fÓ-ˆŒÜ2°6×;LÑ;LÓMˆŒà�‰Ör!   c                ó4  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          R	S[R,          R
S[S[,          /	# r%  )r<   r=   r,  rY   r   )r   r   s   "€r   r   rJ  3  s¯   ø€ ÷ B
ñ B
á—<‘< $Õ&ðB
ñ Ÿ™ tÕ+ðB
ñ Ÿ™ tÕ+ð	B
ñ
 —|‘| dÕ*ðB
ñ —‘˜tÕ#ðB
ñ   $�;ðB
ñ # T�kðB
ñ ˜D•[ðB
ñ 
Ñ)Õ	)ñB
r!   c	           
     ó„  € Ve   TMV P                   P                  pV P                  VVVVVVVR7      p
V
^ ,          pVR,          pV P                  V4      pRpVEe˜   V P                   P                  f–   V P
                  ^8X  d   RV P                   n        MrV P
                  ^8”  dQ   VP                  \        P                  8X  g    VP                  \        P                  8X  d   RV P                   n        MRV P                   n        V P                   P                  R8X  dK   \        4       pV P
                  ^8X  d'   V! VP                  4       VP                  4       4      pMŽV! WÕ4      pM…V P                   P                  R8X  d>   \        4       pV! VP                  R	V P
                  4      VP                  R	4      4      pM-V P                   P                  R8X  d   \        4       pV! WÕ4      pV'       g"   V3V
R,          ,           pVe   V3V,           # T# \        VVV
P                   V
P"                  R7      # )
ae  
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
    `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
Nr(  Ú
regressionÚsingle_label_classificationÚmulti_label_classificationr©   r)  r×   r~   )r   rh  r­  rM  Úproblem_typerL  r`   r<   r²   ru   r   r¨  r   r.  r   r   rt  ru  )r5   r9   rW   rX   r:   r&  r+  rg  rh  r  r  rs  Úpooled_outputr©  rØ  r+  rç   s   &&&&&&&&&,       r   rA   Ú'FunnelForSequenceClassification.forward2  sÙ  € ð& &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆà—+‘+ØØ)Ø)Ø'Ø/Ø!5Ø#ð ó 
ˆð $ A�JÐØ)¨$Õ/ˆØ—‘ Ó/ˆàˆØÓØ�{‰{×'Ñ'Ò/Ø—?‘? aÔ'Ø/;�D—K‘KÕ,Ø—_‘_ qÔ(¨f¯l©l¼e¿j¹jÔ.HÈFÏLÉLÔ\a×\eÑ\eÔLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ô7Ü"›9�Ø—?‘? aÔ'Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# FÓ3‘DØ—‘×)Ñ)Ð-JÔJÜ+Ó-�Ù §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR‘Ø—‘×)Ñ)Ð-IÔIÜ,Ó.�Ù Ó/�çØ�Y ¨¥Õ,ˆFØ)-Ò)9�T�G˜fÕ$ÐE¸vÐEä'ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r!   )rM  r   r­  rL  r  r0  rL   s   @@r   rH  rH  !  s1   ù‡ € ÷ó ð ÷B
ñ B
ó ÷B
ð B
r!   rH  c                   ó^   a a€ ] tR tRt oV3R lV 3R llt]RV3R lR ll4       tRtVtV ;t	# )ÚFunnelForMultipleChoiceix  c                ó$   <€ V ^8„  d   QhRS[ RR/# r   r   )r   r   s   "€r   r   Ú$FunnelForMultipleChoice.__annotate__z  r   r!   c                óˆ   <€ \         SV `  V4       \        V4      V n        \	        V^4      V n        V P                  4        R# r¡  )r&   r'   rÜ  r­  rÈ  rM  râ  r4   s   &&€r   r'   Ú FunnelForMultipleChoice.__init__z  s4   ø€ Ü‰Ñ˜Ô ä% fÓ-ˆŒÜ2°6¸1Ó=ˆŒà�‰Ör!   c                ó4  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          R	S[R,          R
S[S[,          /	# r%  )r<   r=   r,  rY   r   )r   r   s   "€r   r   rZ  ƒ  s¤   ø€ ÷ ;
ñ ;
á—<‘< $Õ&ð;
ñ Ÿ™ tÕ+ð;
ñ Ÿ™ tÕ+ð	;
ñ
 —|‘| dÕ*ð;
ñ —‘˜tÕ#ð;
ñ   $�;ð;
ñ # T�kð;
ñ ˜D•[ð;
ñ 
Ñ*Õ	*ñ;
r!   c	           
     óH  € Ve   TMV P                   P                  pVe   VP                  ^,          MVP                  ^,          p
Ve"   VP                  RVP	                  R4      4      MRpVe"   VP                  RVP	                  R4      4      MRpVe"   VP                  RVP	                  R4      4      MRpVe2   VP                  RVP	                  R4      VP	                  R4      4      MRpV P                  VVVVVVVR7      pV^ ,          pVR,          pV P                  V4      pVP                  RV
4      pRpVe   \        4       pV! Wõ4      pV'       g"   V3VR,          ,           pVe   V3V,           # T# \        VVVP                  VP                  R7      # )a"  
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
    num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
    `input_ids` above)
Nr(  r©   r)  r~   éþÿÿÿr×   )r   rh  r±   r.  r]   r­  rM  r   r   rt  ru  )r5   r9   rW   rX   r:   r&  r+  rg  rh  r  Únum_choicesr  rs  rU  r©  Úreshaped_logitsrØ  r+  rç   s   &&&&&&&&&,         r   rA   ÚFunnelForMultipleChoice.forward‚  s«  € ð& &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆØ,5Ò,A�i—o‘o aÖ(À}×GZÑGZÐ[\ÕG]ˆà>GÒ>S�I—N‘N 2 y§~¡~°bÓ'9Ô:ÐY]ˆ	ØM[ÒMg˜×,Ñ,¨R°×1DÑ1DÀRÓ1HÔIÐmqˆØM[ÒMg˜×,Ñ,¨R°×1DÑ1DÀRÓ1HÔIÐmqˆð Ò(ð ×Ñ˜r =×#5Ñ#5°bÓ#9¸=×;MÑ;MÈbÓ;QÔRàð 	ð —+‘+ØØ)Ø)Ø'Ø/Ø!5Ø#ð ó 
ˆð $ A�JÐØ)¨$Õ/ˆØ—‘ Ó/ˆØ Ÿ+™+ b¨+Ó6ˆàˆØÒÜ'Ó)ˆHÙ˜OÓ4ˆDçØ%Ð'¨'°"­+Õ5ˆFØ)-Ò)9�T�G˜fÕ$ÐE¸vÐEä(ØØ"Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r!   )rM  r­  r  r0  rL   s   @@r   rX  rX  x  s-   ù‡ € ÷ó ð ÷;
ñ ;
ó ÷;
ð ;
r!   rX  c                   ó^   a a€ ] tR tRt oV3R lV 3R llt]RV3R lR ll4       tRtVtV ;t	# )ÚFunnelForTokenClassificationiÁ  c                ó$   <€ V ^8„  d   QhRS[ RR/# r   r   )r   r   s   "€r   r   Ú)FunnelForTokenClassification.__annotate__Ã  s   ø€ ÷ 	ñ 	™|ð 	°ñ 	r!   c                ó2  <€ \         SV `  V4       VP                  V n        \        V4      V n        \
        P                  ! VP                  4      V n        \
        P                  ! VP                  VP                  4      V n        V P                  4        R # r?   )r&   r'   rL  r	  r­  r   r1   r2   r3   r  r*   rM  râ  r4   s   &&€r   r'   Ú%FunnelForTokenClassification.__init__Ã  si   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä! &Ó)ˆŒÜ—z’z &×"7Ñ"7Ó8ˆŒÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Ör!   c                ó4  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          R	S[R,          R
S[S[,          /	# r%  )r<   r=   r,  rY   r   )r   r   s   "€r   r   rf  Ï  s¤   ø€ ÷ .
ñ .
á—<‘< $Õ&ð.
ñ Ÿ™ tÕ+ð.
ñ Ÿ™ tÕ+ð	.
ñ
 —|‘| dÕ*ð.
ñ —‘˜tÕ#ð.
ñ   $�;ð.
ñ # T�kð.
ñ ˜D•[ð.
ñ 
Ñ&Õ	&ñ.
r!   c	           
     óÜ  € Ve   TMV P                   P                  pV P                  VVVVVVVR7      p
V
^ ,          pV P                  V4      pV P	                  V4      pRpVe=   \        4       pV! VP                  RV P                  4      VP                  R4      4      pV'       g"   V3V
R,          ,           pVe   V3V,           # T# \        VVV
P                  V
P                  R7      # )zÃ
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
Nr(  r©   r)  r~   )r   rh  r­  r3   rM  r   r.  rL  r   rt  ru  )r5   r9   rW   rX   r:   r&  r+  rg  rh  r  r  rs  r©  rØ  r+  rç   s   &&&&&&&&&,      r   rA   Ú$FunnelForTokenClassification.forwardÎ  sò   € ð" &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆà—+‘+ØØ)Ø)Ø'Ø/Ø!5Ø#ð ó 
ˆð $ A�JÐØ ŸL™LÐ):Ó;ÐØ—‘Ð!2Ó3ˆàˆØÒÜ'Ó)ˆHÙ˜FŸK™K¨¨D¯O©OÓ<¸f¿k¹kÈ"»oÓNˆDçØ�Y ¨¥Õ,ˆFØ)-Ò)9�T�G˜fÕ$ÐE¸vÐEä$ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r!   )rM  r3   r­  rL  r  r0  rL   s   @@r   rd  rd  Á  s-   ù‡ € ÷	ó 	ð ÷.
ñ .
ó ÷.
ð .
r!   rd  c                   ó^   a a€ ] tR tRt oV3R lV 3R llt]RV3R lR ll4       tRtVtV ;t	# )ÚFunnelForQuestionAnsweringi   c                ó$   <€ V ^8„  d   QhRS[ RR/# r   r   )r   r   s   "€r   r   Ú'FunnelForQuestionAnswering.__annotate__  r  r!   c                óè   <€ \         SV `  V4       VP                  V n        \        V4      V n        \
        P                  ! VP                  VP                  4      V n        V P                  4        R # r?   )
r&   r'   rL  r	  r­  r   r  r*   Ú
qa_outputsrâ  r4   s   &&€r   r'   Ú#FunnelForQuestionAnswering.__init__  sS   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä! &Ó)ˆŒÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Ör!   c                ó\  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[R,          R
S[R,          RS[S[,          /
# )r   r9   NrW   rX   r:   Ústart_positionsÚend_positionsr+  rg  rh  r   )r<   r=   r,  rY   r   )r   r   s   "€r   r   ro    s·   ø€ ÷ ;
ñ ;
á—<‘< $Õ&ð;
ñ Ÿ™ tÕ+ð;
ñ Ÿ™ tÕ+ð	;
ñ
 —|‘| dÕ*ð;
ñ Ÿ™¨Õ,ð;
ñ —|‘| dÕ*ð;
ñ   $�;ð;
ñ # T�kð;
ñ ˜D•[ð;
ñ 
Ñ-Õ	-ñ;
r!   c
           
     óp  € V	e   T	MV P                   P                  p	V P                  VVVVVVV	R7      pV^ ,          pV P                  V4      pVP	                  ^RR7      w  rïVP                  R4      P                  4       pVP                  R4      P                  4       pR pVeÄ   VeÀ   \        VP                  4       4      ^8”  d   VP                  R4      p\        VP                  4       4      ^8”  d   VP                  R4      pVP                  ^4      pVP                  ^ V4      pVP                  ^ V4      p\        VR7      pV! Wå4      pV! Wö4      pVV,           ^,          pV	'       g"   Wï3VR,          ,           pVe   V3V,           # T# \        VVVVP                  VP                  R7      # )Nr(  r{   )Úignore_indexr©   )rØ  Ústart_logitsÚ
end_logitsrt  ru  r~   )r   rh  r­  rq  r  r¨  Ú
contiguousr™  r]   ÚsquezeÚclampr   r   rt  ru  )r5   r9   rW   rX   r:   rt  ru  r+  rg  rh  r  r  rs  r©  rx  ry  Ú
total_lossÚignored_indexr+  Ú
start_lossÚend_lossrç   s   &&&&&&&&&&,           r   rA   Ú"FunnelForQuestionAnswering.forward  s´  € ð &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆà—+‘+ØØ)Ø)Ø'Ø/Ø!5Ø#ð ó 
ˆð $ A�JÐà—‘Ð!2Ó3ˆØ#)§<¡<°°r <Ó#:Ñ ˆØ#×+Ñ+¨BÓ/×:Ñ:Ó<ˆØ×'Ñ'¨Ó+×6Ñ6Ó8ˆ
àˆ
ØÒ&¨=Ò+Dä�?×'Ñ'Ó)Ó*¨QÔ.Ø"1×"8Ñ"8¸Ó"<�Ü�=×%Ñ%Ó'Ó(¨1Ô,Ø -× 5Ñ 5°bÓ 9�à(×-Ñ-¨aÓ0ˆMØ-×3Ñ3°A°}ÓEˆOØ)×/Ñ/°°=ÓAˆMä'°]ÔCˆHÙ! ,Ó@ˆJÙ 
Ó:ˆHØ$ xÕ/°1Õ4ˆJçØ"Ð/°'¸"µ+Õ=ˆFØ/9Ò/E�Z�M FÕ*ÐQÈ6ÐQä+ØØ%Ø!Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r!   )r­  rL  rq  )	NNNNNNNNNr0  rL   s   @@r   rm  rm     s-   ù‡ € ÷ó ð ÷;
ñ ;
ó ÷;
ð ;
r!   rm  )TF)	rÜ  r2  rX  r  rm  rH  rd  r	  r¬  ):rï   Údataclassesr   Únumpyr¶  r<   r   Útorch.nnr   r   r   Ú r   r¸  Úactivationsr   Úmodeling_outputsr	   r
   r   r   r   r   Úmodeling_utilsr   Úutilsr   r   r   Úconfiguration_funnelr   Ú
get_loggerrE   Úloggerr0  ÚModuler   rN   rù   rû   r;  rM  r[  rŠ  rŒ  r�  r¬  rÈ  rÖ  rÜ  r	  r  r2  rH  rX  rd  rm  Ú__all__rm  r!   r   Ú<module>r�     si  ðñ (å !ã Û Ý ß AÑ Aå &Ý !÷÷ õ .ß 9Ñ 9Ý .ð 
×	Ò	˜HÓ	%€ð 
€ô�r—y‘yô ô"A ˜rŸy™yô A õHô MG "§)¡)ô MGô`+˜BŸI™Iô +ô&E�"—)‘)ô Eô&<u�B—I‘Iô <u÷~ô,.u�B—I‘Iô .uôb R§Y¡Yô ð  ô_˜Oó _ó ð_ô>'˜rŸy™yô 'ñ ðôð
 ô7 ó 7ó óð7ñ ðôô?Ð+ó ?óð?ðD ôZ
Ð'ó Z
ó ðZ
ñz ðôôM
Ð0ó M
óðM
ð` ôB
Ð-ó B
ó ðB
ñJ ðôôN
Ð&;ó N
óðN
ðb ôE
Ð3ó E
ó ðE
ðP ô;
Ð#8ó ;
ó ð;
ð| ôG
Ð!6ó G
ó ðG
òT
‚r!   