+
    œ°³j±†  ã                   óh  € R t ^ RIt^ RIt^ RIHt ^ RIHtHtHt ^RIH	t
 ^RIHtHt ^RIHt ^RIHtHtHtHtHtHtHt ^RIHt ^R	IHtHt ^R
IHt ]P>                  ! ] 4      t!] ! R R]4      4       t" ! R R]PF                  4      t$ ! R R]PF                  4      t% ! R R]PF                  4      t& ! R R]PF                  4      t' ! R R]PF                  4      t( ! R R]PF                  4      t) ! R R]PF                  4      t* ! R R]PF                  4      t+] ! R R]"4      4       t, ! R R ]"4      t- ! R! R"]PF                  4      t.]! R#R$7       ! R% R&]"4      4       t/] ! R' R(]"4      4       t0] ! R) R*]"4      4       t1 ! R+ R,]PF                  4      t2] ! R- R.]"4      4       t3R/ t4. R0Ot5R# )1zPyTorch MPNet model.N)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELoss)Úinitialization)ÚACT2FNÚgelu)Úcreate_bidirectional_mask)ÚBaseModelOutputÚBaseModelOutputWithPoolingÚMaskedLMOutputÚMultipleChoiceModelOutputÚQuestionAnsweringModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚPreTrainedModel)Úauto_docstringÚlogging©ÚMPNetConfigc                   ój   a a€ ] tR t^+t oRt]P                  ! 4       V 3R l4       tV3R ltRt	Vt
V ;t# )ÚMPNetPreTrainedModelÚmpnetc                ó„  <€ \         SV `  V4       \        V\        4      '       d#   \        P
                  ! VP                  4       R# \        V\        4      '       db   \        P                  ! VP                  \        P                  ! VP                  P                  R,          4      P                  R4      4       R# R# )zInitialize the weightsNéÿÿÿÿ©é   r   )ÚsuperÚ_init_weightsÚ
isinstanceÚMPNetLMHeadÚinitÚzeros_ÚbiasÚMPNetEmbeddingsÚcopy_Úposition_idsÚtorchÚarangeÚshapeÚexpand)ÚselfÚmoduleÚ	__class__s   &&€Úi/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/mpnet/modeling_mpnet.pyr   Ú"MPNetPreTrainedModel._init_weights0   s|   ø€ ô 	‰Ñ˜fÔ%Ü�fœk×*Ò*Ü�KŠK˜Ÿ™Ö$Ü˜¤×0Ò0Ü�JŠJ�v×*Ñ*¬E¯LªL¸×9LÑ9L×9RÑ9RÐSUÕ9VÓ,W×,^Ñ,^Ð_fÓ,gÖhñ 1ó    c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # )é   Úconfigr   )ÚformatÚ__classdict__s   "€r.   Ú__annotate__Ú!MPNetPreTrainedModel.__annotate__+   s   ø‡ ‚ áÑò r0   © )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Úbase_model_prefixr'   Úno_gradr   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r-   r5   s   @@r.   r   r   +   s.   ù‡ € ð  Ðà
‡]‚]ƒ_ôió ði÷ … r0   r   c                   óB   a a€ ] tR t^:t oV 3R ltRR ltR tRtVtV ;t	# )r$   c                ób  <€ \         SV `  4        ^V n        \        P                  ! VP
                  VP                  V P                  R7      V n        \        P                  ! VP                  VP                  V P                  R7      V n	        \        P                  ! VP                  VP                  R7      V n
        \        P                  ! VP                  4      V n        \        P                  ! \         P"                  ! VP                  4      P%                  R4      RR7      V n        R# )r   )Úpadding_idx©ÚepsF)Ú
persistentNr   )r   Ú__init__rF   r   Ú	EmbeddingÚ
vocab_sizeÚhidden_sizeÚword_embeddingsÚmax_position_embeddingsÚposition_embeddingsÚ	LayerNormÚlayer_norm_epsÚDropoutÚhidden_dropout_probÚdropoutÚBufferr'   r(   r*   r&   ©r+   r3   r-   s   &&€r.   rJ   ÚMPNetEmbeddings.__init__;   sÍ   ø€ Ü‰ÑÔØˆÔÜ!Ÿ|š|¨F×,=Ñ,=¸v×?QÑ?QÐ_c×_oÑ_oÔpˆÔÜ#%§<¢<Ø×*Ñ*¨F×,>Ñ,>ÈD×L\ÑL\ô$
ˆÔ ô Ÿš f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z’z &×"<Ñ"<Ó=ˆŒÜŸIšI¤e§l¢l°6×3QÑ3QÓ&R×&YÑ&YÐZaÓ&bÐotÔuˆÖr0   c                óœ  € Vf,   Ve   \        WP                  4      pMV P                  V4      pVe   VP                  4       pMVP                  4       R R pV^,          pVf   V P                  RR V13,          pVf   V P                  V4      pV P                  V4      pW7,           pV P                  V4      pV P                  V4      pV# )NºNNNr   )	Ú"create_position_ids_from_input_idsrF   Ú&create_position_ids_from_inputs_embedsÚsizer&   rN   rP   rQ   rU   )	r+   Ú	input_idsr&   Úinputs_embedsÚkwargsÚinput_shapeÚ
seq_lengthrP   Ú
embeddingss	   &&&&,    r.   ÚforwardÚMPNetEmbeddings.forwardG   sÍ   € ØÒØÒ$ÜAÀ)×M]ÑM]Ó^‘à#×JÑJÈ=ÓY�àÒ Ø#Ÿ.™.Ó*‰Kà'×,Ñ,Ó.¨s°Ð3ˆKà  •^ˆ
àÒØ×,Ñ,¨Q°°°¨^Õ<ˆLàÒ Ø ×0Ñ0°Ó;ˆMØ"×6Ñ6°|ÓDÐà"Õ8ˆ
Ø—^‘^ JÓ/ˆ
Ø—\‘\ *Ó-ˆ
ØÐr0   c                ó.  € VP                  4       RR pV^,          p\        P                  ! V P                  ^,           W0P                  ,           ^,           \        P                  VP
                  R7      pVP                  ^ 4      P                  V4      # )z¯
We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.

Args:
    inputs_embeds: torch.Tensor

Returns: torch.Tensor
N)ÚdtypeÚdevicer   )r]   r'   r(   rF   Úlongrh   Ú	unsqueezer*   )r+   r_   ra   Úsequence_lengthr&   s   &&   r.   r\   Ú6MPNetEmbeddings.create_position_ids_from_inputs_embedsa   s~   € ð $×(Ñ(Ó*¨3¨BÐ/ˆØ% a�.ˆä—|’|Ø×Ñ˜qÕ  /×4DÑ4DÕ"DÀqÕ"HÔPU×PZÑPZÐcp×cwÑcwô
ˆð ×%Ñ% aÓ(×/Ñ/°Ó<Ð<r0   )rQ   rU   rF   rP   r&   rN   )NNN)
r9   r:   r;   r<   rJ   rd   r\   r@   rA   rB   rC   s   @@r.   r$   r$   :   s   ù‡ € õ
vô÷4=ò =r0   r$   c                   ó<   a a€ ] tR t^st oV 3R ltRR ltRtVtV ;t# )ÚMPNetSelfAttentionc                ó^  <€ \         SV `  4        VP                  VP                  ,          ^ 8w  d9   \	        VR4      '       g'   \        RVP                   RVP                   R24      hVP                  V n        \        VP                  VP                  ,          4      V n        V P                  V P                  ,          V n        \        P                  ! VP                  V P                  4      V n        \        P                  ! VP                  V P                  4      V n        \        P                  ! VP                  V P                  4      V n        \        P                  ! VP                  VP                  4      V n        \        P                  ! VP                   4      V n        R# )é    Úembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads (Ú)N)r   rJ   rM   Únum_attention_headsÚhasattrÚ
ValueErrorÚintÚattention_head_sizeÚall_head_sizer   ÚLinearÚqÚkÚvÚorS   Úattention_probs_dropout_probrU   rW   s   &&€r.   rJ   ÚMPNetSelfAttention.__init__t   s=  ø€ Ü‰ÑÔØ×Ñ × :Ñ :Õ:¸aÔ?ÌÐPVÐXh×HiÒHiÜØ# F×$6Ñ$6Ð#7ð 8Ø ×4Ñ4Ð5°Qð8óð ð
 $*×#=Ñ#=ˆÔ Ü#& v×'9Ñ'9¸F×<VÑ<VÕ'VÓ#WˆÔ Ø!×5Ñ5¸×8PÑ8PÕPˆÔä—’˜6×-Ñ-¨t×/AÑ/AÓBˆŒÜ—’˜6×-Ñ-¨t×/AÑ/AÓBˆŒÜ—’˜6×-Ñ-¨t×/AÑ/AÓBˆŒÜ—’˜6×-Ñ-¨v×/AÑ/AÓBˆŒä—z’z &×"EÑ"EÓFˆŽr0   c                ó´  € VP                   R R p. VORNV P                  N5pV P                  V4      P                  V4      P	                  ^^4      pV P                  V4      P                  V4      P	                  ^^4      p	V P                  V4      P                  V4      P	                  ^^4      p
\        P                  ! W‰P	                  RR4      4      pV\        P                  ! V P                  4      ,          pVe	   W³,          pVe	   W²,           p\        P                  P                  VRR7      pV P                  V4      p\        P                  ! WÊ4      pVP                  ^ ^^^4      P!                  4       pVP#                  4       R R V P$                  3,           pVP                  ! V!  pV P'                  V4      pV'       d   Wü3pV# V3pV# )N©Údimr   éþÿÿÿ)r)   rw   rz   ÚviewÚ	transposer{   r|   r'   ÚmatmulÚmathÚsqrtr   Ú
functionalÚsoftmaxrU   ÚpermuteÚ
contiguousr]   rx   r}   )r+   Úhidden_statesÚattention_maskÚposition_biasÚoutput_attentionsr`   ra   Úhidden_shaperz   r{   r|   Úattention_scoresÚattention_probsÚcÚnew_c_shaper}   Úoutputss   &&&&&,           r.   rd   ÚMPNetSelfAttention.forward‡   s¦  € ð $×)Ñ)¨#¨2Ð.ˆØC˜ÐC bÐC¨$×*BÑ*BÑCˆØ�F‰F�=Ó!×&Ñ& |Ó4×>Ñ>¸qÀ!ÓDˆØ�F‰F�=Ó!×&Ñ& |Ó4×>Ñ>¸qÀ!ÓDˆØ�F‰F�=Ó!×&Ñ& |Ó4×>Ñ>¸qÀ!ÓDˆô !Ÿ<š<¨¯;©;°r¸2Ó+>Ó?ÐØ+¬d¯iªi¸×8PÑ8PÓ.QÕQÐð Ò$ØÕ-ÐàÒ%Ø/Õ@Ðô Ÿ-™-×/Ñ/Ð0@ÀbÐ/ÓIˆàŸ,™, Ó7ˆä�LŠL˜Ó,ˆà�I‰I�a˜˜A˜qÓ!×,Ñ,Ó.ˆØ—f‘f“h˜s �m t×'9Ñ'9Ð&;Õ;ˆØ�FŠF�KÑ ˆà�F‰F�1‹Iˆç*;�1Ð&ˆØˆð CDÀˆØˆr0   )rx   rw   rU   r{   rs   r}   rz   r|   ©NNF©	r9   r:   r;   r<   rJ   rd   r@   rA   rB   rC   s   @@r.   rn   rn   s   s   ù‡ € õG÷&'ô 'r0   rn   c                   ó<   a a€ ] tR t^±t oV 3R ltRR ltRtVtV ;t# )ÚMPNetAttentionc                óð   <€ \         SV `  4        \        V4      V n        \        P
                  ! VP                  VP                  R 7      V n        \        P                  ! VP                  4      V n
        R# ©rG   N)r   rJ   rn   Úattnr   rQ   rM   rR   rS   rT   rU   rW   s   &&€r.   rJ   ÚMPNetAttention.__init__²   sN   ø€ Ü‰ÑÔÜ& vÓ.ˆŒ	ÜŸš f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z’z &×"<Ñ"<Ó=ˆŽr0   c                ó®   € V P                  VVVVR 7      pV P                  V P                  V^ ,          4      V,           4      pV3VR,          ,           pV# ))r�   ºr   NN)rž   rQ   rU   )	r+   r�   rŽ   r�   r�   r`   Úself_outputsÚattention_outputr–   s	   &&&&&,   r.   rd   ÚMPNetAttention.forward¸   s]   € ð —y‘yØØØØ/ð	 !ó 
ˆð  Ÿ>™>¨$¯,©,°|ÀAµÓ*GÈ-Õ*WÓXÐØ#Ð%¨°RÕ(8Õ8ˆØˆr0   )rQ   rž   rU   r˜   r™   rC   s   @@r.   r›   r›   ±   s   ù‡ € õ>÷ô r0   r›   c                   óD   a a€ ] tR t^Ìt oV 3R ltV3R lR ltRtVtV ;t# )ÚMPNetIntermediatec                ó"  <€ \         SV `  4        \        P                  ! VP                  VP
                  4      V n        \        VP                  \        4      '       d   \        VP                  ,          V n        R # VP                  V n        R # ©N)r   rJ   r   ry   rM   Úintermediate_sizeÚdenser   Ú
hidden_actÚstrr   Úintermediate_act_fnrW   s   &&€r.   rJ   ÚMPNetIntermediate.__init__Í   s`   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3KÑ3KÓLˆŒ
Ü�f×'Ñ'¬×-Ò-Ü'-¨f×.?Ñ.?Õ'@ˆDÖ$à'-×'8Ñ'8ˆDÖ$r0   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r2   r�   Úreturn©r'   ÚTensor)r4   r5   s   "€r.   r6   ÚMPNetIntermediate.__annotate__Õ   s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r0   c                óJ   € V P                  V4      pV P                  V4      pV# r¨   ©rª   r­   )r+   r�   s   &&r.   rd   ÚMPNetIntermediate.forwardÕ   s&   € ØŸ
™
 =Ó1ˆØ×0Ñ0°Ó?ˆØÐr0   r¶   r™   rC   s   @@r.   r¦   r¦   Ì   s   ù‡ € õ9÷÷ ð r0   r¦   c                   óD   a a€ ] tR t^Üt oV 3R ltV3R lR ltRtVtV ;t# )ÚMPNetOutputc                ó0  <€ \         SV `  4        \        P                  ! VP                  VP
                  4      V n        \        P                  ! VP
                  VP                  R 7      V n        \        P                  ! VP                  4      V n        R# r�   )r   rJ   r   ry   r©   rM   rª   rQ   rR   rS   rT   rU   rW   s   &&€r.   rJ   ÚMPNetOutput.__init__Ý   s`   ø€ Ü‰ÑÔÜ—Y’Y˜v×7Ñ7¸×9KÑ9KÓLˆŒ
ÜŸš f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z’z &×"<Ñ"<Ó=ˆŽr0   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# )r2   r�   Úinput_tensorr±   r²   )r4   r5   s   "€r.   r6   ÚMPNetOutput.__annotate__ã   s1   ø€ ÷ ñ ¡U§\¡\ð ÁÇÁð ÑRW×R^ÑR^ñ r0   c                óx   € V P                  V4      pV P                  V4      pV P                  W,           4      pV# r¨   )rª   rU   rQ   )r+   r�   r½   s   &&&r.   rd   ÚMPNetOutput.forwardã   s5   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØŸ™ }Õ'CÓDˆØÐr0   )rQ   rª   rU   r™   rC   s   @@r.   r¹   r¹   Ü   s   ù‡ € õ>÷÷ ð r0   r¹   c                   ó<   a a€ ] tR t^êt oV 3R ltRR ltRtVtV ;t# )Ú
MPNetLayerc                ó„   <€ \         SV `  4        \        V4      V n        \	        V4      V n        \        V4      V n        R # r¨   )r   rJ   r›   Ú	attentionr¦   Úintermediater¹   ÚoutputrW   s   &&€r.   rJ   ÚMPNetLayer.__init__ë   s3   ø€ Ü‰ÑÔÜ'¨Ó/ˆŒÜ-¨fÓ5ˆÔÜ! &Ó)ˆŽr0   c                ó¬   € V P                  VVVVR 7      pV^ ,          pVR,          pV P                  V4      p	V P                  W—4      p
V
3V,           pV# ))r�   r�   r¡   ©rÄ   rÅ   rÆ   )r+   r�   rŽ   r�   r�   r`   Úself_attention_outputsr£   r–   Úintermediate_outputÚlayer_outputs   &&&&&,     r.   rd   ÚMPNetLayer.forwardñ   sm   € ð "&§¡ØØØ'Ø/ð	 "0ó "
Ðð 2°!Õ4ÐØ(¨Õ,ˆà"×/Ñ/Ð0@ÓAÐØ—{‘{Ð#6ÓIˆØ�/ GÕ+ˆØˆr0   rÉ   r˜   r™   rC   s   @@r.   rÂ   rÂ   ê   s   ù‡ € õ*÷ô r0   rÂ   c                   óf   a a€ ] tR tRt oV 3R ltRV3R lR lltR	R lt]R
R l4       tRt	Vt
V ;t# )ÚMPNetEncoderi  c                óJ  <€ \         SV `  4        Wn        VP                  V n        \
        P                  ! \        VP                  4       Uu. uF  p\        V4      NK  	  up4      V n
        \
        P                  ! VP                  V P                  4      V n        R # u upi r¨   )r   rJ   r3   rs   Ún_headsr   Ú
ModuleListÚrangeÚnum_hidden_layersrÂ   ÚlayerrK   Úrelative_attention_num_bucketsÚrelative_attention_bias)r+   r3   Ú_r-   s   && €r.   rJ   ÚMPNetEncoder.__init__	  sw   ø€ Ü‰ÑÔØŒØ×1Ñ1ˆŒÜ—]’]ÄÀf×F^ÑF^Ô@_Ó#`Ñ@_¸1¤J¨vÖ$6Ñ@_Ñ#`ÓaˆŒ
Ü')§|¢|°F×4YÑ4YÐ[_×[gÑ[gÓ'hˆÖ$ùò $as   ÁB c          
      ón   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[RS[RS[/# )r2   r�   rŽ   Nr�   Úoutput_hidden_statesÚreturn_dict)r'   r³   Úbool)r4   r5   s   "€r.   r6   ÚMPNetEncoder.__annotate__  sJ   ø€ ÷ &
ñ &
á—|‘|ð&
ñ Ÿ™ tÕ+ð&
ñ  ð	&
ñ
 #ð&
ñ ñ&
r0   c                óÀ  € V P                  V4      pV'       d   RMR pV'       d   RMR p	\        V P                  4       FF  w  r«V'       d
   W�3,           pV! VVV3RV/VB pV^ ,          pV'       g   K6  Wœ^,          3,           p	KH  	  V'       d
   W�3,           pV'       g1   \        ;QJ d    . R WV	3 4       F  NK  	  5# ! R WV	3 4       4      # \	        VVV	R7      # )Nr�   c              3   ó0   "  € T F  qf   K  Vx € K  	  R # 5ir¨   r8   )Ú.0r|   s   & r.   Ú	<genexpr>Ú'MPNetEncoder.forward.<locals>.<genexpr>1  s   é € ÐhÑ$V˜qŸšÓ$Vùs   ‚Œ
)Úlast_hidden_stater�   Ú
attentionsr8   )Úcompute_position_biasÚ	enumeraterÕ   Útupler
   )r+   r�   rŽ   r�   rÛ   rÜ   r`   r�   Úall_hidden_statesÚall_attentionsÚiÚlayer_moduleÚlayer_outputss   &&&&&&,      r.   rd   ÚMPNetEncoder.forward  sê   € ð ×2Ñ2°=ÓAˆß"6™B¸DÐß0™°dˆÜ(¨¯©Ö4‰OˆAß#Ø$5Ð8HÕ$HÐ!á(ØØØñð #4ð	ð
 ñˆMð *¨!Õ,ˆMç Ñ Ø!/ÀÕ3CÐ2EÕ!E’ñ  5÷"  Ø 1Ð4DÕ DÐçß”5Ñh ]À~Ñ$VÓh—5Ðh�5Ñh ]À~Ñ$VÓhÓhÐhÜØ+Ø+Ø%ô
ð 	
r0   c                ó`  € VP                  ^ 4      VP                  ^4      VP                  ^4      repVe   VR,          pVR,          pMZ\        P                  ! V\        P                  R7      R,          p\        P                  ! V\        P                  R7      R,          pW‡,
          p	V P	                  W“R7      p
V
P                  VP                  4      p
V P                  V
4      pVP                  . RO4      P                  ^ 4      pVP                  VRWV34      P                  4       pV# )	rp   )rg   )Únum_buckets)rZ   rZ   N)rZ   NrZ   )rZ   N)NrZ   )r2   rp   r   r   )r]   r'   r(   ri   Úrelative_position_bucketÚtorh   r×   r‹   rj   r*   rŒ   )r+   Úxr&   rð   ÚbszÚqlenÚklenÚcontext_positionÚmemory_positionÚrelative_positionÚ	rp_bucketÚvaluess   &&&&        r.   ræ   Ú"MPNetEncoder.compute_position_bias8  sî   € ØŸ&™& ›) Q§V¡V¨A£Y°·±°q³	�4ˆØÒ#Ø+¨JÕ7ÐØ*¨:Õ6‰Oä$Ÿ|š|¨D¼¿
¹
ÔCÀGÕLÐÜ#Ÿlšl¨4´u·z±zÔBÀ7ÕKˆOà+Õ>Ðà×1Ñ1Ð2CÐ1Ó]ˆ	Ø—L‘L §¡Ó*ˆ	Ø×-Ñ-¨iÓ8ˆØ—‘¢	Ó*×4Ñ4°QÓ7ˆØ—‘  R¨Ð4Ó5×@Ñ@ÓBˆØˆr0   c                ó`  € ^ pV ) pV^,          pW4^ 8  P                  \        P                  4      V,          ,          p\        P                  ! V4      pV^,          pWE8  pV\        P                  ! VP                  4       V,          4      \        P                  ! W%,          4      ,          W,
          ,          P                  \        P                  4      ,           p\        P                  ! V\        P                  ! Wq^,
          4      4      pV\        P                  ! WdV4      ,          pV# )rp   )
rò   r'   ri   ÚabsÚlogÚfloatr‡   ÚminÚ	full_likeÚwhere)rù   rð   Úmax_distanceÚretÚnÚ	max_exactÚis_smallÚval_if_larges   &&&     r.   rñ   Ú%MPNetEncoder.relative_position_bucketJ  s×   € àˆØÐˆà˜ÕˆØ�A‘�z‰zœ%Ÿ*™*Ó%¨Õ3Õ3ˆÜ�IŠI�a‹Lˆà 1Õ$ˆ	Ø‘=ˆà Ü�IŠI�a—g‘g“i )Õ+Ó,¬t¯xªx¸Õ8PÓ/QÕQÐU`ÕUlÕmß
‰"ŒU�Z‰Z‹.õˆô —y’y ¬u¯ª¸|Ð[\Í_Ó/]Ó^ˆØŒu�{Š{˜8¨Ó5Õ5ˆØˆ
r0   )r3   rÕ   rÑ   r×   )NFFF)Né    )r  é€   )r9   r:   r;   r<   rJ   rd   ræ   Ústaticmethodrñ   r@   rA   rB   rC   s   @@r.   rÏ   rÏ     s4   ù‡ € õi÷&
ò &
ôPð$ óó ÷ð r0   rÏ   c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚMPNetPooleri`  c                ó¸   <€ \         SV `  4        \        P                  ! VP                  VP                  4      V n        \        P                  ! 4       V n        R # r¨   )r   rJ   r   ry   rM   rª   ÚTanhÚ
activationrW   s   &&€r.   rJ   ÚMPNetPooler.__init__a  s9   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ'š'›)ˆŽr0   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r°   r²   )r4   r5   s   "€r.   r6   ÚMPNetPooler.__annotate__f  s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r0   c                ó\   € VR,          pV P                  V4      pV P                  V4      pV# )rZ   )rZ   rp   )rª   r  )r+   r�   Úfirst_token_tensorÚpooled_outputs   &&  r.   rd   ÚMPNetPooler.forwardf  s2   € ð +¨4Õ0ÐØŸ
™
Ð#5Ó6ˆØŸ™¨Ó6ˆØÐr0   )r  rª   r™   rC   s   @@r.   r  r  `  s   ù‡ € õ$÷
÷ ð r0   r  c                   ób   a a€ ] tR tRt oRV 3R lltR tR t]R	V3R lR ll4       tRt	Vt
V ;t# )
Ú
MPNetModelio  c                óÆ   <€ \         SV `  V4       Wn        \        V4      V n        \        V4      V n        V'       d   \        V4      MRV n        V P                  4        R# )z^
add_pooling_layer (bool, *optional*, defaults to `True`):
    Whether to add a pooling layer
N)
r   rJ   r3   r$   rc   rÏ   Úencoderr  ÚpoolerÚ	post_init)r+   r3   Úadd_pooling_layerr-   s   &&&€r.   rJ   ÚMPNetModel.__init__q  sK   ø€ ô
 	‰Ñ˜Ô ØŒä)¨&Ó1ˆŒÜ# FÓ+ˆŒß->”k &Ô)ÀDˆŒð 	�‰Ör0   c                ó.   € V P                   P                  # r¨   ©rc   rN   ©r+   s   &r.   Úget_input_embeddingsÚMPNetModel.get_input_embeddings€  s   € Ø�‰×.Ñ.Ð.r0   c                ó&   € WP                   n        R # r¨   r#  )r+   Úvalues   &&r.   Úset_input_embeddingsÚMPNetModel.set_input_embeddingsƒ  s   € Ø*/�‰Ö'r0   c                ó0  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[R,          R	S[S[ P
                  ,          S[,          /# )
r2   r^   NrŽ   r&   r_   r�   rÛ   rÜ   r±   )r'   Ú
LongTensorÚFloatTensorrÝ   rè   r³   r   )r4   r5   s   "€r.   r6   ÚMPNetModel.__annotate__‡  s¢   ø€ ÷ 0
ñ 0
á×#Ñ# dÕ*ð0
ñ ×)Ñ)¨DÕ0ð0
ñ ×&Ñ&¨Õ-ð	0
ñ
 ×(Ñ(¨4Õ/ð0
ñ   $�;ð0
ñ # T�kð0
ñ ˜D•[ð0
ñ 
‰u�|‰|Õ	Ñ9Õ	9ñ0
r0   c                óD  € Ve   TMV P                   P                  pVe   TMV P                   P                  pVe   TMV P                   P                  pVe   Ve   \	        R4      hVf   Vf   \	        R4      hV P                  WVR7      p	\        V P                   V	VR7      pV P                  V	VVVVR7      p
V
^ ,          pV P                  e   V P                  V4      MR pV'       g   W¼3V
R,          ,           # \        VVV
P                  V
P                  R7      # )NzDYou cannot specify both input_ids and inputs_embeds at the same timez5You have to specify either input_ids or inputs_embeds)r^   r&   r_   )r3   r_   rŽ   )rŽ   r�   rÛ   rÜ   r¡   )rä   Úpooler_outputr�   rå   )r3   r�   rÛ   rÜ   ru   rc   r	   r  r  r   r�   rå   )r+   r^   rŽ   r&   r_   r�   rÛ   rÜ   r`   Úembedding_outputÚencoder_outputsÚsequence_outputr  s   &&&&&&&&,    r.   rd   ÚMPNetModel.forward†  s0  € ð 2CÒ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ò$DÑ È$Ï+É+×JjÑJjð 	ð &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆàÒ  ]Ò%>ÜÐcÓdÐdØÒ =Ò#8ÜÐTÓUÐUàŸ?™?°YÐiv˜?ÓwÐä2Ø—;‘;Ø*Ø)ô
ˆð Ÿ,™,ØØ)Ø/Ø!5Ø#ð 'ó 
ˆð *¨!Õ,ˆØ8<¿¹Ò8O˜Ÿ™ OÔ4ÐUYˆçØ#Ð3°oÀbÕ6IÕIÐIä)Ø-Ø'Ø)×7Ñ7Ø&×1Ñ1ô	
ð 	
r0   )r3   rc   r  r  )T)NNNNNNN)r9   r:   r;   r<   rJ   r%  r)  r   rd   r@   rA   rB   rC   s   @@r.   r  r  o  s2   ù‡ € ÷ò/ò0ð ÷0
ñ 0
ó ÷0
ð 0
r0   r  c                   ój   a a€ ] tR tRt oRRRR/tV 3R ltR tR t]RV3R	 lR
 ll4       t	Rt
VtV ;t# )ÚMPNetForMaskedLMiº  zlm_head.decoder.weightz'mpnet.embeddings.word_embeddings.weightzlm_head.decoder.biaszlm_head.biasc                óŠ   <€ \         SV `  V4       \        VR R7      V n        \	        V4      V n        V P                  4        R# ©F)r   N)r   rJ   r  r   r    Úlm_headr  rW   s   &&€r.   rJ   ÚMPNetForMaskedLM.__init__À  s6   ø€ Ü‰Ñ˜Ô ä ¸%Ô@ˆŒ
Ü" 6Ó*ˆŒð 	�‰Ör0   c                ó.   € V P                   P                  # r¨   )r9  Údecoderr$  s   &r.   Úget_output_embeddingsÚ&MPNetForMaskedLM.get_output_embeddingsÉ  s   € Ø�|‰|×#Ñ#Ð#r0   c                ó\   € WP                   n        VP                  V P                   n        R # r¨   )r9  r<  r#   )r+   Únew_embeddingss   &&r.   Úset_output_embeddingsÚ&MPNetForMaskedLM.set_output_embeddingsÌ  s   € Ø-�‰ÔØ*×/Ñ/ˆ�‰Ör0   c                óX  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          R	S[R,          R
S[S[ P
                  ,          S[,          /	# ©r2   r^   NrŽ   r&   r_   Úlabelsr�   rÛ   rÜ   r±   )r'   r,  r-  rÝ   rè   r³   r   )r4   r5   s   "€r.   r6   ÚMPNetForMaskedLM.__annotate__Ñ  s¶   ø€ ÷ /
ñ /
á×#Ñ# dÕ*ð/
ñ ×)Ñ)¨DÕ0ð/
ñ ×&Ñ&¨Õ-ð	/
ñ
 ×(Ñ(¨4Õ/ð/
ñ × Ñ  4Õ'ð/
ñ   $�;ð/
ñ # T�kð/
ñ ˜D•[ð/
ñ 
‰u�|‰|Õ	™~Õ	-ñ/
r0   c	           
     óÎ  € Ve   TMV P                   P                  pV P                  VVVVVVVR7      p
V
^ ,          pV P                  V4      pRpVeG   \	        4       pV! VP                  RV P                   P                  4      VP                  R4      4      pV'       g"   V3V
R,          ,           pVe   V3V,           # T# \        VVV
P                  V
P                  R7      # )az  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
    config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
    loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
N©rŽ   r&   r_   r�   rÛ   rÜ   ºr2   NN©ÚlossÚlogitsr�   rå   r   )
r3   rÜ   r   r9  r   r„   rL   r   r�   rå   )r+   r^   rŽ   r&   r_   rE  r�   rÛ   rÜ   r`   r–   r3  Úprediction_scoresÚmasked_lm_lossÚloss_fctrÆ   s   &&&&&&&&&,      r.   rd   ÚMPNetForMaskedLM.forwardÐ  sò   € ð& &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆà—*‘*ØØ)Ø%Ø'Ø/Ø!5Ø#ð ó 
ˆð " !�*ˆØ ŸL™L¨Ó9ÐàˆØÒÜ'Ó)ˆHÙ%Ð&7×&<Ñ&<¸RÀÇÁ×AWÑAWÓ&XÐZ`×ZeÑZeÐfhÓZiÓjˆNçØ'Ð)¨G°B­KÕ7ˆFØ3AÒ3M�^Ð%¨Õ.ÐYÐSYÐYäØØ$Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r0   )r9  r   ©NNNNNNNN)r9   r:   r;   r<   Ú_tied_weights_keysrJ   r=  rA  r   rd   r@   rA   rB   rC   s   @@r.   r6  r6  º  sE   ù‡ € à Ð"KØ ðÐõ
ò$ò0ð ÷/
ñ /
ó ÷/
ð /
r0   r6  c                   ó<   a a€ ] tR tRt oRtV 3R ltR tRtVtV ;t	# )r    i  z5MPNet Head for masked and permuted language modeling.c                ó¼  <€ \         SV `  4        \        P                  ! VP                  VP                  4      V n        \        P                  ! VP                  VP                  R 7      V n        \        P                  ! VP                  VP                  RR7      V n
        \        P                  ! \        P                  ! VP                  4      4      V n        R# )rG   T)r#   N)r   rJ   r   ry   rM   rª   rQ   rR   Ú
layer_normrL   r<  Ú	Parameterr'   Úzerosr#   rW   s   &&€r.   rJ   ÚMPNetLMHead.__init__  s‰   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ,š, v×'9Ñ'9¸v×?TÑ?TÔUˆŒä—y’y ×!3Ñ!3°V×5FÑ5FÈTÔRˆŒÜ—L’L¤§¢¨V×->Ñ->Ó!?Ó@ˆŽ	r0   c                ó‚   € V P                  V4      p\        V4      pV P                  V4      pV P                  V4      pV# r¨   )rª   r   rU  r<  ©r+   Úfeaturesr`   ró   s   &&, r.   rd   ÚMPNetLMHead.forward  s;   € Ø�J‰J�xÓ ˆÜ�‹GˆØ�O‰O˜AÓˆð �L‰L˜‹Oˆàˆr0   )r#   r<  rª   rU  ©
r9   r:   r;   r<   Ú__doc__rJ   rd   r@   rA   rB   rC   s   @@r.   r    r      s   ù‡ € Ù?õA÷ò r0   r    z�
    MPNet Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled
    output) e.g. for GLUE tasks.
    )Úcustom_introc                   óR   a a€ ] tR tRt oV 3R lt]RV3R lR ll4       tRtVtV ;t	# )ÚMPNetForSequenceClassificationi  c                ó¬   <€ \         SV `  V4       VP                  V n        \        VR R7      V n        \        V4      V n        V P                  4        R# r8  )r   rJ   Ú
num_labelsr  r   ÚMPNetClassificationHeadÚ
classifierr  rW   s   &&€r.   rJ   Ú'MPNetForSequenceClassification.__init__   sC   ø€ Ü‰Ñ˜Ô à ×+Ñ+ˆŒÜ ¸%Ô@ˆŒ
Ü1°&Ó9ˆŒð 	�‰Ör0   c                óX  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          R	S[R,          R
S[S[ P
                  ,          S[,          /	# rD  )r'   r,  r-  rÝ   rè   r³   r   )r4   r5   s   "€r.   r6   Ú+MPNetForSequenceClassification.__annotate__+  sÂ   ø€ ÷ @
ñ @
á×#Ñ# dÕ*ð@
ñ ×)Ñ)¨DÕ0ð@
ñ ×&Ñ&¨Õ-ð	@
ñ
 ×(Ñ(¨4Õ/ð@
ñ × Ñ  4Õ'ð@
ñ   $�;ð@
ñ # T�kð@
ñ ˜D•[ð@
ñ 
‰u�|‰|Õ	Ñ7Õ	7ñ@
r0   c	           
     ór  € Ve   TMV P                   P                  pV P                  VVVVVVVR7      p
V
^ ,          pV P                  V4      pRpVEe˜   V P                   P                  f–   V P
                  ^8X  d   RV P                   n        MrV P
                  ^8”  dQ   VP                  \        P                  8X  g    VP                  \        P                  8X  d   RV P                   n        MRV P                   n        V P                   P                  R8X  dK   \        4       pV P
                  ^8X  d'   V! VP                  4       VP                  4       4      pMŽV! WÅ4      pM…V P                   P                  R8X  d>   \        4       pV! VP                  RV P
                  4      VP                  R4      4      pM-V P                   P                  R8X  d   \        4       pV! WÅ4      pV'       g"   V3V
R,          ,           pVe   V3V,           # T# \        VVV
P                   V
P"                  R7      # )	ae  
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
    `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
NrH  Ú
regressionÚsingle_label_classificationÚmulti_label_classificationrI  rJ  r   )r3   rÜ   r   re  Úproblem_typerc  rg   r'   ri   rv   r   Úsqueezer   r„   r   r   r�   rå   ©r+   r^   rŽ   r&   r_   rE  r�   rÛ   rÜ   r`   r–   r3  rL  rK  rO  rÆ   s   &&&&&&&&&,      r.   rd   Ú&MPNetForSequenceClassification.forward*  sÎ  € ð( &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆà—*‘*ØØ)Ø%Ø'Ø/Ø!5Ø#ð ó 
ˆð " !�*ˆØ—‘ Ó1ˆàˆØÓØ�{‰{×'Ñ'Ò/Ø—?‘? aÔ'Ø/;�D—K‘KÕ,Ø—_‘_ qÔ(¨f¯l©l¼e¿j¹jÔ.HÈFÏLÉLÔ\a×\eÑ\eÔLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ô7Ü"›9�Ø—?‘? aÔ'Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# FÓ3‘DØ—‘×)Ñ)Ð-JÔJÜ+Ó-�Ù §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR‘Ø—‘×)Ñ)Ð-IÔIÜ,Ó.�Ù Ó/�ßØ�Y ¨¥Õ,ˆFØ)-Ò)9�T�G˜fÕ$ÐE¸vÐEä'ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r0   )re  r   rc  rQ  ©
r9   r:   r;   r<   rJ   r   rd   r@   rA   rB   rC   s   @@r.   ra  ra    s,   ù‡ € õð ÷@
ñ @
ó ÷@
ð @
r0   ra  c                   óR   a a€ ] tR tRt oV 3R lt]RV3R lR ll4       tRtVtV ;t	# )ÚMPNetForMultipleChoicein  c                óü   <€ \         SV `  V4       \        V4      V n        \        P
                  ! VP                  4      V n        \        P                  ! VP                  ^4      V n
        V P                  4        R# )r   N)r   rJ   r  r   r   rS   rT   rU   ry   rM   re  r  rW   s   &&€r.   rJ   ÚMPNetForMultipleChoice.__init__p  sV   ø€ Ü‰Ñ˜Ô ä Ó'ˆŒ
Ü—z’z &×"<Ñ"<Ó=ˆŒÜŸ)š) F×$6Ñ$6¸Ó:ˆŒð 	�‰Ör0   c                óX  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          R	S[R,          R
S[S[ P
                  ,          S[,          /	# rD  )r'   r,  r-  rÝ   rè   r³   r   )r4   r5   s   "€r.   r6   Ú#MPNetForMultipleChoice.__annotate__{  sÂ   ø€ ÷ L
ñ L
á×#Ñ# dÕ*ðL
ñ ×)Ñ)¨DÕ0ðL
ñ ×&Ñ&¨Õ-ð	L
ñ
 ×(Ñ(¨4Õ/ðL
ñ × Ñ  4Õ'ðL
ñ   $�;ðL
ñ # T�kðL
ñ ˜D•[ðL
ñ 
‰u�|‰|Õ	Ñ8Õ	8ñL
r0   c	           
     óZ  € Ve   TMV P                   P                  pVe   VP                  ^,          MVP                  ^,          p
Ve"   VP                  RVP	                  R4      4      MRpVe"   VP                  RVP	                  R4      4      MRpVe"   VP                  RVP	                  R4      4      MRpVe2   VP                  RVP	                  R4      VP	                  R4      4      MRpV P                  VVVVVVVR7      pV^,          pV P                  V4      pV P                  V4      pVP                  RV
4      pRpVe   \        4       pV! VV4      pV'       g"   V3VR,          ,           pVe   V3V,           # T# \        VVVP                  VP                  R7      # )a(  
input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
    Indices of input sequence tokens in the vocabulary.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are input IDs?](../glossary#input-ids)
position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
    Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
    config.max_position_embeddings - 1]`.

    [What are position IDs?](../glossary#position-ids)
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):
    Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
    is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
    model's internal embedding lookup matrix.
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
    num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
    `input_ids` above)
N)r&   rŽ   r_   r�   rÛ   rÜ   rI  rJ  r   rƒ   )r3   rÜ   r)   r„   r]   r   rU   re  r   r   r�   rå   )r+   r^   rŽ   r&   r_   rE  r�   rÛ   rÜ   r`   Únum_choicesÚflat_input_idsÚflat_position_idsÚflat_attention_maskÚflat_inputs_embedsr–   r  rL  Úreshaped_logitsrK  rO  rÆ   s   &&&&&&&&&,            r.   rd   ÚMPNetForMultipleChoice.forwardz  s³  € ðH &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆØ,5Ò,A�i—o‘o aÖ(À}×GZÑGZÐ[\ÕG]ˆàCLÒCX˜Ÿ™¨¨I¯N©N¸2Ó,>Ô?Ð^bˆØLXÒLd˜L×-Ñ-¨b°,×2CÑ2CÀBÓ2GÔHÐjnÐØR`ÒRl˜n×1Ñ1°"°n×6IÑ6IÈ"Ó6MÔNÐrvÐð Ò(ð ×Ñ˜r =×#5Ñ#5°bÓ#9¸=×;MÑ;MÈbÓ;QÔRàð 	ð —*‘*ØØ*Ø.Ø,Ø/Ø!5Ø#ð ó 
ˆð   �
ˆàŸ™ ]Ó3ˆØ—‘ Ó/ˆØ Ÿ+™+ b¨+Ó6ˆàˆØÒÜ'Ó)ˆHÙ˜O¨VÓ4ˆDçØ%Ð'¨'°"­+Õ5ˆFØ)-Ò)9�T�G˜fÕ$ÐE¸vÐEä(ØØ"Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r0   )re  rU   r   rQ  rq  rC   s   @@r.   rs  rs  n  s,   ù‡ € õð ÷L
ñ L
ó ÷L
ð L
r0   rs  c                   óR   a a€ ] tR tRt oV 3R lt]RV3R lR ll4       tRtVtV ;t	# )ÚMPNetForTokenClassificationiÊ  c                ó6  <€ \         SV `  V4       VP                  V n        \        VR R7      V n        \
        P                  ! VP                  4      V n        \
        P                  ! VP                  VP                  4      V n        V P                  4        R# r8  )r   rJ   rc  r  r   r   rS   rT   rU   ry   rM   re  r  rW   s   &&€r.   rJ   Ú$MPNetForTokenClassification.__init__Ì  sk   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä ¸%Ô@ˆŒ
Ü—z’z &×"<Ñ"<Ó=ˆŒÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Ör0   c                óX  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          R	S[R,          R
S[S[ P
                  ,          S[,          /	# rD  )r'   r,  r-  rÝ   rè   r³   r   )r4   r5   s   "€r.   r6   Ú(MPNetForTokenClassification.__annotate__Ø  s·   ø€ ÷ 0
ñ 0
á×#Ñ# dÕ*ð0
ñ ×)Ñ)¨DÕ0ð0
ñ ×&Ñ&¨Õ-ð	0
ñ
 ×(Ñ(¨4Õ/ð0
ñ × Ñ  4Õ'ð0
ñ   $�;ð0
ñ # T�kð0
ñ ˜D•[ð0
ñ 
‰u�|‰|Õ	Ñ4Õ	4ñ0
r0   c	           
     óÜ  € Ve   TMV P                   P                  pV P                  VVVVVVVR7      p
V
^ ,          pV P                  V4      pV P	                  V4      pRpVe=   \        4       pV! VP                  RV P                  4      VP                  R4      4      pV'       g"   V3V
R,          ,           pVe   V3V,           # T# \        VVV
P                  V
P                  R7      # )zÃ
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
NrH  rI  rJ  r   )r3   rÜ   r   rU   re  r   r„   rc  r   r�   rå   ro  s   &&&&&&&&&,      r.   rd   Ú#MPNetForTokenClassification.forward×  sî   € ð$ &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆà—*‘*ØØ)Ø%Ø'Ø/Ø!5Ø#ð ó 
ˆð " !�*ˆàŸ,™, Ó7ˆØ—‘ Ó1ˆàˆØÒÜ'Ó)ˆHÙ˜FŸK™K¨¨D¯O©OÓ<¸f¿k¹kÈ"»oÓNˆDçØ�Y ¨¥Õ,ˆFØ)-Ò)9�T�G˜fÕ$ÐE¸vÐEä$ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r0   )re  rU   r   rc  rQ  rq  rC   s   @@r.   r�  r�  Ê  s(   ù‡ € õ	ð ÷0
ñ 0
ó ÷0
ð 0
r0   r�  c                   ó<   a a€ ] tR tRt oRtV 3R ltR tRtVtV ;t	# )rd  i  z-Head for sentence-level classification tasks.c                ó.  <€ \         SV `  4        \        P                  ! VP                  VP                  4      V n        \        P                  ! VP                  4      V n        \        P                  ! VP                  VP                  4      V n
        R # r¨   )r   rJ   r   ry   rM   rª   rS   rT   rU   rc  Úout_projrW   s   &&€r.   rJ   Ú MPNetClassificationHead.__init__  s`   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
Ü—z’z &×"<Ñ"<Ó=ˆŒÜŸ	š	 &×"4Ñ"4°f×6GÑ6GÓHˆŽr0   c                óÌ   € VR,          pV P                  V4      pV P                  V4      p\        P                  ! V4      pV P                  V4      pV P	                  V4      pV# )rZ   )rZ   rp   rZ   )rU   rª   r'   ÚtanhrŠ  rZ  s   &&, r.   rd   ÚMPNetClassificationHead.forward  sS   € Ø�WÕˆØ�L‰L˜‹OˆØ�J‰J�q‹MˆÜ�JŠJ�q‹MˆØ�L‰L˜‹OˆØ�M‰M˜!ÓˆØˆr0   )rª   rU   rŠ  r]  rC   s   @@r.   rd  rd    s   ù‡ € Ù7õI÷ò r0   rd  c                   óR   a a€ ] tR tRt oV 3R lt]RV3R lR ll4       tRtVtV ;t	# )ÚMPNetForQuestionAnsweringi  c                óì   <€ \         SV `  V4       VP                  V n        \        VR R7      V n        \
        P                  ! VP                  VP                  4      V n        V P                  4        R# r8  )
r   rJ   rc  r  r   r   ry   rM   Ú
qa_outputsr  rW   s   &&€r.   rJ   Ú"MPNetForQuestionAnswering.__init__   sU   ø€ Ü‰Ñ˜Ô à ×+Ñ+ˆŒÜ ¸%Ô@ˆŒ
ÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Ör0   c                ó€  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[R,          R
S[R,          RS[S[ P
                  ,          S[,          /
# )r2   r^   NrŽ   r&   r_   Ústart_positionsÚend_positionsr�   rÛ   rÜ   r±   )r'   r,  r-  rÝ   rè   r³   r   )r4   r5   s   "€r.   r6   Ú&MPNetForQuestionAnswering.__annotate__+  sÌ   ø€ ÷ ;
ñ ;
á×#Ñ# dÕ*ð;
ñ ×)Ñ)¨DÕ0ð;
ñ ×&Ñ&¨Õ-ð	;
ñ
 ×(Ñ(¨4Õ/ð;
ñ ×)Ñ)¨DÕ0ð;
ñ ×'Ñ'¨$Õ.ð;
ñ   $�;ð;
ñ # T�kð;
ñ ˜D•[ð;
ñ 
‰u�|‰|Õ	Ñ;Õ	;ñ;
r0   c
           
     óp  € V	e   T	MV P                   P                  p	V P                  VVVVVVV	R7      pV^ ,          pV P                  V4      pVP	                  ^RR7      w  rïVP                  R4      P                  4       pVP                  R4      P                  4       pR pVeÄ   VeÀ   \        VP                  4       4      ^8”  d   VP                  R4      p\        VP                  4       4      ^8”  d   VP                  R4      pVP                  ^4      pVP                  ^ V4      pVP                  ^ V4      p\        VR7      pV! Wå4      pV! Wö4      pVV,           ^,          pV	'       g"   Wï3VR,          ,           pVe   V3V,           # T# \        VVVVP                  VP                  R7      # )NrH  r�   )Úignore_indexrI  )rK  Ústart_logitsÚ
end_logitsr�   rå   r   )r3   rÜ   r   r’  Úsplitrn  rŒ   Úlenr]   Úclampr   r   r�   rå   )r+   r^   rŽ   r&   r_   r•  r–  r�   rÛ   rÜ   r`   r–   r3  rL  rš  r›  Ú
total_lossÚignored_indexrO  Ú
start_lossÚend_lossrÆ   s   &&&&&&&&&&,           r.   rd   Ú!MPNetForQuestionAnswering.forward*  s²  € ð &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆà—*‘*ØØ)Ø%Ø'Ø/Ø!5Ø#ð ó 
ˆð " !�*ˆà—‘ Ó1ˆØ#)§<¡<°°r <Ó#:Ñ ˆØ#×+Ñ+¨BÓ/×:Ñ:Ó<ˆØ×'Ñ'¨Ó+×6Ñ6Ó8ˆ
àˆ
ØÒ&¨=Ò+Dä�?×'Ñ'Ó)Ó*¨QÔ.Ø"1×"9Ñ"9¸"Ó"=�Ü�=×%Ñ%Ó'Ó(¨1Ô,Ø -× 5Ñ 5°bÓ 9�à(×-Ñ-¨aÓ0ˆMØ-×3Ñ3°A°}ÓEˆOØ)×/Ñ/°°=ÓAˆMä'°]ÔCˆHÙ! ,Ó@ˆJÙ 
Ó:ˆHØ$ xÕ/°1Õ4ˆJçØ"Ð/°'¸"µ+Õ=ˆFØ/9Ò/E�Z�M FÕ*ÐQÈ6ÐQä+ØØ%Ø!Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r0   )r   rc  r’  )	NNNNNNNNNrq  rC   s   @@r.   r�  r�    s(   ù‡ € õð ÷;
ñ ;
ó ÷;
ð ;
r0   r�  c                óÊ   € V P                  V4      P                  4       p\        P                  ! V^R7      P	                  V4      V,          pVP                  4       V,           # )zå
Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols
are ignored. This is modified from fairseq's `utils.make_positions`. :param torch.Tensor x: :return torch.Tensor:
r�   )Únerv   r'   ÚcumsumÚtype_asri   )r^   rF   ÚmaskÚincremental_indicess   &&  r.   r[   r[   i  sP   € ð �<‰<˜Ó$×(Ñ(Ó*€DÜŸ,š, t°Ô3×;Ñ;¸DÓAÀDÕHÐØ×#Ñ#Ó%¨Õ3Ð3r0   )r6  rs  r�  ra  r�  rÂ   r  r   )6r^  r‡   r'   r   Útorch.nnr   r   r   Ú r   r!   Úactivationsr   r   Úmasking_utilsr	   Úmodeling_outputsr
   r   r   r   r   r   r   Úmodeling_utilsr   Úutilsr   r   Úconfiguration_mpnetr   Ú
get_loggerr9   Úloggerr   ÚModuler$   rn   r›   r¦   r¹   rÂ   rÏ   r  r  r6  r    ra  rs  r�  rd  r�  r[   Ú__all__r8   r0   r.   Ú<module>r¶     sÔ  ðñ ã ã Ý ß AÑ Aå &ß 'Ý 6÷÷ ñ õ .ß ,Ý ,ð 
×	Ò	˜HÓ	%€ð ôi˜?ó ió ðiô6=�b—i‘iô 6=ôr;˜Ÿ™ô ;ô|�R—Y‘Yô ô6˜Ÿ	™	ô ô �"—)‘)ô ô�—‘ô ô<T�2—9‘9ô Tôp�"—)‘)ô ð ôG
Ð%ó G
ó ðG
ôTF
Ð+ô F
ôR�"—)‘)ô ñ, ðôôL
Ð%9ó L
óðL
ð^ ôX
Ð1ó X
ó ðX
ðv ô=
Ð"6ó =
ó ð=
ô@˜bŸi™iô ð& ôG
Ð 4ó G
ó ðG
òT4ò	‚r0   