+
    œ°³jNô  ã                   óŒ  € ^ RI Ht ^ RIHt ^ RIt^ RIHt ^ RIHtHtH	t	 ^RI
Ht ^RIHt ^RIHtHtHt ^RIHt ^R	IHtHt ^R
IHt ^RIHtHtHtHtHtH t H!t!H"t"H#t# ^RI$H%t%H&t& ^RI'H(t( ^RI)H*t* ^RI+H,t,H-t-H.t.H/t/ ^RI0H1t1H2t2 ^RI3H4t4 ^RI5H6t6 ]/Pn                  ! ]84      t9 ! R R]Pt                  4      t;RPR R llt< ! R R]Pt                  4      t= ! R R]Pt                  4      t> ! R R]Pt                  4      t? ! R R]Pt                  4      t@ ! R R ]Pt                  4      tA ! R! R"]Pt                  4      tB ! R# R$]4      tC ! R% R&]Pt                  4      tD ! R' R(]Pt                  4      tE ! R) R*]Pt                  4      tF ! R+ R,]Pt                  4      tG]. ! R- R.]&4      4       tH].! R/R07       ! R1 R2]H4      4       tI].! R3R07      ] ! R4 R5],4      4       4       tJ ! R6 R7]Pt                  4      tK].! R8R07       ! R9 R:]H4      4       tL ! R; R<]Pt                  4      tM].! R=R07       ! R> R?]H]4      4       tN]. ! R@ RA]H4      4       tO ! RB RC]Pt                  4      tP].! RDR07       ! RE RF]H4      4       tQ].! RGR07       ! RH RI]H4      4       tR]. ! RJ RK]H4      4       tS]. ! RL RM]H4      4       tT]. ! RN RO]H4      4       tU. RQOtVR# )Ré    )ÚCallable)Ú	dataclassN)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELoss)Úinitialization)ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚGradientCheckpointingLayer)	Ú)BaseModelOutputWithPastAndCrossAttentionsÚ,BaseModelOutputWithPoolingAndCrossAttentionsÚ!CausalLMOutputWithCrossAttentionsÚMaskedLMOutputÚMultipleChoiceModelOutputÚNextSentencePredictorOutputÚQuestionAnsweringModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)Úapply_chunking_to_forward)ÚModelOutputÚTransformersKwargsÚauto_docstringÚlogging)Úcan_return_tupleÚmerge_with_config_defaults)Úcapture_outputs)ÚErnieConfigc                   óL   a a€ ] tR t^9t oRtV 3R ltRV3R lR lltRtVtV ;t	# )ÚErnieEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c                óø  <€ \         SV `  4        \        P                  ! VP                  VP
                  VP                  R 7      V n        \        P                  ! VP                  VP
                  4      V n	        \        P                  ! VP                  VP
                  4      V n        \        P                  ! VP
                  VP                  R7      V n        \        P                  ! VP                  4      V n        \        P"                  ! \$        P&                  ! VP                  4      P)                  R4      RR7      V n        \        P"                  ! \$        P,                  ! V P*                  P/                  4       \$        P0                  R7      RR7      V n        VP4                  V n        VP4                  '       d3   \        P                  ! VP6                  VP
                  4      V n        R# R# ))Úpadding_idx©ÚepsF)Ú
persistent)ÚdtypeN©é   éÿÿÿÿ)ÚsuperÚ__init__ÚnnÚ	EmbeddingÚ
vocab_sizeÚhidden_sizeÚpad_token_idÚword_embeddingsÚmax_position_embeddingsÚposition_embeddingsÚtype_vocab_sizeÚtoken_type_embeddingsÚ	LayerNormÚlayer_norm_epsÚDropoutÚhidden_dropout_probÚdropoutÚBufferÚtorchÚarangeÚexpandÚposition_idsÚzerosÚsizeÚlongÚtoken_type_idsÚuse_task_idÚtask_type_vocab_sizeÚtask_type_embeddings©ÚselfÚconfigÚ	__class__s   &&€Úi/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/ernie/modeling_ernie.pyr2   ÚErnieEmbeddings.__init__<   sL  ø€ Ü‰ÑÔÜ!Ÿ|š|¨F×,=Ñ,=¸v×?QÑ?QÐ_e×_rÑ_rÔsˆÔÜ#%§<¢<°×0NÑ0NÐPV×PbÑPbÓ#cˆÔ Ü%'§\¢\°&×2HÑ2HÈ&×J\ÑJ\Ó%]ˆÔ"äŸš f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z’z &×"<Ñ"<Ó=ˆŒäŸIšI¤e§l¢l°6×3QÑ3QÓ&R×&YÑ&YÐZaÓ&bÐotÔuˆÔÜ Ÿiši¬¯ª°D×4EÑ4E×4JÑ4JÓ4LÔTY×T^ÑT^Ô(_ÐlqÔrˆÔà!×-Ñ-ˆÔØ××ÐÜ(*¯ª°V×5PÑ5PÐRX×RdÑRdÓ(eˆDÖ%ñ ó    c                ó  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[RS[ P                  /# )	é   Ú	input_idsNrJ   Útask_type_idsrF   Úinputs_embedsÚpast_key_values_lengthÚreturn)rC   Ú
LongTensorÚFloatTensorÚintÚTensor)ÚformatÚ__classdict__s   "€rR   Ú__annotate__ÚErnieEmbeddings.__annotate__L   sŒ   ø€ ÷ 3ñ 3á×#Ñ# dÕ*ð3ñ ×(Ñ(¨4Õ/ð3ñ ×'Ñ'¨$Õ.ð	3ñ
 ×&Ñ&¨Õ-ð3ñ ×(Ñ(¨4Õ/ð3ñ !$ð3ñ 
�‰ñ3rT   c                óÎ  € Ve   VP                  4       pMVP                  4       R R pVw  r‰Vf   V P                  RWiV,           13,          pVf¦   \        V R4      '       dY   V P                  P	                  VP
                  ^ ,          R4      p
\        P                  ! V
^VR7      p
V
P	                  W‰4      pM;\        P                  ! V\        P                  V P                  P                  R7      pVf   V P                  V4      pV P                  V4      pVP                  VP                  4      pW[,           pV P                  V4      pWÍ,           pV P                  '       dY   Vf<   \        P                  ! V\        P                  V P                  P                  R7      pV P!                  V4      pWÎ,          pV P#                  V4      pV P%                  V4      pV# )NºNNNrJ   )ÚdimÚindex)r-   Údevicer0   )rH   rF   ÚhasattrrJ   rE   ÚshaperC   ÚgatherrG   rI   rh   r8   r<   Útor:   rK   rM   r=   rA   )rO   rW   rJ   rX   rF   rY   rZ   Úinput_shapeÚ
batch_sizeÚ
seq_lengthÚbuffered_token_type_idsr<   Ú
embeddingsr:   rM   s   &&&&&&&        rR   ÚforwardÚErnieEmbeddings.forwardL   s°  € ð Ò Ø#Ÿ.™.Ó*‰Kà'×,Ñ,Ó.¨s°Ð3ˆKà!,Ñˆ
àÒØ×,Ñ,¨QÐ0FÐVlÕIlÐ0lÐ-lÕmˆLð
 Ò!Ü�tÐ-×.Ò.à*.×*=Ñ*=×*DÑ*DÀ\×EWÑEWÐXYÕEZÐ\^Ó*_Ð'Ü*/¯,ª,Ð7NÐTUÐ]iÔ*jÐ'Ø!8×!?Ñ!?À
Ó!W‘ä!&§¢¨[ÄÇ
Á
ÐSW×SdÑSd×SkÑSkÔ!l�àÒ Ø ×0Ñ0°Ó;ˆMØ $× :Ñ :¸>Ó JÐð &×(Ñ(Ð)>×)EÑ)EÓFˆØ"Õ:ˆ
à"×6Ñ6°|ÓDÐØÕ5ˆ
ð ××ÐØÒ$Ü %§¢¨K¼u¿z¹zÐRV×RcÑRc×RjÑRjÔ k�Ø#'×#<Ñ#<¸]Ó#KÐ ØÕ.ˆJà—^‘^ JÓ/ˆ
Ø—\‘\ *Ó-ˆ
ØÐrT   )	r=   rA   r:   rF   rM   r<   rJ   rK   r8   )NNNNNr   )
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r2   rr   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©rQ   ra   s   @@rR   r'   r'   9   s   ù‡ € ÙQõf÷ 3÷ 3ò 3rT   r'   c                ó  € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R,          R\        R	\
        \        ,          /# )
rV   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNÚscalingrA   Úkwargs)r3   ÚModulerC   r_   Úfloatr   r   )r`   s   "rR   rb   rb   ‚   s{   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô �T�\ð%ô ð%ô Ô'Õ(ñ%rT   c                 ó¶  € Vf   VP                  R4      R,          p\        P                  ! WP                  ^^4      4      V,          pVe	   W„,           p\        P
                  P                  VRR7      p\        P
                  P                  W†V P                  R7      p\        P                  ! Wƒ4      p	V	P                  ^^4      P                  4       p	W˜3# )N©rf   )ÚpÚtrainingr0   ç      à¿)
rH   rC   ÚmatmulÚ	transposer3   Ú
functionalÚsoftmaxrA   rŠ   Ú
contiguous)
r~   r   r€   r�   r‚   rƒ   rA   r„   Úattn_weightsÚattn_outputs
   &&&&&&&,  rR   Úeager_attention_forwardr“   ‚   s±   € ð ‚Ø—*‘*˜R“. DÕ(ˆô —<’< §}¡}°Q¸Ó':Ó;¸gÕE€LàÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2Ð(Ó>€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€Lä—,’,˜|Ó3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$rT   c                   óL   a a€ ] tR t^žt oRV 3R lltRV3R lR lltRtVtV ;t# )ÚErnieSelfAttentionc                ót  <€ \         SV `  4        VP                  VP                  ,          ^ 8w  d9   \	        VR4      '       g'   \        RVP                   RVP                   R24      hWn        VP                  V n        \        VP                  VP                  ,          4      V n        V P                  V P                  ,          V n	        V P                  R,          V n
        \        P                  ! VP                  V P                  4      V n        \        P                  ! VP                  V P                  4      V n        \        P                  ! VP                  V P                  4      V n        \        P                   ! VP"                  4      V n        VP&                  V n        W n        W0n        R# ©r   Úembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads (Ú)Nr‹   )r1   r2   r6   Únum_attention_headsri   Ú
ValueErrorrP   r^   Úattention_head_sizeÚall_head_sizerƒ   r3   ÚLinearr   r€   r�   r?   Úattention_probs_dropout_probrA   Ú
is_decoderÚ	is_causalÚ	layer_idx©rO   rP   r¡   r¢   rQ   s   &&&&€rR   r2   ÚErnieSelfAttention.__init__Ÿ   sM  ø€ Ü‰ÑÔØ×Ñ × :Ñ :Õ:¸aÔ?ÌÐPVÐXh×HiÒHiÜØ# F×$6Ñ$6Ð#7ð 8Ø ×4Ñ4Ð5°Qð8óð ð Œà#)×#=Ñ#=ˆÔ Ü#& v×'9Ñ'9¸F×<VÑ<VÕ'VÓ#WˆÔ Ø!×5Ñ5¸×8PÑ8PÕPˆÔØ×/Ñ/°Õ5ˆŒä—Y’Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
Ü—9’9˜V×/Ñ/°×1CÑ1CÓDˆŒÜ—Y’Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
ä—z’z &×"EÑ"EÓFˆŒà ×+Ñ+ˆŒØ"ŒØ"ŽrT   c                ó°   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  ,          /# )rV   Úhidden_statesr‚   NÚpast_key_valuesr„   r[   ©rC   r_   r]   r
   r   r   Útuple)r`   ra   s   "€rR   rb   ÚErnieSelfAttention.__annotate__·   s_   ø€ ÷ ')ñ ')á—|‘|ð')ñ ×)Ñ)¨DÕ0ð')ñ  �ð	')ñ
 Ñ+Õ,ð')ñ 
‰u�|‰|Õ	ñ')rT   c                ó   € VP                   R R p. VORNV P                  N5pV P                  V4      P                  ! V!  P	                  ^^4      pV P                  V4      P                  ! V!  P	                  ^^4      pV P                  V4      P                  ! V!  P	                  ^^4      p	VeC   Tp
\        V\        4      '       d   VP                  p
V
P                  W‰V P                  4      w  r‰\        P                  ! V P                  P                  \         4      pT! V VVV	V3RV P"                  '       g   RMV P$                  P&                  RV P(                  /VB w  rÍVP*                  ! . VORN5!  P-                  4       pWÍ3# )NrA   ç        rƒ   r0   )rj   rœ   r   Úviewr�   r€   r�   Ú
isinstancer   Úself_attention_cacheÚupdater¢   r   Úget_interfacerP   Ú_attn_implementationr“   rŠ   rA   r‰   rƒ   Úreshaper�   )rO   r¦   r‚   r§   r„   rm   Úhidden_shapeÚquery_layerÚ	key_layerÚvalue_layerÚcurrent_past_key_valuesÚattention_interfacer’   r‘   s   &&&&,         rR   rr   ÚErnieSelfAttention.forward·   s‰  € ð $×)Ñ)¨#¨2Ð.ˆØC˜ÐC bÐC¨$×*BÑ*BÑCˆð —j‘j Ó/×4Ò4°lÑC×MÑMÈaÐQRÓSˆØ—H‘H˜]Ó+×0Ò0°,Ñ?×IÑIÈ!ÈQÓOˆ	Ø—j‘j Ó/×4Ò4°lÑC×MÑMÈaÐQRÓSˆàÒ&à&5Ð#Ü˜/Ô+>×?Ò?Ø*9×*NÑ*NÐ'ð &=×%CÑ%CÀIÐ\`×\jÑ\jÓ%kÑ"ˆIä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð  $Ÿ}Ÿ}˜}‘C°$·,±,·.±.ð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØÐ(Ð(rT   )r�   rœ   rP   rA   r¡   r    r€   r¢   rš   r   rƒ   r�   ©FN)NN©	rt   ru   rv   rw   r2   rr   ry   rz   r{   r|   s   @@rR   r•   r•   ž   s   ù‡ € ÷#÷0')÷ ')ò ')rT   r•   c                   óL   a a€ ] tR t^át oRV 3R lltRV3R lR lltRtVtV ;t# )ÚErnieCrossAttentionc                óR  <€ \         SV `  4        VP                  VP                  ,          ^ 8w  d9   \	        VR4      '       g'   \        RVP                   RVP                   R24      hWn        VP                  V n        \        VP                  VP                  ,          4      V n        V P                  V P                  ,          V n	        V P                  R,          V n
        \        P                  ! VP                  V P                  4      V n        \        P                  ! VP                  V P                  4      V n        \        P                  ! VP                  V P                  4      V n        \        P                   ! VP"                  4      V n        W n        W0n        R# r—   )r1   r2   r6   rš   ri   r›   rP   r^   rœ   r�   rƒ   r3   rž   r   r€   r�   r?   rŸ   rA   r¡   r¢   r£   s   &&&&€rR   r2   ÚErnieCrossAttention.__init__â   s@  ø€ Ü‰ÑÔØ×Ñ × :Ñ :Õ:¸aÔ?ÌÐPVÐXh×HiÒHiÜØ# F×$6Ñ$6Ð#7ð 8Ø ×4Ñ4Ð5°Qð8óð ð Œà#)×#=Ñ#=ˆÔ Ü#& v×'9Ñ'9¸F×<VÑ<VÕ'VÓ#WˆÔ Ø!×5Ñ5¸×8PÑ8PÕPˆÔØ×/Ñ/°Õ5ˆŒä—Y’Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
Ü—9’9˜V×/Ñ/°×1CÑ1CÓDˆŒÜ—Y’Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
ä—z’z &×"EÑ"EÓFˆŒà"ŒØ"ŽrT   c                óØ   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  ,          /# )rV   r¦   Úencoder_hidden_statesNr‚   r§   r„   r[   )rC   r_   r]   r   r   r   r©   )r`   ra   s   "€rR   rb   Ú ErnieCrossAttention.__annotate__ù   su   ø€ ÷ 1)ñ 1)á—|‘|ð1)ñ  %×0Ñ0°4Õ7ð1)ñ ×)Ñ)¨DÕ0ð	1)ñ
 -¨tÕ3ð1)ñ Ñ+Õ,ð1)ñ 
‰u�|‰|Õ	ñ1)rT   c                ó˜  € VP                   R R p. VORNV P                  N5pV P                  V4      P                  V4      P	                  ^^4      pVe&   VP
                  P                  V P                  4      MRp	Vel   V	'       dd   VP                  P                  V P                  ,          P                  p
VP                  P                  V P                  ,          P                  pMÅ. VP                   R R ORNV P                  N5pV P                  V4      P                  V4      P	                  ^^4      p
V P                  V4      P                  V4      P	                  ^^4      pVeB   VP                  P                  W«V P                  4      w  r«RVP
                  V P                  &   \        P                   ! V P"                  P$                  \&        4      pT! V VV
VV3RV P(                  '       g   RMV P*                  P,                  RV P.                  /VB w  rïVP0                  ! . VORN5!  P3                  4       pWï3# )NFTrA   r¬   rƒ   r0   )rj   rœ   r   r­   r�   Ú
is_updatedÚgetr¢   Úcross_attention_cacheÚlayersÚkeysÚvaluesr€   r�   r°   r   r±   rP   r²   r“   rŠ   rA   r‰   rƒ   r³   r�   )rO   r¦   rÂ   r‚   r§   r„   rm   r´   rµ   rÅ   r¶   r·   Úkv_shaper¹   r’   r‘   s   &&&&&,          rR   rr   ÚErnieCrossAttention.forwardù   s  € ð $×)Ñ)¨#¨2Ð.ˆàC˜ÐC bÐC¨$×*BÑ*BÑCˆð —j‘j Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆàGVÒGb�_×/Ñ/×3Ñ3°D·N±NÔCÐhmˆ
ØÒ&¯:à'×=Ñ=×DÑDÀTÇ^Á^ÕT×YÑYˆIØ)×?Ñ?×FÑFÀtÇ~Á~ÕV×]Ñ]‰KàXÐ.×4Ñ4°S°bÐ9ÐX¸2ÐX¸t×?WÑ?WÑXˆHØŸ™Ð!6Ó7×<Ñ<¸XÓF×PÑPÐQRÐTUÓVˆIØŸ*™*Ð%:Ó;×@Ñ@ÀÓJ×TÑTÐUVÐXYÓZˆKàÒ*à)8×)NÑ)N×)UÑ)UØ¨D¯N©Nó*Ñ&�	ð >B�×*Ñ*¨4¯>©>Ñ:ä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð  $Ÿ}Ÿ}˜}‘C°$·,±,·.±.ð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØÐ(Ð(rT   )r�   rœ   rP   rA   r¡   r€   r¢   rš   r   rƒ   r�   r»   )NNNr¼   r|   s   @@rR   r¾   r¾   á   s   ù‡ € ÷#÷.1)÷ 1)ò 1)rT   r¾   c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚErnieSelfOutputi-  c                ó0  <€ \         SV `  4        \        P                  ! VP                  VP                  4      V n        \        P                  ! VP                  VP                  R 7      V n        \        P                  ! VP                  4      V n
        R# ©r*   N)r1   r2   r3   rž   r6   Údenser=   r>   r?   r@   rA   rN   s   &&€rR   r2   ÚErnieSelfOutput.__init__.  s`   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸš f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z’z &×"<Ñ"<Ó=ˆŽrT   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# ©rV   r¦   Úinput_tensorr[   ©rC   r_   )r`   ra   s   "€rR   rb   ÚErnieSelfOutput.__annotate__4  ó1   ø€ ÷ ñ ¡U§\¡\ð ÁÇÁð ÑRW×R^ÑR^ñ rT   c                óx   € V P                  V4      pV P                  V4      pV P                  W,           4      pV# ©N©rÑ   rA   r=   ©rO   r¦   rÕ   s   &&&rR   rr   ÚErnieSelfOutput.forward4  ó5   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØŸ™ }Õ'CÓDˆØÐrT   ©r=   rÑ   rA   r¼   r|   s   @@rR   rÎ   rÎ   -  ó   ù‡ € õ>÷÷ ð rT   rÎ   c                   óL   a a€ ] tR tRt oRV 3R lltRV3R lR lltRtVtV ;t# )ÚErnieAttentioni;  c                ó–   <€ \         SV `  4        W@n        V'       d   \        M\        pV! WVR 7      V n        \        V4      V n        R# )©r¡   r¢   N)r1   r2   Úis_cross_attentionr¾   r•   rO   rÎ   Úoutput)rO   rP   r¡   r¢   rå   Úattention_classrQ   s   &&&&& €rR   r2   ÚErnieAttention.__init__<  s9   ø€ Ü‰ÑÔØ"4Ôß1CÕ-ÔI[ˆÙ# FÈ9ÔUˆŒ	Ü% fÓ-ˆŽrT   c                ó   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  ,          /# ©	rV   r¦   r‚   NrÂ   Úencoder_attention_maskr§   r„   r[   r¨   )r`   ra   s   "€rR   rb   ÚErnieAttention.__annotate__C  s‰   ø€ ÷ .ñ .á—|‘|ð.ñ ×)Ñ)¨DÕ0ð.ñ  %×0Ñ0°4Õ7ð	.ñ
 !&× 1Ñ 1°DÕ 8ð.ñ  �ð.ñ Ñ+Õ,ð.ñ 
‰u�|‰|Õ	ñ.rT   c           	     óŒ   € V P                   '       g   TMTpV P                  ! V3R VRVRV/VB w  rxV P                  Wq4      pWx3# )rÂ   r‚   r§   )rå   rO   ræ   )	rO   r¦   r‚   rÂ   rë   r§   r„   Úattention_outputr‘   s	   &&&&&&,  rR   rr   ÚErnieAttention.forwardC  sm   € ð 04×/F×/FÐ/F™ÐLbˆØ)-¯ªØñ*
à"7ð*
ð *ð*
ð ,ð	*
ð
 ñ*
Ñ&Ðð  Ÿ;™;Ð'7ÓGÐØÐ-Ð-rT   )rå   ræ   rO   )FNF©NNNNr¼   r|   s   @@rR   râ   râ   ;  s   ù‡ € ÷.÷.÷ .ò .rT   râ   c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚErnieIntermediateiX  c                ó"  <€ \         SV `  4        \        P                  ! VP                  VP
                  4      V n        \        VP                  \        4      '       d   \        VP                  ,          V n        R # VP                  V n        R # rÚ   )r1   r2   r3   rž   r6   Úintermediate_sizerÑ   r®   Ú
hidden_actÚstrr	   Úintermediate_act_fnrN   s   &&€rR   r2   ÚErnieIntermediate.__init__Y  s`   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3KÑ3KÓLˆŒ
Ü�f×'Ñ'¬×-Ò-Ü'-¨f×.?Ñ.?Õ'@ˆDÖ$à'-×'8Ñ'8ˆDÖ$rT   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©rV   r¦   r[   rÖ   )r`   ra   s   "€rR   rb   ÚErnieIntermediate.__annotate__a  s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ rT   c                óJ   € V P                  V4      pV P                  V4      pV# rÚ   ©rÑ   r÷   ©rO   r¦   s   &&rR   rr   ÚErnieIntermediate.forwarda  s&   € ØŸ
™
 =Ó1ˆØ×0Ñ0°Ó?ˆØÐrT   rý   r¼   r|   s   @@rR   rò   rò   X  s   ù‡ € õ9÷÷ ð rT   rò   c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚErnieOutputig  c                ó0  <€ \         SV `  4        \        P                  ! VP                  VP
                  4      V n        \        P                  ! VP
                  VP                  R 7      V n        \        P                  ! VP                  4      V n        R# rÐ   )r1   r2   r3   rž   rô   r6   rÑ   r=   r>   r?   r@   rA   rN   s   &&€rR   r2   ÚErnieOutput.__init__h  s`   ø€ Ü‰ÑÔÜ—Y’Y˜v×7Ñ7¸×9KÑ9KÓLˆŒ
ÜŸš f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z’z &×"<Ñ"<Ó=ˆŽrT   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# rÔ   rÖ   )r`   ra   s   "€rR   rb   ÚErnieOutput.__annotate__n  rØ   rT   c                óx   € V P                  V4      pV P                  V4      pV P                  W,           4      pV# rÚ   rÛ   rÜ   s   &&&rR   rr   ÚErnieOutput.forwardn  rÞ   rT   rß   r¼   r|   s   @@rR   r  r  g  rà   rT   r  c                   óR   a a€ ] tR tRt oRV 3R lltRV3R lR lltR tRtVtV ;t	# )	Ú
ErnieLayeriu  c                óœ  <€ \         SV `  4        VP                  V n        ^V n        \	        WP
                  VR7      V n        VP
                  V n        VP                  V n        V P                  '       d5   V P
                  '       g   \        V  R24      h\	        VRVRR7      V n	        \        V4      V n        \        V4      V n        R# )r/   rä   z> should be used as a decoder model if cross attention is addedFT)r¡   r¢   rå   N)r1   r2   Úchunk_size_feed_forwardÚseq_len_dimrâ   r    Ú	attentionÚadd_cross_attentionr›   Úcrossattentionrò   Úintermediater  ræ   )rO   rP   r¢   rQ   s   &&&€rR   r2   ÚErnieLayer.__init__v  s°   ø€ Ü‰ÑÔØ'-×'EÑ'EˆÔ$ØˆÔÜ'¨×:KÑ:KÐW`ÔaˆŒØ ×+Ñ+ˆŒØ#)×#=Ñ#=ˆÔ Ø×#×#Ð#Ø—?—?�?Ü  D 6Ð)gÐ!hÓiÐiÜ"0ØØØ#Ø#'ô	#ˆDÔô .¨fÓ5ˆÔÜ! &Ó)ˆŽrT   c                óð   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[S[,          RS[ P                  /# rê   )rC   r_   r]   r
   r   r   )r`   ra   s   "€rR   rb   ÚErnieLayer.__annotate__‰  s„   ø€ ÷ %ñ %á—|‘|ð%ñ ×)Ñ)¨DÕ0ð%ñ  %×0Ñ0°4Õ7ð	%ñ
 !&× 1Ñ 1°DÕ 8ð%ñ  �ð%ñ Ñ+Õ,ð%ñ 
�‰ñ%rT   c                ó8  € V P                   ! VV3R V/VB w  rxTp	V P                  '       dB   Ve>   \        V R4      '       g   \        RV  R24      hV P                  ! VRVV3R V/VB w  r¨T
p	\        V P                  V P                  V P                  V	4      pV# )r§   Nr  z'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`)	r  r    ri   r›   r  r   Úfeed_forward_chunkr  r  )rO   r¦   r‚   rÂ   rë   r§   r„   Úself_attention_outputÚ_rî   Úcross_attention_outputÚlayer_outputs   &&&&&&,     rR   rr   ÚErnieLayer.forward‰  sâ   € ð $(§>¢>ØØñ$
ð ,ð$
ð ñ	$
Ñ Ðð 1Ðà�?�?ˆ?Ð4Ò@Ü˜4Ð!1×2Ò2Ü Ø=¸d¸Vð DDð Dóð ð
 )-×(;Ò(;Ø%ØØ%Ø&ñ	)ð
 !0ð)ð ñ)Ñ%Ð"ð  6Ðä0Ø×#Ñ# T×%AÑ%AÀ4×CSÑCSÐUeó
ˆð ÐrT   c                óJ   € V P                  V4      pV P                  W!4      pV# rÚ   )r  ræ   )rO   rî   Úintermediate_outputr  s   &&  rR   r  ÚErnieLayer.feed_forward_chunk°  s)   € Ø"×/Ñ/Ð0@ÓAÐØ—{‘{Ð#6ÓIˆØÐrT   )r  r  r  r  r  r    ræ   r  rÚ   rð   )
rt   ru   rv   rw   r2   rr   r  ry   rz   r{   r|   s   @@rR   r	  r	  u  s   ù‡ € ÷*÷&%ò %÷Nò rT   r	  c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚErniePooleri¶  c                ó¸   <€ \         SV `  4        \        P                  ! VP                  VP                  4      V n        \        P                  ! 4       V n        R # rÚ   )r1   r2   r3   rž   r6   rÑ   ÚTanhÚ
activationrN   s   &&€rR   r2   ÚErniePooler.__init__·  s9   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ'š'›)ˆŽrT   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rú   rÖ   )r`   ra   s   "€rR   rb   ÚErniePooler.__annotate__¼  s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ rT   c                ó\   € VR,          pV P                  V4      pV P                  V4      pV# )re   )re   r   )rÑ   r"  )rO   r¦   Úfirst_token_tensorÚpooled_outputs   &&  rR   rr   ÚErniePooler.forward¼  s2   € ð +¨4Õ0ÐØŸ
™
Ð#5Ó6ˆØŸ™¨Ó6ˆØÐrT   )r"  rÑ   r¼   r|   s   @@rR   r  r  ¶  s   ù‡ € õ$÷
÷ ð rT   r  c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚErniePredictionHeadTransformiÅ  c                ó‚  <€ \         SV `  4        \        P                  ! VP                  VP                  4      V n        \        VP                  \        4      '       d   \        VP                  ,          V n
        MVP                  V n
        \        P                  ! VP                  VP                  R 7      V n        R# rÐ   )r1   r2   r3   rž   r6   rÑ   r®   rõ   rö   r	   Útransform_act_fnr=   r>   rN   s   &&€rR   r2   Ú%ErniePredictionHeadTransform.__init__Æ  s~   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
Ü�f×'Ñ'¬×-Ò-Ü$*¨6×+<Ñ+<Õ$=ˆDÕ!à$*×$5Ñ$5ˆDÔ!ÜŸš f×&8Ñ&8¸f×>SÑ>SÔTˆŽrT   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rú   rÖ   )r`   ra   s   "€rR   rb   Ú)ErniePredictionHeadTransform.__annotate__Ï  s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ rT   c                ól   € V P                  V4      pV P                  V4      pV P                  V4      pV# rÚ   )rÑ   r-  r=   rþ   s   &&rR   rr   Ú$ErniePredictionHeadTransform.forwardÏ  s4   € ØŸ
™
 =Ó1ˆØ×-Ñ-¨mÓ<ˆØŸ™ }Ó5ˆØÐrT   )r=   rÑ   r-  r¼   r|   s   @@rR   r+  r+  Å  s   ù‡ € õU÷÷ ð rT   r+  c                   ó8   a a€ ] tR tRt oV 3R ltR tRtVtV ;t# )ÚErnieLMPredictionHeadiÖ  c                ó  <€ \         SV `  4        \        V4      V n        \        P
                  ! VP                  VP                  R R7      V n        \        P                  ! \        P                  ! VP                  4      4      V n        R# )T)ÚbiasN)r1   r2   r+  Ú	transformr3   rž   r6   r5   ÚdecoderÚ	ParameterrC   rG   r6  rN   s   &&€rR   r2   ÚErnieLMPredictionHead.__init__×  s[   ø€ Ü‰ÑÔÜ5°fÓ=ˆŒô —y’y ×!3Ñ!3°V×5FÑ5FÈTÔRˆŒÜ—L’L¤§¢¨V×->Ñ->Ó!?Ó@ˆŽ	rT   c                óJ   € V P                  V4      pV P                  V4      pV# rÚ   )r7  r8  rþ   s   &&rR   rr   ÚErnieLMPredictionHead.forwardà  s$   € ØŸ™ }Ó5ˆØŸ™ ]Ó3ˆØÐrT   )r6  r8  r7  r¼   r|   s   @@rR   r4  r4  Ö  s   ù‡ € õA÷ò rT   r4  c                   óH   a a€ ] tR tRt oV 3R ltRV3R lR lltRtVtV ;t# )ÚErnieEncoderiæ  c           
     óÊ   <€ \         SV `  4        Wn        \        P                  ! \        VP                  4       Uu. uF  p\        WR 7      NK  	  up4      V n        R# u upi ))r¢   N)	r1   r2   rP   r3   Ú
ModuleListÚrangeÚnum_hidden_layersr	  Úlayer)rO   rP   ÚirQ   s   && €rR   r2   ÚErnieEncoder.__init__ç  sH   ø€ Ü‰ÑÔØŒÜ—]’]ÌUÐSY×SkÑSkÔMlÓ#mÑMlÈ¤J¨v×$CÑMlÑ#mÓnˆŽ
ùÒ#ms   ½A c                ó$  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[,          R	S[S[ P                  ,          S[,          /# )
rV   r¦   r‚   NrÂ   rë   r§   Ú	use_cacher„   r[   )	rC   r_   r]   r
   Úboolr   r   r©   r   )r`   ra   s   "€rR   rb   ÚErnieEncoder.__annotate__ì  s�   ø€ ÷ 
ñ 
á—|‘|ð
ñ ×)Ñ)¨DÕ0ð
ñ  %×0Ñ0°4Õ7ð	
ñ
 !&× 1Ñ 1°DÕ 8ð
ñ  �ð
ñ ˜$•;ð
ñ Ñ+Õ,ð
ñ 
‰u�|‰|Õ	ÑHÕ	Hñ
rT   c                ó”   € \        V P                  4       F  w  r‰V	! VVV3R VRV/VB pK  	  \        TV'       d   VR7      # RR7      # )rë   r§   N)Úlast_hidden_stater§   )Ú	enumeraterC  r   )
rO   r¦   r‚   rÂ   rë   r§   rG  r„   rD  Úlayer_modules
   &&&&&&&,  rR   rr   ÚErnieEncoder.forwardì  sn   € ô  )¨¯©Ö4‰OˆAÙ(ØØØ%ñð (>ð	ð
 !0ðð ñŠMñ  5ô 9Ø+ß/8˜Oô
ð 	
à>Bô
ð 	
rT   )rP   rC  )NNNNNr¼   r|   s   @@rR   r>  r>  æ  s   ù‡ € õo÷

÷ 
ò 
rT   r>  c                   ó†   a a€ ] tR tRt o]tRtRtRtRt	Rt
RtR]R]R]/t]P"                  ! 4       V 3R l4       tRtVtV ;t# )	ÚErniePreTrainedModeli  ÚernieTr¦   Ú
attentionsÚcross_attentionsc                óÄ  <€ \         SV `  V4       \        V\        4      '       d#   \        P
                  ! VP                  4       R# \        V\        4      '       d‚   \        P                  ! VP                  \        P                  ! VP                  P                  R,          4      P                  R4      4       \        P
                  ! VP                  4       R# R# )zInitialize the weightsNr0   r.   )r1   Ú_init_weightsr®   r4  ÚinitÚzeros_r6  r'   Úcopy_rF   rC   rD   rj   rE   rJ   )rO   r~   rQ   s   &&€rR   rU  Ú"ErniePreTrainedModel._init_weights  s�   ø€ ô 	‰Ñ˜fÔ%Ü�fÔ3×4Ò4Ü�KŠK˜Ÿ™Ö$Ü˜¤×0Ò0Ü�JŠJ�v×*Ñ*¬E¯LªL¸×9LÑ9L×9RÑ9RÐSUÕ9VÓ,W×,^Ñ,^Ð_fÓ,gÔhÜ�KŠK˜×-Ñ-Ö.ñ 1rT   © )rt   ru   rv   rw   r%   Úconfig_classÚbase_model_prefixÚsupports_gradient_checkpointingÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_supports_attention_backendr	  r•   r¾   Ú_can_record_outputsrC   Úno_gradrU  ry   rz   r{   r|   s   @@rR   rP  rP    se   ù‡ € à€LØÐØ&*Ð#ØÐØ€NØÐØ"&Ðà˜ØÐ(ØÐ/ðÐð ‡]‚]ƒ_ô/ó ÷/ð /rT   rP  a
  
    The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of
    cross-attention is added between the self-attention layers, following the architecture described in [Attention is
    all you need](https://huggingface.co/papers/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit,
    Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.

    To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set
    to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and
    `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass.
    )Úcustom_introc                   ó‚   a a€ ] tR tRt oR.tR
V 3R lltR tR t]]	]
RV3R lR ll4       4       4       tR tR	tVtV ;t# )Ú
ErnieModeli   r	  c                óÔ   <€ \         SV `  V4       Wn        RV n        \	        V4      V n        \        V4      V n        V'       d   \        V4      MRV n	        V P                  4        R# )z^
add_pooling_layer (bool, *optional*, defaults to `True`):
    Whether to add a pooling layer
FN)r1   r2   rP   Úgradient_checkpointingr'   rq   r>  Úencoderr  ÚpoolerÚ	post_init)rO   rP   Úadd_pooling_layerrQ   s   &&&€rR   r2   ÚErnieModel.__init__/  sS   ø€ ô
 	‰Ñ˜Ô ØŒØ&+ˆÔ#ä)¨&Ó1ˆŒÜ# FÓ+ˆŒç->”k &Ô)ÀDˆŒð 	�‰ÖrT   c                ó.   € V P                   P                  # rÚ   ©rq   r8   ©rO   s   &rR   Úget_input_embeddingsÚErnieModel.get_input_embeddings@  s   € Ø�‰×.Ñ.Ð.rT   c                ó&   € WP                   n        R # rÚ   ro  )rO   r�   s   &&rR   Úset_input_embeddingsÚErnieModel.set_input_embeddingsC  s   € Ø*/�‰Ö'rT   c                óÒ  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[ P                  R,          R
S[R,          RS[R,          RS[S[,          RS[S[ P                  ,          S[,          /# )rV   rW   Nr‚   rJ   rX   rF   rY   rÂ   rë   r§   rG  r„   r[   )rC   r_   r
   rH  r   r   r©   r   )r`   ra   s   "€rR   rb   ÚErnieModel.__annotate__I  sö   ø€ ÷ H
ñ H
á—<‘< $Õ&ðH
ñ Ÿ™ tÕ+ðH
ñ Ÿ™ tÕ+ð	H
ñ
 —|‘| dÕ*ðH
ñ —l‘l TÕ)ðH
ñ —|‘| dÕ*ðH
ñ  %Ÿ|™|¨dÕ2ðH
ñ !&§¡¨tÕ 3ðH
ñ  �ðH
ñ ˜$•;ðH
ñ Ñ+Õ,ðH
ñ 
‰u�|‰|Õ	ÑKÕ	KñH
rT   c                óì  € VRJ VRJ,          '       d   \        R4      hV P                  P                  '       d   V
e   T
MV P                  P                  p
MRp
V
'       do   V	fk   Vf   V P                  P                  '       d5   \        \        V P                  R7      \        V P                  R7      4      M\        V P                  R7      p	V	e   V	P                  4       M^ pV P                  VVVVVVR7      pV P                  VVVVV	R7      w  r(V P                  ! V3RVRVR	VR
V	RV
RV/VB pV^ ,          pV P                  e   V P                  V4      MRp\        VVVP                  R7      # )áª  
task_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Task type embedding is a special embedding to represent the characteristic of different tasks, such as
    word-aware pre-training task, structure-aware pre-training task and semantic-aware pre-training task. We
    assign a `task_type_id` to each task and the `task_type_id` is in the range `[0,
    config.task_type_vocab_size-1]
Nz:You must specify exactly one of input_ids or inputs_embedsF)rP   )rW   rF   rJ   rX   rY   rZ   )r‚   rë   Úembedding_outputrÂ   r§   r‚   rÂ   rë   r§   rG  rF   )rK  Úpooler_outputr§   )r›   rP   r    rG  Úis_encoder_decoderr   r   Úget_seq_lengthrq   Ú_create_attention_masksri  rj  r   r§   )rO   rW   r‚   rJ   rX   rF   rY   rÂ   rë   r§   rG  r„   rZ   rz  Úencoder_outputsÚsequence_outputr(  s   &&&&&&&&&&&,     rR   rr   ÚErnieModel.forwardF  s¥  € ð0 ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZà�;‰;×!×!Ð!Ø%.Ò%:™	ÀÇÁ×@UÑ@U‰IàˆIç˜Ò0ð )Ò4¸¿¹×8V×8VÐ8Vô $¤L¸¿¹Ô$DÄlÐZ^×ZeÑZeÔFfÔgä!¨¯©Ô5ð ð FUÒE` ×!?Ñ!?Ô!AÐfgÐØŸ?™?ØØ%Ø)à'Ø'Ø#9ð +ó 
Ðð 26×1MÑ1MØ)Ø#9Ø-Ø"7Ø+ð 2Nó 2
Ñ.ˆð Ÿ,š,Øñ	
à)ð	
ð #8ð	
ð $:ð		
ð
 ,ð	
ð  ð	
ð &ð	
ð ñ	
ˆð *¨!Õ,ˆØ8<¿¹Ò8O˜Ÿ™ OÔ4ÐUYˆä;Ø-Ø'Ø+×;Ñ;ô
ð 	
rT   c                óÞ   € V P                   P                  '       d   \        V P                   VVVR 7      pM\        V P                   VVR7      pVe   \        V P                   VVVR7      pW3# ))rP   rY   r‚   r§   )rP   rY   r‚   )rP   rY   r‚   rÂ   )rP   r    r   r   )rO   r‚   rë   rz  rÂ   r§   s   &&&&&&rR   r~  Ú"ErnieModel._create_attention_masks“  su   € ð �;‰;×!×!Ð!Ü/Ø—{‘{Ø.Ø-Ø /ô	‰Nô 7Ø—{‘{Ø.Ø-ôˆNð "Ò-Ü%>Ø—{‘{Ø.Ø5Ø&;ô	&Ð"ð Ð5Ð5rT   )rP   rq   ri  rh  rj  )T)
NNNNNNNNNN)rt   ru   rv   rw   Ú_no_split_modulesr2   rq  rt  r#   r$   r    rr   r~  ry   rz   r{   r|   s   @@rR   rf  rf     sU   ù‡ € ð &˜Ð÷ò"/ò0ð  ØØ÷H
ñ H
ó ó ó  ðH
÷T6ò 6rT   rf  z1
    Output type of [`ErnieForPreTraining`].
    c                   óD   a € ] tR tRt o RtRtRtRtRtRt	V 3R lt
RtV tR# )ÚErnieForPreTrainingOutputi´  ar  
loss (*optional*, returned when `labels` is provided, `torch.FloatTensor` of shape `(1,)`):
    Total loss as the sum of the masked language modeling loss and the next sequence prediction
    (classification) loss.
prediction_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
    Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
seq_relationship_logits (`torch.FloatTensor` of shape `(batch_size, 2)`):
    Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation
    before SoftMax).
Nc                ó   <€ V ^8„  d   Qh/ S[ P                  R,          ;R&   S[ P                  R,          ;R&   S[ P                  R,          ;R&   S[S[ P                  ,          R,          ;R&   S[S[ P                  ,          R,          ;R&   # )rV   NÚlossÚprediction_logitsÚseq_relationship_logitsr¦   rR  )rC   r]   r©   )r`   ra   s   "€rR   rb   Ú&ErnieForPreTrainingOutput.__annotate__´  s‡   ø‡ ‚ ñ$ ×
Ñ
˜dÕ
"Ñ)ñ% ñ& ×(Ñ(¨4Õ/Ñ6ñ' ñ( #×.Ñ.°Õ5Ñ<ñ) ñ* ™×*Ñ*Õ+¨dÕ2Ñ9ñ+ ñ, ‘e×'Ñ'Õ(¨4Õ/Ñ6ò- rT   rZ  )rt   ru   rv   rw   rx   rˆ  r‰  rŠ  r¦   rR  Ú__annotate_func__ry   rz   )ra   s   @rR   r†  r†  ´  s.   ø‡ € ñ	ð &*€DØ26ÐØ8<ÐØ59€MØ26€J÷- ƒ rT   r†  c                   ó8   a a€ ] tR tRt oV 3R ltR tRtVtV ;t# )ÚErniePreTrainingHeadsiÍ  c                ó�   <€ \         SV `  4        \        V4      V n        \        P
                  ! VP                  ^4      V n        R# ©rV   N)r1   r2   r4  Úpredictionsr3   rž   r6   Úseq_relationshiprN   s   &&€rR   r2   ÚErniePreTrainingHeads.__init__Î  s4   ø€ Ü‰ÑÔÜ0°Ó8ˆÔÜ "§	¢	¨&×*<Ñ*<¸aÓ @ˆÖrT   c                óL   € V P                  V4      pV P                  V4      pW43# rÚ   ©r‘  r’  )rO   r€  r(  Úprediction_scoresÚseq_relationship_scores   &&&  rR   rr   ÚErniePreTrainingHeads.forwardÓ  s-   € Ø ×,Ñ,¨_Ó=ÐØ!%×!6Ñ!6°}Ó!EÐØ Ð8Ð8rT   r•  r¼   r|   s   @@rR   rŽ  rŽ  Í  s   ù‡ € õA÷
9ò 9rT   rŽ  z©
    Ernie Model with two heads on top as done during the pretraining: a `masked language modeling` head and a `next
    sentence prediction (classification)` head.
    c                   ót   a a€ ] tR tRt oRRRR/tV 3R ltR tR t]]	RV3R	 lR
 ll4       4       t
RtVtV ;t# )ÚErnieForPreTrainingiÙ  úcls.predictions.decoder.biasúcls.predictions.biasúcls.predictions.decoder.weightú'ernie.embeddings.word_embeddings.weightc                ó†   <€ \         SV `  V4       \        V4      V n        \	        V4      V n        V P                  4        R # rÚ   )r1   r2   rf  rQ  rŽ  Úclsrk  rN   s   &&€rR   r2   ÚErnieForPreTraining.__init__å  s4   ø€ Ü‰Ñ˜Ô ä Ó'ˆŒ
Ü(¨Ó0ˆŒð 	�‰ÖrT   c                óB   € V P                   P                  P                  # rÚ   ©r   r‘  r8  rp  s   &rR   Úget_output_embeddingsÚ)ErnieForPreTraining.get_output_embeddingsî  ó   € Ø�x‰x×#Ñ#×+Ñ+Ð+rT   c                ó„   € WP                   P                  n        VP                  V P                   P                  n        R # rÚ   ©r   r‘  r8  r6  ©rO   Únew_embeddingss   &&rR   Úset_output_embeddingsÚ)ErnieForPreTraining.set_output_embeddingsñ  ó*   € Ø'5�‰×ÑÔ$Ø$2×$7Ñ$7ˆ�‰×ÑÖ!rT   c                óª  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[ P                  R,          R
S[S[,          RS[S[ P                  ,          S[,          /
# )rV   rW   Nr‚   rJ   rX   rF   rY   ÚlabelsÚnext_sentence_labelr„   r[   )rC   r_   r   r   r©   r†  )r`   ra   s   "€rR   rb   Ú ErnieForPreTraining.__annotate__÷  sØ   ø€ ÷ H
ñ H
á—<‘< $Õ&ðH
ñ Ÿ™ tÕ+ðH
ñ Ÿ™ tÕ+ð	H
ñ
 —|‘| dÕ*ðH
ñ —l‘l TÕ)ðH
ñ —|‘| dÕ*ðH
ñ —‘˜tÕ#ðH
ñ #Ÿ\™\¨DÕ0ðH
ñ Ñ+Õ,ðH
ñ 
‰u�|‰|Õ	Ñ8Õ	8ñH
rT   c	                óÆ  € V P                   ! V3RVRVRVRVRVRR/V	B p
V
R,          w  r¼V P                  W¼4      w  rÞR	pVe|   Vex   \        4       pV! VP                  RV P                  P
                  4      VP                  R4      4      pV! VP                  R^4      VP                  R4      4      pVV,           p\        VVVV
P                  V
P                  R
7      # )aj  
task_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Task type embedding is a special embedding to represent the characteristic of different tasks, such as
    word-aware pre-training task, structure-aware pre-training task and semantic-aware pre-training task. We
    assign a `task_type_id` to each task and the `task_type_id` is in the range `[0,
    config.task_type_vocab_size-1]
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
    config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked),
    the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
next_sentence_label (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the next sequence prediction (classification) loss. Input should be a sequence
    pair (see `input_ids` docstring) Indices should be in `[0, 1]`:

    - 0 indicates sequence B is a continuation of sequence A,
    - 1 indicates sequence B is a random sequence.

Example:

```python
>>> from transformers import AutoTokenizer, ErnieForPreTraining
>>> import torch

>>> tokenizer = AutoTokenizer.from_pretrained("nghuyong/ernie-1.0-base-zh")
>>> model = ErnieForPreTraining.from_pretrained("nghuyong/ernie-1.0-base-zh")

>>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
>>> outputs = model(**inputs)

>>> prediction_logits = outputs.prediction_logits
>>> seq_relationship_logits = outputs.seq_relationship_logits
```
r‚   rJ   rX   rF   rY   Úreturn_dictT:NrV   NN)rˆ  r‰  rŠ  r¦   rR  r0   )	rQ  r   r   r­   rP   r5   r†  r¦   rR  )rO   rW   r‚   rJ   rX   rF   rY   r¯  r°  r„   Úoutputsr€  r(  r–  r—  Ú
total_lossÚloss_fctÚmasked_lm_lossÚnext_sentence_losss   &&&&&&&&&,         rR   rr   ÚErnieForPreTraining.forwardõ  s  € ð^ —*’*Øñ	
à)ð	
ð *ð	
ð (ð		
ð
 &ð	
ð (ð	
ð ð	
ð ñ	
ˆð *1°­Ñ&ˆØ48·H±H¸_Ó4\Ñ1Ðàˆ
ØÒÐ"5Ò"AÜ'Ó)ˆHÙ%Ð&7×&<Ñ&<¸RÀÇÁ×AWÑAWÓ&XÐZ`×ZeÑZeÐfhÓZiÓjˆNÙ!)Ð*@×*EÑ*EÀbÈ!Ó*LÐNa×NfÑNfÐgiÓNjÓ!kÐØ'Ð*<Õ<ˆJä(ØØ/Ø$:Ø!×/Ñ/Ø×)Ñ)ô
ð 	
rT   ©r   rQ  ©NNNNNNNN©rt   ru   rv   rw   Ú_tied_weights_keysr2   r¤  r«  r"   r    rr   ry   rz   r{   r|   s   @@rR   rš  rš  Ù  sT   ù‡ € ð 	'Ð(>Ø(Ð*SðÐõ
ò,ò8ð Ø÷H
ñ H
ó ó ÷H
ð H
rT   rš  c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚErnieOnlyMLMHeadiB  c                óD   <€ \         SV `  4        \        V4      V n        R # rÚ   )r1   r2   r4  r‘  rN   s   &&€rR   r2   ÚErnieOnlyMLMHead.__init__C  s   ø€ Ü‰ÑÔÜ0°Ó8ˆÖrT   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )rV   r€  r[   rÖ   )r`   ra   s   "€rR   rb   ÚErnieOnlyMLMHead.__annotate__G  s#   ø€ ÷ !ñ !¡u§|¡|ð !¹¿¹ñ !rT   c                ó(   € V P                  V4      pV# rÚ   ©r‘  )rO   r€  r–  s   && rR   rr   ÚErnieOnlyMLMHead.forwardG  s   € Ø ×,Ñ,¨_Ó=ÐØ Ð rT   rÅ  r¼   r|   s   @@rR   r¿  r¿  B  s   ù‡ € õ9÷!÷ !ð !rT   r¿  zQ
    Ernie Model with a `language modeling` head on top for CLM fine-tuning.
    c                   ót   a a€ ] tR tRt oRRRR/tV 3R ltR tR t]]	RV3R	 lR
 ll4       4       t
RtVtV ;t# )ÚErnieForCausalLMiL  r�  rž  r›  rœ  c                óØ   <€ \         SV `  V4       VP                  '       g   \        P	                  R 4       \        VRR7      V n        \        V4      V n        V P                  4        R# )zMIf you want to use `ErnieForCausalLM` as a standalone, add `is_decoder=True.`F©rl  N©
r1   r2   r    ÚloggerÚwarningrf  rQ  r¿  r   rk  rN   s   &&€rR   r2   ÚErnieForCausalLM.__init__W  sO   ø€ Ü‰Ñ˜Ô à× × Ð Ü�N‰NÐjÔkä ¸%Ô@ˆŒ
Ü# FÓ+ˆŒð 	�‰ÖrT   c                óB   € V P                   P                  P                  # rÚ   r£  rp  s   &rR   r¤  Ú&ErnieForCausalLM.get_output_embeddingsc  r¦  rT   c                ó„   € WP                   P                  n        VP                  V P                   P                  n        R # rÚ   r¨  r©  s   &&rR   r«  Ú&ErnieForCausalLM.set_output_embeddingsf  r­  rT   c                óH  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[ P                  R,          R
S[ P                  R,          RS[S[ P                  ,          R,          RS[R,          RS[S[ P                  ,          RS[S[,          RS[S[ P                  ,          S[,          /# )rV   rW   Nr‚   rJ   rX   rF   rY   rÂ   rë   r¯  r§   rG  Úlogits_to_keepr„   r[   )	rC   r_   ÚlistrH  r^   r   r   r©   r   )r`   ra   s   "€rR   rb   ÚErnieForCausalLM.__annotate__l  s  ø€ ÷ =
ñ =
á—<‘< $Õ&ð=
ñ Ÿ™ tÕ+ð=
ñ Ÿ™ tÕ+ð	=
ñ
 —|‘| dÕ*ð=
ñ —l‘l TÕ)ð=
ñ —|‘| dÕ*ð=
ñ  %Ÿ|™|¨dÕ2ð=
ñ !&§¡¨tÕ 3ð=
ñ —‘˜tÕ#ð=
ñ ™eŸl™lÕ+¨dÕ2ð=
ñ ˜$•;ð=
ñ ™eŸl™lÕ*ð=
ñ Ñ+Õ,ð=
ñ 
‰u�|‰|Õ	Ñ@Õ	@ñ=
rT   c                óÊ  € V	e   RpV P                   ! V3RVRVRVRVRVRVR	VR
V
RVRR/
VB pVP                  p\        V\        4      '       d   \	        V) R4      MTpV P                  VRVR3,          4      pRpV	e-   V P                  ! RRVRV	RV P                  P                  /VB p\        VVVP                  VP                  VP                  VP                  R7      # )aA  
task_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Task type embedding is a special embedding to represent the characteristic of different tasks, such as
    word-aware pre-training task, structure-aware pre-training task and semantic-aware pre-training task. We
    assign a `task_type_id` to each task and the `task_type_id` is in the range `[0,
    config.task_type_vocab_size-1]
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in
    `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are
    ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
NFr‚   rJ   rX   rF   rY   rÂ   rë   r§   rG  r³  Tre   Úlogitsr¯  r5   )rˆ  rØ  r§   r¦   rR  rS  rZ  )rQ  rK  r®   r^   Úslicer   Úloss_functionrP   r5   r   r§   r¦   rR  rS  )rO   rW   r‚   rJ   rX   rF   rY   rÂ   rë   r¯  r§   rG  rÔ  r„   r´  r¦   Úslice_indicesrØ  rˆ  s   &&&&&&&&&&&&&,     rR   rr   ÚErnieForCausalLM.forwardj  sL  € ð: ÒØˆIà@DÇ
Â
ØñA
à)ðA
ð *ðA
ð (ð	A
ð
 &ðA
ð (ðA
ð #8ðA
ð $:ðA
ð ,ðA
ð  ðA
ð ðA
ð ñA
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜-¨¨=¸!Ð(;Õ<Ó=ˆàˆØÒØ×%Ò%Ñp¨VÐp¸FÐpÈtÏ{É{×OeÑOeÐpÐioÑpˆDä0ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø$×5Ñ5ô
ð 	
rT   rº  )NNNNNNNNNNNr   r¼  r|   s   @@rR   rÈ  rÈ  L  sP   ù‡ € ð 	)Ð*SØ&Ð(>ðÐõ

ò,ò8ð Ø÷=
ñ =
ó ó ÷=
ð =
rT   rÈ  c                   ót   a a€ ] tR tRt oRRRR/tV 3R ltR tR t]]	RV3R	 lR
 ll4       4       t
RtVtV ;t# )ÚErnieForMaskedLMi¬  r›  rœ  r�  rž  c                óØ   <€ \         SV `  V4       VP                  '       d   \        P	                  R 4       \        VRR7      V n        \        V4      V n        V P                  4        R# )zlIf you want to use `ErnieForMaskedLM` make sure `config.is_decoder=False` for bi-directional self-attention.FrÊ  NrË  rN   s   &&€rR   r2   ÚErnieForMaskedLM.__init__³  sU   ø€ Ü‰Ñ˜Ô à××ÐÜ�N‰Nð1ôô
   ¸%Ô@ˆŒ
Ü# FÓ+ˆŒð 	�‰ÖrT   c                óB   € V P                   P                  P                  # rÚ   r£  rp  s   &rR   r¤  Ú&ErnieForMaskedLM.get_output_embeddingsÂ  r¦  rT   c                ó„   € WP                   P                  n        VP                  V P                   P                  n        R # rÚ   r¨  r©  s   &&rR   r«  Ú&ErnieForMaskedLM.set_output_embeddingsÅ  r­  rT   c                óÒ  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[ P                  R,          R
S[ P                  R,          RS[S[,          RS[S[ P                  ,          S[,          /# )rV   rW   Nr‚   rJ   rX   rF   rY   rÂ   rë   r¯  r„   r[   )rC   r_   r   r   r©   r   )r`   ra   s   "€rR   rb   ÚErnieForMaskedLM.__annotate__Ë  sÞ   ø€ ÷ 2
ñ 2
á—<‘< $Õ&ð2
ñ Ÿ™ tÕ+ð2
ñ Ÿ™ tÕ+ð	2
ñ
 —|‘| dÕ*ð2
ñ —l‘l TÕ)ð2
ñ —|‘| dÕ*ð2
ñ  %Ÿ|™|¨dÕ2ð2
ñ !&§¡¨tÕ 3ð2
ñ —‘˜tÕ#ð2
ñ Ñ+Õ,ð2
ñ 
‰u�|‰|Õ	™~Õ	-ñ2
rT   c
                óZ  € V P                   ! V3RVRVRVRVRVRVRVRR	/V
B pV^ ,          pV P                  V4      pR
pV	eG   \        4       pV! VP                  RV P                  P
                  4      V	P                  R4      4      p\        VVVP                  VP                  R7      # )a#  
task_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Task type embedding is a special embedding to represent the characteristic of different tasks, such as
    word-aware pre-training task, structure-aware pre-training task and semantic-aware pre-training task. We
    assign a `task_type_id` to each task and the `task_type_id` is in the range `[0,
    config.task_type_vocab_size-1]
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
    config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
    loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
r‚   rJ   rX   rF   rY   rÂ   rë   r³  TN©rˆ  rØ  r¦   rR  r0   )	rQ  r   r   r­   rP   r5   r   r¦   rR  )rO   rW   r‚   rJ   rX   rF   rY   rÂ   rë   r¯  r„   r´  r€  r–  r·  r¶  s   &&&&&&&&&&,     rR   rr   ÚErnieForMaskedLM.forwardÉ  sð   € ð4 —*’*Øñ
à)ð
ð *ð
ð (ð	
ð
 &ð
ð (ð
ð #8ð
ð $:ð
ð ð
ð ñ
ˆð " !�*ˆØ ŸH™H _Ó5ÐàˆØÒÜ'Ó)ˆHÙ%Ð&7×&<Ñ&<¸RÀÇÁ×AWÑAWÓ&XÐZ`×ZeÑZeÐfhÓZiÓjˆNäØØ$Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
rT   rº  )	NNNNNNNNNr¼  r|   s   @@rR   rÞ  rÞ  ¬  sP   ù‡ € ð 	'Ð(>Ø(Ð*SðÐõ
ò,ò8ð Ø÷2
ñ 2
ó ó ÷2
ð 2
rT   rÞ  c                   ó8   a a€ ] tR tRt oV 3R ltR tRtVtV ;t# )ÚErnieOnlyNSPHeadi   c                óp   <€ \         SV `  4        \        P                  ! VP                  ^4      V n        R# r�  )r1   r2   r3   rž   r6   r’  rN   s   &&€rR   r2   ÚErnieOnlyNSPHead.__init__  s'   ø€ Ü‰ÑÔÜ "§	¢	¨&×*<Ñ*<¸aÓ @ˆÖrT   c                ó(   € V P                  V4      pV# rÚ   ©r’  )rO   r(  r—  s   && rR   rr   ÚErnieOnlyNSPHead.forward  s   € Ø!%×!6Ñ!6°}Ó!EÐØ%Ð%rT   rï  r¼   r|   s   @@rR   rë  rë     s   ù‡ € õA÷&ò &rT   rë  zU
    Ernie Model with a `next sentence prediction (classification)` head on top.
    c                   ó\   a a€ ] tR tRt oV 3R lt]]RV3R lR ll4       4       tRtVt	V ;t
# )ÚErnieForNextSentencePredictioni
  c                ó†   <€ \         SV `  V4       \        V4      V n        \	        V4      V n        V P                  4        R # rÚ   )r1   r2   rf  rQ  rë  r   rk  rN   s   &&€rR   r2   Ú'ErnieForNextSentencePrediction.__init__  s4   ø€ Ü‰Ñ˜Ô ä Ó'ˆŒ
Ü# FÓ+ˆŒð 	�‰ÖrT   c                ó‚  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[S[,          R
S[S[ P                  ,          S[,          /	# ©rV   rW   Nr‚   rJ   rX   rF   rY   r¯  r„   r[   )rC   r_   r   r   r©   r   )r`   ra   s   "€rR   rb   Ú+ErnieForNextSentencePrediction.__annotate__  sÄ   ø€ ÷ D
ñ D
á—<‘< $Õ&ðD
ñ Ÿ™ tÕ+ðD
ñ Ÿ™ tÕ+ð	D
ñ
 —|‘| dÕ*ðD
ñ —l‘l TÕ)ðD
ñ —|‘| dÕ*ðD
ñ —‘˜tÕ#ðD
ñ Ñ+Õ,ðD
ñ 
‰u�|‰|Õ	Ñ:Õ	:ñD
rT   c                ó*  € V P                   ! V3RVRVRVRVRVRR/VB p	V	^,          p
V P                  V
4      pRpVe3   \        4       pV! VP                  R
^4      VP                  R
4      4      p\	        VVV	P
                  V	P                  R	7      # )aÇ  
task_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Task type embedding is a special embedding to represent the characteristic of different tasks, such as
    word-aware pre-training task, structure-aware pre-training task and semantic-aware pre-training task. We
    assign a `task_type_id` to each task and the `task_type_id` is in the range `[0,
    config.task_type_vocab_size-1]
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair
    (see `input_ids` docstring). Indices should be in `[0, 1]`:

    - 0 indicates sequence B is a continuation of sequence A,
    - 1 indicates sequence B is a random sequence.

Example:

```python
>>> from transformers import AutoTokenizer, ErnieForNextSentencePrediction
>>> import torch

>>> tokenizer = AutoTokenizer.from_pretrained("nghuyong/ernie-1.0-base-zh")
>>> model = ErnieForNextSentencePrediction.from_pretrained("nghuyong/ernie-1.0-base-zh")

>>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced."
>>> next_sentence = "The sky is blue due to the shorter wavelength of blue light."
>>> encoding = tokenizer(prompt, next_sentence, return_tensors="pt")

>>> outputs = model(**encoding, labels=torch.LongTensor([1]))
>>> logits = outputs.logits
>>> assert logits[0, 0] < logits[0, 1]  # next sentence was random
```
r‚   rJ   rX   rF   rY   r³  TNrè  r0   )rQ  r   r   r­   r   r¦   rR  )rO   rW   r‚   rJ   rX   rF   rY   r¯  r„   r´  r(  Úseq_relationship_scoresr¸  r¶  s   &&&&&&&&,     rR   rr   Ú&ErnieForNextSentencePrediction.forward  sÑ   € ðZ —*’*Øñ	
à)ð	
ð *ð	
ð (ð		
ð
 &ð	
ð (ð	
ð ð	
ð ñ	
ˆð   �
ˆà"&§(¡(¨=Ó"9Ðà!ÐØÒÜ'Ó)ˆHÙ!)Ð*A×*FÑ*FÀrÈ1Ó*MÈvÏ{É{Ð[]ËÓ!_Ðä*Ø#Ø*Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
rT   rº  ©NNNNNNN©rt   ru   rv   rw   r2   r"   r    rr   ry   rz   r{   r|   s   @@rR   rò  rò  
  s4   ù‡ € õð Ø÷D
ñ D
ó ó ÷D
ð D
rT   rò  z�
    Ernie Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled
    output) e.g. for GLUE tasks.
    c                   ó\   a a€ ] tR tRt oV 3R lt]]RV3R lR ll4       4       tRtVt	V ;t
# )ÚErnieForSequenceClassificationib  c                óv  <€ \         SV `  V4       VP                  V n        Wn        \	        V4      V n        VP                  e   VP                  MVP                  p\        P                  ! V4      V n
        \        P                  ! VP                  VP                  4      V n        V P                  4        R # rÚ   )r1   r2   Ú
num_labelsrP   rf  rQ  Úclassifier_dropoutr@   r3   r?   rA   rž   r6   Ú
classifierrk  ©rO   rP   r  rQ   s   && €rR   r2   Ú'ErnieForSequenceClassification.__init__i  s�   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒØŒä Ó'ˆŒ
à)/×)BÑ)BÒ)NˆF×%Ò%ÐTZ×TnÑTnð 	ô —z’zÐ"4Ó5ˆŒÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰ÖrT   c                ó‚  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[S[,          R
S[S[ P                  ,          S[,          /	# rö  )rC   r_   r   r   r©   r   )r`   ra   s   "€rR   rb   Ú+ErnieForSequenceClassification.__annotate__z  sÄ   ø€ ÷ B
ñ B
á—<‘< $Õ&ðB
ñ Ÿ™ tÕ+ðB
ñ Ÿ™ tÕ+ð	B
ñ
 —|‘| dÕ*ðB
ñ —l‘l TÕ)ðB
ñ —|‘| dÕ*ðB
ñ —‘˜tÕ#ðB
ñ Ñ+Õ,ðB
ñ 
‰u�|‰|Õ	Ñ7Õ	7ñB
rT   c                ó  € V P                   ! V3RVRVRVRVRVRR/VB p	V	^,          p
V P                  V
4      p
V P                  V
4      pRpVEe˜   V P                  P                  f–   V P
                  ^8X  d   R	V P                  n        MrV P
                  ^8”  dQ   VP                  \        P                  8X  g    VP                  \        P                  8X  d   R
V P                  n        MRV P                  n        V P                  P                  R	8X  dK   \        4       pV P
                  ^8X  d'   V! VP                  4       VP                  4       4      pMŽV! W·4      pM…V P                  P                  R
8X  d>   \        4       pV! VP                  RV P
                  4      VP                  R4      4      pM-V P                  P                  R8X  d   \        4       pV! W·4      p\        VVV	P                   V	P"                  R7      # )a  
task_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Task type embedding is a special embedding to represent the characteristic of different tasks, such as
    word-aware pre-training task, structure-aware pre-training task and semantic-aware pre-training task. We
    assign a `task_type_id` to each task and the `task_type_id` is in the range `[0,
    config.task_type_vocab_size-1]
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
    `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
r‚   rJ   rX   rF   rY   r³  TNÚ
regressionÚsingle_label_classificationÚmulti_label_classificationrè  r0   )rQ  rA   r  rP   Úproblem_typer   r-   rC   rI   r^   r   Úsqueezer   r­   r   r   r¦   rR  )rO   rW   r‚   rJ   rX   rF   rY   r¯  r„   r´  r(  rØ  rˆ  r¶  s   &&&&&&&&,     rR   rr   Ú&ErnieForSequenceClassification.forwardx  sÉ  € ð0 —*’*Øñ	
à)ð	
ð *ð	
ð (ð		
ð
 &ð	
ð (ð	
ð ð	
ð ñ	
ˆð   �
ˆàŸ™ ]Ó3ˆØ—‘ Ó/ˆàˆØÓØ�{‰{×'Ñ'Ò/Ø—?‘? aÔ'Ø/;�D—K‘KÕ,Ø—_‘_ qÔ(¨f¯l©l¼e¿j¹jÔ.HÈFÏLÉLÔ\a×\eÑ\eÔLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ô7Ü"›9�Ø—?‘? aÔ'Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# FÓ3‘DØ—‘×)Ñ)Ð-JÔJÜ+Ó-�Ù §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR‘Ø—‘×)Ñ)Ð-IÔIÜ,Ó.�Ù Ó/�ä'ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
rT   )r  rP   rA   rQ  r   rû  rü  r|   s   @@rR   rþ  rþ  b  s4   ù‡ € õð Ø÷B
ñ B
ó ó ÷B
ð B
rT   rþ  c                   ó\   a a€ ] tR tRt oV 3R lt]]RV3R lR ll4       4       tRtVt	V ;t
# )ÚErnieForMultipleChoicei¿  c                ó4  <€ \         SV `  V4       \        V4      V n        VP                  e   VP                  MVP
                  p\        P                  ! V4      V n        \        P                  ! VP                  ^4      V n        V P                  4        R # rÚ   )r1   r2   rf  rQ  r  r@   r3   r?   rA   rž   r6   r  rk  r  s   && €rR   r2   ÚErnieForMultipleChoice.__init__Á  su   ø€ Ü‰Ñ˜Ô ä Ó'ˆŒ
à)/×)BÑ)BÒ)NˆF×%Ò%ÐTZ×TnÑTnð 	ô —z’zÐ"4Ó5ˆŒÜŸ)š) F×$6Ñ$6¸Ó:ˆŒð 	�‰ÖrT   c                ó‚  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[S[,          R
S[S[ P                  ,          S[,          /	# rö  )rC   r_   r   r   r©   r   )r`   ra   s   "€rR   rb   Ú#ErnieForMultipleChoice.__annotate__Ð  sÄ   ø€ ÷ U
ñ U
á—<‘< $Õ&ðU
ñ Ÿ™ tÕ+ðU
ñ Ÿ™ tÕ+ð	U
ñ
 —|‘| dÕ*ðU
ñ —l‘l TÕ)ðU
ñ —|‘| dÕ*ðU
ñ —‘˜tÕ#ðU
ñ Ñ+Õ,ðU
ñ 
‰u�|‰|Õ	Ñ8Õ	8ñU
rT   c                ó*  € Ve   VP                   ^,          MVP                   ^,          p	Ve"   VP                  R
VP                  R
4      4      MRpVe"   VP                  R
VP                  R
4      4      MRpVe"   VP                  R
VP                  R
4      4      MRpVe"   VP                  R
VP                  R
4      4      MRpVe2   VP                  R
VP                  R4      VP                  R
4      4      MRpV P                  ! V3RVRVRVRVRVRR/VB p
V
^,          pV P	                  V4      pV P                  V4      pVP                  R
V	4      pRpVe   \        4       pV! W×4      p\        VVV
P                  V
P                  R	7      # )aQ  
input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
    Indices of input sequence tokens in the vocabulary.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are input IDs?](../glossary#input-ids)
token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
    Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
    1]`:

    - 0 corresponds to a *sentence A* token,
    - 1 corresponds to a *sentence B* token.

    [What are token type IDs?](../glossary#token-type-ids)
task_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
    Task type embedding is a special embedding to represent the characteristic of different tasks, such as
    word-aware pre-training task, structure-aware pre-training task and semantic-aware pre-training task. We
    assign a `task_type_id` to each task and the `task_type_id` is in the range `[0,
    config.task_type_vocab_size-1]
position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
    Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
    config.max_position_embeddings - 1]`.

    [What are position IDs?](../glossary#position-ids)
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):
    Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
    is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
    model's internal embedding lookup matrix.
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
    num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
    `input_ids` above)
Nr‚   rJ   rX   rF   rY   r³  Trè  r0   éþÿÿÿ)
rj   r­   rH   rQ  rA   r  r   r   r¦   rR  )rO   rW   r‚   rJ   rX   rF   rY   r¯  r„   Únum_choicesr´  r(  rØ  Úreshaped_logitsrˆ  r¶  s   &&&&&&&&,       rR   rr   ÚErnieForMultipleChoice.forwardÎ  sÀ  € ð` -6Ò,A�i—o‘o aÖ(À}×GZÑGZÐ[\ÕG]ˆà>GÒ>S�I—N‘N 2 y§~¡~°bÓ'9Ô:ÐY]ˆ	ØM[ÒMg˜×,Ñ,¨R°×1DÑ1DÀRÓ1HÔIÐmqˆØM[ÒMg˜×,Ñ,¨R°×1DÑ1DÀRÓ1HÔIÐmqˆØGSÒG_�|×(Ñ(¨¨\×->Ñ->¸rÓ-BÔCÐeiˆð Ò(ð ×Ñ˜r =×#5Ñ#5°bÓ#9¸=×;MÑ;MÈbÓ;QÔRàð 	ð —*’*Øñ	
à)ð	
ð *ð	
ð (ð		
ð
 &ð	
ð (ð	
ð ð	
ð ñ	
ˆð   �
ˆàŸ™ ]Ó3ˆØ—‘ Ó/ˆØ Ÿ+™+ b¨+Ó6ˆàˆØÒÜ'Ó)ˆHÙ˜OÓ4ˆDä(ØØ"Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
rT   )r  rA   rQ  rû  rü  r|   s   @@rR   r  r  ¿  s4   ù‡ € õð Ø÷U
ñ U
ó ó ÷U
ð U
rT   r  c                   ó\   a a€ ] tR tRt oV 3R lt]]RV3R lR ll4       4       tRtVt	V ;t
# )ÚErnieForTokenClassificationi(  c                ón  <€ \         SV `  V4       VP                  V n        \        VR R7      V n        VP
                  e   VP
                  MVP                  p\        P                  ! V4      V n	        \        P                  ! VP                  VP                  4      V n        V P                  4        R# ©FrÊ  N)r1   r2   r   rf  rQ  r  r@   r3   r?   rA   rž   r6   r  rk  r  s   && €rR   r2   Ú$ErnieForTokenClassification.__init__*  sŠ   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä ¸%Ô@ˆŒ
à)/×)BÑ)BÒ)NˆF×%Ò%ÐTZ×TnÑTnð 	ô —z’zÐ"4Ó5ˆŒÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰ÖrT   c                ó‚  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[S[,          R
S[S[ P                  ,          S[,          /	# rö  )rC   r_   r   r   r©   r   )r`   ra   s   "€rR   rb   Ú(ErnieForTokenClassification.__annotate__:  s¹   ø€ ÷ .
ñ .
á—<‘< $Õ&ð.
ñ Ÿ™ tÕ+ð.
ñ Ÿ™ tÕ+ð	.
ñ
 —|‘| dÕ*ð.
ñ —l‘l TÕ)ð.
ñ —|‘| dÕ*ð.
ñ —‘˜tÕ#ð.
ñ Ñ+Õ,ð.
ñ 
‰u�|‰|Õ	Ñ4Õ	4ñ.
rT   c                ó`  € V P                   ! V3RVRVRVRVRVRR/VB p	V	^ ,          p
V P                  V
4      p
V P                  V
4      pRpVe=   \        4       pV! VP	                  R
V P
                  4      VP	                  R
4      4      p\        VVV	P                  V	P                  R	7      # )al  
task_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Task type embedding is a special embedding to represent the characteristic of different tasks, such as
    word-aware pre-training task, structure-aware pre-training task and semantic-aware pre-training task. We
    assign a `task_type_id` to each task and the `task_type_id` is in the range `[0,
    config.task_type_vocab_size-1]
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
r‚   rJ   rX   rF   rY   r³  TNrè  r0   )	rQ  rA   r  r   r­   r   r   r¦   rR  )rO   rW   r‚   rJ   rX   rF   rY   r¯  r„   r´  r€  rØ  rˆ  r¶  s   &&&&&&&&,     rR   rr   Ú#ErnieForTokenClassification.forward8  sÛ   € ð, —*’*Øñ	
à)ð	
ð *ð	
ð (ð		
ð
 &ð	
ð (ð	
ð ð	
ð ñ	
ˆð " !�*ˆàŸ,™, Ó7ˆØ—‘ Ó1ˆàˆØÒÜ'Ó)ˆHÙ˜FŸK™K¨¨D¯O©OÓ<¸f¿k¹kÈ"»oÓNˆDä$ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
rT   )r  rA   rQ  r   rû  rü  r|   s   @@rR   r  r  (  s0   ù‡ € õð Ø÷.
ñ .
ó ó ÷.
ð .
rT   r  c                   ó\   a a€ ] tR tRt oV 3R lt]]RV3R lR ll4       4       tRtVt	V ;t
# )ÚErnieForQuestionAnsweringik  c                óì   <€ \         SV `  V4       VP                  V n        \        VR R7      V n        \
        P                  ! VP                  VP                  4      V n        V P                  4        R# r  )
r1   r2   r   rf  rQ  r3   rž   r6   Ú
qa_outputsrk  rN   s   &&€rR   r2   Ú"ErnieForQuestionAnswering.__init__m  sU   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä ¸%Ô@ˆŒ
ÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰ÖrT   c                óª  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[ P                  R,          R
S[S[,          RS[S[ P                  ,          S[,          /
# )rV   rW   Nr‚   rJ   rX   rF   rY   Ústart_positionsÚend_positionsr„   r[   )rC   r_   r   r   r©   r   )r`   ra   s   "€rR   rb   Ú&ErnieForQuestionAnswering.__annotate__y  sÌ   ø€ ÷ <
ñ <
á—<‘< $Õ&ð<
ñ Ÿ™ tÕ+ð<
ñ Ÿ™ tÕ+ð	<
ñ
 —|‘| dÕ*ð<
ñ —l‘l TÕ)ð<
ñ —|‘| dÕ*ð<
ñ Ÿ™¨Õ,ð<
ñ —|‘| dÕ*ð<
ñ Ñ+Õ,ð<
ñ 
‰u�|‰|Õ	Ñ;Õ	;ñ<
rT   c	                óô  € V P                   ! V3RVRVRVRVRVRR/V	B p
V
^ ,          pV P                  V4      pVP                  ^RR7      w  rÞVP                  R4      P	                  4       pVP                  R4      P	                  4       pR	pVeÄ   VeÀ   \        VP                  4       4      ^8”  d   VP                  R4      p\        VP                  4       4      ^8”  d   VP                  R4      pVP                  ^4      pVP                  ^ V4      pVP                  ^ V4      p\        VR
7      pV! W×4      pV! Wè4      pVV,           ^,          p\        VVVV
P                  V
P                  R7      # )ry  r‚   rJ   rX   rF   rY   r³  Trˆ   N)Úignore_index)rˆ  Ústart_logitsÚ
end_logitsr¦   rR  r0   )rQ  r%  Úsplitr  r�   ÚlenrH   Úclampr   r   r¦   rR  )rO   rW   r‚   rJ   rX   rF   rY   r(  r)  r„   r´  r€  rØ  r-  r.  rµ  Úignored_indexr¶  Ú
start_lossÚend_losss   &&&&&&&&&,          rR   rr   Ú!ErnieForQuestionAnswering.forwardw  sž  € ð* —*’*Øñ	
à)ð	
ð *ð	
ð (ð		
ð
 &ð	
ð (ð	
ð ð	
ð ñ	
ˆð " !�*ˆà—‘ Ó1ˆØ#)§<¡<°°r <Ó#:Ñ ˆØ#×+Ñ+¨BÓ/×:Ñ:Ó<ˆØ×'Ñ'¨Ó+×6Ñ6Ó8ˆ
àˆ
ØÒ&¨=Ò+Dä�?×'Ñ'Ó)Ó*¨QÔ.Ø"1×"9Ñ"9¸"Ó"=�Ü�=×%Ñ%Ó'Ó(¨1Ô,Ø -× 5Ñ 5°bÓ 9�à(×-Ñ-¨aÓ0ˆMØ-×3Ñ3°A°}ÓEˆOØ)×/Ñ/°°=ÓAˆMä'°]ÔCˆHÙ! ,Ó@ˆJÙ 
Ó:ˆHØ$ xÕ/°1Õ4ˆJä+ØØ%Ø!Ø!×/Ñ/Ø×)Ñ)ô
ð 	
rT   )rQ  r   r%  r»  rü  r|   s   @@rR   r#  r#  k  s0   ù‡ € õð Ø÷<
ñ <
ó ó ÷<
ð <
rT   r#  )Nr¬   )
rÈ  rÞ  r  rò  rš  r#  rþ  r  rf  rP  )WÚcollections.abcr   Údataclassesr   rC   Útorch.nnr3   r   r   r   Ú r   rV  Úactivationsr	   Úcache_utilsr
   r   r   Ú
generationr   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   r   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úpytorch_utilsr   Úutilsr   r   r    r!   Úutils.genericr"   r#   Úutils.output_capturingr$   Úconfiguration_ernier%   Ú
get_loggerrt   rÌ  r…   r'   r“   r•   r¾   rÎ   râ   rò   r  r	  r  r+  r4  r>  rP  rf  r†  rŽ  rš  r¿  rÈ  rÞ  rë  rò  rþ  r  r  r#  Ú__all__rZ  rT   rR   Ú<module>rI     s	  ðõ* %Ý !ã Ý ß AÑ Aå &Ý !ß CÑ CÝ )ß JÝ 9÷
÷ 
õ 
÷ GÝ &Ý 6ß MÓ Mß IÝ 5Ý ,ð 
×	Ò	˜HÓ	%€ôF�b—i‘iô F÷R%ô8@)˜Ÿ™ô @)ôFI)˜"Ÿ)™)ô I)ôX�b—i‘iô ô.�R—Y‘Yô .ô:˜Ÿ	™	ô ô�"—)‘)ô ô>Ð+ô >ôB�"—)‘)ô ô 2§9¡9ô ô"˜BŸI™Iô ô 
�2—9‘9ô 
ð@ ô/˜?ó /ó ð/ñ2 ð	ôôE6Ð%ó E6óðE6ñP ðôð
 ô7 ó 7ó óð7ô&	9˜BŸI™Iô 	9ñ ðôô`
Ð.ó `
óð`
ôF!�r—y‘yô !ñ ðôô
X
Ð+¨_ó X
óð
X
ðv ôP
Ð+ó P
ó ðP
ôf&�r—y‘yô &ñ ðôô
P
Ð%9ó P
óð
P
ñf ðôôT
Ð%9ó T
óðT
ðn ôe
Ð1ó e
ó ðe
ðP ô?
Ð"6ó ?
ó ð?
ðD ôI
Ð 4ó I
ó ðI
òX‚rT   