+
    œ°³jŠß  ã                   ó  € R t ^ RIHt ^ RIHt ^ RIt^ RIHt ^ RIHtH	t	H
t
 ^RIHt ^RIHtHt ^RIHtHtHt ^R	IHt ^R
IHtHt ^RIHt ^RIHtHtHtH t H!t!H"t"H#t#H$t$ ^RI%H&t&H't' ^RI(H)t) ^RI*H+t+ ^RI,H-t-H.t.H/t/H0t0 ^RI1H2t2H3t3 ^RI4H5t5 ^RI6H7t7 ]0Pp                  ! ]94      t: ! R R]Pv                  4      t<RKR R llt= ! R R]Pv                  4      t> ! R R]Pv                  4      t? ! R R]Pv                  4      t@ ! R R]Pv                  4      tA ! R  R!]Pv                  4      tB ! R" R#]Pv                  4      tC ! R$ R%]4      tD ! R& R']Pv                  4      tE ! R( R)]Pv                  4      tF ! R* R+]Pv                  4      tG]/ ! R, R-]'4      4       tH]/! R.R/7      ] ! R0 R1]-4      4       4       tI]/ ! R2 R3]H4      4       tJ ! R4 R5]Pv                  4      tK ! R6 R7]Pv                  4      tL]/! R8R/7       ! R9 R:]H4      4       tM]/! R;R/7       ! R< R=]H4      4       tN]/! R>R/7       ! R? R@]H4      4       tO]/! RAR/7       ! RB RC]H4      4       tP]/ ! RD RE]H4      4       tQ]/ ! RF RG]H4      4       tR]/! RHR/7       ! RI RJ]H]4      4       tS. RLOtTR# )MzPyTorch ELECTRA model.)ÚCallable)Ú	dataclassN)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELoss)Úinitialization)ÚACT2FNÚget_activation)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚGradientCheckpointingLayer)Ú"BaseModelOutputWithCrossAttentionsÚ)BaseModelOutputWithPastAndCrossAttentionsÚ!CausalLMOutputWithCrossAttentionsÚMaskedLMOutputÚMultipleChoiceModelOutputÚQuestionAnsweringModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)Úapply_chunking_to_forward)ÚModelOutputÚTransformersKwargsÚauto_docstringÚlogging)Úcan_return_tupleÚmerge_with_config_defaults)Úcapture_outputs©ÚElectraConfigc                   óL   a a€ ] tR t^8t oRtV 3R ltRV3R lR lltRtVtV ;t	# )ÚElectraEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c                óN  <€ \         SV `  4        \        P                  ! VP                  VP
                  VP                  R 7      V n        \        P                  ! VP                  VP
                  4      V n	        \        P                  ! VP                  VP
                  4      V n        \        P                  ! VP
                  VP                  R7      V n        \        P                  ! VP                  4      V n        \        P"                  ! \$        P&                  ! VP                  4      P)                  R4      RR7      V n        \        P"                  ! \$        P,                  ! V P*                  P/                  4       \$        P0                  R7      RR7      V n        R# ))Úpadding_idx©ÚepsF)Ú
persistent©ÚdtypeN©é   éÿÿÿÿ)ÚsuperÚ__init__r   Ú	EmbeddingÚ
vocab_sizeÚembedding_sizeÚpad_token_idÚword_embeddingsÚmax_position_embeddingsÚposition_embeddingsÚtype_vocab_sizeÚtoken_type_embeddingsÚ	LayerNormÚlayer_norm_epsÚDropoutÚhidden_dropout_probÚdropoutÚBufferÚtorchÚarangeÚexpandÚposition_idsÚzerosÚsizeÚlongÚtoken_type_ids©ÚselfÚconfigÚ	__class__s   &&€Úm/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/electra/modeling_electra.pyr4   ÚElectraEmbeddings.__init__;   s  ø€ Ü‰ÑÔÜ!Ÿ|š|¨F×,=Ñ,=¸v×?TÑ?TÐbh×buÑbuÔvˆÔÜ#%§<¢<°×0NÑ0NÐPV×PeÑPeÓ#fˆÔ Ü%'§\¢\°&×2HÑ2HÈ&×J_ÑJ_Ó%`ˆÔ"äŸš f×&;Ñ&;À×AVÑAVÔWˆŒÜ—z’z &×"<Ñ"<Ó=ˆŒô ŸIšI¤e§l¢l°6×3QÑ3QÓ&R×&YÑ&YÐZaÓ&bÐotÔuˆÔÜ Ÿiši¬¯ª°D×4EÑ4E×4JÑ4JÓ4LÔTY×T^ÑT^Ô(_ÐlqÔrˆÖó    c                óÚ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[RS[ P                  /# )é   Ú	input_idsNrK   rG   Úinputs_embedsÚpast_key_values_lengthÚreturn)rD   Ú
LongTensorÚFloatTensorÚintÚTensor)ÚformatÚ__classdict__s   "€rP   Ú__annotate__ÚElectraEmbeddings.__annotate__I   sw   ø€ ÷ (ñ (á×#Ñ# dÕ*ð(ñ ×(Ñ(¨4Õ/ð(ñ ×&Ñ&¨Õ-ð	(ñ
 ×(Ñ(¨4Õ/ð(ñ !$ð(ñ 
�‰ñ(rR   c                óÄ  € Ve   VP                  4       pMVP                  4       R R pVw  rxVf   V P                  RWXV,           13,          pVf¦   \        V R4      '       dY   V P                  P	                  VP
                  ^ ,          R4      p	\        P                  ! V	^VR7      p	V	P	                  Wx4      pM;\        P                  ! V\        P                  V P                  P                  R7      pVf   V P                  V4      pV P                  V4      p
WJ,           pV P                  V4      pW¼,           pV P                  V4      pV P                  V4      pV# )NºNNNrK   )ÚdimÚindex)r/   Údevicer2   )rI   rG   ÚhasattrrK   rF   ÚshaperD   ÚgatherrH   rJ   re   r9   r=   r;   r>   rB   )rM   rU   rK   rG   rV   rW   Úinput_shapeÚ
batch_sizeÚ
seq_lengthÚbuffered_token_type_idsr=   Ú
embeddingsr;   s   &&&&&&       rP   ÚforwardÚElectraEmbeddings.forwardI   sG  € ð Ò Ø#Ÿ.™.Ó*‰Kà'×,Ñ,Ó.¨s°Ð3ˆKà!,Ñˆ
àÒØ×,Ñ,¨QÐ0FÐVlÕIlÐ0lÐ-lÕmˆLð
 Ò!Ü�tÐ-×.Ò.à*.×*=Ñ*=×*DÑ*DÀ\×EWÑEWÐXYÕEZÐ\^Ó*_Ð'Ü*/¯,ª,Ð7NÐTUÐ]iÔ*jÐ'Ø!8×!?Ñ!?À
Ó!W‘ä!&§¢¨[ÄÇ
Á
ÐSW×SdÑSd×SkÑSkÔ!l�àÒ Ø ×0Ñ0°Ó;ˆMØ $× :Ñ :¸>Ó JÐØ"Õ:ˆ
à"×6Ñ6°|ÓDÐØÕ5ˆ
à—^‘^ JÓ/ˆ
Ø—\‘\ *Ó-ˆ
ØÐrR   )r>   rB   r;   rG   r=   rK   r9   )NNNNé    ©
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r4   rn   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©rO   r^   s   @@rP   r(   r(   8   s   ù‡ € ÙQõs÷(÷ (ò (rR   r(   c                ó  € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R,          R\        R	\
        \        ,          /# )
rT   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNÚscalingrB   Úkwargs)r   ÚModulerD   r\   Úfloatr   r   )r]   s   "rP   r_   r_   u   s{   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô �T�\ð%ô ð%ô Ô'Õ(ñ%rR   c                 ó¶  € Vf   VP                  R4      R,          p\        P                  ! WP                  ^^4      4      V,          pVe	   W„,           p\        P
                  P                  VRR7      p\        P
                  P                  W†V P                  R7      p\        P                  ! Wƒ4      p	V	P                  ^^4      P                  4       p	W˜3# )N©rc   )ÚpÚtrainingr2   ç      à¿)
rI   rD   ÚmatmulÚ	transposer   Ú
functionalÚsoftmaxrB   rˆ   Ú
contiguous)
r|   r}   r~   r   r€   r�   rB   r‚   Úattn_weightsÚattn_outputs
   &&&&&&&,  rP   Úeager_attention_forwardr‘   u   s±   € ð ‚Ø—*‘*˜R“. DÕ(ˆô —<’< §}¡}°Q¸Ó':Ó;¸gÕE€LàÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2Ð(Ó>€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€Lä—,’,˜|Ó3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$rR   c                   óL   a a€ ] tR t^’t oRV 3R lltRV3R lR lltRtVtV ;t# )ÚElectraSelfAttentionc                ót  <€ \         SV `  4        VP                  VP                  ,          ^ 8w  d9   \	        VR4      '       g'   \        RVP                   RVP                   R24      hWn        VP                  V n        \        VP                  VP                  ,          4      V n        V P                  V P                  ,          V n	        V P                  R,          V n
        \        P                  ! VP                  V P                  4      V n        \        P                  ! VP                  V P                  4      V n        \        P                  ! VP                  V P                  4      V n        \        P                   ! VP"                  4      V n        VP&                  V n        W n        W0n        R# ©rp   r7   zThe hidden size (z6) is not a multiple of the number of attention heads (Ú)Nr‰   )r3   r4   Úhidden_sizeÚnum_attention_headsrf   Ú
ValueErrorrN   r[   Úattention_head_sizeÚall_head_sizer�   r   ÚLinearr}   r~   r   r@   Úattention_probs_dropout_probrB   Ú
is_decoderÚ	is_causalÚ	layer_idx©rM   rN   rŸ   r    rO   s   &&&&€rP   r4   ÚElectraSelfAttention.__init__“   sM  ø€ Ü‰ÑÔØ×Ñ × :Ñ :Õ:¸aÔ?ÌÐPVÐXh×HiÒHiÜØ# F×$6Ñ$6Ð#7ð 8Ø ×4Ñ4Ð5°Qð8óð ð Œà#)×#=Ñ#=ˆÔ Ü#& v×'9Ñ'9¸F×<VÑ<VÕ'VÓ#WˆÔ Ø!×5Ñ5¸×8PÑ8PÕPˆÔØ×/Ñ/°Õ5ˆŒä—Y’Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
Ü—9’9˜V×/Ñ/°×1CÑ1CÓDˆŒÜ—Y’Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
ä—z’z &×"EÑ"EÓFˆŒà ×+Ñ+ˆŒØ"ŒØ"ŽrR   c                ó°   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  ,          /# )rT   Úhidden_statesr€   NÚpast_key_valuesr‚   rX   ©rD   r\   rZ   r   r   r   Útuple)r]   r^   s   "€rP   r_   Ú!ElectraSelfAttention.__annotate__«   s_   ø€ ÷ ')ñ ')á—|‘|ð')ñ ×)Ñ)¨DÕ0ð')ñ  �ð	')ñ
 Ñ+Õ,ð')ñ 
‰u�|‰|Õ	ñ')rR   c                ó   € VP                   R R p. VORNV P                  N5pV P                  V4      P                  ! V!  P	                  ^^4      pV P                  V4      P                  ! V!  P	                  ^^4      pV P                  V4      P                  ! V!  P	                  ^^4      p	VeC   Tp
\        V\        4      '       d   VP                  p
V
P                  W‰V P                  4      w  r‰\        P                  ! V P                  P                  \         4      pT! V VVV	V3RV P"                  '       g   RMV P$                  P&                  RV P(                  /VB w  rÍVP*                  ! . VORN5!  P-                  4       pWÍ3# )NrB   ç        r�   r2   )rg   rš   r}   Úviewr‹   r~   r   Ú
isinstancer   Úself_attention_cacheÚupdater    r   Úget_interfacerN   Ú_attn_implementationr‘   rˆ   rB   r‡   r�   ÚreshaperŽ   )rM   r¤   r€   r¥   r‚   ri   Úhidden_shapeÚquery_layerÚ	key_layerÚvalue_layerÚcurrent_past_key_valuesÚattention_interfacer�   r�   s   &&&&,         rP   rn   ÚElectraSelfAttention.forward«   s‰  € ð $×)Ñ)¨#¨2Ð.ˆØC˜ÐC bÐC¨$×*BÑ*BÑCˆð —j‘j Ó/×4Ò4°lÑC×MÑMÈaÐQRÓSˆØ—H‘H˜]Ó+×0Ò0°,Ñ?×IÑIÈ!ÈQÓOˆ	Ø—j‘j Ó/×4Ò4°lÑC×MÑMÈaÐQRÓSˆàÒ&à&5Ð#Ü˜/Ô+>×?Ò?Ø*9×*NÑ*NÐ'ð &=×%CÑ%CÀIÐ\`×\jÑ\jÓ%kÑ"ˆIä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð  $Ÿ}Ÿ}˜}‘C°$·,±,·.±.ð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØÐ(Ð(rR   )r›   rš   rN   rB   rŸ   rž   r~   r    r˜   r}   r�   r   ©FN)NN©	rr   rs   rt   ru   r4   rn   rw   rx   ry   rz   s   @@rP   r“   r“   ’   s   ù‡ € ÷#÷0')÷ ')ò ')rR   r“   c                   óL   a a€ ] tR t^Öt oRV 3R lltRV3R lR lltRtVtV ;t# )ÚElectraCrossAttentionc                óR  <€ \         SV `  4        VP                  VP                  ,          ^ 8w  d9   \	        VR4      '       g'   \        RVP                   RVP                   R24      hWn        VP                  V n        \        VP                  VP                  ,          4      V n        V P                  V P                  ,          V n	        V P                  R,          V n
        \        P                  ! VP                  V P                  4      V n        \        P                  ! VP                  V P                  4      V n        \        P                  ! VP                  V P                  4      V n        \        P                   ! VP"                  4      V n        W n        W0n        R# r•   )r3   r4   r—   r˜   rf   r™   rN   r[   rš   r›   r�   r   rœ   r}   r~   r   r@   r�   rB   rŸ   r    r¡   s   &&&&€rP   r4   ÚElectraCrossAttention.__init__×   s@  ø€ Ü‰ÑÔØ×Ñ × :Ñ :Õ:¸aÔ?ÌÐPVÐXh×HiÒHiÜØ# F×$6Ñ$6Ð#7ð 8Ø ×4Ñ4Ð5°Qð8óð ð Œà#)×#=Ñ#=ˆÔ Ü#& v×'9Ñ'9¸F×<VÑ<VÕ'VÓ#WˆÔ Ø!×5Ñ5¸×8PÑ8PÕPˆÔØ×/Ñ/°Õ5ˆŒä—Y’Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
Ü—9’9˜V×/Ñ/°×1CÑ1CÓDˆŒÜ—Y’Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
ä—z’z &×"EÑ"EÓFˆŒà"ŒØ"ŽrR   c                óØ   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  ,          /# )rT   r¤   Úencoder_hidden_statesNr€   r¥   r‚   rX   )rD   r\   rZ   r   r   r   r§   )r]   r^   s   "€rP   r_   Ú"ElectraCrossAttention.__annotate__î   su   ø€ ÷ 1)ñ 1)á—|‘|ð1)ñ  %×0Ñ0°4Õ7ð1)ñ ×)Ñ)¨DÕ0ð	1)ñ
 -¨tÕ3ð1)ñ Ñ+Õ,ð1)ñ 
‰u�|‰|Õ	ñ1)rR   c                ó˜  € VP                   R R p. VORNV P                  N5pV P                  V4      P                  V4      P	                  ^^4      pVe&   VP
                  P                  V P                  4      MRp	Vel   V	'       dd   VP                  P                  V P                  ,          P                  p
VP                  P                  V P                  ,          P                  pMÅ. VP                   R R ORNV P                  N5pV P                  V4      P                  V4      P	                  ^^4      p
V P                  V4      P                  V4      P	                  ^^4      pVeB   VP                  P                  W«V P                  4      w  r«RVP
                  V P                  &   \        P                   ! V P"                  P$                  \&        4      pT! V VV
VV3RV P(                  '       g   RMV P*                  P,                  RV P.                  /VB w  rïVP0                  ! . VORN5!  P3                  4       pWï3# )NFTrB   rª   r�   r2   )rg   rš   r}   r«   r‹   Ú
is_updatedÚgetr    Úcross_attention_cacheÚlayersÚkeysÚvaluesr~   r   r®   r   r¯   rN   r°   r‘   rˆ   rB   r‡   r�   r±   rŽ   )rM   r¤   rÀ   r€   r¥   r‚   ri   r²   r³   rÃ   r´   rµ   Úkv_shaper·   r�   r�   s   &&&&&,          rP   rn   ÚElectraCrossAttention.forwardî   s  € ð $×)Ñ)¨#¨2Ð.ˆàC˜ÐC bÐC¨$×*BÑ*BÑCˆð —j‘j Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆàGVÒGb�_×/Ñ/×3Ñ3°D·N±NÔCÐhmˆ
ØÒ&¯:à'×=Ñ=×DÑDÀTÇ^Á^ÕT×YÑYˆIØ)×?Ñ?×FÑFÀtÇ~Á~ÕV×]Ñ]‰KàXÐ.×4Ñ4°S°bÐ9ÐX¸2ÐX¸t×?WÑ?WÑXˆHØŸ™Ð!6Ó7×<Ñ<¸XÓF×PÑPÐQRÐTUÓVˆIØŸ*™*Ð%:Ó;×@Ñ@ÀÓJ×TÑTÐUVÐXYÓZˆKàÒ*à)8×)NÑ)N×)UÑ)UØ¨D¯N©Nó*Ñ&�	ð >B�×*Ñ*¨4¯>©>Ñ:ä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð  $Ÿ}Ÿ}˜}‘C°$·,±,·.±.ð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØÐ(Ð(rR   )r›   rš   rN   rB   rŸ   r~   r    r˜   r}   r�   r   r¹   )NNNrº   rz   s   @@rP   r¼   r¼   Ö   s   ù‡ € ÷#÷.1)÷ 1)ò 1)rR   r¼   c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚElectraSelfOutputi#  c                ó0  <€ \         SV `  4        \        P                  ! VP                  VP                  4      V n        \        P                  ! VP                  VP                  R 7      V n        \        P                  ! VP                  4      V n
        R# ©r+   N)r3   r4   r   rœ   r—   Údenser>   r?   r@   rA   rB   rL   s   &&€rP   r4   ÚElectraSelfOutput.__init__$  s`   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸš f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z’z &×"<Ñ"<Ó=ˆŽrR   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# ©rT   r¤   Úinput_tensorrX   ©rD   r\   )r]   r^   s   "€rP   r_   ÚElectraSelfOutput.__annotate__*  ó1   ø€ ÷ ñ ¡U§\¡\ð ÁÇÁð ÑRW×R^ÑR^ñ rR   c                óx   € V P                  V4      pV P                  V4      pV P                  W,           4      pV# ©N©rÏ   rB   r>   ©rM   r¤   rÓ   s   &&&rP   rn   ÚElectraSelfOutput.forward*  ó5   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØŸ™ }Õ'CÓDˆØÐrR   ©r>   rÏ   rB   rº   rz   s   @@rP   rÌ   rÌ   #  ó   ù‡ € õ>÷÷ ð rR   rÌ   c                   óL   a a€ ] tR tRt oRV 3R lltRV3R lR lltRtVtV ;t# )ÚElectraAttentioni2  c                ó–   <€ \         SV `  4        W@n        V'       d   \        M\        pV! WVR 7      V n        \        V4      V n        R# )©rŸ   r    N)r3   r4   Úis_cross_attentionr¼   r“   rM   rÌ   Úoutput)rM   rN   rŸ   r    rã   Úattention_classrO   s   &&&&& €rP   r4   ÚElectraAttention.__init__3  s9   ø€ Ü‰ÑÔØ"4Ôß3EÕ/ÔK_ˆÙ# FÈ9ÔUˆŒ	Ü'¨Ó/ˆŽrR   c                ó   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  ,          /# ©	rT   r¤   r€   NrÀ   Úencoder_attention_maskr¥   r‚   rX   r¦   )r]   r^   s   "€rP   r_   ÚElectraAttention.__annotate__:  s‰   ø€ ÷ .ñ .á—|‘|ð.ñ ×)Ñ)¨DÕ0ð.ñ  %×0Ñ0°4Õ7ð	.ñ
 !&× 1Ñ 1°DÕ 8ð.ñ  �ð.ñ Ñ+Õ,ð.ñ 
‰u�|‰|Õ	ñ.rR   c           	     óŒ   € V P                   '       g   TMTpV P                  ! V3R VRVRV/VB w  rxV P                  Wq4      pWx3# )rÀ   r€   r¥   )rã   rM   rä   )	rM   r¤   r€   rÀ   ré   r¥   r‚   Úattention_outputr�   s	   &&&&&&,  rP   rn   ÚElectraAttention.forward:  sm   € ð 04×/F×/FÐ/F™ÐLbˆØ)-¯ªØñ*
à"7ð*
ð *ð*
ð ,ð	*
ð
 ñ*
Ñ&Ðð  Ÿ;™;Ð'7ÓGÐØÐ-Ð-rR   )rã   rä   rM   )FNF©NNNNrº   rz   s   @@rP   rà   rà   2  s   ù‡ € ÷0÷.÷ .ò .rR   rà   c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚElectraIntermediateiP  c                ó"  <€ \         SV `  4        \        P                  ! VP                  VP
                  4      V n        \        VP                  \        4      '       d   \        VP                  ,          V n        R # VP                  V n        R # rØ   )r3   r4   r   rœ   r—   Úintermediate_sizerÏ   r¬   Ú
hidden_actÚstrr	   Úintermediate_act_fnrL   s   &&€rP   r4   ÚElectraIntermediate.__init__Q  s`   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3KÑ3KÓLˆŒ
Ü�f×'Ñ'¬×-Ò-Ü'-¨f×.?Ñ.?Õ'@ˆDÖ$à'-×'8Ñ'8ˆDÖ$rR   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )rT   r¤   rX   rÔ   )r]   r^   s   "€rP   r_   Ú ElectraIntermediate.__annotate__Y  s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ rR   c                óJ   € V P                  V4      pV P                  V4      pV# rØ   ©rÏ   rõ   )rM   r¤   s   &&rP   rn   ÚElectraIntermediate.forwardY  s&   € ØŸ
™
 =Ó1ˆØ×0Ñ0°Ó?ˆØÐrR   rú   rº   rz   s   @@rP   rð   rð   P  s   ù‡ € õ9÷÷ ð rR   rð   c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚElectraOutputi`  c                ó0  <€ \         SV `  4        \        P                  ! VP                  VP
                  4      V n        \        P                  ! VP
                  VP                  R 7      V n        \        P                  ! VP                  4      V n        R# rÎ   )r3   r4   r   rœ   rò   r—   rÏ   r>   r?   r@   rA   rB   rL   s   &&€rP   r4   ÚElectraOutput.__init__a  s`   ø€ Ü‰ÑÔÜ—Y’Y˜v×7Ñ7¸×9KÑ9KÓLˆŒ
ÜŸš f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z’z &×"<Ñ"<Ó=ˆŽrR   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# rÒ   rÔ   )r]   r^   s   "€rP   r_   ÚElectraOutput.__annotate__g  rÖ   rR   c                óx   € V P                  V4      pV P                  V4      pV P                  W,           4      pV# rØ   rÙ   rÚ   s   &&&rP   rn   ÚElectraOutput.forwardg  rÜ   rR   rÝ   rº   rz   s   @@rP   rý   rý   `  rÞ   rR   rý   c                   óR   a a€ ] tR tRt oRV 3R lltRV3R lR lltR tRtVtV ;t	# )	ÚElectraLayerio  c                óœ  <€ \         SV `  4        VP                  V n        ^V n        \	        WP
                  VR7      V n        VP
                  V n        VP                  V n        V P                  '       d5   V P
                  '       g   \        V  R24      h\	        VRVRR7      V n	        \        V4      V n        \        V4      V n        R# )r1   râ   z> should be used as a decoder model if cross attention is addedFT)rŸ   r    rã   N)r3   r4   Úchunk_size_feed_forwardÚseq_len_dimrà   rž   Ú	attentionÚadd_cross_attentionr™   Úcrossattentionrð   Úintermediaterý   rä   )rM   rN   r    rO   s   &&&€rP   r4   ÚElectraLayer.__init__p  s°   ø€ Ü‰ÑÔØ'-×'EÑ'EˆÔ$ØˆÔÜ)¨&×<MÑ<MÐYbÔcˆŒØ ×+Ñ+ˆŒØ#)×#=Ñ#=ˆÔ Ø×#×#Ð#Ø—?—?�?Ü  D 6Ð)gÐ!hÓiÐiÜ"2ØØØ#Ø#'ô	#ˆDÔô 0°Ó7ˆÔÜ# FÓ+ˆŽrR   c                óð   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[S[,          RS[ P                  /# rè   )rD   r\   rZ   r   r   r   )r]   r^   s   "€rP   r_   ÚElectraLayer.__annotate__ƒ  s„   ø€ ÷ %ñ %á—|‘|ð%ñ ×)Ñ)¨DÕ0ð%ñ  %×0Ñ0°4Õ7ð	%ñ
 !&× 1Ñ 1°DÕ 8ð%ñ  �ð%ñ Ñ+Õ,ð%ñ 
�‰ñ%rR   c                ó8  € V P                   ! VV3R V/VB w  rxTp	V P                  '       dB   Ve>   \        V R4      '       g   \        RV  R24      hV P                  ! VRVV3R V/VB w  r¨T
p	\        V P                  V P                  V P                  V	4      pV# )r¥   Nr  z'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`)	r	  rž   rf   r™   r  r   Úfeed_forward_chunkr  r  )rM   r¤   r€   rÀ   ré   r¥   r‚   Úself_attention_outputÚ_rì   Úcross_attention_outputÚlayer_outputs   &&&&&&,     rP   rn   ÚElectraLayer.forwardƒ  sâ   € ð $(§>¢>ØØñ$
ð ,ð$
ð ñ	$
Ñ Ðð 1Ðà�?�?ˆ?Ð4Ò@Ü˜4Ð!1×2Ò2Ü Ø=¸d¸Vð DDð Dóð ð
 )-×(;Ò(;Ø%ØØ%Ø&ñ	)ð
 !0ð)ð ñ)Ñ%Ð"ð  6Ðä0Ø×#Ñ# T×%AÑ%AÀ4×CSÑCSÐUeó
ˆð ÐrR   c                óJ   € V P                  V4      pV P                  W!4      pV# rØ   )r  rä   )rM   rì   Úintermediate_outputr  s   &&  rP   r  ÚElectraLayer.feed_forward_chunkª  s)   € Ø"×/Ñ/Ð0@ÓAÐØ—{‘{Ð#6ÓIˆØÐrR   )r
  r	  r  r  r  rž   rä   r  rØ   rî   )
rr   rs   rt   ru   r4   rn   r  rw   rx   ry   rz   s   @@rP   r  r  o  s   ù‡ € ÷,÷&%ò %÷Nò rR   r  c                   óH   a a€ ] tR tRt oV 3R ltRV3R lR lltRtVtV ;t# )ÚElectraEncoderi±  c           
     óÊ   <€ \         SV `  4        Wn        \        P                  ! \        VP                  4       Uu. uF  p\        WR 7      NK  	  up4      V n        R# u upi ))r    N)	r3   r4   rN   r   Ú
ModuleListÚrangeÚnum_hidden_layersr  Úlayer)rM   rN   ÚirO   s   && €rP   r4   ÚElectraEncoder.__init__²  sH   ø€ Ü‰ÑÔØŒÜ—]’]ÌuÐU[×UmÑUmÔOnÓ#oÑOnÈ!¤L°×$EÑOnÑ#oÓpˆŽ
ùÒ#os   ½A c                ó$  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[,          R	S[S[ P                  ,          S[,          /# )
rT   r¤   r€   NrÀ   ré   r¥   Ú	use_cacher‚   rX   )	rD   r\   rZ   r   Úboolr   r   r§   r   )r]   r^   s   "€rP   r_   ÚElectraEncoder.__annotate__·  s�   ø€ ÷ 
ñ 
á—|‘|ð
ñ ×)Ñ)¨DÕ0ð
ñ  %×0Ñ0°4Õ7ð	
ñ
 !&× 1Ñ 1°DÕ 8ð
ñ  �ð
ñ ˜$•;ð
ñ Ñ+Õ,ð
ñ 
‰u�|‰|Õ	ÑHÕ	Hñ
rR   c                ó”   € \        V P                  4       F  w  r‰V	! VVV3R VRV/VB pK  	  \        TV'       d   VR7      # RR7      # )ré   r¥   N©Úlast_hidden_stater¥   )Ú	enumerater   r   )
rM   r¤   r€   rÀ   ré   r¥   r$  r‚   r!  Úlayer_modules
   &&&&&&&,  rP   rn   ÚElectraEncoder.forward·  sn   € ô  )¨¯©Ö4‰OˆAÙ(ØØØ%ñð (>ð	ð
 !0ðð ñŠMñ  5ô 9Ø+ß/8˜Oô
ð 	
à>Bô
ð 	
rR   )rN   r   )NNNNNrº   rz   s   @@rP   r  r  ±  s   ù‡ € õq÷

÷ 
ò 
rR   r  c                   ó<   a a€ ] tR tRt oRtV 3R ltR tRtVtV ;t	# )ÚElectraDiscriminatorPredictionsiÑ  zEPrediction module for the discriminator, made up of two dense layers.c                ó  <€ \         SV `  4        \        P                  ! VP                  VP                  4      V n        \        VP                  4      V n        \        P                  ! VP                  ^4      V n	        Wn
        R# ©r1   N)r3   r4   r   rœ   r—   rÏ   r
   ró   Ú
activationÚdense_predictionrN   rL   s   &&€rP   r4   Ú(ElectraDiscriminatorPredictions.__init__Ô  s\   ø€ Ü‰ÑÔä—Y’Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
Ü(¨×):Ñ):Ó;ˆŒÜ "§	¢	¨&×*<Ñ*<¸aÓ @ˆÔØŽrR   c                óŠ   € V P                  V4      pV P                  V4      pV P                  V4      P                  R4      pV# r0   )rÏ   r1  r2  Úsqueeze)rM   Údiscriminator_hidden_statesr¤   Úlogitss   &&  rP   rn   Ú'ElectraDiscriminatorPredictions.forwardÜ  s?   € ØŸ
™
Ð#>Ó?ˆØŸ™¨Ó6ˆØ×&Ñ& }Ó5×=Ñ=¸bÓAˆàˆrR   )r1  rN   rÏ   r2  rq   rz   s   @@rP   r.  r.  Ñ  s   ù‡ € ÙOõ÷ò rR   r.  c                   ó<   a a€ ] tR tRt oRtV 3R ltR tRtVtV ;t	# )ÚElectraGeneratorPredictionsiä  zAPrediction module for the generator, made up of two dense layers.c                ó  <€ \         SV `  4        \        R 4      V n        \        P
                  ! VP                  VP                  R7      V n        \        P                  ! VP                  VP                  4      V n
        R# )Úgelur+   N)r3   r4   r
   r1  r   r>   r7   r?   rœ   r—   rÏ   rL   s   &&€rP   r4   Ú$ElectraGeneratorPredictions.__init__ç  sV   ø€ Ü‰ÑÔä(¨Ó0ˆŒÜŸš f×&;Ñ&;À×AVÑAVÔWˆŒÜ—Y’Y˜v×1Ñ1°6×3HÑ3HÓIˆŽ
rR   c                ól   € V P                  V4      pV P                  V4      pV P                  V4      pV# rØ   )rÏ   r1  r>   )rM   Úgenerator_hidden_statesr¤   s   && rP   rn   Ú#ElectraGeneratorPredictions.forwardî  s3   € ØŸ
™
Ð#:Ó;ˆØŸ™¨Ó6ˆØŸ™ }Ó5ˆàÐrR   )r>   r1  rÏ   rq   rz   s   @@rP   r:  r:  ä  s   ù‡ € ÙKõJ÷ò rR   r:  c                   ó^   a a€ ] tR tRt o]tRtRtRtRt	Rt
RtR]R]R]/tV 3R ltRtVtV ;t# )	ÚElectraPreTrainedModeliö  ÚelectraTr¤   Ú
attentionsÚcross_attentionsc                óT  <€ \         SV `  V4       \        V\        4      '       d‚   \        P
                  ! VP                  \        P                  ! VP                  P                  R,          4      P                  R4      4       \        P                  ! VP                  4       R# R# )r1   Nr2   r0   )r3   Ú_init_weightsr¬   r(   ÚinitÚcopy_rG   rD   rE   rg   rF   Úzeros_rK   )rM   r|   rO   s   &&€rP   rG  Ú$ElectraPreTrainedModel._init_weights  sp   ø€ Ü‰Ñ˜fÔ%Ü�fÔ/×0Ò0Ü�JŠJ�v×*Ñ*¬E¯LªL¸×9LÑ9L×9RÑ9RÐSUÕ9VÓ,W×,^Ñ,^Ð_fÓ,gÔhÜ�KŠK˜×-Ñ-Ö.ñ 1rR   © )rr   rs   rt   ru   r&   Úconfig_classÚbase_model_prefixÚsupports_gradient_checkpointingÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_supports_attention_backendr  r“   r¼   Ú_can_record_outputsrG  rw   rx   ry   rz   s   @@rP   rB  rB  ö  sP   ù‡ € à €LØ!ÐØ&*Ð#ØÐØ€NØÐØ"&Ðà˜ØÐ*ØÐ1ðÐ÷/õ /rR   rB  z3
    Output type of [`ElectraForPreTraining`].
    )Úcustom_introc                   ó@   a € ] tR tRt o RtRtRtRtRtV 3R lt	Rt
V tR# )ÚElectraForPreTrainingOutputi  a  
loss (*optional*, returned when `labels` is provided, `torch.FloatTensor` of shape `(1,)`):
    Total loss of the ELECTRA objective.
logits (`torch.FloatTensor` of shape `(batch_size, sequence_length)`):
    Prediction scores of the head (scores for each token before SoftMax).
Nc                óò   <€ V ^8„  d   Qh/ S[ P                  R,          ;R&   S[ P                  R,          ;R&   S[S[ P                  ,          R,          ;R&   S[S[ P                  ,          R,          ;R&   # )rT   NÚlossr7  r¤   rD  )rD   rZ   r§   )r]   r^   s   "€rP   r_   Ú(ElectraForPreTrainingOutput.__annotate__  so   ø‡ ‚ ñ ×
Ñ
˜dÕ
"Ñ)ñ ñ ×Ñ Õ$Ñ+ñ ñ  ™×*Ñ*Õ+¨dÕ2Ñ9ñ! ñ" ‘e×'Ñ'Õ(¨4Õ/Ñ6ò# rR   rL  )rr   rs   rt   ru   rv   rY  r7  r¤   rD  Ú__annotate_func__rw   rx   )r^   s   @rP   rW  rW    s'   ø‡ € ñð &*€DØ'+€FØ59€MØ26€J÷# ƒ rR   rW  c                   óx   a a€ ] tR tRt oV 3R ltR tR t]]]	R	V3R lR ll4       4       4       t
R tRtVtV ;t# )
ÚElectraModeli   c                ó6  <€ \         SV `  V4       \        V4      V n        VP                  VP
                  8w  d1   \        P                  ! VP                  VP
                  4      V n        \        V4      V n
        Wn        R V n        V P                  4        R# r¹   )r3   r4   r(   rm   r7   r—   r   rœ   Úembeddings_projectr  ÚencoderrN   Úgradient_checkpointingÚ	post_initrL   s   &&€rP   r4   ÚElectraModel.__init__"  sr   ø€ Ü‰Ñ˜Ô Ü+¨FÓ3ˆŒà× Ñ  F×$6Ñ$6Ô6Ü&(§i¢i°×0EÑ0EÀv×GYÑGYÓ&ZˆDÔ#ä% fÓ-ˆŒØŒØ&+ˆÔ#à�‰ÖrR   c                ó.   € V P                   P                  # rØ   ©rm   r9   ©rM   s   &rP   Úget_input_embeddingsÚ!ElectraModel.get_input_embeddings/  s   € Ø�‰×.Ñ.Ð.rR   c                ó&   € WP                   n        R # rØ   re  )rM   r   s   &&rP   Úset_input_embeddingsÚ!ElectraModel.set_input_embeddings2  s   € Ø*/�‰Ö'rR   c                óÎ  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[S[ P                  ,          R,          R
S[R,          RS[S[,          RS[S[ P                  ,          S[,          /# )rT   rU   Nr€   rK   rG   rV   rÀ   ré   r¥   r$  r‚   rX   )	rD   r\   ÚlistrZ   r%  r   r   r§   r   )r]   r^   s   "€rP   r_   ÚElectraModel.__annotate__8  sá   ø€ ÷ >
ñ >
á—<‘< $Õ&ð>
ñ Ÿ™ tÕ+ð>
ñ Ÿ™ tÕ+ð	>
ñ
 —l‘l TÕ)ð>
ñ —|‘| dÕ*ð>
ñ  %Ÿ|™|¨dÕ2ð>
ñ !&§¡¨tÕ 3ð>
ñ ™e×/Ñ/Õ0°4Õ7ð>
ñ ˜$•;ð>
ñ Ñ+Õ,ð>
ñ 
‰u�|‰|Õ	ÑAÕ	Añ>
rR   c
                óî  € VR J VR J,          '       d   \        R4      hV P                  P                  '       d   V	e   T	MV P                  P                  p	MRp	V	'       do   Vfk   Vf   V P                  P                  '       d5   \        \        V P                  R7      \        V P                  R7      4      M\        V P                  R7      pVe   VP                  4       M^ pV P                  VVVVVR7      p\        V R4      '       d   V P                  V4      pV P                  VVVVVR7      w  r'V P                  ! V3RVRVR	VR
VRV	RV/V
B p\        VP                  VP                  R7      # )Nz:You must specify exactly one of input_ids or inputs_embedsF)rN   )rU   rG   rK   rV   rW   r_  )r€   ré   Úembedding_outputrÀ   r¥   r€   rÀ   ré   r¥   r$  rG   r(  )r™   rN   rž   r$  Úis_encoder_decoderr   r   Úget_seq_lengthrm   rf   r_  Ú_create_attention_masksr`  r   r)  r¥   )rM   rU   r€   rK   rG   rV   rÀ   ré   r¥   r$  r‚   rW   rp  Úencoder_outputss   &&&&&&&&&&,   rP   rn   ÚElectraModel.forward5  s¡  € ð  ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZà�;‰;×!×!Ð!Ø%.Ò%:™	ÀÇÁ×@UÑ@U‰IàˆIç˜Ò0ð )Ò4¸¿¹×8V×8VÐ8Vô $¤L¸¿¹Ô$DÄlÐZ^×ZeÑZeÔFfÔgä!¨¯©Ô5ð ð FUÒE` ×!?Ñ!?Ô!AÐfgÐàŸ?™?ØØ%Ø)Ø'Ø#9ð +ó 
Ðô �4Ð-×.Ò.Ø#×6Ñ6Ð7GÓHÐà15×1MÑ1MØ)Ø#9Ø-Ø"7Ø+ð 2Nó 2
Ñ.ˆð Ÿ,š,Øñ	
à)ð	
ð #8ð	
ð $:ð		
ð
 ,ð	
ð  ð	
ð &ð	
ð ñ	
ˆô 9Ø-×?Ñ?Ø+×;Ñ;ô
ð 	
rR   c                óÞ   € V P                   P                  '       d   \        V P                   VVVR 7      pM\        V P                   VVR7      pVe   \        V P                   VVVR7      pW3# ))rN   rV   r€   r¥   )rN   rV   r€   )rN   rV   r€   rÀ   )rN   rž   r   r   )rM   r€   ré   rp  rÀ   r¥   s   &&&&&&rP   rs  Ú$ElectraModel._create_attention_masksy  su   € ð �;‰;×!×!Ð!Ü/Ø—{‘{Ø.Ø-Ø /ô	‰Nô 7Ø—{‘{Ø.Ø-ôˆNð "Ò-Ü%>Ø—{‘{Ø.Ø5Ø&;ô	&Ð"ð Ð5Ð5rR   )rN   rm   r_  r`  ra  )	NNNNNNNNN)rr   rs   rt   ru   r4   rg  rj  r#   r$   r    rn   rs  rw   rx   ry   rz   s   @@rP   r]  r]     sH   ù‡ € õò/ò0ð  ØØ÷>
ñ >
ó ó ó  ð>
÷B6ò 6rR   r]  c                   ó<   a a€ ] tR tRt oRtV 3R ltR tRtVtV ;t	# )ÚElectraClassificationHeadiš  z-Head for sentence-level classification tasks.c                ó†  <€ \         SV `  4        \        P                  ! VP                  VP                  4      V n        VP                  e   VP                  MVP                  p\        R4      V n	        \        P                  ! V4      V n        \        P                  ! VP                  VP                  4      V n        R # )Nr<  )r3   r4   r   rœ   r—   rÏ   Úclassifier_dropoutrA   r
   r1  r@   rB   Ú
num_labelsÚout_proj©rM   rN   r{  rO   s   && €rP   r4   Ú"ElectraClassificationHead.__init__�  s‹   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
à)/×)BÑ)BÒ)NˆF×%Ò%ÐTZ×TnÑTnð 	ô )¨Ó0ˆŒÜ—z’zÐ"4Ó5ˆŒÜŸ	š	 &×"4Ñ"4°f×6GÑ6GÓHˆŽrR   c                óÂ   € VR,          pV P                  V4      pV P                  V4      pV P                  V4      pV P                  V4      pV P                  V4      pV# )rb   )rb   rp   rb   )rB   rÏ   r1  r}  )rM   Úfeaturesr‚   Úxs   &&, rP   rn   Ú!ElectraClassificationHead.forward§  sT   € Ø�WÕˆØ�L‰L˜‹OˆØ�J‰J�q‹MˆØ�O‰O˜AÓˆØ�L‰L˜‹OˆØ�M‰M˜!ÓˆØˆrR   )r1  rÏ   rB   r}  rq   rz   s   @@rP   ry  ry  š  s   ù‡ € Ù7õI÷ò rR   ry  c                   óX   a a€ ] tR tRt oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )	ÚElectraSequenceSummaryi²  aƒ  
Compute a single vector summary of a sequence hidden states.

Args:
    config ([`ElectraConfig`]):
        The config used by the model. Relevant arguments in the config class of the model are (refer to the actual
        config class of your model for the default values it uses):

        - **summary_type** (`str`) -- The method to use to make this summary. Accepted values are:

            - `"last"` -- Take the last token hidden state (like XLNet)
            - `"first"` -- Take the first token hidden state (like Bert)
            - `"mean"` -- Take the mean of all tokens hidden states
            - `"cls_index"` -- Supply a Tensor of classification token position (GPT/GPT-2)
            - `"attn"` -- Not implemented now, use multi-head attention

        - **summary_use_proj** (`bool`) -- Add a projection after the vector extraction.
        - **summary_proj_to_labels** (`bool`) -- If `True`, the projection outputs to `config.num_labels` classes
          (otherwise to `config.hidden_size`).
        - **summary_activation** (`Optional[str]`) -- Set to `"tanh"` to add a tanh activation to the output,
          another string or `None` will add no activation.
        - **summary_first_dropout** (`float`) -- Optional dropout probability before the projection and activation.
        - **summary_last_dropout** (`float`)-- Optional dropout probability after the projection and activation.
c                ó    <€ V ^8„  d   QhRS[ /# )rT   rN   r%   )r]   r^   s   "€rP   r_   Ú#ElectraSequenceSummary.__annotate__Ì  s   ø€ ÷ Hñ H™}ñ HrR   c                óÞ  <€ \         SV `  4        \        VR R4      V n        V P                  R8X  d   \        h\
        P                  ! 4       V n        \        VR4      '       d‡   VP                  '       du   \        VR4      '       d1   VP                  '       d   VP                  ^ 8”  d   VP                  pMVP                  p\
        P                  ! VP                  V4      V n        \        VRR4      pV'       d   \        V4      M\
        P                  ! 4       V n        \
        P                  ! 4       V n        \        VR4      '       d7   VP"                  ^ 8”  d&   \
        P$                  ! VP"                  4      V n        \
        P                  ! 4       V n        \        VR4      '       d;   VP(                  ^ 8”  d(   \
        P$                  ! VP(                  4      V n        R# R# R# )	Úsummary_typeÚlastÚattnÚsummary_use_projÚsummary_proj_to_labelsÚsummary_activationNÚsummary_first_dropoutÚsummary_last_dropout)r3   r4   Úgetattrr‰  ÚNotImplementedErrorr   ÚIdentityÚsummaryrf   rŒ  r�  r|  r—   rœ   r
   r1  Úfirst_dropoutr�  r@   Úlast_dropoutr�  )rM   rN   Únum_classesÚactivation_stringrO   s   &&  €rP   r4   ÚElectraSequenceSummary.__init__Ì  sg  ø€ Ü‰ÑÔä# F¨N¸FÓCˆÔØ×Ñ Ô&ô &Ð%ä—{’{“}ˆŒÜ�6Ð-×.Ò.°6×3J×3JÐ3JÜ�vÐ7×8Ò8¸V×=Z×=ZÐ=ZÐ_e×_pÑ_pÐstÔ_tØ$×/Ñ/‘à$×0Ñ0�ÜŸ9š9 V×%7Ñ%7¸ÓEˆDŒLä# FÐ,@À$ÓGÐßIZ¤NÐ3DÔ$EÔ`b×`kÒ`kÓ`mˆŒäŸ[š[›]ˆÔÜ�6Ð2×3Ò3¸×8TÑ8TÐWXÔ8XÜ!#§¢¨F×,HÑ,HÓ!IˆDÔäŸKšK›MˆÔÜ�6Ð1×2Ò2°v×7RÑ7RÐUVÔ7VÜ "§
¢
¨6×+FÑ+FÓ GˆDÖñ 8WÑ2rR   c                óv   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  /# )rT   r¤   Ú	cls_indexNrX   )rD   rZ   rY   )r]   r^   s   "€rP   r_   r‡  é  s<   ø€ ÷ )ñ )Ù"×.Ñ.ð)Ù;@×;KÑ;KÈdÕ;Rð)á	×	Ñ	ñ)rR   c                ó8  € V P                   R8X  d   VR	,          pEM9V P                   R8X  d   VR
,          pEMV P                   R8X  d   VP                  ^R7      pMùV P                   R8X  dÒ   VfH   \        P                  ! VR,          VP                  R,          ^,
          \        P
                  R7      pMdVP                  R4      P                  R4      pVP                  RVP                  4       ^,
          ,          VP                  R4      3,           4      pVP                  RV4      P                  R4      pMV P                   R8X  d   \        hV P                  X4      pV P                  V4      pV P                  V4      pV P!                  V4      pV# )a#  
Compute a single vector summary of a sequence hidden states.

Args:
    hidden_states (`torch.FloatTensor` of shape `[batch_size, seq_len, hidden_size]`):
        The hidden states of the last layer.
    cls_index (`torch.LongTensor` of shape `[batch_size]` or `[batch_size, ...]` where ... are optional leading dimensions of `hidden_states`, *optional*):
        Used if `summary_type == "cls_index"` and takes the last token of the sequence as classification token.

Returns:
    `torch.FloatTensor`: The summary of the sequence hidden states.
rŠ  ÚfirstÚmeanr†   r›  r.   r‹  r2   )rb   r2   )rb   rp   ).:Nr1   Nrb   éþÿÿÿ)r2   )r‰  rž  rD   Ú	full_likerg   rJ   Ú	unsqueezerF   rc   rI   rh   r5  r’  r•  r”  r1  r–  )rM   r¤   r›  rä   s   &&& rP   rn   ÚElectraSequenceSummary.forwardé  s\  € ð ×Ñ Ô&Ø" 5Õ)ŠFØ×Ñ 'Ô)Ø" 4Õ(ŠFØ×Ñ &Ô(Ø"×'Ñ'¨AÐ'Ó.‰FØ×Ñ +Ô-ØÒ Ü!ŸOšOØ! *Õ-Ø!×'Ñ'¨Õ+¨aÕ/ÜŸ*™*ô‘	ð &×/Ñ/°Ó3×=Ñ=¸bÓA�	Ø%×,Ñ,¨U°i·m±m³oÈÕ6IÕ-JÈm×N`ÑN`ÐacÓNdÐMfÕ-fÓg�	à"×)Ñ)¨"¨iÓ8×@Ñ@ÀÓD‰FØ×Ñ &Ô(Ü%Ð%à×#Ñ# FÓ+ˆØ—‘˜fÓ%ˆØ—‘ Ó(ˆØ×"Ñ" 6Ó*ˆàˆrR   )r1  r•  r–  r”  r‰  rØ   rq   rz   s   @@rP   r…  r…  ²  s%   ù‡ € ñ÷2Hó H÷:)÷ )ò )rR   r…  zŸ
    ELECTRA Model transformer with a sequence classification/regression head on top (a linear layer on top of the
    pooled output) e.g. for GLUE tasks.
    c                   ó\   a a€ ] tR tRt oV 3R lt]]RV3R lR ll4       4       tRtVt	V ;t
# )Ú ElectraForSequenceClassificationi  c                ó´   <€ \         SV `  V4       VP                  V n        Wn        \	        V4      V n        \        V4      V n        V P                  4        R # rØ   )	r3   r4   r|  rN   r]  rC  ry  Ú
classifierrb  rL   s   &&€rP   r4   Ú)ElectraForSequenceClassification.__init__  sF   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒØŒÜ# FÓ+ˆŒÜ3°FÓ;ˆŒð 	�‰ÖrR   c                óZ  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          R	S[S[ P                  ,          S[,          /# ©
rT   rU   Nr€   rK   rG   rV   Úlabelsr‚   rX   )rD   r\   r   r   r§   r   )r]   r^   s   "€rP   r_   Ú-ElectraForSequenceClassification.__annotate__(  s¦   ø€ ÷ 9
ñ 9
á—<‘< $Õ&ð9
ñ Ÿ™ tÕ+ð9
ñ Ÿ™ tÕ+ð	9
ñ
 —l‘l TÕ)ð9
ñ —|‘| dÕ*ð9
ñ —‘˜tÕ#ð9
ñ Ñ+Õ,ð9
ñ 
‰u�|‰|Õ	Ñ7Õ	7ñ9
rR   c                óò  € V P                   ! V3RVRVRVRVRR/VB pV^ ,          p	V P                  V	4      p
RpVEe˜   V P                  P                  f–   V P                  ^8X  d   RV P                  n        MrV P                  ^8”  dQ   VP
                  \        P                  8X  g    VP
                  \        P                  8X  d   R	V P                  n        MR
V P                  n        V P                  P                  R8X  dK   \        4       pV P                  ^8X  d'   V! V
P                  4       VP                  4       4      pMŽV! W¦4      pM…V P                  P                  R	8X  d>   \        4       pV! V
P                  RV P                  4      VP                  R4      4      pM-V P                  P                  R
8X  d   \        4       pV! W¦4      p\        VV
VP                  VP                   R7      # )ae  
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
    `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
r€   rK   rG   rV   Úreturn_dictTNÚ
regressionÚsingle_label_classificationÚmulti_label_classification©rY  r7  r¤   rD  r2   )rC  r¦  rN   Úproblem_typer|  r/   rD   rJ   r[   r   r5  r   r«   r   r   r¤   rD  )rM   rU   r€   rK   rG   rV   rª  r‚   r6  Úsequence_outputr7  rY  Úloss_fcts   &&&&&&&,     rP   rn   Ú(ElectraForSequenceClassification.forward&  s³  € ð$ '+§l¢lØñ'
à)ð'
ð *ð'
ð &ð	'
ð
 (ð'
ð ð'
ð ñ'
Ð#ð 6°aÕ8ˆØ—‘ Ó1ˆàˆØÓØ�{‰{×'Ñ'Ò/Ø—?‘? aÔ'Ø/;�D—K‘KÕ,Ø—_‘_ qÔ(¨f¯l©l¼e¿j¹jÔ.HÈFÏLÉLÔ\a×\eÑ\eÔLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ô7Ü"›9�Ø—?‘? aÔ'Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# FÓ3‘DØ—‘×)Ñ)Ð-JÔJÜ+Ó-�Ù §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR‘Ø—‘×)Ñ)Ð-IÔIÜ,Ó.�Ù Ó/�ä'ØØØ5×CÑCØ2×=Ñ=ô	
ð 	
rR   )r¦  rN   rC  r|  ©NNNNNN©rr   rs   rt   ru   r4   r"   r    rn   rw   rx   ry   rz   s   @@rP   r¤  r¤    s0   ù‡ € õð Ø÷9
ñ 9
ó ó ÷9
ð 9
rR   r¤  zÊ
    Electra model with a binary classification head on top as used during pretraining for identifying generated tokens.

    It is recommended to load the discriminator checkpoint into that model.
    c                   ó\   a a€ ] tR tRt oV 3R lt]]RV3R lR ll4       4       tRtVt	V ;t
# )ÚElectraForPreTrainingid  c                ó†   <€ \         SV `  V4       \        V4      V n        \	        V4      V n        V P                  4        R # rØ   )r3   r4   r]  rC  r.  Údiscriminator_predictionsrb  rL   s   &&€rP   r4   ÚElectraForPreTraining.__init__l  s3   ø€ Ü‰Ñ˜Ô ä# FÓ+ˆŒÜ)HÈÓ)PˆÔ&à�‰ÖrR   c                óZ  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          R	S[S[ P                  ,          S[,          /# r©  )rD   r\   r   r   r§   rW  )r]   r^   s   "€rP   r_   Ú"ElectraForPreTraining.__annotate__v  s°   ø€ ÷ F
ñ F
á—<‘< $Õ&ðF
ñ Ÿ™ tÕ+ðF
ñ Ÿ™ tÕ+ð	F
ñ
 —l‘l TÕ)ðF
ñ —|‘| dÕ*ðF
ñ —‘˜tÕ#ðF
ñ Ñ+Õ,ðF
ñ 
‰u�|‰|Õ	Ñ:Õ	:ñF
rR   c                óB  € V P                   ! V3RVRVRVRVRR/VB pV^ ,          p	V P                  V	4      p
RpVeÁ   \        P                  ! 4       pVep   VP	                  R	V	P
                  ^,          4      ^8H  pV
P	                  R	V	P
                  ^,          4      V,          pWm,          pV! WïP                  4       4      pM8V! V
P	                  R	V	P
                  ^,          4      VP                  4       4      p\        VV
VP                  VP                  R7      # )
a½  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the ELECTRA loss. Input should be a sequence of tokens (see `input_ids` docstring)
    Indices should be in `[0, 1]`:

    - 0 indicates the token is an original token,
    - 1 indicates the token was replaced.

Examples:

```python
>>> from transformers import ElectraForPreTraining, AutoTokenizer
>>> import torch

>>> discriminator = ElectraForPreTraining.from_pretrained("google/electra-base-discriminator")
>>> tokenizer = AutoTokenizer.from_pretrained("google/electra-base-discriminator")

>>> sentence = "The quick brown fox jumps over the lazy dog"
>>> fake_sentence = "The quick brown fox fake over the lazy dog"

>>> fake_tokens = tokenizer.tokenize(fake_sentence, add_special_tokens=True)
>>> fake_inputs = tokenizer.encode(fake_sentence, return_tensors="pt")
>>> discriminator_outputs = discriminator(fake_inputs)
>>> predictions = torch.round((torch.sign(discriminator_outputs[0]) + 1) / 2)

>>> fake_tokens
['[CLS]', 'the', 'quick', 'brown', 'fox', 'fake', 'over', 'the', 'lazy', 'dog', '[SEP]']

>>> predictions.squeeze().tolist()
[0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0]
```r€   rK   rG   rV   r­  TNr±  r2   )
rC  r»  r   r   r«   rg   r„   rW  r¤   rD  )rM   rU   r€   rK   rG   rV   rª  r‚   r6  Údiscriminator_sequence_outputr7  rY  r´  Úactive_lossÚactive_logitsÚactive_labelss   &&&&&&&,        rP   rn   ÚElectraForPreTraining.forwardt  sE  € ðV '+§l¢lØñ'
à)ð'
ð *ð'
ð &ð	'
ð
 (ð'
ð ð'
ð ñ'
Ð#ð )DÀAÕ(FÐ%à×/Ñ/Ð0MÓNˆàˆØÒÜ×+Ò+Ó-ˆHØÒ)Ø,×1Ñ1°"Ð6S×6YÑ6YÐZ[Õ6\Ó]ÐabÑb�Ø &§¡¨BÐ0M×0SÑ0SÐTUÕ0VÓ WÐXcÕ d�Ø &Õ 3�Ù ×/BÑ/BÓ/DÓE‘á §¡¨BÐ0M×0SÑ0SÐTUÕ0VÓ WÐY_×YeÑYeÓYgÓh�ä*ØØØ5×CÑCØ2×=Ñ=ô	
ð 	
rR   )r»  rC  r¶  r·  rz   s   @@rP   r¹  r¹  d  s4   ù‡ € õð Ø÷F
ñ F
ó ó ÷F
ð F
rR   r¹  zý
    Electra model with a language modeling head on top.

    Even though both the discriminator and generator may be loaded into this model, the generator is the only model of
    the two to have been trained for the masked language modeling task.
    c                   óp   a a€ ] tR tRt oRR/tV 3R ltR tR t]]	R
V3R lR ll4       4       t
R	tVtV ;t# )ÚElectraForMaskedLMi¿  úgenerator_lm_head.weightú)electra.embeddings.word_embeddings.weightc                óæ   <€ \         SV `  V4       \        V4      V n        \	        V4      V n        \        P                  ! VP                  VP                  4      V n
        V P                  4        R # rØ   )r3   r4   r]  rC  r:  Úgenerator_predictionsr   rœ   r7   r6   Úgenerator_lm_headrb  rL   s   &&€rP   r4   ÚElectraForMaskedLM.__init__Ê  sR   ø€ Ü‰Ñ˜Ô ä# FÓ+ˆŒÜ%@ÀÓ%HˆÔ"ä!#§¢¨6×+@Ñ+@À&×BSÑBSÓ!TˆÔà�‰ÖrR   c                ó   € V P                   # rØ   ©rË  rf  s   &rP   Úget_output_embeddingsÚ(ElectraForMaskedLM.get_output_embeddingsÔ  ó   € Ø×%Ñ%Ð%rR   c                ó   € Wn         R # rØ   rÎ  )rM   r9   s   &&rP   Úset_output_embeddingsÚ(ElectraForMaskedLM.set_output_embeddings×  s   € Ø!0ÖrR   c                óZ  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          R	S[S[ P                  ,          S[,          /# r©  )rD   r\   r   r   r§   r   )r]   r^   s   "€rP   r_   ÚElectraForMaskedLM.__annotate__Ü  s¥   ø€ ÷ )
ñ )
á—<‘< $Õ&ð)
ñ Ÿ™ tÕ+ð)
ñ Ÿ™ tÕ+ð	)
ñ
 —l‘l TÕ)ð)
ñ —|‘| dÕ*ð)
ñ —‘˜tÕ#ð)
ñ Ñ+Õ,ð)
ñ 
‰u�|‰|Õ	™~Õ	-ñ)
rR   c                ó†  € V P                   ! V3RVRVRVRVRR/VB pV^ ,          p	V P                  V	4      p
V P                  V
4      p
RpVeR   \        P                  ! 4       pV! V
P                  R	V P                  P                  4      VP                  R	4      4      p\        VV
VP                  VP                  R7      # )
az  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
    config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
    loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
r€   rK   rG   rV   r­  TNr±  r2   )rC  rÊ  rË  r   r   r«   rN   r6   r   r¤   rD  )rM   rU   r€   rK   rG   rV   rª  r‚   r?  Úgenerator_sequence_outputÚprediction_scoresrY  r´  s   &&&&&&&,     rP   rn   ÚElectraForMaskedLM.forwardÚ  sð   € ð$ #'§,¢,Øñ#
à)ð#
ð *ð#
ð &ð	#
ð
 (ð#
ð ð#
ð ñ#
Ðð %<¸AÕ$>Ð!à ×6Ñ6Ð7PÓQÐØ ×2Ñ2Ð3DÓEÐàˆàÒÜ×*Ò*Ó,ˆHÙÐ-×2Ñ2°2°t·{±{×7MÑ7MÓNÐPV×P[ÑP[Ð\^ÓP_Ó`ˆDäØØ$Ø1×?Ñ?Ø.×9Ñ9ô	
ð 	
rR   ©rC  rË  rÊ  r¶  ©rr   rs   rt   ru   Ú_tied_weights_keysr4   rÏ  rÓ  r"   r    rn   rw   rx   ry   rz   s   @@rP   rÆ  rÆ  ¿  sH   ù‡ € ð 5Ð6aÐbÐõò&ò1ð Ø÷)
ñ )
ó ó ÷)
ð )
rR   rÆ  z‰
    Electra model with a token classification head on top.

    Both the discriminator and generator may be loaded into this model.
    c                   ó\   a a€ ] tR tRt oV 3R lt]]RV3R lR ll4       4       tRtVt	V ;t
# )ÚElectraForTokenClassificationi  c                ój  <€ \         SV `  V4       VP                  V n        \        V4      V n        VP
                  e   VP
                  MVP                  p\        P                  ! V4      V n	        \        P                  ! VP                  VP                  4      V n        V P                  4        R # rØ   )r3   r4   r|  r]  rC  r{  rA   r   r@   rB   rœ   r—   r¦  rb  r~  s   && €rP   r4   Ú&ElectraForTokenClassification.__init__  s†   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä# FÓ+ˆŒà)/×)BÑ)BÒ)NˆF×%Ò%ÐTZ×TnÑTnð 	ô —z’zÐ"4Ó5ˆŒÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒà�‰ÖrR   c                óZ  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          R	S[S[ P                  ,          S[,          /# r©  )rD   r\   r   r   r§   r   )r]   r^   s   "€rP   r_   Ú*ElectraForTokenClassification.__annotate__  s¦   ø€ ÷ &
ñ &
á—<‘< $Õ&ð&
ñ Ÿ™ tÕ+ð&
ñ Ÿ™ tÕ+ð	&
ñ
 —l‘l TÕ)ð&
ñ —|‘| dÕ*ð&
ñ —‘˜tÕ#ð&
ñ Ñ+Õ,ð&
ñ 
‰u�|‰|Õ	Ñ4Õ	4ñ&
rR   c                ó\  € V P                   ! V3RVRVRVRVRR/VB pV^ ,          p	V P                  V	4      p	V P                  V	4      p
RpVe=   \        4       pV! V
P	                  R	V P
                  4      VP	                  R	4      4      p\        VV
VP                  VP                  R7      # )
zÃ
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
r€   rK   rG   rV   r­  TNr±  r2   )	rC  rB   r¦  r   r«   r|  r   r¤   rD  )rM   rU   r€   rK   rG   rV   rª  r‚   r6  rÀ  r7  rY  r´  s   &&&&&&&,     rP   rn   Ú%ElectraForTokenClassification.forward  sØ   € ð  '+§l¢lØñ'
à)ð'
ð *ð'
ð &ð	'
ð
 (ð'
ð ð'
ð ñ'
Ð#ð )DÀAÕ(FÐ%à(,¯©Ð5RÓ(SÐ%Ø—‘Ð!>Ó?ˆàˆØÒÜ'Ó)ˆHÙ˜FŸK™K¨¨D¯O©OÓ<¸f¿k¹kÈ"»oÓNˆDä$ØØØ5×CÑCØ2×=Ñ=ô	
ð 	
rR   )r¦  rB   rC  r|  r¶  r·  rz   s   @@rP   rß  rß    s0   ù‡ € õð Ø÷&
ñ &
ó ó ÷&
ð &
rR   rß  c                   ód   a a€ ] tR tRt o]tRtV 3R lt]]	RV3R lR ll4       4       t
RtVtV ;t# )ÚElectraForQuestionAnsweringiH  rC  c                óè   <€ \         SV `  V4       VP                  V n        \        V4      V n        \
        P                  ! VP                  VP                  4      V n        V P                  4        R # rØ   )
r3   r4   r|  r]  rC  r   rœ   r—   Ú
qa_outputsrb  rL   s   &&€rP   r4   Ú$ElectraForQuestionAnswering.__init__M  sS   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä# FÓ+ˆŒÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰ÖrR   c                ó‚  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[S[,          R
S[S[ P                  ,          S[,          /	# )rT   rU   Nr€   rK   rG   rV   Ústart_positionsÚend_positionsr‚   rX   )rD   r\   r   r   r§   r   )r]   r^   s   "€rP   r_   Ú(ElectraForQuestionAnswering.__annotate__Y  s¹   ø€ ÷ 3
ñ 3
á—<‘< $Õ&ð3
ñ Ÿ™ tÕ+ð3
ñ Ÿ™ tÕ+ð	3
ñ
 —l‘l TÕ)ð3
ñ —|‘| dÕ*ð3
ñ Ÿ™¨Õ,ð3
ñ —|‘| dÕ*ð3
ñ Ñ+Õ,ð3
ñ 
‰u�|‰|Õ	Ñ;Õ	;ñ3
rR   c                óð  € V P                   ! V3R VRVRVRVRR/VB p	V	^ ,          p
V P                  V
4      pVP                  ^R
R7      w  rÍVP                  R
4      P	                  4       pVP                  R
4      P	                  4       pRpVeÄ   VeÀ   \        VP                  4       4      ^8”  d   VP                  R
4      p\        VP                  4       4      ^8”  d   VP                  R
4      pVP                  ^4      pVP                  ^ V4      pVP                  ^ V4      p\        VR7      pV! WÆ4      pV! W×4      pVV,           ^,          p\        VVVV	P                  V	P                  R	7      # )r€   rK   rG   rV   r­  Tr†   N)Úignore_index)rY  Ústart_logitsÚ
end_logitsr¤   rD  r2   )rC  ré  Úsplitr5  rŽ   ÚlenrI   Úclampr   r   r¤   rD  )rM   rU   r€   rK   rG   rV   rì  rí  r‚   r6  r³  r7  rñ  rò  Ú
total_lossÚignored_indexr´  Ú
start_lossÚend_losss   &&&&&&&&,          rP   rn   Ú#ElectraForQuestionAnswering.forwardW  s–  € ð '+§l¢lØñ'
à)ð'
ð *ð'
ð &ð	'
ð
 (ð'
ð ð'
ð ñ'
Ð#ð 6°aÕ8ˆà—‘ Ó1ˆØ#)§<¡<°°r <Ó#:Ñ ˆØ#×+Ñ+¨BÓ/×:Ñ:Ó<ˆØ×'Ñ'¨Ó+×6Ñ6Ó8ˆ
àˆ
ØÒ&¨=Ò+Dä�?×'Ñ'Ó)Ó*¨QÔ.Ø"1×"9Ñ"9¸"Ó"=�Ü�=×%Ñ%Ó'Ó(¨1Ô,Ø -× 5Ñ 5°bÓ 9�à(×-Ñ-¨aÓ0ˆMØ-×3Ñ3°A°}ÓEˆOØ)×/Ñ/°°=ÓAˆMä'°]ÔCˆHÙ! ,Ó@ˆJÙ 
Ó:ˆHØ$ xÕ/°1Õ4ˆJä+ØØ%Ø!Ø5×CÑCØ2×=Ñ=ô
ð 	
rR   )rC  r|  ré  )NNNNNNN)rr   rs   rt   ru   r&   rM  rN  r4   r"   r    rn   rw   rx   ry   rz   s   @@rP   rç  rç  H  s;   ù‡ € à €LØ!Ðõð Ø÷3
ñ 3
ó ó ÷3
ð 3
rR   rç  c                   ó\   a a€ ] tR tRt oV 3R lt]]RV3R lR ll4       4       tRtVt	V ;t
# )ÚElectraForMultipleChoicei�  c                óÒ   <€ \         SV `  V4       \        V4      V n        \	        V4      V n        \        P                  ! VP                  ^4      V n	        V P                  4        R# r0  )r3   r4   r]  rC  r…  Úsequence_summaryr   rœ   r—   r¦  rb  rL   s   &&€rP   r4   Ú!ElectraForMultipleChoice.__init__‘  sM   ø€ Ü‰Ñ˜Ô ä# FÓ+ˆŒÜ 6°vÓ >ˆÔÜŸ)š) F×$6Ñ$6¸Ó:ˆŒð 	�‰ÖrR   c                óZ  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          R	S[S[ P                  ,          S[,          /# r©  )rD   r\   r   r   r§   r   )r]   r^   s   "€rP   r_   Ú%ElectraForMultipleChoice.__annotate__�  s°   ø€ ÷ N
ñ N
á—<‘< $Õ&ðN
ñ Ÿ™ tÕ+ðN
ñ Ÿ™ tÕ+ð	N
ñ
 —l‘l TÕ)ðN
ñ —|‘| dÕ*ðN
ñ —‘˜tÕ#ðN
ñ Ñ+Õ,ðN
ñ 
‰u�|‰|Õ	Ñ8Õ	8ñN
rR   c                ó&  € Ve   VP                   ^,          MVP                   ^,          pVe"   VP                  R	VP                  R	4      4      MRpVe"   VP                  R	VP                  R	4      4      MRpVe"   VP                  R	VP                  R	4      4      MRpVe"   VP                  R	VP                  R	4      4      MRpVe2   VP                  R	VP                  R
4      VP                  R	4      4      MRpV P                  ! V3RVRVRVRVRR/VB p	V	^ ,          p
V P	                  V
4      pV P                  V4      pVP                  R	V4      pRpVe   \        4       pV! WÖ4      p\        VVV	P                  V	P                  R7      # )a›  
input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
    Indices of input sequence tokens in the vocabulary.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are input IDs?](../glossary#input-ids)
token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
    Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
    1]`:

    - 0 corresponds to a *sentence A* token,
    - 1 corresponds to a *sentence B* token.

    [What are token type IDs?](../glossary#token-type-ids)
position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
    Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
    config.max_position_embeddings - 1]`.

    [What are position IDs?](../glossary#position-ids)
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):
    Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
    is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
    model's internal embedding lookup matrix.
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
    num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
    `input_ids` above)
Nr€   rK   rG   rV   r­  Tr±  r2   rŸ  )
rg   r«   rI   rC  rþ  r¦  r   r   r¤   rD  )rM   rU   r€   rK   rG   rV   rª  r‚   Únum_choicesr6  r³  Úpooled_outputr7  Úreshaped_logitsrY  r´  s   &&&&&&&,        rP   rn   Ú ElectraForMultipleChoice.forward›  sº  € ðT -6Ò,A�i—o‘o aÖ(À}×GZÑGZÐ[\ÕG]ˆà>GÒ>S�I—N‘N 2 y§~¡~°bÓ'9Ô:ÐY]ˆ	ØM[ÒMg˜×,Ñ,¨R°×1DÑ1DÀRÓ1HÔIÐmqˆØM[ÒMg˜×,Ñ,¨R°×1DÑ1DÀRÓ1HÔIÐmqˆØGSÒG_�|×(Ñ(¨¨\×->Ñ->¸rÓ-BÔCÐeiˆð Ò(ð ×Ñ˜r =×#5Ñ#5°bÓ#9¸=×;MÑ;MÈbÓ;QÔRàð 	ð '+§l¢lØñ'
à)ð'
ð *ð'
ð &ð	'
ð
 (ð'
ð ð'
ð ñ'
Ð#ð 6°aÕ8ˆà×-Ñ-¨oÓ>ˆØ—‘ Ó/ˆØ Ÿ+™+ b¨+Ó6ˆàˆØÒÜ'Ó)ˆHÙ˜OÓ4ˆDä(ØØ"Ø5×CÑCØ2×=Ñ=ô	
ð 	
rR   )r¦  rC  rþ  r¶  r·  rz   s   @@rP   rü  rü  �  s4   ù‡ € õð Ø÷N
ñ N
ó ó ÷N
ð N
rR   rü  zS
    ELECTRA Model with a `language modeling` head on top for CLM fine-tuning.
    c                   óp   a a€ ] tR tRt oRR/tV 3R ltR tR t]]	R
V3R lR ll4       4       t
R	tVtV ;t# )ÚElectraForCausalLMiî  rÇ  rÈ  c                ó4  <€ \         SV `  V4       VP                  '       g   \        P	                  R 4       \        V4      V n        \        V4      V n        \        P                  ! VP                  VP                  4      V n        V P                  4        R# )zOIf you want to use `ElectraForCausalLM` as a standalone, add `is_decoder=True.`N)r3   r4   rž   ÚloggerÚwarningr]  rC  r:  rÊ  r   rœ   r7   r6   rË  rb  rL   s   &&€rP   r4   ÚElectraForCausalLM.__init__ö  sk   ø€ Ü‰Ñ˜Ô à× × Ð Ü�N‰NÐlÔmä# FÓ+ˆŒÜ%@ÀÓ%HˆÔ"Ü!#§¢¨6×+@Ñ+@À&×BSÑBSÓ!TˆÔà�‰ÖrR   c                ó   € V P                   # rØ   rÎ  rf  s   &rP   rÏ  Ú(ElectraForCausalLM.get_output_embeddings  rÑ  rR   c                ó   € Wn         R # rØ   rÎ  )rM   Únew_embeddingss   &&rP   rÓ  Ú(ElectraForCausalLM.set_output_embeddings  s   € Ø!/ÖrR   c                óü  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[ P                  R,          R
S[R,          RS[R,          RS[S[ P                  ,          RS[S[,          RS[S[ P                  ,          S[,          /# )rT   rU   Nr€   rK   rG   rV   rÀ   ré   rª  r¥   r$  Úlogits_to_keepr‚   rX   )	rD   r\   r   r%  r[   r   r   r§   r   )r]   r^   s   "€rP   r_   ÚElectraForCausalLM.__annotate__
  s
  ø€ ÷ F
ñ F
á—<‘< $Õ&ðF
ñ Ÿ™ tÕ+ðF
ñ Ÿ™ tÕ+ð	F
ñ
 —l‘l TÕ)ðF
ñ —|‘| dÕ*ðF
ñ  %Ÿ|™|¨dÕ2ðF
ñ !&§¡¨tÕ 3ðF
ñ —‘˜tÕ#ðF
ñ  �ðF
ñ ˜$•;ðF
ñ ™eŸl™lÕ*ðF
ñ Ñ+Õ,ðF
ñ 
‰u�|‰|Õ	Ñ@Õ	@ñF
rR   c                óä  € Ve   Rp
V P                   ! V3RVRVRVRVRVRVR	V	R
V
RR/	VB pVP                  p\        V\        4      '       d   \	        V) R4      MTpV P                  V P                  VRVR3,          4      4      pRpVe-   V P                  ! RRVRVRV P                  P                  /VB p\        VVVP                  VP                  VP                  VP                  R7      # )a³  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in
    `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are
    ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`

Example:

```python
>>> from transformers import AutoTokenizer, ElectraForCausalLM, ElectraConfig
>>> import torch

>>> tokenizer = AutoTokenizer.from_pretrained("google/electra-base-generator")
>>> config = ElectraConfig.from_pretrained("google/electra-base-generator")
>>> config.is_decoder = True
>>> model = ElectraForCausalLM.from_pretrained("google/electra-base-generator", config=config)

>>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
>>> outputs = model(**inputs)

>>> prediction_logits = outputs.logits
```NFr€   rK   rG   rV   rÀ   ré   r¥   r$  r­  Trb   r7  rª  r6   )rY  r7  r¥   r¤   rD  rE  rL  )rC  r)  r¬   r[   ÚslicerË  rÊ  Úloss_functionrN   r6   r   r¥   r¤   rD  rE  )rM   rU   r€   rK   rG   rV   rÀ   ré   rª  r¥   r$  r  r‚   Úoutputsr¤   Úslice_indicesr7  rY  s   &&&&&&&&&&&&,     rP   rn   ÚElectraForCausalLM.forward  sG  € ðN ÒØˆIà=A¿\º\Øñ>
à)ð>
ð *ð>
ð &ð	>
ð
 (ð>
ð #8ð>
ð $:ð>
ð ,ð>
ð  ð>
ð ð>
ð ñ>
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ×'Ñ'¨×(BÑ(BÀ=ÐQRÐTaÐcdÐQdÕCeÓ(fÓgˆàˆØÒØ×%Ò%Ñp¨VÐp¸FÐpÈtÏ{É{×OeÑOeÐpÐioÑpˆDä0ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø$×5Ñ5ô
ð 	
rR   rÛ  )NNNNNNNNNNrp   rÜ  rz   s   @@rP   r  r  î  sL   ù‡ € ð 5Ð6aÐbÐõ
ò&ò0ð Ø÷F
ñ F
ó ó ÷F
ð F
rR   r  )Nrª   )	r  rÆ  rü  r¹  rç  r¤  rß  r]  rB  )Urv   Úcollections.abcr   Údataclassesr   rD   r   Útorch.nnr   r   r   Ú r   rH  Úactivationsr	   r
   Úcache_utilsr   r   r   Ú
generationr   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úpytorch_utilsr   Úutilsr   r   r    r!   Úutils.genericr"   r#   Úutils.output_capturingr$   Úconfiguration_electrar&   Ú
get_loggerrr   r
  rƒ   r(   r‘   r“   r¼   rÌ   rà   rð   rý   r  r  r.  r:  rB  rW  r]  ry  r…  r¤  r¹  rÆ  rß  rç  rü  r  Ú__all__rL  rR   rP   Ú<module>r.     sÑ  ðñ å $Ý !ã Ý ß AÑ Aå &ß 1ß CÑ CÝ )ß JÝ 9÷	÷ 	ó 	÷ GÝ &Ý 6÷ó ÷ JÝ 5Ý 0ð 
×	Ò	˜HÓ	%€ô9˜Ÿ	™	ô 9÷z%ô:@)˜2Ÿ9™9ô @)ôHI)˜BŸI™Iô I)ôZ˜Ÿ	™	ô ô.�r—y‘yô .ô<˜"Ÿ)™)ô ô �B—I‘Iô ô>Ð-ô >ôD
�R—Y‘Yô 
ô@ b§i¡iô ô& "§)¡)ô ð$ ô/˜_ó /ó ð/ñ* ðôð
 ô7 +ó 7ó óð7ð ôv6Ð)ó v6ó ðv6ôr §	¡	ô ô0`˜RŸY™Yô `ñF ðôôF
Ð'=ó F
óðF
ñR ðôôQ
Ð2ó Q
óðQ
ñh ðôô>
Ð/ó >
óð>
ñB ðôô6
Ð$:ó 6
óð6
ðr ôC
Ð"8ó C
ó ðC
ðL ô[
Ð5ó [
ó ð[
ñ| ðôô
]
Ð/°ó ]
óð
]
ò@
‚rR   