+
    œ°³jiÚ  ã                   ó�  € R t ^ RIHt ^ RIHt ^ RIt^ RIHt ^ RIHtH	t	H
t
 ^RIHt ^RIHt ^RIHtHtHt ^R	IHt ^R
IHtHt ^RIHt ^RIHtHtHtHtH t H!t!H"t"H#t#H$t$ ^RI%H&t&H't' ^RI(H)t) ^RI*H+t+ ^RI,H-t-H.t.H/t/H0t0 ^RI1H2t2H3t3 ^RI4H5t5 ^RI6H7t7 ]0Pp                  ! ]94      t: ! R R]Pv                  4      t<RQR R llt= ! R R]Pv                  4      t> ! R R]Pv                  4      t? ! R R]Pv                  4      t@ ! R R]Pv                  4      tA ! R  R!]Pv                  4      tB ! R" R#]Pv                  4      tC ! R$ R%]4      tD ! R& R']Pv                  4      tE ! R( R)]Pv                  4      tF ! R* R+]Pv                  4      tG ! R, R-]Pv                  4      tH ! R. R/]Pv                  4      tI ! R0 R1]Pv                  4      tJ ! R2 R3]Pv                  4      tK]/ ! R4 R5]'4      4       tL]/! R6R77      ] ! R8 R9]-4      4       4       tM]/! R:R77       ! R; R<]L4      4       tN]/! R=R77       ! R> R?]L4      4       tO]/! R@R77       ! RA RB]L]4      4       tP]/ ! RC RD]L4      4       tQ]/! RER77       ! RF RG]L4      4       tR]/! RHR77       ! RI RJ]L4      4       tS]/ ! RK RL]L4      4       tT]/ ! RM RN]L4      4       tU]/ ! RO RP]L4      4       tV. RROtWR# )SzPyTorch BERT model.)ÚCallable)Ú	dataclassN)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELoss)Úinitialization)ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚGradientCheckpointingLayer)	Ú)BaseModelOutputWithPastAndCrossAttentionsÚ,BaseModelOutputWithPoolingAndCrossAttentionsÚ!CausalLMOutputWithCrossAttentionsÚMaskedLMOutputÚMultipleChoiceModelOutputÚNextSentencePredictorOutputÚQuestionAnsweringModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)Úapply_chunking_to_forward)ÚModelOutputÚTransformersKwargsÚauto_docstringÚlogging)Úcan_return_tupleÚmerge_with_config_defaults)Úcapture_outputs)Ú
BertConfigc                   óL   a a€ ] tR t^5t oRtV 3R ltRV3R lR lltRtVtV ;t	# )ÚBertEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c                óN  <€ \         SV `  4        \        P                  ! VP                  VP
                  VP                  R 7      V n        \        P                  ! VP                  VP
                  4      V n	        \        P                  ! VP                  VP
                  4      V n        \        P                  ! VP
                  VP                  R7      V n        \        P                  ! VP                  4      V n        \        P"                  ! \$        P&                  ! VP                  4      P)                  R4      RR7      V n        \        P"                  ! \$        P,                  ! V P*                  P/                  4       \$        P0                  R7      RR7      V n        R# ))Úpadding_idx©ÚepsF)Ú
persistent)ÚdtypeN©é   éÿÿÿÿ)ÚsuperÚ__init__r   Ú	EmbeddingÚ
vocab_sizeÚhidden_sizeÚpad_token_idÚword_embeddingsÚmax_position_embeddingsÚposition_embeddingsÚtype_vocab_sizeÚtoken_type_embeddingsÚ	LayerNormÚlayer_norm_epsÚDropoutÚhidden_dropout_probÚdropoutÚBufferÚtorchÚarangeÚexpandÚposition_idsÚzerosÚsizeÚlongÚtoken_type_ids©ÚselfÚconfigÚ	__class__s   &&€Úg/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/bert/modeling_bert.pyr2   ÚBertEmbeddings.__init__8   s  ø€ Ü‰ÑÔÜ!Ÿ|š|¨F×,=Ñ,=¸v×?QÑ?QÐ_e×_rÑ_rÔsˆÔÜ#%§<¢<°×0NÑ0NÐPV×PbÑPbÓ#cˆÔ Ü%'§\¢\°&×2HÑ2HÈ&×J\ÑJ\Ó%]ˆÔ"äŸš f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z’z &×"<Ñ"<Ó=ˆŒäŸIšI¤e§l¢l°6×3QÑ3QÓ&R×&YÑ&YÐZaÓ&bÐotÔuˆÔÜ Ÿiši¬¯ª°D×4EÑ4E×4JÑ4JÓ4LÔTY×T^ÑT^Ô(_ÐlqÔrˆÖó    c                óÚ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[RS[ P                  /# )é   Ú	input_idsNrI   rE   Úinputs_embedsÚpast_key_values_lengthÚreturn)rB   Ú
LongTensorÚFloatTensorÚintÚTensor)ÚformatÚ__classdict__s   "€rN   Ú__annotate__ÚBertEmbeddings.__annotate__D   sw   ø€ ÷ (ñ (á×#Ñ# dÕ*ð(ñ ×(Ñ(¨4Õ/ð(ñ ×&Ñ&¨Õ-ð	(ñ
 ×(Ñ(¨4Õ/ð(ñ !$ð(ñ 
�‰ñ(rP   c                óÄ  € Ve   VP                  4       pMVP                  4       R R pVw  rxVf   V P                  RWXV,           13,          pVf¦   \        V R4      '       dY   V P                  P	                  VP
                  ^ ,          R4      p	\        P                  ! V	^VR7      p	V	P	                  Wx4      pM;\        P                  ! V\        P                  V P                  P                  R7      pVf   V P                  V4      pV P                  V4      p
WJ,           pV P                  V4      pW¼,           pV P                  V4      pV P                  V4      pV# )NºNNNrI   )ÚdimÚindex)r-   Údevicer0   )rG   rE   ÚhasattrrI   rD   ÚshaperB   ÚgatherrF   rH   rc   r7   r;   r9   r<   r@   )rK   rS   rI   rE   rT   rU   Úinput_shapeÚ
batch_sizeÚ
seq_lengthÚbuffered_token_type_idsr;   Ú
embeddingsr9   s   &&&&&&       rN   ÚforwardÚBertEmbeddings.forwardD   sG  € ð Ò Ø#Ÿ.™.Ó*‰Kà'×,Ñ,Ó.¨s°Ð3ˆKà!,Ñˆ
àÒØ×,Ñ,¨QÐ0FÐVlÕIlÐ0lÐ-lÕmˆLð
 Ò!Ü�tÐ-×.Ò.à*.×*=Ñ*=×*DÑ*DÀ\×EWÑEWÐXYÕEZÐ\^Ó*_Ð'Ü*/¯,ª,Ð7NÐTUÐ]iÔ*jÐ'Ø!8×!?Ñ!?À
Ó!W‘ä!&§¢¨[ÄÇ
Á
ÐSW×SdÑSd×SkÑSkÔ!l�àÒ Ø ×0Ñ0°Ó;ˆMØ $× :Ñ :¸>Ó JÐØ"Õ:ˆ
à"×6Ñ6°|ÓDÐØÕ5ˆ
à—^‘^ JÓ/ˆ
Ø—\‘\ *Ó-ˆ
ØÐrP   )r<   r@   r9   rE   r;   rI   r7   )NNNNé    )
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r2   rl   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©rM   r\   s   @@rN   r'   r'   5   s   ù‡ € ÙQõ
s÷(÷ (ò (rP   r'   c                ó  € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R,          R\        R	\
        \        ,          /# )
rR   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNÚscalingr@   Úkwargs)r   ÚModulerB   rZ   Úfloatr   r   )r[   s   "rN   r]   r]   o   s{   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô �T�\ð%ô ð%ô Ô'Õ(ñ%rP   c                 ó¶  € Vf   VP                  R4      R,          p\        P                  ! WP                  ^^4      4      V,          pVe	   W„,           p\        P
                  P                  VRR7      p\        P
                  P                  W†V P                  R7      p\        P                  ! Wƒ4      p	V	P                  ^^4      P                  4       p	W˜3# )N©ra   )ÚpÚtrainingr0   ç      à¿)
rG   rB   ÚmatmulÚ	transposer   Ú
functionalÚsoftmaxr@   r…   Ú
contiguous)
ry   rz   r{   r|   r}   r~   r@   r   Úattn_weightsÚattn_outputs
   &&&&&&&,  rN   Úeager_attention_forwardrŽ   o   s±   € ð ‚Ø—*‘*˜R“. DÕ(ˆô —<’< §}¡}°Q¸Ó':Ó;¸gÕE€LàÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2Ð(Ó>€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€Lä—,’,˜|Ó3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$rP   c                   óL   a a€ ] tR t^‹t oRV 3R lltRV3R lR lltRtVtV ;t# )ÚBertSelfAttentionc                ót  <€ \         SV `  4        VP                  VP                  ,          ^ 8w  d9   \	        VR4      '       g'   \        RVP                   RVP                   R24      hWn        VP                  V n        \        VP                  VP                  ,          4      V n        V P                  V P                  ,          V n	        V P                  R,          V n
        \        P                  ! VP                  V P                  4      V n        \        P                  ! VP                  V P                  4      V n        \        P                  ! VP                  V P                  4      V n        \        P                   ! VP"                  4      V n        VP&                  V n        W n        W0n        R# ©rn   Úembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads (Ú)Nr†   )r1   r2   r5   Únum_attention_headsrd   Ú
ValueErrorrL   rY   Úattention_head_sizeÚall_head_sizer~   r   ÚLinearrz   r{   r|   r>   Úattention_probs_dropout_probr@   Ú
is_decoderÚ	is_causalÚ	layer_idx©rK   rL   rœ   r�   rM   s   &&&&€rN   r2   ÚBertSelfAttention.__init__Œ   sM  ø€ Ü‰ÑÔØ×Ñ × :Ñ :Õ:¸aÔ?ÌÐPVÐXh×HiÒHiÜØ# F×$6Ñ$6Ð#7ð 8Ø ×4Ñ4Ð5°Qð8óð ð Œà#)×#=Ñ#=ˆÔ Ü#& v×'9Ñ'9¸F×<VÑ<VÕ'VÓ#WˆÔ Ø!×5Ñ5¸×8PÑ8PÕPˆÔØ×/Ñ/°Õ5ˆŒä—Y’Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
Ü—9’9˜V×/Ñ/°×1CÑ1CÓDˆŒÜ—Y’Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
ä—z’z &×"EÑ"EÓFˆŒà ×+Ñ+ˆŒØ"ŒØ"ŽrP   c                ó°   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  ,          /# )rR   Úhidden_statesr}   NÚpast_key_valuesr   rV   ©rB   rZ   rX   r
   r   r   Útuple)r[   r\   s   "€rN   r]   ÚBertSelfAttention.__annotate__¤   s_   ø€ ÷ ')ñ ')á—|‘|ð')ñ ×)Ñ)¨DÕ0ð')ñ  �ð	')ñ
 Ñ+Õ,ð')ñ 
‰u�|‰|Õ	ñ')rP   c                ó   € VP                   R R p. VORNV P                  N5pV P                  V4      P                  ! V!  P	                  ^^4      pV P                  V4      P                  ! V!  P	                  ^^4      pV P                  V4      P                  ! V!  P	                  ^^4      p	VeC   Tp
\        V\        4      '       d   VP                  p
V
P                  W‰V P                  4      w  r‰\        P                  ! V P                  P                  \         4      pT! V VVV	V3RV P"                  '       g   RMV P$                  P&                  RV P(                  /VB w  rÍVP*                  ! . VORN5!  P-                  4       pWÍ3# )Nr@   ç        r~   r0   )re   r—   rz   Úviewrˆ   r{   r|   Ú
isinstancer   Úself_attention_cacheÚupdater�   r   Úget_interfacerL   Ú_attn_implementationrŽ   r…   r@   r„   r~   Úreshaper‹   )rK   r¡   r}   r¢   r   rg   Úhidden_shapeÚquery_layerÚ	key_layerÚvalue_layerÚcurrent_past_key_valuesÚattention_interfacer�   rŒ   s   &&&&,         rN   rl   ÚBertSelfAttention.forward¤   s‰  € ð $×)Ñ)¨#¨2Ð.ˆØC˜ÐC bÐC¨$×*BÑ*BÑCˆð —j‘j Ó/×4Ò4°lÑC×MÑMÈaÐQRÓSˆØ—H‘H˜]Ó+×0Ò0°,Ñ?×IÑIÈ!ÈQÓOˆ	Ø—j‘j Ó/×4Ò4°lÑC×MÑMÈaÐQRÓSˆàÒ&à&5Ð#Ü˜/Ô+>×?Ò?Ø*9×*NÑ*NÐ'ð &=×%CÑ%CÀIÐ\`×\jÑ\jÓ%kÑ"ˆIä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð  $Ÿ}Ÿ}˜}‘C°$·,±,·.±.ð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØÐ(Ð(rP   )r˜   r—   rL   r@   rœ   r›   r{   r�   r•   rz   r~   r|   ©FN)NN©	ro   rp   rq   rr   r2   rl   rt   ru   rv   rw   s   @@rN   r�   r�   ‹   s   ù‡ € ÷#÷0')÷ ')ò ')rP   r�   c                   óL   a a€ ] tR t^Ît oRV 3R lltRV3R lR lltRtVtV ;t# )ÚBertCrossAttentionc                óR  <€ \         SV `  4        VP                  VP                  ,          ^ 8w  d9   \	        VR4      '       g'   \        RVP                   RVP                   R24      hWn        VP                  V n        \        VP                  VP                  ,          4      V n        V P                  V P                  ,          V n	        V P                  R,          V n
        \        P                  ! VP                  V P                  4      V n        \        P                  ! VP                  V P                  4      V n        \        P                  ! VP                  V P                  4      V n        \        P                   ! VP"                  4      V n        W n        W0n        R# r’   )r1   r2   r5   r•   rd   r–   rL   rY   r—   r˜   r~   r   r™   rz   r{   r|   r>   rš   r@   rœ   r�   rž   s   &&&&€rN   r2   ÚBertCrossAttention.__init__Ï   s@  ø€ Ü‰ÑÔØ×Ñ × :Ñ :Õ:¸aÔ?ÌÐPVÐXh×HiÒHiÜØ# F×$6Ñ$6Ð#7ð 8Ø ×4Ñ4Ð5°Qð8óð ð Œà#)×#=Ñ#=ˆÔ Ü#& v×'9Ñ'9¸F×<VÑ<VÕ'VÓ#WˆÔ Ø!×5Ñ5¸×8PÑ8PÕPˆÔØ×/Ñ/°Õ5ˆŒä—Y’Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
Ü—9’9˜V×/Ñ/°×1CÑ1CÓDˆŒÜ—Y’Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
ä—z’z &×"EÑ"EÓFˆŒà"ŒØ"ŽrP   c                óØ   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  ,          /# )rR   r¡   Úencoder_hidden_statesNr}   r¢   r   rV   )rB   rZ   rX   r   r   r   r¤   )r[   r\   s   "€rN   r]   ÚBertCrossAttention.__annotate__æ   su   ø€ ÷ 1)ñ 1)á—|‘|ð1)ñ  %×0Ñ0°4Õ7ð1)ñ ×)Ñ)¨DÕ0ð	1)ñ
 -¨tÕ3ð1)ñ Ñ+Õ,ð1)ñ 
‰u�|‰|Õ	ñ1)rP   c                ó˜  € VP                   R R p. VORNV P                  N5pV P                  V4      P                  V4      P	                  ^^4      pVe&   VP
                  P                  V P                  4      MRp	Vel   V	'       dd   VP                  P                  V P                  ,          P                  p
VP                  P                  V P                  ,          P                  pMÅ. VP                   R R ORNV P                  N5pV P                  V4      P                  V4      P	                  ^^4      p
V P                  V4      P                  V4      P	                  ^^4      pVeB   VP                  P                  W«V P                  4      w  r«RVP
                  V P                  &   \        P                   ! V P"                  P$                  \&        4      pT! V VV
VV3RV P(                  '       g   RMV P*                  P,                  RV P.                  /VB w  rïVP0                  ! . VORN5!  P3                  4       pWï3# )NFTr@   r§   r~   r0   )re   r—   rz   r¨   rˆ   Ú
is_updatedÚgetr�   Úcross_attention_cacheÚlayersÚkeysÚvaluesr{   r|   r«   r   r¬   rL   r­   rŽ   r…   r@   r„   r~   r®   r‹   )rK   r¡   r½   r}   r¢   r   rg   r¯   r°   rÀ   r±   r²   Úkv_shaper´   r�   rŒ   s   &&&&&,          rN   rl   ÚBertCrossAttention.forwardæ   s  € ð $×)Ñ)¨#¨2Ð.ˆàC˜ÐC bÐC¨$×*BÑ*BÑCˆð —j‘j Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆàGVÒGb�_×/Ñ/×3Ñ3°D·N±NÔCÐhmˆ
ØÒ&¯:à'×=Ñ=×DÑDÀTÇ^Á^ÕT×YÑYˆIØ)×?Ñ?×FÑFÀtÇ~Á~ÕV×]Ñ]‰KàXÐ.×4Ñ4°S°bÐ9ÐX¸2ÐX¸t×?WÑ?WÑXˆHØŸ™Ð!6Ó7×<Ñ<¸XÓF×PÑPÐQRÐTUÓVˆIØŸ*™*Ð%:Ó;×@Ñ@ÀÓJ×TÑTÐUVÐXYÓZˆKàÒ*à)8×)NÑ)N×)UÑ)UØ¨D¯N©Nó*Ñ&�	ð >B�×*Ñ*¨4¯>©>Ñ:ä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð  $Ÿ}Ÿ}˜}‘C°$·,±,·.±.ð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØÐ(Ð(rP   )r˜   r—   rL   r@   rœ   r{   r�   r•   rz   r~   r|   r¶   )NNNr·   rw   s   @@rN   r¹   r¹   Î   s   ù‡ € ÷#÷.1)÷ 1)ò 1)rP   r¹   c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚBertSelfOutputi  c                ó0  <€ \         SV `  4        \        P                  ! VP                  VP                  4      V n        \        P                  ! VP                  VP                  R 7      V n        \        P                  ! VP                  4      V n
        R# ©r*   N)r1   r2   r   r™   r5   Údenser<   r=   r>   r?   r@   rJ   s   &&€rN   r2   ÚBertSelfOutput.__init__  s`   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸš f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z’z &×"<Ñ"<Ó=ˆŽrP   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# ©rR   r¡   Úinput_tensorrV   ©rB   rZ   )r[   r\   s   "€rN   r]   ÚBertSelfOutput.__annotate__!  ó1   ø€ ÷ ñ ¡U§\¡\ð ÁÇÁð ÑRW×R^ÑR^ñ rP   c                óx   € V P                  V4      pV P                  V4      pV P                  W,           4      pV# ©N©rÌ   r@   r<   ©rK   r¡   rÐ   s   &&&rN   rl   ÚBertSelfOutput.forward!  ó5   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØŸ™ }Õ'CÓDˆØÐrP   ©r<   rÌ   r@   r·   rw   s   @@rN   rÉ   rÉ     ó   ù‡ € õ>÷÷ ð rP   rÉ   c                   óL   a a€ ] tR tRt oRV 3R lltRV3R lR lltRtVtV ;t# )ÚBertAttentioni(  c                ó–   <€ \         SV `  4        W@n        V'       d   \        M\        pV! WVR 7      V n        \        V4      V n        R# )©rœ   r�   N)r1   r2   Úis_cross_attentionr¹   r�   rK   rÉ   Úoutput)rK   rL   rœ   r�   rà   Úattention_classrM   s   &&&&& €rN   r2   ÚBertAttention.__init__)  s9   ø€ Ü‰ÑÔØ"4Ôß0BÕ,ÔHYˆÙ# FÈ9ÔUˆŒ	Ü$ VÓ,ˆŽrP   c                ó   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  ,          /# ©	rR   r¡   r}   Nr½   Úencoder_attention_maskr¢   r   rV   r£   )r[   r\   s   "€rN   r]   ÚBertAttention.__annotate__0  s‰   ø€ ÷ .ñ .á—|‘|ð.ñ ×)Ñ)¨DÕ0ð.ñ  %×0Ñ0°4Õ7ð	.ñ
 !&× 1Ñ 1°DÕ 8ð.ñ  �ð.ñ Ñ+Õ,ð.ñ 
‰u�|‰|Õ	ñ.rP   c           	     óŒ   € V P                   '       g   TMTpV P                  ! V3R VRVRV/VB w  rxV P                  Wq4      pWx3# )r½   r}   r¢   )rà   rK   rá   )	rK   r¡   r}   r½   ræ   r¢   r   Úattention_outputrŒ   s	   &&&&&&,  rN   rl   ÚBertAttention.forward0  sm   € ð 04×/F×/FÐ/F™ÐLbˆØ)-¯ªØñ*
à"7ð*
ð *ð*
ð ,ð	*
ð
 ñ*
Ñ&Ðð  Ÿ;™;Ð'7ÓGÐØÐ-Ð-rP   )rà   rá   rK   )FNF©NNNNr·   rw   s   @@rN   rÝ   rÝ   (  s   ù‡ € ÷-÷.÷ .ò .rP   rÝ   c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚBertIntermediateiE  c                ó"  <€ \         SV `  4        \        P                  ! VP                  VP
                  4      V n        \        VP                  \        4      '       d   \        VP                  ,          V n        R # VP                  V n        R # rÕ   )r1   r2   r   r™   r5   Úintermediate_sizerÌ   r©   Ú
hidden_actÚstrr	   Úintermediate_act_fnrJ   s   &&€rN   r2   ÚBertIntermediate.__init__F  s`   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3KÑ3KÓLˆŒ
Ü�f×'Ñ'¬×-Ò-Ü'-¨f×.?Ñ.?Õ'@ˆDÖ$à'-×'8Ñ'8ˆDÖ$rP   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©rR   r¡   rV   rÑ   )r[   r\   s   "€rN   r]   ÚBertIntermediate.__annotate__N  s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ rP   c                óJ   € V P                  V4      pV P                  V4      pV# rÕ   ©rÌ   rò   ©rK   r¡   s   &&rN   rl   ÚBertIntermediate.forwardN  s&   € ØŸ
™
 =Ó1ˆØ×0Ñ0°Ó?ˆØÐrP   rø   r·   rw   s   @@rN   rí   rí   E  s   ù‡ € õ9÷÷ ð rP   rí   c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )Ú
BertOutputiT  c                ó0  <€ \         SV `  4        \        P                  ! VP                  VP
                  4      V n        \        P                  ! VP
                  VP                  R 7      V n        \        P                  ! VP                  4      V n        R# rË   )r1   r2   r   r™   rï   r5   rÌ   r<   r=   r>   r?   r@   rJ   s   &&€rN   r2   ÚBertOutput.__init__U  s`   ø€ Ü‰ÑÔÜ—Y’Y˜v×7Ñ7¸×9KÑ9KÓLˆŒ
ÜŸš f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z’z &×"<Ñ"<Ó=ˆŽrP   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# rÏ   rÑ   )r[   r\   s   "€rN   r]   ÚBertOutput.__annotate__[  rÓ   rP   c                óx   € V P                  V4      pV P                  V4      pV P                  W,           4      pV# rÕ   rÖ   r×   s   &&&rN   rl   ÚBertOutput.forward[  rÙ   rP   rÚ   r·   rw   s   @@rN   rü   rü   T  rÛ   rP   rü   c                   óR   a a€ ] tR tRt oRV 3R lltRV3R lR lltR tRtVtV ;t	# )	Ú	BertLayerib  c                óœ  <€ \         SV `  4        VP                  V n        ^V n        \	        WP
                  VR7      V n        VP
                  V n        VP                  V n        V P                  '       d5   V P
                  '       g   \        V  R24      h\	        VRVRR7      V n	        \        V4      V n        \        V4      V n        R# )r/   rß   z> should be used as a decoder model if cross attention is addedFT)rœ   r�   rà   N)r1   r2   Úchunk_size_feed_forwardÚseq_len_dimrÝ   r›   Ú	attentionÚadd_cross_attentionr–   Úcrossattentionrí   Úintermediaterü   rá   )rK   rL   r�   rM   s   &&&€rN   r2   ÚBertLayer.__init__c  s°   ø€ Ü‰ÑÔØ'-×'EÑ'EˆÔ$ØˆÔÜ& v×9JÑ9JÐV_Ô`ˆŒØ ×+Ñ+ˆŒØ#)×#=Ñ#=ˆÔ Ø×#×#Ð#Ø—?—?�?Ü  D 6Ð)gÐ!hÓiÐiÜ"/ØØØ#Ø#'ô	#ˆDÔô -¨VÓ4ˆÔÜ  Ó(ˆŽrP   c                óð   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[S[,          RS[ P                  /# rå   )rB   rZ   rX   r
   r   r   )r[   r\   s   "€rN   r]   ÚBertLayer.__annotate__v  s„   ø€ ÷ %ñ %á—|‘|ð%ñ ×)Ñ)¨DÕ0ð%ñ  %×0Ñ0°4Õ7ð	%ñ
 !&× 1Ñ 1°DÕ 8ð%ñ  �ð%ñ Ñ+Õ,ð%ñ 
�‰ñ%rP   c                ó8  € V P                   ! VV3R V/VB w  rxTp	V P                  '       dB   Ve>   \        V R4      '       g   \        RV  R24      hV P                  ! VRVV3R V/VB w  r¨T
p	\        V P                  V P                  V P                  V	4      pV# )r¢   Nr
  z'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`)	r  r›   rd   r–   r
  r   Úfeed_forward_chunkr  r  )rK   r¡   r}   r½   ræ   r¢   r   Úself_attention_outputÚ_ré   Úcross_attention_outputÚlayer_outputs   &&&&&&,     rN   rl   ÚBertLayer.forwardv  sâ   € ð $(§>¢>ØØñ$
ð ,ð$
ð ñ	$
Ñ Ðð 1Ðà�?�?ˆ?Ð4Ò@Ü˜4Ð!1×2Ò2Ü Ø=¸d¸Vð DDð Dóð ð
 )-×(;Ò(;Ø%ØØ%Ø&ñ	)ð
 !0ð)ð ñ)Ñ%Ð"ð  6Ðä0Ø×#Ñ# T×%AÑ%AÀ4×CSÑCSÐUeó
ˆð ÐrP   c                óJ   € V P                  V4      pV P                  W!4      pV# rÕ   )r  rá   )rK   ré   Úintermediate_outputr  s   &&  rN   r  ÚBertLayer.feed_forward_chunk�  s)   € Ø"×/Ñ/Ð0@ÓAÐØ—{‘{Ð#6ÓIˆØÐrP   )r	  r  r  r
  r  r›   rá   r  rÕ   rë   )
ro   rp   rq   rr   r2   rl   r  rt   ru   rv   rw   s   @@rN   r  r  b  s   ù‡ € ÷)÷&%ò %÷Nò rP   r  c                   óH   a a€ ] tR tRt oV 3R ltRV3R lR lltRtVtV ;t# )ÚBertEncoderi£  c           
     óÊ   <€ \         SV `  4        Wn        \        P                  ! \        VP                  4       Uu. uF  p\        WR 7      NK  	  up4      V n        R# u upi ))r�   N)	r1   r2   rL   r   Ú
ModuleListÚrangeÚnum_hidden_layersr  Úlayer)rK   rL   ÚirM   s   && €rN   r2   ÚBertEncoder.__init__¤  sH   ø€ Ü‰ÑÔØŒÜ—]’]ÌEÐRX×RjÑRjÔLkÓ#lÑLkÀq¤I¨f×$BÑLkÑ#lÓmˆŽ
ùÒ#ls   ½A c                ó$  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[,          R	S[S[ P                  ,          S[,          /# )
rR   r¡   r}   Nr½   ræ   r¢   Ú	use_cacher   rV   )	rB   rZ   rX   r
   Úboolr   r   r¤   r   )r[   r\   s   "€rN   r]   ÚBertEncoder.__annotate__©  s�   ø€ ÷ 
ñ 
á—|‘|ð
ñ ×)Ñ)¨DÕ0ð
ñ  %×0Ñ0°4Õ7ð	
ñ
 !&× 1Ñ 1°DÕ 8ð
ñ  �ð
ñ ˜$•;ð
ñ Ñ+Õ,ð
ñ 
‰u�|‰|Õ	ÑHÕ	Hñ
rP   c                ó”   € \        V P                  4       F  w  r‰V	! VVV3R VRV/VB pK  	  \        TV'       d   VR7      # RR7      # )ræ   r¢   N)Úlast_hidden_stater¢   )Ú	enumerater  r   )
rK   r¡   r}   r½   ræ   r¢   r#  r   r   Úlayer_modules
   &&&&&&&,  rN   rl   ÚBertEncoder.forward©  sn   € ô  )¨¯©Ö4‰OˆAÙ(ØØØ%ñð (>ð	ð
 !0ðð ñŠMñ  5ô 9Ø+ß/8˜Oô
ð 	
à>Bô
ð 	
rP   )rL   r  )NNNNNr·   rw   s   @@rN   r  r  £  s   ù‡ € õn÷

÷ 
ò 
rP   r  c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )Ú
BertPooleriÃ  c                ó¸   <€ \         SV `  4        \        P                  ! VP                  VP                  4      V n        \        P                  ! 4       V n        R # rÕ   )r1   r2   r   r™   r5   rÌ   ÚTanhÚ
activationrJ   s   &&€rN   r2   ÚBertPooler.__init__Ä  s9   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ'š'›)ˆŽrP   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rõ   rÑ   )r[   r\   s   "€rN   r]   ÚBertPooler.__annotate__É  s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ rP   c                ó\   € VR,          pV P                  V4      pV P                  V4      pV# )r`   )r`   rn   )rÌ   r/  )rK   r¡   Úfirst_token_tensorÚpooled_outputs   &&  rN   rl   ÚBertPooler.forwardÉ  s2   € ð +¨4Õ0ÐØŸ
™
Ð#5Ó6ˆØŸ™¨Ó6ˆØÐrP   )r/  rÌ   r·   rw   s   @@rN   r,  r,  Ã  s   ù‡ € õ$÷
÷ ð rP   r,  c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚBertPredictionHeadTransformiÒ  c                ó‚  <€ \         SV `  4        \        P                  ! VP                  VP                  4      V n        \        VP                  \        4      '       d   \        VP                  ,          V n
        MVP                  V n
        \        P                  ! VP                  VP                  R 7      V n        R# rË   )r1   r2   r   r™   r5   rÌ   r©   rð   rñ   r	   Útransform_act_fnr<   r=   rJ   s   &&€rN   r2   Ú$BertPredictionHeadTransform.__init__Ó  s~   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
Ü�f×'Ñ'¬×-Ò-Ü$*¨6×+<Ñ+<Õ$=ˆDÕ!à$*×$5Ñ$5ˆDÔ!ÜŸš f×&8Ñ&8¸f×>SÑ>SÔTˆŽrP   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rõ   rÑ   )r[   r\   s   "€rN   r]   Ú(BertPredictionHeadTransform.__annotate__Ü  s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ rP   c                ól   € V P                  V4      pV P                  V4      pV P                  V4      pV# rÕ   )rÌ   r:  r<   rù   s   &&rN   rl   Ú#BertPredictionHeadTransform.forwardÜ  s4   € ØŸ
™
 =Ó1ˆØ×-Ñ-¨mÓ<ˆØŸ™ }Ó5ˆØÐrP   )r<   rÌ   r:  r·   rw   s   @@rN   r8  r8  Ò  s   ù‡ € õU÷÷ ð rP   r8  c                   ó8   a a€ ] tR tRt oV 3R ltR tRtVtV ;t# )ÚBertLMPredictionHeadiã  c                ó  <€ \         SV `  4        \        V4      V n        \        P
                  ! VP                  VP                  R R7      V n        \        P                  ! \        P                  ! VP                  4      4      V n        R# )T)ÚbiasN)r1   r2   r8  Ú	transformr   r™   r5   r4   ÚdecoderÚ	ParameterrB   rF   rC  rJ   s   &&€rN   r2   ÚBertLMPredictionHead.__init__ä  s[   ø€ Ü‰ÑÔÜ4°VÓ<ˆŒô —y’y ×!3Ñ!3°V×5FÑ5FÈTÔRˆŒÜ—L’L¤§¢¨V×->Ñ->Ó!?Ó@ˆŽ	rP   c                óJ   € V P                  V4      pV P                  V4      pV# rÕ   )rD  rE  rù   s   &&rN   rl   ÚBertLMPredictionHead.forwardí  s$   € ØŸ™ }Ó5ˆØŸ™ ]Ó3ˆØÐrP   )rC  rE  rD  r·   rw   s   @@rN   rA  rA  ã  s   ù‡ € õA÷ò rP   rA  c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚBertOnlyMLMHeadió  c                óD   <€ \         SV `  4        \        V4      V n        R # rÕ   )r1   r2   rA  ÚpredictionsrJ   s   &&€rN   r2   ÚBertOnlyMLMHead.__init__ô  s   ø€ Ü‰ÑÔÜ/°Ó7ˆÖrP   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )rR   Úsequence_outputrV   rÑ   )r[   r\   s   "€rN   r]   ÚBertOnlyMLMHead.__annotate__ø  s#   ø€ ÷ !ñ !¡u§|¡|ð !¹¿¹ñ !rP   c                ó(   € V P                  V4      pV# rÕ   ©rM  )rK   rP  Úprediction_scoress   && rN   rl   ÚBertOnlyMLMHead.forwardø  s   € Ø ×,Ñ,¨_Ó=ÐØ Ð rP   rS  r·   rw   s   @@rN   rK  rK  ó  s   ù‡ € õ8÷!÷ !ð !rP   rK  c                   ó8   a a€ ] tR tRt oV 3R ltR tRtVtV ;t# )ÚBertOnlyNSPHeadiý  c                óp   <€ \         SV `  4        \        P                  ! VP                  ^4      V n        R# ©rR   N)r1   r2   r   r™   r5   Úseq_relationshiprJ   s   &&€rN   r2   ÚBertOnlyNSPHead.__init__þ  s'   ø€ Ü‰ÑÔÜ "§	¢	¨&×*<Ñ*<¸aÓ @ˆÖrP   c                ó(   € V P                  V4      pV# rÕ   ©rZ  )rK   r5  Úseq_relationship_scores   && rN   rl   ÚBertOnlyNSPHead.forward  s   € Ø!%×!6Ñ!6°}Ó!EÐØ%Ð%rP   r]  r·   rw   s   @@rN   rW  rW  ý  s   ù‡ € õA÷&ò &rP   rW  c                   ó8   a a€ ] tR tRt oV 3R ltR tRtVtV ;t# )ÚBertPreTrainingHeadsi  c                ó�   <€ \         SV `  4        \        V4      V n        \        P
                  ! VP                  ^4      V n        R# rY  )r1   r2   rA  rM  r   r™   r5   rZ  rJ   s   &&€rN   r2   ÚBertPreTrainingHeads.__init__  s4   ø€ Ü‰ÑÔÜ/°Ó7ˆÔÜ "§	¢	¨&×*<Ñ*<¸aÓ @ˆÖrP   c                óL   € V P                  V4      pV P                  V4      pW43# rÕ   ©rM  rZ  )rK   rP  r5  rT  r^  s   &&&  rN   rl   ÚBertPreTrainingHeads.forward  s-   € Ø ×,Ñ,¨_Ó=ÐØ!%×!6Ñ!6°}Ó!EÐØ Ð8Ð8rP   re  r·   rw   s   @@rN   ra  ra    s   ù‡ € õA÷
9ò 9rP   ra  c                   ó†   a a€ ] tR tRt o]tRtRtRtRt	Rt
RtR]R]R]/t]P"                  ! 4       V 3R l4       tRtVtV ;t# )	ÚBertPreTrainedModeli  ÚbertTr¡   Ú
attentionsÚcross_attentionsc                óÄ  <€ \         SV `  V4       \        V\        4      '       d#   \        P
                  ! VP                  4       R# \        V\        4      '       d‚   \        P                  ! VP                  \        P                  ! VP                  P                  R,          4      P                  R4      4       \        P
                  ! VP                  4       R# R# )zInitialize the weightsNr0   r.   )r1   Ú_init_weightsr©   rA  ÚinitÚzeros_rC  r'   Úcopy_rE   rB   rC   re   rD   rI   )rK   ry   rM   s   &&€rN   rm  Ú!BertPreTrainedModel._init_weights"  s�   ø€ ô 	‰Ñ˜fÔ%Ü�fÔ2×3Ò3Ü�KŠK˜Ÿ™Ö$Ü˜¤×/Ò/Ü�JŠJ�v×*Ñ*¬E¯LªL¸×9LÑ9L×9RÑ9RÐSUÕ9VÓ,W×,^Ñ,^Ð_fÓ,gÔhÜ�KŠK˜×-Ñ-Ö.ñ 0rP   © )ro   rp   rq   rr   r%   Úconfig_classÚbase_model_prefixÚsupports_gradient_checkpointingÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_supports_attention_backendr  r�   r¹   Ú_can_record_outputsrB   Úno_gradrm  rt   ru   rv   rw   s   @@rN   rh  rh    se   ù‡ € à€LØÐØ&*Ð#ØÐØ€NØÐØ"&Ðà˜ØÐ'ØÐ.ðÐð ‡]‚]ƒ_ô/ó ÷/ð /rP   rh  z0
    Output type of [`BertForPreTraining`].
    )Úcustom_introc                   óD   a € ] tR tRt o RtRtRtRtRtRt	V 3R lt
RtV tR# )ÚBertForPreTrainingOutputi-  ar  
loss (*optional*, returned when `labels` is provided, `torch.FloatTensor` of shape `(1,)`):
    Total loss as the sum of the masked language modeling loss and the next sequence prediction
    (classification) loss.
prediction_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
    Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
seq_relationship_logits (`torch.FloatTensor` of shape `(batch_size, 2)`):
    Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation
    before SoftMax).
Nc                ó   <€ V ^8„  d   Qh/ S[ P                  R,          ;R&   S[ P                  R,          ;R&   S[ P                  R,          ;R&   S[S[ P                  ,          R,          ;R&   S[S[ P                  ,          R,          ;R&   # )rR   NÚlossÚprediction_logitsÚseq_relationship_logitsr¡   rj  )rB   rX   r¤   )r[   r\   s   "€rN   r]   Ú%BertForPreTrainingOutput.__annotate__-  s‡   ø‡ ‚ ñ$ ×
Ñ
˜dÕ
"Ñ)ñ% ñ& ×(Ñ(¨4Õ/Ñ6ñ' ñ( #×.Ñ.°Õ5Ñ<ñ) ñ* ™×*Ñ*Õ+¨dÕ2Ñ9ñ+ ñ, ‘e×'Ñ'Õ(¨4Õ/Ñ6ò- rP   rr  )ro   rp   rq   rr   rs   r€  r�  r‚  r¡   rj  Ú__annotate_func__rt   ru   )r\   s   @rN   r~  r~  -  s.   ø‡ € ñ	ð &*€DØ26ÐØ8<ÐØ59€MØ26€J÷- ƒ rP   r~  a
  
    The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of
    cross-attention is added between the self-attention layers, following the architecture described in [Attention is
    all you need](https://huggingface.co/papers/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit,
    Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.

    To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set
    to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and
    `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass.
    c                   ó„   a a€ ] tR tRt oRR.tRV 3R lltR tR t]]	]
RV3R lR ll4       4       4       tR	 tR
tVtV ;t# )Ú	BertModeliF  r'   r  c                óÔ   <€ \         SV `  V4       Wn        RV n        \	        V4      V n        \        V4      V n        V'       d   \        V4      MRV n	        V P                  4        R# )z^
add_pooling_layer (bool, *optional*, defaults to `True`):
    Whether to add a pooling layer
FN)r1   r2   rL   Úgradient_checkpointingr'   rk   r  Úencoderr,  ÚpoolerÚ	post_init)rK   rL   Úadd_pooling_layerrM   s   &&&€rN   r2   ÚBertModel.__init__U  sS   ø€ ô
 	‰Ñ˜Ô ØŒØ&+ˆÔ#ä(¨Ó0ˆŒÜ" 6Ó*ˆŒç,=”j Ô(À4ˆŒð 	�‰ÖrP   c                ó.   € V P                   P                  # rÕ   ©rk   r7   ©rK   s   &rN   Úget_input_embeddingsÚBertModel.get_input_embeddingsf  s   € Ø�‰×.Ñ.Ð.rP   c                ó&   € WP                   n        R # rÕ   r�  )rK   r|   s   &&rN   Úset_input_embeddingsÚBertModel.set_input_embeddingsi  s   € Ø*/�‰Ö'rP   c                óª  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[R,          R
S[R,          RS[S[,          RS[S[ P                  ,          S[,          /# )rR   rS   Nr}   rI   rE   rT   r½   ræ   r¢   r#  r   rV   )rB   rZ   r
   r$  r   r   r¤   r   )r[   r\   s   "€rN   r]   ÚBertModel.__annotate__o  sÕ   ø€ ÷ ?
ñ ?
á—<‘< $Õ&ð?
ñ Ÿ™ tÕ+ð?
ñ Ÿ™ tÕ+ð	?
ñ
 —l‘l TÕ)ð?
ñ —|‘| dÕ*ð?
ñ  %Ÿ|™|¨dÕ2ð?
ñ !&§¡¨tÕ 3ð?
ñ  �ð?
ñ ˜$•;ð?
ñ Ñ+Õ,ð?
ñ 
‰u�|‰|Õ	ÑKÕ	Kñ?
rP   c
                óð  € VR J VR J,          '       d   \        R4      hV P                  P                  '       d   V	e   T	MV P                  P                  p	MRp	V	'       do   Vfk   Vf   V P                  P                  '       d5   \        \        V P                  R7      \        V P                  R7      4      M\        V P                  R7      pVe   VP                  4       M^ pV P                  VVVVVR7      pV P                  VVVVVR7      w  r'V P                  ! V3RVRVRVR	VR
V	RV/V
B pVP                  pV P                  e   V P                  V4      MR p\        VVVP                  R7      # )Nz:You must specify exactly one of input_ids or inputs_embedsF)rL   )rS   rE   rI   rT   rU   )r}   ræ   Úembedding_outputr½   r¢   r}   r½   ræ   r¢   r#  rE   )r'  Úpooler_outputr¢   )r–   rL   r›   r#  Úis_encoder_decoderr   r   Úget_seq_lengthrk   Ú_create_attention_masksr‰  r'  rŠ  r   r¢   )rK   rS   r}   rI   rE   rT   r½   ræ   r¢   r#  r   rU   r™  Úencoder_outputsrP  r5  s   &&&&&&&&&&,     rN   rl   ÚBertModel.forwardl  s£  € ð  ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZà�;‰;×!×!Ð!Ø%.Ò%:™	ÀÇÁ×@UÑ@U‰IàˆIç˜Ò0ð )Ò4¸¿¹×8V×8VÐ8Vô $¤L¸¿¹Ô$DÄlÐZ^×ZeÑZeÔFfÔgä!¨¯©Ô5ð ð FUÒE` ×!?Ñ!?Ô!AÐfgÐàŸ?™?ØØ%Ø)Ø'Ø#9ð +ó 
Ðð 26×1MÑ1MØ)Ø#9Ø-Ø"7Ø+ð 2Nó 2
Ñ.ˆð Ÿ,š,Øñ	
à)ð	
ð #8ð	
ð $:ð		
ð
 ,ð	
ð  ð	
ð &ð	
ð ñ	
ˆð *×;Ñ;ˆØ8<¿¹Ò8O˜Ÿ™ OÔ4ÐUYˆä;Ø-Ø'Ø+×;Ñ;ô
ð 	
rP   c                óÞ   € V P                   P                  '       d   \        V P                   VVVR 7      pM\        V P                   VVR7      pVe   \        V P                   VVVR7      pW3# ))rL   rT   r}   r¢   )rL   rT   r}   )rL   rT   r}   r½   )rL   r›   r   r   )rK   r}   ræ   r™  r½   r¢   s   &&&&&&rN   r�  Ú!BertModel._create_attention_masks°  su   € ð �;‰;×!×!Ð!Ü/Ø—{‘{Ø.Ø-Ø /ô	‰Nô 7Ø—{‘{Ø.Ø-ôˆNð "Ò-Ü%>Ø—{‘{Ø.Ø5Ø&;ô	&Ð"ð Ð5Ð5rP   )rL   rk   r‰  rˆ  rŠ  )T)	NNNNNNNNN)ro   rp   rq   rr   Ú_no_split_modulesr2   r‘  r”  r#   r$   r    rl   r�  rt   ru   rv   rw   s   @@rN   r†  r†  F  sU   ù‡ € ð *¨;Ð7Ð÷ò"/ò0ð  ØØ÷?
ñ ?
ó ó ó  ð?
÷B6ò 6rP   r†  z¨
    Bert Model with two heads on top as done during the pretraining: a `masked language modeling` head and a `next
    sentence prediction (classification)` head.
    c                   ót   a a€ ] tR tRt oRRRR/tV 3R ltR tR t]]	RV3R	 lR
 ll4       4       t
RtVtV ;t# )ÚBertForPreTrainingiÑ  úcls.predictions.decoder.weightú&bert.embeddings.word_embeddings.weightúcls.predictions.decoder.biasúcls.predictions.biasc                ó†   <€ \         SV `  V4       \        V4      V n        \	        V4      V n        V P                  4        R # rÕ   )r1   r2   r†  ri  ra  Úclsr‹  rJ   s   &&€rN   r2   ÚBertForPreTraining.__init__Ý  s4   ø€ Ü‰Ñ˜Ô ä˜fÓ%ˆŒ	Ü'¨Ó/ˆŒð 	�‰ÖrP   c                óB   € V P                   P                  P                  # rÕ   ©rª  rM  rE  r�  s   &rN   Úget_output_embeddingsÚ(BertForPreTraining.get_output_embeddingsæ  ó   € Ø�x‰x×#Ñ#×+Ñ+Ð+rP   c                ó„   € WP                   P                  n        VP                  V P                   P                  n        R # rÕ   ©rª  rM  rE  rC  ©rK   Únew_embeddingss   &&rN   Úset_output_embeddingsÚ(BertForPreTraining.set_output_embeddingsé  ó*   € Ø'5�‰×ÑÔ$Ø$2×$7Ñ$7ˆ�‰×ÑÖ!rP   c                ó‚  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[S[,          R
S[S[ P                  ,          S[,          /	# )rR   rS   Nr}   rI   rE   rT   ÚlabelsÚnext_sentence_labelr   rV   )rB   rZ   r   r   r¤   r~  )r[   r\   s   "€rN   r]   ÚBertForPreTraining.__annotate__ï  sÄ   ø€ ÷ A
ñ A
á—<‘< $Õ&ðA
ñ Ÿ™ tÕ+ðA
ñ Ÿ™ tÕ+ð	A
ñ
 —l‘l TÕ)ðA
ñ —|‘| dÕ*ðA
ñ —‘˜tÕ#ðA
ñ #Ÿ\™\¨DÕ0ðA
ñ Ñ+Õ,ðA
ñ 
‰u�|‰|Õ	Ñ7Õ	7ñA
rP   c                óÂ  € V P                   ! V3RVRVRVRVRR/VB p	V	R,          w  r«V P                  W«4      w  rÍRpVe|   Vex   \        4       pV! VP                  R
V P                  P
                  4      VP                  R
4      4      pV! VP                  R
^4      VP                  R
4      4      pVV,           p\        VVVV	P                  V	P                  R	7      # )aÅ  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
    config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked),
    the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
next_sentence_label (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the next sequence prediction (classification) loss. Input should be a sequence
    pair (see `input_ids` docstring) Indices should be in `[0, 1]`:

    - 0 indicates sequence B is a continuation of sequence A,
    - 1 indicates sequence B is a random sequence.

Example:

```python
>>> from transformers import AutoTokenizer, BertForPreTraining
>>> import torch

>>> tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-uncased")
>>> model = BertForPreTraining.from_pretrained("google-bert/bert-base-uncased")

>>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
>>> outputs = model(**inputs)

>>> prediction_logits = outputs.prediction_logits
>>> seq_relationship_logits = outputs.seq_relationship_logits
```
r}   rI   rE   rT   Úreturn_dictT:NrR   NN)r€  r�  r‚  r¡   rj  r0   )	ri  rª  r   r¨   rL   r4   r~  r¡   rj  )rK   rS   r}   rI   rE   rT   r¹  rº  r   ÚoutputsrP  r5  rT  r^  Ú
total_lossÚloss_fctÚmasked_lm_lossÚnext_sentence_losss   &&&&&&&&,         rN   rl   ÚBertForPreTraining.forwardí  s  € ðR —)’)Øñ
à)ð
ð *ð
ð &ð	
ð
 (ð
ð ð
ð ñ
ˆð *1°­Ñ&ˆØ48·H±H¸_Ó4\Ñ1Ðàˆ
ØÒÐ"5Ò"AÜ'Ó)ˆHÙ%Ð&7×&<Ñ&<¸RÀÇÁ×AWÑAWÓ&XÐZ`×ZeÑZeÐfhÓZiÓjˆNÙ!)Ð*@×*EÑ*EÀbÈ!Ó*LÐNa×NfÑNfÐgiÓNjÓ!kÐØ'Ð*<Õ<ˆJä'ØØ/Ø$:Ø!×/Ñ/Ø×)Ñ)ô
ð 	
rP   ©ri  rª  ©NNNNNNN©ro   rp   rq   rr   Ú_tied_weights_keysr2   r®  rµ  r"   r    rl   rt   ru   rv   rw   s   @@rN   r¤  r¤  Ñ  sT   ù‡ € ð 	)Ð*RØ&Ð(>ðÐõ
ò,ò8ð Ø÷A
ñ A
ó ó ÷A
ð A
rP   r¤  zP
    Bert Model with a `language modeling` head on top for CLM fine-tuning.
    c                   ót   a a€ ] tR tRt oRRRR/tV 3R ltR tR t]]	RV3R	 lR
 ll4       4       t
RtVtV ;t# )ÚBertLMHeadModeli3  r¥  r¦  r§  r¨  c                óØ   <€ \         SV `  V4       VP                  '       g   \        P	                  R 4       \        VRR7      V n        \        V4      V n        V P                  4        R# )zLIf you want to use `BertLMHeadModel` as a standalone, add `is_decoder=True.`F©rŒ  N©
r1   r2   r›   ÚloggerÚwarningr†  ri  rK  rª  r‹  rJ   s   &&€rN   r2   ÚBertLMHeadModel.__init__>  sO   ø€ Ü‰Ñ˜Ô à× × Ð Ü�N‰NÐiÔjä˜f¸Ô>ˆŒ	Ü" 6Ó*ˆŒð 	�‰ÖrP   c                óB   € V P                   P                  P                  # rÕ   r­  r�  s   &rN   r®  Ú%BertLMHeadModel.get_output_embeddingsJ  r°  rP   c                ó„   € WP                   P                  n        VP                  V P                   P                  n        R # rÕ   r²  r³  s   &&rN   rµ  Ú%BertLMHeadModel.set_output_embeddingsM  r·  rP   c                óü  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[ P                  R,          R
S[R,          RS[R,          RS[S[ P                  ,          RS[S[,          RS[S[ P                  ,          S[,          /# )rR   rS   Nr}   rI   rE   rT   r½   ræ   r¹  r¢   r#  Úlogits_to_keepr   rV   )	rB   rZ   r
   r$  rY   r   r   r¤   r   )r[   r\   s   "€rN   r]   ÚBertLMHeadModel.__annotate__S  sû   ø€ ÷ 6
ñ 6
á—<‘< $Õ&ð6
ñ Ÿ™ tÕ+ð6
ñ Ÿ™ tÕ+ð	6
ñ
 —l‘l TÕ)ð6
ñ —|‘| dÕ*ð6
ñ  %Ÿ|™|¨dÕ2ð6
ñ !&§¡¨tÕ 3ð6
ñ —‘˜tÕ#ð6
ñ  �ð6
ñ ˜$•;ð6
ñ ™eŸl™lÕ*ð6
ñ Ñ+Õ,ð6
ñ 
‰u�|‰|Õ	Ñ@Õ	@ñ6
rP   c                óÆ  € Ve   Rp
V P                   ! V3RVRVRVRVRVRVR	V	R
V
RR/	VB pVP                  p\        V\        4      '       d   \	        V) R4      MTpV P                  VRVR3,          4      pRpVe-   V P                  ! RRVRVRV P                  P                  /VB p\        VVVP                  VP                  VP                  VP                  R7      # )a˜  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in
    `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are
    ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
NFr}   rI   rE   rT   r½   ræ   r¢   r#  r½  Tr`   Úlogitsr¹  r4   )r€  rØ  r¢   r¡   rj  rk  rr  )ri  r'  r©   rY   Úslicerª  Úloss_functionrL   r4   r   r¢   r¡   rj  rk  )rK   rS   r}   rI   rE   rT   r½   ræ   r¹  r¢   r#  rÕ  r   r¾  r¡   Úslice_indicesrØ  r€  s   &&&&&&&&&&&&,     rN   rl   ÚBertLMHeadModel.forwardQ  sA  € ð. ÒØˆIà@DÇ	Â	ØñA
à)ðA
ð *ðA
ð &ð	A
ð
 (ðA
ð #8ðA
ð $:ðA
ð ,ðA
ð  ðA
ð ðA
ð ñA
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜-¨¨=¸!Ð(;Õ<Ó=ˆàˆØÒØ×%Ò%Ñp¨VÐp¸FÐpÈtÏ{É{×OeÑOeÐpÐioÑpˆDä0ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø$×5Ñ5ô
ð 	
rP   rÄ  )NNNNNNNNNNrn   rÆ  rw   s   @@rN   rÉ  rÉ  3  sP   ù‡ € ð 	)Ð*RØ&Ð(>ðÐõ

ò,ò8ð Ø÷6
ñ 6
ó ó ÷6
ð 6
rP   rÉ  c                   ót   a a€ ] tR tRt oRRRR/tV 3R ltR tR t]]	RV3R	 lR
 ll4       4       t
RtVtV ;t# )ÚBertForMaskedLMiŒ  r¥  r¦  r§  r¨  c                óØ   <€ \         SV `  V4       VP                  '       d   \        P	                  R 4       \        VRR7      V n        \        V4      V n        V P                  4        R# )zkIf you want to use `BertForMaskedLM` make sure `config.is_decoder=False` for bi-directional self-attention.FrË  NrÌ  rJ   s   &&€rN   r2   ÚBertForMaskedLM.__init__“  sU   ø€ Ü‰Ñ˜Ô à××ÐÜ�N‰Nð1ôô
 ˜f¸Ô>ˆŒ	Ü" 6Ó*ˆŒð 	�‰ÖrP   c                óB   € V P                   P                  P                  # rÕ   r­  r�  s   &rN   r®  Ú%BertForMaskedLM.get_output_embeddings¢  r°  rP   c                ó„   € WP                   P                  n        VP                  V P                   P                  n        R # rÕ   r²  r³  s   &&rN   rµ  Ú%BertForMaskedLM.set_output_embeddings¥  r·  rP   c                óª  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[ P                  R,          R
S[S[,          RS[S[ P                  ,          S[,          /
# )rR   rS   Nr}   rI   rE   rT   r½   ræ   r¹  r   rV   )rB   rZ   r   r   r¤   r   )r[   r\   s   "€rN   r]   ÚBertForMaskedLM.__annotate__«  sË   ø€ ÷ +
ñ +
á—<‘< $Õ&ð+
ñ Ÿ™ tÕ+ð+
ñ Ÿ™ tÕ+ð	+
ñ
 —l‘l TÕ)ð+
ñ —|‘| dÕ*ð+
ñ  %Ÿ|™|¨dÕ2ð+
ñ !&§¡¨tÕ 3ð+
ñ —‘˜tÕ#ð+
ñ Ñ+Õ,ð+
ñ 
‰u�|‰|Õ	™~Õ	-ñ+
rP   c	                óV  € V P                   ! V3RVRVRVRVRVRVRR/V	B p
V
^ ,          pV P                  V4      pR	pVeG   \        4       pV! VP                  RV P                  P
                  4      VP                  R4      4      p\        VVV
P                  V
P                  R
7      # )az  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
    config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
    loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
r}   rI   rE   rT   r½   ræ   r½  TN©r€  rØ  r¡   rj  r0   )	ri  rª  r   r¨   rL   r4   r   r¡   rj  )rK   rS   r}   rI   rE   rT   r½   ræ   r¹  r   r¾  rP  rT  rÁ  rÀ  s   &&&&&&&&&,     rN   rl   ÚBertForMaskedLM.forward©  sæ   € ð( —)’)Øñ

à)ð

ð *ð

ð &ð	

ð
 (ð

ð #8ð

ð $:ð

ð ð

ð ñ

ˆð " !�*ˆØ ŸH™H _Ó5ÐàˆØÒÜ'Ó)ˆHÙ%Ð&7×&<Ñ&<¸RÀÇÁ×AWÑAWÓ&XÐZ`×ZeÑZeÐfhÓZiÓjˆNäØØ$Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
rP   rÄ  )NNNNNNNNrÆ  rw   s   @@rN   rÞ  rÞ  Œ  sP   ù‡ € ð 	)Ð*RØ&Ð(>ðÐõ
ò,ò8ð Ø÷+
ñ +
ó ó ÷+
ð +
rP   rÞ  zT
    Bert Model with a `next sentence prediction (classification)` head on top.
    c                   ó\   a a€ ] tR tRt oV 3R lt]]RV3R lR ll4       4       tRtVt	V ;t
# )ÚBertForNextSentencePredictioniÙ  c                ó†   <€ \         SV `  V4       \        V4      V n        \	        V4      V n        V P                  4        R # rÕ   )r1   r2   r†  ri  rW  rª  r‹  rJ   s   &&€rN   r2   Ú&BertForNextSentencePrediction.__init__ß  s4   ø€ Ü‰Ñ˜Ô ä˜fÓ%ˆŒ	Ü" 6Ó*ˆŒð 	�‰ÖrP   c                óZ  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          R	S[S[ P                  ,          S[,          /# ©
rR   rS   Nr}   rI   rE   rT   r¹  r   rV   )rB   rZ   r   r   r¤   r   )r[   r\   s   "€rN   r]   Ú*BertForNextSentencePrediction.__annotate__ê  s¦   ø€ ÷ =
ñ =
á—<‘< $Õ&ð=
ñ Ÿ™ tÕ+ð=
ñ Ÿ™ tÕ+ð	=
ñ
 —l‘l TÕ)ð=
ñ —|‘| dÕ*ð=
ñ —‘˜tÕ#ð=
ñ Ñ+Õ,ð=
ñ 
‰u�|‰|Õ	Ñ:Õ	:ñ=
rP   c                ó&  € V P                   ! V3RVRVRVRVRR/VB pV^,          p	V P                  V	4      p
RpVe3   \        4       pV! V
P                  R	^4      VP                  R	4      4      p\	        VV
VP
                  VP                  R7      # )
a"  
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair
    (see `input_ids` docstring). Indices should be in `[0, 1]`:

    - 0 indicates sequence B is a continuation of sequence A,
    - 1 indicates sequence B is a random sequence.

Example:

```python
>>> from transformers import AutoTokenizer, BertForNextSentencePrediction
>>> import torch

>>> tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-uncased")
>>> model = BertForNextSentencePrediction.from_pretrained("google-bert/bert-base-uncased")

>>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced."
>>> next_sentence = "The sky is blue due to the shorter wavelength of blue light."
>>> encoding = tokenizer(prompt, next_sentence, return_tensors="pt")

>>> outputs = model(**encoding, labels=torch.LongTensor([1]))
>>> logits = outputs.logits
>>> assert logits[0, 0] < logits[0, 1]  # next sentence was random
```
r}   rI   rE   rT   r½  TNrè  r0   )ri  rª  r   r¨   r   r¡   rj  )rK   rS   r}   rI   rE   rT   r¹  r   r¾  r5  Úseq_relationship_scoresrÂ  rÀ  s   &&&&&&&,     rN   rl   Ú%BertForNextSentencePrediction.forwardè  sÇ   € ðN —)’)Øñ
à)ð
ð *ð
ð &ð	
ð
 (ð
ð ð
ð ñ
ˆð   �
ˆà"&§(¡(¨=Ó"9Ðà!ÐØÒÜ'Ó)ˆHÙ!)Ð*A×*FÑ*FÀrÈ1Ó*MÈvÏ{É{Ð[]ËÓ!_Ðä*Ø#Ø*Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
rP   rÄ  ©NNNNNN©ro   rp   rq   rr   r2   r"   r    rl   rt   ru   rv   rw   s   @@rN   rë  rë  Ù  s0   ù‡ € õð Ø÷=
ñ =
ó ó ÷=
ð =
rP   rë  zœ
    Bert Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled
    output) e.g. for GLUE tasks.
    c                   ó\   a a€ ] tR tRt oV 3R lt]]RV3R lR ll4       4       tRtVt	V ;t
# )ÚBertForSequenceClassificationi*  c                óv  <€ \         SV `  V4       VP                  V n        Wn        \	        V4      V n        VP                  e   VP                  MVP                  p\        P                  ! V4      V n
        \        P                  ! VP                  VP                  4      V n        V P                  4        R # rÕ   )r1   r2   Ú
num_labelsrL   r†  ri  Úclassifier_dropoutr?   r   r>   r@   r™   r5   Ú
classifierr‹  ©rK   rL   rú  rM   s   && €rN   r2   Ú&BertForSequenceClassification.__init__1  s�   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒØŒä˜fÓ%ˆŒ	à)/×)BÑ)BÒ)NˆF×%Ò%ÐTZ×TnÑTnð 	ô —z’zÐ"4Ó5ˆŒÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰ÖrP   c                óZ  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          R	S[S[ P                  ,          S[,          /# rï  )rB   rZ   r   r   r¤   r   )r[   r\   s   "€rN   r]   Ú*BertForSequenceClassification.__annotate__B  s¦   ø€ ÷ ;
ñ ;
á—<‘< $Õ&ð;
ñ Ÿ™ tÕ+ð;
ñ Ÿ™ tÕ+ð	;
ñ
 —l‘l TÕ)ð;
ñ —|‘| dÕ*ð;
ñ —‘˜tÕ#ð;
ñ Ñ+Õ,ð;
ñ 
‰u�|‰|Õ	Ñ7Õ	7ñ;
rP   c                ó  € V P                   ! V3RVRVRVRVRR/VB pV^,          p	V P                  V	4      p	V P                  V	4      p
RpVEe˜   V P                  P                  f–   V P
                  ^8X  d   RV P                  n        MrV P
                  ^8”  dQ   VP                  \        P                  8X  g    VP                  \        P                  8X  d   R	V P                  n        MR
V P                  n        V P                  P                  R8X  dK   \        4       pV P
                  ^8X  d'   V! V
P                  4       VP                  4       4      pMŽV! W¦4      pM…V P                  P                  R	8X  d>   \        4       pV! V
P                  RV P
                  4      VP                  R4      4      pM-V P                  P                  R
8X  d   \        4       pV! W¦4      p\        VV
VP                   VP"                  R7      # )ae  
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
    `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
r}   rI   rE   rT   r½  TNÚ
regressionÚsingle_label_classificationÚmulti_label_classificationrè  r0   )ri  r@   rû  rL   Úproblem_typerù  r-   rB   rH   rY   r   Úsqueezer   r¨   r   r   r¡   rj  )rK   rS   r}   rI   rE   rT   r¹  r   r¾  r5  rØ  r€  rÀ  s   &&&&&&&,     rN   rl   Ú%BertForSequenceClassification.forward@  s¿  € ð$ —)’)Øñ
à)ð
ð *ð
ð &ð	
ð
 (ð
ð ð
ð ñ
ˆð   �
ˆàŸ™ ]Ó3ˆØ—‘ Ó/ˆàˆØÓØ�{‰{×'Ñ'Ò/Ø—?‘? aÔ'Ø/;�D—K‘KÕ,Ø—_‘_ qÔ(¨f¯l©l¼e¿j¹jÔ.HÈFÏLÉLÔ\a×\eÑ\eÔLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ô7Ü"›9�Ø—?‘? aÔ'Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# FÓ3‘DØ—‘×)Ñ)Ð-JÔJÜ+Ó-�Ù §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR‘Ø—‘×)Ñ)Ð-IÔIÜ,Ó.�Ù Ó/�ä'ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
rP   )ri  rû  rL   r@   rù  rô  rõ  rw   s   @@rN   r÷  r÷  *  s0   ù‡ € õð Ø÷;
ñ ;
ó ó ÷;
ð ;
rP   r÷  c                   ó\   a a€ ] tR tRt oV 3R lt]]RV3R lR ll4       4       tRtVt	V ;t
# )ÚBertForMultipleChoicei€  c                ó4  <€ \         SV `  V4       \        V4      V n        VP                  e   VP                  MVP
                  p\        P                  ! V4      V n        \        P                  ! VP                  ^4      V n        V P                  4        R # rÕ   )r1   r2   r†  ri  rú  r?   r   r>   r@   r™   r5   rû  r‹  rü  s   && €rN   r2   ÚBertForMultipleChoice.__init__‚  su   ø€ Ü‰Ñ˜Ô ä˜fÓ%ˆŒ	à)/×)BÑ)BÒ)NˆF×%Ò%ÐTZ×TnÑTnð 	ô —z’zÐ"4Ó5ˆŒÜŸ)š) F×$6Ñ$6¸Ó:ˆŒð 	�‰ÖrP   c                óZ  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          R	S[S[ P                  ,          S[,          /# rï  )rB   rZ   r   r   r¤   r   )r[   r\   s   "€rN   r]   Ú"BertForMultipleChoice.__annotate__‘  s°   ø€ ÷ N
ñ N
á—<‘< $Õ&ðN
ñ Ÿ™ tÕ+ðN
ñ Ÿ™ tÕ+ð	N
ñ
 —l‘l TÕ)ðN
ñ —|‘| dÕ*ðN
ñ —‘˜tÕ#ðN
ñ Ñ+Õ,ðN
ñ 
‰u�|‰|Õ	Ñ8Õ	8ñN
rP   c                ó&  € Ve   VP                   ^,          MVP                   ^,          pVe"   VP                  R	VP                  R	4      4      MRpVe"   VP                  R	VP                  R	4      4      MRpVe"   VP                  R	VP                  R	4      4      MRpVe"   VP                  R	VP                  R	4      4      MRpVe2   VP                  R	VP                  R
4      VP                  R	4      4      MRpV P                  ! V3RVRVRVRVRR/VB p	V	^,          p
V P	                  V
4      p
V P                  V
4      pVP                  R	V4      pRpVe   \        4       pV! WÆ4      p\        VVV	P                  V	P                  R7      # )a›  
input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
    Indices of input sequence tokens in the vocabulary.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are input IDs?](../glossary#input-ids)
token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
    Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
    1]`:

    - 0 corresponds to a *sentence A* token,
    - 1 corresponds to a *sentence B* token.

    [What are token type IDs?](../glossary#token-type-ids)
position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
    Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
    config.max_position_embeddings - 1]`.

    [What are position IDs?](../glossary#position-ids)
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):
    Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
    is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
    model's internal embedding lookup matrix.
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
    num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
    `input_ids` above)
Nr}   rI   rE   rT   r½  Trè  r0   éþÿÿÿ)
re   r¨   rG   ri  r@   rû  r   r   r¡   rj  )rK   rS   r}   rI   rE   rT   r¹  r   Únum_choicesr¾  r5  rØ  Úreshaped_logitsr€  rÀ  s   &&&&&&&,       rN   rl   ÚBertForMultipleChoice.forward�  s¶  € ðT -6Ò,A�i—o‘o aÖ(À}×GZÑGZÐ[\ÕG]ˆà>GÒ>S�I—N‘N 2 y§~¡~°bÓ'9Ô:ÐY]ˆ	ØM[ÒMg˜×,Ñ,¨R°×1DÑ1DÀRÓ1HÔIÐmqˆØM[ÒMg˜×,Ñ,¨R°×1DÑ1DÀRÓ1HÔIÐmqˆØGSÒG_�|×(Ñ(¨¨\×->Ñ->¸rÓ-BÔCÐeiˆð Ò(ð ×Ñ˜r =×#5Ñ#5°bÓ#9¸=×;MÑ;MÈbÓ;QÔRàð 	ð —)’)Øñ
à)ð
ð *ð
ð &ð	
ð
 (ð
ð ð
ð ñ
ˆð   �
ˆàŸ™ ]Ó3ˆØ—‘ Ó/ˆØ Ÿ+™+ b¨+Ó6ˆàˆØÒÜ'Ó)ˆHÙ˜OÓ4ˆDä(ØØ"Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
rP   )ri  rû  r@   rô  rõ  rw   s   @@rN   r  r  €  s4   ù‡ € õð Ø÷N
ñ N
ó ó ÷N
ð N
rP   r  c                   ó\   a a€ ] tR tRt oV 3R lt]]RV3R lR ll4       4       tRtVt	V ;t
# )ÚBertForTokenClassificationiâ  c                ón  <€ \         SV `  V4       VP                  V n        \        VR R7      V n        VP
                  e   VP
                  MVP                  p\        P                  ! V4      V n	        \        P                  ! VP                  VP                  4      V n        V P                  4        R# ©FrË  N)r1   r2   rù  r†  ri  rú  r?   r   r>   r@   r™   r5   rû  r‹  rü  s   && €rN   r2   Ú#BertForTokenClassification.__init__ä  sŠ   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä˜f¸Ô>ˆŒ	à)/×)BÑ)BÒ)NˆF×%Ò%ÐTZ×TnÑTnð 	ô —z’zÐ"4Ó5ˆŒÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰ÖrP   c                óZ  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          R	S[S[ P                  ,          S[,          /# rï  )rB   rZ   r   r   r¤   r   )r[   r\   s   "€rN   r]   Ú'BertForTokenClassification.__annotate__ô  s¦   ø€ ÷ '
ñ '
á—<‘< $Õ&ð'
ñ Ÿ™ tÕ+ð'
ñ Ÿ™ tÕ+ð	'
ñ
 —l‘l TÕ)ð'
ñ —|‘| dÕ*ð'
ñ —‘˜tÕ#ð'
ñ Ñ+Õ,ð'
ñ 
‰u�|‰|Õ	Ñ4Õ	4ñ'
rP   c                ó\  € V P                   ! V3RVRVRVRVRR/VB pV^ ,          p	V P                  V	4      p	V P                  V	4      p
RpVe=   \        4       pV! V
P	                  R	V P
                  4      VP	                  R	4      4      p\        VV
VP                  VP                  R7      # )
zÃ
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
r}   rI   rE   rT   r½  TNrè  r0   )	ri  r@   rû  r   r¨   rù  r   r¡   rj  )rK   rS   r}   rI   rE   rT   r¹  r   r¾  rP  rØ  r€  rÀ  s   &&&&&&&,     rN   rl   Ú"BertForTokenClassification.forwardò  sÑ   € ð  —)’)Øñ
à)ð
ð *ð
ð &ð	
ð
 (ð
ð ð
ð ñ
ˆð " !�*ˆàŸ,™, Ó7ˆØ—‘ Ó1ˆàˆØÒÜ'Ó)ˆHÙ˜FŸK™K¨¨D¯O©OÓ<¸f¿k¹kÈ"»oÓNˆDä$ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
rP   )ri  rû  r@   rù  rô  rõ  rw   s   @@rN   r  r  â  s0   ù‡ € õð Ø÷'
ñ '
ó ó ÷'
ð '
rP   r  c                   ó\   a a€ ] tR tRt oV 3R lt]]RV3R lR ll4       4       tRtVt	V ;t
# )ÚBertForQuestionAnsweringi  c                óì   <€ \         SV `  V4       VP                  V n        \        VR R7      V n        \
        P                  ! VP                  VP                  4      V n        V P                  4        R# r  )
r1   r2   rù  r†  ri  r   r™   r5   Ú
qa_outputsr‹  rJ   s   &&€rN   r2   Ú!BertForQuestionAnswering.__init__   sU   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä˜f¸Ô>ˆŒ	ÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰ÖrP   c                ó‚  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[S[,          R
S[S[ P                  ,          S[,          /	# )rR   rS   Nr}   rI   rE   rT   Ústart_positionsÚend_positionsr   rV   )rB   rZ   r   r   r¤   r   )r[   r\   s   "€rN   r]   Ú%BertForQuestionAnswering.__annotate__,  s¹   ø€ ÷ 3
ñ 3
á—<‘< $Õ&ð3
ñ Ÿ™ tÕ+ð3
ñ Ÿ™ tÕ+ð	3
ñ
 —l‘l TÕ)ð3
ñ —|‘| dÕ*ð3
ñ Ÿ™¨Õ,ð3
ñ —|‘| dÕ*ð3
ñ Ñ+Õ,ð3
ñ 
‰u�|‰|Õ	Ñ;Õ	;ñ3
rP   c                óð  € V P                   ! V3R VRVRVRVRR/VB p	V	^ ,          p
V P                  V
4      pVP                  ^R
R7      w  rÍVP                  R
4      P	                  4       pVP                  R
4      P	                  4       pRpVeÄ   VeÀ   \        VP                  4       4      ^8”  d   VP                  R
4      p\        VP                  4       4      ^8”  d   VP                  R
4      pVP                  ^4      pVP                  ^ V4      pVP                  ^ V4      p\        VR7      pV! WÆ4      pV! W×4      pVV,           ^,          p\        VVVV	P                  V	P                  R	7      # )r}   rI   rE   rT   r½  Trƒ   N)Úignore_index)r€  Ústart_logitsÚ
end_logitsr¡   rj  r0   )ri  r  Úsplitr  r‹   ÚlenrG   Úclampr   r   r¡   rj  )rK   rS   r}   rI   rE   rT   r!  r"  r   r¾  rP  rØ  r&  r'  r¿  Úignored_indexrÀ  Ú
start_lossÚend_losss   &&&&&&&&,          rN   rl   Ú BertForQuestionAnswering.forward*  s”  € ð —)’)Øñ
à)ð
ð *ð
ð &ð	
ð
 (ð
ð ð
ð ñ
ˆð " !�*ˆà—‘ Ó1ˆØ#)§<¡<°°r <Ó#:Ñ ˆØ#×+Ñ+¨BÓ/×:Ñ:Ó<ˆØ×'Ñ'¨Ó+×6Ñ6Ó8ˆ
àˆ
ØÒ&¨=Ò+Dä�?×'Ñ'Ó)Ó*¨QÔ.Ø"1×"9Ñ"9¸"Ó"=�Ü�=×%Ñ%Ó'Ó(¨1Ô,Ø -× 5Ñ 5°bÓ 9�à(×-Ñ-¨aÓ0ˆMØ-×3Ñ3°A°}ÓEˆOØ)×/Ñ/°°=ÓAˆMä'°]ÔCˆHÙ! ,Ó@ˆJÙ 
Ó:ˆHØ$ xÕ/°1Õ4ˆJä+ØØ%Ø!Ø!×/Ñ/Ø×)Ñ)ô
ð 	
rP   )ri  rù  r  rÅ  rõ  rw   s   @@rN   r  r    s0   ù‡ € õð Ø÷3
ñ 3
ó ó ÷3
ð 3
rP   r  )Nr§   )rÞ  r  rë  r¤  r  r÷  r  r  rÉ  r†  rh  )Xrs   Úcollections.abcr   Údataclassesr   rB   r   Útorch.nnr   r   r   Ú r   rn  Úactivationsr	   Úcache_utilsr
   r   r   Ú
generationr   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   r   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úpytorch_utilsr   Úutilsr   r   r    r!   Úutils.genericr"   r#   Úutils.output_capturingr$   Úconfiguration_bertr%   Ú
get_loggerro   rÍ  r€   r'   rŽ   r�   r¹   rÉ   rÝ   rí   rü   r  r  r,  r8  rA  rK  rW  ra  rh  r~  r†  r¤  rÉ  rÞ  rë  r÷  r  r  r  Ú__all__rr  rP   rN   Ú<module>rB     s
  ðñ å $Ý !ã Ý ß AÑ Aå &Ý !ß CÑ CÝ )ß JÝ 9÷
÷ 
õ 
÷ GÝ &Ý 6ß MÓ Mß IÝ 5Ý *ð 
×	Ò	˜HÓ	%€ô7�R—Y‘Yô 7÷t%ô8@)˜Ÿ	™	ô @)ôFI)˜Ÿ™ô I)ôX�R—Y‘Yô ô.�B—I‘Iô .ô:�r—y‘yô ô�—‘ô ô>Ð*ô >ôB
�"—)‘)ô 
ô@�—‘ô ô "§)¡)ô ô"˜2Ÿ9™9ô ô !�b—i‘iô !ô&�b—i‘iô &ô	9˜2Ÿ9™9ô 	9ð ô/˜/ó /ó ð/ñ2 ðôð
 ô7˜{ó 7ó óð7ñ& ð	ôô|6Ð#ó |6óð|6ñ~ ðôôY
Ð,ó Y
óðY
ñx ðôô
Q
Ð)¨?ó Q
óð
Q
ðh ôI
Ð)ó I
ó ðI
ñX ðôô
I
Ð$7ó I
óð
I
ñX ðôôM
Ð$7ó M
óðM
ð` ô^
Ð/ó ^
ó ð^
ðB ô8
Ð!4ó 8
ó ð8
ðv ô@
Ð2ó @
ó ð@
òF‚rP   