+
    œ°³j¡Æ  ã                   ó  € ^ RI Ht ^ RIHt ^ RIt^ RIHt ^ RIHtHtH	t	 ^RI
Ht ^RIHt ^RIHt ^R	IHt ^R
IHtHtHtHtHtHtHtHt ^RIHtHt ^RIH t  ^RI!H"t"H#t#H$t$H%t% ^RI&H't'H(t( ^RI)H*t* ^RI+H,t, ]%PZ                  ! ].4      t/ ! R R]P`                  4      t1R]Pd                  R]1/t3 ! R R]P`                  4      t4RVR R llt5 ! R R]P`                  4      t6 ! R R]P`                  4      t7 ! R R]P`                  4      t8 ! R R ]P`                  4      t9 ! R! R"]P`                  4      t: ! R# R$]P`                  4      t; ! R% R&]P`                  4      t< ! R' R(]P`                  4      t= ! R) R*]P`                  4      t> ! R+ R,]P`                  4      t? ! R- R.]4      t@ ! R/ R0]P`                  4      tA ! R1 R2]P`                  4      tB ! R3 R4]P`                  4      tC ! R5 R6]P`                  4      tD ! R7 R8]P`                  4      tE ! R9 R:]P`                  4      tF]$ ! R; R<]4      4       tG]$! R=R>7      ] ! R? R@]"4      4       4       tH]$ ! RA RB]G4      4       tI]$! RCR>7       ! RD RE]G4      4       tJ]$ ! RF RG]G4      4       tK ! RH RI]P`                  4      tL]$! RJR>7       ! RK RL]G4      4       tM]$! RMR>7       ! RN RO]G4      4       tN]$ ! RP RQ]G4      4       tO]$ ! RR RS]G4      4       tP]$ ! RT RU]G4      4       tQ. RWOtRR# )Xé    )ÚCallable)Ú	dataclassN)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELoss)Úinitialization)ÚACT2FN)Úcreate_bidirectional_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚBaseModelOutputWithPoolingÚMaskedLMOutputÚMultipleChoiceModelOutputÚNextSentencePredictorOutputÚQuestionAnsweringModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚModelOutputÚTransformersKwargsÚauto_docstringÚlogging)Úcan_return_tupleÚmerge_with_config_defaults)Úcapture_outputs©ÚMobileBertConfigc                   óH   a a€ ] tR t^7t oRV 3R lltV3R lR ltRtVtV ;t# )ÚNoNormc                óà   <€ \         SV `  4        \        P                  ! \        P
                  ! V4      4      V n        \        P                  ! \        P                  ! V4      4      V n        R # ©N)	ÚsuperÚ__init__r   Ú	ParameterÚtorchÚzerosÚbiasÚonesÚweight)ÚselfÚ	feat_sizeÚepsÚ	__class__s   &&&€Ús/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/mobilebert/modeling_mobilebert.pyr&   ÚNoNorm.__init__8   s@   ø€ Ü‰ÑÔÜ—L’L¤§¢¨YÓ!7Ó8ˆŒ	Ü—l’l¤5§:¢:¨iÓ#8Ó9ˆŽó    c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )é   Úinput_tensorÚreturn©r(   ÚTensor)ÚformatÚ__classdict__s   "€r1   Ú__annotate__ÚNoNorm.__annotate__=   s#   ø€ ÷ 6ñ 6¡E§L¡Lð 6±U·\±\ñ 6r3   c                óH   € WP                   ,          V P                  ,           # r$   )r,   r*   )r-   r6   s   &&r1   ÚforwardÚNoNorm.forward=   s   € ØŸk™kÕ)¨D¯I©IÕ5Ð5r3   )r*   r,   r$   ©	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r&   r?   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r0   r;   s   @@r1   r"   r"   7   s   ù‡ € ÷:÷
6÷ 6ð 6r3   r"   Ú
layer_normÚno_normc                   óL   a a€ ] tR t^Dt oRtV 3R ltRV3R lR lltRtVtV ;t	# )ÚMobileBertEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c                ó”  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  V n        \
        P                  ! VP                  VP                  VP                  R 7      V n	        \
        P                  ! VP                  VP                  4      V n        \
        P                  ! VP                  VP                  4      V n        V P                  '       d   ^M^pV P                  V,          p\
        P                  ! W1P                  4      V n        \         VP"                  ,          ! VP                  4      V n        \
        P&                  ! VP(                  4      V n        \
        P,                  ! \.        P0                  ! VP                  4      P3                  R4      RR7      V n        R# ))Úpadding_idxF)Ú
persistentN©é   éÿÿÿÿ)r%   r&   Útrigram_inputÚembedding_sizeÚhidden_sizer   Ú	EmbeddingÚ
vocab_sizeÚpad_token_idÚword_embeddingsÚmax_position_embeddingsÚposition_embeddingsÚtype_vocab_sizeÚtoken_type_embeddingsÚLinearÚembedding_transformationÚNORM2FNÚnormalization_typeÚ	LayerNormÚDropoutÚhidden_dropout_probÚdropoutÚBufferr(   ÚarangeÚexpandÚposition_ids)r-   ÚconfigÚembed_dim_multiplierÚembedded_input_sizer0   s   &&  €r1   r&   ÚMobileBertEmbeddings.__init__G   s@  ø€ Ü‰ÑÔØ#×1Ñ1ˆÔØ$×3Ñ3ˆÔØ!×-Ñ-ˆÔä!Ÿ|š|¨F×,=Ñ,=¸v×?TÑ?TÐbh×buÑbuÔvˆÔÜ#%§<¢<°×0NÑ0NÐPV×PbÑPbÓ#cˆÔ Ü%'§\¢\°&×2HÑ2HÈ&×J\ÑJ\Ó%]ˆÔ"à$(×$6×$6Ð$6™q¸AÐØ"×1Ñ1Ð4HÕHÐÜ(*¯	ª	Ð2E×GYÑGYÓ(ZˆÔ%ä  ×!:Ñ!:Ö;¸F×<NÑ<NÓOˆŒÜ—z’z &×"<Ñ"<Ó=ˆŒô ŸIšI¤e§l¢l°6×3QÑ3QÓ&R×&YÑ&YÐZaÓ&bÐotÔuˆÖr3   c          
      óÔ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  /# )r5   Ú	input_idsNÚtoken_type_idsrj   Úinputs_embedsr7   )r(   Ú
LongTensorÚFloatTensorr9   )r:   r;   s   "€r1   r<   Ú!MobileBertEmbeddings.__annotate__[   sm   ø€ ÷ 0ñ 0á×#Ñ# dÕ*ð0ñ ×(Ñ(¨4Õ/ð0ñ ×&Ñ&¨Õ-ð	0ñ
 ×(Ñ(¨4Õ/ð0ñ 
�‰ñ0r3   c           
     ón  € Ve   VP                  4       pMVP                  4       R R pV^,          pVf   V P                  RR V13,          pVf<   \        P                  ! V\        P                  V P                  P
                  R7      pVf   V P                  V4      pV P                  '       dr   \        P                  ! \        P                  P                  VR,          . RORR7      V\        P                  P                  VRR R13,          . R	ORR7      .^R7      pV P                  '       g   V P                  V P                  8w  d   V P                  V4      pV P                  V4      pV P!                  V4      pWG,           V,           p	V P#                  V	4      p	V P%                  V	4      p	V	# )
NºNNN)ÚdtypeÚdeviceç        )Úvalue©ÚdimrS   )rw   :rR   NN)r   r   r   rR   r   r   )r   r   rR   r   r   r   )Úsizerj   r(   r)   Úlongry   rZ   rT   Úcatr   Ú
functionalÚpadrU   rV   r`   r\   r^   rc   rf   )
r-   rp   rq   rj   rr   Úinput_shapeÚ
seq_lengthr\   r^   Ú
embeddingss
   &&&&&     r1   r?   ÚMobileBertEmbeddings.forward[   s†  € ð Ò Ø#Ÿ.™.Ó*‰Kà'×,Ñ,Ó.¨s°Ð3ˆKà  •^ˆ
àÒØ×,Ñ,¨Q°°°¨^Õ<ˆLàÒ!Ü"Ÿ[š[¨¼E¿J¹JÈt×O`ÑO`×OgÑOgÔhˆNØÒ Ø ×0Ñ0°Ó;ˆMà××Ðô "ŸIšIä—M‘M×%Ñ% m°EÕ&:Ò<NÐVYÐ%ÓZØ!Ü—M‘M×%Ñ% m°A°s¸°s°FÕ&;Ò=OÐWZÐ%Ó[ðð
 ôˆMð ××Ð ×!4Ñ!4¸×8HÑ8HÔ!HØ ×9Ñ9¸-ÓHˆMð #×6Ñ6°|ÓDÐØ $× :Ñ :¸>Ó JÐØ"Õ8Ð;PÕPˆ
Ø—^‘^ JÓ/ˆ
Ø—\‘\ *Ó-ˆ
ØÐr3   )
rc   rf   rU   r`   rV   r\   rj   r^   rT   rZ   )NNNN)
rB   rC   rD   rE   Ú__doc__r&   r?   rF   rG   rH   rI   s   @@r1   rM   rM   D   s   ù‡ € ÙQõv÷(0÷ 0ò 0r3   rM   c                ó  € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R,          R\        R	\
        \        ,          /# )
r5   ÚmoduleÚqueryÚkeyr{   Úattention_maskNÚscalingrf   Úkwargs)r   ÚModuler(   r9   Úfloatr   r   )r:   s   "r1   r<   r<   �   s{   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô �T�\ð%ô ð%ô Ô'Õ(ñ%r3   c                 ó¶  € Vf   VP                  R4      R,          p\        P                  ! WP                  ^^4      4      V,          pVe	   W„,           p\        P
                  P                  VRR7      p\        P
                  P                  W†V P                  R7      p\        P                  ! Wƒ4      p	V	P                  ^^4      P                  4       p	W˜3# )Nr|   )ÚpÚtrainingrS   ç      à¿)
r~   r(   ÚmatmulÚ	transposer   r�   Úsoftmaxrf   r“   Ú
contiguous)
r‰   rŠ   r‹   r{   rŒ   r�   rf   rŽ   Úattn_weightsÚattn_outputs
   &&&&&&&,  r1   Úeager_attention_forwardr›   �   s±   € ð ‚Ø—*‘*˜R“. DÕ(ˆô —<’< §}¡}°Q¸Ó':Ó;¸gÕE€LàÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2Ð(Ó>€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€Lä—,’,˜|Ó3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r3   c                   óH   a a€ ] tR t^«t oV 3R ltRV3R lR lltRtVtV ;t# )ÚMobileBertSelfAttentionc                óÐ  <€ \         SV `  4        Wn        VP                  V n        \	        VP
                  VP                  ,          4      V n        V P                  V P                  ,          V n        V P                  R,          V n        \        P                  ! VP
                  V P                  4      V n        \        P                  ! VP
                  V P                  4      V n        \        P                  ! VP                  '       d   VP
                  MVP                  V P                  4      V n        \        P                   ! VP"                  4      V n        RV n        R# )g      à?FNr”   )r%   r&   rk   Únum_attention_headsÚintÚtrue_hidden_sizeÚattention_head_sizeÚall_head_sizer�   r   r_   rŠ   r‹   Úuse_bottleneck_attentionrV   r{   rd   Úattention_probs_dropout_probrf   Ú	is_causal©r-   rk   r0   s   &&€r1   r&   Ú MobileBertSelfAttention.__init__¬   sú   ø€ Ü‰ÑÔØŒØ#)×#=Ñ#=ˆÔ Ü#& v×'>Ñ'>À×A[ÑA[Õ'[Ó#\ˆÔ Ø!×5Ñ5¸×8PÑ8PÕPˆÔØ×/Ñ/°Õ5ˆŒä—Y’Y˜v×6Ñ6¸×8JÑ8JÓKˆŒ
Ü—9’9˜V×4Ñ4°d×6HÑ6HÓIˆŒÜ—Y’YØ'-×'F×'FÐ'FˆF×#Ò#ÈF×L^ÑL^Ð`d×`rÑ`ró
ˆŒ
ô —z’z &×"EÑ"EÓFˆŒàˆŽr3   c                óÐ   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[ P                  R,          RS[S[,          RS[S[ P                  ,          /# )r5   Úquery_tensorÚ
key_tensorÚvalue_tensorrŒ   NrŽ   r7   ©r(   r9   rt   r   r   Útuple)r:   r;   s   "€r1   r<   Ú$MobileBertSelfAttention.__annotate__½   sm   ø€ ÷ )ñ )á—l‘lð)ñ —L‘Lð)ñ —l‘lð	)ñ
 ×)Ñ)¨DÕ0ð)ñ Ñ+Õ,ð)ñ 
‰u�|‰|Õ	ñ)r3   c                ó”  € VP                   R R p. VORNV P                  N5pV P                  V4      P                  ! V!  P	                  ^^4      pV P                  V4      P                  ! V!  P	                  ^^4      p	V P                  V4      P                  ! V!  P	                  ^^4      p
\        P                  ! V P                  P                  \        4      pT! V VV	V
V3RV P                  '       g   RMV P                  P                  RV P                  /VB w  rÍVP                   ! . VORN5!  P#                  4       pWÍ3# )Nrf   rz   r�   rS   )Úshaper¢   rŠ   Úviewr–   r‹   r{   r   Úget_interfacerk   Ú_attn_implementationr›   r“   rf   r’   r�   Úreshaper˜   )r-   rª   r«   r¬   rŒ   rŽ   rƒ   Úhidden_shapeÚquery_layerÚ	key_layerÚvalue_layerÚattention_interfacerš   r™   s   &&&&&,        r1   r?   ÚMobileBertSelfAttention.forward½   sE  € ð #×(Ñ(¨¨"Ð-ˆØC˜ÐC bÐC¨$×*BÑ*BÑCˆð —j‘j Ó.×3Ò3°\ÑB×LÑLÈQÐPQÓRˆØ—H‘H˜ZÓ(×-Ò-¨|Ñ<×FÑFÀqÈ!ÓLˆ	Ø—j‘j Ó.×3Ò3°\ÑB×LÑLÈQÐPQÓRˆä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð  $Ÿ}Ÿ}˜}‘C°$·,±,·.±.ð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØÐ(Ð(r3   )
r£   r¢   rk   rf   r¦   r‹   rŸ   rŠ   r�   r{   r$   rA   rI   s   @@r1   r�   r�   «   s   ù‡ € õ÷")÷ )ò )r3   r�   c                   óD   a a€ ] tR t^ßt oV 3R ltV3R lR ltRtVtV ;t# )ÚMobileBertSelfOutputc                óˆ  <€ \         SV `  4        VP                  V n        \        P                  ! VP
                  VP
                  4      V n        \        VP                  ,          ! VP
                  VP                  R 7      V n
        V P                  '       g(   \        P                  ! VP                  4      V n        R# R# ©©r/   N)r%   r&   Úuse_bottleneckr   r_   r¡   Údensera   rb   Úlayer_norm_epsrc   rd   re   rf   r§   s   &&€r1   r&   ÚMobileBertSelfOutput.__init__à   s‡   ø€ Ü‰ÑÔØ$×3Ñ3ˆÔÜ—Y’Y˜v×6Ñ6¸×8OÑ8OÓPˆŒ
Ü  ×!:Ñ!:Ö;¸F×<SÑ<SÐY_×YnÑYnÔoˆŒØ×"×"Ð"ÜŸ:š: f×&@Ñ&@ÓAˆDŽLñ #r3   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# ©r5   Úhidden_statesÚresidual_tensorr7   r8   )r:   r;   s   "€r1   r<   Ú!MobileBertSelfOutput.__annotate__è   s1   ø€ ÷ ñ ¡U§\¡\ð ÁEÇLÁLð ÑUZ×UaÑUañ r3   c                óœ   € V P                  V4      pV P                  '       g   V P                  V4      pV P                  W2,           4      pV# r$   )rÂ   rÁ   rf   rc   ©r-   rÇ   rÈ   Úlayer_outputss   &&& r1   r?   ÚMobileBertSelfOutput.forwardè   sA   € ØŸ
™
 =Ó1ˆØ×"×"Ð"Ø ŸL™L¨Ó7ˆMØŸ™ }Õ'FÓGˆØÐr3   )rc   rÂ   rf   rÁ   rA   rI   s   @@r1   r½   r½   ß   s   ù‡ € õB÷÷ ð r3   r½   c                   óH   a a€ ] tR t^ðt oV 3R ltRV3R lR lltRtVtV ;t# )ÚMobileBertAttentionc                ód   <€ \         SV `  4        \        V4      V n        \	        V4      V n        R # r$   )r%   r&   r�   r-   r½   Úoutputr§   s   &&€r1   r&   ÚMobileBertAttention.__init__ñ   s&   ø€ Ü‰ÑÔÜ+¨FÓ3ˆŒ	Ü*¨6Ó2ˆŽr3   c                óê   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[ P                  RS[ P                  R,          RS[S[,          RS[S[ P                  ,          /# )	r5   rª   r«   r¬   Úlayer_inputrŒ   NrŽ   r7   r­   )r:   r;   s   "€r1   r<   Ú MobileBertAttention.__annotate__ö   s{   ø€ ÷ .ñ .á—l‘lð.ñ —L‘Lð.ñ —l‘lð	.ñ
 —\‘\ð.ñ ×)Ñ)¨DÕ0ð.ñ Ñ+Õ,ð.ñ 
‰u�|‰|Õ	ñ.r3   c                óZ   € V P                   ! VVVV3/ VB w  rxV P                  Wt4      pWx3# r$   )r-   rÑ   )	r-   rª   r«   r¬   rÔ   rŒ   rŽ   Úattention_outputr™   s	   &&&&&&,  r1   r?   ÚMobileBertAttention.forwardö   sG   € ð *.¯ªØØØØñ	*
ð
 ñ*
Ñ&Ðð  Ÿ;™;Ð'7ÓEÐØÐ-Ð-r3   )rÑ   r-   r$   rA   rI   s   @@r1   rÏ   rÏ   ð   s   ù‡ € õ3÷
.÷ .ò .r3   rÏ   c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚMobileBertIntermediatei  c                ó"  <€ \         SV `  4        \        P                  ! VP                  VP
                  4      V n        \        VP                  \        4      '       d   \        VP                  ,          V n        R # VP                  V n        R # r$   )r%   r&   r   r_   r¡   Úintermediate_sizerÂ   Ú
isinstanceÚ
hidden_actÚstrr
   Úintermediate_act_fnr§   s   &&€r1   r&   ÚMobileBertIntermediate.__init__  s`   ø€ Ü‰ÑÔÜ—Y’Y˜v×6Ñ6¸×8PÑ8PÓQˆŒ
Ü�f×'Ñ'¬×-Ò-Ü'-¨f×.?Ñ.?Õ'@ˆDÖ$à'-×'8Ñ'8ˆDÖ$r3   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r5   rÇ   r7   r8   )r:   r;   s   "€r1   r<   Ú#MobileBertIntermediate.__annotate__  ó#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r3   c                óJ   € V P                  V4      pV P                  V4      pV# r$   ©rÂ   rà   ©r-   rÇ   s   &&r1   r?   ÚMobileBertIntermediate.forward  s&   € ØŸ
™
 =Ó1ˆØ×0Ñ0°Ó?ˆØÐr3   rç   rA   rI   s   @@r1   rÚ   rÚ     s   ù‡ € õ9÷÷ ð r3   rÚ   c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚOutputBottlenecki  c                ó>  <€ \         SV `  4        \        P                  ! VP                  VP
                  4      V n        \        VP                  ,          ! VP
                  VP                  R 7      V n
        \        P                  ! VP                  4      V n        R# r¿   )r%   r&   r   r_   r¡   rV   rÂ   ra   rb   rÃ   rc   rd   re   rf   r§   s   &&€r1   r&   ÚOutputBottleneck.__init__  sh   ø€ Ü‰ÑÔÜ—Y’Y˜v×6Ñ6¸×8JÑ8JÓKˆŒ
Ü  ×!:Ñ!:Ö;¸F×<NÑ<NÐTZ×TiÑTiÔjˆŒÜ—z’z &×"<Ñ"<Ó=ˆŽr3   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# rÆ   r8   )r:   r;   s   "€r1   r<   ÚOutputBottleneck.__annotate__"  s1   ø€ ÷ ñ ¡U§\¡\ð ÁEÇLÁLð ÑUZ×UaÑUañ r3   c                óx   € V P                  V4      pV P                  V4      pV P                  W2,           4      pV# r$   )rÂ   rf   rc   rË   s   &&& r1   r?   ÚOutputBottleneck.forward"  s5   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØŸ™ }Õ'FÓGˆØÐr3   )rc   rÂ   rf   rA   rI   s   @@r1   rë   rë     s   ù‡ € õ>÷÷ ð r3   rë   c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚMobileBertOutputi)  c                ó�  <€ \         SV `  4        VP                  V n        \        P                  ! VP
                  VP                  4      V n        \        VP                  ,          ! VP                  4      V n
        V P                  '       g(   \        P                  ! VP                  4      V n        R # \        V4      V n        R # r$   )r%   r&   rÁ   r   r_   rÜ   r¡   rÂ   ra   rb   rc   rd   re   rf   rë   Ú
bottleneckr§   s   &&€r1   r&   ÚMobileBertOutput.__init__*  s…   ø€ Ü‰ÑÔØ$×3Ñ3ˆÔÜ—Y’Y˜v×7Ñ7¸×9PÑ9PÓQˆŒ
Ü  ×!:Ñ!:Ö;¸F×<SÑ<SÓTˆŒØ×"×"Ð"ÜŸ:š: f×&@Ñ&@ÓAˆDŽLä.¨vÓ6ˆDŽOr3   c                ó‚   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[ P                  /# )r5   Úintermediate_statesÚresidual_tensor_1Úresidual_tensor_2r7   r8   )r:   r;   s   "€r1   r<   ÚMobileBertOutput.__annotate__4  s?   ø€ ÷ 
ñ 
Ù#(§<¡<ð
ÙDIÇLÁLð
Ùej×eqÑeqð
á	�‰ñ
r3   c                óð   € V P                  V4      pV P                  '       g+   V P                  V4      pV P                  WB,           4      pV# V P                  WB,           4      pV P	                  WC4      pV# r$   )rÂ   rÁ   rf   rc   rõ   )r-   rø   rù   rú   Úlayer_outputs   &&&& r1   r?   ÚMobileBertOutput.forward4  sm   € ð —z‘zÐ"5Ó6ˆØ×"×"Ð"ØŸ<™<¨Ó5ˆLØŸ>™>¨,Õ*JÓKˆLð Ðð  Ÿ>™>¨,Õ*JÓKˆLØŸ?™?¨<ÓKˆLØÐr3   )rc   rõ   rÂ   rf   rÁ   rA   rI   s   @@r1   ró   ró   )  s   ù‡ € õ7÷
÷ 
ð 
r3   ró   c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚBottleneckLayeriA  c                óô   <€ \         SV `  4        \        P                  ! VP                  VP
                  4      V n        \        VP                  ,          ! VP
                  VP                  R 7      V n
        R# r¿   )r%   r&   r   r_   rV   Úintra_bottleneck_sizerÂ   ra   rb   rÃ   rc   r§   s   &&€r1   r&   ÚBottleneckLayer.__init__B  sR   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3OÑ3OÓPˆŒ
Ü  ×!:Ñ!:Ö;¸F×<XÑ<XÐ^d×^sÑ^sÔtˆŽr3   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rã   r8   )r:   r;   s   "€r1   r<   ÚBottleneckLayer.__annotate__G  s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r3   c                óJ   € V P                  V4      pV P                  V4      pV# r$   ©rÂ   rc   )r-   rÇ   rÔ   s   && r1   r?   ÚBottleneckLayer.forwardG  s$   € Ø—j‘j Ó/ˆØ—n‘n [Ó1ˆØÐr3   ©rc   rÂ   rA   rI   s   @@r1   r   r   A  s   ù‡ € õu÷
÷ ð r3   r   c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )Ú
BottleneckiM  c                óÐ   <€ \         SV `  4        VP                  V n        VP                  V n        \	        V4      V n        V P                  '       d   \	        V4      V n        R # R # r$   )r%   r&   Úkey_query_shared_bottleneckr¤   r   ÚinputÚ	attentionr§   s   &&€r1   r&   ÚBottleneck.__init__N  sS   ø€ Ü‰ÑÔØ+1×+MÑ+MˆÔ(Ø(.×(GÑ(GˆÔ%Ü$ VÓ,ˆŒ
Ø×+×+Ð+Ü,¨VÓ4ˆDŽNñ ,r3   c                ó^   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          /# rã   ©r(   r9   r®   )r:   r;   s   "€r1   r<   ÚBottleneck.__annotate__V  s,   ø€ ÷ ]ñ ]¡U§\¡\ð ]±e¹E¿L¹LÕ6Iñ ]r3   c                ó²   € V P                  V4      pV P                  '       d   V3^,          # V P                  '       d   V P                  V4      pW3W3# WW3# ©é   )r  r¤   r  r  )r-   rÇ   Úbottlenecked_hidden_statesÚshared_attention_inputs   &&  r1   r?   ÚBottleneck.forwardV  s^   € ð" &*§Z¡Z°Ó%>Ð"Ø×(×(Ð(Ø.Ð0°1Õ4Ð4Ø×-×-Ð-Ø%)§^¡^°MÓ%BÐ"Ø*ÀMÐnÐnà!°-Ð\Ð\r3   )r  r  r  r¤   rA   rI   s   @@r1   r  r  M  s   ù‡ € õ5÷]÷ ]ð ]r3   r  c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )Ú	FFNOutputiq  c                óô   <€ \         SV `  4        \        P                  ! VP                  VP
                  4      V n        \        VP                  ,          ! VP
                  VP                  R 7      V n
        R# r¿   )r%   r&   r   r_   rÜ   r¡   rÂ   ra   rb   rÃ   rc   r§   s   &&€r1   r&   ÚFFNOutput.__init__r  sR   ø€ Ü‰ÑÔÜ—Y’Y˜v×7Ñ7¸×9PÑ9PÓQˆŒ
Ü  ×!:Ñ!:Ö;¸F×<SÑ<SÐY_×YnÑYnÔoˆŽr3   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# rÆ   r8   )r:   r;   s   "€r1   r<   ÚFFNOutput.__annotate__w  s1   ø€ ÷ ñ ¡U§\¡\ð ÁEÇLÁLð ÑUZ×UaÑUañ r3   c                óV   € V P                  V4      pV P                  W2,           4      pV# r$   r  rË   s   &&& r1   r?   ÚFFNOutput.forwardw  s'   € ØŸ
™
 =Ó1ˆØŸ™ }Õ'FÓGˆØÐr3   r	  rA   rI   s   @@r1   r  r  q  s   ù‡ € õp÷
÷ ð r3   r  c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚFFNLayeri}  c                ód   <€ \         SV `  4        \        V4      V n        \	        V4      V n        R # r$   )r%   r&   rÚ   Úintermediater  rÑ   r§   s   &&€r1   r&   ÚFFNLayer.__init__~  s'   ø€ Ü‰ÑÔÜ2°6Ó:ˆÔÜ Ó'ˆŽr3   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rã   r8   )r:   r;   s   "€r1   r<   ÚFFNLayer.__annotate__ƒ  rå   r3   c                óJ   € V P                  V4      pV P                  W!4      pV# r$   ©r%  rÑ   )r-   rÇ   Úintermediate_outputrÌ   s   &&  r1   r?   ÚFFNLayer.forwardƒ  s(   € Ø"×/Ñ/°Ó>ÐØŸ™Ð$7ÓGˆØÐr3   r*  rA   rI   s   @@r1   r#  r#  }  s   ù‡ € õ(÷
÷ ð r3   r#  c                   óH   a a€ ] tR tRt oV 3R ltRV3R lR lltRtVtV ;t# )ÚMobileBertLayeri‰  c                óØ  <€ \         SV `  4        VP                  V n        VP                  V n        \	        V4      V n        \        V4      V n        \        V4      V n	        V P                  '       d   \        V4      V n        VP                  ^8”  dQ   \        P                  ! \        VP                  ^,
          4       Uu. uF  p\        V4      NK  	  up4      V n        R# R# u upi )rR   N)r%   r&   rÁ   Únum_feedforward_networksrÏ   r  rÚ   r%  ró   rÑ   r  rõ   r   Ú
ModuleListÚranger#  Úffn©r-   rk   Ú_r0   s   && €r1   r&   ÚMobileBertLayer.__init__Š  s´   ø€ Ü‰ÑÔØ$×3Ñ3ˆÔØ(.×(GÑ(GˆÔ%ä,¨VÓ4ˆŒÜ2°6Ó:ˆÔÜ& vÓ.ˆŒØ××ÐÜ(¨Ó0ˆDŒOØ×*Ñ*¨QÔ.Ü—}’}ÄÀf×FeÑFeÐhiÕFiÔ@jÓ%kÑ@j¸1¤h¨vÖ&6Ñ@jÑ%kÓlˆDŽHñ /ùÚ%ks   ÃC'c                óŒ   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[S[,          RS[ P                  /# ©r5   rÇ   rŒ   NrŽ   r7   )r(   r9   rt   r   r   )r:   r;   s   "€r1   r<   ÚMobileBertLayer.__annotate__—  sL   ø€ ÷ ñ á—|‘|ðñ ×)Ñ)¨DÕ0ðñ Ñ+Õ,ð	ñ
 
�‰ñr3   c                óB  € V P                   '       d   V P                  V4      w  rErgMV.^,          w  rErgV P                  ! VVVVV3/ VB w  r‰Tp
V P                  ^8w  d   V P                   F  pV! V
4      p
K  	  V P                  V
4      pV P                  WÊV4      pV# r  )rÁ   rõ   r  r0  r3  r%  rÑ   )r-   rÇ   rŒ   rŽ   rª   r«   r¬   rÔ   Úself_attention_outputr5  r×   Ú
ffn_moduler+  rý   s   &&&,          r1   r?   ÚMobileBertLayer.forward—  s¾   € ð ××ÐØBFÇ/Á/ÐR_ÓB`Ñ?ˆL l°KàCPÀ/ÐTUÕBUÑ?ˆL là#'§>¢>ØØØØØñ$
ð ñ$
Ñ Ðð 1Ðà×(Ñ(¨AÔ-Ø"Ÿhœh�
Ù#-Ð.>Ó#?Ò ñ 'ð #×/Ñ/Ð0@ÓAÐØ—{‘{Ð#6È-ÓXˆØÐr3   )r  rõ   r3  r%  r0  rÑ   rÁ   r$   rA   rI   s   @@r1   r.  r.  ‰  s   ù‡ € õm÷÷ ò r3   r.  c                   óH   a a€ ] tR tRt oV 3R ltRV3R lR lltRtVtV ;t# )ÚMobileBertEncoderiµ  c                ó¼   <€ \         SV `  4        \        P                  ! \	        VP
                  4       Uu. uF  p\        V4      NK  	  up4      V n        R # u upi r$   )r%   r&   r   r1  r2  Únum_hidden_layersr.  Úlayerr4  s   && €r1   r&   ÚMobileBertEncoder.__init__¶  sB   ø€ Ü‰ÑÔÜ—]’]ÄUÈ6×KcÑKcÔEdÓ#eÑEdÀ¤O°FÖ$;ÑEdÑ#eÓfˆŽ
ùÒ#es   ·Ac          	      óˆ   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[S[,          RS[S[,          /# r8  )r(   r9   rt   r   r   r®   r   )r:   r;   s   "€r1   r<   ÚMobileBertEncoder.__annotate__º  sS   ø€ ÷ @ñ @á—|‘|ð@ñ ×)Ñ)¨DÕ0ð@ñ Ñ+Õ,ð	@ñ
 
‘Õ	 ñ@r3   c                ój   € \        V P                  4       F  w  rEV! VV3/ VB pK  	  \        VR 7      # ))Úlast_hidden_state)Ú	enumeraterB  r   )r-   rÇ   rŒ   rŽ   ÚiÚlayer_modules   &&&,  r1   r?   ÚMobileBertEncoder.forwardº  s@   € ô  )¨¯©Ö4‰OˆAÙ(ØØñð ñŠMñ  5ô °Ô?Ð?r3   )rB  r$   rA   rI   s   @@r1   r?  r?  µ  s   ù‡ € õg÷@÷ @ò @r3   r?  c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚMobileBertPooleriÉ  c                óÎ   <€ \         SV `  4        VP                  V n        V P                  '       d3   \        P
                  ! VP                  VP                  4      V n        R # R # r$   )r%   r&   Úclassifier_activationÚdo_activater   r_   rV   rÂ   r§   s   &&€r1   r&   ÚMobileBertPooler.__init__Ê  sK   ø€ Ü‰ÑÔØ!×7Ñ7ˆÔØ××ÐÜŸš 6×#5Ñ#5°v×7IÑ7IÓJˆDŽJñ r3   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rã   r8   )r:   r;   s   "€r1   r<   ÚMobileBertPooler.__annotate__Ð  s#   ø€ ÷ 	!ñ 	!¡U§\¡\ð 	!±e·l±lñ 	!r3   c                óŽ   € VR,          pV P                   '       g   V# V P                  V4      p\        P                  ! V4      pV# )rw   )rw   r   )rP  rÂ   r(   Útanh)r-   rÇ   Úfirst_token_tensorÚpooled_outputs   &&  r1   r?   ÚMobileBertPooler.forwardÐ  sD   € ð +¨4Õ0ÐØ××ÐØ%Ð%à ŸJ™JÐ'9Ó:ˆMÜ!ŸJšJ }Ó5ˆMØ Ð r3   )rÂ   rP  rA   rI   s   @@r1   rM  rM  É  s   ù‡ € õK÷	!÷ 	!ð 	!r3   rM  c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )Ú!MobileBertPredictionHeadTransformiÜ  c                ó|  <€ \         SV `  4        \        P                  ! VP                  VP                  4      V n        \        VP                  \        4      '       d   \        VP                  ,          V n
        MVP                  V n
        \        R ,          ! VP                  VP                  R7      V n        R# )rJ   rÀ   N)r%   r&   r   r_   rV   rÂ   rÝ   rÞ   rß   r
   Útransform_act_fnra   rÃ   rc   r§   s   &&€r1   r&   Ú*MobileBertPredictionHeadTransform.__init__Ý  s   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
Ü�f×'Ñ'¬×-Ò-Ü$*¨6×+<Ñ+<Õ$=ˆDÕ!à$*×$5Ñ$5ˆDÔ!Ü  Ö.¨v×/AÑ/AÀv×G\ÑG\Ô]ˆŽr3   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rã   r8   )r:   r;   s   "€r1   r<   Ú.MobileBertPredictionHeadTransform.__annotate__æ  ó#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r3   c                ól   € V P                  V4      pV P                  V4      pV P                  V4      pV# r$   )rÂ   r\  rc   rè   s   &&r1   r?   Ú)MobileBertPredictionHeadTransform.forwardæ  s4   € ØŸ
™
 =Ó1ˆØ×-Ñ-¨mÓ<ˆØŸ™ }Ó5ˆØÐr3   )rc   rÂ   r\  rA   rI   s   @@r1   rZ  rZ  Ü  s   ù‡ € õ^÷÷ ð r3   rZ  c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚMobileBertLMPredictionHeadií  c                ó   <€ \         SV `  4        \        V4      V n        \        P
                  ! VP                  VP                  VP                  ,
          R R7      V n	        \        P
                  ! VP                  VP                  RR7      V n
        \        P                  ! \        P                  ! VP                  4      4      V n        R# )F)r*   TN)r%   r&   rZ  Ú	transformr   r_   rX   rV   rU   rÂ   Údecoderr'   r(   r)   r*   r§   s   &&€r1   r&   Ú#MobileBertLMPredictionHead.__init__î  sˆ   ø€ Ü‰ÑÔÜ:¸6ÓBˆŒô —Y’Y˜v×0Ñ0°&×2DÑ2DÀv×G\ÑG\Õ2\ÐchÔiˆŒ
Ü—y’y ×!6Ñ!6¸×8IÑ8IÐPTÔUˆŒÜ—L’L¤§¢¨V×->Ñ->Ó!?Ó@ˆŽ	r3   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rã   r8   )r:   r;   s   "€r1   r<   Ú'MobileBertLMPredictionHead.__annotate__÷  r`  r3   c                ó  € V P                  V4      pVP                  \        P                  ! V P                  P
                  P                  4       V P                  P
                  .^ R7      4      pWP                  P                  ,          pV# )r   r|   )	rf  r•   r(   r€   rg  r,   ÚtrÂ   r*   rè   s   &&r1   r?   Ú"MobileBertLMPredictionHead.forward÷  si   € ØŸ™ }Ó5ˆØ%×,Ñ,¬U¯YªY¸¿¹×8KÑ8K×8MÑ8MÓ8OÐQU×Q[ÑQ[×QbÑQbÐ7cÐijÔ-kÓlˆØŸ™×*Ñ*Õ*ˆØÐr3   )r*   rg  rÂ   rf  rA   rI   s   @@r1   rd  rd  í  s   ù‡ € õA÷÷ ð r3   rd  c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚMobileBertOnlyMLMHeadiþ  c                óD   <€ \         SV `  4        \        V4      V n        R # r$   )r%   r&   rd  Úpredictionsr§   s   &&€r1   r&   ÚMobileBertOnlyMLMHead.__init__ÿ  s   ø€ Ü‰ÑÔÜ5°fÓ=ˆÖr3   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r5   Úsequence_outputr7   r8   )r:   r;   s   "€r1   r<   Ú"MobileBertOnlyMLMHead.__annotate__  s#   ø€ ÷ !ñ !¡u§|¡|ð !¹¿¹ñ !r3   c                ó(   € V P                  V4      pV# r$   ©rq  )r-   rt  Úprediction_scoress   && r1   r?   ÚMobileBertOnlyMLMHead.forward  s   € Ø ×,Ñ,¨_Ó=ÐØ Ð r3   rw  rA   rI   s   @@r1   ro  ro  þ  s   ù‡ € õ>÷!÷ !ð !r3   ro  c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚMobileBertPreTrainingHeadsi  c                ó�   <€ \         SV `  4        \        V4      V n        \        P
                  ! VP                  ^4      V n        R# ©r5   N)r%   r&   rd  rq  r   r_   rV   Úseq_relationshipr§   s   &&€r1   r&   Ú#MobileBertPreTrainingHeads.__init__	  s4   ø€ Ü‰ÑÔÜ5°fÓ=ˆÔÜ "§	¢	¨&×*<Ñ*<¸aÓ @ˆÖr3   c                óx   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[S[ P                  ,          /# )r5   rt  rW  r7   r  )r:   r;   s   "€r1   r<   Ú'MobileBertPreTrainingHeads.__annotate__  s7   ø€ ÷ 9ñ 9¡u§|¡|ð 9ÁEÇLÁLð 9ÑUZÑ[`×[gÑ[gÕUhñ 9r3   c                óL   € V P                  V4      pV P                  V4      pW43# r$   ©rq  r~  )r-   rt  rW  rx  Úseq_relationship_scores   &&&  r1   r?   Ú"MobileBertPreTrainingHeads.forward  s-   € Ø ×,Ñ,¨_Ó=ÐØ!%×!6Ñ!6°}Ó!EÐØ Ð8Ð8r3   rƒ  rA   rI   s   @@r1   r{  r{    s   ù‡ € õA÷
9÷ 9ð 9r3   r{  c                   óŠ   a a€ ] tR tRt oRtRtRtRtRtRt	R]
R]/t]P                  ! 4       V 3R l4       tV3R ltRtVtV ;t# )	ÚMobileBertPreTrainedModeli  Ú
mobilebertTrÇ   Ú
attentionsc                ó4  <€ \         SV `  V4       \        V\        4      '       dC   \        P
                  ! VP                  4       \        P                  ! VP                  4       R# \        V\        4      '       d#   \        P
                  ! VP                  4       R# \        V\        4      '       db   \        P                  ! VP                  \        P                  ! VP                  P                  R,          4      P!                  R4      4       R# R# )zInitialize the weightsNrS   rQ   )r%   Ú_init_weightsrÝ   r"   ÚinitÚzeros_r*   Úones_r,   rd  rM   Úcopy_rj   r(   rh   r±   ri   )r-   r‰   r0   s   &&€r1   r‹  Ú'MobileBertPreTrainedModel._init_weights"  s«   ø€ ô 	‰Ñ˜fÔ%Ü�fœf×%Ò%Ü�KŠK˜Ÿ™Ô$Ü�JŠJ�v—}‘}Ö%Ü˜Ô :×;Ò;Ü�KŠK˜Ÿ™Ö$Ü˜Ô 4×5Ò5Ü�JŠJ�v×*Ñ*¬E¯LªL¸×9LÑ9L×9RÑ9RÐSUÕ9VÓ,W×,^Ñ,^Ð_fÓ,gÖhñ 6r3   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # )r5   rk   r   )r:   r;   s   "€r1   r<   Ú&MobileBertPreTrainedModel.__annotate__  s   ø‡ ‚ áÑò r3   © )rB   rC   rD   rE   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_supports_attention_backendr.  r�   Ú_can_record_outputsr(   Úno_gradr‹  Ú__annotate_func__rF   rG   rH   rI   s   @@r1   r‡  r‡    s`   ù‡ € ð %ÐØ&*Ð#ØÐØ€NØÐØ"&Ðà˜ØÐ-ðÐð
 ‡]‚]ƒ_ô	ió ð	i÷ … r3   r‡  z6
    Output type of [`MobileBertForPreTraining`].
    )Úcustom_introc                   óD   a € ] tR tRt o RtRtRtRtRtRt	V 3R lt
RtV tR# )ÚMobileBertForPreTrainingOutputi/  ar  
loss (*optional*, returned when `labels` is provided, `torch.FloatTensor` of shape `(1,)`):
    Total loss as the sum of the masked language modeling loss and the next sequence prediction
    (classification) loss.
prediction_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
    Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
seq_relationship_logits (`torch.FloatTensor` of shape `(batch_size, 2)`):
    Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation
    before SoftMax).
Nc                ó   <€ V ^8„  d   Qh/ S[ P                  R,          ;R&   S[ P                  R,          ;R&   S[ P                  R,          ;R&   S[S[ P                  ,          R,          ;R&   S[S[ P                  ,          R,          ;R&   # )r5   NÚlossÚprediction_logitsÚseq_relationship_logitsrÇ   r‰  )r(   rt   r®   )r:   r;   s   "€r1   r<   Ú+MobileBertForPreTrainingOutput.__annotate__/  s‡   ø‡ ‚ ñ$ ×
Ñ
˜dÕ
"Ñ)ñ% ñ& ×(Ñ(¨4Õ/Ñ6ñ' ñ( #×.Ñ.°Õ5Ñ<ñ) ñ* ™×*Ñ*Õ+¨dÕ2Ñ9ñ+ ñ, ‘e×'Ñ'Õ(¨4Õ/Ñ6ò- r3   r“  )rB   rC   rD   rE   r‡   r¡  r¢  r£  rÇ   r‰  rœ  rF   rG   )r;   s   @r1   rŸ  rŸ  /  s.   ø‡ € ñ	ð &*€DØ26ÐØ8<ÐØ59€MØ26€J÷- ƒ r3   rŸ  c                   óz   a a€ ] tR tRt oRtR	V 3R lltR tR t]]	]
R
V3R lR ll4       4       4       tRtVtV ;t# )ÚMobileBertModeliH  z*
https://huggingface.co/papers/2004.02984
c                óÔ   <€ \         SV `  V4       Wn        RV n        \	        V4      V n        \        V4      V n        V'       d   \        V4      MRV n	        V P                  4        R# )z^
add_pooling_layer (bool, *optional*, defaults to `True`):
    Whether to add a pooling layer
FN)r%   r&   rk   Úgradient_checkpointingrM   r…   r?  ÚencoderrM  ÚpoolerÚ	post_init)r-   rk   Úadd_pooling_layerr0   s   &&&€r1   r&   ÚMobileBertModel.__init__N  sT   ø€ ô
 	‰Ñ˜Ô ØŒØ&+ˆÔ#ä.¨vÓ6ˆŒÜ(¨Ó0ˆŒç2CÔ& vÔ.ÈˆŒð 	�‰Ör3   c                ó.   € V P                   P                  # r$   ©r…   rZ   ©r-   s   &r1   Úget_input_embeddingsÚ$MobileBertModel.get_input_embeddings_  s   € Ø�‰×.Ñ.Ð.r3   c                ó&   € WP                   n        R # r$   r¯  )r-   r{   s   &&r1   Úset_input_embeddingsÚ$MobileBertModel.set_input_embeddingsb  s   € Ø*/�‰Ö'r3   c                ó  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          RS[S[,          /# )	r5   rp   NrŒ   rq   rj   rr   rŽ   r7   )r(   rs   rt   r   r   r®   r   )r:   r;   s   "€r1   r<   ÚMobileBertModel.__annotate__h  s”   ø€ ÷ $
ñ $
á×#Ñ# dÕ*ð$
ñ ×)Ñ)¨DÕ0ð$
ñ ×(Ñ(¨4Õ/ð	$
ñ
 ×&Ñ&¨Õ-ð$
ñ ×(Ñ(¨4Õ/ð$
ñ Ñ+Õ,ð$
ñ 
Ñ+Õ	+ñ$
r3   c                ó0  € VR J VR J,          '       d   \        R4      hV P                  VVVVR7      p\        V P                  VVR7      pV P                  ! V3RV/VB pV^ ,          p	V P
                  e   V P                  V	4      MR p
\        V	V
R7      # )Nz:You must specify exactly one of input_ids or inputs_embeds)rp   rj   rq   rr   )rk   rr   rŒ   rŒ   )rG  Úpooler_output)Ú
ValueErrorr…   r   rk   r©  rª  r   )r-   rp   rŒ   rq   rj   rr   rŽ   Úembedding_outputÚencoder_outputsrt  rW  s   &&&&&&,    r1   r?   ÚMobileBertModel.forwarde  s¼   € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàŸ?™?ØØ%Ø)Ø'ð	 +ó 
Ðô 3Ø—;‘;Ø*Ø)ô
ˆð Ÿ,š,Øñ
à)ð
ð ñ
ˆð
 *¨!Õ,ˆØ8<¿¹Ò8O˜Ÿ™ OÔ4ÐUYˆä)Ø-Ø'ô
ð 	
r3   )rk   r…   r©  r¨  rª  )T)NNNNN)rB   rC   rD   rE   r‡   r&   r±  r´  r   r   r   r?   rF   rG   rH   rI   s   @@r1   r¦  r¦  H  sG   ù‡ € ñ÷ò"/ò0ð  ØØ÷$
ñ $
ó ó ó  ÷$
ð $
r3   r¦  z®
    MobileBert Model with two heads on top as done during the pretraining: a `masked language modeling` head and a
    `next sentence prediction (classification)` head.
    c                   ó�   a a€ ] tR tRt oRRRR/tV 3R ltR tR tRV3R	 lV 3R
 lllt]	]
RV3R lR ll4       4       tRtVtV ;t# )ÚMobileBertForPreTrainingi�  úcls.predictions.decoder.biasúcls.predictions.biasúcls.predictions.decoder.weightú,mobilebert.embeddings.word_embeddings.weightc                ó†   <€ \         SV `  V4       \        V4      V n        \	        V4      V n        V P                  4        R # r$   )r%   r&   r¦  rˆ  r{  Úclsr«  r§   s   &&€r1   r&   Ú!MobileBertForPreTraining.__init__›  s4   ø€ Ü‰Ñ˜Ô Ü)¨&Ó1ˆŒÜ-¨fÓ5ˆŒð 	�‰Ör3   c                óB   € V P                   P                  P                  # r$   ©rÅ  rq  rg  r°  s   &r1   Úget_output_embeddingsÚ.MobileBertForPreTraining.get_output_embeddings£  ó   € Ø�x‰x×#Ñ#×+Ñ+Ð+r3   c                ó„   € WP                   P                  n        VP                  V P                   P                  n        R # r$   ©rÅ  rq  rg  r*   ©r-   Únew_embeddingss   &&r1   Úset_output_embeddingsÚ.MobileBertForPreTraining.set_output_embeddings¦  ó*   € Ø'5�‰×ÑÔ$Ø$2×$7Ñ$7ˆ�‰×ÑÖ!r3   c                óH   <€ V ^8„  d   QhRS[ R,          RS[P                  /# ©r5   Únew_num_tokensNr7   ©r    r   rW   )r:   r;   s   "€r1   r<   Ú%MobileBertForPreTraining.__annotate__ª  s'   ø€ ÷ Nñ N±c¸Dµjð NÉBÏLÉLñ Nr3   c                óº   <€ V P                  V P                  P                  P                  VR R7      V P                  P                  n        \        SV `  VR7      # ©T)rÕ  Ú
transposed)rÕ  ©Ú_get_resized_lm_headrÅ  rq  rÂ   r%   Úresize_token_embeddings©r-   rÕ  r0   s   &&€r1   rÝ  Ú0MobileBertForPreTraining.resize_token_embeddingsª  sR   ø€ à%)×%>Ñ%>Ø�H‰H× Ñ ×&Ñ&°~ÐRVð &?ó &
ˆ�‰×ÑÔ"ô ‰wÑ.¸nÐ.ÓMÐMr3   c                ó^  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[S[,          R
S[S[,          /	# )r5   rp   NrŒ   rq   rj   rr   ÚlabelsÚnext_sentence_labelrŽ   r7   )r(   rs   rt   r   r   r®   rŸ  )r:   r;   s   "€r1   r<   r×  ´  sÉ   ø€ ÷ @
ñ @
á×#Ñ# dÕ*ð@
ñ ×)Ñ)¨DÕ0ð@
ñ ×(Ñ(¨4Õ/ð	@
ñ
 ×&Ñ&¨Õ-ð@
ñ ×(Ñ(¨4Õ/ð@
ñ × Ñ  4Õ'ð@
ñ #×-Ñ-°Õ4ð@
ñ Ñ+Õ,ð@
ñ 
Ñ/Õ	/ñ@
r3   c                óÂ  € V P                   ! V3RVRVRVRVRR/VB p	V	R,          w  r«V P                  W«4      w  rÍRpVe|   Vex   \        4       pV! VP                  R
V P                  P
                  4      VP                  R
4      4      pV! VP                  R
^4      VP                  R
4      4      pVV,           p\        VVVV	P                  V	P                  R	7      # )a  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
    config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
    loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
next_sentence_label (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair
    (see `input_ids` docstring) Indices should be in `[0, 1]`:

    - 0 indicates sequence B is a continuation of sequence A,
    - 1 indicates sequence B is a random sequence.

Examples:

```python
>>> from transformers import AutoTokenizer, MobileBertForPreTraining
>>> import torch

>>> tokenizer = AutoTokenizer.from_pretrained("google/mobilebert-uncased")
>>> model = MobileBertForPreTraining.from_pretrained("google/mobilebert-uncased")

>>> input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0)
>>> # Batch size 1
>>> outputs = model(input_ids)

>>> prediction_logits = outputs.prediction_logits
>>> seq_relationship_logits = outputs.seq_relationship_logits
```rŒ   rq   rj   rr   Úreturn_dictT:Nr5   NN)r¡  r¢  r£  rÇ   r‰  rS   )	rˆ  rÅ  r   r²   rk   rX   rŸ  rÇ   r‰  )r-   rp   rŒ   rq   rj   rr   rá  râ  rŽ   Úoutputsrt  rW  rx  r„  Ú
total_lossÚloss_fctÚmasked_lm_lossÚnext_sentence_losss   &&&&&&&&,         r1   r?   Ú MobileBertForPreTraining.forward²  s  € ðR —/’/Øñ
à)ð
ð *ð
ð &ð	
ð
 (ð
ð ð
ð ñ
ˆð *1°­Ñ&ˆØ48·H±H¸_Ó4\Ñ1Ðàˆ
ØÒÐ"5Ò"AÜ'Ó)ˆHÙ%Ð&7×&<Ñ&<¸RÀÇÁ×AWÑAWÓ&XÐZ`×ZeÑZeÐfhÓZiÓjˆNÙ!)Ð*@×*EÑ*EÀbÈ!Ó*LÐNa×NfÑNfÐgiÓNjÓ!kÐØ'Ð*<Õ<ˆJä-ØØ/Ø$:Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r3   ©rÅ  rˆ  r$   ©NNNNNNN©rB   rC   rD   rE   Ú_tied_weights_keysr&   rÉ  rÐ  rÝ  r   r   r?   rF   rG   rH   rI   s   @@r1   r¿  r¿  �  s`   ù‡ € ð 	'Ð(>Ø(Ð*XðÐõ
ò,ò8÷Nõ Nð Ø÷@
ñ @
ó ó ÷@
ð @
r3   r¿  c                   ó�   a a€ ] tR tRt oRRRR/tV 3R ltR tR tRV3R	 lV 3R
 lllt]	]
RV3R lR ll4       4       tRtVtV ;t# )ÚMobileBertForMaskedLMi÷  rÀ  rÁ  rÂ  rÃ  c                ó–   <€ \         SV `  V4       \        VR R7      V n        \	        V4      V n        Wn        V P                  4        R# ©F)r¬  N)r%   r&   r¦  rˆ  ro  rÅ  rk   r«  r§   s   &&€r1   r&   ÚMobileBertForMaskedLM.__init__þ  s;   ø€ Ü‰Ñ˜Ô Ü)¨&ÀEÔJˆŒÜ(¨Ó0ˆŒØŒð 	�‰Ör3   c                óB   € V P                   P                  P                  # r$   rÈ  r°  s   &r1   rÉ  Ú+MobileBertForMaskedLM.get_output_embeddings  rË  r3   c                ó„   € WP                   P                  n        VP                  V P                   P                  n        R # r$   rÍ  rÎ  s   &&r1   rÐ  Ú+MobileBertForMaskedLM.set_output_embeddings
  rÒ  r3   c                óH   <€ V ^8„  d   QhRS[ R,          RS[P                  /# rÔ  rÖ  )r:   r;   s   "€r1   r<   Ú"MobileBertForMaskedLM.__annotate__  s'   ø€ ÷ Nñ N±c¸Dµjð NÉBÏLÉLñ Nr3   c                óº   <€ V P                  V P                  P                  P                  VR R7      V P                  P                  n        \        SV `  VR7      # rÙ  rÛ  rÞ  s   &&€r1   rÝ  Ú-MobileBertForMaskedLM.resize_token_embeddings  sR   ø€ à%)×%>Ñ%>Ø�H‰H× Ñ ×&Ñ&°~ÐRVð &?ó &
ˆ�‰×ÑÔ"ô ‰wÑ.¸nÐ.ÓMÐMr3   c                ó6  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          R	S[S[,          /# ©
r5   rp   NrŒ   rq   rj   rr   rá  rŽ   r7   )r(   rs   rt   r   r   r®   r   )r:   r;   s   "€r1   r<   rù    s¨   ø€ ÷ '
ñ '
á×#Ñ# dÕ*ð'
ñ ×)Ñ)¨DÕ0ð'
ñ ×(Ñ(¨4Õ/ð	'
ñ
 ×&Ñ&¨Õ-ð'
ñ ×(Ñ(¨4Õ/ð'
ñ × Ñ  4Õ'ð'
ñ Ñ+Õ,ð'
ñ 
‘Õ	ñ'
r3   c                óN  € V P                   ! V3RVRVRVRVRR/VB pV^ ,          p	V P                  V	4      p
RpVeG   \        4       pV! V
P                  R	V P                  P
                  4      VP                  R	4      4      p\        VV
VP                  VP                  R7      # )
az  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
    config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
    loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
rŒ   rq   rj   rr   rä  TN©r¡  ÚlogitsrÇ   r‰  rS   )	rˆ  rÅ  r   r²   rk   rX   r   rÇ   r‰  )r-   rp   rŒ   rq   rj   rr   rá  rŽ   rå  rt  rx  rè  rç  s   &&&&&&&,     r1   r?   ÚMobileBertForMaskedLM.forward  sÒ   € ð$ —/’/Øñ
à)ð
ð *ð
ð &ð	
ð
 (ð
ð ð
ð ñ
ˆð " !�*ˆØ ŸH™H _Ó5ÐàˆØÒÜ'Ó)ˆHÙ%Ð&7×&<Ñ&<¸RÀÇÁ×AWÑAWÓ&XÐZ`×ZeÑZeÐfhÓZiÓjˆNäØØ$Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r3   )rÅ  rk   rˆ  r$   ©NNNNNNrí  rI   s   @@r1   rð  rð  ÷  s\   ù‡ € ð 	'Ð(>Ø(Ð*XðÐõ
ò,ò8÷Nõ Nð Ø÷'
ñ '
ó ó ÷'
ð '
r3   rð  c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚMobileBertOnlyNSPHeadiA  c                óp   <€ \         SV `  4        \        P                  ! VP                  ^4      V n        R# r}  )r%   r&   r   r_   rV   r~  r§   s   &&€r1   r&   ÚMobileBertOnlyNSPHead.__init__B  s'   ø€ Ü‰ÑÔÜ "§	¢	¨&×*<Ñ*<¸aÓ @ˆÖr3   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r5   rW  r7   r8   )r:   r;   s   "€r1   r<   Ú"MobileBertOnlyNSPHead.__annotate__F  s#   ø€ ÷ &ñ &¡U§\¡\ð &±e·l±lñ &r3   c                ó(   € V P                  V4      pV# r$   ©r~  )r-   rW  r„  s   && r1   r?   ÚMobileBertOnlyNSPHead.forwardF  s   € Ø!%×!6Ñ!6°}Ó!EÐØ%Ð%r3   r
  rA   rI   s   @@r1   r  r  A  s   ù‡ € õA÷&÷ &ð &r3   r  zZ
    MobileBert Model with a `next sentence prediction (classification)` head on top.
    c                   ó\   a a€ ] tR tRt oV 3R lt]]RV3R lR ll4       4       tRtVt	V ;t
# )Ú#MobileBertForNextSentencePredictioniK  c                ó†   <€ \         SV `  V4       \        V4      V n        \	        V4      V n        V P                  4        R # r$   )r%   r&   r¦  rˆ  r  rÅ  r«  r§   s   &&€r1   r&   Ú,MobileBertForNextSentencePrediction.__init__Q  s4   ø€ Ü‰Ñ˜Ô ä)¨&Ó1ˆŒÜ(¨Ó0ˆŒð 	�‰Ör3   c                ó6  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          R	S[S[,          /# rý  )r(   rs   rt   r   r   r®   r   )r:   r;   s   "€r1   r<   Ú0MobileBertForNextSentencePrediction.__annotate__\  s©   ø€ ÷ ;
ñ ;
á×#Ñ# dÕ*ð;
ñ ×)Ñ)¨DÕ0ð;
ñ ×(Ñ(¨4Õ/ð	;
ñ
 ×&Ñ&¨Õ-ð;
ñ ×(Ñ(¨4Õ/ð;
ñ × Ñ  4Õ'ð;
ñ Ñ+Õ,ð;
ñ 
Ñ,Õ	,ñ;
r3   c                ó&  € V P                   ! V3RVRVRVRVRR/VB pV^,          p	V P                  V	4      p
RpVe3   \        4       pV! V
P                  R	^4      VP                  R	4      4      p\	        VV
VP
                  VP                  R7      # )
aú  
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair
    (see `input_ids` docstring) Indices should be in `[0, 1]`.

    - 0 indicates sequence B is a continuation of sequence A,
    - 1 indicates sequence B is a random sequence.

Examples:

```python
>>> from transformers import AutoTokenizer, MobileBertForNextSentencePrediction
>>> import torch

>>> tokenizer = AutoTokenizer.from_pretrained("google/mobilebert-uncased")
>>> model = MobileBertForNextSentencePrediction.from_pretrained("google/mobilebert-uncased")

>>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced."
>>> next_sentence = "The sky is blue due to the shorter wavelength of blue light."
>>> encoding = tokenizer(prompt, next_sentence, return_tensors="pt")

>>> outputs = model(**encoding, labels=torch.LongTensor([1]))
>>> loss = outputs.loss
>>> logits = outputs.logits
```rŒ   rq   rj   rr   rä  TNrÿ  rS   )rˆ  rÅ  r   r²   r   rÇ   r‰  )r-   rp   rŒ   rq   rj   rr   rá  rŽ   rå  rW  r„  ré  rç  s   &&&&&&&,     r1   r?   Ú+MobileBertForNextSentencePrediction.forwardZ  sÇ   € ðL —/’/Øñ
à)ð
ð *ð
ð &ð	
ð
 (ð
ð ð
ð ñ
ˆð   �
ˆØ!%§¡¨-Ó!8Ðà!ÐØÒÜ'Ó)ˆHÙ!)Ð*@×*EÑ*EÀbÈ!Ó*LÈfÏkÉkÐZ\ËoÓ!^Ðä*Ø#Ø)Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r3   rë  r  ©rB   rC   rD   rE   r&   r   r   r?   rF   rG   rH   rI   s   @@r1   r  r  K  s0   ù‡ € õð Ø÷;
ñ ;
ó ó ÷;
ð ;
r3   r  z¢
    MobileBert Model transformer with a sequence classification/regression head on top (a linear layer on top of the
    pooled output) e.g. for GLUE tasks.
    c                   ó\   a a€ ] tR tRt oV 3R lt]]RV3R lR ll4       4       tRtVt	V ;t
# )Ú#MobileBertForSequenceClassificationiš  c                óv  <€ \         SV `  V4       VP                  V n        Wn        \	        V4      V n        VP                  e   VP                  MVP                  p\        P                  ! V4      V n
        \        P                  ! VP                  VP                  4      V n        V P                  4        R # r$   )r%   r&   Ú
num_labelsrk   r¦  rˆ  Úclassifier_dropoutre   r   rd   rf   r_   rV   Ú
classifierr«  ©r-   rk   r  r0   s   && €r1   r&   Ú,MobileBertForSequenceClassification.__init__¢  s�   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒØŒä)¨&Ó1ˆŒà)/×)BÑ)BÒ)NˆF×%Ò%ÐTZ×TnÑTnð 	ô —z’zÐ"4Ó5ˆŒÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Ör3   c                óZ  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          R	S[S[ P                  ,          S[,          /# rý  )r(   r9   r   r   r®   r   )r:   r;   s   "€r1   r<   Ú0MobileBertForSequenceClassification.__annotate__³  s¦   ø€ ÷ ;
ñ ;
á—<‘< $Õ&ð;
ñ Ÿ™ tÕ+ð;
ñ Ÿ™ tÕ+ð	;
ñ
 —l‘l TÕ)ð;
ñ —|‘| dÕ*ð;
ñ —‘˜tÕ#ð;
ñ Ñ+Õ,ð;
ñ 
‰u�|‰|Õ	Ñ7Õ	7ñ;
r3   c                ó  € V P                   ! V3RVRVRVRVRR/VB pV^,          p	V P                  V	4      p	V P                  V	4      p
RpVEe˜   V P                  P                  f–   V P
                  ^8X  d   RV P                  n        MrV P
                  ^8”  dQ   VP                  \        P                  8X  g    VP                  \        P                  8X  d   R	V P                  n        MR
V P                  n        V P                  P                  R8X  dK   \        4       pV P
                  ^8X  d'   V! V
P                  4       VP                  4       4      pMŽV! W¦4      pM…V P                  P                  R	8X  d>   \        4       pV! V
P                  RV P
                  4      VP                  R4      4      pM-V P                  P                  R
8X  d   \        4       pV! W¦4      p\        VV
VP                   VP"                  R7      # )ae  
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
    `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
rŒ   rq   rj   rr   rä  TNÚ
regressionÚsingle_label_classificationÚmulti_label_classificationrÿ  rS   )rˆ  rf   r  rk   Úproblem_typer  rx   r(   r   r    r   Úsqueezer   r²   r   r   rÇ   r‰  )r-   rp   rŒ   rq   rj   rr   rá  rŽ   rå  rW  r   r¡  rç  s   &&&&&&&,     r1   r?   Ú+MobileBertForSequenceClassification.forward±  s¿  € ð$ —/’/Øñ
à)ð
ð *ð
ð &ð	
ð
 (ð
ð ð
ð ñ
ˆð   �
ˆàŸ™ ]Ó3ˆØ—‘ Ó/ˆàˆØÓØ�{‰{×'Ñ'Ò/Ø—?‘? aÔ'Ø/;�D—K‘KÕ,Ø—_‘_ qÔ(¨f¯l©l¼e¿j¹jÔ.HÈFÏLÉLÔ\a×\eÑ\eÔLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ô7Ü"›9�Ø—?‘? aÔ'Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# FÓ3‘DØ—‘×)Ñ)Ð-JÔJÜ+Ó-�Ù §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR‘Ø—‘×)Ñ)Ð-IÔIÜ,Ó.�Ù Ó/�ä'ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r3   )r  rk   rf   rˆ  r  r  r  rI   s   @@r1   r  r  š  s0   ù‡ € õð Ø÷;
ñ ;
ó ó ÷;
ð ;
r3   r  c                   ó\   a a€ ] tR tRt oV 3R lt]]RV3R lR ll4       4       tRtVt	V ;t
# )ÚMobileBertForQuestionAnsweringiñ  c                óì   <€ \         SV `  V4       VP                  V n        \        VR R7      V n        \
        P                  ! VP                  VP                  4      V n        V P                  4        R# rò  )
r%   r&   r  r¦  rˆ  r   r_   rV   Ú
qa_outputsr«  r§   s   &&€r1   r&   Ú'MobileBertForQuestionAnswering.__init__ô  sU   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä)¨&ÀEÔJˆŒÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Ör3   c                ó‚  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[S[,          R
S[S[ P                  ,          S[,          /	# )r5   rp   NrŒ   rq   rj   rr   Ústart_positionsÚend_positionsrŽ   r7   )r(   r9   r   r   r®   r   )r:   r;   s   "€r1   r<   Ú+MobileBertForQuestionAnswering.__annotate__   s¹   ø€ ÷ 3
ñ 3
á—<‘< $Õ&ð3
ñ Ÿ™ tÕ+ð3
ñ Ÿ™ tÕ+ð	3
ñ
 —l‘l TÕ)ð3
ñ —|‘| dÕ*ð3
ñ Ÿ™¨Õ,ð3
ñ —|‘| dÕ*ð3
ñ Ñ+Õ,ð3
ñ 
‰u�|‰|Õ	Ñ;Õ	;ñ3
r3   c                óð  € V P                   ! V3R VRVRVRVRR/VB p	V	^ ,          p
V P                  V
4      pVP                  ^R
R7      w  rÍVP                  R
4      P	                  4       pVP                  R
4      P	                  4       pRpVeÄ   VeÀ   \        VP                  4       4      ^8”  d   VP                  R
4      p\        VP                  4       4      ^8”  d   VP                  R
4      pVP                  ^4      pVP                  ^ V4      pVP                  ^ V4      p\        VR7      pV! WÆ4      pV! W×4      pVV,           ^,          p\        VVVV	P                  V	P                  R	7      # )rŒ   rq   rj   rr   rä  Tr|   N)Úignore_index)r¡  Ústart_logitsÚ
end_logitsrÇ   r‰  rS   )rˆ  r)  Úsplitr$  r˜   Úlenr~   Úclampr   r   rÇ   r‰  )r-   rp   rŒ   rq   rj   rr   r,  r-  rŽ   rå  rt  r   r1  r2  ræ  Úignored_indexrç  Ú
start_lossÚend_losss   &&&&&&&&,          r1   r?   Ú&MobileBertForQuestionAnswering.forwardþ  s”  € ð —/’/Øñ
à)ð
ð *ð
ð &ð	
ð
 (ð
ð ð
ð ñ
ˆð " !�*ˆà—‘ Ó1ˆØ#)§<¡<°°r <Ó#:Ñ ˆØ#×+Ñ+¨BÓ/×:Ñ:Ó<ˆØ×'Ñ'¨Ó+×6Ñ6Ó8ˆ
àˆ
ØÒ&¨=Ò+Dä�?×'Ñ'Ó)Ó*¨QÔ.Ø"1×"9Ñ"9¸"Ó"=�Ü�=×%Ñ%Ó'Ó(¨1Ô,Ø -× 5Ñ 5°bÓ 9�à(×-Ñ-¨aÓ0ˆMØ-×3Ñ3°A°}ÓEˆOØ)×/Ñ/°°=ÓAˆMä'°]ÔCˆHÙ! ,Ó@ˆJÙ 
Ó:ˆHØ$ xÕ/°1Õ4ˆJä+ØØ%Ø!Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r3   )rˆ  r  r)  rì  r  rI   s   @@r1   r'  r'  ñ  s0   ù‡ € õð Ø÷3
ñ 3
ó ó ÷3
ð 3
r3   r'  c                   ó\   a a€ ] tR tRt oV 3R lt]]RV3R lR ll4       4       tRtVt	V ;t
# )ÚMobileBertForMultipleChoicei6  c                ó4  <€ \         SV `  V4       \        V4      V n        VP                  e   VP                  MVP
                  p\        P                  ! V4      V n        \        P                  ! VP                  ^4      V n        V P                  4        R # r$   )r%   r&   r¦  rˆ  r  re   r   rd   rf   r_   rV   r  r«  r  s   && €r1   r&   Ú$MobileBertForMultipleChoice.__init__9  su   ø€ Ü‰Ñ˜Ô ä)¨&Ó1ˆŒà)/×)BÑ)BÒ)NˆF×%Ò%ÐTZ×TnÑTnð 	ô —z’zÐ"4Ó5ˆŒÜŸ)š) F×$6Ñ$6¸Ó:ˆŒð 	�‰Ör3   c                óZ  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          R	S[S[ P                  ,          S[,          /# rý  )r(   r9   r   r   r®   r   )r:   r;   s   "€r1   r<   Ú(MobileBertForMultipleChoice.__annotate__H  s°   ø€ ÷ N
ñ N
á—<‘< $Õ&ðN
ñ Ÿ™ tÕ+ðN
ñ Ÿ™ tÕ+ð	N
ñ
 —l‘l TÕ)ðN
ñ —|‘| dÕ*ðN
ñ —‘˜tÕ#ðN
ñ Ñ+Õ,ðN
ñ 
‰u�|‰|Õ	Ñ8Õ	8ñN
r3   c                ó&  € Ve   VP                   ^,          MVP                   ^,          pVe"   VP                  R	VP                  R	4      4      MRpVe"   VP                  R	VP                  R	4      4      MRpVe"   VP                  R	VP                  R	4      4      MRpVe"   VP                  R	VP                  R	4      4      MRpVe2   VP                  R	VP                  R
4      VP                  R	4      4      MRpV P                  ! V3RVRVRVRVRR/VB p	V	^,          p
V P	                  V
4      p
V P                  V
4      pVP                  R	V4      pRpVe   \        4       pV! WÆ4      p\        VVV	P                  V	P                  R7      # )a›  
input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
    Indices of input sequence tokens in the vocabulary.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are input IDs?](../glossary#input-ids)
token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
    Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
    1]`:

    - 0 corresponds to a *sentence A* token,
    - 1 corresponds to a *sentence B* token.

    [What are token type IDs?](../glossary#token-type-ids)
position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
    Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
    config.max_position_embeddings - 1]`.

    [What are position IDs?](../glossary#position-ids)
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):
    Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
    is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
    model's internal embedding lookup matrix.
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
    num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
    `input_ids` above)
NrŒ   rq   rj   rr   rä  Trÿ  rS   éþÿÿÿ)
r±   r²   r~   rˆ  rf   r  r   r   rÇ   r‰  )r-   rp   rŒ   rq   rj   rr   rá  rŽ   Únum_choicesrå  rW  r   Úreshaped_logitsr¡  rç  s   &&&&&&&,       r1   r?   Ú#MobileBertForMultipleChoice.forwardF  s¶  € ðT -6Ò,A�i—o‘o aÖ(À}×GZÑGZÐ[\ÕG]ˆà>GÒ>S�I—N‘N 2 y§~¡~°bÓ'9Ô:ÐY]ˆ	ØM[ÒMg˜×,Ñ,¨R°×1DÑ1DÀRÓ1HÔIÐmqˆØM[ÒMg˜×,Ñ,¨R°×1DÑ1DÀRÓ1HÔIÐmqˆØGSÒG_�|×(Ñ(¨¨\×->Ñ->¸rÓ-BÔCÐeiˆð Ò(ð ×Ñ˜r =×#5Ñ#5°bÓ#9¸=×;MÑ;MÈbÓ;QÔRàð 	ð —/’/Øñ
à)ð
ð *ð
ð &ð	
ð
 (ð
ð ð
ð ñ
ˆð   �
ˆàŸ™ ]Ó3ˆØ—‘ Ó/ˆØ Ÿ+™+ b¨+Ó6ˆàˆØÒÜ'Ó)ˆHÙ˜OÓ4ˆDä(ØØ"Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r3   )r  rf   rˆ  r  r  rI   s   @@r1   r;  r;  6  s4   ù‡ € õð Ø÷N
ñ N
ó ó ÷N
ð N
r3   r;  c                   ó\   a a€ ] tR tRt oV 3R lt]]RV3R lR ll4       4       tRtVt	V ;t
# )Ú MobileBertForTokenClassificationi™  c                ón  <€ \         SV `  V4       VP                  V n        \        VR R7      V n        VP
                  e   VP
                  MVP                  p\        P                  ! V4      V n	        \        P                  ! VP                  VP                  4      V n        V P                  4        R# rò  )r%   r&   r  r¦  rˆ  r  re   r   rd   rf   r_   rV   r  r«  r  s   && €r1   r&   Ú)MobileBertForTokenClassification.__init__œ  sŠ   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä)¨&ÀEÔJˆŒà)/×)BÑ)BÒ)NˆF×%Ò%ÐTZ×TnÑTnð 	ô —z’zÐ"4Ó5ˆŒÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Ör3   c                óZ  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          R	S[S[ P                  ,          S[,          /# rý  )r(   r9   r   r   r®   r   )r:   r;   s   "€r1   r<   Ú-MobileBertForTokenClassification.__annotate__¬  s¦   ø€ ÷ '
ñ '
á—<‘< $Õ&ð'
ñ Ÿ™ tÕ+ð'
ñ Ÿ™ tÕ+ð	'
ñ
 —l‘l TÕ)ð'
ñ —|‘| dÕ*ð'
ñ —‘˜tÕ#ð'
ñ Ñ+Õ,ð'
ñ 
‰u�|‰|Õ	Ñ4Õ	4ñ'
r3   c                ó\  € V P                   ! V3RVRVRVRVRR/VB pV^ ,          p	V P                  V	4      p	V P                  V	4      p
RpVe=   \        4       pV! V
P	                  R	V P
                  4      VP	                  R	4      4      p\        VV
VP                  VP                  R7      # )
zÃ
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
rŒ   rq   rj   rr   rä  TNrÿ  rS   )	rˆ  rf   r  r   r²   r  r   rÇ   r‰  )r-   rp   rŒ   rq   rj   rr   rá  rŽ   rå  rt  r   r¡  rç  s   &&&&&&&,     r1   r?   Ú(MobileBertForTokenClassification.forwardª  sÑ   € ð  —/’/Øñ
à)ð
ð *ð
ð &ð	
ð
 (ð
ð ð
ð ñ
ˆð " !�*ˆàŸ,™, Ó7ˆØ—‘ Ó1ˆàˆØÒÜ'Ó)ˆHÙ˜FŸK™K¨¨D¯O©OÓ<¸f¿k¹kÈ"»oÓNˆDä$ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r3   )r  rf   rˆ  r  r  r  rI   s   @@r1   rF  rF  ™  s0   ù‡ € õð Ø÷'
ñ '
ó ó ÷'
ð '
r3   rF  )Nrz   )
rð  r;  r  r¿  r'  r  rF  r.  r¦  r‡  )SÚcollections.abcr   Údataclassesr   r(   r   Útorch.nnr   r   r   Ú r	   rŒ  Úactivationsr
   Úmasking_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.genericr   r   Úutils.output_capturingr   Úconfiguration_mobilebertr    Ú
get_loggerrB   Úloggerr�   r"   rc   ra   rM   r›   r�   r½   rÏ   rÚ   rë   ró   r   r  r  r#  r.  r?  rM  rZ  rd  ro  r{  r‡  rŸ  r¦  r¿  rð  r  r  r  r'  r;  rF  Ú__all__r“  r3   r1   Ú<module>r^     s-  ðõ. %Ý !ã Ý ß AÑ Aå &Ý !Ý 6Ý 9÷	÷ 	ó 	÷ GÝ &ß MÓ Mß IÝ 5Ý 6ð 
×	Ò	˜HÓ	%€ô6ˆR�Y‰Yô 6ð ˜Ÿ™ y°&Ð
9€ôG˜2Ÿ9™9ô G÷V%ô81)˜bŸi™iô 1)ôh˜2Ÿ9™9ô ô".˜"Ÿ)™)ô .ô8˜RŸY™Yô ô�r—y‘yô ô�r—y‘yô ô0	�b—i‘iô 	ô!]�—‘ô !]ôH	�—	‘	ô 	ô	ˆr�y‰yô 	ô)Ð0ô )ôX@˜Ÿ	™	ô @ô(!�r—y‘yô !ô&¨¯	©	ô ô" §¡ô ô"!˜BŸI™Iô !ô	9 §¡ô 	9ð ôi ó ió ðiñ4 ðôð
 ô7 [ó 7ó óð7ð& ôC
Ð/ó C
ó ðC
ñL ðôô_
Ð8ó _
óð_
ðD ôF
Ð5ó F
ó ðF
ôR&˜BŸI™Iô &ñ ðôô
G
Ð*Có G
óð
G
ñT ðôôM
Ð*Có M
óðM
ð` ô@
Ð%>ó @
ó ð@
ðF ô^
Ð";ó ^
ó ð^
ðB ô8
Ð'@ó 8
ó ð8
òv‚r3   