+
    œ°³jx“  ã                   ó°  € R t ^ RIHt ^ RIHt ^ RIt^ RIHt ^ RIHtH	t	H
t
 ^RIHt ^RIHt ^RIHt ^R	IHtHtHtHtHtHtHt ^R
IHtHt ^RIHt ^RIH t  ^RI!H"t"H#t#H$t$H%t% ^RI&H't'H(t( ^RI)H*t* ^RI+H,t, ]%PZ                  ! ].4      t/ ! R R]P`                  4      t1R7R R llt2 ! R R]P`                  4      t3 ! R R]P`                  4      t4 ! R R]P`                  4      t5 ! R R]P`                  4      t6]$ ! R R]4      4       t7]$! RR 7      ] ! R! R"]"4      4       4       t8]$ ! R# R$]74      4       t9]$! R%R 7       ! R& R']74      4       t: ! R( R)]P`                  4      t; ! R* R+]P`                  4      t<]$ ! R, R-]74      4       t=]$! R.R 7       ! R/ R0]74      4       t>]$ ! R1 R2]74      4       t?]$ ! R3 R4]74      4       t@]$ ! R5 R6]74      4       tA. R8OtBR# )9zPyTorch ALBERT model.)ÚCallable)Ú	dataclassN)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELoss)Úinitialization)ÚACT2FN)Úcreate_bidirectional_mask)ÚBaseModelOutputÚBaseModelOutputWithPoolingÚMaskedLMOutputÚMultipleChoiceModelOutputÚQuestionAnsweringModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)Úapply_chunking_to_forward)ÚModelOutputÚTransformersKwargsÚauto_docstringÚlogging)Úcan_return_tupleÚmerge_with_config_defaults)Úcapture_outputs©ÚAlbertConfigc                   óX   a a€ ] tR t^1t oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )ÚAlbertEmbeddingszI
Construct the embeddings from word, position and token_type embeddings.
c                ó    <€ V ^8„  d   QhRS[ /# ©é   Úconfigr   )ÚformatÚ__classdict__s   "€Úk/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/albert/modeling_albert.pyÚ__annotate__ÚAlbertEmbeddings.__annotate__6   s   ø€ ÷ sñ s™|ñ só    c                óN  <€ \         SV `  4        \        P                  ! VP                  VP
                  VP                  R 7      V n        \        P                  ! VP                  VP
                  4      V n	        \        P                  ! VP                  VP
                  4      V n        \        P                  ! VP
                  VP                  R7      V n        \        P                  ! VP                  4      V n        \        P"                  ! \$        P&                  ! VP                  4      P)                  R4      RR7      V n        \        P"                  ! \$        P,                  ! V P*                  P/                  4       \$        P0                  R7      RR7      V n        R# ))Úpadding_idx©ÚepsF)Ú
persistent)ÚdtypeN©é   éÿÿÿÿ)ÚsuperÚ__init__r   Ú	EmbeddingÚ
vocab_sizeÚembedding_sizeÚpad_token_idÚword_embeddingsÚmax_position_embeddingsÚposition_embeddingsÚtype_vocab_sizeÚtoken_type_embeddingsÚ	LayerNormÚlayer_norm_epsÚDropoutÚhidden_dropout_probÚdropoutÚBufferÚtorchÚarangeÚexpandÚposition_idsÚzerosÚsizeÚlongÚtoken_type_ids©Úselfr$   Ú	__class__s   &&€r'   r5   ÚAlbertEmbeddings.__init__6   s  ø€ Ü‰ÑÔÜ!Ÿ|š|¨F×,=Ñ,=¸v×?TÑ?TÐbh×buÑbuÔvˆÔÜ#%§<¢<°×0NÑ0NÐPV×PeÑPeÓ#fˆÔ Ü%'§\¢\°&×2HÑ2HÈ&×J_ÑJ_Ó%`ˆÔ"äŸš f×&;Ñ&;À×AVÑAVÔWˆŒÜ—z’z &×"<Ñ"<Ó=ˆŒô ŸIšI¤e§l¢l°6×3QÑ3QÓ&R×&YÑ&YÐZaÓ&bÐotÔuˆÔÜ Ÿiši¬¯ª°D×4EÑ4E×4JÑ4JÓ4LÔTY×T^ÑT^Ô(_ÐlqÔrˆÖr*   c          
      óÔ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  /# )r#   Ú	input_idsNrL   rH   Úinputs_embedsÚreturn)rE   Ú
LongTensorÚFloatTensorÚTensor)r%   r&   s   "€r'   r(   r)   C   sm   ø€ ÷ 'ñ 'á×#Ñ# dÕ*ð'ñ ×(Ñ(¨4Õ/ð'ñ ×&Ñ&¨Õ-ð	'ñ
 ×(Ñ(¨4Õ/ð'ñ 
�‰ñ'r*   c                ó¸  € Ve   VP                  4       pMVP                  4       R R pVw  rgVf   V P                  RR V13,          pVf¦   \        V R4      '       dY   V P                  P	                  VP
                  ^ ,          R4      p\        P                  ! V^VR7      pVP	                  Wg4      pM;\        P                  ! V\        P                  V P                  P                  R7      pVf   V P                  V4      pV P                  V4      p	WI,           p
V P                  V4      pW«,           p
V P                  V
4      p
V P                  V
4      p
V
# )NºNNNrL   )ÚdimÚindex)r0   Údevicer3   )rJ   rH   ÚhasattrrL   rG   ÚshaperE   ÚgatherrI   rK   r\   r:   r>   r<   r?   rC   )rN   rR   rL   rH   rS   Úinput_shapeÚ
batch_sizeÚ
seq_lengthÚbuffered_token_type_idsr>   Ú
embeddingsr<   s   &&&&&       r'   ÚforwardÚAlbertEmbeddings.forwardC   s@  € ð Ò Ø#Ÿ.™.Ó*‰Kà'×,Ñ,Ó.¨s°Ð3ˆKà!,Ñˆ
àÒØ×,Ñ,¨Q°°°¨^Õ<ˆLð
 Ò!Ü�tÐ-×.Ò.à*.×*=Ñ*=×*DÑ*DÀ\×EWÑEWÐXYÕEZÐ\^Ó*_Ð'Ü*/¯,ª,Ð7NÐTUÐ]iÔ*jÐ'Ø!8×!?Ñ!?À
Ó!W‘ä!&§¢¨[ÄÇ
Á
ÐSW×SdÑSd×SkÑSkÔ!l�àÒ Ø ×0Ñ0°Ó;ˆMØ $× :Ñ :¸>Ó JÐØ"Õ:ˆ
à"×6Ñ6°|ÓDÐØÕ5ˆ
à—^‘^ JÓ/ˆ
Ø—\‘\ *Ó-ˆ
ØÐr*   )r?   rC   r<   rH   r>   rL   r:   )NNNN)
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r5   re   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©rO   r&   s   @@r'   r    r    1   s%   ù‡ € ñ÷só s÷'÷ 'ò 'r*   r    c                ó  € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R,          R\        R	\
        \        ,          /# )
r#   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNÚscalingrC   Úkwargs)r   ÚModulerE   rW   Úfloatr   r   )r%   s   "r'   r(   r(   n   s{   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô �T�\ð%ô ð%ô Ô'Õ(ñ%r*   c                 ó¶  € Vf   VP                  R4      R,          p\        P                  ! WP                  ^^4      4      V,          pVe	   W„,           p\        P
                  P                  VRR7      p\        P
                  P                  W†V P                  R7      p\        P                  ! Wƒ4      p	V	P                  ^^4      P                  4       p	W˜3# )N©rZ   )ÚpÚtrainingr3   ç      à¿)
rJ   rE   ÚmatmulÚ	transposer   Ú
functionalÚsoftmaxrC   r}   Ú
contiguous)
rq   rr   rs   rt   ru   rv   rC   rw   Úattn_weightsÚattn_outputs
   &&&&&&&,  r'   Úeager_attention_forwardr†   n   s±   € ð ‚Ø—*‘*˜R“. DÕ(ˆô —<’< §}¡}°Q¸Ó':Ó;¸gÕE€LàÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2Ð(Ó>€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€Lä—,’,˜|Ó3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r*   c                   óT   a a€ ] tR t^Št oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚAlbertAttentionc                ó    <€ V ^8„  d   QhRS[ /# r"   r   )r%   r&   s   "€r'   r(   ÚAlbertAttention.__annotate__‹   s   ø€ ÷ ñ ™|ñ r*   c                ób  <€ \         SV `  4        VP                  VP                  ,          ^ 8w  d8   \	        VR4      '       g&   \        RVP                   RVP                   24      hWn        VP                  V n        VP                  V n        VP                  VP                  ,          V n        V P                  V P                  ,          V n        V P                  R,          V n	        \        P                  ! VP                  4      V n        \        P                  ! VP                  4      V n        \        P                   ! VP                  V P                  4      V n        \        P                   ! VP                  V P                  4      V n        \        P                   ! VP                  V P                  4      V n        \        P                   ! VP                  VP                  4      V n        \        P*                  ! VP                  VP,                  R7      V n        RV n        R# )é    r8   zThe hidden size (z6) is not a multiple of the number of attention heads (r-   FNr~   )r4   r5   Úhidden_sizeÚnum_attention_headsr]   Ú
ValueErrorr$   Úattention_head_sizeÚall_head_sizerv   r   rA   Úattention_probs_dropout_probÚattention_dropoutrB   Úoutput_dropoutÚLinearrr   rs   rt   Údenser?   r@   Ú	is_causalrM   s   &&€r'   r5   ÚAlbertAttention.__init__‹   s˜  ø€ Ü‰ÑÔØ×Ñ × :Ñ :Õ:¸aÔ?ÌÐPVÐXh×HiÒHiÜØ# F×$6Ñ$6Ð#7ð 8Ø ×4Ñ4Ð5ð7óð ð Œà#)×#=Ñ#=ˆÔ Ø!×-Ñ-ˆÔØ#)×#5Ñ#5¸×9SÑ9SÕ#SˆÔ Ø!×5Ñ5¸×8PÑ8PÕPˆÔØ×/Ñ/°Õ5ˆŒä!#§¢¨F×,OÑ,OÓ!PˆÔÜ Ÿjšj¨×)CÑ)CÓDˆÔä—Y’Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
Ü—9’9˜V×/Ñ/°×1CÑ1CÓDˆŒÜ—Y’Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
Ü—Y’Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
äŸš f×&8Ñ&8¸f×>SÑ>SÔTˆŒàˆŽr*   c          
      ó¶   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[S[,          RS[S[ P                  S[ P                  3,          /# ©r#   Úhidden_statesru   Nrw   rT   )rE   rW   rV   r   r   Útuple)r%   r&   s   "€r'   r(   rŠ   ¦   sZ   ø€ ÷ ")ñ ")á—|‘|ð")ñ ×)Ñ)¨DÕ0ð")ñ Ñ+Õ,ð	")ñ
 
‰u�|‰|™UŸ\™\Ð)Õ	*ñ")r*   c                ó  € VP                   R R p. VORNV P                  N5pV P                  V4      P                  ! V!  P	                  ^^4      pV P                  V4      P                  ! V!  P	                  ^^4      pV P                  V4      P                  ! V!  P	                  ^^4      p\        P                  ! V P                  P                  \        4      p	T	! V VVVV3RV P                  '       g   RMV P                  P                  RV P                  /VB w  r«V
P                   ! . VORN5!  P#                  4       p
V P%                  V
4      p
V P'                  V
4      p
V P)                  W,           4      p
W«3# )NrC   ç        rv   r3   )r^   r�   rr   Úviewr€   rs   rt   r   Úget_interfacer$   Ú_attn_implementationr†   r}   r“   r|   rv   Úreshaperƒ   r–   r”   r?   )rN   r›   ru   rw   r`   Úhidden_shapeÚquery_layerÚ	key_layerÚvalue_layerÚattention_interfacer…   r„   s   &&&,        r'   re   ÚAlbertAttention.forward¦   sx  € ð $×)Ñ)¨#¨2Ð.ˆØC˜ÐC bÐC¨$×*BÑ*BÑCˆð —j‘j Ó/×4Ò4°lÑC×MÑMÈaÐQRÓSˆØ—H‘H˜]Ó+×0Ò0°,Ñ?×IÑIÈ!ÈQÓOˆ	Ø—j‘j Ó/×4Ò4°lÑC×MÑMÈaÐQRÓSˆä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2H×2JÑ2Jð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆà—j‘j Ó-ˆØ×)Ñ)¨+Ó6ˆØ—n‘n ]Õ%@ÓAˆàÐ(Ð(r*   )r?   r‘   r“   r�   r$   r–   r�   r—   rs   rŽ   r”   rr   rv   rt   ©N©	rg   rh   ri   rj   r5   re   rl   rm   rn   ro   s   @@r'   rˆ   rˆ   Š   s   ù‡ € ÷ó ÷6")÷ ")ò ")r*   rˆ   c                   óf   a a€ ] tR t^Ët oV3R lV 3R lltRV3R lR lltV3R lR ltRtVtV ;t	# )	ÚAlbertLayerc                ó    <€ V ^8„  d   QhRS[ /# r"   r   )r%   r&   s   "€r'   r(   ÚAlbertLayer.__annotate__Ì   s   ø€ ÷ >ñ >™|ñ >r*   c                ó$  <€ \         SV `  4        Wn        VP                  V n        ^V n        \
        P                  ! VP                  VP                  R7      V n	        \        V4      V n        \
        P                  ! VP                  VP                  4      V n        \
        P                  ! VP                  VP                  4      V n        \         VP"                  ,          V n        \
        P&                  ! VP(                  4      V n        R# )r2   r-   N)r4   r5   r$   Úchunk_size_feed_forwardÚseq_len_dimr   r?   r�   r@   Úfull_layer_layer_normrˆ   Ú	attentionr•   Úintermediate_sizeÚffnÚ
ffn_outputr	   Ú
hidden_actÚ
activationrA   rB   rC   rM   s   &&€r'   r5   ÚAlbertLayer.__init__Ì   s¸   ø€ Ü‰ÑÔàŒØ'-×'EÑ'EˆÔ$ØˆÔÜ%'§\¢\°&×2DÑ2DÈ&×J_ÑJ_Ô%`ˆÔ"Ü(¨Ó0ˆŒÜ—9’9˜V×/Ñ/°×1IÑ1IÓJˆŒÜŸ)š) F×$<Ñ$<¸f×>PÑ>PÓQˆŒÜ  ×!2Ñ!2Õ3ˆŒÜ—z’z &×"<Ñ"<Ó=ˆŽr*   c                óŒ   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[S[,          RS[ P                  /# rš   ©rE   rW   rV   r   r   )r%   r&   s   "€r'   r(   r®   Ù   sL   ø€ ÷ ñ á—|‘|ðñ ×)Ñ)¨DÕ0ðñ Ñ+Õ,ð	ñ
 
�‰ñr*   c                ó¶   € V P                   ! W3/ VB w  rE\        V P                  V P                  V P                  V4      pV P                  Wd,           4      pV# r©   )r³   r   Úff_chunkr°   r±   r²   )rN   r›   ru   rw   Úattention_outputÚ_r¶   s   &&&,   r'   re   ÚAlbertLayer.forwardÙ   s\   € ð #Ÿnšn¨]ÑUÈfÑUÑÐä.Ø�M‰MØ×(Ñ(Ø×ÑØó	
ˆ
ð ×2Ñ2°:Õ3PÓQˆØÐr*   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r#   r¾   rT   ©rE   rW   )r%   r&   s   "€r'   r(   r®   ê   s#   ø€ ÷ ñ ©¯©ð ¹%¿,¹,ñ r*   c                ól   € V P                  V4      pV P                  V4      pV P                  V4      pV# r©   )rµ   r¸   r¶   )rN   r¾   r¶   s   && r'   r½   ÚAlbertLayer.ff_chunkê   s3   € Ø—X‘XÐ.Ó/ˆ
Ø—_‘_ ZÓ0ˆ
Ø—_‘_ ZÓ0ˆ
ØÐr*   )	r¸   r³   r°   r$   rC   rµ   r¶   r²   r±   r©   )
rg   rh   ri   rj   r5   re   r½   rl   rm   rn   ro   s   @@r'   r¬   r¬   Ë   s(   ù‡ € ÷>ó >÷ò ÷"÷ ð r*   r¬   c                   óT   a a€ ] tR t^ñt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚAlbertLayerGroupc                ó    <€ V ^8„  d   QhRS[ /# r"   r   )r%   r&   s   "€r'   r(   ÚAlbertLayerGroup.__annotate__ò   s   ø€ ÷ iñ i™|ñ ir*   c                ó¼   <€ \         SV `  4        \        P                  ! \	        VP
                  4       Uu. uF  p\        V4      NK  	  up4      V n        R # u upi r©   )r4   r5   r   Ú
ModuleListÚrangeÚinner_group_numr¬   Úalbert_layers©rN   r$   r¿   rO   s   && €r'   r5   ÚAlbertLayerGroup.__init__ò   sC   ø€ Ü‰ÑÔäŸ]š]ÌÈv×OeÑOeÔIfÓ+gÑIfÀA¬K¸Ö,?ÑIfÑ+gÓhˆÖùÒ+gs   ·Ac                óŒ   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[S[,          RS[ P                  /# rš   r»   )r%   r&   s   "€r'   r(   rÈ   ÷   sL   ø€ ÷ ñ á—|‘|ðñ ×)Ñ)¨DÕ0ðñ Ñ+Õ,ð	ñ
 
�‰ñr*   c                óT   € \        V P                  4       F  w  rEV! W3/ VB pK  	  V# r©   )Ú	enumeraterÍ   )rN   r›   ru   rw   Úlayer_indexÚalbert_layers   &&&,  r'   re   ÚAlbertLayerGroup.forward÷   s2   € ô *3°4×3EÑ3EÖ)FÑ%ˆKÙ(¨ÑQÈ&ÑQŠMñ *GàÐr*   )rÍ   r©   rª   ro   s   @@r'   rÆ   rÆ   ñ   s    ù‡ € ÷ió i÷
÷ ò r*   rÆ   c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚAlbertTransformeri  c                ó    <€ V ^8„  d   QhRS[ /# r"   r   )r%   r&   s   "€r'   r(   ÚAlbertTransformer.__annotate__  s   ø€ ÷ vñ v™|ñ vr*   c                ó(  <€ \         SV `  4        Wn        \        P                  ! VP
                  VP                  4      V n        \        P                  ! \        VP                  4       Uu. uF  p\        V4      NK  	  up4      V n        R # u upi r©   )r4   r5   r$   r   r•   r8   r�   Úembedding_hidden_mapping_inrÊ   rË   Únum_hidden_groupsrÆ   Úalbert_layer_groupsrÎ   s   && €r'   r5   ÚAlbertTransformer.__init__  sj   ø€ Ü‰ÑÔàŒÜ+-¯9ª9°V×5JÑ5JÈF×L^ÑL^Ó+_ˆÔ(Ü#%§=¢=ÔTYÐZ`×ZrÑZrÔTsÓ1tÑTsÈqÔ2BÀ6Ö2JÑTsÑ1tÓ#uˆÖ ùÒ1ts   Á-Bc          	      óˆ   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[S[,          RS[S[,          /# rš   )rE   rW   rV   r   r   r   rœ   )r%   r&   s   "€r'   r(   rÙ   
  sS   ø€ ÷ @ñ @á—|‘|ð@ñ ×)Ñ)¨DÕ0ð@ñ Ñ+Õ,ð	@ñ
 
™5Õ	 ñ@r*   c                ó>  € V P                  V4      p\        V P                  P                  4       F^  p\	        W@P                  P                  V P                  P
                  ,          ,          4      pV P                  V,          ! VV3/ VB pK`  	  \        VR 7      # ))Úlast_hidden_state)rÛ   rË   r$   Únum_hidden_layersÚintrÜ   rÝ   r   )rN   r›   ru   rw   ÚiÚ	group_idxs   &&&,  r'   re   ÚAlbertTransformer.forward
  s…   € ð ×8Ñ8¸ÓGˆä�t—{‘{×4Ñ4Ö5ˆAä˜A§¡×!>Ñ!>ÀÇÁ×A^ÑA^Õ!^Õ_Ó`ˆIà ×4Ñ4°YÖ?ØØñð ñŠMñ	 6ô °Ô?Ð?r*   )rÝ   r$   rÛ   r©   rª   ro   s   @@r'   r×   r×     s#   ù‡ € ÷vó v÷@÷ @ò @r*   r×   c                   ó~   a a€ ] tR tRt o]tRtRtRtRt	Rt
R]R]/t]P                  ! 4       V 3R l4       tRtVtV ;t# )ÚAlbertPreTrainedModeli  ÚalbertTr›   Ú
attentionsc                óÄ  <€ \         SV `  V4       \        V\        4      '       d#   \        P
                  ! VP                  4       R# \        V\        4      '       d‚   \        P                  ! VP                  \        P                  ! VP                  P                  R,          4      P                  R4      4       \        P
                  ! VP                  4       R# R# )zInitialize the weights.Nr3   r1   )r4   Ú_init_weightsÚ
isinstanceÚAlbertMLMHeadÚinitÚzeros_Úbiasr    Úcopy_rH   rE   rF   r^   rG   rL   )rN   rq   rO   s   &&€r'   rì   Ú#AlbertPreTrainedModel._init_weights,  s�   ø€ ô 	‰Ñ˜fÔ%Ü�fœm×,Ò,Ü�KŠK˜Ÿ™Ö$Ü˜Ô 0×1Ò1Ü�JŠJ�v×*Ñ*¬E¯LªL¸×9LÑ9L×9RÑ9RÐSUÕ9VÓ,W×,^Ñ,^Ð_fÓ,gÔhÜ�KŠK˜×-Ñ-Ö.ñ 2r*   © )rg   rh   ri   rj   r   Úconfig_classÚbase_model_prefixÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_supports_attention_backendr¬   rˆ   Ú_can_record_outputsrE   Úno_gradrì   rl   rm   rn   ro   s   @@r'   rè   rè     sX   ù‡ € à€LØ ÐØÐØ€NØÐØ"&Ðà˜Ø�oðÐð
 ‡]‚]ƒ_ô/ó ÷/ð /r*   rè   z2
    Output type of [`AlbertForPreTraining`].
    )Úcustom_introc                   óD   a € ] tR tRt o RtRtRtRtRtRt	V 3R lt
RtV tR# )ÚAlbertForPreTrainingOutputi7  ae  
loss (*optional*, returned when `labels` is provided, `torch.FloatTensor` of shape `(1,)`):
    Total loss as the sum of the masked language modeling loss and the next sequence prediction
    (classification) loss.
prediction_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
    Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
sop_logits (`torch.FloatTensor` of shape `(batch_size, 2)`):
    Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation
    before SoftMax).
Nc                ó   <€ V ^8„  d   Qh/ S[ P                  R,          ;R&   S[ P                  R,          ;R&   S[ P                  R,          ;R&   S[S[ P                  ,          R,          ;R&   S[S[ P                  ,          R,          ;R&   # )r#   NÚlossÚprediction_logitsÚ
sop_logitsr›   rê   )rE   rV   rœ   )r%   r&   s   "€r'   r(   Ú'AlbertForPreTrainingOutput.__annotate__7  s‡   ø‡ ‚ ñ$ ×
Ñ
˜dÕ
"Ñ)ñ% ñ& ×(Ñ(¨4Õ/Ñ6ñ' ñ( ×!Ñ! DÕ(Ñ/ñ) ñ* ™×*Ñ*Õ+¨dÕ2Ñ9ñ+ ñ, ‘e×'Ñ'Õ(¨4Õ/Ñ6ò- r*   rô   )rg   rh   ri   rj   rk   r  r  r  r›   rê   Ú__annotate_func__rl   rm   )r&   s   @r'   rÿ   rÿ   7  s-   ø‡ € ñ	ð &*€DØ26ÐØ+/€JØ59€MØ26€J÷- ƒ r*   rÿ   c                   ó¢   a a€ ] tR tRt o]tRtRV3R lV 3R llltV3R lR ltV3R lR lt	]
]]RV3R	 lR
 ll4       4       4       tRtVtV ;t# )ÚAlbertModeliP  ré   c                ó&   <€ V ^8„  d   QhRS[ RS[/# )r#   r$   Úadd_pooling_layer)r   Úbool)r%   r&   s   "€r'   r(   ÚAlbertModel.__annotate__U  s   ø€ ÷ ñ ™|ð Áñ r*   c                óv  <€ \         SV `  V4       Wn        \        V4      V n        \        V4      V n        V'       dL   \        P                  ! VP                  VP                  4      V n
        \        P                  ! 4       V n        MRV n
        RV n        VP                  V n        V P                  4        R# )z^
add_pooling_layer (bool, *optional*, defaults to `True`):
    Whether to add a pooling layer
N)r4   r5   r$   r    rd   r×   Úencoderr   r•   r�   ÚpoolerÚTanhÚpooler_activationr¡   Úattn_implementationÚ	post_init)rN   r$   r	  rO   s   &&&€r'   r5   ÚAlbertModel.__init__U  s‡   ø€ ô
 	‰Ñ˜Ô àŒÜ*¨6Ó2ˆŒÜ(¨Ó0ˆŒßÜŸ)š) F×$6Ñ$6¸×8JÑ8JÓKˆDŒKÜ%'§W¢W£YˆDÕ"àˆDŒKØ%)ˆDÔ"à#)×#>Ñ#>ˆÔ ð 	�‰Ör*   c                ó4   <€ V ^8„  d   QhRS[ P                  /# ©r#   rT   ©r   r6   )r%   r&   s   "€r'   r(   r  k  s   ø€ ÷ /ñ /¡b§l¡lñ /r*   c                ó.   € V P                   P                  # r©   ©rd   r:   ©rN   s   &r'   Úget_input_embeddingsÚ AlbertModel.get_input_embeddingsk  s   € Ø�‰×.Ñ.Ð.r*   c                ó8   <€ V ^8„  d   QhRS[ P                  RR/# )r#   rt   rT   Nr  )r%   r&   s   "€r'   r(   r  n  s   ø€ ÷ 0ñ 0©"¯,©,ð 0¸4ñ 0r*   c                ó&   € WP                   n        R # r©   r  )rN   rt   s   &&r'   Úset_input_embeddingsÚ AlbertModel.set_input_embeddingsn  s   € Ø*/�‰Ö'r*   c                ó  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          RS[S[,          /# )	r#   rR   Nru   rL   rH   rS   rw   rT   )rE   rU   rV   r   r   r   rœ   )r%   r&   s   "€r'   r(   r  t  s“   ø€ ÷ $
ñ $
á×#Ñ# dÕ*ð$
ñ ×)Ñ)¨DÕ0ð$
ñ ×(Ñ(¨4Õ/ð	$
ñ
 ×&Ñ&¨Õ-ð$
ñ ×(Ñ(¨4Õ/ð$
ñ Ñ+Õ,ð$
ñ 
$¡eÕ	+ñ$
r*   c                óZ  € VR J VR J,          '       d   \        R4      hV P                  WW5R7      p\        V P                  VVR7      pV P                  ! VV3RV/VB pV^ ,          p	V P
                  e(   V P                  V P                  V	R,          4      4      MR p
\        V	V
R7      # )Nz:You must specify exactly one of input_ids or inputs_embeds)rH   rL   rS   )r$   rS   ru   rH   )rá   Úpooler_output)rY   rŒ   )r�   rd   r
   r$   r  r  r  r   )rN   rR   ru   rL   rH   rS   rw   Úembedding_outputÚencoder_outputsÚsequence_outputÚpooled_outputs   &&&&&&,    r'   re   ÚAlbertModel.forwardq  sÌ   € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàŸ?™?ØÀð +ó 
Ðô 3Ø—;‘;Ø*Ø)ô
ˆð Ÿ,š,ØØñ
ð &ð
ð ñ	
ˆð *¨!Õ,ˆàVZ×VaÑVaÒVm˜×.Ñ.¨t¯{©{¸?È4Õ;PÓ/QÔRÐswˆä)Ø-Ø'ô
ð 	
r*   )r  r$   rd   r  r  r  )T)NNNNN)rg   rh   ri   rj   r   rõ   rö   r5   r  r  r   r   r   re   rl   rm   rn   ro   s   @@r'   r  r  P  s\   ù‡ € à€LØ Ð÷õ ÷,/ð /÷0ð 0ð  ØØ÷$
ñ $
ó ó ó  ÷$
ð $
r*   r  z«
    Albert Model with two heads on top as done during the pretraining: a `masked language modeling` head and a
    `sentence order prediction (classification)` head.
    c                   óª   a a€ ] tR tRt oRRRR/tV3R lV 3R lltV3R lR	 ltV3R
 lR ltV3R lR lt]	]
RV3R lR ll4       4       tRtVtV ;t# )ÚAlbertForPreTrainingi›  úpredictions.decoder.weightú(albert.embeddings.word_embeddings.weightúpredictions.decoder.biasúpredictions.biasc                ó    <€ V ^8„  d   QhRS[ /# r"   r   )r%   r&   s   "€r'   r(   Ú!AlbertForPreTraining.__annotate__§  ó   ø€ ÷ ñ ™|ñ r*   c                ó¦   <€ \         SV `  V4       \        V4      V n        \	        V4      V n        \        V4      V n        V P                  4        R # r©   )	r4   r5   r  ré   rî   ÚpredictionsÚAlbertSOPHeadÚsop_classifierr  rM   s   &&€r'   r5   ÚAlbertForPreTraining.__init__§  sB   ø€ Ü‰Ñ˜Ô ä! &Ó)ˆŒÜ(¨Ó0ˆÔÜ+¨FÓ3ˆÔð 	�‰Ör*   c                ó4   <€ V ^8„  d   QhRS[ P                  /# r  ©r   r•   )r%   r&   s   "€r'   r(   r/  ±  ó   ø€ ÷ (ñ (¡r§y¡yñ (r*   c                ó.   € V P                   P                  # r©   ©r2  Údecoderr  s   &r'   Úget_output_embeddingsÚ*AlbertForPreTraining.get_output_embeddings±  ó   € Ø×Ñ×'Ñ'Ð'r*   c                ó8   <€ V ^8„  d   QhRS[ P                  RR/# ©r#   Únew_embeddingsrT   Nr7  )r%   r&   s   "€r'   r(   r/  ´  s   ø€ ÷ 2ñ 2±B·I±Ið 2À$ñ 2r*   c                ó&   € WP                   n        R # r©   r:  ©rN   rA  s   &&r'   Úset_output_embeddingsÚ*AlbertForPreTraining.set_output_embeddings´  s   € Ø#1×ÑÖ r*   c                ó4   <€ V ^8„  d   QhRS[ P                  /# r  r  )r%   r&   s   "€r'   r(   r/  ·  ó   ø€ ÷ 6ñ 6¡b§l¡lñ 6r*   c                óB   € V P                   P                  P                  # r©   ©ré   rd   r:   r  s   &r'   r  Ú)AlbertForPreTraining.get_input_embeddings·  ó   € Ø�{‰{×%Ñ%×5Ñ5Ð5r*   c                ó^  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[S[,          R
S[S[,          /	# )r#   rR   Nru   rL   rH   rS   ÚlabelsÚsentence_order_labelrw   rT   ©rE   rU   rV   r   r   rÿ   rœ   )r%   r&   s   "€r'   r(   r/  ¼  sÈ   ø€ ÷ A
ñ A
á×#Ñ# dÕ*ðA
ñ ×)Ñ)¨DÕ0ðA
ñ ×(Ñ(¨4Õ/ð	A
ñ
 ×&Ñ&¨Õ-ðA
ñ ×(Ñ(¨4Õ/ðA
ñ × Ñ  4Õ'ðA
ñ $×.Ñ.°Õ5ðA
ñ Ñ+Õ,ðA
ñ 
$¡eÕ	+ñA
r*   c                óà  € V P                   ! V3RVRVRVRVRR/VB p	V	R,          w  r«V P                  V
4      pV P                  V4      pRpVe|   Vex   \        4       pV! VP	                  R
V P
                  P                  4      VP	                  R
4      4      pV! VP	                  R
^4      VP	                  R
4      4      pVV,           p\        VVVV	P                  V	P                  R	7      # )aæ  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
    config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
    loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
sentence_order_label (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair
    (see `input_ids` docstring) Indices should be in `[0, 1]`. `0` indicates original order (sequence A, then
    sequence B), `1` indicates switched order (sequence B, then sequence A).

Example:

```python
>>> from transformers import AutoTokenizer, AlbertForPreTraining
>>> import torch

>>> tokenizer = AutoTokenizer.from_pretrained("albert/albert-base-v2")
>>> model = AlbertForPreTraining.from_pretrained("albert/albert-base-v2")

>>> input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0)
>>> # Batch size 1
>>> outputs = model(input_ids)

>>> prediction_logits = outputs.prediction_logits
>>> sop_logits = outputs.sop_logits
```ru   rL   rH   rS   Úreturn_dictT:Nr#   NN)r  r  r  r›   rê   r3   )
ré   r2  r4  r   rŸ   r$   r7   rÿ   r›   rê   )rN   rR   ru   rL   rH   rS   rM  rN  rw   Úoutputsr%  r&  Úprediction_scoresÚ
sop_scoresÚ
total_lossÚloss_fctÚmasked_lm_lossÚsentence_order_losss   &&&&&&&&,         r'   re   ÚAlbertForPreTraining.forwardº  s!  € ðN —+’+Øñ
à)ð
ð *ð
ð &ð	
ð
 (ð
ð ð
ð ñ
ˆð *1°­Ñ&ˆà ×,Ñ,¨_Ó=ÐØ×(Ñ(¨Ó7ˆ
àˆ
ØÒÐ"6Ò"BÜ'Ó)ˆHÙ%Ð&7×&<Ñ&<¸RÀÇÁ×AWÑAWÓ&XÐZ`×ZeÑZeÐfhÓZiÓjˆNÙ"*¨:¯?©?¸2¸qÓ+AÐCW×C\ÑC\Ð]_ÓC`Ó"aÐØ'Ð*=Õ=ˆJä)ØØ/Ø!Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r*   )ré   r2  r4  ©NNNNNNN©rg   rh   ri   rj   Ú_tied_weights_keysr5   r<  rD  r  r   r   re   rl   rm   rn   ro   s   @@r'   r)  r)  ›  sm   ù‡ € ð 	%Ð&PØ"Ð$6ðÐ÷
ó ÷(ð (÷2ð 2÷6ð 6ð Ø÷A
ñ A
ó ó ÷A
ð A
r*   r)  c                   óP   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltRtVtV ;t# )rî   i   c                ó    <€ V ^8„  d   QhRS[ /# r"   r   )r%   r&   s   "€r'   r(   ÚAlbertMLMHead.__annotate__  s   ø€ ÷ 4ñ 4™|ñ 4r*   c                óð  <€ \         SV `  4        \        P                  ! VP                  VP
                  R 7      V n        \        P                  ! \        P                  ! VP                  4      4      V n
        \        P                  ! VP                  VP                  4      V n        \        P                  ! VP                  VP                  4      V n        \        VP                   ,          V n        R# )r-   N)r4   r5   r   r?   r8   r@   Ú	ParameterrE   rI   r7   rñ   r•   r�   r–   r;  r	   r·   r¸   rM   s   &&€r'   r5   ÚAlbertMLMHead.__init__  s™   ø€ Ü‰ÑÔäŸš f×&;Ñ&;À×AVÑAVÔWˆŒÜ—L’L¤§¢¨V×->Ñ->Ó!?Ó@ˆŒ	Ü—Y’Y˜v×1Ñ1°6×3HÑ3HÓIˆŒ
Ü—y’y ×!6Ñ!6¸×8IÑ8IÓJˆŒÜ  ×!2Ñ!2Õ3ˆŽr*   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r#   r›   rT   rÂ   )r%   r&   s   "€r'   r(   r_  
  s#   ø€ ÷ !ñ !¡U§\¡\ð !±e·l±lñ !r*   c                ó’   € V P                  V4      pV P                  V4      pV P                  V4      pV P                  V4      pTpV# r©   )r–   r¸   r?   r;  )rN   r›   rS  s   && r'   re   ÚAlbertMLMHead.forward
  sF   € ØŸ
™
 =Ó1ˆØŸ™¨Ó6ˆØŸ™ }Ó5ˆØŸ™ ]Ó3ˆà)Ðà Ð r*   )r?   r¸   rñ   r;  r–   rª   ro   s   @@r'   rî   rî      s   ù‡ € ÷4ó 4÷!÷ !ð !r*   rî   c                   óP   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltRtVtV ;t# )r3  i  c                ó    <€ V ^8„  d   QhRS[ /# r"   r   )r%   r&   s   "€r'   r(   ÚAlbertSOPHead.__annotate__  s   ø€ ÷ Kñ K™|ñ Kr*   c                óÎ   <€ \         SV `  4        \        P                  ! VP                  4      V n        \        P                  ! VP                  VP                  4      V n	        R # r©   )
r4   r5   r   rA   Úclassifier_dropout_probrC   r•   r�   Ú
num_labelsÚ
classifierrM   s   &&€r'   r5   ÚAlbertSOPHead.__init__  sB   ø€ Ü‰ÑÔä—z’z &×"@Ñ"@ÓAˆŒÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŽr*   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r#   r&  rT   rÂ   )r%   r&   s   "€r'   r(   rh    s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r*   c                óJ   € V P                  V4      pV P                  V4      pV# r©   )rC   rl  )rN   r&  Údropout_pooled_outputÚlogitss   &&  r'   re   ÚAlbertSOPHead.forward  s%   € Ø $§¡¨]Ó ;ÐØ—‘Ð!6Ó7ˆØˆr*   )rl  rC   rª   ro   s   @@r'   r3  r3    s    ù‡ € ÷Kó K÷÷ ð r*   r3  c                   óž   a a€ ] tR tRt oRRRR/tV 3R ltV3R lR ltV3R	 lR
 ltV3R lR lt]	]
RV3R lR ll4       4       tRtVtV ;t# )ÚAlbertForMaskedLMi"  r*  r+  r,  r-  c                óŠ   <€ \         SV `  V4       \        VR R7      V n        \	        V4      V n        V P                  4        R# ©F)r	  N)r4   r5   r  ré   rî   r2  r  rM   s   &&€r'   r5   ÚAlbertForMaskedLM.__init__)  s7   ø€ Ü‰Ñ˜Ô ä! &¸EÔBˆŒÜ(¨Ó0ˆÔð 	�‰Ör*   c                ó4   <€ V ^8„  d   QhRS[ P                  /# r  r7  )r%   r&   s   "€r'   r(   ÚAlbertForMaskedLM.__annotate__2  r8  r*   c                ó.   € V P                   P                  # r©   r:  r  s   &r'   r<  Ú'AlbertForMaskedLM.get_output_embeddings2  r>  r*   c                ó8   <€ V ^8„  d   QhRS[ P                  RR/# r@  r7  )r%   r&   s   "€r'   r(   ry  5  s   ø€ ÷ 4ñ 4±B·I±Ið 4À$ñ 4r*   c                ó\   € WP                   n        VP                  V P                   n        R # r©   )r2  r;  rñ   rC  s   &&r'   rD  Ú'AlbertForMaskedLM.set_output_embeddings5  s"   € Ø#1×ÑÔ Ø .× 3Ñ 3ˆ×ÑÖr*   c                ó4   <€ V ^8„  d   QhRS[ P                  /# r  r  )r%   r&   s   "€r'   r(   ry  9  rG  r*   c                óB   € V P                   P                  P                  # r©   rI  r  s   &r'   r  Ú&AlbertForMaskedLM.get_input_embeddings9  rK  r*   c                ó6  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          R	S[S[,          /# ©
r#   rR   Nru   rL   rH   rS   rM  rw   rT   )rE   rU   rV   r   r   r   rœ   )r%   r&   s   "€r'   r(   ry  >  s²   ø€ ÷ D
ñ D
á×#Ñ# dÕ*ðD
ñ ×)Ñ)¨DÕ0ðD
ñ ×(Ñ(¨4Õ/ð	D
ñ
 ×&Ñ&¨Õ-ðD
ñ ×(Ñ(¨4Õ/ðD
ñ × Ñ  4Õ'ðD
ñ Ñ+Õ,ðD
ñ 
™%Õ	ñD
r*   c                óP  € V P                   ! R
RVRVRVRVRVRR/VB pV^ ,          p	V P                  V	4      p
RpVeG   \        4       pV! V
P                  RV P                  P
                  4      VP                  R4      4      p\        VV
VP                  VP                  R	7      # )a$  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
    config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
    loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`

Example:

```python
>>> import torch
>>> from transformers import AutoTokenizer, AlbertForMaskedLM

>>> tokenizer = AutoTokenizer.from_pretrained("albert/albert-base-v2")
>>> model = AlbertForMaskedLM.from_pretrained("albert/albert-base-v2")

>>> # add mask_token
>>> inputs = tokenizer("The capital of [MASK] is Paris.", return_tensors="pt")
>>> with torch.no_grad():
...     logits = model(**inputs).logits

>>> # retrieve index of [MASK]
>>> mask_token_index = (inputs.input_ids == tokenizer.mask_token_id)[0].nonzero(as_tuple=True)[0]
>>> predicted_token_id = logits[0, mask_token_index].argmax(axis=-1)
>>> tokenizer.decode(predicted_token_id)
'france'
```

```python
>>> labels = tokenizer("The capital of France is Paris.", return_tensors="pt")["input_ids"]
>>> labels = torch.where(inputs.input_ids == tokenizer.mask_token_id, labels, -100)
>>> outputs = model(**inputs, labels=labels)
>>> round(outputs.loss.item(), 2)
0.81
```
rR   ru   rL   rH   rS   rQ  TN©r  rq  r›   rê   rô   r3   )	ré   r2  r   rŸ   r$   r7   r   r›   rê   )rN   rR   ru   rL   rH   rS   rM  rw   rR  Úsequence_outputsrS  rW  rV  s   &&&&&&&,     r'   re   ÚAlbertForMaskedLM.forward<  sÜ   € ð^ —+’+ñ 
Øð
à)ð
ð *ð
ð &ð	
ð
 (ð
ð ð
ð ñ
ˆð # 1�:Ðà ×,Ñ,Ð-=Ó>ÐàˆØÒÜ'Ó)ˆHÙ%Ð&7×&<Ñ&<¸RÀÇÁ×AWÑAWÓ&XÐZ`×ZeÑZeÐfhÓZiÓjˆNäØØ$Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r*   )ré   r2  ©NNNNNNr[  ro   s   @@r'   rt  rt  "  sh   ù‡ € ð 	%Ð&PØ"Ð$6ðÐõ
÷(ð (÷4ð 4÷6ð 6ð Ø÷D
ñ D
ó ó ÷D
ð D
r*   rt  zž
    Albert Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled
    output) e.g. for GLUE tasks.
    c                   óh   a a€ ] tR tRt oV3R lV 3R llt]]RV3R lR ll4       4       tRtVt	V ;t
# )ÚAlbertForSequenceClassificationi…  c                ó    <€ V ^8„  d   QhRS[ /# r"   r   )r%   r&   s   "€r'   r(   Ú,AlbertForSequenceClassification.__annotate__Œ  s   ø€ ÷ 
ñ 
™|ñ 
r*   c                óR  <€ \         SV `  V4       VP                  V n        Wn        \	        V4      V n        \        P                  ! VP                  4      V n	        \        P                  ! VP                  V P                  P                  4      V n        V P                  4        R # r©   )r4   r5   rk  r$   r  ré   r   rA   rj  rC   r•   r�   rl  r  rM   s   &&€r'   r5   Ú(AlbertForSequenceClassification.__init__Œ  sr   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒØŒä! &Ó)ˆŒÜ—z’z &×"@Ñ"@ÓAˆŒÜŸ)š) F×$6Ñ$6¸¿¹×8NÑ8NÓOˆŒð 	�‰Ör*   c                ó6  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          R	S[S[,          /# rƒ  )rE   rU   rV   r   r   r   rœ   )r%   r&   s   "€r'   r(   rŒ  š  s¨   ø€ ÷ ;
ñ ;
á×#Ñ# dÕ*ð;
ñ ×)Ñ)¨DÕ0ð;
ñ ×(Ñ(¨4Õ/ð	;
ñ
 ×&Ñ&¨Õ-ð;
ñ ×(Ñ(¨4Õ/ð;
ñ × Ñ  4Õ'ð;
ñ Ñ+Õ,ð;
ñ 
"¡EÕ	)ñ;
r*   c                ó  € V P                   ! RRVRVRVRVRVRR/VB pV^,          p	V P                  V	4      p	V P                  V	4      p
RpVEe˜   V P                  P                  f–   V P
                  ^8X  d   R	V P                  n        MrV P
                  ^8”  dQ   VP                  \        P                  8X  g    VP                  \        P                  8X  d   R
V P                  n        MRV P                  n        V P                  P                  R	8X  dK   \        4       pV P
                  ^8X  d'   V! V
P                  4       VP                  4       4      pMŽV! W¦4      pM…V P                  P                  R
8X  d>   \        4       pV! V
P                  RV P
                  4      VP                  R4      4      pM-V P                  P                  R8X  d   \        4       pV! W¦4      p\        VV
VP                   VP"                  R7      # )ae  
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
    `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
rR   ru   rL   rH   rS   rQ  TNÚ
regressionÚsingle_label_classificationÚmulti_label_classificationr…  rô   r3   )ré   rC   rl  r$   Úproblem_typerk  r0   rE   rK   rã   r   Úsqueezer   rŸ   r   r   r›   rê   )rN   rR   ru   rL   rH   rS   rM  rw   rR  r&  rq  r  rV  s   &&&&&&&,     r'   re   Ú'AlbertForSequenceClassification.forward˜  sÄ  € ð$ —+’+ñ 
Øð
à)ð
ð *ð
ð &ð	
ð
 (ð
ð ð
ð ñ
ˆð   �
ˆàŸ™ ]Ó3ˆØ—‘ Ó/ˆàˆØÓØ�{‰{×'Ñ'Ò/Ø—?‘? aÔ'Ø/;�D—K‘KÕ,Ø—_‘_ qÔ(¨f¯l©l¼e¿j¹jÔ.HÈFÏLÉLÔ\a×\eÑ\eÔLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ô7Ü"›9�Ø—?‘? aÔ'Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# FÓ3‘DØ—‘×)Ñ)Ð-JÔJÜ+Ó-�Ù §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR‘Ø—‘×)Ñ)Ð-IÔIÜ,Ó.�Ù Ó/�ä'ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r*   )ré   rl  r$   rC   rk  rˆ  ©rg   rh   ri   rj   r5   r   r   re   rl   rm   rn   ro   s   @@r'   rŠ  rŠ  …  s5   ù‡ € ÷
ó 
ð Ø÷;
ñ ;
ó ó ÷;
ð ;
r*   rŠ  c                   óh   a a€ ] tR tRt oV3R lV 3R llt]]RV3R lR ll4       4       tRtVt	V ;t
# )ÚAlbertForTokenClassificationiØ  c                ó    <€ V ^8„  d   QhRS[ /# r"   r   )r%   r&   s   "€r'   r(   Ú)AlbertForTokenClassification.__annotate__Ú  s   ø€ ÷ ñ ™|ñ r*   c                ó‚  <€ \         SV `  V4       VP                  V n        \        VR R7      V n        VP
                  e   VP
                  MVP                  p\        P                  ! V4      V n	        \        P                  ! VP                  V P                  P                  4      V n        V P                  4        R# rv  )r4   r5   rk  r  ré   rj  rB   r   rA   rC   r•   r�   r$   rl  r  )rN   r$   rj  rO   s   && €r'   r5   Ú%AlbertForTokenClassification.__init__Ú  s“   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä! &¸EÔBˆŒð ×-Ñ-Ò9ð ×*Ò*à×+Ñ+ð 	 ô
 —z’zÐ"9Ó:ˆŒÜŸ)š) F×$6Ñ$6¸¿¹×8NÑ8NÓOˆŒð 	�‰Ör*   c                ó6  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          R	S[S[,          /# rƒ  )rE   rU   rV   r   r   r   rœ   )r%   r&   s   "€r'   r(   r›  ì  s¨   ø€ ÷ '
ñ '
á×#Ñ# dÕ*ð'
ñ ×)Ñ)¨DÕ0ð'
ñ ×(Ñ(¨4Õ/ð	'
ñ
 ×&Ñ&¨Õ-ð'
ñ ×(Ñ(¨4Õ/ð'
ñ × Ñ  4Õ'ð'
ñ Ñ+Õ,ð'
ñ 
¡Õ	&ñ'
r*   c                ó\  € V P                   ! V3RVRVRVRVRR/VB pV^ ,          p	V P                  V	4      p	V P                  V	4      p
RpVe=   \        4       pV! V
P	                  R	V P
                  4      VP	                  R	4      4      p\        VV
VP                  VP                  R7      # )
zÃ
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
ru   rL   rH   rS   rQ  TNr…  r3   )	ré   rC   rl  r   rŸ   rk  r   r›   rê   )rN   rR   ru   rL   rH   rS   rM  rw   rR  r%  rq  r  rV  s   &&&&&&&,     r'   re   Ú$AlbertForTokenClassification.forwardê  sÑ   € ð  —+’+Øñ
à)ð
ð *ð
ð &ð	
ð
 (ð
ð ð
ð ñ
ˆð " !�*ˆàŸ,™, Ó7ˆØ—‘ Ó1ˆàˆØÒÜ'Ó)ˆHÙ˜FŸK™K¨¨D¯O©OÓ<¸f¿k¹kÈ"»oÓNˆDä$ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r*   )ré   rl  rC   rk  rˆ  r—  ro   s   @@r'   r™  r™  Ø  s5   ù‡ € ÷ó ð  Ø÷'
ñ '
ó ó ÷'
ð '
r*   r™  c                   óh   a a€ ] tR tRt oV3R lV 3R llt]]RV3R lR ll4       4       tRtVt	V ;t
# )ÚAlbertForQuestionAnsweringi  c                ó    <€ V ^8„  d   QhRS[ /# r"   r   )r%   r&   s   "€r'   r(   Ú'AlbertForQuestionAnswering.__annotate__  r0  r*   c                óì   <€ \         SV `  V4       VP                  V n        \        VR R7      V n        \
        P                  ! VP                  VP                  4      V n        V P                  4        R# rv  )
r4   r5   rk  r  ré   r   r•   r�   Ú
qa_outputsr  rM   s   &&€r'   r5   Ú#AlbertForQuestionAnswering.__init__  sU   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä! &¸EÔBˆŒÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Ör*   c                ó^  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[S[,          R
S[S[,          /	# )r#   rR   Nru   rL   rH   rS   Ústart_positionsÚend_positionsrw   rT   rO  )r%   r&   s   "€r'   r(   r¤  $  s½   ø€ ÷ 3
ñ 3
á×#Ñ# dÕ*ð3
ñ ×)Ñ)¨DÕ0ð3
ñ ×(Ñ(¨4Õ/ð	3
ñ
 ×&Ñ&¨Õ-ð3
ñ ×(Ñ(¨4Õ/ð3
ñ ×)Ñ)¨DÕ0ð3
ñ ×'Ñ'¨$Õ.ð3
ñ Ñ+Õ,ð3
ñ 
$¡eÕ	+ñ3
r*   c                óò  € V P                   ! RR VRVRVRVRVRR/VB p	V	^ ,          p
V P                  V
4      pVP                  ^RR7      w  rÍVP                  R4      P	                  4       pVP                  R4      P	                  4       pRpVeÄ   VeÀ   \        VP                  4       4      ^8”  d   VP                  R4      p\        VP                  4       4      ^8”  d   VP                  R4      pVP                  ^4      pVP                  ^ V4      pVP                  ^ V4      p\        VR	7      pV! WÆ4      pV! W×4      pVV,           ^,          p\        VVVV	P                  V	P                  R
7      # )rR   ru   rL   rH   rS   rQ  Tr{   N)Úignore_index)r  Ústart_logitsÚ
end_logitsr›   rê   rô   r3   )ré   r¦  Úsplitr•  rƒ   ÚlenrJ   Úclampr   r   r›   rê   )rN   rR   ru   rL   rH   rS   r©  rª  rw   rR  r%  rq  r­  r®  rU  Úignored_indexrV  Ú
start_lossÚend_losss   &&&&&&&&,          r'   re   Ú"AlbertForQuestionAnswering.forward"  s™  € ð —+’+ñ 
Øð
à)ð
ð *ð
ð &ð	
ð
 (ð
ð ð
ð ñ
ˆð " !�*ˆà#Ÿ™¨Ó?ˆØ#)§<¡<°°r <Ó#:Ñ ˆØ#×+Ñ+¨BÓ/×:Ñ:Ó<ˆØ×'Ñ'¨Ó+×6Ñ6Ó8ˆ
àˆ
ØÒ&¨=Ò+Dä�?×'Ñ'Ó)Ó*¨QÔ.Ø"1×"9Ñ"9¸"Ó"=�Ü�=×%Ñ%Ó'Ó(¨1Ô,Ø -× 5Ñ 5°bÓ 9�à(×-Ñ-¨aÓ0ˆMØ-×3Ñ3°A°}ÓEˆOØ)×/Ñ/°°=ÓAˆMä'°]ÔCˆHÙ! ,Ó@ˆJÙ 
Ó:ˆHØ$ xÕ/°1Õ4ˆJä+ØØ%Ø!Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r*   )ré   rk  r¦  rZ  r—  ro   s   @@r'   r¢  r¢    s5   ù‡ € ÷ó ð Ø÷3
ñ 3
ó ó ÷3
ð 3
r*   r¢  c                   óh   a a€ ] tR tRt oV3R lV 3R llt]]RV3R lR ll4       4       tRtVt	V ;t
# )ÚAlbertForMultipleChoiceiZ  c                ó    <€ V ^8„  d   QhRS[ /# r"   r   )r%   r&   s   "€r'   r(   Ú$AlbertForMultipleChoice.__annotate__\  r0  r*   c                óü   <€ \         SV `  V4       \        V4      V n        \        P
                  ! VP                  4      V n        \        P                  ! VP                  ^4      V n
        V P                  4        R# )r2   N)r4   r5   r  ré   r   rA   rj  rC   r•   r�   rl  r  rM   s   &&€r'   r5   Ú AlbertForMultipleChoice.__init__\  sV   ø€ Ü‰Ñ˜Ô ä! &Ó)ˆŒÜ—z’z &×"@Ñ"@ÓAˆŒÜŸ)š) F×$6Ñ$6¸Ó:ˆŒð 	�‰Ör*   c                ó6  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          R	S[S[,          /# rƒ  rO  )r%   r&   s   "€r'   r(   r¹  h  s²   ø€ ÷ M
ñ M
á×#Ñ# dÕ*ðM
ñ ×)Ñ)¨DÕ0ðM
ñ ×(Ñ(¨4Õ/ð	M
ñ
 ×&Ñ&¨Õ-ðM
ñ ×(Ñ(¨4Õ/ðM
ñ × Ñ  4Õ'ðM
ñ Ñ+Õ,ðM
ñ 
$¡eÕ	+ñM
r*   c                ó&  € Ve   VP                   ^,          MVP                   ^,          pVe"   VP                  R	VP                  R	4      4      MRpVe"   VP                  R	VP                  R	4      4      MRpVe"   VP                  R	VP                  R	4      4      MRpVe"   VP                  R	VP                  R	4      4      MRpVe2   VP                  R	VP                  R
4      VP                  R	4      4      MRpV P                  ! V3RVRVRVRVRR/VB p	V	^,          p
V P	                  V
4      p
V P                  V
4      pVP                  R	V4      pRpVe   \        4       pV! WÆ4      p\        VVV	P                  V	P                  R7      # )a›  
input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
    Indices of input sequence tokens in the vocabulary.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and
    [`PreTrainedTokenizer.encode`] for details.

    [What are input IDs?](../glossary#input-ids)
token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
    Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
    1]`:

    - 0 corresponds to a *sentence A* token,
    - 1 corresponds to a *sentence B* token.

    [What are token type IDs?](../glossary#token-type-ids)
position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
    Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
    config.max_position_embeddings - 1]`.

    [What are position IDs?](../glossary#position-ids)
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):
    Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
    is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
    model's internal embedding lookup matrix.
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
    num_choices-1]` where *num_choices* is the size of the second dimension of the input tensors. (see
    *input_ids* above)
Nru   rL   rH   rS   rQ  Tr…  r3   éþÿÿÿ)
r^   rŸ   rJ   ré   rC   rl  r   r   r›   rê   )rN   rR   ru   rL   rH   rS   rM  rw   Únum_choicesrR  r&  rq  Úreshaped_logitsr  rV  s   &&&&&&&,       r'   re   ÚAlbertForMultipleChoice.forwardf  s¶  € ðT -6Ò,A�i—o‘o aÖ(À}×GZÑGZÐ[\ÕG]ˆà>GÒ>S�I—N‘N 2 y§~¡~°bÓ'9Ô:ÐY]ˆ	ØM[ÒMg˜×,Ñ,¨R°×1DÑ1DÀRÓ1HÔIÐmqˆØM[ÒMg˜×,Ñ,¨R°×1DÑ1DÀRÓ1HÔIÐmqˆØGSÒG_�|×(Ñ(¨¨\×->Ñ->¸rÓ-BÔCÐeiˆð Ò(ð ×Ñ˜r =×#5Ñ#5°bÓ#9¸=×;MÑ;MÈbÓ;QÔRàð 	ð
 —+’+Øñ
à)ð
ð *ð
ð &ð	
ð
 (ð
ð ð
ð ñ
ˆð   �
ˆàŸ™ ]Ó3ˆØ#Ÿ™¨}Ó=ˆØ Ÿ+™+ b¨+Ó6ˆàˆØÒÜ'Ó)ˆHÙ˜OÓ4ˆDä(ØØ"Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r*   )ré   rl  rC   rˆ  r—  ro   s   @@r'   r·  r·  Z  s9   ù‡ € ÷ó ð Ø÷M
ñ M
ó ó ÷M
ð M
r*   r·  )Nrž   )rè   r  r)  rt  rŠ  r™  r¢  r·  )Crk   Úcollections.abcr   Údataclassesr   rE   r   Útorch.nnr   r   r   Ú r   rï   Úactivationsr	   Úmasking_utilsr
   Úmodeling_outputsr   r   r   r   r   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úpytorch_utilsr   Úutilsr   r   r   r   Úutils.genericr   r   Úutils.output_capturingr   Úconfiguration_albertr   Ú
get_loggerrg   Úloggerrx   r    r†   rˆ   r¬   rÆ   r×   rè   rÿ   r  r)  rî   r3  rt  rŠ  r™  r¢  r·  Ú__all__rô   r*   r'   Ú<module>rÓ     s"  ðñ å $Ý !ã Ý ß AÑ Aå &Ý !Ý 6÷÷ ñ ÷ GÝ &õ÷ NÓ Mß IÝ 5Ý .ð 
×	Ò	˜HÓ	%€ô9�r—y‘yô 9÷z%ô8>)�b—i‘iô >)ôB#�"—)‘)ô #ôL�r—y‘yô ô"@˜Ÿ	™	ô @ð: ô/˜Oó /ó ð/ñ. ðôð
 ô7 ó 7ó óð7ð& ôG
Ð'ó G
ó ðG
ñT ðôô\
Ð0ó \
óð\
ô~!�B—I‘Iô !ô*
�B—I‘Iô 
ð ô_
Ð-ó _
ó ð_
ñD ðôôJ
Ð&;ó J
óðJ
ðZ ô:
Ð#8ó :
ó ð:
ðz ô@
Ð!6ó @
ó ð@
ðF ôZ
Ð3ó Z
ó ðZ
òz	‚r*   