+
    œ°³jÞå  ã                   ó˜  € R t ^ RIHt ^ RIt^ RIHt ^ RIHtHtHt ^RI	H
t ^RIHtHt ^RIHtHtHt ^RIHt ^R	IHtHt ^R
IHt ^RIHtHtHtHtHtH t H!t!H"t" ^RI#H$t$H%t% ^RI&H't' ^RI(H)t) ^RI*H+t+H,t,H-t- ^RI.H/t/H0t0 ^RI1H2t2 ^RI3H4t4 ]-Pj                  ! ]64      t7 ! R R]Pp                  4      t9RCR R llt: ! R R]Pp                  4      t; ! R R]Pp                  4      t< ! R R]Pp                  4      t= ! R R]Pp                  4      t> ! R R ]Pp                  4      t? ! R! R"]Pp                  4      t@ ! R# R$]Pp                  4      tA ! R% R&]4      tB ! R' R(]Pp                  4      tC ! R) R*]Pp                  4      tD], ! R+ R,]%4      4       tE],! R-R.7       ! R/ R0]E4      4       tF],! R1R.7       ! R2 R3]E]4      4       tG], ! R4 R5]E4      4       tH ! R6 R7]Pp                  4      tI],! R8R.7       ! R9 R:]E4      4       tJ], ! R; R<]E4      4       tK], ! R= R>]E4      4       tL ! R? R@]Pp                  4      tM], ! RA RB]E4      4       tN. RDOtOR# )EzPyTorch X-MOD model.)ÚCallableN)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELoss)Úinitialization)ÚACT2FNÚgelu)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚGradientCheckpointingLayer)Ú)BaseModelOutputWithPastAndCrossAttentionsÚ,BaseModelOutputWithPoolingAndCrossAttentionsÚ!CausalLMOutputWithCrossAttentionsÚMaskedLMOutputÚMultipleChoiceModelOutputÚQuestionAnsweringModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)Úapply_chunking_to_forward)ÚTransformersKwargsÚauto_docstringÚlogging)Úcan_return_tupleÚmerge_with_config_defaults)Úcapture_outputs)Ú
XmodConfigc                   óp   a a€ ] tR t^3t oRtV 3R ltRV3R lR llt]R 4       t]R	R l4       t	Rt
VtV ;t# )
ÚXmodEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c                óˆ  <€ \         SV `  4        \        P                  ! VP                  VP
                  VP                  R 7      V n        \        P                  ! VP                  VP
                  4      V n	        \        P                  ! VP
                  VP                  R7      V n
        \        P                  ! VP                  4      V n        \        P                  ! \         P"                  ! VP$                  4      P'                  R4      RR7      V n        \        P                  ! \         P*                  ! V P(                  P-                  4       \         P.                  R7      RR7      V n        VP                  V n        \        P                  ! VP$                  VP
                  V P2                  R 7      V n        R# ))Úpadding_idx©ÚepsF)Ú
persistent)ÚdtypeN©é   éÿÿÿÿ)ÚsuperÚ__init__r   Ú	EmbeddingÚ
vocab_sizeÚhidden_sizeÚpad_token_idÚword_embeddingsÚtype_vocab_sizeÚtoken_type_embeddingsÚ	LayerNormÚlayer_norm_epsÚDropoutÚhidden_dropout_probÚdropoutÚBufferÚtorchÚarangeÚmax_position_embeddingsÚexpandÚposition_idsÚzerosÚsizeÚlongÚtoken_type_idsr'   Úposition_embeddings©ÚselfÚconfigÚ	__class__s   &&€Úg/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/xmod/modeling_xmod.pyr0   ÚXmodEmbeddings.__init__6   s%  ø€ Ü‰ÑÔÜ!Ÿ|š|¨F×,=Ñ,=¸v×?QÑ?QÐ_e×_rÑ_rÔsˆÔÜ%'§\¢\°&×2HÑ2HÈ&×J\ÑJ\Ó%]ˆÔ"äŸš f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z’z &×"<Ñ"<Ó=ˆŒäŸIšI¤e§l¢l°6×3QÑ3QÓ&R×&YÑ&YÐZaÓ&bÐotÔuˆÔÜ Ÿiši¬¯ª°D×4EÑ4E×4JÑ4JÓ4LÔTY×T^ÑT^Ô(_ÐlqÔrˆÔà!×.Ñ.ˆÔÜ#%§<¢<Ø×*Ñ*¨F×,>Ñ,>ÈD×L\ÑL\ô$
ˆÖ ó    c                óÚ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[RS[ P                  /# )é   Ú	input_idsNrF   rB   Úinputs_embedsÚpast_key_values_lengthÚreturn)r>   Ú
LongTensorÚFloatTensorÚintÚTensor)ÚformatÚ__classdict__s   "€rL   Ú__annotate__ÚXmodEmbeddings.__annotate__F   sw   ø€ ÷ .ñ .á×#Ñ# dÕ*ð.ñ ×(Ñ(¨4Õ/ð.ñ ×&Ñ&¨Õ-ð	.ñ
 ×(Ñ(¨4Õ/ð.ñ !$ð.ñ 
�‰ñ.rN   c                ó4  € Vf=   Ve   V P                  WP                  V4      pMV P                  W@P                  4      pVe   VP                  4       pMVP                  4       R R pVw  rxVf¿   \	        V R4      '       dr   V P
                  P                  VP                  4      P                  VP                  ^ ,          R4      p	\        P                  ! V	^VR7      p	V	P                  Wx4      pM;\        P                  ! V\        P                  V P                  P                  R7      pVf   V P                  V4      pV P!                  V4      p
WJ,           pV P#                  V4      pW¼,           pV P%                  V4      pV P'                  V4      pV# )NrF   )ÚdimÚindex©r+   Údevicer.   )Ú"create_position_ids_from_input_idsr'   Ú&create_position_ids_from_inputs_embedsrD   ÚhasattrrF   Útora   rA   Úshaper>   ÚgatherrC   rE   rB   r5   r7   rG   r8   r<   )rI   rQ   rF   rB   rR   rS   Úinput_shapeÚ
batch_sizeÚ
seq_lengthÚbuffered_token_type_idsr7   Ú
embeddingsrG   s   &&&&&&       rL   ÚforwardÚXmodEmbeddings.forwardF   st  € ð ÒØÒ$à#×FÑFØ×/Ñ/Ð1Gó ‘ð  $×JÑJÈ=×ZjÑZjÓk�àÒ Ø#Ÿ.™.Ó*‰Kà'×,Ñ,Ó.¨s°Ð3ˆKà!,Ñˆ
ð
 Ò!Ü�tÐ-×.Ò.à*.×*=Ñ*=×*@Ñ*@À×ATÑATÓ*U×*\Ñ*\Ð]i×]oÑ]oÐpqÕ]rÐtvÓ*wÐ'Ü*/¯,ª,Ð7NÐTUÐ]iÔ*jÐ'Ø!8×!?Ñ!?À
Ó!W‘ä!&§¢¨[ÄÇ
Á
ÐSW×SdÑSd×SkÑSkÔ!l�àÒ Ø ×0Ñ0°Ó;ˆMØ $× :Ñ :¸>Ó JÐØ"Õ:ˆ
à"×6Ñ6°|ÓDÐØÕ5ˆ
à—^‘^ JÓ/ˆ
Ø—\‘\ *Ó-ˆ
ØÐrN   c                ó  € V P                  4       RR pV^,          p\        P                  ! V^,           W1,           ^,           \        P                  V P                  R7      pVP                  ^ 4      P                  V4      # )z¯
We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.

Args:
    inputs_embeds: torch.Tensor

Returns: torch.Tensor
Nr`   r.   )rD   r>   r?   rE   ra   Ú	unsqueezerA   )rR   r'   rh   Úsequence_lengthrB   s   &&   rL   rc   Ú5XmodEmbeddings.create_position_ids_from_inputs_embedsv   sn   € ð $×(Ñ(Ó*¨3¨BÐ/ˆØ% a�.ˆä—|’|Ø˜!�O˜_Õ:¸QÕ>ÄeÇjÁjÐYf×YmÑYmô
ˆð ×%Ñ% aÓ(×/Ñ/°Ó<Ð<rN   c                óØ   € V P                  V4      P                  4       p\        P                  ! V^R7      P	                  V4      V,           V,          pVP                  4       V,           # )zí
Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols
are ignored. This is modified from fairseq's `utils.make_positions`.

Args:
    x: torch.Tensor x:

Returns: torch.Tensor
©r^   )ÚnerW   r>   ÚcumsumÚtype_asrE   )rQ   r'   rS   ÚmaskÚincremental_indicess   &&&  rL   rb   Ú1XmodEmbeddings.create_position_ids_from_input_idsˆ   sW   € ð �|‰|˜KÓ(×,Ñ,Ó.ˆÜ$Ÿ|š|¨D°aÔ8×@Ñ@ÀÓFÐI_Õ_ÐcgÕgÐØ"×'Ñ'Ó)¨KÕ7Ð7rN   )r8   r<   r'   rG   rB   r7   rF   r5   )NNNNé    )r{   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r0   rm   Ústaticmethodrc   rb   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©rK   rZ   s   @@rL   r%   r%   3   sE   ù‡ € ÙQõ
÷ .ò .ð` ñ=ó ð=ð" ó8ó ÷8ð 8rN   r%   c                ó  € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R,          R\        R	\
        \        ,          /# )
rP   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNÚscalingr<   Úkwargs)r   ÚModuler>   rX   Úfloatr   r   )rY   s   "rL   r[   r[   š   s{   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô �T�\ð%ô ð%ô Ô'Õ(ñ%rN   c                 ó¶  € Vf   VP                  R4      R,          p\        P                  ! WP                  ^^4      4      V,          pVe	   W„,           p\        P
                  P                  VRR7      p\        P
                  P                  W†V P                  R7      p\        P                  ! Wƒ4      p	V	P                  ^^4      P                  4       p	W˜3# )Nrt   )ÚpÚtrainingr.   ç      à¿)
rD   r>   ÚmatmulÚ	transposer   Ú
functionalÚsoftmaxr<   r’   Ú
contiguous)
r‡   rˆ   r‰   rŠ   r‹   rŒ   r<   r�   Úattn_weightsÚattn_outputs
   &&&&&&&,  rL   Úeager_attention_forwardr›   š   s±   € ð ‚Ø—*‘*˜R“. DÕ(ˆô —<’< §}¡}°Q¸Ó':Ó;¸gÕE€LàÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2Ð(Ó>€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€Lä—,’,˜|Ó3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$rN   c                   óL   a a€ ] tR t^·t oRV 3R lltRV3R lR lltRtVtV ;t# )ÚXmodSelfAttentionc                ót  <€ \         SV `  4        VP                  VP                  ,          ^ 8w  d9   \	        VR4      '       g'   \        RVP                   RVP                   R24      hWn        VP                  V n        \        VP                  VP                  ,          4      V n        V P                  V P                  ,          V n	        V P                  R,          V n
        \        P                  ! VP                  V P                  4      V n        \        P                  ! VP                  V P                  4      V n        \        P                  ! VP                  V P                  4      V n        \        P                   ! VP"                  4      V n        VP&                  V n        W n        W0n        R# ©r{   Úembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads (Ú)Nr“   )r/   r0   r3   Únum_attention_headsrd   Ú
ValueErrorrJ   rW   Úattention_head_sizeÚall_head_sizerŒ   r   ÚLinearrˆ   r‰   rŠ   r:   Úattention_probs_dropout_probr<   Ú
is_decoderÚ	is_causalÚ	layer_idx©rI   rJ   r©   rª   rK   s   &&&&€rL   r0   ÚXmodSelfAttention.__init__¸   sM  ø€ Ü‰ÑÔØ×Ñ × :Ñ :Õ:¸aÔ?ÌÐPVÐXh×HiÒHiÜØ# F×$6Ñ$6Ð#7ð 8Ø ×4Ñ4Ð5°Qð8óð ð Œà#)×#=Ñ#=ˆÔ Ü#& v×'9Ñ'9¸F×<VÑ<VÕ'VÓ#WˆÔ Ø!×5Ñ5¸×8PÑ8PÕPˆÔØ×/Ñ/°Õ5ˆŒä—Y’Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
Ü—9’9˜V×/Ñ/°×1CÑ1CÓDˆŒÜ—Y’Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
ä—z’z &×"EÑ"EÓFˆŒà ×+Ñ+ˆŒØ"ŒØ"ŽrN   c                ó°   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  ,          /# )rP   Úhidden_statesr‹   NÚpast_key_valuesr�   rT   )r>   rX   rV   r
   r   r   Útuple)rY   rZ   s   "€rL   r[   ÚXmodSelfAttention.__annotate__Ð   s_   ø€ ÷ ')ñ ')á—|‘|ð')ñ ×)Ñ)¨DÕ0ð')ñ  �ð	')ñ
 Ñ+Õ,ð')ñ 
‰u�|‰|Õ	ñ')rN   c                ó   € VP                   R R p. VORNV P                  N5pV P                  V4      P                  ! V!  P	                  ^^4      pV P                  V4      P                  ! V!  P	                  ^^4      pV P                  V4      P                  ! V!  P	                  ^^4      p	VeC   Tp
\        V\        4      '       d   VP                  p
V
P                  W‰V P                  4      w  r‰\        P                  ! V P                  P                  \         4      pT! V VVV	V3RV P"                  '       g   RMV P$                  P&                  RV P(                  /VB w  rÍVP*                  ! . VORN5!  P-                  4       pWÍ3# )Nr<   ç        rŒ   r.   )rf   r¤   rˆ   Úviewr•   r‰   rŠ   Ú
isinstancer   Úself_attention_cacheÚupdaterª   r   Úget_interfacerJ   Ú_attn_implementationr›   r’   r<   r‘   rŒ   Úreshaper˜   )rI   r®   r‹   r¯   r�   rh   Úhidden_shapeÚquery_layerÚ	key_layerÚvalue_layerÚcurrent_past_key_valuesÚattention_interfacerš   r™   s   &&&&,         rL   rm   ÚXmodSelfAttention.forwardÐ   s‰  € ð $×)Ñ)¨#¨2Ð.ˆØC˜ÐC bÐC¨$×*BÑ*BÑCˆð —j‘j Ó/×4Ò4°lÑC×MÑMÈaÐQRÓSˆØ—H‘H˜]Ó+×0Ò0°,Ñ?×IÑIÈ!ÈQÓOˆ	Ø—j‘j Ó/×4Ò4°lÑC×MÑMÈaÐQRÓSˆàÒ&à&5Ð#Ü˜/Ô+>×?Ò?Ø*9×*NÑ*NÐ'ð &=×%CÑ%CÀIÐ\`×\jÑ\jÓ%kÑ"ˆIä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð  $Ÿ}Ÿ}˜}‘C°$·,±,·.±.ð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØÐ(Ð(rN   )r¥   r¤   rJ   r<   r©   r¨   r‰   rª   r¢   rˆ   rŒ   rŠ   ©FN)NN©	r|   r}   r~   r   r0   rm   r‚   rƒ   r„   r…   s   @@rL   r�   r�   ·   s   ù‡ € ÷#÷0')÷ ')ò ')rN   r�   c                   óL   a a€ ] tR t^ût oRV 3R lltRV3R lR lltRtVtV ;t# )ÚXmodCrossAttentionc                óR  <€ \         SV `  4        VP                  VP                  ,          ^ 8w  d9   \	        VR4      '       g'   \        RVP                   RVP                   R24      hWn        VP                  V n        \        VP                  VP                  ,          4      V n        V P                  V P                  ,          V n	        V P                  R,          V n
        \        P                  ! VP                  V P                  4      V n        \        P                  ! VP                  V P                  4      V n        \        P                  ! VP                  V P                  4      V n        \        P                   ! VP"                  4      V n        W n        W0n        R# rŸ   )r/   r0   r3   r¢   rd   r£   rJ   rW   r¤   r¥   rŒ   r   r¦   rˆ   r‰   rŠ   r:   r§   r<   r©   rª   r«   s   &&&&€rL   r0   ÚXmodCrossAttention.__init__ü   s@  ø€ Ü‰ÑÔØ×Ñ × :Ñ :Õ:¸aÔ?ÌÐPVÐXh×HiÒHiÜØ# F×$6Ñ$6Ð#7ð 8Ø ×4Ñ4Ð5°Qð8óð ð Œà#)×#=Ñ#=ˆÔ Ü#& v×'9Ñ'9¸F×<VÑ<VÕ'VÓ#WˆÔ Ø!×5Ñ5¸×8PÑ8PÕPˆÔØ×/Ñ/°Õ5ˆŒä—Y’Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
Ü—9’9˜V×/Ñ/°×1CÑ1CÓDˆŒÜ—Y’Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
ä—z’z &×"EÑ"EÓFˆŒà"ŒØ"ŽrN   c                óØ   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  ,          /# )rP   r®   Úencoder_hidden_statesNr‹   r¯   r�   rT   )r>   rX   rV   r   r   r   r°   )rY   rZ   s   "€rL   r[   ÚXmodCrossAttention.__annotate__  su   ø€ ÷ 1)ñ 1)á—|‘|ð1)ñ  %×0Ñ0°4Õ7ð1)ñ ×)Ñ)¨DÕ0ð	1)ñ
 -¨tÕ3ð1)ñ Ñ+Õ,ð1)ñ 
‰u�|‰|Õ	ñ1)rN   c                ó˜  € VP                   R R p. VORNV P                  N5pV P                  V4      P                  V4      P	                  ^^4      pVe&   VP
                  P                  V P                  4      MRp	Vel   V	'       dd   VP                  P                  V P                  ,          P                  p
VP                  P                  V P                  ,          P                  pMÅ. VP                   R R ORNV P                  N5pV P                  V4      P                  V4      P	                  ^^4      p
V P                  V4      P                  V4      P	                  ^^4      pVeB   VP                  P                  W«V P                  4      w  r«RVP
                  V P                  &   \        P                   ! V P"                  P$                  \&        4      pT! V VV
VV3RV P(                  '       g   RMV P*                  P,                  RV P.                  /VB w  rïVP0                  ! . VORN5!  P3                  4       pWï3# )NFTr<   r³   rŒ   r.   )rf   r¤   rˆ   r´   r•   Ú
is_updatedÚgetrª   Úcross_attention_cacheÚlayersÚkeysÚvaluesr‰   rŠ   r·   r   r¸   rJ   r¹   r›   r’   r<   r‘   rŒ   rº   r˜   )rI   r®   rÉ   r‹   r¯   r�   rh   r»   r¼   rÌ   r½   r¾   Úkv_shaperÀ   rš   r™   s   &&&&&,          rL   rm   ÚXmodCrossAttention.forward  s  € ð $×)Ñ)¨#¨2Ð.ˆàC˜ÐC bÐC¨$×*BÑ*BÑCˆð —j‘j Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆàGVÒGb�_×/Ñ/×3Ñ3°D·N±NÔCÐhmˆ
ØÒ&¯:à'×=Ñ=×DÑDÀTÇ^Á^ÕT×YÑYˆIØ)×?Ñ?×FÑFÀtÇ~Á~ÕV×]Ñ]‰KàXÐ.×4Ñ4°S°bÐ9ÐX¸2ÐX¸t×?WÑ?WÑXˆHØŸ™Ð!6Ó7×<Ñ<¸XÓF×PÑPÐQRÐTUÓVˆIØŸ*™*Ð%:Ó;×@Ñ@ÀÓJ×TÑTÐUVÐXYÓZˆKàÒ*à)8×)NÑ)N×)UÑ)UØ¨D¯N©Nó*Ñ&�	ð >B�×*Ñ*¨4¯>©>Ñ:ä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð  $Ÿ}Ÿ}˜}‘C°$·,±,·.±.ð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØÐ(Ð(rN   )r¥   r¤   rJ   r<   r©   r‰   rª   r¢   rˆ   rŒ   rŠ   rÂ   )NNNrÃ   r…   s   @@rL   rÅ   rÅ   û   s   ù‡ € ÷#÷.1)÷ 1)ò 1)rN   rÅ   c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚXmodSelfOutputiG  c                ó0  <€ \         SV `  4        \        P                  ! VP                  VP                  4      V n        \        P                  ! VP                  VP                  R 7      V n        \        P                  ! VP                  4      V n
        R# ©r(   N)r/   r0   r   r¦   r3   Údenser8   r9   r:   r;   r<   rH   s   &&€rL   r0   ÚXmodSelfOutput.__init__I  s`   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸš f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z’z &×"<Ñ"<Ó=ˆŽrN   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# )rP   r®   Úinput_tensorrT   ©r>   rX   )rY   rZ   s   "€rL   r[   ÚXmodSelfOutput.__annotate__O  s1   ø€ ÷ ñ ¡U§\¡\ð ÁÇÁð ÑRW×R^ÑR^ñ rN   c                óZ   € V P                  V4      pV P                  V4      pW,           pV# ©N)rØ   r<   )rI   r®   rÛ   s   &&&rL   rm   ÚXmodSelfOutput.forwardO  s,   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØ%Õ4ˆØÐrN   )r8   rØ   r<   rÃ   r…   s   @@rL   rÕ   rÕ   G  s   ù‡ € õ>÷÷ ð rN   rÕ   c                   óL   a a€ ] tR tRt oRV 3R lltRV3R lR lltRtVtV ;t# )ÚXmodAttentioniV  c                ó¸   <€ \         SV `  4        W@n        V'       d   \        M\        pV! WVR 7      V n        \        V4      V n        VP                  V n        R# )©r©   rª   N)	r/   r0   Úis_cross_attentionrÅ   r�   rI   rÕ   ÚoutputÚpre_norm)rI   rJ   r©   rª   rå   Úattention_classrK   s   &&&&& €rL   r0   ÚXmodAttention.__init__W  sD   ø€ Ü‰ÑÔØ"4Ôß0BÕ,ÔHYˆÙ# FÈ9ÔUˆŒ	Ü$ VÓ,ˆŒàŸ™ˆŽrN   c                ó4  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[S[ P                  ,          ,          R,          RS[S[,          RS[S[ P                  ,          /# )	rP   r®   r‹   NrÉ   Úencoder_attention_maskr¯   r�   rT   ©r>   rX   rV   r°   r   r   )rY   rZ   s   "€rL   r[   ÚXmodAttention.__annotate__`  sš   ø€ ÷ .ñ .á—|‘|ð.ñ ×)Ñ)¨DÕ0ð.ñ  %×0Ñ0°4Õ7ð	.ñ
 !&× 1Ñ 1°DÕ 8ð.ñ ™u¡U×%6Ñ%6Õ7Õ8¸4Õ?ð.ñ Ñ+Õ,ð.ñ 
‰u�|‰|Õ	ñ.rN   c           	     óD  € TpV P                   '       d   V P                  P                  V4      pV P                  '       g   TMTpV P                  ! V3R VRVRV/VB w  r‰V P                  W‡4      pV P                   '       g   V P                  P                  V4      pW‰3# )rÉ   r‹   r¯   )rç   ræ   r8   rå   rI   )
rI   r®   r‹   rÉ   rë   r¯   r�   ÚresidualÚattention_outputr™   s
   &&&&&&,   rL   rm   ÚXmodAttention.forward`  s®   € ð !ˆØ�=�=ˆ=Ø ŸK™K×1Ñ1°-Ó@ˆMà/3×/F×/FÐ/F™ÐLbˆØ)-¯ªØñ*
à"7ð*
ð *ð*
ð ,ð	*
ð
 ñ*
Ñ&Ðð  Ÿ;™;Ð'7ÓBÐà�}�}ˆ}Ø#Ÿ{™{×4Ñ4Ð5EÓFÐàÐ-Ð-rN   )rå   ræ   rç   rI   )FNF©NNNNrÃ   r…   s   @@rL   râ   râ   V  s   ù‡ € ÷(÷.÷ .ò .rN   râ   c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚXmodIntermediatei~  c                ó"  <€ \         SV `  4        \        P                  ! VP                  VP
                  4      V n        \        VP                  \        4      '       d   \        VP                  ,          V n        R # VP                  V n        R # rß   )r/   r0   r   r¦   r3   Úintermediate_sizerØ   rµ   Ú
hidden_actÚstrr   Úintermediate_act_fnrH   s   &&€rL   r0   ÚXmodIntermediate.__init__  s`   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3KÑ3KÓLˆŒ
Ü�f×'Ñ'¬×-Ò-Ü'-¨f×.?Ñ.?Õ'@ˆDÖ$à'-×'8Ñ'8ˆDÖ$rN   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©rP   r®   rT   rÜ   )rY   rZ   s   "€rL   r[   ÚXmodIntermediate.__annotate__‡  s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ rN   c                óJ   € V P                  V4      pV P                  V4      pV# rß   ©rØ   rù   ©rI   r®   s   &&rL   rm   ÚXmodIntermediate.forward‡  s&   € ØŸ
™
 =Ó1ˆØ×0Ñ0°Ó?ˆØÐrN   rÿ   rÃ   r…   s   @@rL   rô   rô   ~  s   ù‡ € õ9÷÷ ð rN   rô   c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚXmodAdapteri�  c                óÆ  <€ \         SV `  4        VP                  VP                  ,          V n        \
        P                  ! VP                  V P                  4      V n        \
        P                  ! V P                  VP                  4      V n        \        VP                  \        4      '       d   \        VP                  ,          V n        R # VP                  V n        R # rß   )r/   r0   r3   Úadapter_reduction_factorÚbottleneck_sizer   r¦   Údense1Údense2rµ   r÷   rø   r   Úadapter_act_fnrH   s   &&€rL   r0   ÚXmodAdapter.__init__Ž  s—   ø€ Ü‰ÑÔØ%×1Ñ1°V×5TÑ5TÕTˆÔÜ—i’i × 2Ñ 2°D×4HÑ4HÓIˆŒÜ—i’i × 4Ñ 4°f×6HÑ6HÓIˆŒÜ�f×'Ñ'¬×-Ò-Ü"(¨×):Ñ):Õ";ˆDÖà"(×"3Ñ"3ˆDÖrN   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rü   rÜ   )rY   rZ   s   "€rL   r[   ÚXmodAdapter.__annotate__˜  s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ rN   c                ól   € V P                  V4      pV P                  V4      pV P                  V4      pV# rß   )r  r	  r  r   s   &&rL   rm   ÚXmodAdapter.forward˜  s4   € ØŸ™ MÓ2ˆØ×+Ñ+¨MÓ:ˆØŸ™ MÓ2ˆØÐrN   )r	  r  r  r  rÃ   r…   s   @@rL   r  r  �  s   ù‡ € õ4÷÷ ð rN   r  c                   óV   a a€ ] tR tRt oV 3R ltV3R lR ltV3R lR ltRtVtV ;t	# )Ú
XmodOutputiŸ  c                ó¨  <€ \         SV `  4        \        P                  ! VP                  VP
                  4      V n        \        P                  ! VP
                  VP                  R 7      V n        VP                  V n	        \        P                  ! VP                  4      V n        VP                  '       d3   \        P                  ! VP
                  VP                  R 7      V n        MRV n        VP                  V n        \        P                  ! / 4      V n        VP"                   F$  p\%        V4      V P                   \'        V4      &   K&  	  R# r×   )r/   r0   r   r¦   rö   r3   rØ   r8   r9   Úln_before_adapterr:   r;   r<   Úadapter_layer_normÚadapter_reuse_layer_normÚ
ModuleDictÚadapter_modulesÚ	languagesr  rø   )rI   rJ   ÚlanguagerK   s   && €rL   r0   ÚXmodOutput.__init__   sé   ø€ Ü‰ÑÔÜ—Y’Y˜v×7Ñ7¸×9KÑ9KÓLˆŒ
ÜŸš f×&8Ñ&8¸f×>SÑ>SÔTˆŒØ!'×!9Ñ!9ˆÔÜ—z’z &×"<Ñ"<Ó=ˆŒØ×$×$Ð$Ü&(§l¢l°6×3EÑ3EÈ6×K`ÑK`Ô&aˆDÕ#à&*ˆDÔ#Ø(.×(GÑ(GˆÔ%Ü!Ÿ}š}¨RÓ0ˆÔØ×(Ô(ˆHÜ2=¸fÓ2EˆD× Ñ ¤ X£Ó/ó )rN   c                ó‚   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[ P                  /# )rP   r®   rÛ   Úlang_idsrT   rÜ   )rY   rZ   s   "€rL   r[   ÚXmodOutput.__annotate__¯  s?   ø€ ÷ ñ ¡U§\¡\ð ÁÇÁð ÑY^×YeÑYeð Ñjo×jvÑjvñ rN   c                ó|   € V P                  V4      pV P                  V4      pW,           pV P                  W14      pV# rß   )rØ   r<   Úlang_adapter)rI   r®   rÛ   r  s   &&&&rL   rm   ÚXmodOutput.forward¯  s<   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØ%Õ4ˆØ×)Ñ)¨(ÓBˆØÐrN   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )rP   r  r®   rÜ   )rY   rZ   s   "€rL   r[   r  ¶  s#   ø€ ÷ ñ ¡U§\¡\ð Á%Ç,Á,ñ rN   c                óæ  € V P                   '       g   TpV P                  e   V P                  V4      pM#V P                  '       d   V P                  V4      pV P                   '       d   Tp\        P
                  ! V4      p\        V P                  P                  4       4       F.  w  rVW8H  pW',          pV P                  V,          ! V4      p	W”V&   K0  	  V P                  V4      pVX,          pV# rß   )
r  r  r  r8   r>   Ú
zeros_likeÚ	enumerater  rÐ   r<   )
rI   r  r®   rï   Únew_hidden_statesÚadapter_idxÚlang_keyÚ	lang_maskÚlang_hidden_statesÚadapted_lang_hidden_statess
   &&&       rL   r  ÚXmodOutput.lang_adapter¶  sÞ   € Ø×%×%Ð%Ø$ˆHà×"Ñ"Ò.Ø ×3Ñ3°MÓB‰MØ×*×*Ð*Ø ŸN™N¨=Ó9ˆMà×!×!Ð!Ø$ˆHä!×,Ò,¨]Ó;ÐÜ%.¨t×/CÑ/C×/HÑ/HÓ/JÖ%KÑ!ˆKØ Ñ/ˆIØ!.Õ!9ÐØ)-×)=Ñ)=¸hÖ)GÐHZÓ)[Ð&Ø+E˜iÓ(ñ	 &Lð Ÿ™Ð%6Ó7ˆØ˜Õ!ˆØÐrN   )r8   r  r  r  rØ   r<   r  )
r|   r}   r~   r   r0   rm   r  r‚   rƒ   r„   r…   s   @@rL   r  r  Ÿ  s$   ù‡ € õF÷ð ÷÷ ð rN   r  c                   óR   a a€ ] tR tRt oRV 3R lltRV3R lR lltR tRtVtV ;t	# )	Ú	XmodLayeriÎ  c                ó¾  <€ \         SV `  4        VP                  V n        ^V n        \	        WP
                  VR7      V n        VP
                  V n        VP                  V n        V P                  '       d5   V P
                  '       g   \        V  R24      h\	        VRVRR7      V n	        \        V4      V n        \        V4      V n        VP                  V n        R# )r-   rä   z> should be used as a decoder model if cross attention is addedFT)r©   rª   rå   N)r/   r0   Úchunk_size_feed_forwardÚseq_len_dimrâ   r¨   Ú	attentionÚadd_cross_attentionr£   Úcrossattentionrô   Úintermediater  ræ   rç   )rI   rJ   rª   rK   s   &&&€rL   r0   ÚXmodLayer.__init__Ï  s»   ø€ Ü‰ÑÔØ'-×'EÑ'EˆÔ$ØˆÔÜ& v×9JÑ9JÐV_Ô`ˆŒØ ×+Ñ+ˆŒØ#)×#=Ñ#=ˆÔ Ø×#×#Ð#Ø—?—?�?Ü  D 6Ð)gÐ!hÓiÐiÜ"/ØØØ#Ø#'ô	#ˆDÔô -¨VÓ4ˆÔÜ  Ó(ˆŒØŸ™ˆŽrN   c                ó>  <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[S[ P                  ,          ,          R,          RS[S[,          R	S[ P                  /# )
rP   r®   r  r‹   NrÉ   rë   r¯   r�   rT   rì   )rY   rZ   s   "€rL   r[   ÚXmodLayer.__annotate__ã  s£   ø€ ÷ 0ñ 0á—|‘|ð0ñ —,‘,ð0ñ ×)Ñ)¨DÕ0ð	0ñ
  %×0Ñ0°4Õ7ð0ñ !&× 1Ñ 1°DÕ 8ð0ñ ™u¡U×%6Ñ%6Õ7Õ8¸4Õ?ð0ñ Ñ+Õ,ð0ñ 
�‰ñ0rN   c                ó  € V P                   ! VV3R V/VB w  r‰Tp
V P                  '       dB   Ve>   \        V R4      '       g   \        RV  R24      hV P                  ! V
RVV3R V/VB w  r¹Tp
T
pV P
                  '       d   V P                  P                  V
4      p
\        V P                  V P                  V P                  V
4      pV P                  WÜV4      pV P
                  '       g   V P                  P                  V4      pV# )r¯   Nr2  z'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`)r0  r¨   rd   r£   r2  rç   ræ   r8   r   Úfeed_forward_chunkr.  r/  )rI   r®   r  r‹   rÉ   rë   r¯   r�   Úself_attention_outputÚ_rð   Úcross_attention_outputrï   Úintermediate_outputÚlayer_outputs   &&&&&&&,       rL   rm   ÚXmodLayer.forwardã  s7  € ð $(§>¢>ØØñ$
ð ,ð$
ð ñ	$
Ñ Ðð 1Ðà�?�?ˆ?Ð4Ò@Ü˜4Ð!1×2Ò2Ü Ø=¸d¸Vð DDð Dóð ð
 )-×(;Ò(;Ø ØØ%Ø&ñ	)ð
 !0ð)ð ñ)Ñ%Ð"ð  6Ðà#ˆØ�=�=ˆ=Ø#Ÿ{™{×4Ñ4Ð5EÓFÐÜ7Ø×#Ñ#Ø×(Ñ(Ø×ÑØó	
Ðð —{‘{Ð#6À(ÓKˆØ�}�}ˆ}ØŸ;™;×0Ñ0°Ó>ˆLàÐrN   c                ó$   € V P                  V4      # rß   )r3  )rI   rð   s   &&rL   r8  ÚXmodLayer.feed_forward_chunk  s   € Ø× Ñ Ð!1Ó2Ð2rN   )	r1  r0  r.  r2  r3  r¨   ræ   rç   r/  rß   rò   )
r|   r}   r~   r   r0   rm   r8  r‚   rƒ   r„   r…   s   @@rL   r,  r,  Î  s   ù‡ € ÷(÷(0ò 0÷d3ò 3rN   r,  c                   óH   a a€ ] tR tRt oV 3R ltRV3R lR lltRtVtV ;t# )ÚXmodEncoderi  c           
     óv  <€ \         SV `  4        Wn        \        P                  ! \        VP                  4       Uu. uF  p\        WR 7      NK  	  up4      V n        VP                  V n
        V P                  '       d4   \        P                  ! VP                  VP                  R7      V n        R# R# u upi ))rª   r(   N)r/   r0   rJ   r   Ú
ModuleListÚrangeÚnum_hidden_layersr,  Úlayerrç   Úis_pre_normr8   r3   r9   )rI   rJ   ÚirK   s   && €rL   r0   ÚXmodEncoder.__init__  s†   ø€ Ü‰ÑÔØŒÜ—]’]ÌEÐRX×RjÑRjÔLkÓ#lÑLkÀq¤I¨f×$BÑLkÑ#lÓmˆŒ
Ø!Ÿ?™?ˆÔØ××ÐÜŸ\š\¨&×*<Ñ*<À&×BWÑBWÔXˆDŽNñ ùò $ms   ½B6c                ór  <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[S[ P                  ,          ,          R,          RS[R,          R	S[S[,          R
S[S[ P                  ,          S[,          /	# )rP   r®   r  r‹   NrÉ   rë   r¯   Ú	use_cacher�   rT   )r>   rX   rV   r°   Úboolr   r   r   )rY   rZ   s   "€rL   r[   ÚXmodEncoder.__annotate__"  s¼   ø€ ÷ 
ñ 
á—|‘|ð
ñ —,‘,ð
ñ ×)Ñ)¨DÕ0ð	
ñ
  %×0Ñ0°4Õ7ð
ñ !&× 1Ñ 1°DÕ 8ð
ñ ™u¡U×%6Ñ%6Õ7Õ8¸4Õ?ð
ñ ˜$•;ð
ñ Ñ+Õ,ð
ñ 
‰u�|‰|Õ	ÑHÕ	Hñ
rN   c           	     óØ   € \        V P                  4       F  w  ršV
! VVVVVV3/ VB pK  	  V P                  '       d   V P                  V4      p\	        TV'       d   VR7      # R R7      # )N)Úlast_hidden_stater¯   )r#  rG  rH  r8   r   )rI   r®   r  r‹   rÉ   rë   r¯   rL  r�   rI  Úlayer_modules   &&&&&&&&,  rL   rm   ÚXmodEncoder.forward"  s}   € ô  )¨¯©Ö4‰OˆAÙ(ØØØØ%Ø&Øñð ñŠMñ  5ð ××ÐØ ŸN™N¨=Ó9ˆMä8Ø+ß/8˜Oô
ð 	
à>Bô
ð 	
rN   )r8   rJ   rH  rG  )NNNNNrÃ   r…   s   @@rL   rB  rB    s   ù‡ € õY÷
÷ 
ò 
rN   rB  c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )Ú
XmodPooleriB  c                ó¸   <€ \         SV `  4        \        P                  ! VP                  VP                  4      V n        \        P                  ! 4       V n        R # rß   )r/   r0   r   r¦   r3   rØ   ÚTanhÚ
activationrH   s   &&€rL   r0   ÚXmodPooler.__init__C  s9   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ'š'›)ˆŽrN   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rü   rÜ   )rY   rZ   s   "€rL   r[   ÚXmodPooler.__annotate__H  s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ rN   c                ó\   € VR,          pV P                  V4      pV P                  V4      pV# )ºNNN)r\  r{   )rØ   rW  )rI   r®   Úfirst_token_tensorÚpooled_outputs   &&  rL   rm   ÚXmodPooler.forwardH  s2   € ð +¨4Õ0ÐØŸ
™
Ð#5Ó6ˆØŸ™¨Ó6ˆØÐrN   )rW  rØ   rÃ   r…   s   @@rL   rT  rT  B  s   ù‡ € õ$÷
÷ ð rN   rT  c                   ó¦   a a€ ] tR tRt o]tRtRt. ROtRt	Rt
RtRtR]R]R]/t]P$                  ! 4       V 3R l4       tV3R lR	 ltR
 tRtVtV ;t# )ÚXmodPreTrainedModeliQ  ÚrobertaTr®   Ú
attentionsÚcross_attentionsc                óÄ  <€ \         SV `  V4       \        V\        4      '       d#   \        P
                  ! VP                  4       R# \        V\        4      '       d‚   \        P                  ! VP                  \        P                  ! VP                  P                  R,          4      P                  R4      4       \        P
                  ! VP                  4       R# R# )zInitialize the weightsNr.   r,   )r/   Ú_init_weightsrµ   Ú
XmodLMHeadÚinitÚzeros_Úbiasr%   Úcopy_rB   r>   r?   rf   rA   rF   )rI   r‡   rK   s   &&€rL   rf  Ú!XmodPreTrainedModel._init_weightsa  sŽ   ø€ ô 	‰Ñ˜fÔ%Ü�fœj×)Ò)Ü�KŠK˜Ÿ™Ö$Ü˜¤×/Ò/Ü�JŠJ�v×*Ñ*¬E¯LªL¸×9LÑ9L×9RÑ9RÐSUÕ9VÓ,W×,^Ñ,^Ð_fÓ,gÔhÜ�KŠK˜×-Ñ-Ö.ñ 0rN   c                ó    <€ V ^8„  d   QhRS[ /# )rP   r  )rø   )rY   rZ   s   "€rL   r[   Ú XmodPreTrainedModel.__annotate__k  s   ø€ ÷ 0ñ 0©Sñ 0rN   c           	     óº   € WP                   P                  9  d1   \        V  RV R\        V P                   P                  4       24      hWP                   n        R# )z¼
Set the default language code for the model. This is used when the language is not specified in the input.

Args:
    language (`str`): The language code, such as `"en_XX"` or `"de_DE"`.
z does not have an adapter for z. Supported languages: N)rJ   r  r£   ÚlistÚdefault_language)rI   r  s   &&rL   Úset_default_languageÚ(XmodPreTrainedModel.set_default_languagek  sW   € ð Ÿ;™;×0Ñ0Ô0ÜØ�&Ð6°x°jÐ@WÔX\Ð]a×]hÑ]h×]rÑ]rÓXsÐWtÐuóð ð (0�‰Ö$rN   c                ó  € \         P                  R4       V P                  P                  P	                  4        F
  pRVn        K  	  \         P                  R4       V P                  P                  P                   F  pVP                  P                  e3   VP                  P                  P	                  4        F
  pRVn        K  	  VP                  P                  P	                  4        F
  pRVn        K  	  K�  	  R# )z‰
Freeze the embeddings and language adapters of the model. Usually, this is applied before the model is
fine-tuned on a downstream task.
zFreezing embeddingsFzFreezing adaptersN)ÚloggerÚinforb  rl   Ú
parametersÚrequires_gradÚencoderrG  ræ   r  r  )rI   Ú	parameterrG  s   &  rL   Ú'freeze_embeddings_and_language_adaptersÚ;XmodPreTrainedModel.freeze_embeddings_and_language_adaptersx  s¹   € ô
 	�‰Ð)Ô*ØŸ™×0Ñ0×;Ñ;Ö=ˆIØ&+ˆIÖ#ñ >ä�‰Ð'Ô(Ø—\‘\×)Ñ)×/Ô/ˆEØ�|‰|×.Ñ.Ò:Ø!&§¡×!@Ñ!@×!KÑ!KÖ!M�IØ.3�IÖ+ñ "Nà"Ÿ\™\×9Ñ9×DÑDÖF�	Ø*/�	Ö'ó Gó	 0rN   © )r%   r�   rÅ   )r|   r}   r~   r   r#   Úconfig_classÚbase_model_prefixÚsupports_gradient_checkpointingÚno_split_modulesÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_supports_attention_backendr,  r�   rÅ   Ú_can_record_outputsr>   Úno_gradrf  rr  r{  r‚   rƒ   r„   r…   s   @@rL   ra  ra  Q  sz   ù‡ € à€LØ!ÐØ&*Ð#ÚTÐØÐØ€NØÐØ"&Ðà˜ØÐ'ØÐ.ðÐð ‡]‚]ƒ_ô/ó ð/÷0ð 0÷0ò 0rN   ra  a0  
    The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of
    cross-attention is added between the self-attention layers, following the architecture described in *Attention is
    all you need*_ by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz
    Kaiser and Illia Polosukhin.

    To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set
    to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and
    `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass.

    .. _*Attention is all you need*: https://huggingface.co/papers/1706.03762
    )Úcustom_introc                   ó|   a a€ ] tR tRt oR	V 3R lltR tR t]]]	R
V3R lR ll4       4       4       t
R tRtVtV ;t# )Ú	XmodModeli‰  c                óÔ   <€ \         SV `  V4       Wn        RV n        \	        V4      V n        \        V4      V n        V'       d   \        V4      MRV n	        V P                  4        R# )z^
add_pooling_layer (bool, *optional*, defaults to `True`):
    Whether to add a pooling layer
FN)r/   r0   rJ   Úgradient_checkpointingr%   rl   rB  ry  rT  ÚpoolerÚ	post_init)rI   rJ   Úadd_pooling_layerrK   s   &&&€rL   r0   ÚXmodModel.__init__˜  sS   ø€ ô
 	‰Ñ˜Ô ØŒØ&+ˆÔ#ä(¨Ó0ˆŒÜ" 6Ó*ˆŒç,=”j Ô(À4ˆŒð 	�‰ÖrN   c                ó.   € V P                   P                  # rß   ©rl   r5   ©rI   s   &rL   Úget_input_embeddingsÚXmodModel.get_input_embeddingsª  s   € Ø�‰×.Ñ.Ð.rN   c                ó&   € WP                   n        R # rß   r’  )rI   rŠ   s   &&rL   Úset_input_embeddingsÚXmodModel.set_input_embeddings®  s   € Ø*/�‰Ö'rN   c                óö  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[ P                  R,          R
S[S[ P                  ,          R,          RS[R,          RS[S[,          RS[S[ P                  ,          S[	,          /# )rP   rQ   Nr  r‹   rF   rB   rR   rÉ   rë   r¯   rL  r�   rT   )
r>   rX   rU   rp  rV   rM  r   r   r°   r   )rY   rZ   s   "€rL   r[   ÚXmodModel.__annotate__´  s  ø€ ÷ O
ñ O
á—<‘< $Õ&ðO
ñ ×"Ñ" TÕ)ðO
ñ Ÿ™ tÕ+ð	O
ñ
 Ÿ™ tÕ+ðO
ñ —l‘l TÕ)ðO
ñ —|‘| dÕ*ðO
ñ  %Ÿ|™|¨dÕ2ðO
ñ !&§¡¨tÕ 3ðO
ñ ™e×/Ñ/Õ0°4Õ7ðO
ñ ˜$•;ðO
ñ Ñ+Õ,ðO
ñ 
‰u�|‰|Õ	ÑKÕ	KñO
rN   c                óÞ  € VRJ VRJ,          '       d   \        R4      hV P                  P                  '       d   V
e   T
MV P                  P                  p
MRp
V
'       do   V	fk   Vf   V P                  P                  '       d5   \        \        V P                  R7      \        V P                  R7      4      M\        V P                  R7      p	Ve   VP                  ^ ,          MVP                  ^ ,          pVe   VP                  MVP                  pV	e   V	P                  4       M^ pVf¯   V P                  P                  f   \        R4      h\        V P                  P                  ^ ,          P                  P                  P!                  4       4      pVP#                  V P                  P                  4      pV\$        P&                  ! WÍR7      ,          pV P)                  VVVVVR7      pV P+                  VVVVV	R7      w  r8V P                  ! V3R	VR
VRVRVRV	RV
RV/VB pV^ ,          pV P,                  e   V P-                  V4      MRp\/        VVVP0                  R7      # )úý
lang_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Indices of the language adapters that should be activated for each sample, respectively. Default: the index
    that corresponds to `self.config.default_language`.
Nz:You must specify exactly one of input_ids or inputs_embedsF)rJ   zPInput language unknown. Please call `XmodPreTrainedModel.set_default_language()`)ra   )rQ   rB   rF   rR   rS   )r‹   rë   Úembedding_outputrÉ   r¯   r  r‹   rÉ   rë   r¯   rL  rB   )rP  Úpooler_outputr¯   )r£   rJ   r¨   rL  Úis_encoder_decoderr   r   rf   ra   Úget_seq_lengthrq  rp  ry  rG  ræ   r  rÐ   r_   r>   Úonesrl   Ú_create_attention_masksr�  r   r¯   )rI   rQ   r  r‹   rF   rB   rR   rÉ   rë   r¯   rL  r�   ri   ra   rS   Úadapter_languagesÚdefault_lang_idr�  Úencoder_outputsÚsequence_outputr^  s   &&&&&&&&&&&,         rL   rm   ÚXmodModel.forward±  sa  € ð, ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZà�;‰;×!×!Ð!Ø%.Ò%:™	ÀÇÁ×@UÑ@U‰IàˆIç˜Ò0ð )Ò4¸¿¹×8V×8VÐ8Vô $¤L¸¿¹Ô$DÄlÐZ^×ZeÑZeÔFfÔgä!¨¯©Ô5ð ð ,5Ò+@�Y—_‘_ QÖ'Àm×FYÑFYÐZ[ÕF\ˆ
Ø%.Ò%:�×!Ò!À×@TÑ@TˆØETÒE` ×!?Ñ!?Ô!AÐfgÐàÒØ�{‰{×+Ñ+Ò3Ü Ð!sÓtÐtÜ $ T§\¡\×%7Ñ%7¸Õ%:×%AÑ%A×%QÑ%Q×%VÑ%VÓ%XÓ YÐØ/×5Ñ5°d·k±k×6RÑ6RÓSˆOØ&¬¯ª°JÔ)NÕNˆHàŸ?™?ØØ%Ø)Ø'Ø#9ð +ó 
Ðð 26×1MÑ1MØ)Ø#9Ø-Ø"7Ø+ð 2Nó 2
Ñ.ˆð Ÿ,š,Øñ

àð

ð *ð

ð #8ð	

ð
 $:ð

ð ,ð

ð  ð

ð &ð

ð ñ

ˆð *¨!Õ,ˆØ8<¿¹Ò8O˜Ÿ™ OÔ4ÐUYˆä;Ø-Ø'Ø+×;Ñ;ô
ð 	
rN   c                óÞ   € V P                   P                  '       d   \        V P                   VVVR 7      pM\        V P                   VVR7      pVe   \        V P                   VVVR7      pW3# ))rJ   rR   r‹   r¯   )rJ   rR   r‹   )rJ   rR   r‹   rÉ   )rJ   r¨   r   r   )rI   r‹   rë   r�  rÉ   r¯   s   &&&&&&rL   r¢  Ú!XmodModel._create_attention_masks  su   € ð �;‰;×!×!Ð!Ü/Ø—{‘{Ø.Ø-Ø /ô	‰Nô 7Ø—{‘{Ø.Ø-ôˆNð "Ò-Ü%>Ø—{‘{Ø.Ø5Ø&;ô	&Ð"ð Ð5Ð5rN   )rJ   rl   ry  rŒ  r�  )T)
NNNNNNNNNN)r|   r}   r~   r   r0   r”  r—  r!   r"   r   rm   r¢  r‚   rƒ   r„   r…   s   @@rL   rŠ  rŠ  ‰  sK   ù‡ € ÷ò$/ò0ð  ØØ÷O
ñ O
ó ó ó  ðO
÷d6ò 6rN   rŠ  zQ
    X-MOD Model with a `language modeling` head on top for CLM fine-tuning.
    c                   ót   a a€ ] tR tRt oRRRR/tV 3R ltR tR t]]	RV3R	 lR
 ll4       4       t
RtVtV ;t# )ÚXmodForCausalLMi'  úlm_head.decoder.weightú)roberta.embeddings.word_embeddings.weightúlm_head.decoder.biasúlm_head.biasc                óØ   <€ \         SV `  V4       VP                  '       g   \        P	                  R 4       \        VRR7      V n        \        V4      V n        V P                  4        R# )zLIf you want to use `XmodLMHeadModel` as a standalone, add `is_decoder=True.`F©r�  N©
r/   r0   r¨   ru  ÚwarningrŠ  rb  rg  Úlm_headrŽ  rH   s   &&€rL   r0   ÚXmodForCausalLM.__init__3  sO   ø€ Ü‰Ñ˜Ô à× × Ð Ü�N‰NÐiÔjä  ¸5ÔAˆŒÜ! &Ó)ˆŒð 	�‰ÖrN   c                ó.   € V P                   P                  # rß   ©r´  Údecoderr“  s   &rL   Úget_output_embeddingsÚ%XmodForCausalLM.get_output_embeddings@  ó   € Ø�|‰|×#Ñ#Ð#rN   c                ó&   € WP                   n        R # rß   r·  ©rI   Únew_embeddingss   &&rL   Úset_output_embeddingsÚ%XmodForCausalLM.set_output_embeddingsD  ó   € Ø-�‰ÖrN   c                óX  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[ P                  R,          R
S[ P                  R,          RS[S[S[ P                  ,          ,          R,          RS[R,          RS[S[ P                  ,          RS[S[,          RS[S[ P                  ,          S[	,          /# )rP   rQ   Nr  r‹   rF   rB   rR   rÉ   rë   Úlabelsr¯   rL  Úlogits_to_keepr�   rT   )
r>   rU   rV   r°   rM  rW   rX   r   r   r   )rY   rZ   s   "€rL   r[   ÚXmodForCausalLM.__annotate__I  sA  ø€ ÷ L
ñ L
á×#Ñ# dÕ*ðL
ñ ×"Ñ" TÕ)ðL
ñ ×)Ñ)¨DÕ0ð	L
ñ
 ×(Ñ(¨4Õ/ðL
ñ ×&Ñ&¨Õ-ðL
ñ ×(Ñ(¨4Õ/ðL
ñ  %×0Ñ0°4Õ7ðL
ñ !&× 1Ñ 1°DÕ 8ðL
ñ × Ñ  4Õ'ðL
ñ ™u¡U×%6Ñ%6Õ7Õ8¸4Õ?ðL
ñ ˜$•;ðL
ñ ™eŸl™lÕ*ðL
ñ Ñ+Õ,ðL
ñ 
‰u�|‰|Õ	Ñ@Õ	@ñL
rN   c                óÊ  € V	e   RpV P                   ! V3RVRVRVRVRVRVR	VR
V
RVRR/
VB pVP                  p\        V\        4      '       d   \	        V) R4      MTpV P                  VRVR3,          4      pRpV	e-   V P                  ! RRVRV	RV P                  P                  /VB p\        VVVP                  VP                  VP                  VP                  R7      # )a³  
lang_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Indices of the language adapters that should be activated for each sample, respectively. Default: the index
    that corresponds to `self.config.default_language`.
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in
    `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are
    ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`

Example:

```python
>>> from transformers import AutoTokenizer, XmodForCausalLM, AutoConfig
>>> import torch

>>> tokenizer = AutoTokenizer.from_pretrained("FacebookAI/xlm-roberta-base")
>>> config = AutoConfig.from_pretrained("facebook/xmod-base")
>>> config.is_decoder = True
>>> model = XmodForCausalLM.from_pretrained("facebook/xmod-base", config=config)
>>> model.set_default_language("en_XX")

>>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
>>> outputs = model(**inputs)

>>> prediction_logits = outputs.logits
```NFr  r‹   rF   rB   rR   rÉ   rë   r¯   rL  Úreturn_dictTr\  ÚlogitsrÃ  r2   )ÚlossrÈ  r¯   r®   rc  rd  r}  )rb  rP  rµ   rW   Úslicer´  Úloss_functionrJ   r2   r   r¯   r®   rc  rd  )rI   rQ   r  r‹   rF   rB   rR   rÉ   rë   rÃ  r¯   rL  rÄ  r�   Úoutputsr®   Úslice_indicesrÈ  rÉ  s   &&&&&&&&&&&&&,     rL   rm   ÚXmodForCausalLM.forwardG  sM  € ðX ÒØˆIà@DÇÂØñA
àðA
ð *ðA
ð *ð	A
ð
 &ðA
ð (ðA
ð #8ðA
ð $:ðA
ð ,ðA
ð  ðA
ð ðA
ð ñA
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒØ×%Ò%Ñp¨VÐp¸FÐpÈtÏ{É{×OeÑOeÐpÐioÑpˆDä0ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø$×5Ñ5ô
ð 	
rN   ©r´  rb  )NNNNNNNNNNNr{   ©r|   r}   r~   r   Ú_tied_weights_keysr0   r¹  r¿  r    r   rm   r‚   rƒ   r„   r…   s   @@rL   r«  r«  '  sS   ù‡ € ð 	!Ð"MØ ðÐõ
ò$ò.ð Ø÷L
ñ L
ó ó ÷L
ð L
rN   r«  c                   ót   a a€ ] tR tRt oRRRR/tV 3R ltR tR t]]	RV3R	 lR
 ll4       4       t
RtVtV ;t# )ÚXmodForMaskedLMi˜  r¬  r­  r®  r¯  c                óØ   <€ \         SV `  V4       VP                  '       d   \        P	                  R 4       \        VRR7      V n        \        V4      V n        V P                  4        R# )zkIf you want to use `XmodForMaskedLM` make sure `config.is_decoder=False` for bi-directional self-attention.Fr±  Nr²  rH   s   &&€rL   r0   ÚXmodForMaskedLM.__init__   sU   ø€ Ü‰Ñ˜Ô à××ÐÜ�N‰Nð1ôô
 ! ¸5ÔAˆŒÜ! &Ó)ˆŒð 	�‰ÖrN   c                ó.   € V P                   P                  # rß   r·  r“  s   &rL   r¹  Ú%XmodForMaskedLM.get_output_embeddings°  r»  rN   c                ó&   € WP                   n        R # rß   r·  r½  s   &&rL   r¿  Ú%XmodForMaskedLM.set_output_embeddings´  rÁ  rN   c                óÒ  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[ P                  R,          R
S[ P                  R,          RS[S[,          RS[S[ P                  ,          S[,          /# )rP   rQ   Nr  r‹   rF   rB   rR   rÉ   rë   rÃ  r�   rT   )r>   rU   rV   r   r   r°   rX   r   )rY   rZ   s   "€rL   r[   ÚXmodForMaskedLM.__annotate__¹  sð   ø€ ÷ /
ñ /
á×#Ñ# dÕ*ð/
ñ ×"Ñ" TÕ)ð/
ñ ×)Ñ)¨DÕ0ð	/
ñ
 ×(Ñ(¨4Õ/ð/
ñ ×&Ñ&¨Õ-ð/
ñ ×(Ñ(¨4Õ/ð/
ñ  %×0Ñ0°4Õ7ð/
ñ !&× 1Ñ 1°DÕ 8ð/
ñ × Ñ  4Õ'ð/
ñ Ñ+Õ,ð/
ñ 
‰u�|‰|Õ	™~Õ	-ñ/
rN   c
                óZ  € V P                   ! V3RVRVRVRVRVRVRVRR	/V
B pV^ ,          pV P                  V4      pR
pV	eG   \        4       pV! VP                  RV P                  P
                  4      V	P                  R4      4      p\        VVVP                  VP                  R7      # )av  
lang_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Indices of the language adapters that should be activated for each sample, respectively. Default: the index
    that corresponds to `self.config.default_language`.
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
    config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
    loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
r  r‹   rF   rB   rR   rÉ   rë   rÇ  TN©rÉ  rÈ  r®   rc  r.   )	rb  r´  r   r´   rJ   r2   r   r®   rc  )rI   rQ   r  r‹   rF   rB   rR   rÉ   rë   rÃ  r�   rÌ  r¦  Úprediction_scoresÚmasked_lm_lossÚloss_fcts   &&&&&&&&&&,     rL   rm   ÚXmodForMaskedLM.forward·  sð   € ð0 —,’,Øñ
àð
ð *ð
ð *ð	
ð
 &ð
ð (ð
ð #8ð
ð $:ð
ð ð
ð ñ
ˆð " !�*ˆØ ŸL™L¨Ó9ÐàˆØÒÜ'Ó)ˆHÙ%Ð&7×&<Ñ&<¸RÀÇÁ×AWÑAWÓ&XÐZ`×ZeÑZeÐfhÓZiÓjˆNäØØ$Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
rN   rÏ  )	NNNNNNNNNrÐ  r…   s   @@rL   rÓ  rÓ  ˜  sO   ù‡ € ð 	!Ð"MØ ðÐõò $ò.ð Ø÷/
ñ /
ó ó ÷/
ð /
rN   rÓ  c                   ó<   a a€ ] tR tRt oRtV 3R ltR tRtVtV ;t	# )rg  iì  z*Roberta Head for masked language modeling.c                ó¸  <€ \         SV `  4        \        P                  ! VP                  VP                  4      V n        \        P                  ! VP                  VP                  R 7      V n        \        P                  ! VP                  VP                  4      V n
        \        P                  ! \        P                  ! VP                  4      4      V n        R# r×   )r/   r0   r   r¦   r3   rØ   r8   r9   Ú
layer_normr2   r¸  Ú	Parameterr>   rC   rj  rH   s   &&€rL   r0   ÚXmodLMHead.__init__ï  s‡   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ,š, v×'9Ñ'9¸v×?TÑ?TÔUˆŒä—y’y ×!3Ñ!3°V×5FÑ5FÓGˆŒÜ—L’L¤§¢¨V×->Ñ->Ó!?Ó@ˆŽ	rN   c                ó‚   € V P                  V4      p\        V4      pV P                  V4      pV P                  V4      pV# rß   )rØ   r	   rä  r¸  ©rI   Úfeaturesr�   Úxs   &&, rL   rm   ÚXmodLMHead.forward÷  s;   € Ø�J‰J�xÓ ˆÜ�‹GˆØ�O‰O˜AÓˆð �L‰L˜‹OˆàˆrN   )rj  r¸  rØ   rä  ©
r|   r}   r~   r   r€   r0   rm   r‚   rƒ   r„   r…   s   @@rL   rg  rg  ì  s   ù‡ € Ù4õA÷ò rN   rg  z�
    X-MOD Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled
    output) e.g. for GLUE tasks.
    c                   ó\   a a€ ] tR tRt oV 3R lt]]RV3R lR ll4       4       tRtVt	V ;t
# )ÚXmodForSequenceClassificationi  c                ó¸   <€ \         SV `  V4       VP                  V n        Wn        \	        VR R7      V n        \        V4      V n        V P                  4        R# ©Fr±  N)	r/   r0   Ú
num_labelsrJ   rŠ  rb  ÚXmodClassificationHeadÚ
classifierrŽ  rH   s   &&€rL   r0   Ú&XmodForSequenceClassification.__init__
  sH   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒØŒä  ¸5ÔAˆŒÜ0°Ó8ˆŒð 	�‰ÖrN   c                ó‚  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[S[,          R
S[S[ P                  ,          S[,          /	# ©rP   rQ   Nr  r‹   rF   rB   rR   rÃ  r�   rT   )r>   rU   rV   r   r   r°   rX   r   )rY   rZ   s   "€rL   r[   Ú*XmodForSequenceClassification.__annotate__  sÇ   ø€ ÷ =
ñ =
á×#Ñ# dÕ*ð=
ñ ×"Ñ" TÕ)ð=
ñ ×)Ñ)¨DÕ0ð	=
ñ
 ×(Ñ(¨4Õ/ð=
ñ ×&Ñ&¨Õ-ð=
ñ ×(Ñ(¨4Õ/ð=
ñ × Ñ  4Õ'ð=
ñ Ñ+Õ,ð=
ñ 
‰u�|‰|Õ	Ñ7Õ	7ñ=
rN   c                óö  € V P                   ! V3RVRVRVRVRVRR/VB p	V	^ ,          p
V P                  V
4      pRpVEe˜   V P                  P                  f–   V P                  ^8X  d   R	V P                  n        MrV P                  ^8”  dQ   VP
                  \        P                  8X  g    VP
                  \        P                  8X  d   R
V P                  n        MRV P                  n        V P                  P                  R	8X  dK   \        4       pV P                  ^8X  d'   V! VP                  4       VP                  4       4      pMŽV! W·4      pM…V P                  P                  R
8X  d>   \        4       pV! VP                  RV P                  4      VP                  R4      4      pM-V P                  P                  R8X  d   \        4       pV! W·4      p\        VVV	P                  V	P                   R7      # )aa  
lang_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Indices of the language adapters that should be activated for each sample, respectively. Default: the index
    that corresponds to `self.config.default_language`.
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
    `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
r  r‹   rF   rB   rR   rÇ  TNÚ
regressionÚsingle_label_classificationÚmulti_label_classificationrÝ  r.   )rb  ró  rJ   Úproblem_typerñ  r+   r>   rE   rW   r   Úsqueezer   r´   r   r   r®   rc  ©rI   rQ   r  r‹   rF   rB   rR   rÃ  r�   rÌ  r¦  rÈ  rÉ  rà  s   &&&&&&&&,     rL   rm   Ú%XmodForSequenceClassification.forward  s»  € ð, —,’,Øñ	
àð	
ð *ð	
ð *ð		
ð
 &ð	
ð (ð	
ð ð	
ð ñ	
ˆð " !�*ˆØ—‘ Ó1ˆàˆØÓØ�{‰{×'Ñ'Ò/Ø—?‘? aÔ'Ø/;�D—K‘KÕ,Ø—_‘_ qÔ(¨f¯l©l¼e¿j¹jÔ.HÈFÏLÉLÔ\a×\eÑ\eÔLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ô7Ü"›9�Ø—?‘? aÔ'Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# FÓ3‘DØ—‘×)Ñ)Ð-JÔJÜ+Ó-�Ù §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR‘Ø—‘×)Ñ)Ð-IÔIÜ,Ó.�Ù Ó/�ä'ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
rN   )ró  rJ   rñ  rb  ©NNNNNNN©r|   r}   r~   r   r0   r    r   rm   r‚   rƒ   r„   r…   s   @@rL   rî  rî    s0   ù‡ € õ	ð Ø÷=
ñ =
ó ó ÷=
ð =
rN   rî  c                   ó\   a a€ ] tR tRt oV 3R lt]]RV3R lR ll4       4       tRtVt	V ;t
# )ÚXmodForMultipleChoiceiW  c                óü   <€ \         SV `  V4       \        V4      V n        \        P
                  ! VP                  4      V n        \        P                  ! VP                  ^4      V n
        V P                  4        R# )r-   N)r/   r0   rŠ  rb  r   r:   r;   r<   r¦   r3   ró  rŽ  rH   s   &&€rL   r0   ÚXmodForMultipleChoice.__init__Z  sV   ø€ Ü‰Ñ˜Ô ä  Ó(ˆŒÜ—z’z &×"<Ñ"<Ó=ˆŒÜŸ)š) F×$6Ñ$6¸Ó:ˆŒð 	�‰ÖrN   c                ó‚  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[S[,          R
S[S[ P                  ,          S[,          /	# )rP   rQ   Nr  rF   r‹   rÃ  rB   rR   r�   rT   )r>   rU   rV   r   r   r°   rX   r   )rY   rZ   s   "€rL   r[   Ú"XmodForMultipleChoice.__annotate__f  sÒ   ø€ ÷ S
ñ S
á×#Ñ# dÕ*ðS
ñ ×"Ñ" TÕ)ðS
ñ ×(Ñ(¨4Õ/ð	S
ñ
 ×)Ñ)¨DÕ0ðS
ñ × Ñ  4Õ'ðS
ñ ×&Ñ&¨Õ-ðS
ñ ×(Ñ(¨4Õ/ðS
ñ Ñ+Õ,ðS
ñ 
‰u�|‰|Õ	Ñ8Õ	8ñS
rN   c                ó¤  € Ve   VP                   ^,          MVP                   ^,          p	Ve"   VP                  R
VP                  R
4      4      MRp
Ve7   VP                  VP                  ^ 4      VP                  ^4      ,          4      MRpVe"   VP                  R
VP                  R
4      4      MRpVe"   VP                  R
VP                  R
4      4      MRpVe"   VP                  R
VP                  R
4      4      MRpVe2   VP                  R
VP                  R4      VP                  R
4      4      MRpV P                  ! V
3RVRVRVRVRVRR/VB pV^,          pV P                  V4      pV P                  V4      pVP                  R
V	4      pRpVe   \        4       pV! VV4      p\        VVVP                  VP                  R	7      # )a¤  
input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
    Indices of input sequence tokens in the vocabulary.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are input IDs?](../glossary#input-ids)
lang_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
    Indices of the language adapters that should be activated for each sample, respectively. Default: the index
    that corresponds to `self.config.default_language`.
token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
    Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
    1]`:

    - 0 corresponds to a *sentence A* token,
    - 1 corresponds to a *sentence B* token.

    [What are token type IDs?](../glossary#token-type-ids)
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
    num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
    `input_ids` above)
position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
    Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
    config.max_position_embeddings - 1]`.

    [What are position IDs?](../glossary#position-ids)
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):
    Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
    is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
    model's internal embedding lookup matrix.
Nr  rB   rF   r‹   rR   rÇ  TrÝ  r.   éþÿÿÿ)rf   r´   rD   Úrepeatrb  r<   ró  r   r   r®   rc  )rI   rQ   r  rF   r‹   rÃ  rB   rR   r�   Únum_choicesÚflat_input_idsÚflat_lang_idsÚflat_position_idsÚflat_token_type_idsÚflat_attention_maskÚflat_inputs_embedsrÌ  r^  rÈ  Úreshaped_logitsrÉ  rà  s   &&&&&&&&,             rL   rm   ÚXmodForMultipleChoice.forwardd  sò  € ð\ -6Ò,A�i—o‘o aÖ(À}×GZÑGZÐ[\ÕG]ˆàCLÒCX˜Ÿ™¨¨I¯N©N¸2Ó,>Ô?Ð^bˆØRZÒRf˜Ÿ™¨	¯©°qÓ(9¸I¿N¹NÈ1Ó<MÕ(MÔNÐlpˆØLXÒLd˜L×-Ñ-¨b°,×2CÑ2CÀBÓ2GÔHÐjnÐØR`ÒRl˜n×1Ñ1°"°n×6IÑ6IÈ"Ó6MÔNÐrvÐØR`ÒRl˜n×1Ñ1°"°n×6IÑ6IÈ"Ó6MÔNÐrvÐð Ò(ð ×Ñ˜r =×#5Ñ#5°bÓ#9¸=×;MÑ;MÈbÓ;QÔRàð 	ð —,’,Øñ	
à"ð	
ð +ð	
ð /ð		
ð
 /ð	
ð -ð	
ð ð	
ð ñ	
ˆð   �
ˆàŸ™ ]Ó3ˆØ—‘ Ó/ˆØ Ÿ+™+ b¨+Ó6ˆàˆØÒÜ'Ó)ˆHÙ˜O¨VÓ4ˆDä(ØØ"Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
rN   )ró  r<   rb  r   r  r…   s   @@rL   r  r  W  s4   ù‡ € õð Ø÷S
ñ S
ó ó ÷S
ð S
rN   r  c                   ó\   a a€ ] tR tRt oV 3R lt]]RV3R lR ll4       4       tRtVt	V ;t
# )ÚXmodForTokenClassificationi¼  c                ón  <€ \         SV `  V4       VP                  V n        \        VR R7      V n        VP
                  e   VP
                  MVP                  p\        P                  ! V4      V n	        \        P                  ! VP                  VP                  4      V n        V P                  4        R# rð  )r/   r0   rñ  rŠ  rb  Úclassifier_dropoutr;   r   r:   r<   r¦   r3   ró  rŽ  ©rI   rJ   r  rK   s   && €rL   r0   Ú#XmodForTokenClassification.__init__¿  sŠ   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä  ¸5ÔAˆŒà)/×)BÑ)BÒ)NˆF×%Ò%ÐTZ×TnÑTnð 	ô —z’zÐ"4Ó5ˆŒÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰ÖrN   c                ó‚  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[S[,          R
S[S[ P                  ,          S[,          /	# rö  )r>   rU   rV   r   r   r°   rX   r   )rY   rZ   s   "€rL   r[   Ú'XmodForTokenClassification.__annotate__Ï  sÇ   ø€ ÷ ,
ñ ,
á×#Ñ# dÕ*ð,
ñ ×"Ñ" TÕ)ð,
ñ ×)Ñ)¨DÕ0ð	,
ñ
 ×(Ñ(¨4Õ/ð,
ñ ×&Ñ&¨Õ-ð,
ñ ×(Ñ(¨4Õ/ð,
ñ × Ñ  4Õ'ð,
ñ Ñ+Õ,ð,
ñ 
‰u�|‰|Õ	Ñ4Õ	4ñ,
rN   c                ó`  € V P                   ! V3RVRVRVRVRVRR/VB p	V	^ ,          p
V P                  V
4      p
V P                  V
4      pRpVe=   \        4       pV! VP	                  R
V P
                  4      VP	                  R
4      4      p\        VVV	P                  V	P                  R	7      # )a¿  
lang_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Indices of the language adapters that should be activated for each sample, respectively. Default: the index
    that corresponds to `self.config.default_language`.
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
r  r‹   rF   rB   rR   rÇ  TNrÝ  r.   )	rb  r<   ró  r   r´   rñ  r   r®   rc  rþ  s   &&&&&&&&,     rL   rm   Ú"XmodForTokenClassification.forwardÍ  sÛ   € ð( —,’,Øñ	
àð	
ð *ð	
ð *ð		
ð
 &ð	
ð (ð	
ð ð	
ð ñ	
ˆð " !�*ˆàŸ,™, Ó7ˆØ—‘ Ó1ˆàˆØÒÜ'Ó)ˆHÙ˜FŸK™K¨¨D¯O©OÓ<¸f¿k¹kÈ"»oÓNˆDä$ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
rN   )ró  r<   rñ  rb  r   r  r…   s   @@rL   r  r  ¼  s0   ù‡ € õð Ø÷,
ñ ,
ó ó ÷,
ð ,
rN   r  c                   ó<   a a€ ] tR tRt oRtV 3R ltR tRtVtV ;t	# )rò  iÿ  z-Head for sentence-level classification tasks.c                óf  <€ \         SV `  4        \        P                  ! VP                  VP                  4      V n        VP                  e   VP                  MVP                  p\        P                  ! V4      V n	        \        P                  ! VP                  VP                  4      V n        R # rß   )r/   r0   r   r¦   r3   rØ   r  r;   r:   r<   rñ  Úout_projr  s   && €rL   r0   ÚXmodClassificationHead.__init__  s   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
à)/×)BÑ)BÒ)NˆF×%Ò%ÐTZ×TnÑTnð 	ô —z’zÐ"4Ó5ˆŒÜŸ	š	 &×"4Ñ"4°f×6GÑ6GÓHˆŽrN   c                óÌ   € VR,          pV P                  V4      pV P                  V4      p\        P                  ! V4      pV P                  V4      pV P	                  V4      pV# )r\  )r\  r{   r\  )r<   rØ   r>   Útanhr   rè  s   &&, rL   rm   ÚXmodClassificationHead.forward  sS   € Ø�WÕˆØ�L‰L˜‹OˆØ�J‰J�q‹MˆÜ�JŠJ�q‹MˆØ�L‰L˜‹OˆØ�M‰M˜!ÓˆØˆrN   )rØ   r<   r   rì  r…   s   @@rL   rò  rò  ÿ  s   ù‡ € Ù7õI÷ò rN   rò  c                   ó\   a a€ ] tR tRt oV 3R lt]]RV3R lR ll4       4       tRtVt	V ;t
# )ÚXmodForQuestionAnsweringi  c                óì   <€ \         SV `  V4       VP                  V n        \        VR R7      V n        \
        P                  ! VP                  VP                  4      V n        V P                  4        R# rð  )
r/   r0   rñ  rŠ  rb  r   r¦   r3   Ú
qa_outputsrŽ  rH   s   &&€rL   r0   Ú!XmodForQuestionAnswering.__init__  sU   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä  ¸5ÔAˆŒÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰ÖrN   c                óª  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[ P                  R,          R
S[S[,          RS[S[ P                  ,          S[,          /
# )rP   rQ   Nr  r‹   rF   rB   rR   Ústart_positionsÚend_positionsr�   rT   )r>   rU   rV   r   r   r°   rX   r   )rY   rZ   s   "€rL   r[   Ú%XmodForQuestionAnswering.__annotate__$  sÜ   ø€ ÷ :
ñ :
á×#Ñ# dÕ*ð:
ñ ×"Ñ" TÕ)ð:
ñ ×)Ñ)¨DÕ0ð	:
ñ
 ×(Ñ(¨4Õ/ð:
ñ ×&Ñ&¨Õ-ð:
ñ ×(Ñ(¨4Õ/ð:
ñ ×)Ñ)¨DÕ0ð:
ñ ×'Ñ'¨$Õ.ð:
ñ Ñ+Õ,ð:
ñ 
‰u�|‰|Õ	Ñ;Õ	;ñ:
rN   c	                óô  € V P                   ! V3RVRVRVRVRVRR/V	B p
V
^ ,          pV P                  V4      pVP                  ^RR7      w  rÞVP                  R4      P	                  4       pVP                  R4      P	                  4       pR	pVeÄ   VeÀ   \        VP                  4       4      ^8”  d   VP                  R4      p\        VP                  4       4      ^8”  d   VP                  R4      pVP                  ^4      pVP                  ^ V4      pVP                  ^ V4      p\        VR
7      pV! W×4      pV! Wè4      pVV,           ^,          p\        VVVV
P                  V
P                  R7      # )rœ  r  r‹   rF   rB   rR   rÇ  Trt   N)Úignore_index)rÉ  Ústart_logitsÚ
end_logitsr®   rc  r.   )rb  r(  Úsplitrý  r˜   ÚlenrD   Úclampr   r   r®   rc  )rI   rQ   r  r‹   rF   rB   rR   r+  r,  r�   rÌ  r¦  rÈ  r0  r1  Ú
total_lossÚignored_indexrà  Ú
start_lossÚend_losss   &&&&&&&&&,          rL   rm   Ú XmodForQuestionAnswering.forward"  sž  € ð& —,’,Øñ	
àð	
ð *ð	
ð *ð		
ð
 &ð	
ð (ð	
ð ð	
ð ñ	
ˆð " !�*ˆà—‘ Ó1ˆØ#)§<¡<°°r <Ó#:Ñ ˆØ#×+Ñ+¨BÓ/×:Ñ:Ó<ˆØ×'Ñ'¨Ó+×6Ñ6Ó8ˆ
àˆ
ØÒ&¨=Ò+Dä�?×'Ñ'Ó)Ó*¨QÔ.Ø"1×"9Ñ"9¸"Ó"=�Ü�=×%Ñ%Ó'Ó(¨1Ô,Ø -× 5Ñ 5°bÓ 9�à(×-Ñ-¨aÓ0ˆMØ-×3Ñ3°A°}ÓEˆOØ)×/Ñ/°°=ÓAˆMä'°]ÔCˆHÙ! ,Ó@ˆJÙ 
Ó:ˆHØ$ xÕ/°1Õ4ˆJä+ØØ%Ø!Ø!×/Ñ/Ø×)Ñ)ô
ð 	
rN   )rñ  r(  rb  )NNNNNNNNr  r…   s   @@rL   r&  r&    s0   ù‡ € õð Ø÷:
ñ :
ó ó ÷:
ð :
rN   r&  )Nr³   )r«  rÓ  r  r&  rî  r  rŠ  ra  )Pr€   Úcollections.abcr   r>   r   Útorch.nnr   r   r   Ú r   rh  Úactivationsr   r	   Úcache_utilsr
   r   r   Ú
generationr   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úpytorch_utilsr   Úutilsr   r   r   Úutils.genericr    r!   Úutils.output_capturingr"   Úconfiguration_xmodr#   Ú
get_loggerr|   ru  rŽ   r%   r›   r�   rÅ   rÕ   râ   rô   r  r  r,  rB  rT  ra  rŠ  r«  rÓ  rg  rî  r  r  rò  r&  Ú__all__r}  rN   rL   Ú<module>rL     sk  ðñ å $ã Ý ß AÑ Aå &ß 'ß CÑ CÝ )ß JÝ 9÷	÷ 	ó 	÷ GÝ &Ý 6ß @Ñ @ß IÝ 5Ý *ð 
×	Ò	˜HÓ	%€ôc8�R—Y‘Yô c8÷N%ô:@)˜Ÿ	™	ô @)ôHI)˜Ÿ™ô I)ôX�R—Y‘Yô ô$.�B—I‘Iô $.ôP�r—y‘yô ô�"—)‘)ô ô$,�—‘ô ,ô^H3Ð*ô H3ôV%
�"—)‘)ô %
ôR�—‘ô ð ô40˜/ó 40ó ð40ñn ðôôM6Ð#ó M6óðM6ñ` ðôô
i
Ð)¨?ó i
óð
i
ðX ôO
Ð)ó O
ó ðO
ôf�—‘ô ñ, ðôôL
Ð$7ó L
óðL
ð^ ôa
Ð/ó a
ó ða
ðH ô>
Ð!4ó >
ó ð>
ôD˜RŸY™Yô ð, ôH
Ð2ó H
ó ðH
òV	‚rN   