+
    œ°³j@«  ã                   ó^  € R t ^ RIt^ RIHt ^ RIt^ RIHt ^ RIHtHtH	t	 ^RI
Ht ^RIHtHt ^RIHtHt ^RIHt ^R	IHtHtHtHtHt ^R
IHt ^RIHtHt ^RI H!t! ^RI"H#t#H$t$H%t%H&t& ^RI'H(t( ^RI)H*t*H+t+ ^RI,H-t-H.t. ^RI/H0t0 ]&Pb                  ! ]24      t3R t4]! R4      RAR l4       t5R t6R t7R t8 ! R R]Pr                  4      t: ! R R]Pr                  4      t; ! R R]Pr                  4      t<RBR R llt=]! ]54       ! R  R!]Pr                  4      4       t> ! R" R#]Pr                  4      t? ! R$ R%]Pr                  4      t@ ! R& R']Pr                  4      tA ! R( R)]Pr                  4      tB ! R* R+]4      tC ! R, R-]Pr                  4      tD ! R. R/]Pr                  4      tE]$ ! R0 R1]4      4       tF]$ ! R2 R3]F4      4       tG]$ ! R4 R5]F4      4       tH ! R6 R7]Pr                  4      tI]$! R8R97       ! R: R;]F4      4       tJ]$ ! R< R=]F4      4       tK ! R> R?]Pr                  4      tLR@ tM. RCOtNR# )DzPyTorch ESM model.N)ÚCallable)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELoss)Úinitialization)Úuse_kernel_forward_from_hubÚuse_kernelized_func)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚGradientCheckpointingLayer)Ú"BaseModelOutputWithCrossAttentionsÚ,BaseModelOutputWithPoolingAndCrossAttentionsÚMaskedLMOutputÚSequenceClassifierOutputÚTokenClassifierOutput)Údynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputs©Ú	EsmConfigc                óÌ   € V RRV P                   R,          ^,          13,          pV RV P                   R,          ^,          R13,          p\        P                  ! V) V3RR7      # )z*Rotates half the hidden dims of the input..N©Údiméÿÿÿÿ)ÚshapeÚtorchÚcat)ÚxÚx1Úx2s   &  Úe/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/esm/modeling_esm.pyÚrotate_halfr,   0   sZ   € à	
ˆ3Ð"�!—'‘'˜"•+ Õ"Ð"Ð"Õ	#€BØ	
ˆ3�—‘˜•˜qÕ Ñ"Ð"Õ	#€BÜ�9Š9�r�c˜2�Y BÔ'Ð'ó    Úrotary_pos_embc                ó’  € V P                   pVP                  V4      pVP                  V4      pV P                  4       V,          \        V P                  4       4      V,          ,           pVP                  4       V,          \        VP                  4       4      V,          ,           pVP	                  V4      VP	                  V4      3# )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)ÚdtypeÚ	unsqueezeÚfloatr,   Úto)ÚqÚkÚcosÚsinÚunsqueeze_dimÚoriginal_dtypeÚq_embedÚk_embeds   &&&&&   r+   Úapply_rotary_pos_embr<   7   sŽ   € ð& —W‘W€NØ
�-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�w‰w‹y˜3�¤;¨q¯w©w«yÓ#9¸CÕ#?Õ@€GØ�w‰w‹y˜3�¤;¨q¯w©w«yÓ#9¸CÕ#?Õ@€GØ�:‰:�nÓ% w§z¡z°.Ó'AÐAÐAr-   c                óŽ   € V R,          R\         P                  ! V \        P                  ! R4      ,          4      ,           ,          # )zg
This is the gelu implementation from the original ESM repo. Using F.gelu yields subtly wrong results.
g      à?ç      ð?g       @)r&   ÚerfÚmathÚsqrt©r(   s   &r+   ÚgelurC   R   s.   € ð ˆs�7�cœEŸIšI a¬$¯)ª)°C«.Õ&8Ó9Õ9Õ:Ð:r-   c                ó2   € W P                  RR4      ,           # )zJMake layer symmetric in final two dimensions, used for contact prediction.r$   éþÿÿÿ)Ú	transposerB   s   &r+   Ú
symmetrizerG   Y   s   € à�{‰{˜2˜rÓ"Õ"Ð"r-   c                óº   € V P                  RRR7      pV P                  RRR7      pV P                  RRR7      pW,          pVP                  V4       W,
          pV# )z=Perform average product correct, used for contact prediction.T)Úkeepdimsr$   rE   )r$   rE   )ÚsumÚdiv_)r(   Úa1Úa2Úa12ÚavgÚ
normalizeds   &     r+   Úaverage_product_correctrQ   ^   sW   € à	
�‰ˆr˜DˆÓ	!€BØ	
�‰ˆr˜DˆÓ	!€BØ
�%‰%� 4ˆ%Ó
(€Cà
�'€CØ‡H�HˆS„MØ•€JØÐr-   c                   óÖ   a a€ ] tR t^jt oRt]! RRR7      RV3R lV 3R lll4       t]]! RRR7      RV3R lR ll4       4       t]	P                  ! 4       ]RR	 l4       4       tR
tVtV ;t# )ÚEsmRotaryEmbeddingzî
Rotary position embeddings.
Implementation based on [ModernBERT's RotaryEmbedding](https://github.com/huggingface/transformers/blob/aad13b87ed59f2afcfaebc985f403301887a35fc/src/transformers/models/modernbert/modeling_modernbert.py#L94).
Údevicez5.18)Úversionc                ó    <€ V ^8„  d   QhRS[ /# ©é   Úconfigr   )ÚformatÚ__classdict__s   "€r+   Ú__annotate__ÚEsmRotaryEmbedding.__annotate__q   s   ø€ ÷ Cñ C™yñ Cr-   c                óÈ   <€ \         SV `  4        Wn        / V n        V P	                  V P                  4      w  r4\
        P                  ! V4      V n        \        V R V4       R# )Úattention_scalingN)	ÚsuperÚ__init__rY   Ú	rope_typeÚcompute_default_rope_parametersr   ÚBufferÚinv_freqÚsetattr)ÚselfrY   rT   Úcurr_inv_freqÚcurr_attention_scalingÚ	__class__s   &&&  €r+   ra   ÚEsmRotaryEmbedding.__init__p   sR   ø€ ä‰ÑÔàŒØˆŒà04×0TÑ0TÐUY×U`ÑU`Ó0aÑ-ˆÜŸ	š	 -Ó0ˆŒÜ�Ð)Ð+AÖBr-   c                óP   <€ V ^8„  d   QhRS[ RS[S[P                  S[3,          /# )rX   rY   Úreturn)r    Útupler&   ÚTensorr2   )rZ   r[   s   "€r+   r\   r]   }   s.   ø€ ÷ 5ñ 5±	ð 5ÑUZÑ[`×[gÑ[gÑinÐ[nÕUoñ 5r-   c           	     ó"  € V P                   p\        V RR4      ;'       g    V P                  V P                  ,          pRpRV\        P
                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.

Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Úhead_dimNr>   ©r0   )Ú
rope_thetaÚgetattrÚhidden_sizeÚnum_attention_headsr&   Úaranger2   r3   )rY   rT   ÚkwargsÚbaser#   Úattention_factorre   s   &&,    r+   rc   Ú2EsmRotaryEmbedding.compute_default_rope_parameters{   sy   € ð × Ñ ˆÜ�f˜j¨$Ó/×cÐc°6×3EÑ3EÈ×IcÑIcÕ3cˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r-   c                óˆ  € \        V R 4      p\        V R4      pVR	,          P                  4       P                  VP                  ^ ,          R
^4      P	                  VP
                  4      pVR,          P                  4       p\        VP
                  P                  \        4      '       d2   VP
                  P                  R8w  d   VP
                  P                  MRp\        VRR7      ;_uu_ 4        VP                  4       VP                  4       ,          P                  ^^4      p	\        P                  ! W™3R
R7      p
V
P                  4       V,          pV
P                  4       V,          pRRR4       XP	                  VP                  R7      XP	                  VP                  R7      3#   + '       g   i     LH; i)re   r_   NÚmpsÚcpuF)Údevice_typeÚenabledr"   rr   )NºNNNNr$   )r�   Nr�   )rt   r2   Úexpandr%   r3   rT   Ú
isinstanceÚtypeÚstrr   rF   r&   r'   r6   r7   r0   )rg   r(   Úposition_idsÚ
layer_typere   r_   Úinv_freq_expandedÚposition_ids_expandedr   ÚfreqsÚembr6   r7   s   &&&&         r+   ÚforwardÚEsmRotaryEmbedding.forward�   sQ  € ô ˜4 Ó,ˆÜ# DÐ*=Ó>Ðà$ ]Õ3×9Ñ9Ó;×BÑBÀ<×CUÑCUÐVWÕCXÐZ\Ð^_Ó`×cÑcÐde×dlÑdlÓmÐØ ,¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆÜ¨¸U×CÖCØ&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÕN×YÑYÐZ[Ð]^Ó_ˆEÜ—)’)˜U˜N°Ô3ˆCØ—'‘'“)Ð/Õ/ˆCØ—'‘'“)Ð/Õ/ˆC÷	 Dð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷ D×Cús   Ã4A=F1Æ1G	)rY   re   rb   ©N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r   ra   Ústaticmethodrc   r&   Úno_gradr   rŒ   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©rj   r[   s   @@r+   rS   rS   j   s|   ù‡ € ññ
 �X vÔ.÷Cô Có /ðCð Ù�X vÔ.÷5ñ 5ó /ó ð5ð& ‡]‚]ƒ_Øó<ó ó ÷<ð <r-   rS   c                   óL   a a€ ] tR t^£t oRtRV3R lV 3R llltR tRtVtV ;t	# )ÚEsmContactPredictionHeadzWPerforms symmetrization, apc, and computes a logistic regression on the output featuresc                ó&   <€ V ^8„  d   QhRS[ RS[ /# )rX   Úin_featuresÚeos_idx)Úint)rZ   r[   s   "€r+   r\   Ú%EsmContactPredictionHead.__annotate__¦   s   ø€ ÷ 
'ñ 
'áð
'ñ ñ	
'r-   c                óª   <€ \         SV `  4        Wn        W0n        \        P
                  ! V^V4      V n        \        P                  ! 4       V n        R# )é   N)	r`   ra   r�   rž   r   ÚLinearÚ
regressionÚSigmoidÚ
activation)rg   r�   Úbiasrž   rj   s   &&&&€r+   ra   Ú!EsmContactPredictionHead.__init__¦   s<   ø€ ô 	‰ÑÔØ&ÔØŒÜŸ)š) K°°DÓ9ˆŒÜŸ*š*›,ˆŽr-   c                óZ  € VP                  V P                  4      P                  V4      pVP                  ^4      VP                  ^4      ,          pW#R,          ,          pVRRR1RR13,          pVR,          pVP	                  4       w  rErgpVP                  WEV,          Ww4      pVP                  V P                  P                  P                  4      p\        \        V4      4      pVP                  ^ ^^^4      pV P                  V P                  V4      P                  ^4      4      # )r¢   N.)r�   NNr�   r�   r$   ).ºr¢   NNrª   )Únerž   r3   r1   ÚsizeÚviewr¤   ÚweightrT   rQ   rG   Úpermuter¦   Úsqueeze)	rg   ÚtokensÚ
attentionsÚeos_maskÚ
batch_sizeÚlayersÚheadsÚseqlenÚ_s	   &&&      r+   rŒ   Ú EsmContactPredictionHead.forward²   s  € à—9‘9˜TŸ\™\Ó*×-Ñ-¨jÓ9ˆØ×%Ñ% aÓ(¨8×+=Ñ+=¸aÓ+@Õ@ˆØÐ+>Õ"?Õ?ˆ
Ø  S b S¨#¨2¨# Õ.ˆ
à Õ,ˆ
Ø/9¯©Ó/@Ñ,ˆ
˜E¨1Ø—_‘_ Z¸%µÀÓPˆ
ð  —]‘]Ø�O‰O×"Ñ"×)Ñ)ó
ˆ
ô -¬Z¸
Ó-CÓDˆ
Ø×'Ñ'¨¨1¨a°Ó3ˆ
Ø�‰˜tŸ™¨zÓ:×BÑBÀ1ÓEÓFÐFr-   )r¦   rž   r�   r¤   )TrX   ©
r�   r�   r‘   r’   r“   ra   rŒ   r–   r—   r˜   r™   s   @@r+   r›   r›   £   s   ù‡ € Ùa÷
'õ 
'÷Gò Gr-   r›   c                   óF   a a€ ] tR t^Æt oRtV 3R ltRR ltR tRtVt	V ;t
# )ÚEsmEmbeddingszN
Same as BertEmbeddings with a tiny tweak for positional embeddings indexing.
c                ó4  <€ \         SV `  4        \        P                  ! VP                  VP
                  VP                  R 7      V n        VP                  '       d3   \        P                  ! VP
                  VP                  R7      V n        MRV n        \        P                  ! VP                  4      V n        \        VRR4      V n        \        P"                  ! \$        P&                  ! VP(                  4      P+                  R4      RR7      V n        VP                  V n        V P                   R8X  d=   \        P                  ! VP(                  VP
                  V P.                  R 7      V n        VP2                  V n        VP4                  V n        R# ))Úpadding_idx©ÚepsNÚposition_embedding_typeÚabsoluteF)Ú
persistent©r¢   r$   )r`   ra   r   Ú	EmbeddingÚ
vocab_sizeru   Úpad_token_idÚword_embeddingsÚemb_layer_norm_beforeÚ	LayerNormÚlayer_norm_epsÚ
layer_normÚDropoutÚhidden_dropout_probÚdropoutrt   rÁ   rd   r&   rw   Úmax_position_embeddingsr‚   r†   r¾   Úposition_embeddingsÚtoken_dropoutÚmask_token_id©rg   rY   rj   s   &&€r+   ra   ÚEsmEmbeddings.__init__Ë   s$  ø€ Ü‰ÑÔÜ!Ÿ|š|¨F×,=Ñ,=¸v×?QÑ?QÐ_e×_rÑ_rÔsˆÔà×'×'Ð'Ü Ÿlšl¨6×+=Ñ+=À6×CXÑCXÔYˆD�Oà"ˆDŒOÜ—z’z &×"<Ñ"<Ó=ˆŒä'.¨vÐ7PÐR\Ó']ˆÔ$ÜŸIšI¤e§l¢l°6×3QÑ3QÓ&R×&YÑ&YÐZaÓ&bÐotÔuˆÔà!×.Ñ.ˆÔØ×'Ñ'¨:Ô5Ü')§|¢|Ø×.Ñ.°×0BÑ0BÐPT×P`ÑP`ô(ˆDÔ$ð $×1Ñ1ˆÔØ#×1Ñ1ˆÖr-   c                óF  € Vf,   Ve   \        WP                  4      pMV P                  V4      pVf   V P                  V4      pTpV P                  '       dÌ   VeÈ   VP                  WP                  8H  P                  R4      R4      pRpVe   VP                  R4      MVP                  ^,          pWP                  8H  P                  R4      P                  4       V,          pV^V,
          ,          ^V,
          R,          ,          P                  VP                  4      pV P                  R8X  d   V P                  V4      p	WY,           pV P                  e   V P                  V4      pVe1   WRP                  R4      ,          P                  VP                  4      pV# )Nç        rÂ   r$   g¸…ëQ¸¾?)r�   NN)Ú"create_position_ids_from_input_idsr¾   Ú&create_position_ids_from_inputs_embedsrÈ   rÒ   Úmasked_fillrÓ   r1   rJ   r%   r2   r3   r0   rÁ   rÑ   rÌ   )
rg   Ú	input_idsÚattention_maskr†   Úinputs_embedsÚ
embeddingsÚmask_ratio_trainÚsrc_lengthsÚmask_ratio_observedrÑ   s
   &&&&&     r+   rŒ   ÚEsmEmbeddings.forwardà   sƒ  € ð ÒØÒ$äAÀ)×M]ÑM]Ó^‘à#×JÑJÈ=ÓY�àÒ Ø ×0Ñ0°Ó;ˆMð #ˆ
ð ××Ð )Ò"7Ø#×/Ñ/°×>PÑ>PÑ1P×0[Ñ0[Ð\^Ó0_ÐadÓeˆJØ)ÐØ4BÒ4N˜.×,Ñ,¨RÔ0ÐT]×TcÑTcÐdeÕTfˆKØ#,×0BÑ0BÑ#B×"GÑ"GÈÓ"K×"QÑ"QÓ"SÐVaÕ"aÐØ$¨Ð,<Õ(<Õ=ÀÐEXÕAXÐZgÕ@hÕh×lÑlØ× Ñ óˆJð ×'Ñ'¨:Ô5Ø"&×":Ñ":¸<Ó"HÐØ#Õ9ˆJà�?‰?Ò&ØŸ™¨Ó4ˆJØÒ%Ø$×'?Ñ'?ÀÓ'CÕC×GÑGÈ
×HXÑHXÓYˆJð Ðr-   c                ó.  € VP                  4       RR pV^,          p\        P                  ! V P                  ^,           W0P                  ,           ^,           \        P                  VP
                  R7      pVP                  ^ 4      P                  V4      # )z¯
We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.

Args:
    inputs_embeds: torch.Tensor

Returns: torch.Tensor
N)r0   rT   r$   )r¬   r&   rw   r¾   ÚlongrT   r1   r‚   )rg   rÝ   Úinput_shapeÚsequence_lengthr†   s   &&   r+   rÙ   Ú4EsmEmbeddings.create_position_ids_from_inputs_embeds  s~   € ð $×(Ñ(Ó*¨3¨BÐ/ˆØ% a�.ˆä—|’|Ø×Ñ˜qÕ  /×4DÑ4DÕ"DÀqÕ"HÔPU×PZÑPZÐcp×cwÑcwô
ˆð ×%Ñ% aÓ(×/Ñ/°Ó<Ð<r-   )	rÏ   rÌ   rÓ   r¾   rÁ   rÑ   r†   rÒ   rÈ   ©NNNN)r�   r�   r‘   r’   r“   ra   rŒ   rÙ   r–   r—   r˜   r™   s   @@r+   r¼   r¼   Æ   s   ù‡ € ñõ2ô*/÷b=ò =r-   r¼   c                ó  € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R,          R\        R	\
        \        ,          /# )
rX   ÚmoduleÚqueryÚkeyÚvaluerÜ   NÚscalingrÏ   rx   )r   ÚModuler&   ro   r2   r   r   )rZ   s   "r+   r\   r\   $  s{   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô �T�\ð%ô ð%ô Ô'Õ(ñ%r-   c                 ó¶  € Vf   VP                  R4      R,          p\        P                  ! WP                  ^^4      4      V,          pVe	   W„,           p\        P
                  P                  VRR7      p\        P
                  P                  W†V P                  R7      p\        P                  ! Wƒ4      p	V	P                  ^^4      P                  4       p	W˜3# )Nr"   )ÚpÚtrainingr$   ç      à¿)
r¬   r&   ÚmatmulrF   r   Ú
functionalÚsoftmaxrÏ   rò   Ú
contiguous)
rê   rë   rì   rí   rÜ   rî   rÏ   rx   Úattn_weightsÚattn_outputs
   &&&&&&&,  r+   Úeager_attention_forwardrú   $  s±   € ð ‚Ø—*‘*˜R“. DÕ(ˆô —<’< §}¡}°Q¸Ó':Ó;¸gÕE€LàÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2Ð(Ó>€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€Lä—,’,˜|Ó3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r-   c                   óL   a a€ ] tR tRt oRV 3R lltRV3R lR lltRtVtV ;t# )ÚEsmSelfAttentioni@  c                ó–  <€ \         SV `  4        Wn        VP                  VP                  ,          ^ 8w  d9   \        VR4      '       g'   \        RVP                   RVP                   R24      hVP                  V n        \        VP                  VP                  ,          4      V n        V P                  V P                  ,          V n	        \        P                  ! VP                  V P                  4      V n        \        P                  ! VP                  V P                  4      V n        \        P                  ! VP                  V P                  4      V n        VP                  V n        T;'       g    \#        VRR4      V n        RV n        VP(                  V n        W0n        V P(                  ;'       d    V'       * V n        R# )	é    Úembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads (Ú)rÁ   rÂ   r>   N)r`   ra   rY   ru   rv   ÚhasattrÚ
ValueErrorrŸ   Úattention_head_sizeÚall_head_sizer   r£   rë   rì   rí   Úattention_probs_dropout_probrÏ   rt   rÁ   rî   Ú
is_decoderÚ	layer_idxÚ	is_causal)rg   rY   rÁ   r  Úis_cross_attentionrj   s   &&&&&€r+   ra   ÚEsmSelfAttention.__init__B  so  ø€ Ü‰ÑÔØŒà×Ñ × :Ñ :Õ:¸aÔ?ÌÐPVÐXh×HiÒHiÜØ# F×$6Ñ$6Ð#7ð 8Ø ×4Ñ4Ð5°Qð8óð ð
 $*×#=Ñ#=ˆÔ Ü#& v×'9Ñ'9¸F×<VÑ<VÕ'VÓ#WˆÔ Ø!×5Ñ5¸×8PÑ8PÕPˆÔä—Y’Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
Ü—9’9˜V×/Ñ/°×1CÑ1CÓDˆŒÜ—Y’Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
à×:Ñ:ˆŒà'>÷ (
ð (
Ä'ØÐ-¨zóC
ˆÔ$ð ˆŒØ ×+Ñ+ˆŒØ"ŒØŸ™×CÐCÐ1CÔ-CˆŽr-   c                ó  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          RS[S[ P                  ,          /# )	rX   Úhidden_statesrÜ   NÚencoder_hidden_statesÚencoder_attention_maskrÑ   rx   rm   )r&   ro   ÚFloatTensorr   r   rn   )rZ   r[   s   "€r+   r\   ÚEsmSelfAttention.__annotate__^  sŽ   ø€ ÷ .)ñ .)á—|‘|ð.)ñ ×)Ñ)¨DÕ0ð.)ñ  %×0Ñ0°4Õ7ð	.)ñ
 !&× 1Ñ 1°DÕ 8ð.)ñ #Ÿ\™\¨DÕ0ð.)ñ Ñ+Õ,ð.)ñ 
‰u�|‰|Õ	ñ.)r-   c                ó>  € VP                   R R p. VORNV P                  N5pV P                  V4      P                  V4      P	                  ^^4      p	VR Jp
V
'       d   TMTpV
'       d   TMTpV P                  V4      P                  V4      P	                  ^^4      pV P                  V4      P                  V4      P	                  ^^4      pW�P                  R,          ,          p	V P                  R8X  d   Vw  rï\        WœWï^R7      w  rœ\        P                  ! V P                  P                  \        4      pT! V V	VVV3RV P                  '       g   RMV P                  RV P                   /VB w  ppVP"                  ! . VORN5!  P%                  4       pVV3# )NÚrotary)r8   rÏ   r×   rî   r$   ró   )r%   r  rë   r­   rF   rì   rí   rÁ   r<   r   Úget_interfacerY   Ú_attn_implementationrú   rò   rÏ   rî   Úreshaper÷   )rg   r  rÜ   r  r  rÑ   rx   rå   Úhidden_shapeÚquery_layerr	  Úcurrent_statesÚ	key_layerÚvalue_layerr6   r7   Úattention_interfacerù   rø   s   &&&&&&,            r+   rŒ   ÚEsmSelfAttention.forward^  sŸ  € ð $×)Ñ)¨#¨2Ð.ˆØC˜ÐC bÐC¨$×*BÑ*BÑCˆà—j‘j Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆà2¸$Ð>Ðß2DÑ.È-ˆß3EÑ/È>ˆØ—H‘H˜^Ó,×1Ñ1°,Ó?×IÑIÈ!ÈQÓOˆ	Ø—j‘j Ó0×5Ñ5°lÓC×MÑMÈaÐQRÓSˆð "×$<Ñ$<¸dÕ$BÕBˆà×'Ñ'¨8Ô3Ø*‰HˆCÜ%9¸+ÐRUÐjkÔ%lÑ"ˆKä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð  $Ÿ}Ÿ}˜}‘C°$·,±,ð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆ�\ð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ˜LÐ(Ð(r-   )r  r  rY   rÏ   r  r  rì   r  rv   rÁ   rë   rî   rí   )NNFrè   ©	r�   r�   r‘   r’   ra   rŒ   r–   r—   r˜   r™   s   @@r+   rü   rü   @  s   ù‡ € ÷D÷8.)÷ .)ò .)r-   rü   c                   ó8   a a€ ] tR tRt oV 3R ltR tRtVtV ;t# )ÚEsmSelfOutputi�  c                óÎ   <€ \         SV `  4        \        P                  ! VP                  VP                  4      V n        \        P                  ! VP                  4      V n        R # rŽ   )	r`   ra   r   r£   ru   ÚdenserÍ   rÎ   rÏ   rÔ   s   &&€r+   ra   ÚEsmSelfOutput.__init__�  sB   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
Ü—z’z &×"<Ñ"<Ó=ˆŽr-   c                óZ   € V P                  V4      pV P                  V4      pW,           pV# rŽ   ©r!  rÏ   ©rg   r  Úinput_tensors   &&&r+   rŒ   ÚEsmSelfOutput.forward•  ó,   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØ%Õ4ˆØÐr-   r$  r  r™   s   @@r+   r  r  �  ó   ù‡ € õ>÷
ò r-   r  c                   óL   a a€ ] tR tRt oRV 3R lltRV3R lR lltRtVtV ;t# )ÚEsmAttentioniœ  c                óÊ   <€ \         SV `  4        \        WVR 7      V n        \	        V4      V n        \        P                  ! VP                  VP                  R7      V n        R# ))r  r	  r¿   N)
r`   ra   rü   rg   r  Úoutputr   rÊ   ru   rË   )rg   rY   r  r	  rj   s   &&&&€r+   ra   ÚEsmAttention.__init__�  sG   ø€ Ü‰ÑÔÜ$ VÐUgÔhˆŒ	Ü# FÓ+ˆŒäŸš f×&8Ñ&8¸f×>SÑ>SÔTˆŽr-   c                óX   <€ V ^8„  d   QhRS[ P                  R,          RS[S[,          /# ©rX   rÑ   Nrx   ©r&   ro   r   r   )rZ   r[   s   "€r+   r\   ÚEsmAttention.__annotate__¤  s0   ø€ ÷ ñ ñ #Ÿ\™\¨DÕ0ðñ Ñ+Õ,ñr-   c                ó„   € V P                  V4      pV P                  ! V3R VRVRVRV/VB w  r‰V P                  W�4      pV# )rÜ   r  r  rÑ   )rÊ   rg   r-  )
rg   r  rÜ   r  r  rÑ   rx   Úhidden_states_lnrù   r¸   s
   &&&&&&,   r+   rŒ   ÚEsmAttention.forward¤  sl   € ð  Ÿ>™>¨-Ó8ÐØŸšØñ
à)ð
ð #8ð
ð $:ð	
ð
 !4ð
ð ñ
‰ˆð —k‘k +Ó=ˆØÐr-   )rÊ   r-  rg   )NFrè   r  r™   s   @@r+   r+  r+  œ  s   ù‡ € ÷U÷÷ ò r-   r+  c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚEsmIntermediateiº  c                ó„   <€ \         SV `  4        \        P                  ! VP                  VP
                  4      V n        R # rŽ   )r`   ra   r   r£   ru   Úintermediate_sizer!  rÔ   s   &&€r+   ra   ÚEsmIntermediate.__init__»  s,   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3KÑ3KÓLˆŽ
r-   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©rX   r  rm   ©r&   ro   )rZ   r[   s   "€r+   r\   ÚEsmIntermediate.__annotate__¿  s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r-   c                ó>   € V P                  V4      p\        V4      pV# rŽ   )r!  rC   )rg   r  s   &&r+   rŒ   ÚEsmIntermediate.forward¿  s    € ØŸ
™
 =Ó1ˆÜ˜]Ó+ˆØÐr-   )r!  r  r™   s   @@r+   r7  r7  º  s   ù‡ € õM÷÷ ð r-   r7  c                   ó8   a a€ ] tR tRt oV 3R ltR tRtVtV ;t# )Ú	EsmOutputiÅ  c                óÎ   <€ \         SV `  4        \        P                  ! VP                  VP
                  4      V n        \        P                  ! VP                  4      V n	        R # rŽ   )
r`   ra   r   r£   r9  ru   r!  rÍ   rÎ   rÏ   rÔ   s   &&€r+   ra   ÚEsmOutput.__init__Æ  sB   ø€ Ü‰ÑÔÜ—Y’Y˜v×7Ñ7¸×9KÑ9KÓLˆŒ
Ü—z’z &×"<Ñ"<Ó=ˆŽr-   c                óZ   € V P                  V4      pV P                  V4      pW,           pV# rŽ   r$  r%  s   &&&r+   rŒ   ÚEsmOutput.forwardË  r(  r-   r$  r  r™   s   @@r+   rB  rB  Å  r)  r-   rB  c                   óN   a a€ ] tR tRt oV 3R ltRV3R lR lltR tRtVtV ;t	# )ÚEsmLayeriÒ  c                óâ  <€ \         SV `  4        VP                  V n        ^V n        \	        V4      V n        VP                  V n        VP                  V n        V P                  '       d3   V P                  '       g   \        V  R24      h\	        VRR7      V n	        \        V4      V n        \        V4      V n        \        P                  ! VP                   VP"                  R7      V n        R# )r¢   z> should be used as a decoder model if cross attention is addedT)r	  r¿   N)r`   ra   Úchunk_size_feed_forwardÚseq_len_dimr+  Ú	attentionr  Úadd_cross_attentionÚRuntimeErrorÚcrossattentionr7  ÚintermediaterB  r-  r   rÊ   ru   rË   rÔ   s   &&€r+   ra   ÚEsmLayer.__init__Ó  s¹   ø€ Ü‰ÑÔØ'-×'EÑ'EˆÔ$ØˆÔÜ% fÓ-ˆŒØ ×+Ñ+ˆŒØ#)×#=Ñ#=ˆÔ Ø×#×#Ð#Ø—?—?�?Ü" d VÐ+iÐ#jÓkÐkÜ".¨vÈ$Ô"OˆDÔÜ+¨FÓ3ˆÔÜ Ó'ˆŒÜŸš f×&8Ñ&8¸f×>SÑ>SÔTˆŽr-   c                óX   <€ V ^8„  d   QhRS[ P                  R,          RS[S[,          /# r0  r1  )rZ   r[   s   "€r+   r\   ÚEsmLayer.__annotate__â  s0   ø€ ÷ !ñ !ñ #Ÿ\™\¨DÕ0ð!ñ Ñ+Õ,ñ!r-   c                óú   € V P                   ! V3R VRV/VB pV P                  '       dA   Ve=   \        V R4      '       g   \        RV  R24      hV P                  ! V3R VRVRVRV/VB pV P                  V4      pV# )rÜ   rÑ   rO  z'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`r  r  )rL  r  r  ÚAttributeErrorrO  Úfeed_forward_chunk)	rg   r  rÜ   r  r  rÑ   rx   Úattention_outputÚlayer_outputs	   &&&&&&,  r+   rŒ   ÚEsmLayer.forwardâ  sÑ   € ð  Ÿ>š>Øñ
à)ð
ð !4ð
ð ñ	
Ðð �?�?ˆ?Ð4Ò@Ü˜4Ð!1×2Ò2Ü$Ø=¸d¸Vð D`ð `óð ð
  $×2Ò2Ø ñ à-ð ð '<ð ð (>ð	 ð
 %8ð ð ñ Ðð ×.Ñ.Ð/?Ó@ˆØÐr-   c                ól   € V P                  V4      pV P                  V4      pV P                  W14      pV# rŽ   )rÊ   rP  r-  )rg   rW  Úattention_output_lnÚintermediate_outputrX  s   &&   r+   rV  ÚEsmLayer.feed_forward_chunk  s9   € Ø"Ÿn™nÐ-=Ó>ÐØ"×/Ñ/Ð0CÓDÐØ—{‘{Ð#6ÓIˆØÐr-   )	rÊ   rM  rL  rJ  rO  rP  r  r-  rK  rè   )
r�   r�   r‘   r’   ra   rŒ   rV  r–   r—   r˜   r™   s   @@r+   rH  rH  Ò  s    ù‡ € õU÷!ò !÷Fò r-   rH  c                   óR   a a€ ] tR tRt oV 3R lt]RV3R lR ll4       tRtVtV ;t	# )Ú
EsmEncoderi  c                ó8  <€ \         SV `  4        Wn        \        P                  ! \        VP                  4       Uu. uF  p\        V4      NK  	  up4      V n        \        P                  ! VP                  VP                  R 7      V n        RV n        R# u upi )r¿   FN)r`   ra   rY   r   Ú
ModuleListÚrangeÚnum_hidden_layersrH  ÚlayerrÊ   ru   rË   Úemb_layer_norm_afterÚgradient_checkpointing)rg   rY   r¸   rj   s   && €r+   ra   ÚEsmEncoder.__init__  sq   ø€ Ü‰ÑÔØŒÜ—]’]¼eÀF×D\ÑD\Ô>]Ó#^Ñ>]¸¤H¨VÖ$4Ñ>]Ñ#^Ó_ˆŒ
Ü$&§L¢L°×1CÑ1CÈ×I^ÑI^Ô$_ˆÔ!Ø&+ˆÖ#ùò $_s   ½Bc                óX   <€ V ^8„  d   QhRS[ P                  R,          RS[S[,          /# r0  r1  )rZ   r[   s   "€r+   r\   ÚEsmEncoder.__annotate__  s4   ø€ ÷ Sñ Sñ #Ÿ\™\¨DÕ0ðSñ Ñ+Õ,ñSr-   c                ó¾   € \        V P                  4       F  w  rxV! V3R VRVRVRV/VB pK  	  V P                  '       d   V P                  V4      p\        VR7      # )rÜ   r  r  rÑ   )Úlast_hidden_state)Ú	enumeraterd  re  r   )	rg   r  rÜ   r  r  rÑ   rx   ÚiÚlayer_modules	   &&&&&&,  r+   rŒ   ÚEsmEncoder.forward  s   € ô  )¨¯©Ö4‰OˆAÙ(Øñà-ðð '<ðð (>ð	ð
 %8ðð ñŠMñ  5ð ×$×$Ð$Ø ×5Ñ5°mÓDˆMä1ÀMÔRÐRr-   )rY   re  rf  rd  rè   )
r�   r�   r‘   r’   ra   r   rŒ   r–   r—   r˜   r™   s   @@r+   r_  r_    s,   ù‡ € õ,ð ÷Sñ Só ÷Sð Sr-   r_  c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )Ú	EsmPooleri/  c                ó¸   <€ \         SV `  4        \        P                  ! VP                  VP                  4      V n        \        P                  ! 4       V n        R # rŽ   )r`   ra   r   r£   ru   r!  ÚTanhr¦   rÔ   s   &&€r+   ra   ÚEsmPooler.__init__0  s9   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ'š'›)ˆŽr-   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r<  r=  )rZ   r[   s   "€r+   r\   ÚEsmPooler.__annotate__5  s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r-   c                ó\   € VR,          pV P                  V4      pV P                  V4      pV# )r�   )r�   rþ   )r!  r¦   )rg   r  Úfirst_token_tensorÚpooled_outputs   &&  r+   rŒ   ÚEsmPooler.forward5  s2   € ð +¨4Õ0ÐØŸ
™
Ð#5Ó6ˆØŸ™¨Ó6ˆØÐr-   )r¦   r!  r  r™   s   @@r+   rq  rq  /  s   ù‡ € õ$÷
÷ ð r-   rq  c                   óÎ   a a€ ] tR tRt oRtRtRt. ROtR.tRt	Rt
RtRtR]R]! ]^RR	7      .R
]! ]^RR	7      ./t]P$                  ! 4       V 3R l4       tR tV3R ltRtVtV ;t# )ÚEsmPreTrainedModeli>  ÚesmTFzposition_embeddings.weightr  r²   rL  )ÚindexÚ
layer_nameÚcross_attentionsrO  c                ó0  <€ \         SV `  V4       \        V\        4      '       d#   \        P
                  ! VP                  4       R# \        V\        4      '       db   \        P                  ! VP                  \        P                  ! VP                  P                  R,          4      P                  R4      4       R# \        V\        4      '       dA   VP                  VP                   4      w  r#\        P                  ! \#        VR4      V4       R# R# )zInitialize the weightsre   Nr$   rÄ   )r`   Ú_init_weightsrƒ   Ú	EsmLMHeadÚinitÚzeros_r§   r¼   Úcopy_r†   r&   rw   r%   r‚   rS   rc   rY   rt   )rg   rê   rh   r¸   rj   s   &&  €r+   r‚  Ú EsmPreTrainedModel._init_weightsS  s¶   ø€ ô 	‰Ñ˜fÔ%Ü�fœi×(Ò(Ü�KŠK˜Ÿ™Ö$Ü˜¤×.Ò.Ü�JŠJ�v×*Ñ*¬E¯LªL¸×9LÑ9L×9RÑ9RÐSUÕ9VÓ,W×,^Ñ,^Ð_fÓ,gÖhÜ˜Ô 2×3Ò3Ø%×EÑEÀfÇmÁmÓTÑˆMÜ�JŠJ”w˜v zÓ2°MÖBñ 4r-   c                ó   € R # rŽ   © ©rg   s   &r+   Úget_output_embeddingsÚ(EsmPreTrainedModel.get_output_embeddings_  s   € ñ r-   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # rW   r   )rZ   r[   s   "€r+   r\   ÚEsmPreTrainedModel.__annotate__>  s   ø‡ ‚ áÑò r-   r‰  )rH  Ú#EsmFoldTriangularSelfAttentionBlockr¼   )r�   r�   r‘   r’   Úbase_model_prefixÚsupports_gradient_checkpointingÚaccepts_loss_kwargsÚ_no_split_modulesÚ"_keys_to_ignore_on_load_unexpectedÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_supports_attention_backendrH  r   rü   Ú_can_record_outputsr&   r•   r‚  r‹  Ú__annotate_func__r–   r—   r˜   r™   s   @@r+   r|  r|  >  sŸ   ù‡ € ð ÐØ&*Ð#ØÐÚ\ÐØ*FÐ)GÐ&ØÐØ€NØÐØ"&Ðð 	˜Ø‘~Ð&6¸aÈKÔXÐYØÙÐ+°1ÐAQÔRð
ðÐð ‡]‚]ƒ_ô	Có ð	Cò÷C … r-   r|  c                   óŒ   a a€ ] tR tRt oRtRV 3R lltR tR tR t]	]
]RV3R lR ll4       4       4       tR	 tR
 tRtVtV ;t# )ÚEsmModelie  aë  

The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of
cross-attention is added between the self-attention layers, following the architecture described in [Attention is
all you need](https://huggingface.co/papers/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit,
Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.

To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set
to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and
`add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass.
c                óÌ  <€ \         SV `  V4       Wn        \        V4      V n        RV n        \        VRR4      V n        V P                  R8X  d   \        VR7      V n        \        V4      V n
        V'       d   \        V4      MRV n        \        VP                  VP                  ,          RR7      V n        V P#                  V P$                  4       V P'                  4        R# )z^
add_pooling_layer (bool, *optional*, defaults to `True`):
    Whether to add a pooling layer
NrÁ   rÂ   r  )rY   T)r�   r§   )r`   ra   rY   r¼   rÞ   Úrotary_embeddingsrt   rÁ   rS   r_  Úencoderrq  Úpoolerr›   rc  rv   Úcontact_headÚ"_register_load_state_dict_pre_hookÚ	load_hookÚ	post_init)rg   rY   Úadd_pooling_layerrj   s   &&&€r+   ra   ÚEsmModel.__init__s  s·   ø€ ô
 	‰Ñ˜Ô ØŒä'¨Ó/ˆŒà!%ˆÔÜ'.¨vÐ7PÐR\Ó']ˆÔ$Ø×'Ñ'¨8Ô3Ü%7¸vÔ%FˆDÔ"ä! &Ó)ˆŒç+<”i Ô'À$ˆŒä4Ø×0Ñ0°6×3MÑ3MÕMÐTXô
ˆÔð 	×/Ñ/°·±Ô?à�‰Ör-   c                óÂ   a€ S R2pWA9  dS   \        V3R l\        VP                  4       4       4       4      pV'       d   W^ ,          ,          W&   V F  pW K  	  R# R# )aÇ  Remap per-layer rotary inv_freq keys from old checkpoints to the new model-level location.

Old checkpoints stored inv_freq per attention layer at:
    {prefix}encoder.layer.{i}.attention.self.rotary_embeddings.inv_freq
New code stores a single shared inv_freq at:
    {prefix}rotary_embeddings.inv_freq
The old checkpoint values must be preserved (not recomputed) because they may
have been saved in float16, matching the precision used during training.
zrotary_embeddings.inv_freqc              3   óŒ   <"  € T F9  pVP                  S4      '       g   K  VP                  R 4      '       g   K5  Vx € K;  	  R# 5i)z*.attention.self.rotary_embeddings.inv_freqN)Ú
startswithÚendswith)Ú.0r5   Úprefixs   & €r+   Ú	<genexpr>Ú%EsmModel.load_hook.<locals>.<genexpr>š  s8   øé € ð á0�AØ—<‘< ×'ô à,-¯J©JÐ7c×,d÷ ’Û0ùs   ƒA¡Aº
AN)ÚsortedÚlistÚkeys)rg   Ú
state_dictr¬  ÚargsÚnew_keyÚold_keysr5   s   &&f*   r+   r£  ÚEsmModel.load_hookŽ  sc   ø€ ð �HÐ6Ð7ˆØÔ$Üô ä˜jŸo™oÓ/Ô0óó ˆH÷
 Ø&0¸!µÕ&=�
Ñ#Û�Ø’Mó ñ %r-   c                ó.   € V P                   P                  # rŽ   ©rÞ   rÈ   rŠ  s   &r+   Úget_input_embeddingsÚEsmModel.get_input_embeddings¤  s   € Ø�‰×.Ñ.Ð.r-   c                ó&   € WP                   n        R # rŽ   r¸  )rg   rí   s   &&r+   Úset_input_embeddingsÚEsmModel.set_input_embeddings§  s   € Ø*/�‰Ö'r-   c                óZ  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          R	S[S[ P                  ,          S[,          /# )
rX   rÛ   NrÜ   r†   rÝ   r  r  rx   rm   )r&   ro   r   r   rn   r   )rZ   r[   s   "€r+   r\   ÚEsmModel.__annotate__­  s°   ø€ ÷ F
ñ F
á—<‘< $Õ&ðF
ñ Ÿ™ tÕ+ðF
ñ —l‘l TÕ)ð	F
ñ
 —|‘| dÕ*ðF
ñ  %Ÿ|™|¨dÕ2ðF
ñ !&§¡¨tÕ 3ðF
ñ Ñ+Õ,ðF
ñ 
‰u�|‰|Õ	ÑKÕ	KñF
r-   c                ó  € VRJ VRJ,          '       d   \        R4      hVf   V P                  VVVR7      pV P                  VVVVRR7      w  r&V P                  R8X  dZ   VfD   VP                  ^,          p\
        P                  ! W„P                  R7      P                  ^ 4      pV P                  WC4      p	MRp	V P                  ! V3RVRVR	VR
V	/VB p
V
^ ,          pV P                  e   V P                  V4      MRp\        VVR7      # )aæ  
input_ids (`torch.LongTensor` of shape `((batch_size, sequence_length))`):
    Indices of input sequence tokens in the vocabulary.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are input IDs?](../glossary#input-ids)
position_ids (`torch.LongTensor` of shape `((batch_size, sequence_length))`, *optional*):
    Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
    config.max_position_embeddings - 1]`.

    [What are position IDs?](../glossary#position-ids)
inputs_embeds (`torch.FloatTensor` of shape `((batch_size, sequence_length), hidden_size)`, *optional*):
    Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
    is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
    model's internal embedding lookup matrix.
Nz:You must specify exactly one of input_ids or inputs_embeds)rÛ   rÜ   r†   )rÜ   r  Úembedding_outputr  Úpast_key_valuesr  )rT   rÜ   r  r  rÑ   )rk  Úpooler_output)r  rÞ   Ú_create_attention_masksrÁ   r%   r&   rw   rT   r1   rž  rŸ  r   r   )rg   rÛ   rÜ   r†   rÝ   r  r  rx   Úseq_lenrÑ   Úencoder_outputsÚsequence_outputry  s   &&&&&&&,     r+   rŒ   ÚEsmModel.forwardª  sJ  € ð> ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ ð !ŸO™OØ#Ø-Ø)ð ,ó ˆMð 26×1MÑ1MØ)Ø#9Ø*Ø"7Ø ð 2Nó 2
Ñ.ˆð ×'Ñ'¨8Ô3ØÒ#Ø'×-Ñ-¨aÕ0�Ü$Ÿ|š|¨G×<PÑ<PÔQ×[Ñ[Ð\]Ó^�Ø"&×"8Ñ"8¸Ó"UÑà"&ÐàŸ,š,Øñ
à)ð
ð #8ð
ð $:ð	
ð
 !4ð
ð ñ
ˆð *¨!Õ,ˆØ8<¿¹Ò8O˜Ÿ™ OÔ4ÐUYˆä;Ø-Ø'ô
ð 	
r-   c                óÞ   € V P                   P                  '       d   \        V P                   VVVR 7      pM\        V P                   VVR7      pVe   \        V P                   VVVR7      pW3# ))rY   rÝ   rÜ   rÂ  )rY   rÝ   rÜ   )rY   rÝ   rÜ   r  )rY   r  r   r
   )rg   rÜ   r  rÁ  r  rÂ  s   &&&&&&r+   rÄ  Ú EsmModel._create_attention_masksö  su   € ð �;‰;×!×!Ð!Ü/Ø—{‘{Ø.Ø-Ø /ô	‰Nô 7Ø—{‘{Ø.Ø-ôˆNð "Ò-Ü%>Ø—{‘{Ø.Ø5Ø&;ô	&Ð"ð Ð5Ð5r-   c                óR  € V ! WR R R7      P                   p\        P                  ! V^R7      pW2P                  ^4      P                  ^4      P                  ^4      ,          pW2P                  ^4      P                  ^4      P                  ^4      ,          pV P	                  W4      # )T)rÜ   Úreturn_dictÚoutput_attentionsr"   )r²   r&   Ústackr1   r¡  )rg   r±   rÜ   Úattnss   &&& r+   Úpredict_contactsÚEsmModel.predict_contacts  sŒ   € Ù�VÈÐ`dÔe×pÑpˆÜ—’˜E qÔ)ˆð
 	×)Ñ)¨!Ó,×6Ñ6°qÓ9×CÑCÀAÓFÕFˆØ×)Ñ)¨!Ó,×6Ñ6°qÓ9×CÑCÀAÓFÕFˆØ× Ñ  Ó/Ð/r-   )rY   r¡  rÞ   rŸ  r   rÁ   rž  )T)NNNNNN)r�   r�   r‘   r’   r“   ra   r£  r¹  r¼  r   r   r   rŒ   rÄ  rÐ  r–   r—   r˜   r™   s   @@r+   rœ  rœ  e  s[   ù‡ € ñ
÷ò6"ò,/ò0ð  ØØ÷F
ñ F
ó ó ó  ðF
òR6÷@	0ò 	0r-   rœ  c                   óv   a a€ ] tR tRt oRR/tV 3R ltR tR t]]	RV3R lR ll4       4       t
R	 tR
tVtV ;t# )ÚEsmForMaskedLMi"  zlm_head.decoder.weightz%esm.embeddings.word_embeddings.weightc                óØ   <€ \         SV `  V4       VP                  '       d   \        P	                  R 4       \        VRR7      V n        \        V4      V n        V P                  4        R# )zjIf you want to use `EsmForMaskedLM` make sure `config.is_decoder=False` for bi-directional self-attention.F©r¥  N)
r`   ra   r  ÚloggerÚwarningrœ  r}  rƒ  Úlm_headr¤  rÔ   s   &&€r+   ra   ÚEsmForMaskedLM.__init__&  sS   ø€ Ü‰Ñ˜Ô à××ÐÜ�N‰Nð1ôô
 ˜F°eÔ<ˆŒÜ  Ó(ˆŒà�‰Ör-   c                ó.   € V P                   P                  # rŽ   ©rØ  ÚdecoderrŠ  s   &r+   r‹  Ú$EsmForMaskedLM.get_output_embeddings4  s   € Ø�|‰|×#Ñ#Ð#r-   c                ó&   € WP                   n        R # rŽ   rÛ  )rg   Únew_embeddingss   &&r+   Úset_output_embeddingsÚ$EsmForMaskedLM.set_output_embeddings7  s   € Ø-�‰Ör-   c                ó^  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[S[,          R
S[S[,          /	# )rX   rÛ   NrÜ   r†   rÝ   r  r  Úlabelsrx   rm   )r&   Ú
LongTensorro   r  r   r   rn   r   )rZ   r[   s   "€r+   r\   ÚEsmForMaskedLM.__annotate__<  s¹   ø€ ÷ *
ñ *
á×#Ñ# dÕ*ð*
ñ Ÿ™ tÕ+ð*
ñ ×&Ñ&¨Õ-ð	*
ñ
 ×(Ñ(¨4Õ/ð*
ñ  %×0Ñ0°4Õ7ð*
ñ !&§¡¨tÕ 3ð*
ñ × Ñ  4Õ'ð*
ñ Ñ+Õ,ð*
ñ 
‘Õ	ñ*
r-   c                ó„  € V P                   ! V3RVRVRVRVRV/VB p	V	^ ,          p
V P                  V
4      pRpVeb   \        4       pVP                  VP                  4      pV! VP                  RV P                  P                  4      VP                  R4      4      p\        VVV	P                  V	P                  R7      # )	az  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
    config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
    loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
rÜ   r†   rÝ   r  r  N©ÚlossÚlogitsr  r²   r$   )r}  rØ  r   r3   rT   r­   rY   rÆ   r   r  r²   )rg   rÛ   rÜ   r†   rÝ   r  r  rã  rx   ÚoutputsrÇ  Úprediction_scoresÚmasked_lm_lossÚloss_fcts   &&&&&&&&,     r+   rŒ   ÚEsmForMaskedLM.forward:  sç   € ð( —(’(Øñ
à)ð
ð &ð
ð (ð	
ð
 #8ð
ð $:ð
ð ñ
ˆð " !�*ˆØ ŸL™L¨Ó9ÐàˆØÒÜ'Ó)ˆHà—Y‘YÐ0×7Ñ7Ó8ˆFÙ%Ð&7×&<Ñ&<¸RÀÇÁ×AWÑAWÓ&XÐZ`×ZeÑZeÐfhÓZiÓjˆNäØØ$Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r-   c                ó:   € V P                   P                  WR 7      # ))rÜ   )r}  rÐ  )rg   r±   rÜ   s   &&&r+   rÐ  ÚEsmForMaskedLM.predict_contactsh  s   € Ø�x‰x×(Ñ(¨Ð(ÓOÐOr-   )r}  rØ  )NNNNNNN)r�   r�   r‘   r’   Ú_tied_weights_keysra   r‹  rà  r   r   rŒ   rÐ  r–   r—   r˜   r™   s   @@r+   rÓ  rÓ  "  sN   ù‡ € à2Ð4[Ð\Ðõò$ò.ð Ø÷*
ñ *
ó ó ð*
÷XPò Pr-   rÓ  c                   ó<   a a€ ] tR tRt oRtV 3R ltR tRtVtV ;t	# )rƒ  il  z&ESM Head for masked language modeling.c                ó¼  <€ \         SV `  4        \        P                  ! VP                  VP                  4      V n        \        P                  ! VP                  VP                  R 7      V n        \        P                  ! VP                  VP                  RR7      V n
        \        P                  ! \        P                  ! VP                  4      4      V n        R# )r¿   F)r§   N)r`   ra   r   r£   ru   r!  rÊ   rË   rÌ   rÆ   rÜ  Ú	Parameterr&   Úzerosr§   rÔ   s   &&€r+   ra   ÚEsmLMHead.__init__o  s‰   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ,š, v×'9Ñ'9¸v×?TÑ?TÔUˆŒä—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒÜ—L’L¤§¢¨V×->Ñ->Ó!?Ó@ˆŽ	r-   c                ó¤   € V P                  V4      p\        V4      pV P                  V4      pV P                  V4      V P                  ,           pV# rŽ   )r!  rC   rÌ   rÜ  r§   ©rg   Úfeaturesrx   r(   s   &&, r+   rŒ   ÚEsmLMHead.forwardw  sD   € Ø�J‰J�xÓ ˆÜ�‹GˆØ�O‰O˜AÓˆð �L‰L˜‹O˜dŸi™iÕ'ˆØˆr-   )r§   rÜ  r!  rÌ   rº   r™   s   @@r+   rƒ  rƒ  l  s   ù‡ € Ù0õA÷ò r-   rƒ  z›
    ESM Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled
    output) e.g. for GLUE tasks.
    )Úcustom_introc                   ó\   a a€ ] tR tRt oV 3R lt]]RV3R lR ll4       4       tRtVt	V ;t
# )ÚEsmForSequenceClassificationi�  c                ó¸   <€ \         SV `  V4       VP                  V n        Wn        \	        VR R7      V n        \        V4      V n        V P                  4        R# ©FrÕ  N)	r`   ra   Ú
num_labelsrY   rœ  r}  ÚEsmClassificationHeadÚ
classifierr¤  rÔ   s   &&€r+   ra   Ú%EsmForSequenceClassification.__init__ˆ  sF   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒØŒä˜F°eÔ<ˆŒÜ/°Ó7ˆŒà�‰Ör-   c                ó  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          RS[S[,          /# ©	rX   rÛ   NrÜ   r†   rÝ   rã  rx   rm   )r&   rä  ro   r  r   r   rn   r   )rZ   r[   s   "€r+   r\   Ú)EsmForSequenceClassification.__annotate__”  s’   ø€ ÷ 8
ñ 8
á×#Ñ# dÕ*ð8
ñ Ÿ™ tÕ+ð8
ñ ×&Ñ&¨Õ-ð	8
ñ
 ×(Ñ(¨4Õ/ð8
ñ × Ñ  4Õ'ð8
ñ Ñ+Õ,ð8
ñ 
Ñ)Õ	)ñ8
r-   c           	     ó   € V P                   ! V3RVRVRV/VB pV^ ,          pV P                  V4      p	Rp
VEe³   VP                  V	P                  4      pV P                  P
                  f–   V P                  ^8X  d   RV P                  n        MrV P                  ^8”  dQ   VP                  \        P                  8X  g    VP                  \        P                  8X  d   RV P                  n        MRV P                  n        V P                  P
                  R8X  dK   \        4       pV P                  ^8X  d'   V! V	P                  4       VP                  4       4      p
MŽV! W•4      p
M…V P                  P
                  R8X  d>   \        4       pV! V	P                  R	V P                  4      VP                  R	4      4      p
M-V P                  P
                  R8X  d   \        4       pV! W•4      p
\!        V
V	VP"                  VP$                  R7      # )
ae  
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
    `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
rÜ   r†   rÝ   Nr¤   Úsingle_label_classificationÚmulti_label_classificationrç  r$   )r}  r  r3   rT   rY   Úproblem_typer   r0   r&   rä   rŸ   r   r°   r   r­   r   r   r  r²   ©rg   rÛ   rÜ   r†   rÝ   rã  rx   rê  rÇ  ré  rè  rí  s   &&&&&&,     r+   rŒ   Ú$EsmForSequenceClassification.forward’  s¯  € ð$ —(’(Øñ
à)ð
ð &ð
ð (ð	
ð
 ñ
ˆð " !�*ˆØ—‘ Ó1ˆàˆØÓØ—Y‘Y˜vŸ}™}Ó-ˆFà�{‰{×'Ñ'Ò/Ø—?‘? aÔ'Ø/;�D—K‘KÕ,Ø—_‘_ qÔ(¨f¯l©l¼e¿j¹jÔ.HÈFÏLÉLÔ\a×\eÑ\eÔLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ô7Ü"›9�Ø—?‘? aÔ'Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# FÓ3‘DØ—‘×)Ñ)Ð-JÔJÜ+Ó-�Ù §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR‘Ø—‘×)Ñ)Ð-IÔIÜ,Ó.�Ù Ó/�ä'ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r-   )r  rY   r}  r   ©NNNNN©r�   r�   r‘   r’   ra   r   r   rŒ   r–   r—   r˜   r™   s   @@r+   rý  rý  �  s0   ù‡ € õð Ø÷8
ñ 8
ó ó ÷8
ð 8
r-   rý  c                   ó\   a a€ ] tR tRt oV 3R lt]]RV3R lR ll4       4       tRtVt	V ;t
# )ÚEsmForTokenClassificationiÏ  c                ó6  <€ \         SV `  V4       VP                  V n        \        VR R7      V n        \
        P                  ! VP                  4      V n        \
        P                  ! VP                  VP                  4      V n        V P                  4        R# rÿ  )r`   ra   r   rœ  r}  r   rÍ   rÎ   rÏ   r£   ru   r  r¤  rÔ   s   &&€r+   ra   Ú"EsmForTokenClassification.__init__Ñ  si   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä˜F°eÔ<ˆŒÜ—z’z &×"<Ñ"<Ó=ˆŒÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒà�‰Ör-   c                ó  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          RS[S[,          /# r  )r&   rä  ro   r  r   r   rn   r   )rZ   r[   s   "€r+   r\   Ú&EsmForTokenClassification.__annotate__Ý  s’   ø€ ÷ '
ñ '
á×#Ñ# dÕ*ð'
ñ Ÿ™ tÕ+ð'
ñ ×&Ñ&¨Õ-ð	'
ñ
 ×(Ñ(¨4Õ/ð'
ñ × Ñ  4Õ'ð'
ñ Ñ+Õ,ð'
ñ 
Ñ&Õ	&ñ'
r-   c           	     óŠ  € V P                   ! V3RVRVRV/VB pV^ ,          pV P                  V4      pV P                  V4      p	Rp
VeX   \        4       pVP	                  V	P
                  4      pV! V	P                  RV P                  4      VP                  R4      4      p
\        V
V	VP                  VP                  R7      # )zÃ
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
rÜ   r†   rÝ   Nrç  r$   )r}  rÏ   r  r   r3   rT   r­   r   r   r  r²   r  s   &&&&&&,     r+   rŒ   Ú!EsmForTokenClassification.forwardÛ  sÏ   € ð  —(’(Øñ
à)ð
ð &ð
ð (ð	
ð
 ñ
ˆð " !�*ˆàŸ,™, Ó7ˆØ—‘ Ó1ˆàˆØÒÜ'Ó)ˆHà—Y‘Y˜vŸ}™}Ó-ˆFÙ˜FŸK™K¨¨D¯O©OÓ<¸f¿k¹kÈ"»oÓNˆDä$ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r-   )r  rÏ   r}  r   r  r  r™   s   @@r+   r  r  Ï  s0   ù‡ € õð Ø÷'
ñ '
ó ó ÷'
ð '
r-   r  c                   ó<   a a€ ] tR tRt oRtV 3R ltR tRtVtV ;t	# )r  i  z-Head for sentence-level classification tasks.c                ó.  <€ \         SV `  4        \        P                  ! VP                  VP                  4      V n        \        P                  ! VP                  4      V n        \        P                  ! VP                  VP                  4      V n
        R # rŽ   )r`   ra   r   r£   ru   r!  rÍ   rÎ   rÏ   r   Úout_projrÔ   s   &&€r+   ra   ÚEsmClassificationHead.__init__
  s`   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
Ü—z’z &×"<Ñ"<Ó=ˆŒÜŸ	š	 &×"4Ñ"4°f×6GÑ6GÓHˆŽr-   c                óÌ   € VR,          pV P                  V4      pV P                  V4      p\        P                  ! V4      pV P                  V4      pV P	                  V4      pV# )r�   )r�   rþ   r�   )rÏ   r!  r&   Útanhr  rø  s   &&, r+   rŒ   ÚEsmClassificationHead.forward  sS   € Ø�WÕˆØ�L‰L˜‹OˆØ�J‰J�q‹MˆÜ�JŠJ�q‹MˆØ�L‰L˜‹OˆØ�M‰M˜!ÓˆØˆr-   )r!  rÏ   r  rº   r™   s   @@r+   r  r    s   ù‡ € Ù7õI÷ò r-   r  c                óÊ   € V P                  V4      P                  4       p\        P                  ! V^R7      P	                  V4      V,          pVP                  4       V,           # )zí
Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols
are ignored. This is modified from fairseq's `utils.make_positions`.

Args:
    x: torch.Tensor x:

Returns: torch.Tensor
r"   )r«   rŸ   r&   ÚcumsumÚtype_asrä   )rÛ   r¾   ÚmaskÚincremental_indicess   &&  r+   rØ   rØ     sP   € ð �<‰<˜Ó$×(Ñ(Ó*€DÜŸ,š, t°Ô3×;Ñ;¸DÓAÀDÕHÐØ×#Ñ#Ó%¨Õ3Ð3r-   )r¢   )Nr×   )rÓ  rý  r  rœ  r|  )Or“   r@   Úcollections.abcr   r&   r   Útorch.nnr   r   r   Ú r   r„  Úintegrationsr   r	   Úmasking_utilsr
   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   Úmodeling_rope_utilsr   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   r   Úconfiguration_esmr    Ú
get_loggerr�   rÖ  r,   r<   rC   rG   rQ   rï   rS   r›   r¼   rú   rü   r  r+  r7  rB  rH  r_  rq  r|  rœ  rÓ  rƒ  rý  r  r  rØ   Ú__all__r‰  r-   r+   Ú<module>r4     s9  ðñ ã Ý $ã Ý ß AÑ Aå &ß Lß JÝ 9÷õ õ 7ß FÝ &ß RÓ RÝ 0ß Gß EÝ (ð 
×	Ò	˜HÓ	%€ò(ñ Ð-Ó.óBó /ðBò4;ò#ò
	ô6<˜Ÿ™ô 6<ôr G˜rŸy™yô  GôFZ=�B—I‘Iô Z=÷|%ñ8 Ð)Ó*ôK)�r—y‘yó K)ó +ðK)ô\
�B—I‘Iô 
ô�2—9‘9ô ô<�b—i‘iô ô
�—	‘	ô 
ô7Ð)ô 7ôtS�—‘ô SôF�—	‘	ô ð ô#˜ó #ó ð#ðL ôy0Ð!ó y0ó ðy0ðx ôFPÐ'ó FPó ðFPôR�—	‘	ô ñ* ðôôE
Ð#5ó E
óðE
ðP ô4
Ð 2ó 4
ó ð4
ôn˜BŸI™Iô ò&4ò ‚r-   