+
    œ°³jsf  ã                   ó"  € ^ RI t ^ RIHt ^ RIt^ RIHt ^ RIHtHtHt ^ RIH	t
 ^RIHt ^RIHtHt ^RIHtHt ^RIHtHt ^R	IHtHtHt ^R
IHtHt ^RIHtHt ^RI H!t! ^RI"H#t#H$t$H%t% ^RI&H't' ^RI(H)t)H*t*H+t+ ^RI,H-t- ^RI.H/t/  ! R R]P`                  4      t1 ! R R]Pd                  4      t3 ! R R]Pd                  4      t4R t5]! R4      R1R l4       t6R2R R llt7]! ]64       ! R R]Pd                  4      4       t8 ! R R ]4      t9]$ ! R! R"]4      4       t:]$ ! R# R$]:4      4       t; ! R% R&]Pd                  4      t<]$ ! R' R(]:4      4       t= ! R) R*]Pd                  4      t>]$! R+R,7       ! R- R.]:4      4       t? ! R/ R0]]:4      t@. R3OtAR# )4é    N)ÚCallable)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELoss)Ú
functional)ÚACT2FN)Úuse_kernel_forward_from_hubÚuse_kernelized_func)Úcreate_bidirectional_maskÚpacked_sequence_mask_function)ÚGenericForTokenClassificationÚGradientCheckpointingLayer)ÚBaseModelOutputÚMaskedLMOutputÚSequenceClassifierOutput)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Údeprecate_kwarg)Úis_flash_attention_requestedÚmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputs©Ú
EsmcConfigc                   óB   a a€ ] tR t^,t oRtV3R lV 3R lltRtVtV ;t# )ÚEsmcLayerNormaL  LayerNorm that returns its input dtype.

The reference fuses each LayerNorm into the projection that follows it, keeping the fp32 reduction
inside the fused op; a bare ``nn.LayerNorm`` under autocast instead hands back fp32 and promotes
the residual stream, leaving the rotary ``cos``/``sin`` as the only bf16 tensors in the block.
c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©é   Úhidden_statesÚreturn©ÚtorchÚTensor)ÚformatÚ__classdict__s   "€Úg/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/esmc/modeling_esmc.pyÚ__annotate__ÚEsmcLayerNorm.__annotate__4   s'   ø€ ÷ Fñ F¡U§\¡\ð F±e·l±lñ Fó    c                óT   <€ \         SV `  V4      P                  VP                  4      # ©N)ÚsuperÚforwardÚtoÚdtype)Úselfr&   Ú	__class__s   &&€r-   r4   ÚEsmcLayerNorm.forward4   s#   ø€ Ü‰w‰˜}Ó-×0Ñ0°×1DÑ1DÓEÐEr0   © )	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r4   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r8   r,   s   @@r-   r"   r"   ,   s   ù‡ € ñ÷F÷ Fó Fr0   r"   c                   óÎ   a a€ ] tR t^8t o]! RRR7      R
V3R lV 3R lll4       t]]! RRR7      R
V3R lR ll4       4       t]P                  ! 4       ]
R 4       4       tR	tVtV ;t# )ÚEsmcRotaryEmbeddingÚdevicez5.18)Úversionc                ó    <€ V ^8„  d   QhRS[ /# ©r%   Úconfigr   )r+   r,   s   "€r-   r.   Ú EsmcRotaryEmbedding.__annotate__:   s   ø€ ÷ Oñ O™zñ Or0   c                óä  <€ \         SV `  4        VP                  V n        VP                  V n        Wn        V P
                  P                  R ,          V n        V P                  pV P                  R8w  d   \        V P                  ,          pV! V P
                  V4      w  q@n
        \        P                  ! VRR7      V n        \        P                  ! VP                  4       RR7      V n        R# )Ú	rope_typeÚdefaultF)Ú
persistentN)r3   Ú__init__Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenrJ   Úrope_parametersrM   Úcompute_default_rope_parametersr   Úattention_scalingÚnnÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)r7   rJ   rF   Úrope_init_fnrY   r8   s   &&&  €r-   rP   ÚEsmcRotaryEmbedding.__init__9   s¬   ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àŒàŸ™×4Ñ4°[ÕAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÔ&Ü.¨t¯~©~Õ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÔ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÔ!NˆÖr0   c                óP   <€ V ^8„  d   QhRS[ RS[S[P                  S[3,          /# )r%   rJ   r'   )r    Útupler)   r*   Úfloat)r+   r,   s   "€r-   r.   rK   L   s.   ø€ ÷ 5ñ 5±
ð 5ÑV[Ñ\a×\hÑ\hÑjoÐ\oÕVpñ 5r0   c           	     ó0  € V P                   R,          p\        V RR4      ;'       g    V P                  V P                  ,          pRpRV\        P
                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚhead_dimNg      ð?©r6   )rT   ÚgetattrÚhidden_sizeÚnum_attention_headsr)   Úaranger`   r5   )rJ   rF   ÚkwargsÚbaseÚdimÚattention_factorrY   s   &&,    r-   rU   Ú3EsmcRotaryEmbedding.compute_default_rope_parametersJ   s~   € ð ×%Ñ% lÕ3ˆÜ�f˜j¨$Ó/×cÐc°6×3EÑ3EÈ×IcÑIcÕ3cˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r0   c                ó^  € V P                   R,          P                  VP                  ^ ,          R	^4      P                  \        P
                  VP                  R7      pVR
,          P                  4       p\        VP                  P                  \        4      '       d2   VP                  P                  R8w  d   VP                  P                  MRp\        VRR7      ;_uu_ 4        W4,          P                  ^^4      p\        P                  ! Wf3R	R7      pVP                  4       V P                  ,          pVP                  4       V P                  ,          p	R R R 4       XP                  VP                   R7      X	P                  VP                   R7      3#   + '       g   i     LH; i)N)r6   rF   ÚmpsÚcpuF)Údevice_typeÚenabled©rk   rd   )NºNNNNéÿÿÿÿ)rt   Nrt   )rY   ÚexpandÚshaper5   r)   r`   rF   Ú
isinstanceÚtypeÚstrr   Ú	transposeÚcatÚcosrV   Úsinr6   )
r7   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedrq   ÚfreqsÚembr}   r~   s
   &&&       r-   r4   ÚEsmcRotaryEmbedding.forward^   s2  € ð �M‰M˜-Õ(×/Ñ/°×0BÑ0BÀ1Õ0EÀrÈ1ÓM×PÑPÔW\×WbÑWbÐkl×ksÑksÐPÓtð 	ð !-¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆä¨¸U×CÖCØ&Õ>×IÑIÈ!ÈQÓOˆEÜ—)’)˜U˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆC÷	 Dð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷ D×Cús   Ã(A4FÆF,	)rV   rJ   rY   rR   r[   rS   rM   r2   )r;   r<   r=   r>   r   rP   ÚstaticmethodrU   r)   Úno_gradr   r4   r@   rA   rB   rC   s   @@r-   rE   rE   8   su   ù‡ € Ù�X vÔ.÷Oô Oó /ðOð  Ù�X vÔ.÷5ñ 5ó /ó ð5ð$ ‡]‚]ƒ_Øñ<ó ó ÷<ð <r0   rE   c                   ó8   a a€ ] tR t^qt oV 3R ltR tRtVtV ;t# )ÚEsmcMLPc                ó  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        \
        P                  ! V P                  V P                  VP                  R 7      V n        \
        P                  ! V P                  V P                  VP                  R 7      V n	        \
        P                  ! V P                  V P                  VP                  R 7      V n
        \        VP                  ,          V n        R# )©ÚbiasN)r3   rP   rJ   rf   Úintermediate_sizerW   ÚLinearÚmlp_biasÚ	gate_projÚup_projÚ	down_projr   Ú
hidden_actÚact_fn©r7   rJ   r8   s   &&€r-   rP   ÚEsmcMLP.__init__r   s¶   ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔÜŸš 4×#3Ñ#3°T×5KÑ5KÐRX×RaÑRaÔbˆŒÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPV×P_ÑP_Ô`ˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRX×RaÑRaÔbˆŒÜ˜V×.Ñ.Õ/ˆŽr0   c                ó�   € V P                  V P                  V P                  V4      4      V P                  V4      ,          4      pV# r2   )r’   r”   r�   r‘   )r7   r   r’   s   && r-   r4   ÚEsmcMLP.forward|   s6   € Ø—N‘N 4§;¡;¨t¯~©~¸aÓ/@Ó#AÀDÇLÁLÐQRÃOÕ#SÓTˆ	ØÐr0   )r”   rJ   r’   r�   rf   r�   r‘   ©	r;   r<   r=   r>   rP   r4   r@   rA   rB   rC   s   @@r-   r‰   r‰   q   s   ù‡ € õ0÷ò r0   r‰   c                óÌ   € V RRV P                   R,          ^,          13,          pV RV P                   R,          ^,          R13,          p\        P                  ! V) V3RR7      # )z*Rotates half the hidden dims of the input..Nrs   ru   )rw   r)   r|   )r   Úx1Úx2s   &  r-   Úrotate_halfr�   �   sZ   € à	
ˆ3Ð"�!—'‘'˜"•+ Õ"Ð"Ð"Õ	#€BØ	
ˆ3�—‘˜•˜qÕ Ñ"Ð"Õ	#€BÜ�9Š9�r�c˜2�Y BÔ'Ð'r0   Úrotary_pos_embc                óÈ   € VP                  V4      pVP                  V4      pW,          \        V 4      V,          ,           pW,          \        V4      V,          ,           pWV3# )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)Ú	unsqueezer�   )ÚqÚkr}   r~   Úunsqueeze_dimÚq_embedÚk_embeds   &&&&&  r-   Úapply_rotary_pos_embr¦   ˆ   sS   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�wœ; q›>¨CÕ/Õ0€GØ�wœ; q›>¨CÕ/Õ0€GØÐÐr0   c                ó  € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R,          R\        R	\
        \        ,          /# )
r%   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNÚscalingÚdropoutri   )rW   ÚModuler)   r*   r`   r   r   )r+   s   "r-   r.   r.   ¢   s{   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô �T�\ð%ô ð%ô Ô'Õ(ñ%r0   c                 ó¶  € Vf   VP                  R4      R,          p\        P                  ! WP                  ^^4      4      V,          pVe	   W„,           p\        P
                  P                  VRR7      p\        P
                  P                  W†V P                  R7      p\        P                  ! Wƒ4      p	V	P                  ^^4      P                  4       p	W˜3# )Nrs   )ÚpÚtrainingru   ç      à¿)
Úsizer)   Úmatmulr{   rW   r   Úsoftmaxr®   r²   Ú
contiguous)
r¨   r©   rª   r«   r¬   r­   r®   ri   Úattn_weightsÚattn_outputs
   &&&&&&&,  r-   Úeager_attention_forwardrº   ¢   s±   € ð ‚Ø—*‘*˜R“. DÕ(ˆô —<’< §}¡}°Q¸Ó':Ó;¸gÕE€LàÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2Ð(Ó>€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€Lä—,’,˜|Ó3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r0   c                   ó\   a a€ ] tR t^¾t oRtRV3R lV 3R llltRV3R lR lltRtVtV ;t	# )	ÚEsmcAttentionz5Multi-head self-attention with QK-LayerNorm and RoPE.c                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# ©r%   rJ   Ú	layer_idxN©r    Úint)r+   r,   s   "€r-   r.   ÚEsmcAttention.__annotate__Â   s#   ø€ ÷ Dñ D™zð D±c¸Dµjñ Dr0   c                ó  <€ \         SV `  4        Wn        W n        \	        VR VP
                  VP                  ,          4      V n        VP                  VP                  ,          V n	        V P                  R,          V n
        VP                  V n        RV n        \        P                  ! VP
                  VP                  V P                  ,          VP                  R7      V n        \        P                  ! VP
                  VP                  V P                  ,          VP                  R7      V n        \        P                  ! VP
                  VP                  V P                  ,          VP                  R7      V n        \        P                  ! VP                  V P                  ,          VP
                  VP                  R7      V n        \)        VP
                  RR7      V n        \)        VP
                  RR7      V n        R# )rc   Fr‹   Nr³   )r3   rP   rJ   r¿   re   rf   rg   rc   Únum_key_value_headsÚnum_key_value_groupsr­   Úattention_dropoutÚ	is_causalrW   rŽ   Úattention_biasÚq_projÚk_projÚv_projÚo_projr"   Úq_normÚk_norm©r7   rJ   r¿   r8   s   &&&€r-   rP   ÚEsmcAttention.__init__Â   ss  ø€ Ü‰ÑÔØŒØ"ŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÕ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Õ$\ˆÔ!Ø—}‘} dÕ*ˆŒØ!'×!9Ñ!9ˆÔØˆŒä—i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×&Ñ&¨¯©Õ6¸×8JÑ8JÐQW×QfÑQfô
ˆŒô $ F×$6Ñ$6¸UÔCˆŒÜ# F×$6Ñ$6¸UÔCˆŽr0   c                ó  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[S[ P                  S[ P                  3,          R,          RS[S[,          RS[S[ P                  S[ P                  R,          3,          /# ©r%   r&   r¬   NÚposition_embeddingsri   r'   ©r)   r*   r_   r   r   )r+   r,   s   "€r-   r.   rÂ   Ü   s~   ø€ ÷ $6ñ $6á—|‘|ð$6ñ Ÿ™ tÕ+ð$6ñ #¡5§<¡<±·±Ð#=Õ>ÀÕEð	$6ñ
 Ñ+Õ,ð$6ñ 
‰u�|‰|™UŸ\™\¨DÕ0Ð0Õ	1ñ$6r0   c                óv  € VP                   R R p. VORNV P                  N5pV P                  V P                  V4      4      P	                  VP
                  4      pV P                  V P                  V4      4      P	                  VP
                  4      pV P                  V4      p	VP                  V4      P                  ^^4      pVP                  V4      P                  ^^4      pV	P                  V4      P                  ^^4      p	Vw  r«\        WxW«4      w  rx\        P                  ! V P                  P                  \         4      pT! V VVV	V3RV P"                  '       g   RMV P$                  RV P&                  /VB w  rÞVP(                  ! . VORN5!  P+                  4       pV P-                  V4      V3# )Nr®   ç        r­   ru   )rw   rc   rÍ   rÉ   r5   r6   rÎ   rÊ   rË   Úviewr{   r¦   r   Úget_interfacerJ   Ú_attn_implementationrº   r²   rÆ   r­   Úreshaper·   rÌ   )r7   r&   r¬   rÓ   ri   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesr}   r~   Úattention_interfacer¹   r¸   s   &&&&,          r-   r4   ÚEsmcAttention.forwardÜ   s   € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ 4§;¡;¨}Ó#=Ó>×AÑAÀ-×BUÑBUÓVˆØ—[‘[ §¡¨]Ó!;Ó<×?Ñ?À×@SÑ@SÓTˆ
Ø—{‘{ =Ó1ˆà#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆØ—_‘_ \Ó2×<Ñ<¸QÀÓBˆ
Ø#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ�{‰{˜;Ó'¨Ð5Ð5r0   )rÆ   rJ   rc   rÇ   rÎ   rÊ   r¿   rÅ   rÌ   rÍ   rÉ   r­   rË   r2   ©NN©
r;   r<   r=   r>   r?   rP   r4   r@   rA   rB   rC   s   @@r-   r¼   r¼   ¾   s#   ù‡ € á?÷Dõ D÷4$6÷ $6ò $6r0   r¼   c                   ó\   a a€ ] tR tRt oRtRV3R lV 3R llltR	V3R lR lltRtVtV ;t	# )
Ú	EsmcLayeri  zRSingle transformer block: pre-norm attention + pre-norm FFN with residual scaling.c                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# r¾   rÀ   )r+   r,   s   "€r-   r.   ÚEsmcLayer.__annotate__  s#   ø€ ÷ Gñ G™zð G±c¸Dµjñ Gr0   c                óH  <€ \         SV `  4        VP                  V n        \        WR 7      V n        \        V4      V n        \        VP                  4      V n        \        VP                  4      V n	        \        P                  ! VP                  ^$,          4      V n        R# ))rJ   r¿   N)r3   rP   rf   r¼   Ú	self_attnr‰   Úmlpr"   Úinput_layernormÚpost_attention_layernormÚmathÚsqrtÚnum_hidden_layersÚscaling_factorrÏ   s   &&&€r-   rP   ÚEsmcLayer.__init__  su   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔä&¨fÔJˆŒä˜6“?ˆŒä,¨V×-?Ñ-?Ó@ˆÔÜ(5°f×6HÑ6HÓ(IˆÔ%ä"Ÿiši¨×(@Ñ(@À2Õ(EÓFˆÖr0   c          
      óÞ   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[S[ P                  S[ P                  3,          R,          RS[S[,          RS[ P                  /# rÒ   rÔ   )r+   r,   s   "€r-   r.   rç     sk   ø€ ÷ ñ á—|‘|ðñ Ÿ™ tÕ+ðñ #¡5§<¡<±·±Ð#=Õ>ÀÕEð	ñ
 Ñ+Õ,ðñ 
�‰ñr0   c                óü   € TpV P                   ! V P                  V4      V3R V/VB w  rgWVV P                  ,          ,           pTpWPP                  V P	                  V4      4      V P                  ,          ,           pV# )rÓ   )ré   rë   rð   rê   rì   )r7   r&   r¬   rÓ   ri   Úresidualr¹   Ú_s   &&&&,   r-   r4   ÚEsmcLayer.forward  s†   € ð !ˆØŸšØ× Ñ  Ó/Øñ
ð !4ð
ð ñ	
‰ˆð !°×1DÑ1DÕ#DÕDˆØ ˆØ §8¡8¨D×,IÑ,IÈ-Ó,XÓ#YÐ\`×\oÑ\oÕ#oÕoˆØÐr0   )rf   rë   rê   rì   rð   ré   r2   râ   rã   rC   s   @@r-   rå   rå     s#   ù‡ € Ù\÷Gõ G÷÷ ò r0   rå   c                   óL   € ] tR tRt]tRtRtRtRt	Rt
RtR]R]/tRR.tR.tR	tR
# )ÚEsmcPreTrainedModeli(  ÚesmcTr&   Ú
attentionsrY   r[   rå   r:   N)r;   r<   r=   r>   r    Úconfig_classÚbase_model_prefixÚsupports_gradient_checkpointingÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_supports_attention_backendrå   r¼   Ú_can_record_outputsÚ"_keys_to_ignore_on_load_unexpectedÚ_no_split_modulesr@   r:   r0   r-   rø   rø   (  sR   † à€LØÐØ&*Ð#ØÐØ€NØÐØ"&Ðà˜Ø�mðÐð +5Ð6IÐ)JÐ&Ø$˜Ôr0   rø   c                   ór   a a€ ] tR tRt oV3R lV 3R llt]]]RV3R lR ll4       4       4       tRt	Vt
V ;t# )Ú	EsmcModeli<  c                ó    <€ V ^8„  d   QhRS[ /# rI   r   )r+   r,   s   "€r-   r.   ÚEsmcModel.__annotate__>  ó   ø€ ÷ ñ ™zñ r0   c                ó¤  <€ \         SV `  V4       \        P                  ! VP                  VP
                  4      V n        \        V4      V n        \        P                  ! \        VP                  4       Uu. uF  p\        V4      NK  	  up4      V n        \        VP
                  R R7      V n        R V n        V P#                  4        R# u upi )Fr‹   N)r3   rP   rW   Ú	EmbeddingÚ
vocab_sizerf   Úembed_tokensrE   Ú
rotary_embÚ
ModuleListÚrangerï   rå   Úlayersr"   ÚnormÚgradient_checkpointingÚ	post_init)r7   rJ   rõ   r8   s   && €r-   rP   ÚEsmcModel.__init__>  s˜   ø€ Ü‰Ñ˜Ô ÜŸLšL¨×):Ñ):¸F×<NÑ<NÓOˆÔÜ-¨fÓ5ˆŒÜ—m’mÄÀf×F^ÑF^Ô@_Ó$`Ñ@_¸1¤Y¨vÖ%6Ñ@_Ñ$`ÓaˆŒÜ! &×"4Ñ"4¸5ÔAˆŒ	Ø&+ˆÔ#Ø�‰Öùò %as   Á8Cc                óÖ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          RS[/# )r%   Ú	input_idsNr¬   r€   Úsequence_idri   r'   )r)   r*   r   r   r   )r+   r,   s   "€r-   r.   r  J  sy   ø€ ÷ L9ñ L9á—<‘< $Õ&ðL9ñ Ÿ™ tÕ+ðL9ñ —l‘l TÕ)ð	L9ñ
 —\‘\ DÕ(ðL9ñ Ñ+Õ,ðL9ñ 
ñL9r0   c                ó  € V P                  V4      pVfC   \        P                  ! VP                  ^,          VP                  R7      P                  ^ 4      pV P                  Wc4      pVeJ   \        V P                  4      '       d   \        R4      h\        V P                  VR\        V4      R7      pM\        V P                  VVR7      pV P                   F  pV! VV3RV/VB pK  	  V P                  V4      p	\        V	R7      # )a®  
position_ids (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
    Position of each token in the sequence, used by RoPE. Defaults to
    ``arange(sequence_length)``.

    ESMC positions are absolute over the whole input, and keep running *across* chain
    boundaries in a multi-chain input. The chain structure is carried separately by
    ``sequence_id``, which masks attention: unlike the packed-sequence format used elsewhere
    in the library, these are therefore not per-chain position ids restarting at 0, and the
    chain structure is deliberately not inferred from them.
sequence_id (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
    Integer chain-ID tensor for chain-aware attention masking. Tokens with the same
    non-negative integer value can attend to each other; tokens with different values
    cannot (cross-chain masking). Padding positions should be set to ``-1`` and inputs
    must be **right-padded** (RoPE uses absolute positions starting at 0). When provided,
    ``attention_mask`` is ignored. Passing ``sequence_id`` builds a custom attention
    mask, which requires ``torch>=2.6``. Multi-chain inputs additionally require a
    non-flash ``attn_implementation`` (``'sdpa'`` / ``'eager'`` / ``'flex_attention'``);
    flash attention only supports the single-chain case.

Examples:

```python
>>> from transformers import AutoTokenizer, EsmcModel

>>> model = EsmcModel.from_pretrained("biohub/ESMC-300M-hf")
>>> tokenizer = AutoTokenizer.from_pretrained("biohub/ESMC-300M-hf")
>>> inputs = tokenizer(["MLKNVQVQLV"], return_tensors="pt")
>>> outputs = model(**inputs)
>>> outputs.last_hidden_state.shape
torch.Size([1, 12, 960])
```
N)rF   zÔ`sequence_id` (chain-aware attention) is not supported with flash attention, which can't represent the block-diagonal chain mask. Re-load the model with attn_implementation='sdpa' (or 'eager' / 'flex_attention').)rJ   Úinputs_embedsr¬   Úand_mask_function)rJ   r  r¬   rÓ   )Úlast_hidden_state)r  r)   rh   rw   rF   r    r  r   rJ   Ú
ValueErrorr   r   r  r  r   )
r7   r  r¬   r€   r  ri   r&   rÓ   ÚlayerÚoutputs
   &&&&&,    r-   r4   ÚEsmcModel.forwardG  s  € ðX ×)Ñ)¨)Ó4ˆØÒÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\×fÑfÐghÓiˆLØ"Ÿo™o¨mÓJÐàÒ"Ü+¨D¯K©K×8Ò8Ü ðRóð ô
 7Ø—{‘{Ø+Ø#Ü"?ÀÓ"Lô	‰Nô 7Ø—{‘{Ø+Ø-ôˆNð —[”[ˆEÙ!ØØñð %8ðð ñ	ŠMñ !ð —‘˜=Ó)ˆä°Ô8Ð8r0   )r  r  r  r  r  )NNNN)r;   r<   r=   r>   rP   r   r   r   r4   r@   rA   rB   rC   s   @@r-   r  r  <  sA   ù‡ € ÷ó ð  ØØ÷L9ñ L9ó ó ó  ÷L9ð L9r0   r  c                   óT   a a€ ] tR tRt oRV3R lV 3R llltV3R lR ltRtVtV ;t# )ÚEsmcMaskedLMHeadi™  c                ó>   <€ V ^8„  d   QhRS[ RS[ RS[ R,          RR/# )r%   rf   r  r�   Nr'   )rÁ   )r+   r,   s   "€r-   r.   ÚEsmcMaskedLMHead.__annotate__š  s5   ø€ ÷ @ñ @¡Cð @±Sð @ÉSÐSWÍZð @Ðcgñ @r0   c                óÖ   <€ \         SV `  4        Ve   TMTp\        P                  ! W4      V n        \        P
                  ! V4      V n        \        P                  ! W24      V n        R # r2   )r3   rP   rW   rŽ   ÚdenseÚ	LayerNormÚ
layer_normÚdecoder)r7   rf   r  r�   r8   s   &&&&€r-   rP   ÚEsmcMaskedLMHead.__init__š  sO   ø€ Ü‰ÑÔØ1BÒ1NÑ-ÐT_ÐÜ—Y’Y˜{Ó>ˆŒ
ÜŸ,š,Ð'8Ó9ˆŒÜ—y’yÐ!2Ó?ˆŽr0   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r$   r(   )r+   r,   s   "€r-   r.   r$  ¡  s#   ø€ ÷ +ñ +¡U§\¡\ð +±e·l±lñ +r0   c                ó”   € V P                  V4      p\        P                  ! V4      pV P                  V4      pV P	                  V4      # r2   )r&  ÚFÚgelur(  r)  )r7   r&   s   &&r-   r4   ÚEsmcMaskedLMHead.forward¡  s;   € ØŸ
™
 =Ó1ˆÜŸš˜}Ó-ˆØŸ™¨Ó6ˆØ�|‰|˜MÓ*Ð*r0   )r)  r&  r(  r2   r™   rC   s   @@r-   r"  r"  ™  s    ù‡ € ÷@õ @÷+÷ +ð +r0   r"  c                   óŒ   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltV3R lR lt]]RV3R lR	 ll4       4       t	R
t
VtV ;t# )ÚEsmcForMaskedLMi¨  c                ó    <€ V ^8„  d   QhRS[ /# rI   r   )r+   r,   s   "€r-   r.   ÚEsmcForMaskedLM.__annotate__ª  s   ø€ ÷ ñ ™zñ r0   c                ó°   <€ \         SV `  V4       \        V4      V n        \	        VP
                  VP                  4      V n        V P                  4        R # r2   )	r3   rP   r  rù   r"  rf   r  Úlm_headr  r•   s   &&€r-   rP   ÚEsmcForMaskedLM.__init__ª  s@   ø€ Ü‰Ñ˜Ô Ü˜fÓ%ˆŒ	Ü'¨×(:Ñ(:¸F×<MÑ<MÓNˆŒØ�‰Ör0   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r%   r'   ©rW   rŽ   )r+   r,   s   "€r-   r.   r3  ³  s   ø€ ÷ $ñ $¡r§y¡yñ $r0   c                ó.   € V P                   P                  # r2   ©r5  r)  )r7   s   &r-   Úget_output_embeddingsÚ%EsmcForMaskedLM.get_output_embeddings³  s   € Ø�|‰|×#Ñ#Ð#r0   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r%   Únew_embeddingsr8  )r+   r,   s   "€r-   r.   r3  ¶  s   ø€ ÷ .ñ .±B·I±Iñ .r0   c                ó&   € WP                   n        R # r2   r:  )r7   r>  s   &&r-   Úset_output_embeddingsÚ%EsmcForMaskedLM.set_output_embeddings¶  s   € Ø-�‰Ör0   c                ó6  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          RS[S[ P                  R	3,          S[,          /# )
r%   r  Nr¬   r€   r  Úlabelsri   r'   .)r)   r*   r   r   r_   r   )r+   r,   s   "€r-   r.   r3  »  s—   ø€ ÷ 3
ñ 3
á—<‘< $Õ&ð3
ñ Ÿ™ tÕ+ð3
ñ —l‘l TÕ)ð	3
ñ
 —\‘\ DÕ(ð3
ñ —‘˜tÕ#ð3
ñ Ñ+Õ,ð3
ñ 
‰u�|‰|˜SÐ Õ	!¡NÕ	2ñ3
r0   c                ó  € V P                   ! R
RVRVRVRVRR/VB pV P                  VP                  4      pRp	Ve*   V P                  ! W…3RV P                  P
                  /VB p	\        V	VVP                  VP                  R	7      # )aH  
sequence_id (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
    Integer chain-ID tensor forwarded to the encoder for chain-aware
    attention masking. See :meth:`EsmcModel.forward` for the encoding.
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for masked language modelling loss.  Positions with label ``-100``
    are ignored.  Other positions must be in ``[0, config.vocab_size)``.

Examples:

```python
>>> from transformers import AutoTokenizer, EsmcForMaskedLM
>>> import torch

>>> model = EsmcForMaskedLM.from_pretrained("biohub/ESMC-300M-hf")
>>> tokenizer = AutoTokenizer.from_pretrained("biohub/ESMC-300M-hf")
>>> inputs = tokenizer(["MLKNVQ<mask>LV"], return_tensors="pt")
>>> outputs = model(**inputs)
>>> outputs.logits.shape
torch.Size([1, 11, 64])
```
r  r¬   r€   r  Úreturn_dictTNr  ©ÚlossÚlogitsr&   rú   r:   )	rù   r5  r  Úloss_functionrJ   r  r   r&   rú   )
r7   r  r¬   r€   r  rC  ri   Úencoder_outputsrH  rG  s
   &&&&&&,   r-   r4   ÚEsmcForMaskedLM.forward¹  s¶   € ðB Ÿ)š)ñ 
Øð
à)ð
ð &ð
ð $ð	
ð
 ð
ð ñ
ˆð —‘˜o×?Ñ?Ó@ˆà$(ˆØÒØ×%Ò% fÑbÀÇÁ×AWÑAWÐbÐ[aÑbˆDäØØØ)×7Ñ7Ø&×1Ñ1ô	
ð 	
r0   )rù   r5  ©NNNNN)r;   r<   r=   r>   rP   r;  r@  r   r   r4   r@   rA   rB   rC   s   @@r-   r1  r1  ¨  sI   ù‡ € ÷ó ÷$ð $÷.ð .ð Ø÷3
ñ 3
ó ó ÷3
ð 3
r0   r1  c                   óH   a a€ ] tR tRt oRtV3R lV 3R lltR tRtVtV ;t	# )ÚEsmcClassificationHeadiñ  z-Head for sentence-level classification tasks.c                ó    <€ V ^8„  d   QhRS[ /# rI   r   )r+   r,   s   "€r-   r.   Ú#EsmcClassificationHead.__annotate__ô  s   ø€ ÷ Iñ I™zñ Ir0   c                ó.  <€ \         SV `  4        \        P                  ! VP                  VP                  4      V n        \        P                  ! VP                  4      V n        \        P                  ! VP                  VP                  4      V n
        R # r2   )r3   rP   rW   rŽ   rf   r&  ÚDropoutÚclassifier_dropoutr®   Ú
num_labelsÚout_projr•   s   &&€r-   rP   ÚEsmcClassificationHead.__init__ô  s`   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
Ü—z’z &×";Ñ";Ó<ˆŒÜŸ	š	 &×"4Ñ"4°f×6GÑ6GÓHˆŽr0   c                óÌ   € VR,          pV P                  V4      pV P                  V4      p\        P                  ! V4      pV P                  V4      pV P	                  V4      pV# )rt   )rt   r   rt   )r®   r&  r)   ÚtanhrU  )r7   Úfeaturesri   r   s   &&, r-   r4   ÚEsmcClassificationHead.forwardú  sS   € Ø�WÕˆØ�L‰L˜‹OˆØ�J‰J�q‹MˆÜ�JŠJ�q‹MˆØ�L‰L˜‹OˆØ�M‰M˜!ÓˆØˆr0   )r&  r®   rU  rã   rC   s   @@r-   rN  rN  ñ  s   ù‡ € Ù7÷Ió I÷ò r0   rN  zœ
    ESMC Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled
    output) e.g. for GLUE tasks.
    )Úcustom_introc                   óh   a a€ ] tR tRt oV3R lV 3R llt]]RV3R lR ll4       4       tRtVt	V ;t
# )ÚEsmcForSequenceClassificationi  c                ó    <€ V ^8„  d   QhRS[ /# rI   r   )r+   r,   s   "€r-   r.   Ú*EsmcForSequenceClassification.__annotate__  r	  r0   c                ó´   <€ \         SV `  V4       VP                  V n        Wn        \	        V4      V n        \        V4      V n        V P                  4        R # r2   )	r3   rP   rT  rJ   r  rù   rN  Ú
classifierr  r•   s   &&€r-   rP   Ú&EsmcForSequenceClassification.__init__  sD   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒØŒÜ˜fÓ%ˆŒ	Ü0°Ó8ˆŒà�‰Ör0   c                ó  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          RS[S[,          /# )	r%   r  Nr¬   r€   r  rC  ri   r'   )r)   Ú
LongTensorr*   ÚFloatTensorr   r   r_   r   )r+   r,   s   "€r-   r.   r_    s’   ø€ ÷ 8
ñ 8
á×#Ñ# dÕ*ð8
ñ Ÿ™ tÕ+ð8
ñ ×&Ñ&¨Õ-ð	8
ñ
 ×(Ñ(¨4Õ/ð8
ñ × Ñ  4Õ'ð8
ñ Ñ+Õ,ð8
ñ 
Ñ)Õ	)ñ8
r0   c           	     ó   € V P                   ! V3RVRVRV/VB pV^ ,          pV P                  V4      p	Rp
VEe³   VP                  V	P                  4      pV P                  P
                  f–   V P                  ^8X  d   RV P                  n        MrV P                  ^8”  dQ   VP                  \        P                  8X  g    VP                  \        P                  8X  d   RV P                  n        MRV P                  n        V P                  P
                  R8X  dK   \        4       pV P                  ^8X  d'   V! V	P                  4       VP                  4       4      p
MŽV! W•4      p
M…V P                  P
                  R8X  d>   \        4       pV! V	P                  R	V P                  4      VP                  R	4      4      p
M-V P                  P
                  R8X  d   \        4       pV! W•4      p
\!        V
V	VP"                  VP$                  R7      # )
ae  
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
    `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
r¬   r€   r  NÚ
regressionÚsingle_label_classificationÚmulti_label_classificationrF  ru   )rù   ra  r5   rF   rJ   Úproblem_typerT  r6   r)   ÚlongrÁ   r   Úsqueezer   r×   r   r   r&   rú   )r7   r  r¬   r€   r  rC  ri   ÚoutputsÚsequence_outputrH  rG  Úloss_fcts   &&&&&&,     r-   r4   Ú%EsmcForSequenceClassification.forward  s¯  € ð$ —)’)Øñ
à)ð
ð &ð
ð (ð	
ð
 ñ
ˆð " !�*ˆØ—‘ Ó1ˆàˆØÓØ—Y‘Y˜vŸ}™}Ó-ˆFà�{‰{×'Ñ'Ò/Ø—?‘? aÔ'Ø/;�D—K‘KÕ,Ø—_‘_ qÔ(¨f¯l©l¼e¿j¹jÔ.HÈFÏLÉLÔ\a×\eÑ\eÔLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ô7Ü"›9�Ø—?‘? aÔ'Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# FÓ3‘DØ—‘×)Ñ)Ð-JÔJÜ+Ó-�Ù §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR‘Ø—‘×)Ñ)Ð-IÔIÜ,Ó.�Ù Ó/�ä'ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r0   )ra  rJ   rù   rT  rL  )r;   r<   r=   r>   rP   r   r   r4   r@   rA   rB   rC   s   @@r-   r]  r]    s5   ù‡ € ÷ó ð Ø÷8
ñ 8
ó ó ÷8
ð 8
r0   r]  c                   ó   € ] tR tRtRtRtR# )ÚEsmcForTokenClassificationiQ  rù   r:   N)r;   r<   r=   r>   rü   r@   r:   r0   r-   rr  rr  Q  s   † ØÔr0   rr  )é   )NrÖ   )r  r1  r]  rr  rø   )Brí   Úcollections.abcr   r)   Útorch.nnrW   r   r   r   r   r-  Úactivationsr   Úintegrationsr	   r
   Úmasking_utilsr   r   Úmodeling_layersr   r   Úmodeling_outputsr   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.deprecationr   Úutils.genericr   r   r   Úutils.output_capturingr   Úconfiguration_esmcr    r'  r"   r¯   rE   r‰   r�   r¦   rº   r¼   rå   rø   r  r"  r1  rN  r]  rr  Ú__all__r:   r0   r-   Ú<module>r„     s–  ðó* Ý $ã Ý ß AÑ AÝ $å !ß Lß Uß Xß YÑ Yß Kß FÝ &ß IÑ IÝ 0ß eÑ eÝ 5Ý *ô	F�B—L‘Lô 	Fô6<˜"Ÿ)™)ô 6<ôrˆb�i‰iô ò (ñ Ð-Ó.óó /ð÷2%ñ8 Ð)Ó*ôA6�B—I‘Ió A6ó +ðA6ôH"Ð*ô "ðJ ô&˜/ó &ó ð&ð& ôY9Ð#ó Y9ó ðY9ôx+�r—y‘yô +ð ôE
Ð)ó E
ó ðE
ôP˜RŸY™Yô ñ& ðôôD
Ð$7ó D
óðD
ôNÐ!>Ð@Sô ò‚r0   