+
    œ°³j^D  ã                   ó¾  € R t ^ RIt^ RIHt ^ RIt^ RIHt ^ RIHt ^ RIH	t
 ^RIHtHt ^RIHt ^RIHtHt ^RIHt ^R	IHt ^R
IHtHtHtHt ^RIHtHt ^RIH t  ^RI!H"t"H#t#H$t$ ^RI%H&t& ^RI'H(t(H)t)H*t*H+t+H,t, ^RI-H.t. ]P^                  ! ]04      t1]! RR7      ] ! R R]&4      4       4       t2 ! R R]Pf                  4      t4 ! R R]+4      t5 ! R R]*4      t6 ! R R](4      t7 ! R R])4      t8] ! R R ].4      4       t9] ! R! R"]94      4       t: ! R# R$]Pv                  4      t<] ! R% R&]94      4       t= ! R' R(]"4      t> ! R) R*]#4      t? ! R+ R,]]94      t@. R-OtAR# ).zPyTorch ESMC model.N)ÚCallable)Ústrict)Ú
functional)Úcreate_bidirectional_maskÚpacked_sequence_mask_function)ÚGenericForTokenClassification)ÚBaseModelOutputÚMaskedLMOutput)ÚALL_ATTENTION_FUNCTIONS)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Úis_flash_attention_requestedÚmerge_with_config_defaults)Úcapture_outputs)ÚEsmClassificationHeadÚEsmForSequenceClassificationÚeager_attention_forward)ÚLlamaConfig)ÚLlamaAttentionÚLlamaDecoderLayerÚLlamaMLPÚLlamaRotaryEmbeddingÚapply_rotary_pos_emb)ÚNomicBertPreTrainedModelzbiohub/ESMC-6B-hf)Ú
checkpointc                   ó²   a a€ ] tR t^7t oRtRt^@tRtRt^Pt	^(t
^t^ t^t^ tRt]! 4       t]! 4       t]! 4       t]! 4       tV 3R ltV 3R ltV3R ltR	tVtV ;t# )
Ú
EsmcConfigah  
mask_token_id (`int`, *optional*, defaults to 32):
    Index of the mask token in the vocabulary (``"<mask>"``), used for masked language modelling.
classifier_dropout (`float`, *optional*, defaults to 0.1):
    Dropout ratio for the classification head.

Examples:

```python
>>> from transformers import EsmcConfig, EsmcModel

>>> # Initializing an ESMC biohub/ESMC-6B-hf style configuration
>>> configuration = EsmcConfig()

>>> # Initializing a model (with random weights) from the configuration
>>> model = EsmcModel(configuration)

>>> # Accessing the model configuration
>>> configuration = model.config
```
Úesmci 
  i   gš™™™™™¹?c                ó|   <€ V P                   f   ^ V n         V P                  f   ^V n        \        SV `  ! R/ VB  R # )N© )Úbos_token_idÚeos_token_idÚsuperÚ__post_init__)ÚselfÚkwargsÚ	__class__s   &,€Úf/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/esmc/modular_esmc.pyr&   ÚEsmcConfig.__post_init__f   s=   ø€ ð ×ÑÒ$Ø !ˆDÔØ×ÑÒ$Ø !ˆDÔÜ‰ÒÑ' Ô'ó    c                ó¦   <€ \         SV `  4        V P                  V P                  8w  d'   \	        R V P                   RV P                   R24      hR# )zFESMC does not support grouped-query attention: `num_key_value_heads` (z$) must equal `num_attention_heads` (z).N)r%   Úvalidate_architectureÚnum_key_value_headsÚnum_attention_headsÚ
ValueError)r'   r)   s   &€r*   r.   Ú EsmcConfig.validate_architectureo   s^   ø€ Ü‰Ñ%Ô'à×#Ñ# t×'?Ñ'?Ô?ÜðØ×,Ñ,Ð-Ð-QÐRV×RjÑRjÐQkÐkmðoóð ñ @r,   c                óø   <€ V ^8„  d   Qh/ S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ R,          ;R&   S[ R,          ;R&   S[ S[S[ ,          ,          R,          ;R	&   S[ R,          ;R
&   S[R,          ;R&   # )é   Ú
vocab_sizeÚhidden_sizeÚintermediate_sizeÚnum_hidden_layersr0   NÚpad_token_idr#   r$   Úmask_token_idÚclassifier_dropout)ÚintÚlistÚfloat)ÚformatÚ__classdict__s   "€r*   Ú__annotate__ÚEsmcConfig.__annotate__7   s°   ø‡ ‚ ñ8 Ññ9 ñ: Ññ; ñ< Ñ!ñ= ñ> Ññ? ñ@ Ñ!ñA ñB ˜•*Ñ ñC ñD ˜•*Ñ ñE ñF ™™S�	•/ DÕ(Ñ,ñG ñL ˜•:Ñ"ñM ñN  �Ñ*òO r,   )r#   r$   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Ú
model_typer5   r6   r7   r8   r0   r9   r#   r$   r:   r;   ÚAttributeErrorÚrms_norm_epsÚpretraining_tpÚ	use_cacheÚkeys_to_ignore_at_inferencer&   r.   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r)   r@   s   @@r*   r   r   7   s€   ù‡ € ñð, €Jð €JØ€KØ!ÐØÐØ!ÐØ €LØ €LØ+,€Lð !#€MØ'*Ðñ "Ó#€LÙ#Ó%€NÙÓ €IÙ"0Ó"2Ðõ(õ÷q … r,   r   c                   óB   a a€ ] tR t^yt oRtV3R lV 3R lltRtVtV ;t# )ÚEsmcLayerNormaL  LayerNorm that returns its input dtype.

The reference fuses each LayerNorm into the projection that follows it, keeping the fp32 reduction
inside the fused op; a bare ``nn.LayerNorm`` under autocast instead hands back fp32 and promotes
the residual stream, leaving the rotary ``cos``/``sin`` as the only bf16 tensors in the block.
c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r4   Úhidden_statesÚreturn©ÚtorchÚTensor)r?   r@   s   "€r*   rA   ÚEsmcLayerNorm.__annotate__�   s'   ø€ ÷ Fñ F¡U§\¡\ð F±e·l±lñ Fr,   c                óT   <€ \         SV `  V4      P                  VP                  4      # ©N)r%   ÚforwardÚtoÚdtype)r'   rW   r)   s   &&€r*   r_   ÚEsmcLayerNorm.forward�   s#   ø€ Ü‰w‰˜}Ó-×0Ñ0°×1DÑ1DÓEÐEr,   r"   )	rC   rD   rE   rF   rG   r_   rO   rP   rQ   rR   s   @@r*   rT   rT   y   s   ù‡ € ñ÷F÷ Fó Fr,   rT   c                   ó   € ] tR t^…tRtR# )ÚEsmcRotaryEmbeddingr"   N©rC   rD   rE   rF   rO   r"   r,   r*   rd   rd   …   ó   † Ûr,   rd   c                   ó   € ] tR t^‰tRtR# )ÚEsmcMLPr"   Nre   r"   r,   r*   rh   rh   ‰   rf   r,   rh   c                   ó\   a a€ ] tR t^�t oRtRV3R lV 3R llltRV3R lR lltRtVtV ;t	# )	ÚEsmcAttentionz5Multi-head self-attention with QK-LayerNorm and RoPE.c                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# ©r4   ÚconfigÚ	layer_idxN©r   r<   )r?   r@   s   "€r*   rA   ÚEsmcAttention.__annotate__�   s#   ø€ ÷ Dñ D™zð D±c¸Dµjñ Dr,   c                ó¤   <€ \         SV `  W4       R V n        \        VP                  R R7      V n        \        VP                  R R7      V n        R# ©F)ÚbiasN)r%   Ú__init__Ú	is_causalrT   r6   Úq_normÚk_norm©r'   rm   rn   r)   s   &&&€r*   rt   ÚEsmcAttention.__init__�   s?   ø€ Ü‰Ñ˜Ô+ØˆŒä# F×$6Ñ$6¸UÔCˆŒÜ# F×$6Ñ$6¸UÔCˆŽr,   c                ó  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[S[ P                  S[ P                  3,          R,          RS[S[,          RS[S[ P                  S[ P                  R,          3,          /# ©r4   rW   Úattention_maskNÚposition_embeddingsr(   rX   ©rZ   r[   Útupler   r   )r?   r@   s   "€r*   rA   rp   —   s~   ø€ ÷ $6ñ $6á—|‘|ð$6ñ Ÿ™ tÕ+ð$6ñ #¡5§<¡<±·±Ð#=Õ>ÀÕEð	$6ñ
 Ñ+Õ,ð$6ñ 
‰u�|‰|™UŸ\™\¨DÕ0Ð0Õ	1ñ$6r,   c                óv  € VP                   R R p. VORNV P                  N5pV P                  V P                  V4      4      P	                  VP
                  4      pV P                  V P                  V4      4      P	                  VP
                  4      pV P                  V4      p	VP                  V4      P                  ^^4      pVP                  V4      P                  ^^4      pV	P                  V4      P                  ^^4      p	Vw  r«\        WxW«4      w  rx\        P                  ! V P                  P                  \         4      pT! V VVV	V3RV P"                  '       g   RMV P$                  RV P&                  /VB w  rÞVP(                  ! . VORN5!  P+                  4       pV P-                  V4      V3# )NÚdropoutg        Úscalingéÿÿÿÿ)ÚshapeÚhead_dimrv   Úq_projr`   ra   rw   Úk_projÚv_projÚviewÚ	transposer   r
   Úget_interfacerm   Ú_attn_implementationr   ÚtrainingÚattention_dropoutr‚   ÚreshapeÚ
contiguousÚo_proj)r'   rW   r|   r}   r(   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚcosÚsinÚattention_interfaceÚattn_outputÚattn_weightss   &&&&,          r*   r_   ÚEsmcAttention.forward—   s   € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ 4§;¡;¨}Ó#=Ó>×AÑAÀ-×BUÑBUÓVˆØ—[‘[ §¡¨]Ó!;Ó<×?Ñ?À×@SÑ@SÓTˆ
Ø—{‘{ =Ó1ˆà#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆØ—_‘_ \Ó2×<Ñ<¸QÀÓBˆ
Ø#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ�{‰{˜;Ó'¨Ð5Ð5r,   )ru   rw   rv   r^   ©NN©
rC   rD   rE   rF   rG   rt   r_   rO   rP   rQ   rR   s   @@r*   rj   rj   �   s#   ù‡ € Ù?÷Dõ D÷$6÷ $6ò $6r,   rj   c                   ó\   a a€ ] tR t^¾t oRtRV3R lV 3R llltRV3R lR lltRtVtV ;t	# )	Ú	EsmcLayerzRSingle transformer block: pre-norm attention + pre-norm FFN with residual scaling.c                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# rl   ro   )r?   r@   s   "€r*   rA   ÚEsmcLayer.__annotate__Á   s#   ø€ ÷ Gñ G™zð G±c¸Dµjñ Gr,   c                óæ   <€ \         SV `  W4       \        VP                  4      V n        \        VP                  4      V n        \        P                  ! VP                  ^$,          4      V n	        R# )é$   N)
r%   rt   rT   r6   Úinput_layernormÚpost_attention_layernormÚmathÚsqrtr8   Úscaling_factorrx   s   &&&€r*   rt   ÚEsmcLayer.__init__Á   sR   ø€ Ü‰Ñ˜Ô+ä,¨V×-?Ñ-?Ó@ˆÔÜ(5°f×6HÑ6HÓ(IˆÔ%ä"Ÿiši¨×(@Ñ(@À2Õ(EÓFˆÖr,   c          
      óÞ   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[S[ P                  S[ P                  3,          R,          RS[S[,          RS[ P                  /# r{   r~   )r?   r@   s   "€r*   rA   r¢   É   sk   ø€ ÷ ñ á—|‘|ðñ Ÿ™ tÕ+ðñ #¡5§<¡<±·±Ð#=Õ>ÀÕEð	ñ
 Ñ+Õ,ðñ 
�‰ñr,   c                óü   € TpV P                   ! V P                  V4      V3R V/VB w  rgWVV P                  ,          ,           pTpWPP                  V P	                  V4      4      V P                  ,          ,           pV# )r}   )Ú	self_attnr¥   r©   Úmlpr¦   )r'   rW   r|   r}   r(   Úresidualrš   Ú_s   &&&&,   r*   r_   ÚEsmcLayer.forwardÉ   s†   € ð !ˆØŸšØ× Ñ  Ó/Øñ
ð !4ð
ð ñ	
‰ˆð !°×1DÑ1DÕ#DÕDˆØ ˆØ §8¡8¨D×,IÑ,IÈ-Ó,XÓ#YÐ\`×\oÑ\oÕ#oÕoˆØÐr,   )r¥   r¦   r©   r^   r�   rž   rR   s   @@r*   r    r    ¾   s#   ù‡ € Ù\÷Gõ G÷÷ ò r,   r    c                   ó@   a € ] tR t^Þt o ]tRtR.tR]R]	/t
R tRtV tR# )ÚEsmcPreTrainedModelr    r    rW   Ú
attentionsc                ó   € \        4       hr^   )rI   )r'   Úmodules   &&r*   Ú_init_weightsÚ!EsmcPreTrainedModel._init_weightsè   s   € ÜÓÐr,   r"   N)rC   rD   rE   rF   r   Úconfig_classÚbase_model_prefixÚ_no_split_modulesr    rj   Ú_can_record_outputsr·   rO   rP   )r@   s   @r*   r³   r³   Þ   s4   ø‡ € à€LØÐØ$˜Ðà˜Ø�mðÐ÷
ð r,   r³   c                   ór   a a€ ] tR t^ìt oV3R lV 3R llt]]]RV3R lR ll4       4       4       tRt	Vt
V ;t# )Ú	EsmcModelc                ó    <€ V ^8„  d   QhRS[ /# ©r4   rm   ©r   )r?   r@   s   "€r*   rA   ÚEsmcModel.__annotate__î   s   ø€ ÷ ñ ™zñ r,   c                ó¤  <€ \         SV `  V4       \        P                  ! VP                  VP
                  4      V n        \        V4      V n        \        P                  ! \        VP                  4       Uu. uF  p\        V4      NK  	  up4      V n        \        VP
                  R R7      V n        R V n        V P#                  4        R# u upi rr   )r%   rt   ÚnnÚ	Embeddingr5   r6   Úembed_tokensrd   Ú
rotary_embÚ
ModuleListÚranger8   r    ÚlayersrT   ÚnormÚgradient_checkpointingÚ	post_init)r'   rm   r°   r)   s   && €r*   rt   ÚEsmcModel.__init__î   s˜   ø€ Ü‰Ñ˜Ô ÜŸLšL¨×):Ñ):¸F×<NÑ<NÓOˆÔÜ-¨fÓ5ˆŒÜ—m’mÄÀf×F^ÑF^Ô@_Ó$`Ñ@_¸1¤Y¨vÖ%6Ñ@_Ñ$`ÓaˆŒÜ! &×"4Ñ"4¸5ÔAˆŒ	Ø&+ˆÔ#Ø�‰Öùò %as   Á8Cc                óÖ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          RS[/# )r4   Ú	input_idsNr|   Úposition_idsÚsequence_idr(   rX   )rZ   r[   r   r   r   )r?   r@   s   "€r*   rA   rÂ   ú   sy   ø€ ÷ L9ñ L9á—<‘< $Õ&ðL9ñ Ÿ™ tÕ+ðL9ñ —l‘l TÕ)ð	L9ñ
 —\‘\ DÕ(ðL9ñ Ñ+Õ,ðL9ñ 
ñL9r,   c                ó  € V P                  V4      pVfC   \        P                  ! VP                  ^,          VP                  R7      P                  ^ 4      pV P                  Wc4      pVeJ   \        V P                  4      '       d   \        R4      h\        V P                  VR\        V4      R7      pM\        V P                  VVR7      pV P                   F  pV! VV3RV/VB pK  	  V P                  V4      p	\        V	R7      # )a®  
position_ids (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
    Position of each token in the sequence, used by RoPE. Defaults to
    ``arange(sequence_length)``.

    ESMC positions are absolute over the whole input, and keep running *across* chain
    boundaries in a multi-chain input. The chain structure is carried separately by
    ``sequence_id``, which masks attention: unlike the packed-sequence format used elsewhere
    in the library, these are therefore not per-chain position ids restarting at 0, and the
    chain structure is deliberately not inferred from them.
sequence_id (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
    Integer chain-ID tensor for chain-aware attention masking. Tokens with the same
    non-negative integer value can attend to each other; tokens with different values
    cannot (cross-chain masking). Padding positions should be set to ``-1`` and inputs
    must be **right-padded** (RoPE uses absolute positions starting at 0). When provided,
    ``attention_mask`` is ignored. Passing ``sequence_id`` builds a custom attention
    mask, which requires ``torch>=2.6``. Multi-chain inputs additionally require a
    non-flash ``attn_implementation`` (``'sdpa'`` / ``'eager'`` / ``'flex_attention'``);
    flash attention only supports the single-chain case.

Examples:

```python
>>> from transformers import AutoTokenizer, EsmcModel

>>> model = EsmcModel.from_pretrained("biohub/ESMC-300M-hf")
>>> tokenizer = AutoTokenizer.from_pretrained("biohub/ESMC-300M-hf")
>>> inputs = tokenizer(["MLKNVQVQLV"], return_tensors="pt")
>>> outputs = model(**inputs)
>>> outputs.last_hidden_state.shape
torch.Size([1, 12, 960])
```
N)ÚdevicezÔ`sequence_id` (chain-aware attention) is not supported with flash attention, which can't represent the block-diagonal chain mask. Re-load the model with attn_implementation='sdpa' (or 'eager' / 'flex_attention').)rm   Úinputs_embedsr|   Úand_mask_function)rm   rÕ   r|   r}   )Úlast_hidden_state)rÆ   rZ   Úaranger„   rÔ   Ú	unsqueezerÇ   r   rm   r1   r   r   rÊ   rË   r   )
r'   rÐ   r|   rÑ   rÒ   r(   rW   r}   ÚlayerÚoutputs
   &&&&&,    r*   r_   ÚEsmcModel.forward÷   s  € ðX ×)Ñ)¨)Ó4ˆØÒÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\×fÑfÐghÓiˆLØ"Ÿo™o¨mÓJÐàÒ"Ü+¨D¯K©K×8Ò8Ü ðRóð ô
 7Ø—{‘{Ø+Ø#Ü"?ÀÓ"Lô	‰Nô 7Ø—{‘{Ø+Ø-ôˆNð —[”[ˆEÙ!ØØñð %8ðð ñ	ŠMñ !ð —‘˜=Ó)ˆä°Ô8Ð8r,   )rÆ   rÌ   rÊ   rË   rÇ   )NNNN)rC   rD   rE   rF   rt   r   r   r   r_   rO   rP   rQ   rR   s   @@r*   r¾   r¾   ì   sA   ù‡ € ÷ó ð  ØØ÷L9ñ L9ó ó ó  ÷L9ð L9r,   r¾   c                   óT   a a€ ] tR tRt oRV3R lV 3R llltV3R lR ltRtVtV ;t# )ÚEsmcMaskedLMHeadiI  c                ó>   <€ V ^8„  d   QhRS[ RS[ RS[ R,          RR/# )r4   r6   r5   r7   NrX   )r<   )r?   r@   s   "€r*   rA   ÚEsmcMaskedLMHead.__annotate__J  s5   ø€ ÷ @ñ @¡Cð @±Sð @ÉSÐSWÍZð @Ðcgñ @r,   c                óÖ   <€ \         SV `  4        Ve   TMTp\        P                  ! W4      V n        \        P
                  ! V4      V n        \        P                  ! W24      V n        R # r^   )r%   rt   rÄ   ÚLinearÚdenseÚ	LayerNormÚ
layer_normÚdecoder)r'   r6   r5   r7   r)   s   &&&&€r*   rt   ÚEsmcMaskedLMHead.__init__J  sO   ø€ Ü‰ÑÔØ1BÒ1NÑ-ÐT_ÐÜ—Y’Y˜{Ó>ˆŒ
ÜŸ,š,Ð'8Ó9ˆŒÜ—y’yÐ!2Ó?ˆŽr,   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rV   rY   )r?   r@   s   "€r*   rA   rà   Q  s#   ø€ ÷ +ñ +¡U§\¡\ð +±e·l±lñ +r,   c                ó”   € V P                  V4      p\        P                  ! V4      pV P                  V4      pV P	                  V4      # r^   )rã   ÚFÚgelurå   ræ   )r'   rW   s   &&r*   r_   ÚEsmcMaskedLMHead.forwardQ  s;   € ØŸ
™
 =Ó1ˆÜŸš˜}Ó-ˆØŸ™¨Ó6ˆØ�|‰|˜MÓ*Ð*r,   )ræ   rã   rå   r^   )	rC   rD   rE   rF   rt   r_   rO   rP   rQ   rR   s   @@r*   rÞ   rÞ   I  s    ù‡ € ÷@õ @÷+÷ +ð +r,   rÞ   c                   óŒ   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltV3R lR lt]]RV3R lR	 ll4       4       t	R
t
VtV ;t# )ÚEsmcForMaskedLMiX  c                ó    <€ V ^8„  d   QhRS[ /# rÀ   rÁ   )r?   r@   s   "€r*   rA   ÚEsmcForMaskedLM.__annotate__Z  s   ø€ ÷ ñ ™zñ r,   c                ó°   <€ \         SV `  V4       \        V4      V n        \	        VP
                  VP                  4      V n        V P                  4        R # r^   )	r%   rt   r¾   r    rÞ   r6   r5   Úlm_headrÍ   ©r'   rm   r)   s   &&€r*   rt   ÚEsmcForMaskedLM.__init__Z  s@   ø€ Ü‰Ñ˜Ô Ü˜fÓ%ˆŒ	Ü'¨×(:Ñ(:¸F×<MÑ<MÓNˆŒØ�‰Ör,   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r4   rX   ©rÄ   râ   )r?   r@   s   "€r*   rA   rð   c  s   ø€ ÷ $ñ $¡r§y¡yñ $r,   c                ó.   € V P                   P                  # r^   ©rò   ræ   )r'   s   &r*   Úget_output_embeddingsÚ%EsmcForMaskedLM.get_output_embeddingsc  s   € Ø�|‰|×#Ñ#Ð#r,   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r4   Únew_embeddingsrö   )r?   r@   s   "€r*   rA   rð   f  s   ø€ ÷ .ñ .±B·I±Iñ .r,   c                ó&   € WP                   n        R # r^   rø   )r'   rü   s   &&r*   Úset_output_embeddingsÚ%EsmcForMaskedLM.set_output_embeddingsf  s   € Ø-�‰Ör,   c                ó6  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          RS[S[ P                  R	3,          S[,          /# )
r4   rÐ   Nr|   rÑ   rÒ   Úlabelsr(   rX   .)rZ   r[   r   r   r   r	   )r?   r@   s   "€r*   rA   rð   k  s—   ø€ ÷ 3
ñ 3
á—<‘< $Õ&ð3
ñ Ÿ™ tÕ+ð3
ñ —l‘l TÕ)ð	3
ñ
 —\‘\ DÕ(ð3
ñ —‘˜tÕ#ð3
ñ Ñ+Õ,ð3
ñ 
‰u�|‰|˜SÐ Õ	!¡NÕ	2ñ3
r,   c                ó  € V P                   ! R
RVRVRVRVRR/VB pV P                  VP                  4      pRp	Ve*   V P                  ! W…3RV P                  P
                  /VB p	\        V	VVP                  VP                  R	7      # )aH  
sequence_id (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
    Integer chain-ID tensor forwarded to the encoder for chain-aware
    attention masking. See :meth:`EsmcModel.forward` for the encoding.
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for masked language modelling loss.  Positions with label ``-100``
    are ignored.  Other positions must be in ``[0, config.vocab_size)``.

Examples:

```python
>>> from transformers import AutoTokenizer, EsmcForMaskedLM
>>> import torch

>>> model = EsmcForMaskedLM.from_pretrained("biohub/ESMC-300M-hf")
>>> tokenizer = AutoTokenizer.from_pretrained("biohub/ESMC-300M-hf")
>>> inputs = tokenizer(["MLKNVQ<mask>LV"], return_tensors="pt")
>>> outputs = model(**inputs)
>>> outputs.logits.shape
torch.Size([1, 11, 64])
```
rÐ   r|   rÑ   rÒ   Úreturn_dictTNr5   )ÚlossÚlogitsrW   r´   r"   )	r    rò   r×   Úloss_functionrm   r5   r	   rW   r´   )
r'   rÐ   r|   rÑ   rÒ   r  r(   Úencoder_outputsr  r  s
   &&&&&&,   r*   r_   ÚEsmcForMaskedLM.forwardi  s¶   € ðB Ÿ)š)ñ 
Øð
à)ð
ð &ð
ð $ð	
ð
 ð
ð ñ
ˆð —‘˜o×?Ñ?Ó@ˆà$(ˆØÒØ×%Ò% fÑbÀÇÁ×AWÑAWÐbÐ[aÑbˆDäØØØ)×7Ñ7Ø&×1Ñ1ô	
ð 	
r,   )r    rò   )NNNNN)rC   rD   rE   rF   rt   rù   rþ   r   r   r_   rO   rP   rQ   rR   s   @@r*   rî   rî   X  sI   ù‡ € ÷ó ÷$ð $÷.ð .ð Ø÷3
ñ 3
ó ó ÷3
ð 3
r,   rî   c                   ó>   a a€ ] tR tRt oV3R lV 3R lltRtVtV ;t# )ÚEsmcClassificationHeadi¡  c                ó    <€ V ^8„  d   QhRS[ /# rÀ   rÁ   )r?   r@   s   "€r*   rA   Ú#EsmcClassificationHead.__annotate__¢  s   ø€ ÷ Iñ I™zñ Ir,   c                ó.  <€ \         SV `  4        \        P                  ! VP                  VP                  4      V n        \        P                  ! VP                  4      V n        \        P                  ! VP                  VP                  4      V n
        R # r^   )r%   rt   rÄ   râ   r6   rã   ÚDropoutr;   r�   Ú
num_labelsÚout_projró   s   &&€r*   rt   ÚEsmcClassificationHead.__init__¢  s`   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
Ü—z’z &×";Ñ";Ó<ˆŒÜŸ	š	 &×"4Ñ"4°f×6GÑ6GÓHˆŽr,   )rã   r�   r  ©rC   rD   rE   rF   rt   rO   rP   rQ   rR   s   @@r*   r
  r
  ¡  s   ù‡ € ÷I÷ Ió Ir,   r
  c                   ó>   a a€ ] tR tRt oV3R lV 3R lltRtVtV ;t# )ÚEsmcForSequenceClassificationi©  c                ó    <€ V ^8„  d   QhRS[ /# rÀ   rÁ   )r?   r@   s   "€r*   rA   Ú*EsmcForSequenceClassification.__annotate__ª  s   ø€ ÷ ñ ™zñ r,   c                ó¨   <€ \         SV `  V4       VP                  V n        \        V4      V n        \        V4      V n        V P                  4        R # r^   )r%   rt   r  r¾   r    r
  Ú
classifierrÍ   ró   s   &&€r*   rt   Ú&EsmcForSequenceClassification.__init__ª  s?   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒÜ˜fÓ%ˆŒ	Ü0°Ó8ˆŒØ�‰Ör,   )r  r    r  r  rR   s   @@r*   r  r  ©  s   ù‡ € ÷÷ ó r,   r  c                   ó   € ] tR tRtRtRtR# )ÚEsmcForTokenClassificationi²  r    r"   N)rC   rD   rE   rF   rº   rO   r"   r,   r*   r  r  ²  s   † ØÔr,   r  )r   r¾   rî   r  r  r³   )BrG   r§   Úcollections.abcr   rZ   Útorch.nnrÄ   Úhuggingface_hub.dataclassesr   r   rê   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r	   Úmodeling_utilsr
   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.genericr   r   Úutils.output_capturingr   Úesm.modeling_esmr   r   r   Úllama.configuration_llamar   Úllama.modeling_llamar   r   r   r   r   Únomic_bert.modeling_nomic_bertr   Ú
get_loggerrC   Úloggerr   rä   rT   rd   rh   rj   r    r³   r¾   ÚModulerÞ   rî   r
  r  r  Ú__all__r"   r,   r*   Ú<module>r/     s|  ðñ ã Ý $ã Ý Ý .Ý $ç UÝ <÷õ 6Ý &÷ó ÷ VÝ 5ß kÑ kÝ 3÷õ õ Fð 
×	Ò	˜HÓ	%€ñ Ð.Ô/Øô=�ó =ó ó 0ð=ô@	F�B—L‘Lô 	Fô	Ð.ô 	ô	ˆhô 	ô.6�Nô .6ôbÐ!ô ð@ ô
Ð2ó 
ó ð
ð ôY9Ð#ó Y9ó ðY9ôx+�r—y‘yô +ð ôE
Ð)ó E
ó ðE
ôPIÐ2ô IôÐ$@ô ôÐ!>Ð@Sô ò‚r,   