+
    œ°³j4  ã                   ó  € R t ^ RIHt ^ RIHt ^ RIHt ^RIHt ^RI	H
t
 ]
P                  ! ]4      tRR/t. RNR	NR
NRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNR NR!NR"NR#NR$NR%NR&NR'NR(Nt ! R) R*]4      tR*.tR+# ),zTokenization classes for ESMC.)Ú	Tokenizer)ÚBPE)ÚTemplateProcessing)ÚTokenizersBackend)ÚloggingÚtokenizer_fileztokenizer.jsonú<cls>ú<pad>ú<eos>ú<unk>ÚLÚAÚGÚVÚSÚEÚRÚTÚIÚDÚPÚKÚQÚNÚFÚYÚMÚHÚWÚCÚXÚBÚUÚZÚOÚ.Ú-Ú|ú<mask>c                   óJ   a a€ ] tR t^Ct oRt]tRR.t]t	RV 3R llt
RtVtV ;t# )ÚEsmcTokenizera1  
Construct an ESMC tokenizer.

This tokenizer is a character-level tokenizer backed by the HuggingFace `tokenizers` library.
It wraps every sequence with ``<cls>`` and ``<eos>`` tokens and supports a ``|`` chain-break
token for multi-chain inputs.

Args:
    unk_token (`str`, *optional*, defaults to `"<unk>"`):
        The unknown token.
    cls_token (`str`, *optional*, defaults to `"<cls>"`):
        The classification token (prepended to every sequence).
    pad_token (`str`, *optional*, defaults to `"<pad>"`):
        The padding token.
    mask_token (`str`, *optional*, defaults to `"<mask>"`):
        The mask token, used for masked language modelling.
    eos_token (`str`, *optional*, defaults to `"<eos>"`):
        The end-of-sequence token (appended to every sequence).
    bos_token (`str`, *optional*, defaults to `"<cls>"`):
        The beginning-of-sequence token (prepended to every sequence). When unset, uses cls_token.
    chain_break_token (`str`, *optional*, defaults to `"|"`):
        Token inserted between chains in multi-chain protein inputs. Registered as a model-specific
        special token, so it is exposed as `chain_break_token` / `chain_break_token_id`.
    extra_special_tokens (`list[str]`, *optional*):
        Additional special tokens to register with the tokenizer (round-tripped from a saved config).

Examples:

```python
>>> from transformers import EsmcTokenizer

>>> tokenizer = EsmcTokenizer()
>>> tokenizer("ACDEFGHIKLMNPQRSTVWY")["input_ids"]
[0, 5, 23, 13, 9, 18, 6, 21, 12, 15, 4, 20, 17, 14, 16, 10, 8, 11, 7, 22, 19, 2]
```
Ú	input_idsÚattention_maskc	                óÌ  <€ \        \        4       U
Uu/ uF  w  r«WºbK	  	  pp
p\        \        V. \	        V4      R 7      4      V n        \        SV `  ! RRTRVf   TMTRVRVRVRVRVR	V/V	B  \        V P                   R
V P                   2V P                  WÀP                  ,          3V P                  WÀP                  ,          3.R7      V P
                  n        R# u upp
i ))ÚmergesÚ	unk_tokenr/   Ú	bos_tokenNÚ	cls_tokenÚ	pad_tokenÚ
mask_tokenÚ	eos_tokenÚchain_break_tokenÚextra_special_tokensz $A )ÚsingleÚspecial_tokens© )Ú	enumerateÚSEQUENCE_VOCABr   r   ÚstrÚ
_tokenizerÚsuperÚ__init__r   r1   r4   Úpost_processor)Úselfr/   r1   r2   r3   r4   r0   r5   r6   ÚkwargsÚindÚtokÚtoken_to_idÚ	__class__s   &&&&&&&&&,   €Úk/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/esmc/tokenization_esmc.pyr?   ÚEsmcTokenizer.__init__m   s  ø€ ô 1:¼.Ô0IÔJÑ0I¡H C�s’xÑ0IˆÑJô $¤C¨¸BÌ#ÈiË.Ô$YÓZˆŒä‰Òñ 	
Øð	
ð $-Ò#4‘i¸)ð	
ð  ð		
ð
  ð	
ð "ð	
ð  ð	
ð 0ð	
ð "6ð	
ð ò	
ô *<Ø—n‘nÐ% T¨$¯.©.Ð)9Ð:à—‘ ¯^©^Õ!<Ð=Ø—‘ ¯^©^Õ!<Ð=ðô*
ˆ�‰Ö&ùó' Ks   ”C )r=   )r   r   r	   r(   r
   Nr'   N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__ÚVOCAB_FILES_NAMESÚvocab_files_namesÚmodel_input_namesr   Úmodelr?   Ú__static_attributes__Ú__classdictcell__Ú__classcell__)rF   Ú__classdict__s   @@rG   r*   r*   C   s.   ù‡ € ñ#ðJ *ÐØ$Ð&6Ð7ÐØ€E÷%
÷ %
ó    r*   N)rM   Ú
tokenizersr   Útokenizers.modelsr   Útokenizers.processorsr   Útokenization_utils_tokenizersr   Úutilsr   Ú
get_loggerrI   ÚloggerrN   r;   r*   Ú__all__r9   rV   rG   Ú<module>r_      s£  ðñ %å  Ý !Ý 4å >Ý ð 
×	Ò	˜HÓ	%€à%Ð'7Ð8Ð ð"Øð"àð"ð ð"ð ð	"ð
 ð"ð ð"ð ð"ð ð"ð ð"ð ð"ð ð"ð ð"ð ð"ð ð"ð ð"ð  ð!"ð" ð#"ð$ ð%"ð& ð'"ð( ð)"ð* ð+"ð, ð-"ð. ð/"ð0 ð1"ð2 ð3"ð4 ð5"ð6 ð7"ð8 ð9"ð: ð;"ð< ð="ð> ð?"ð@ ðA"ðB ðC"€ôJO
Ð%ô O
ðd Ð
‚rV   