+
    œ°³j  ã                   óx   € R t ^ RIt^RIHt ^RIHt ]P                  ! ]4      tRR/t	R t
 ! R R]4      tR.tR# )	zTokenization classes for ESM.N)ÚPreTrainedTokenizer)ÚloggingÚ
vocab_fileú	vocab.txtc                 óì   € \        V R 4      ;_uu_ 4       pVP                  4       P                  4       pV Uu. uF  q3P                  4       NK  	  upuuRRR4       # u upi   + '       g   i     R# ; i)ÚrN)ÚopenÚreadÚ
splitlinesÚstrip)r   ÚfÚlinesÚls   &   Úi/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/esm/tokenization_esm.pyÚload_vocab_filer      sR   € Ü	ˆj˜#×	Ô	 !Ø—‘“×#Ñ#Ó%ˆÙ#(Ó)¡5˜a—‘–	¡5Ñ)÷ 
Ò	ùâ)÷ 
×	Ð	ús   •#A"¸AÁA"ÁA"Á"A3	c                   óè   a a€ ] tR t^!t oRt]tRR.tRV 3R lltV3R lR lt	V3R lR lt
R	 tR
 tV3R lR ltV3R lR ltRV3R lR lltRV3R lR lltR t]V3R lR l4       tRtVtV ;t# )ÚEsmTokenizerz
Constructs an ESM tokenizer.
Ú	input_idsÚattention_maskc                ób  <€ \        V4      V n        \        \        V P                  4      4      V n        \        V P                  4       UU	u/ uF  w  r‰W˜bK	  	  up	pV n        \        S
V `  ! RR VRVRVRVRV/VB  V P                  V n        V P                  V P                  4       R# u up	pi )Ú	unk_tokenÚ	cls_tokenÚ	pad_tokenÚ
mask_tokenÚ	eos_tokenN© )
r   Ú
all_tokensÚdictÚ	enumerateÚ_id_to_tokenÚ_token_to_idÚsuperÚ__init__Úunique_no_split_tokensÚ_update_trie)Úselfr   r   r   r   r   r   ÚkwargsÚindÚtokÚ	__class__s   &&&&&&&,  €r   r"   ÚEsmTokenizer.__init__)   sº   ø€ ô *¨*Ó5ˆŒÜ ¤¨4¯?©?Ó!;Ó<ˆÔÜ6?ÀÇÁÔ6PÔQÑ6P©(¨#˜SšXÑ6PÒQˆÔÜ‰Òñ 	
Øð	
àð	
ð  ð	
ð "ð		
ð
  ð	
ð ò	
ð '+§o¡oˆÔ#Ø×Ñ˜$×5Ñ5Ö6ùó Rs   ÁB+c                ó&   <€ V ^8„  d   QhRS[ RS[/# ©é   ÚindexÚreturn©ÚintÚstr)ÚformatÚ__classdict__s   "€r   Ú__annotate__ÚEsmTokenizer.__annotate__E   s   ø€ ÷ <ñ <©#ð <±#ñ <ó    c                óL   € V P                   P                  WP                  4      # ©N©r   Úgetr   ©r%   r.   s   &&r   Ú_convert_id_to_tokenÚ!EsmTokenizer._convert_id_to_tokenE   ó   € Ø× Ñ ×$Ñ$ U¯N©NÓ;Ð;r7   c                ó&   <€ V ^8„  d   QhRS[ RS[/# ©r-   Útokenr/   ©r2   r1   )r3   r4   s   "€r   r5   r6   H   s   ø€ ÷ Sñ S©#ð S±#ñ Sr7   c                ó~   € V P                   P                  WP                   P                  V P                  4      4      # r9   ©r    r;   r   ©r%   rB   s   &&r   Ú_convert_token_to_idÚ!EsmTokenizer._convert_token_to_idH   ó.   € Ø× Ñ ×$Ñ$ U×,=Ñ,=×,AÑ,AÀ$Ç.Á.Ó,QÓRÐRr7   c                ó"   € VP                  4       # r9   )Úsplit)r%   Útextr&   s   &&,r   Ú	_tokenizeÚEsmTokenizer._tokenizeK   s   € Ø�z‰z‹|Ðr7   c                óp   € V P                   P                  4       pVP                  V P                  4       V# r9   )r    ÚcopyÚupdateÚadded_tokens_encoder)r%   Ú
base_vocabs   & r   Ú	get_vocabÚEsmTokenizer.get_vocabN   s0   € Ø×&Ñ&×+Ñ+Ó-ˆ
Ø×Ñ˜$×3Ñ3Ô4ØÐr7   c                ó&   <€ V ^8„  d   QhRS[ RS[/# rA   rC   )r3   r4   s   "€r   r5   r6   S   s   ø€ ÷ Sñ S¡ð S©ñ Sr7   c                ó~   € V P                   P                  WP                   P                  V P                  4      4      # r9   rE   rF   s   &&r   Útoken_to_idÚEsmTokenizer.token_to_idS   rI   r7   c                ó&   <€ V ^8„  d   QhRS[ RS[/# r,   r0   )r3   r4   s   "€r   r5   r6   V   s   ø€ ÷ <ñ <¡ð <©ñ <r7   c                óL   € V P                   P                  WP                  4      # r9   r:   r<   s   &&r   Úid_to_tokenÚEsmTokenizer.id_to_tokenV   r?   r7   c                ój   <€ V ^8„  d   QhRS[ S[,          RS[ S[,          R,          RS[ S[,          /# )r-   Útoken_ids_0Útoken_ids_1Nr/   )Úlistr1   )r3   r4   s   "€r   r5   r6   Y   s6   ø€ ÷ ;ñ ;Ù¡�9ð;Ù37¹µ9¸tÕ3Cð;á	‰c�ñ;r7   c                óô   € V P                   .pV P                  .pVf&   V P                  f	   W1,           # W1,           V,           # V P                  f   \        R4      hW1,           V,           V,           V,           # )Nz=Cannot tokenize multiple sequences when EOS token is not set!)Úcls_token_idÚeos_token_idÚ
ValueError)r%   r_   r`   ÚclsÚseps   &&&  r   Ú build_inputs_with_special_tokensÚ-EsmTokenizer.build_inputs_with_special_tokensY   sy   € ð × Ñ Ð!ˆØ× Ñ Ð!ˆØÒØ× Ñ Ò(ØÕ(Ð(àÕ(¨3Õ.Ð.Ø×ÑÒ&ÜÐ\Ó]Ð]ØÕ  3Õ&¨Õ4°sÕ:Ð:r7   c          	      óP   <€ V ^8„  d   QhRS[ RS[ R,          RS[RS[ S[,          /# )r-   r_   r`   NÚalready_has_special_tokensr/   )ra   Úboolr1   )r3   r4   s   "€r   r5   r6   g   s5   ø€ ÷ ñ ÙðÙ.2°T­kðÙ^bðá	‰c�ñr7   c                ó.  € V'       d?   Ve   \        R4      h\        V P                  4      pV Uu. uF  qUV9   d   ^M^ NK  	  up# ^.^ .\        V4      ,          ,           ^.,           pVe#   V^ .\        V4      ,          ^.,           ,          pV# u upi )at  
Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding
special tokens using the tokenizer `prepare_for_model` or `encode_plus` methods.

Args:
    token_ids_0 (`list[int]`):
        List of ids of the first sequence.
    token_ids_1 (`list[int]`, *optional*):
        List of ids of the second sequence.
    already_has_special_tokens (`bool`, *optional*, defaults to `False`):
        Whether or not the token list is already formatted with special tokens for the model.

Returns:
    A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
zYou should not supply a second sequence if the provided sequence of ids is already formatted with special tokens for the model.)re   ÚsetÚall_special_idsÚlen)r%   r_   r`   rk   ro   rB   Úmasks   &&&&   r   Úget_special_tokens_maskÚ$EsmTokenizer.get_special_tokens_maskg   sž   € ÷$ &ØÒ&Ü ðRóð ô
 " $×"6Ñ"6Ó7ˆOÙFQÓRÁk¸U /Ô1‘A°qÒ8ÁkÑRÐRØˆs�q�cœC Ó,Õ,Õ-°°Õ3ˆØÒ"Ø�Q�Cœ#˜kÓ*Õ*¨a¨SÕ0Õ0ˆDØˆùò	 Ss   ±Bc                ó.  € \         P                  P                  Y'       d
   VR ,           MRR,           4      p\        VR4      ;_uu_ 4       pVP	                  RP                  V P
                  4      4       RRR4       V3#   + '       g   i     T3# ; i)Ú-Ú r   ÚwÚ
N)ÚosÚpathÚjoinr   Úwriter   )r%   Úsave_directoryÚfilename_prefixr   r   s   &&&  r   Úsave_vocabularyÚEsmTokenizer.save_vocabulary‡   sl   € Ü—W‘W—\‘\ .ÎO°?ÀSÖ3HÐacÐgrÕ2rÓsˆ
Ü�*˜c×"Ô" aØ�G‰G�D—I‘I˜dŸo™oÓ.Ô/÷ #àˆ}Ð÷ #Ö"àˆ}Ðús   Á+BÂB	c                ó    <€ V ^8„  d   QhRS[ /# )r-   r/   )r1   )r3   r4   s   "€r   r5   r6   Ž   s   ø€ ÷ $ñ $™Cñ $r7   c                ó,   € \        V P                  4      # r9   )rp   r   )r%   s   &r   Ú
vocab_sizeÚEsmTokenizer.vocab_size�   s   € ä�4—?‘?Ó#Ð#r7   )r   r    r   r#   )z<unk>z<cls>z<pad>z<mask>z<eos>r9   )NF)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__ÚVOCAB_FILES_NAMESÚvocab_files_namesÚmodel_input_namesr"   r=   rG   rM   rT   rX   r\   rh   rr   r   Úpropertyrƒ   Ú__static_attributes__Ú__classdictcell__Ú__classcell__)r)   r4   s   @@r   r   r   !   sŒ   ù‡ € ñð *ÐØ$Ð&6Ð7Ð÷7÷8<ð <÷Sð Sòò÷
Sð S÷<ð <÷;ò ;÷ò ò@ð ÷$ó ÷$ð $r7   r   )r‰   ry   Útokenization_pythonr   Úutilsr   Ú
get_loggerr…   ÚloggerrŠ   r   r   Ú__all__r   r7   r   Ú<module>r–      sO   ðñ $ã 	å 6Ý ð 
×	Ò	˜HÓ	%€à! ;Ð/Ð ò*ôn$Ð&ô n$ðb Ð
‚r7   