+
    œ°³j9  ã                   óÆ   € R t ^ RIt^ RIt^ RIHtHt ]! 4       '       d   ^ RIt^RIHt ^RI	H
t
 ]
P                  ! ]4      tRR/tR t ! R R	4      t ! R
 R]4      tR.tR# )z Tokenization classes for CPMAnt.N)Úis_rjieba_availableÚrequires_backends)ÚPreTrainedTokenizer)ÚloggingÚ
vocab_filez	vocab.txtc                ó  € \         P                  ! 4       p\        V RRR7      ;_uu_ 4       pVP                  4       pRRR4       \	        X4       F  w  rEVP                  R4      pWAV&   K  	  V#   + '       g   i     L;; i)z*Loads a vocabulary file into a dictionary.Úrúutf-8©ÚencodingNÚ
)ÚcollectionsÚOrderedDictÚopenÚ	readlinesÚ	enumerateÚrstrip)r   ÚvocabÚreaderÚtokensÚindexÚtokens   &     Úo/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/cpmant/tokenization_cpmant.pyÚ
load_vocabr   "   sj   € ä×#Ò#Ó%€EÜ	ˆj˜#¨×	0Õ	0°FØ×!Ñ!Ó#ˆ÷ 
1ä! &Ö)‰ˆØ—‘˜TÓ"ˆØˆe‹ñ *ð €L÷ 
1×	0ús   ¬A0Á0B 	c                   ó0   a € ] tR t^-t o RR ltR tRtV tR# )ÚWordpieceTokenizerc                ó*   € Wn         W n        W0n        R # ©N)r   Ú	unk_tokenÚmax_input_chars_per_word)Úselfr   r   r   s   &&&&r   Ú__init__ÚWordpieceTokenizer.__init__.   s   € ØŒ
Ø"ŒØ(@Ö%ó    c                ó˜  € \        V4      p\        V4      V P                  8”  d   V P                  .# ^ p. pV\        V4      8  d„   \        V4      pRpW58  d2   RP	                  W#V 4      pWpP
                  9   d   TpMV^,          pK7  Vf'   VP                  V P                  4       V^,          pK~  VP                  V4       TpK“  V# )é    NÚ )ÚlistÚlenr   r   Újoinr   Úappend)r    r   ÚcharsÚstartÚ
sub_tokensÚendÚ
cur_substrÚsubstrs   &&      r   ÚtokenizeÚWordpieceTokenizer.tokenize3   s»   € Ü�U“ˆÜˆu‹:˜×5Ñ5Ô5Ø—N‘NÐ#Ð#àˆØˆ
Ø”c˜%“jÔ Ü�e“*ˆCØˆJØ”+ØŸ™ ¨SÐ!1Ó2�ØŸZ™ZÔ'Ø!'�JØØ�q•’ØÒ!Ø×!Ñ! $§.¡.Ô1Ø˜•
’à×!Ñ! *Ô-Ø’àÐr#   )r   r   r   N)ú<unk>éÈ   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r!   r1   Ú__static_attributes__Ú__classdictcell__)Ú__classdict__s   @r   r   r   -   s   ø‡ € ôA÷
ð r#   r   c                   óè   a a€ ] tR t^Mt oRt]tRR.tRtRV 3R llt	]
R 4       t]
R 4       t]
R 4       t]
V3R	 lR
 l4       tR tR tV 3R ltR tV3R lR ltR tR tRV3R lR lltRtVtV ;t# )ÚCpmAntTokenizera^  
Construct a CPMAnt tokenizer. Based on byte-level Byte-Pair-Encoding.

Args:
    vocab_file (`str`):
        Path to the vocabulary file.
    bod_token (`str`, *optional*, defaults to `"<d>"`):
        The beginning of document token.
    eod_token (`str`, *optional*, defaults to `"</d>"`):
        The end of document token.
    bos_token (`str`, *optional*, defaults to `"<s>"`):
        The beginning of sequence token.
    eos_token (`str`, *optional*, defaults to `"</s>"`):
        The end of sequence token.
    pad_token (`str`, *optional*, defaults to `"<pad>"`):
        The token used for padding.
    unk_token (`str`, *optional*, defaults to `"<unk>"`):
        The unknown token.
    line_token (`str`, *optional*, defaults to `"</n>"`):
        The line token.
    space_token (`str`, *optional*, defaults to `"</_>"`):
        The space token.
Ú	input_idsÚattention_maskFc                ó6  <€ \        V R .4       W n        W0n        \        V4      V n        V P                  V	,          V P                  R&   V P                  V,          V P                  R&   V P                  V	 V P                  V \
        P                  ! \        V P                  P                  4       R R7      4      V n        V P                  P                  4        UUu/ uF  w  rÍWÜbK	  	  uppV n	        \        V P                  VR7      V n        \        SV `4  ! RRVRVRVR	VR
VRVRVRV	RV
RRRRRR/VB  W˜3 FA  pV P                  P                  VR4      pVf   K%  V P                   P                  VR4       KC  	  V P#                  4        R# u uppi )ÚrjiebaÚ r   c                 ó   € V ^,          # ©é   © ©Úxs   &r   Ú<lambda>Ú*CpmAntTokenizer.__init__.<locals>.<lambda>‚   ó   € ÐZ[Ð\]ÖZ^r#   ©Úkey)r   r   Ú	bod_tokenÚ	eod_tokenÚ	bos_tokenÚ	eos_tokenÚ	pad_tokenr   Ú
line_tokenÚspace_tokenÚpadding_sideÚtoken_type_ids_patternÚ	all_zerosÚ%token_type_ids_include_special_tokensTÚspecial_tokens_patternÚbosNrF   )r   rN   rO   r   Úencoderr   r   ÚsortedÚitemsÚdecoderr   Úwordpiece_tokenizerÚsuperr!   Úadded_tokens_encoderÚpopÚ_added_tokens_decoderÚ_update_total_vocab_size)r    r   rN   rO   rP   rQ   rR   r   rS   rT   rU   ÚkwargsÚkÚvÚspecial_tokenÚtoken_idÚ	__class__s   &&&&&&&&&&&,    €r   r!   ÚCpmAntTokenizer.__init__j   s²  ø€ ô 	˜$  
Ô+Ø"ŒØ"ŒÜ! *Ó-ˆŒØ ŸL™L¨Õ5ˆ�‰�SÑØ!Ÿ\™\¨*Õ5ˆ�‰�TÑà�L‰L˜Ð%Ø�L‰L˜Ð$ä"×.Ò.¬v°d·l±l×6HÑ6HÓ6JÑP^Ô/_Ó`ˆŒØ)-¯©×);Ñ);Ô)=Ô>Ñ)=¡ ˜šÑ)=Ò>ˆŒä#5¸D¿L¹LÐT]Ô#^ˆÔ ä‰Òñ 	
Øð	
àð	
ð  ð	
ð  ð		
ð
  ð	
ð  ð	
ð "ð	
ð $ð	
ð &ð	
ð $/ð	
ð 37ð	
ð $)ð	
ð ò	
ð *Ó6ˆMØ×0Ñ0×4Ñ4°]ÀDÓIˆHØÔ#Ø×*Ñ*×.Ñ.¨x¸Ö>ñ 7ð 	×%Ñ%Ö'ùó/ ?s   Ã!Fc                ó<   € V P                   V P                  ,          # r   )r[   rN   ©r    s   &r   Úbod_token_idÚCpmAntTokenizer.bod_token_idœ   ó   € à�|‰|˜DŸN™NÕ+Ð+r#   c                ó<   € V P                   V P                  ,          # r   )r[   rO   rm   s   &r   Úeod_token_idÚCpmAntTokenizer.eod_token_id    rp   r#   c                ó(   € V P                   R ,          # )r   ©r[   rm   s   &r   Ú
newline_idÚCpmAntTokenizer.newline_id¤   s   € à�|‰|˜DÕ!Ð!r#   c                ó    <€ V ^8„  d   QhRS[ /# )é   Úreturn)Úint)Úformatr;   s   "€r   Ú__annotate__ÚCpmAntTokenizer.__annotate__©   s   ø€ ÷ !ñ !™Cñ !r#   c                ó,   € \        V P                  4      # r   )r(   r[   rm   s   &r   Ú
vocab_sizeÚCpmAntTokenizer.vocab_size¨   s   € ä�4—<‘<Ó Ð r#   c                óB   € \        V P                  3/ V P                  B # r   )Údictr[   ra   rm   s   &r   Ú	get_vocabÚCpmAntTokenizer.get_vocab¬   s   € Ü�D—L‘LÑ> D×$=Ñ$=Ñ>Ð>r#   c                óš   € . p\         P                  ! VR4       F-  pVP                  V P                  P	                  V4      4       K/  	  V# )zTokenize a string.F)rA   ÚcutÚextendr_   r1   )r    ÚtextÚoutput_tokensrH   s   &&  r   Ú	_tokenizeÚCpmAntTokenizer._tokenize¯   sA   € àˆÜ—’˜D %Ö(ˆAØ× Ñ  ×!9Ñ!9×!BÑ!BÀ1Ó!EÖFñ )àÐr#   c                óú   <€ V Uu. uF  q3^ 8¼  g   K  VNK  	  ppV Uu. uF;  qDV P                   8w  g   K  W@P                  8w  g   K'  W@P                  8w  g   K9  VNK=  	  pp\        SV `  ! V3/ VB # u upi u upi )zDecode ids into a string.)Úpad_token_idÚeos_token_idÚbos_token_idr`   Ú_decode)r    Ú	token_idsre   ÚirH   rj   s   &&,  €r   r‘   ÚCpmAntTokenizer._decode¶   s�   ø€ á )Ó4¡	˜1°!©V—Q�Q¡	ˆ	Ð4á ó
Ù �!¨×):Ñ):Ñ$:ŒA¸q×DUÑDUÑ?UŒAÐZ[×_pÑ_pÑZp�AˆA‘yð 	ð 
ô ‰wŠ˜yÑ3¨FÑ3Ð3ùò	 5ùò
s    †A3“A3ŸA8¶A8ÁA8ÁA8c                ó   € WP                   9   # r   ru   ©r    r   s   &&r   ÚcheckÚCpmAntTokenizer.check¾   s   € ØŸ™Ñ$Ð$r#   c                ó6   <€ V ^8„  d   QhRS[ S[,          RS[/# )ry   r   rz   )r'   Ústr)r|   r;   s   "€r   r}   r~   Á   s   ø€ ÷ ñ ©t±C­yð ¹Sñ r#   c                ó$   € R P                  V4      # )r&   )r)   )r    r   s   &&r   Úconvert_tokens_to_stringÚ(CpmAntTokenizer.convert_tokens_to_stringÁ   s   € Ø�w‰w�v‹Ðr#   c                ó~   € V P                   P                  WP                   P                  V P                  4      4      # )z0Converts a token (str) in an id using the vocab.)r[   Úgetr   r–   s   &&r   Ú_convert_token_to_idÚ$CpmAntTokenizer._convert_token_to_idÄ   s*   € à�|‰|×Ñ §|¡|×'7Ñ'7¸¿¹Ó'GÓHÐHr#   c                óL   € V P                   P                  WP                  4      # )z=Converts an index (integer) in a token (str) using the vocab.)r^   rŸ   r   )r    r   s   &&r   Ú_convert_id_to_tokenÚ$CpmAntTokenizer._convert_id_to_tokenÈ   s   € à�|‰|×Ñ §~¡~Ó6Ð6r#   c                óJ   <€ V ^8„  d   QhRS[ RS[ R,          RS[S[ ,          /# )ry   Úsave_directoryÚfilename_prefixNrz   )rš   Útuple)r|   r;   s   "€r   r}   r~   Ì   s-   ø€ ÷ ñ ©cð ÁCÈ$ÅJð ÑZ_Ñ`cÕZdñ r#   c                ó¸  € \         P                  P                  V4      '       dD   \         P                  P                  Y'       d
   VR ,           MR\        R,          ,           4      pMV'       d
   VR ,           MRV,           p^ pRV P
                  9   d.   V P
                  R,          V P
                  R&   V P
                  R RV P
                  9   d.   V P
                  R,          V P
                  R&   V P
                  R \        P                  ! \        V P
                  P                  4       R R7      4      V n        \        VR	R
R7      ;_uu_ 4       pV P
                  P                  4        FG  w  rgWG8w  d   \        P                  RV R24       TpVP                  VR,           4       V^,          pKI  	  RRR4       V3#   + '       g   i     T3# ; i)Ú-r&   r   rB   ú</_>r   ú</n>c                 ó   € V ^,          # rD   rF   rG   s   &r   rI   Ú1CpmAntTokenizer.save_vocabulary.<locals>.<lambda>Ú   rK   r#   rL   Úwr	   r
   zSaving vocabulary to z\: vocabulary indices are not consecutive. Please check that the vocabulary is not corrupted!N)ÚosÚpathÚisdirr)   ÚVOCAB_FILES_NAMESr[   r   r   r\   r]   r   ÚloggerÚwarningÚwrite)r    r¦   r§   r   r   Úwriterr   Útoken_indexs   &&&     r   Úsave_vocabularyÚCpmAntTokenizer.save_vocabularyÌ   ss  € Ü�7‰7�=‰=˜×(Ò(ÜŸ™Ÿ™Ø¾/ °3Ö!6ÈrÔUfÐgsÕUtÕ tó‰J÷ 4C˜/¨CÖ/ÈÈnÕ\ˆJØˆØ�$—,‘,ÔØ#'§<¡<°Õ#4ˆD�L‰L˜Ñ Ø—‘˜SÐ!Ø�4—<‘<ÔØ#'§<¡<°Õ#5ˆD�L‰L˜Ñ Ø—‘˜TÐ"Ü"×.Ò.¬v°d·l±l×6HÑ6HÓ6JÑP^Ô/_Ó`ˆŒÜ�*˜c¨G×4Õ4¸Ø&*§l¡l×&8Ñ&8Ö&:Ñ"�ØÔ'Ü—N‘NØ/°
¨|ð <Nð Nôð (�EØ—‘˜U T�\Ô*Ø˜•
’ñ ';÷ 5ð ˆ}Ð÷ 5Ö4ð ˆ}Ðús   ÅA&GÇG	)rN   r^   r[   rO   r_   )	z<d>z</d>z<s>z</s>z<pad>r3   r¬   r«   Úleftr   )r5   r6   r7   r8   Ú__doc__r³   Úvocab_files_namesÚmodel_input_namesÚadd_prefix_spacer!   Úpropertyrn   rr   rv   r€   r„   r‹   r‘   r—   rœ   r    r£   r¹   r9   r:   Ú__classcell__)rj   r;   s   @@r   r=   r=   M   s²   ù‡ € ñð0 *ÐØ$Ð&6Ð7ÐØÐ÷0(ðd ñ,ó ð,ð ñ,ó ð,ð ñ"ó ð"ð ÷!ó ð!ò?òõ4ò%÷ð òIò7÷÷ ò r#   r=   )r¼   r   r°   Útransformers.utilsr   r   rA   Útokenization_pythonr   Úutilsr   Ú
get_loggerr5   r´   r³   r   r   r=   Ú__all__rF   r#   r   Ú<module>rÇ      sn   ðñ 'ã Û 	ç Eñ ×ÒÛå 6Ý ð 
×	Ò	˜HÓ	%€à! ;Ð/Ð ò÷ñ ô@XÐ)ô Xðv Ð
‚r#   