+
    œ°³jåE  ã                   ó¤   € R t ^ RIt^ RIt^ RIt^ RIt^RIHt ^RIHt ]P                  ! ]
4      tRRRRRR	/tR
 tR tR t ! R R]4      tR.tR# )zTokenization classes for FSMT.N)ÚPreTrainedTokenizer)ÚloggingÚsrc_vocab_filezvocab-src.jsonÚtgt_vocab_filezvocab-tgt.jsonÚmerges_filez
merges.txtc                ót   € \        4       pV ^ ,          pV R,           F  pVP                  W#34       TpK  	  V# )zw
Return set of symbol pairs in a word. word is represented as tuple of symbols (symbols being variable-length
strings)
:é   NN)ÚsetÚadd)ÚwordÚpairsÚ	prev_charÚchars   &   Úk/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/fsmt/tokenization_fsmt.pyÚ	get_pairsr   "   s?   € ô
 ‹E€EØ�Q•€IØ�R—�ˆØ�	‰	�9Ð#Ô$ØŠ	ñ ð €Ló    c                ó.  € V P                  RR4      p \        P                  ! RRV 4      p V P                  RR4      p V P                  RR4      p V P                  RR4      p V P                  R	R
4      p V P                  RR
4      p V P                  RR4      p V P                  RR4      p V P                  RR4      p V P                  RR4      p V P                  RR4      p V P                  RR4      p V P                  RR4      p V P                  RR4      p V P                  RR4      p V P                  RR4      p V P                  RR4      p V P                  RR4      p V P                  R R!4      p V P                  R"R#4      p V P                  R$R%4      p V P                  R&R'4      p V P                  R(R)4      p V P                  R*R+4      p V P                  R,R-4      p \        P                  ! R.RV 4      p V P                  R/R04      p V P                  R1R24      p V P                  R3R44      p V P                  R5R64      p V P                  R7R84      p V P                  R9R:4      p V P                  R;R<4      p V P                  R=R>4      p V P                  R?R@4      p V # )Azr
Port of https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/replace-unicode-punctuation.perl
u   ï¼ŒÚ,u   ã€‚\s*z. u   ã€�u   â€�Ú"u   â€œu   âˆ¶Ú:u   ï¼šu   ï¼ŸÚ?u   ã€Šu   ã€‹u   ï¼‰Ú)u   ï¼�Ú!u   ï¼ˆÚ(u   ï¼›Ú;u   ï¼‘Ú1u   ã€�u   ã€Œu   ï¼�Ú0u   ï¼“Ú3u   ï¼’Ú2u   ï¼•Ú5u   ï¼–Ú6u   ï¼™Ú9u   ï¼—Ú7u   ï¼˜Ú8u   ï¼”Ú4u   ï¼Ž\s*u   ï½žÚ~u   â€™Ú'u   â€¦z...u   â”�Ú-u   ã€ˆÚ<u   ã€‰Ú>u   ã€�Ú[u   ã€‘Ú]u   ï¼…Ú%)ÚreplaceÚreÚsub)Útexts   &r   Úreplace_unicode_punctr1   /   sM  € ð �<‰<˜˜sÓ#€DÜ�6Š6�)˜T 4Ó(€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DÜ�6Š6�)˜T 4Ó(€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜uÓ%€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ€Kr   c                óº   € . pV  FC  p\         P                  ! V4      pVP                  R4      '       d   K2  VP                  V4       KE  	  RP	                  V4      # )zo
Port of https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/remove-non-printing-char.perl
ÚCÚ )ÚunicodedataÚcategoryÚ
startswithÚappendÚjoin)r0   Úoutputr   Úcats   &   r   Úremove_non_printing_charr<   Z   sQ   € ð €FÛˆÜ×"Ò" 4Ó(ˆØ�>‰>˜#×ÒÙØ�‰�dÖñ	 ð
 �7‰7�6‹?Ðr   c                   ó.  a a€ ] tR t^pt oRt]tRR.tRV 3R lltV3R lR lt	]
V3R lR l4       tR	 tR
 tR tR t]
R 4       t]
R 4       tR tR tR tR R ltR tR tR tR!V3R lR lltR"V3R lV 3R llltR!V3R lR lltR tR tRtVtV ;t # )#ÚFSMTTokenizerau  
Construct an FAIRSEQ Transformer tokenizer. Based on Byte-Pair Encoding. The tokenization process is the following:

- Moses preprocessing and tokenization.
- Normalizing all inputs text.
- The arguments `special_tokens` and the function `set_special_tokens`, can be used to add additional symbols (like
  "__classify__") to a vocabulary.
- The argument `langs` defines a pair of languages.

This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
this superclass for more information regarding those methods.

Args:
    langs (`List[str]`, *optional*):
        A list of two languages to translate from and to, for instance `["en", "ru"]`.
    src_vocab_file (`str`, *optional*):
        File containing the vocabulary for the source language.
    tgt_vocab_file (`st`, *optional*):
        File containing the vocabulary for the target language.
    merges_file (`str`, *optional*):
        File containing the merges.
    do_lower_case (`bool`, *optional*, defaults to `False`):
        Whether or not to lowercase the input when tokenizing.
    unk_token (`str`, *optional*, defaults to `"<unk>"`):
        The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
        token instead.
    bos_token (`str`, *optional*, defaults to `"<s>"`):
        The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.

        <Tip>

        When building a sequence using special tokens, this is not the token that is used for the beginning of
        sequence. The token used is the `cls_token`.

        </Tip>

    sep_token (`str`, *optional*, defaults to `"</s>"`):
        The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for
        sequence classification or for a text and a question for question answering. It is also used as the last
        token of a sequence built with special tokens.
    pad_token (`str`, *optional*, defaults to `"<pad>"`):
        The token used for padding, for example when batching sequences of different lengths.

Ú	input_idsÚattention_maskc
                ó^  <€  ^ RI pY°n        Y n        Y0n        Y@n        YPn        / T n        / T n        / T n	        T'       d!   \        T4      ^8X  d   Tw  T n        T n        M\        RT R24      h\        TRR7      ;_uu_ 4       p\        P                   ! T4      T n        RRR4       \        TRR7      ;_uu_ 4       p\        P                   ! T4      pTP%                  4        UUu/ uF	  w  ppTTbK  	  uppT n        RRR4       \        TRR7      ;_uu_ 4       pTP)                  4       P+                  R4      RR pRRR4       X Uu. uF#  p\-        TP+                  4       R,          4      NK%  	  pp\/        \1        T\3        \        T4      4      4      4      T n        / T n        \8        ST `t  ! RR	TR
TRTRTRTRTRTRTRT	/	T
B  R#   \         d    \        R4      hi ; i  + '       g   i     ELP; iu uppi   + '       g   i     EL; i  + '       g   i     LÞ; iu upi )é    NúnYou need to install sacremoses to use XLMTokenizer. See https://pypi.org/project/sacremoses/ for installation.zFarg `langs` needs to be a list of 2 langs, e.g. ['en', 'ru'], but got zw. Usually that means that tokenizer can't find a mapping for the given model path in  and other maps of this tokenizer.úutf-8©ÚencodingÚ
:Né   NÚlangsr   r   r   Údo_lower_caseÚ	unk_tokenÚ	bos_tokenÚ	sep_tokenÚ	pad_tokenéÿÿÿÿ© )Ú
sacremosesÚImportErrorÚsmr   r   r   rJ   Úcache_moses_punct_normalizerÚcache_moses_tokenizerÚcache_moses_detokenizerÚlenÚsrc_langÚtgt_langÚ
ValueErrorÚopenÚjsonÚloadÚencoderÚitemsÚdecoderÚreadÚsplitÚtupleÚdictÚzipÚrangeÚ	bpe_ranksÚcacheÚsuperÚ__init__)ÚselfrI   r   r   r   rJ   rK   rL   rM   rN   ÚkwargsrQ   Úsrc_vocab_handleÚtgt_vocab_handleÚ	tgt_vocabÚkÚvÚmerges_handleÚmergesÚmergeÚ	__class__s   &&&&&&&&&&,         €r   rj   ÚFSMTTokenizer.__init__¡   s*  ø€ ð	Ûð Œà,ÔØ,ÔØ&ÔØ*Ôð -/ˆÔ)à%'ˆÔ"Ø')ˆÔ$ç”S˜“Z 1”_Ø+0Ñ(ˆDŒM˜4�=äØXÐY^ÐX_ð `8ð 8óð ô �.¨7×3Õ3Ð7GÜŸ9š9Ð%5Ó6ˆDŒL÷ 4ä�.¨7×3Õ3Ð7GÜŸ	š	Ð"2Ó3ˆIØ-6¯_©_Ô->Ô?Ñ->¡T Q¨˜A˜qšDÑ->Ò?ˆDŒL÷ 4ô �+¨×0Õ0°MØ"×'Ñ'Ó)×/Ñ/°Ó5°c°rÐ:ˆF÷ 1á8>Ó?¹¨u”%˜Ÿ™› bÕ)Ö*¹ˆÐ?Üœc &¬%´°F³Ó*<Ó=Ó>ˆŒØˆŒ
Ü‰Òñ 	
Øð	
à)ð	
ð *ð	
ð $ð		
ð
 (ð	
ð  ð	
ð  ð	
ð  ð	
ð  ð	
ð ô	
øôM ô 	ÜðMóð ð	ú÷8 4×3Ð3üó @÷ 4×3Ð3ú÷ 1×0üâ?sG   ƒG ÂG)Â?*HÃ)G=
Ã8	HÄ#HÅ)H*ÇG&Ç)G:	Ç=HÈH	ÈH'	c                ó6   <€ V ^8„  d   QhRS[ S[S[3,          /# ©rH   Úreturn)rd   ÚstrÚint)ÚformatÚ__classdict__s   "€r   Ú__annotate__ÚFSMTTokenizer.__annotate__ä   s   ø€ ÷ $ñ $™4¡¡S �>ñ $r   c                ó"   € V P                  4       # ©N)Úget_src_vocab©rk   s   &r   Ú	get_vocabÚFSMTTokenizer.get_vocabä   s   € Ø×!Ñ!Ó#Ð#r   c                ó    <€ V ^8„  d   QhRS[ /# rx   )r{   )r|   r}   s   "€r   r~   r   é   s   ø€ ÷ #ñ #™Cñ #r   c                ó   € V P                   # r�   )Úsrc_vocab_sizerƒ   s   &r   Ú
vocab_sizeÚFSMTTokenizer.vocab_sizeè   s   € à×"Ñ"Ð"r   c                óº   € W P                   9  d+   V P                  P                  VR 7      pW0P                   V&   V P                   V,          P                  V4      # ©©Úlang)rT   rS   ÚMosesPunctNormalizerÚ	normalize)rk   r0   rŽ   Úpunct_normalizers   &&& r   Úmoses_punct_normÚFSMTTokenizer.moses_punct_normì   sP   € Ø×8Ñ8Ô8Ø#Ÿw™w×;Ñ;ÀÐ;ÓFÐØ6F×-Ñ-¨dÑ3Ø×0Ñ0°Õ6×@Ñ@ÀÓFÐFr   c                óÂ   € W P                   9  d+   V P                  P                  VR 7      pW0P                   V&   V P                   V,          P                  VRRRR7      # )r�   TF)Úaggressive_dash_splitsÚ
return_strÚescape)rU   rS   ÚMosesTokenizerÚtokenize)rk   r0   rŽ   Úmoses_tokenizers   &&& r   Úmoses_tokenizeÚFSMTTokenizer.moses_tokenizeò   s_   € Ø×1Ñ1Ô1Ø"Ÿg™g×4Ñ4¸$Ð4Ó?ˆOØ/>×&Ñ& tÑ,Ø×)Ñ)¨$Õ/×8Ñ8Ø¨¸%Èð 9ó 
ð 	
r   c                óº   € W P                   9  d+   V P                  P                  VR 7      pW0P                   V&   V P                   V,          P                  V4      # rŒ   )rV   rS   ÚMosesDetokenizerÚ
detokenize)rk   ÚtokensrŽ   Úmoses_detokenizers   &&& r   Úmoses_detokenizeÚFSMTTokenizer.moses_detokenizeú   sP   € Ø×3Ñ3Ô3Ø $§¡× 8Ñ 8¸dÐ 8Ó CÐØ1B×(Ñ(¨Ñ.Ø×+Ñ+¨DÕ1×<Ñ<¸VÓDÐDr   c                óT   € \        V4      pV P                  W4      p\        V4      pV# r�   )r1   r’   r<   )rk   r0   rŽ   s   &&&r   Úmoses_pipelineÚFSMTTokenizer.moses_pipeline   s+   € Ü$ TÓ*ˆØ×$Ñ$ TÓ0ˆÜ'¨Ó-ˆØˆr   c                ó,   € \        V P                  4      # r�   )rW   r^   rƒ   s   &r   rˆ   ÚFSMTTokenizer.src_vocab_size  ó   € ä�4—<‘<Ó Ð r   c                ó,   € \        V P                  4      # r�   )rW   r`   rƒ   s   &r   Útgt_vocab_sizeÚFSMTTokenizer.tgt_vocab_size
  r©   r   c                óB   € \        V P                  3/ V P                  B # r�   )rd   r^   Úadded_tokens_encoderrƒ   s   &r   r‚   ÚFSMTTokenizer.get_src_vocab  ó   € Ü�D—L‘LÑ> D×$=Ñ$=Ñ>Ð>r   c                óB   € \        V P                  3/ V P                  B # r�   )rd   r`   Úadded_tokens_decoderrƒ   s   &r   Úget_tgt_vocabÚFSMTTokenizer.get_tgt_vocab  r°   r   c                ó‚  a € \        VR R 4      VR,          R,           3,           pVS P                  9   d   S P                  V,          # \        V4      pV'       g
   VR,           #  \        VV 3R lR7      pVS P                  9  d   MçVw  rV. p^ pV\        V4      8  d¤    VP                  WX4      p	VP                  W(V	 4       T	pY(,          T8X  dN   T\        T4      ^,
          8  d7   Y(^,           ,          T8X  d#   TP                  YV,           4       T^,          pK‘  TP                  Y(,          4       T^,          pK³  \        V4      pTp\        V4      ^8X  d   M\        V4      pEK  RP                  V4      pVR8X  d   RpVS P                  V&   V#   \         d    TP                  Y(R  4        Kx  i ; i)Nú</w>c                 óN   <€ SP                   P                  V \        R 4      4      # )Úinf)rg   ÚgetÚfloat)Úpairrk   s   &€r   Ú<lambda>Ú#FSMTTokenizer.bpe.<locals>.<lambda>  s   ø€ °·±×1CÑ1CÀDÌ%ÐPUË,Ô1Wr   ©ÚkeyÚ z
  </w>z
</w>rO   )rc   rh   r   Úminrg   rW   ÚindexÚextendrZ   r8   r9   )
rk   Útokenr   r   ÚbigramÚfirstÚsecondÚnew_wordÚiÚjs
   f&        r   ÚbpeÚFSMTTokenizer.bpe  s„  ø€ Ü�U˜3˜B�ZÓ  E¨"¥I°Õ$6Ð#8Õ8ˆØ�D—J‘JÔØ—:‘:˜eÕ$Ð$Ü˜$“ˆçØ˜6•>Ð!àÜ˜Ô$WÔXˆFØ˜TŸ^™^Ô+ØØ"‰MˆEØˆHØˆAØ”c˜$“i”-ðØŸ
™
 5Ó,�Að
 —O‘O D¨1 IÔ.Ø�Aà•7˜eÔ#¨¬C°«I¸­MÔ(9¸dÀqÅ5½kÈVÔ>SØ—O‘O E¥NÔ3Ø˜•F’Aà—O‘O D¥GÔ,Ø˜•F’AÜ˜X“ˆHØˆDÜ�4‹y˜AŒ~Øä! $›“Ø�x‰x˜‹~ˆØ�:ÔØˆDØ ˆ�
‰
�5ÑØˆøô/ "ô Ø—O‘O D¨ HÔ-Úðús   Â#F ÆF>Æ=F>c                óv  € V P                   pV P                  '       d   VP                  4       pV'       d   VP                  4       pM$V P	                  WR7      pV P                  WR7      p. pV FE  pV'       g   K  VP                  \        V P                  V4      P                  R4      4      4       KG  	  V# )a  
Tokenize a string given language code using Moses.

Details of tokenization:

    - [sacremoses](https://github.com/alvations/sacremoses): port of Moses
    - Install with `pip install sacremoses`

Args:
    - lang: ISO language code (default = 'en') (string). Languages should belong of the model supported
      languages. However, we don't enforce it.
    - bypass_tokenizer: Allow users to preprocess and tokenize the sentences externally (default = False)
      (bool). If True, we only apply BPE.

Returns:
    List of tokens.
r�   rÀ   )	rX   rJ   Úlowerrb   r¥   r›   rÃ   ÚlistrË   )rk   r0   rŽ   Úbypass_tokenizerÚsplit_tokensrÄ   s   &&&&  r   Ú	_tokenizeÚFSMTTokenizer._tokenize@  sš   € ð* �}‰}ˆà××ÐØ—:‘:“<ˆDçØ—:‘:“<‰Dà×&Ñ& tÐ&Ó7ˆDØ×&Ñ& tÐ&Ó7ˆDàˆÛˆEß‰uØ×#Ñ#¤D¨¯©°%«×)>Ñ)>¸sÓ)CÓ$DÖEñ ð Ðr   c                ó~   € V P                   P                  WP                   P                  V P                  4      4      # )z0Converts a token (str) in an id using the vocab.)r^   r¹   rK   )rk   rÄ   s   &&r   Ú_convert_token_to_idÚ"FSMTTokenizer._convert_token_to_idg  s*   € à�|‰|×Ñ §|¡|×'7Ñ'7¸¿¹Ó'GÓHÐHr   c                óL   € V P                   P                  WP                  4      # )z=Converts an index (integer) in a token (str) using the vocab.)r`   r¹   rK   )rk   rÂ   s   &&r   Ú_convert_id_to_tokenÚ"FSMTTokenizer._convert_id_to_tokenk  s   € à�|‰|×Ñ §~¡~Ó6Ð6r   c                óä   € V Uu. uF$  q"P                  RR4      P                  RR4      NK&  	  ppRP                  V4      P                  4       pV P                  WP                  4      pV# u upi )z:Converts a sequence of tokens (string) in a single string.rÀ   r4   r¶   )r-   r9   rb   r¢   rY   )rk   r    Útr0   s   &&  r   Úconvert_tokens_to_stringÚ&FSMTTokenizer.convert_tokens_to_stringo  se   € ñ DJÓJÁ6¸a—)‘)˜C Ó$×,Ñ,¨V°SÖ9Á6ˆÐJØ—‘˜“×&Ñ&Ó(ˆà×$Ñ$ V¯]©]Ó;ˆØˆùò	 Ks   …*A-c                ój   <€ V ^8„  d   QhRS[ S[,          RS[ S[,          R,          RS[ S[,          /# )rH   Útoken_ids_0Útoken_ids_1Nry   )rÏ   r{   )r|   r}   s   "€r   r~   r   y  s6   ø€ ÷ 5ñ 5Ù¡�9ð5Ù37¹µ9¸tÕ3Cð5á	‰c�ñ5r   c                ó`   € V P                   .pVf	   W,           # W,           V,           V,           # )a>  
Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
adding special tokens. A FAIRSEQ Transformer sequence has the following format:

- single sequence: `<s> X </s>`
- pair of sequences: `<s> A </s> B </s>`

Args:
    token_ids_0 (`List[int]`):
        List of IDs to which the special tokens will be added.
    token_ids_1 (`List[int]`, *optional*):
        Optional second list of IDs for sequence pairs.

Returns:
    `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.
)Úsep_token_id)rk   rß   rà   Úseps   &&& r   Ú build_inputs_with_special_tokensÚ.FSMTTokenizer.build_inputs_with_special_tokensy  s6   € ð& × Ñ Ð!ˆð ÒØÕ$Ð$ØÕ  ;Õ.°Õ4Ð4r   c          	      óp   <€ V ^8„  d   QhRS[ S[,          RS[ S[,          R,          RS[RS[ S[,          /# )rH   rß   rà   NÚalready_has_special_tokensry   )rÏ   r{   Úbool)r|   r}   s   "€r   r~   r   “  s>   ø€ ÷ .ñ .Ù¡�9ð.Ù37¹µ9¸tÕ3Cð.Ùhlð.á	‰c�ñ.r   c                óê   <€ V'       d   \         SV `  WRR7      # Ve<   ^ .\        V4      ,          ^.,           ^ .\        V4      ,          ,           ^.,           # ^ .\        V4      ,          ^.,           # )ad  
Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
special tokens using the tokenizer `prepare_for_model` method.

Args:
    token_ids_0 (`List[int]`):
        List of IDs.
    token_ids_1 (`List[int]`, *optional*):
        Optional second list of IDs for sequence pairs.
    already_has_special_tokens (`bool`, *optional*, defaults to `False`):
        Whether or not the token list is already formatted with special tokens for the model.

Returns:
    `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
T)rß   rà   rç   )ri   Úget_special_tokens_maskrW   )rk   rß   rà   rç   ru   s   &&&&€r   rê   Ú%FSMTTokenizer.get_special_tokens_mask“  sx   ø€ ÷& &Ü‘7Ñ2Ø'Ð]að 3ó ð ð Ò"Ø�Cœ#˜kÓ*Õ*¨q¨cÕ1°a°S¼3¸{Ó;KÕ5KÕLÐPQÈsÕRÐRØ�”c˜+Ó&Õ&¨1¨#Õ-Ð-r   c                óJ   <€ V ^8„  d   QhRS[ RS[ R,          RS[S[ ,          /# )rH   Úsave_directoryÚfilename_prefixNry   )rz   rc   )r|   r}   s   "€r   r~   r   ¯  s-   ø€ ÷ ";ñ ";©cð ";ÁCÈ$ÅJð ";ÑZ_Ñ`cÕZdñ ";r   c                ó‚  € \         P                  P                  V4      '       g   \        P	                  R V R24       R# \         P                  P                  Y'       d
   VR,           MR\        R,          ,           4      p\         P                  P                  Y'       d
   VR,           MR\        R,          ,           4      p\         P                  P                  Y'       d
   VR,           MR\        R,          ,           4      p\        VRR	R
7      ;_uu_ 4       pVP                  \        P                  ! V P                  ^RRR7      R,           4       RRR4       \        VRR	R
7      ;_uu_ 4       pV P                  P                  4        UUu/ uF  w  rxW‡bK	  	  p	ppVP                  \        P                  ! V	^RRR7      R,           4       RRR4       ^ p
\        VRR	R
7      ;_uu_ 4       p\        V P                  P                  4       R R7       FV  w  rÍW­8w  d   \        P!                  RV R24       Tp
VP                  RP                  V4      R,           4       V
^,          p
KX  	  RRR4       W4V3#   + '       g   i     EL2; iu uppi   + '       g   i     LÏ; i  + '       g   i     LA; i)zVocabulary path (z) should be a directoryNr'   r4   r   r   r   ÚwrD   rE   TF)ÚindentÚ	sort_keysÚensure_asciirG   c                 ó   € V ^,          # )r   rP   )Úkvs   &r   r¼   Ú/FSMTTokenizer.save_vocabulary.<locals>.<lambda>Ç  s   € ÐY[Ð\]ÖY^r   r¾   zSaving vocabulary to zZ: BPE merge indices are not consecutive. Please check that the tokenizer is not corrupted!rÀ   )ÚosÚpathÚisdirÚloggerÚerrorr9   ÚVOCAB_FILES_NAMESr[   Úwriter\   Údumpsr^   r`   r_   Úsortedrg   Úwarning)rk   rí   rî   r   r   r   Úfrp   rq   ro   rÂ   ÚwriterÚ
bpe_tokensÚtoken_indexs   &&&           r   Úsave_vocabularyÚFSMTTokenizer.save_vocabulary¯  s  € Ü�w‰w�}‰}˜^×,Ò,Ü�L‰LÐ,¨^Ð,<Ð<SÐTÔUÙäŸ™Ÿ™Ø¶o˜_¨sÖ2È2ÔQbÐcsÕQtÕtó
ˆô Ÿ™Ÿ™Ø¶o˜_¨sÖ2È2ÔQbÐcsÕQtÕtó
ˆô —g‘g—l‘lØ¶o˜_¨sÖ2È2ÔQbÐcpÕQqÕqó
ˆô �. #°×8Õ8¸AØ�G‰G”D—J’J˜tŸ|™|°AÀÐTYÔZÐ]aÕaÔb÷ 9ô �. #°×8Õ8¸AØ*.¯,©,×*<Ñ*<Ô*>Ô?Ñ*>¡$ !˜šÑ*>ˆIÑ?Ø�G‰G”D—J’J˜y°¸dÐQVÔWÐZ^Õ^Ô_÷ 9ð ˆÜ�+˜s¨W×5Õ5¸Ü+1°$·.±.×2FÑ2FÓ2HÑN^×+_Ñ'�
ØÔ'Ü—N‘NØ/°¨}ð =Mð Môð (�EØ—‘˜SŸX™X jÓ1°DÕ8Ô9Ø˜•
’ñ ,`÷ 6ð ¨{Ð:Ð:÷' 9×8Ð8üó @÷ 9×8ú÷
 6×5ús7   Ä;JÅ6JÆJ
Æ!3JÇ4BJ.ÊJ	ÊJÊJ+	Ê.J>	c                óD   € V P                   P                  4       pR VR&   V# )NrS   )Ú__dict__Úcopy)rk   Ústates   & r   Ú__getstate__ÚFSMTTokenizer.__getstate__Ó  s"   € Ø—‘×"Ñ"Ó$ˆØˆˆd‰Øˆr   c                ó\   € Wn          ^ RIpY n        R#   \         d    \        R4      hi ; i)rB   NrC   )r  rQ   rR   rS   )rk   ÚdrQ   s   && r   Ú__setstate__ÚFSMTTokenizer.__setstate__Ø  s:   € ØŒð	Ûð Žøô ô 	ÜðMóð ð	ús   ˆ ”+)r  rg   rh   rV   rT   rU   r`   rJ   r^   r   rS   rX   r   rY   r   )	NNNNFz<unk>z<s>z</s>z<pad>)ÚenFr�   )NF)!Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__rü   Úvocab_files_namesÚmodel_input_namesrj   r„   Úpropertyr‰   r’   r›   r¢   r¥   rˆ   r«   r‚   r³   rË   rÒ   rÕ   rØ   rÜ   rä   rê   r  r  r  Ú__static_attributes__Ú__classdictcell__Ú__classcell__)ru   r}   s   @@r   r>   r>   p   sÖ   ù‡ € ñ+ðZ *ÐØ$Ð&6Ð7Ð÷@
÷F$ð $ð ÷#ó ð#òGò
òEòð ñ!ó ð!ð ñ!ó ð!ò?ò?ò*ôX%òNIò7ò÷5ò 5÷4.õ .÷8";ò ";òH÷
ò r   r>   )r  r\   r÷   r.   r5   Útokenization_pythonr   Úutilsr   Ú
get_loggerr  rú   rü   r   r1   r<   r>   Ú__all__rP   r   r   Ú<module>r!     ss   ðñ %ã Û 	Û 	Û å 6Ý ð 
×	Ò	˜HÓ	%€ð Ð&ØÐ&Ø�<ðÐ ò
ò(òV
ô,sÐ'ô sðl Ð
‚r   