+
    œ°³j\  ã                   ó¤  € R t ^ RIt^ RIt^ RIt^ RIHt ^ RIHt ^ RIH	t	 ^ RI
HtHtHtHtHtHt ^ RIHt ^RIHt ^RIHt ^R	IHtHtHt ^R
IHtHt ^RIHt ^RI H!t!H"t" ]! 4       '       d   ^ RI#H$t$ ]PJ                  ! ]&4      t'R R lt(R R lt)R(R R llt*RRRRRRRR/t+R R lt, ! R R4      t-R  R! lt.R" R# lt/R$ R% lt0R(R& R' llt1R# ))zOConversion between Mistral tekken tokenizers and HuggingFace tokenizer formats.N)Ú	lru_cache)ÚPath)Úcopyfile)Ú
AddedTokenÚRegexÚ	TokenizerÚdecodersÚpre_tokenizersÚ
processors)ÚBPE)Úbytes_to_unicode)ÚTokenizersBackend)Úcached_fileÚloggingÚrequires_backends)ÚCHAT_TEMPLATE_FILEÚ#LEGACY_PROCESSOR_CHAT_TEMPLATE_FILE)Úis_mistral_common_available)ÚTEKKEN_VOCAB_FILEÚis_tekken_vocab_filename)Ú
Tekkenizerc                ó‚   € V ^8„  d   QhR\         \        P                  ,          R\         R,          R\         R,          /# )é   Útekken_fileÚchat_templateNÚreturn©ÚstrÚosÚPathLike)Úformats   "Úl/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/integrations/mistral/tokenizer.pyÚ__annotate__r"   *   s6   € ÷ 7ñ 7¬¬b¯k©kÕ(9ð 7Ì#ÐPTÍ*ð 7ÔY\Ð_cÕYcñ 7ó    c                óD  € Ve   V# \        V 4      P                  pV\        ,          pVP                  4       '       d   VP	                  RR7      # V\
        ,          pVP                  4       '       d>   \        VRR7      ;_uu_ 4       p\        P                  ! V4      pRRR4       VR,          # \        4       '       d    ^ RI
Hp V! V 4      # R#   + '       g   i     XR,          # ; i  \         d(   p\        P                  RT  RT R24        Rp?R# Rp?ii ; i)	uL  Resolve the chat template to attach during tekken to HF conversion.

Applies a fixed precedence order:

1. `chat_template` argument (if not `None`) â€” returned unchanged.
2. Sibling `chat_template.jinja` file in the same directory as *tekken_file*.
3. Sibling `chat_template.json` file â€” value of its `"chat_template"` key.
4. Automatic generation via `mistral_common.integrations.chat_templates` (lazy import).
5. `None` if none of the above succeed.

Args:
    tekken_file (`str` or `os.PathLike`): Path to the `tekken.json` file.
    chat_template (`str` or `None`): Explicit chat template string. Only `None`
        triggers the lookup cascade; an empty string is returned as-is.

Returns:
    Resolved chat template string, or `None`.

Raises:
    KeyError: If `chat_template.json` exists but does not contain a `"chat_template"` key.
Núutf-8©Úencodingr   )Ú"convert_tokenizer_to_chat_templatez'Failed to generate chat template from 'z': z#. Falling back to no chat template.)r   Úparentr   Úis_fileÚ	read_textr   ÚopenÚjsonÚloadr   Ú9mistral_common.integrations.chat_templates.chat_templatesr(   Ú	ExceptionÚloggerÚwarning_once)	r   r   r)   Ú
jinja_pathÚ	json_pathÚfÚdatar(   Úexcs	   &&       r!   Ú_resolve_chat_templater8   *   s
  € ð. Ò ØÐä�+Ó×%Ñ%€Fð Ô,Õ,€JØ×Ñ×ÒØ×#Ñ#¨WÐ#Ó5Ð5ð Ô<Õ<€IØ×Ñ×ÒÜ�) g×.Õ.°!Ü—9’9˜Q“<ˆD÷ /à�OÕ$Ð$ô #×$Ò$ð
	õñ 6°kÓBÐBñ ÷' /Ö.à�OÕ$Ð$ûô ô 	Ü×ÑØ9¸+¸ÀcÈ#ÈÐNqÐrôõ ûð		ús$   Â	CÃC- ÃC*	Ã-DÃ8DÄDc                ót   € V ^8„  d   QhR\         \        P                  ,          R\         R\         R,          /# )r   Úpretrained_model_name_or_pathÚfilenamer   Nr   )r    s   "r!   r"   r"   d   s3   € ÷ ñ Ü#&¬¯©Õ#4ðäðô 	ˆ4…Zñ	r#   c                ó$   € \        V V3RRRR/VB # )a  Return the resolved path for *filename* inside *pretrained_model_name_or_path*, or `None`.

Exceptions for missing entries and connection errors are suppressed so that
callers can treat a `None` return as "file not found / not reachable".

Args:
    pretrained_model_name_or_path (`str` or `os.PathLike`): Model id or local directory.
    filename (`str`): File name to look up inside the checkpoint.
    **cache_kwargs: Forwarded to [`~utils.cached_file`].

Returns:
    Resolved local path string, or `None` if the file could not be found.
Ú%_raise_exceptions_for_missing_entriesFÚ'_raise_exceptions_for_connection_errors)r   )r:   r;   Úcache_kwargss   &&,r!   Ú_probe_filer@   d   s5   € ô$ Ø%Øñð /4ðð 16ð	ð
 ñð r#   c          	      ó¤   € V ^8„  d   QhR\         \        P                  ,          R\        R,          R\        \        \         R,          3,          /# )r   r:   Úmistral_formatNr   )r   r   r   ÚboolÚtuple)r    s   "r!   r"   r"      sF   € ÷ C2ñ C2Ü#&¬¯©Õ#4ðC2ä˜4•KðC2ô Œ4”�t•ÐÕñ	C2r#   c                óL  € VRJ d   R
# VP                  RR4       VP                  RR4       VRJ dK   \        4       '       g   \        R4      h\        V \        3/ VB pVf   \        R\         RV  R	24      hRV3# \        4       '       g   R
# \        V \        3/ VB pVRJV3# )uh  Resolve whether to use `MistralCommonBackend` for tokenization.

Probes for `tekken.json` in the checkpoint directory and checks whether
`mistral-common` is installed to determine the appropriate tokenizer backend.

Args:
    pretrained_model_name_or_path (`str` or `os.PathLike`):
        This can be either:

        - a string, the *model id* of a pretrained model hosted on huggingface.co.
        - a path to a *directory* containing model files.
    mistral_format (`bool`, *optional*):
        Tri-state control for tokenizer backend selection:

        - `True` â€” force `MistralCommonBackend` (raises `ImportError` if `mistral-common`
          is not installed, raises `OSError` if `tekken.json` is not found).
        - `False` â€” force standard HuggingFace tokenizer.
        - `None` â€” auto-detect: selects native (`MistralCommonBackend`) when
          `mistral-common` is installed and a `tekken.json` is found, regardless of
          whether HF-format files are also present. Otherwise falls back to the
          standard HuggingFace tokenizer.
    **cache_kwargs:
        Forwarded to [`~utils.cached_file`] (e.g. `cache_dir`, `force_download`,
        `local_files_only`, `revision`, `token`).

Returns:
    `tuple[bool, str | None]`: A tuple of `(use_mistral_format, tekken_file_path)` where
    `use_mistral_format` indicates whether `MistralCommonBackend` should be used, and
    `tekken_file_path` is the resolved path to `tekken.json` (or `None`).

Raises:
    ImportError: If `mistral_format=True` and `mistral-common` is not installed.
    OSError: If `mistral_format=True` and `tekken.json` cannot be found.
FNr=   r>   TzZmistral_format=True requires `mistral-common`. Install it with: pip install mistral-commonzCannot find 'z' at 'zH'. Set `mistral_format=False` to use standard HuggingFace files instead.)FN)Úpopr   ÚImportErrorr@   r   ÚOSError)r:   rB   r?   r   s   &&, r!   Úresolve_mistral_formatrI      sá   € ðN ˜ÓØÐð ×ÑÐ<¸dÔCØ×ÑÐ>ÀÔEà˜ÓÜ*×,Ò,ÜØlóð ô
 "Ð"?ÔARÑcÐVbÑcˆØÒÜØÔ 1Ð2°&Ð9VÐ8Wð XXð Xóð ð �kÐ"Ð"ô '×(Ò(ØÐäÐ;Ô=NÑ_ÐR^Ñ_€KØ˜tÐ# [Ð1Ð1r#   Ú	bos_tokenz<s>Ú	eos_tokenz</s>Ú	pad_tokenz<pad>Ú	unk_tokenz<unk>c                óh   € V ^8„  d   QhR\         R\        \        \        ,          \        3,          /# )r   Úrawr   )ÚdictrD   Úlistr   Úint)r    s   "r!   r"   r"   Í   s+   € ÷ 1@ñ 1@¤ð 1@¬%´´Sµ	¼3°Õ*?ñ 1@r#   c                óV  € V R,          pRpV P                  R4      pVf<   \        \        R.4       \        \        RV4      p\        \        P                  4      p\        R V 4       4      pV\        \        \        V4      4      4      8w  d   \        RV 24      h\        R V 4       4       UUu. uF  w  rVVNK	  	  pppVP                  R	4      pVf   \        V4      p\        \        V4      V4       U	u. uF  q’P                  V	R
7      NK  	  p
p	Wz,           V3# u uppi u up	i )a  Derive the expected special-token strings and special-token count from a parsed tekken.json.

Single source of truth for the special-token layout, shared by
`MistralConverter._parse_tekken_file` (to build the vocab) and
`_check_tekken_vocab_unchanged` (to compare against a live tokenizer), so the two cannot
silently drift apart.

Args:
    raw (`dict`): A parsed tekken.json, i.e. the result of `json.load` on the file.

Returns:
    `tuple[list[str], int]`: The special-token strings, ordered by rank (the real entries
    from *raw* followed by any `<SPECIAL_i>` filler needed to reach `num_special_tokens`),
    and the resolved `num_special_tokens`.

Raises:
    ImportError: If *raw* has no top-level `"special_tokens"` key (old tekken format) and
        `mistral-common` is not installed.
Úconfigz<SPECIAL_{id}>Úspecial_tokenszmistral-commonÚSPECIAL_TOKEN_TEMPLATEc              3   ó2   "  € T F  qR ,          x € K  	  R# 5i)ÚrankN© ©Ú.0Úentrys   & r!   Ú	<genexpr>Ú*_derive_tekken_specials.<locals>.<genexpr>í   s   é € ÐLÑ7K¨e &ŸMšMÓ7Kùs   ‚zGtekken.json special_tokens ranks must be contiguous starting at 0, got c           
   3   ó|   "  € T F2  pVR ,          \        \        VR,          RVR,          4      4      3x € K4  	  R# 5i)rX   Ú	token_strÚvalueN)r   ÚgetattrrZ   s   & r!   r]   r^   ó   s8   é € ð #
á-�ð �6�]œC¤¨¨kÕ(:¸GÀUÈ;ÕEWÓ XÓYÕZÛ-ùs   ‚:<Údefault_num_special_tokens)Úid)Úgetr   ÚMistralConverterrb   r   rQ   ÚDEPRECATED_SPECIAL_TOKENSÚsortedÚrangeÚlenÚ
ValueErrorr    )rO   rT   Úfiller_templateÚspecial_tokens_dictsÚdeclared_ranksÚ_r`   Úspecial_stringsÚnum_special_tokensÚiÚfiller_stringss   &          r!   Ú_derive_tekken_specialsrt   Í   s<  € ð( ��]€FØ&€OàŸ7™7Ð#3Ó4ÐØÒ#äÔ*Ð-=Ð,>Ô?Ü!¤*Ð.FÈÓXˆÜ#¤J×$HÑ$HÓIÐô ÑLÑ7KÓLÓL€NØœœe¤C¨Ó$7Ó8Ó9Ô9ÜÐbÐcqÐbrÐsÓtÐtô #ñ #
á-ó#
ô 
ôñ
‰LˆAó 	ñ
ð ñ ð  Ÿ™Ð$@ÓAÐØÒ!Ü  Ó1Ðä<AÄ#ÀoÓBVÐXjÔ<kÓlÑ<k°q×,Ñ,°Ð,Ö2Ñ<k€NÐlØÕ+Ð-?Ð?Ð?ùóùò ms   Â3D Ã:D&c                   óŒ   a € ] tR tRt o RtRV 3R lR lltV 3R lR lt]V 3R lR l4       tV 3R	 lR
 lt	V 3R lR lt
RtV tR# )rf   i  zPConverter from Mistral tekken BPE vocab to a HuggingFace `tokenizers.Tokenizer`.c                ó&   <€ V ^8„  d   QhRS[ RS[/# )r   Ú
vocab_fileÚadd_prefix_space©r   rC   )r    Ú__classdict__s   "€r!   r"   ÚMistralConverter.__annotate__  s   ø€ ÷ 	>ñ 	>¡3ð 	>¹$ñ 	>r#   c                ó(   € V P                  W4       R# )a!  Parse a raw `tekken.json` file into a ready-to-use converter.

Matches `mistral_common`'s `Tekkenizer.from_file` vocab and special-token layout.

Args:
    vocab_file (`str`): Path to a `tekken.json` file.
    add_prefix_space (`bool`): Whether to add a leading space during tokenization.
N)Ú_parse_tekken_file)Úselfrw   rx   Úkwargss   &&&,r!   Ú__init__ÚMistralConverter.__init__  s   € ð 	×Ñ 
Ö=r#   c                ó*   <€ V ^8„  d   QhRS[ RS[RR/# )r   rw   rx   r   Nry   )r    rz   s   "€r!   r"   r{     s"   ø€ ÷ %*ñ %*©Sð %*ÁDð %*ÈTñ %*r#   c                ó  € \        VRR7      ;_uu_ 4       p\        P                  ! V4      pRRR4       XR,          pVR,          pVP                  R4      p\	        V4      w  r‰V U
u. uF  p
\        V
RR7      NK  	  pp
VR	,          pVe   Wy,
          pVRV pV Uu. uF   p\        P                  ! VR
,          4      NK"  	  pp\        V4       UUu/ uF	  w  ppVVbK  	  pppV P                  V4      w  ppVP                  4        UUu/ uF  w  ppVVV	,           bK  	  ppp\        V4       F  w  pp
VVV
&   K  	  W`n        W n        W°n        VV n        VV n        R#   + '       g   i     EL?; iu up
i u upi u uppi u uppi )zÊParse a tekken.json file and set all instance attributes.

Args:
    vocab_file (`str`): Path to a `tekken.json` file.
    add_prefix_space (`bool`): Whether to add a leading space during tokenization.
r%   r&   NrT   ÚpatternÚdefault_vocab_sizeT)ÚspecialÚvocabÚtoken_bytes)r,   r-   r.   re   rt   r   Úbase64Ú	b64decodeÚ	enumerateÚ_extract_mergesÚitemsr„   rx   Úadditional_special_tokensÚ_precomputed_vocabÚ_precomputed_merges)r~   rw   rx   r5   ÚuntypedrT   r„   Ú
vocab_sizeÚspecial_token_stringsrq   r`   rŽ   Úbpe_ranks_rawÚinner_vocab_sizeÚkÚ	bpe_ranksrX   ÚtokenÚbpe_ranks_dictr‡   ÚmergesÚvs   &&&                   r!   r}   Ú#MistralConverter._parse_tekken_file  s…  € ô �* w×/Õ/°1Ü—i’i “lˆG÷ 0ð ˜Õ"ˆØ˜Õ#ˆØ—Z‘ZÐ 4Ó5ˆ
ä4KÈGÓ4TÑ1ÐáZoÓ$pÑZoÈY¤Z°	À4×%HÑZoÐ!Ð$pð   Õ(ˆØÒ!Ø)Õ>ÐØ)Ð*;Ð+;Ð<ˆMáANÓOÁ¸A”V×%Ò% a¨Õ&6Ö7Áˆ	ÐOÜ9BÀ9Ô9MÔNÑ9M©+¨$°˜% š+Ñ9MˆÑNà×,Ñ,¨^Ó<‰ˆˆvà7<·{±{´}ÔE±}©t¨q°!��AÐ*Õ*Ò*±}ˆÑEÜ(Ð)>Ö?‰OˆD�)Ø#ˆE�)Óñ  @ð ŒØ 0ÔØ)BÔ&Ø"'ˆÔØ#)ˆÖ ÷= 0×/Ð/üò %qùò PùÛNùó Fs#   –E#Á)E7Â&E<ÃFÄFÅ#E4	c          	      óŽ   <€ V ^8„  d   QhRS[ S[S[3,          RS[S[ S[S[3,          S[S[S[S[3,          ,          3,          /# )r   r—   r   )rP   ÚbytesrR   rD   r   rQ   )r    rz   s   "€r!   r"   r{   7  sH   ø€ ÷ ñ ¡4©©s¨
Õ#3ð ¹¹dÁ3ÉÀ8½nÉdÑSXÑY\Ñ^aÐYaÕSbÕNcÐ>cÕ8dñ r#   c                ó,  a a€ \        4       o\        R V3R ll4       p/ p. pS P                  4        F�  w  rEWRV! V4      &   \        V4      ^8X  d   K!  . p\	        ^\        V4      4       F?  pVRV WGR r˜VS 9   g   K  V	S 9   g   K  W‰,           S 9   g   K,  VP                  W‰V34       KA  	  \        VV 3R lR7      pVP                  V4       KŸ  	  \        VR R7      pV U
u. uF   q¡! V
^ ,          4      V! V
^,          4      3NK"  	  pp
W+3# u up
i )zMExtract a unicode vocab and ordered BPE merge list from byte-level BPE ranks.c                ó0   € V ^8„  d   QhR\         R\        /# )r   Úbr   )rž   r   )r    s   "r!   r"   Ú6MistralConverter._extract_merges.<locals>.__annotate__<  s   € ÷ 	Vñ 	V¤Uð 	V¬sñ 	Vr#   c           	      óŽ   <€ R P                  V P                  R4       Uu. uF  pS\        V4      ,          NK  	  up4      # u upi )Ú zlatin-1)ÚjoinÚdecodeÚord)r¡   ÚcharÚbyte_encoders   & €r!   Útoken_bytes_to_stringÚ?MistralConverter._extract_merges.<locals>.token_bytes_to_string;  s;   ø€ à—7‘7ÀÇÁÈÔ@SÓTÑ@S¸˜L¬¨T«×3Ð3Ñ@SÑTÓUÐUùÒTs    ANc                 óD   <€ SV ^ ,          ,          SV ^,          ,          3# )é    rY   )Úxr—   s   &€r!   Ú<lambda>Ú2MistralConverter._extract_merges.<locals>.<lambda>K  s   ø€ °¸1¸Q½4µÀ)ÈAÈaÍDÅ/Ñ0Rr#   )Úkeyc                 ó   € V ^,          # )r   rY   )Úvals   &r!   r¯   r°   N  s   € ¸¸A¾r#   )r   r   r�   rj   ri   Úappendrh   Úextend)r—   rª   r‡   Ú
all_mergesr˜   rX   ÚlocalÚindexÚpiece_lÚpiece_rr³   rš   r©   s   f           @r!   rŒ   Ú MistralConverter._extract_merges6  s  ù€ ô (Ó)ˆä	÷	Vó 
ð	Vð !#ˆØ57ˆ
à$Ÿ?™?Ö,‰KˆEØ26Ñ'¨Ó.Ñ/Ü�5‹z˜QŒÙØˆEÜ˜q¤# e£*Ö-�Ø#(¨¨% =°%¸°-˜Ø˜iÖ'¨G°yÖ,@ÀgÕFWÐ\eÖEeØ—L‘L '°DÐ!9Ö:ñ .ô ˜5Ô&RÔSˆEØ×Ñ˜eÖ$ñ -ô ˜JÑ,>Ô?ˆ
Ù\fÓgÑ\fÐUXÐ(¨¨Q­Ó0Ñ2GÈÈAÍÓ2OÓPÑ\fˆÐgØˆ}Ðùò hs   Ã&&Dc                ó    <€ V ^8„  d   QhRS[ /# ©r   r   ©r   )r    rz   s   "€r!   r"   r{   R  s   ø€ ÷ ñ ™9ñ r#   c                ó¶   € \        \        V P                  V P                  RR7      4      p\	        VP
                  R4      '       d   RVP
                  n        V# )zKBuild a raw `tokenizers.Tokenizer` with BPE model (no pre/post-processing).F)Úfuse_unkÚignore_mergesT)r   r   r�   r�   ÚhasattrÚmodelrÁ   ©r~   Ú	tokenizers   & r!   rÅ   ÚMistralConverter.tokenizerR  sE   € äœc $×"9Ñ"9¸4×;SÑ;SÐ^cÔdÓeˆ	Ü�9—?‘? O×4Ò4Ø,0ˆI�O‰OÔ)ØÐr#   c                ó    <€ V ^8„  d   QhRS[ /# r½   r¾   )r    rz   s   "€r!   r"   r{   Y  s   ø€ ÷ ñ ™9ñ r#   c                ó–  € V P                  4       p\        P                  ! \        P                  ! \	        V P
                  4      RRR7      \        P                  ! V P                  RR7      .4      Vn        \        P                  ! 4       Vn
        VP                  V P                  4       \        P                  ! RR7      Vn        V# )zOBuild a fully configured `tokenizers.Tokenizer` with pre-tokenizer and decoder.ÚisolatedF)ÚbehaviorÚinvert)rx   Ú	use_regex)Útrim_offsets)rÅ   r	   ÚSequenceÚSplitr   r„   Ú	ByteLevelrx   Úpre_tokenizerr   ÚdecoderÚadd_special_tokensrŽ   r
   Úpost_processorrÄ   s   & r!   Ú	convertedÚMistralConverter.convertedY  s™   € à—N‘NÓ$ˆ	Ü"0×"9Ò"9ä×$Ò$¤U¨4¯<©<Ó%8À:ÐV[Ô\Ü×(Ò(¸$×:OÑ:OÐ[`Ôaðó#
ˆ	Ôô %×.Ò.Ó0ˆ	ÔØ×$Ñ$ T×%CÑ%CÔDä#-×#7Ò#7ÀUÔ#Kˆ	Ô àÐr#   )r�   r�   rx   rŽ   r„   N)F)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r€   r}   ÚstaticmethodrŒ   rÅ   rÕ   Ú__static_attributes__Ú__classdictcell__)rz   s   @r!   rf   rf     sE   ø‡ € ÙZ÷	>ò 	>÷%*ð %*ðN ÷ó ð÷6ð ÷ö r#   rf   c                ó0   € V ^8„  d   QhR\         R\        /# )r   rÅ   r   ©r   r   )r    s   "r!   r"   r"   j  s   € ÷ ñ Ô&7ð ¼Cñ r#   c                ó  € V P                   P                  R4      ;'       g    \        V RR4      p\        V\        4      '       d7   \        V4      '       d&   \        P                  P                  V4      '       g   \        R4      hV# )aû  Locate the on-disk `tekken.json` a tokenizer was built from.

Called once by `save_tekken_format`, which passes the resolved path on to
`_check_tekken_vocab_unchanged` to know what to diff against.

Args:
    tokenizer (`TokenizersBackend`): The tokenizer to resolve a source file for. Must
        have been loaded or converted from a native Mistral tekken vocabulary file,
        i.e. its `vocab_file` must still point at an existing file whose name identifies
        it as tekken (see `is_tekken_vocab_filename`).

Returns:
    `str`: The validated path to the tokenizer's source tekken vocabulary file.

Raises:
    OSError: If the tokenizer's `vocab_file` does not resolve to an existing file
        identified as tekken (see `is_tekken_vocab_filename`).
rw   Nz½Cannot save in 'mistral' format: the original tekken.json is not available. Load a native checkpoint (that still contains tekken.json) or install mistral-common to use MistralCommonBackend.)
Úinit_kwargsre   rb   Ú
isinstancer   r   r   ÚpathÚisfilerH   )rÅ   rw   s   & r!   Ú_resolve_tekken_sourceræ   j  sv   € ð& ×&Ñ&×*Ñ*¨<Ó8×bÐb¼GÀIÈ|Ð]aÓ<b€JÜ�z¤3×'Ò'Ô,DÀZ×,PÒ,PÔUW×U\ÑU\×UcÑUcÐdn×UoÒUoÜð:ó
ð 	
ð
 Ðr#   c                ó4   € V ^8„  d   QhR\         R\        RR/# )r   rÅ   rw   r   Nrà   )r    s   "r!   r"   r"   ‡  s#   € ÷ >
ñ >
Ô->ð >
ÌCð >
ÐTXñ >
r#   c                ób  € \        VRR7      ;_uu_ 4       p\        P                  ! V4      pRRR4       XR,          p\        V4      w  rV\	        V4      pVP                  R4      pVf   \        VR,          4      V,           p\        V 4      p	V P                  P                  4        U
u0 uF  qªP                  kK  	  pp
\        W·,
          4      p\        W{,
          4      pW˜8g  pV'       g	   V'       d†   R.pV'       d   VP                  RV 24       V'       d   VP                  R	V 24       V'       d   VP                  R
V	 RV 24       VP                  R4       \        RP                  V4      4      hV'       d   \        RV	 RV R24      hR#   + '       g   i     ELt; iu up
i )a÷  Compare *tokenizer*'s vocab against the `tekken.json` it was converted from.

`save_format="mistral"` writes a native `tekken.json` by copying the source file
directly, so any in-session mutation (`add_tokens`, `add_special_tokens`) would be
silently dropped, and `tekken.json` cannot represent arbitrary added tokens anyway.
This guard is called only from `save_tekken_format`, right before that lossy copy, to
turn a silent drop into a raised error.

Only two things are compared: the total vocab size, and the *set* of added-token
content strings (via `tokenizer.added_tokens_decoder`). This does not check token ids
or other `AddedToken` attributes (e.g. `lstrip`, `rstrip`, `normalized`).

Args:
    tokenizer (`TokenizersBackend`): The tokenizer to check. Never mutated.
    vocab_file (`str`): Path to the source `tekken.json` to compare against.

Raises:
    ValueError: If the vocab size or the added-token set diverges from the source
        `tekken.json`.
    OSError: If `vocab_file` cannot be opened.
r%   r&   NrT   r…   r‡   z“Cannot save in 'mistral' format: the tokenizer vocab state has diverged from its source tekken.json, which cannot represent arbitrary added tokens.zUnexpected added tokens: z!Missing expected special tokens: zVocab size: observed=z, expected=z7Use save_format='hf' instead to preserve these changes.Ú
zGCannot save in 'mistral' format: the tokenizer reports a vocab size of z&, but its source tekken.json declares a   (via default_vocab_size/default_num_special_tokens). Either the vocab was resized in this session, in which case use save_format='hf' to preserve the change, or the source tekken.json is itself inconsistent and should be regenerated with matching vocab and config sizes.)r,   r-   r.   rt   Úsetre   rj   Úadded_tokens_decoderÚvaluesÚcontentrh   r´   rk   r¥   )rÅ   rw   r5   rO   rT   Úexpected_special_tokens_listrq   Úexpected_special_tokensÚexpected_vocab_sizeÚobserved_vocab_sizer˜   Úobserved_added_tokensÚunexpected_tokensÚmissing_tokensÚsize_divergesÚliness   &&              r!   Ú_check_tekken_vocab_unchangedr÷   ‡  s—  € ô, 
ˆj 7×	+Õ	+¨qÜ�iŠi˜‹lˆ÷ 
,ð ��]€FÜ7NÈsÓ7SÑ4Ð Ü!Ð">Ó?Ðà Ÿ*™*Ð%9Ó:ÐØÒ"Ü! # g¥,Ó/Ð2DÕDÐä˜i›.ÐØ8A×8VÑ8V×8]Ñ8]Ô8_Ó`Ñ8_¨uŸ]œ]Ñ8_ÐÐ`äÐ4ÕNÓOÐÜÐ3ÕKÓL€NØ'Ñ>€MçŸNðJð
ˆ÷ Ø�L‰LÐ4Ð5FÐ4GÐHÔIßØ�L‰LÐ<¸^Ð<LÐMÔNßØ�L‰LÐ0Ð1DÐ0EÀ[ÐQdÐPeÐfÔgØ�‰ÐNÔOÜ˜Ÿ™ 5Ó)Ó*Ð*çÜØUØ"Ð#Ð#IÐJ]ÐI^ð _&ð&ó
ð 	
ñ ÷A 
,×	+Ð	+üò as   –FÂ+F,ÆF)	c                ó€   € V ^8„  d   QhR\         R\        \        P                  ,          R\        \        R3,          /# )r   rÅ   Úsave_directoryr   .)r   r   r   r   rD   )r    s   "r!   r"   r"   È  s8   € ÷ *ñ *Ô"3ð *ÄSÌ2Ï;É;ÕEVð *Ô[`ÔadÐfiÐaiÕ[jñ *r#   c                óV  € \        V 4      p\        W4       \        P                  ! VRR7       \        P                  P                  V\        4      p\        P                  P                  V4      '       d&   \        P                  P                  W#4      '       g   \        W#4       V3# )aÓ  Write a tokenizer back out in native Mistral format by copying its source tekken file.

Used by `TokenizersBackend.save_pretrained` for `save_format="mistral"`. The tokenizer's
original tekken vocabulary file (resolved via `vocab_file`) is copied byte-for-byte into
*save_directory* as `tekken.json`, regardless of the source's own basename (e.g. a source
named `tekken_240911.json` is normalized to `tekken.json` on save, so the output directory
stays discoverable by both `resolve_mistral_format` and mistral-common.

Resaving into the same directory the tokenizer was loaded from is a no-op copy: the
source and destination are the same file, so the copy is skipped instead of raising
`shutil.SameFileError`. The divergence guard still runs first, so an in-place resave of
a mutated tokenizer is still rejected.

Args:
    tokenizer (`TokenizersBackend`): The tokenizer to save. Must have been loaded or
        converted from a native Mistral tekken vocabulary file, i.e. its `vocab_file`
        must still point at an existing file identified as tekken (see
        `is_tekken_vocab_filename`), and it must not have diverged from that source
        file (see `_check_tekken_vocab_unchanged`).
    save_directory (`str` or `os.PathLike`): The path to a directory where the tekken
        vocabulary file will be written.

Returns:
    `tuple[str, ...]`: A 1-tuple containing the path of the saved tekken vocabulary file.

Raises:
    OSError: If the tokenizer's `vocab_file` does not resolve to an existing file
        identified as tekken (see `is_tekken_vocab_filename`).
    ValueError: If the tokenizer has diverged from its source tekken vocabulary file
        (added tokens, changed vocab size). See `_check_tekken_vocab_unchanged`.
T)Úexist_ok)
ræ   r÷   r   Úmakedirsrä   r¥   r   ÚexistsÚsamefiler   )rÅ   rù   rw   Údests   &&  r!   Úsave_tekken_formatr   È  sr   € ô@ (¨	Ó2€JÜ! )Ô8ä‡K‚K�¨Õ.ô �7‰7�<‰<˜Ô(9Ó:€DÜ�G‰G�N‰N˜4× Ò ¤R§W¡W×%5Ñ%5°j×%GÒ%GÜ�Ô"Øˆ7€Nr#   c                óJ   € V ^8„  d   QhR\         R\         R,          R\        /# )r   Útokenizer_filer   Nr   )r   r   )r    s   "r!   r"   r"   õ  s*   € ÷ ñ Üðä˜•:ðô ñr#   c           	     ó|   € \        W4      p\        V RR7      p\        RRVP                  4       RV RV/\        B pV# )u  Build a `TokenizersBackend` from a Mistral `tekken.json` file.

The chat template is resolved via a fixed precedence order (see
`_resolve_chat_template`): explicit `chat_template` argument â†’ sibling
`chat_template.jinja` â†’ sibling `chat_template.json` â†’ auto-generation via
`mistral-common` â†’ `None`.

Args:
    tokenizer_file (`str`): Path to the `tekken.json` vocabulary file.
    chat_template (`str`, *optional*): Explicit Jinja2 chat template string.
        When not provided (`None`), the template is resolved automatically
        from sibling files or generated via `mistral-common` if available.

Returns:
    Configured `TokenizersBackend` with BPE model, special token mappings, and
    an attached chat template (or `None` when none could be resolved).
F)rw   rx   Útokenizer_objectrw   r   rY   )r8   rf   r   rÕ   Ú_MAP_SPECIALS)r  r   ÚresolvedÚ	converterÚfasts   &&   r!   Úconvert_tekken_tokenizerr	  õ  sX   € ô* & nÓD€HÜ ¨NÈUÔS€IÜñ Ø"×,Ñ,Ó.ðà!ðð ðô ñ	€Dð €Kr#   )N)2rÛ   r‰   r-   r   Ú	functoolsr   Úpathlibr   Úshutilr   Ú
tokenizersr   r   r   r   r	   r
   Útokenizers.modelsr   Úconvert_slow_tokenizerr   Útokenization_utils_tokenizersr   Úutilsr   r   r   Ú	utils.hubr   r   Úutils.import_utilsr   Ú	constantsr   r   Ú'mistral_common.tokens.tokenizers.tekkenr   Ú
get_loggerr×   r1   r8   r@   rI   r  rt   rf   ræ   r÷   r   r	  rY   r#   r!   Ú<module>r     s¹   ðñ Vã Û Û 	Ý Ý Ý ç Y× YÝ !å 6Ý >ß <Ñ <ß PÝ =ß Bñ × Ò ÝBð 
×	Ò	˜HÓ	%€õ7õt÷6C2ðN �Ø�Ø�Ø�ð	€õ1@÷hfñ fõRõ:>
õB*÷Zñ r#   