+
    œ°³jÎÒ  ã                   óú&  a€ ERg t@0 t R t^ RIt^ RIt^ RIt^ RIt^ RIt^ RIHt ^ RI	H
t
 ^ RIHt ^RIHt ^RIHtHt ^RIHt ^RIHt ^R	IHtHtHtHtHt ^R
IHt ^RIHt ^RI H!t! ^RI"H#t#H$t$H%t%H&t&H't' ]! 4       '       d   ^RI(H)t) MRt)]! 4       '       d   ^RI*H+t+ MRt+]PX                  ! ]-4      t./ t/] ^ k / t0] ^k ]]1]1R,          3,          ! . R]! 4       '       d   RMR3NR]! 4       '       d   RMR3NR]! 4       '       d   RMR3NR]! 4       '       d   RMR3NR]! 4       '       d   RMR3NR]! 4       '       d   RMR3NR]! 4       '       d   RMR3NR]! 4       '       d   RMR3NR]! 4       '       d   RMR3NR ]! 4       '       d   R!MR3NERhNR"]! 4       '       d   RMR3NR#]! 4       '       d   R$MR3NERiNERjNR%]! 4       '       d   R&MR3NR']! 4       '       d   RMR3NERkNR(]! 4       '       d   R)MR3NERlNR*]! 4       '       d   RMR3NR+]! 4       '       d   R,MR3NERmNR-]! 4       '       d   RMR3NERnNR.]! 4       '       d   R/MR3NR0]! 4       '       d   RMR3NERoNR1]! 4       '       d   RMR3NERpNR2]! 4       '       d   RMR3NR3]! 4       '       d   RMR3NERqNR4]! 4       '       d   R5MR3NR6]! 4       '       d   R,MR3NR7]! 4       '       d   RMR3NR8]! 4       '       d   RMR3NR9]! 4       '       d   RMR3NR:]! 4       '       d   RMR3NR;]! 4       '       d   RMR3NR<]! 4       '       d   R=MR3NERrNERsNERtNERuNR>]! 4       '       d   R,MR3NR?]! 4       '       d   R@MR3NRA]! 4       '       d   RBMR3NERvNRC]! 4       '       d   RDMR3NRE]! 4       '       d   RMR3NRF]! 4       '       d   RGMR3NRH]! 4       '       d   RMR3NRI]! 4       '       d   R,MR3NRJ]! 4       '       d   RMR3NERwNRK]! 4       '       d   RLMR3NRM]! 4       '       d   RLMR3NRN]! 4       '       d   ROMR3NRP]! 4       '       d   RQMR3NERxNRR]! 4       '       d   RMR3NRS]! 4       '       d   RMR3NRT]! 4       '       d   RUMR3NRV]! 4       '       d   RWMR3NERyNRX]! 4       '       d   RMR3NRY]! 4       '       d   RZMR3NR[]! 4       '       d   RDMR3NR\]! 4       '       d   RDMR3NR]]! 4       '       d   RDMR3NR^]! 4       '       d   RDMR3NR_]! 4       '       d   RDMR3NR`]! 4       '       d   RDMR3NRa]! 4       '       d   RMR3NRb]! 4       '       d   RMR3NRc]! 4       '       d   RMR3NRd]! 4       '       d   RMR3NRe]! 4       '       d   RMR3NRf]! 4       '       d   RMR3NRg]! 4       '       d   RMR3NRh]! 4       '       d   RMR3NRi]! 4       '       d   RMR3NRj]! 4       '       d   RMR3NRk]! 4       '       d   RlMR3NRm]! 4       '       d   R,MR3NRn]! 4       '       d   RMR3NRo]! 4       '       d   R,MR3NRp]! 4       '       d   ROMR3NERzNRq]! 4       '       d   R,MR3NRr]! 4       '       d   RMR3NRs]! 4       '       d   RtMR3NRu]! 4       '       d   RMR3NRv]! 4       '       d   RMR3NRw]! 4       '       d   RMR3NRx]! 4       '       d   RMR3NRy]! 4       '       d   RMR3NRz]! 4       '       d   R{MR3NER{NR|]! 4       '       d   RMR3NER|NR}]! 4       '       d   R~MR3NR]! 4       '       d   R~MR3NR€]! 4       '       d   R,MR3NR�]! 4       '       d   R,MR3NR‚]! 4       '       d   RMR3NRƒ]! 4       '       d   R,MR3NR„]! 4       '       d   R…MR3NR†]! 4       '       d   RMR3NR‡]! 4       '       d   RMR3NRˆ]! 4       '       d   RMR3NR‰]! 4       '       d   RŠMR3NR‹]! 4       '       d   RŠMR3NRŒ]! 4       '       d   RMR3NR�]! 4       '       d   RŽMR3NR�]! 4       '       d   R�MR3NR‘]! 4       '       d   R’MR3NR“]! 4       '       d   R”MR3NR•]! 4       '       d   R–MR3NR—]! 4       '       d   RMR3NR˜]! 4       '       d   RMR3NER}NR™]! 4       '       d   RšMR3NR›]! 4       '       d   RœMR3NR�]! 4       '       d   ROMR3NRž]! 4       '       d   ROMR3NRŸ]! 4       '       d   R MR3NR¡]! 4       '       d   R¢MR3NR£]! 4       '       d   R¤MR3NR¥]! 4       '       d   R¦MR3NER~NR§]! 4       '       d   RMR3NR¨]! 4       '       d   R…MR3NERNR©]! 4       '       d   RMR3NRª]! 4       '       d   RMR3NR«]! 4       '       d   R¬M]! 4       '       d   RMR3NR­]! 4       '       d   R¬M]! 4       '       d   RMR3NR®]! 4       '       d   R¬M]! 4       '       d   RMR3NR¯]! 4       '       d   R¬M]! 4       '       d   RMR3NR°]! 4       '       d   R¬M]! 4       '       d   RMR3NR±]! 4       '       d   R²MR3NR³]! 4       '       d   RMR3NR´]! 4       '       d   RµMR3NR¶]! 4       '       d   R·MR3NR¸]! 4       '       d   ROMR3NER€NR¹]! 4       '       d   RºMR3NR»]! 4       '       d   RºMR3NR¼]! 4       '       d   RºMR3NR½]! 4       '       d   R¾MR3NER�NR¿]! 4       '       d   RtMR3NRÀ]! 4       '       d   RtMR3NRÁ]! 4       '       d   RMR3NRÂ]! 4       '       d   RÃMR3NRÄ]! 4       '       d   RÃMR3NRÅ]! 4       '       d   RMR3NRÆ]! 4       '       d   RÇMR3NRÈ]! 4       '       d   RMR3NRÉ]! 4       '       d   ROMR3NRÊ]! 4       '       d   RMR3NRË]! 4       '       d   RMR3NRÌ]! 4       '       d   RMR3NRÍ]! 4       '       d   ROMR3NRÎ]! 4       '       d   RMR3NRÏ]! 4       '       d   RMR3NRÐ]! 4       '       d   RÑMR3NRÒ]! 4       '       d   R,MR3NRÓ]! 4       '       d   RMR3NRÔ]! 4       '       d   RMR3NRÕ]! 4       '       d   RMR3NRÖ]! 4       '       d   RtMR3NR×]! 4       '       d   RtMR3NRØ]! 4       '       d   RtMR3NRÙ]! 4       '       d   RÚMR3NRÛ]! 4       '       d   RMR3NER‚NRÜ]! 4       '       d   RMR3NRÝ]! 4       '       d   R,MR3NERƒNRÞ]! 4       '       d   RMR3NRß]! 4       '       d   R¬M]! 4       '       d   RMR3NRà]! 4       '       d   RáMR3NRâ]! 4       '       d   RÇMR3NER„NRã]! 4       '       d   RMR3NRä]! 4       '       d   RMR3NRå]! 4       '       d   RMR3NRæ]! 4       '       d   RMR3NRç]! 4       '       d   RMR3NRè]! 4       '       d   RMR3NRé]! 4       '       d   RMR3NRê]! 4       '       d   RMR3NRë]! 4       '       d   RMR3NRì]! 4       '       d   RíMR3NRî]! 4       '       d   RíMR3NRï]! 4       '       d   RíMR3NRð]! 4       '       d   RMR3NRñ]! 4       '       d   RMR3NRò]! 4       '       d   RMR3NRó]! 4       '       d   RMR3NRô]! 4       '       d   RMR3NRõ]! 4       '       d   RMR3NRö]! 4       '       d   RíMR3NER…NR÷]! 4       '       d   RMR3NRø]! 4       '       d   RDMR3NRù]! 4       '       d   RúMR3NRû]! 4       '       d   RüMR3NRý]! 4       '       d   RMR3NER†NER‡NERˆNRþ]! 4       '       d   RÿMR3NER ]! 4       '       d   ROMR3NER]! 4       '       d   RMR3NER]! 4       '       d   RMR3NER]! 4       '       d   ERMR3NER]! 4       '       d   ERMR3NER]! 4       '       d   RDMR3NER]! 4       '       d   ERMR3NER	]! 4       '       d   ER
MR3NER]! 4       '       d   ERMR3NER]! 4       '       d   ERMR3NER‰NER]! 4       '       d   RMR3NER]! 4       '       d   RMR3NER]! 4       '       d   R,MR3NER]! 4       '       d   RºMR3NER]! 4       '       d   RºMR3NER]! 4       '       d   RDMR3NERŠNER]! 4       '       d   ERMR3NER]! 4       '       d   R…MR3NER]! 4       '       d   RMR3NER]! 4       '       d   ERMR3NER]! 4       '       d   RMR3NER‹NERŒNER]! 4       '       d   R–MR3NER]! 4       '       d   ERMR3NER]! 4       '       d   RMR3NER ]! 4       '       d   RMR3NER�NER!]! 4       '       d   R¬M]! 4       '       d   RMR3NER"]! 4       '       d   R¬M]! 4       '       d   RMR3NERŽNER�NER�NER‘NER#]! 4       '       d   ER$MR3NER%]! 4       '       d   RMR3NER&]! 4       '       d   ER'MR3NER’NER(]! 4       '       d   R…MR3NER)]! 4       '       d   RMR3NER*]! 4       '       d   ER+MR3NER,]! 4       '       d   ROMR3NER-]! 4       '       d   R…MR3NER.]! 4       '       d   RMR3NER/]! 4       '       d   RDMR3N4      t20 ER0kER1kER2kER3kER4kER5kER6kER7kER8kER9kER:kER;kER<kER=kER>kER?kER@kERAkERBkERCkERDkEREkERFkERGkERHkERIkERJkERKkERLkERMkERNkEROkERPkERQkERRkERSkERTkRåkERUkERVkERWkERXkERYkERZkER[kt3] ^k ]3 F   t4]4]29  g   K  ]! 4       '       d   RMR]2]4&   K"  	  ]!! ]#]24      t5]#Pl                  ! 4        U Uu/ uF  w  rWbK	  	  upp t7. ER“Ot8ER\ t9ER] t:ER”ER^ ER_ llt;ER` ERa lt<ER•ERb ERc llt= ! ERd ERe4      t>ERfERe.t?R# u upp i (–  zAuto Tokenizer class.N)ÚOrderedDict)ÚAny)Úis_mistral_common_available)ÚPreTrainedConfig)Úget_class_from_dynamic_moduleÚresolve_trust_remote_code)Úload_gguf_checkpoint)ÚTOKENIZER_CONFIG_FILE)Úextract_commit_hashÚis_g2p_en_availableÚis_sentencepiece_availableÚis_tokenizers_availableÚlogging)Úcached_file)ÚEncoderDecoderConfig)Ú_LazyAutoMapping)ÚCONFIG_MAPPING_NAMESÚ
AutoConfigÚconfig_class_to_model_typeÚmodel_type_to_module_nameÚ!replace_list_option_in_docstrings)ÚTokenizersBackend)ÚSentencePieceBackendÚEvollaModelr   Úaimv2ÚCLIPTokenizerÚalbertÚAlbertTokenizerÚalignÚBertTokenizerÚariaÚaudioflamingo3ÚQwen2TokenizerÚ
aya_visionÚCohereTokenizerÚbarkÚbartÚRobertaTokenizerÚbarthezÚBarthezTokenizerÚbertzbert-generationÚBertGenerationTokenizerÚbig_birdÚBigBirdTokenizerÚbigbird_pegasusÚ
blenderbotÚBlenderbotTokenizerÚblipzblip-2ÚGPT2TokenizerÚbrosÚ	camembertÚCamembertTokenizerÚcanaryÚchinese_clipÚclipÚclipsegÚ
code_llamaÚCodeLlamaTokenizerÚcodegenÚcohereÚcohere2Úcolqwen2ÚconvbertÚcosmos3_omniÚcpmÚCpmTokenizerÚdbrxÚdebertaÚDebertaTokenizerz
deberta-v2ÚDebertaV2TokenizerÚdiffusion_gemmaÚGemmaTokenizerÚ
distilbertÚdprÚDPRQuestionEncoderTokenizerÚelectraÚemu3ÚernieÚesmcÚEsmcTokenizerÚesmfold2Úfalcon_mambaÚGPTNeoXTokenizerÚfastspeech2_conformerÚFastSpeech2ConformerTokenizerÚflavaÚ	flex_olmoÚ	florence2ÚBartTokenizerÚfnetÚFNetTokenizerÚfun_asr_nanoÚfunnelÚFunnelTokenizerÚgemmaÚgemma2Úgemma3Úgemma3_textÚgemma3nÚgemma3n_textÚgitÚglmÚglm4Úglm4_moeÚglm4_moe_liteÚglm4vÚ	glm4v_moeÚ	glm_imageÚglmasrÚgot_ocr2zgpt-sw3ÚGPTSw3TokenizerÚgpt2Úgpt_bigcodeÚgpt_neoÚgpt_neoxÚgptjÚgraniteÚgranite_speech5_ctcÚParakeetTokenizerÚ
granitemoeÚgranitemoehybridÚgranitemoesharedzgrounding-dinoÚgroupvitÚherbertÚHerbertTokenizerÚ
hunyuan_vlÚideficsÚLlamaTokenizerÚidefics2ÚinstructblipÚinstructblipvideoÚinternvlÚjais2Újina_embeddings_v3ÚXLMRobertaTokenizerÚkimi_k25Úkimi_linearzkosmos-2Úlasr_ctcÚLasrTokenizerÚlasr_encoderÚlayoutlmÚ
layoutlmv2ÚLayoutLMv2TokenizerÚ
layoutlmv3ÚLayoutLMv3TokenizerÚ	layoutxlmÚLayoutXLMTokenizerÚledÚLEDTokenizerÚlighton_ocrÚQwen2TokenizerFastÚliltÚ
longformerÚlxmertÚLxmertTokenizerÚm2m_100ÚM2M100TokenizerÚmambaÚmamba2ÚmarianÚMarianTokenizerÚmarkuplmÚMarkupLMTokenizerÚmbartÚMBartTokenizerÚmbart50ÚMBart50Tokenizerzmegatron-bertÚ
metaclip_2Úmimo_v2_flashÚminicpmv4_6Ú	ministralÚMistralCommonBackendÚ
ministral3ÚmistralÚmistral3ÚmixtralÚmlukeÚMLukeTokenizerzmm-grounding-dinoÚ
mobilebertÚMobileBertTokenizerÚmpnetÚMPNetTokenizerÚmptÚmt5ÚT5TokenizerÚmusicgenÚmusicgen_melodyÚmvpÚMvpTokenizerÚnemotron3_5_asrÚnemotron_asr_streamingÚnezhaÚnllbÚNllbTokenizerznllb-moeÚ
nomic_bertÚnougatÚNougatTokenizerÚnystromformerÚolmoÚolmo2Úolmo3Úolmo_hybridÚolmoezomdet-turboÚ	oneformerz
openai-gptÚOpenAIGPTTokenizerÚoptÚovis2Úowlv2ÚowlvitÚparakeet_ctcÚparakeet_rnntÚparakeet_tdtÚpegasusÚPegasusTokenizerÚ	pegasus_xÚ	persimmonÚphiÚ
pix2structÚpixtralÚplbartÚPLBartTokenizerÚpp_formulanetÚqdqbertÚqianfan_ocrÚqwen2Úqwen2_5_omniÚ
qwen2_5_vlÚqwen2_audioÚ	qwen2_moeÚqwen2_vlÚqwen3Úqwen3_5ÚQwen3_5TokenizerÚqwen3_5_moeÚqwen3_5_textÚ	qwen3_asrÚ	qwen3_moeÚ
qwen3_nextÚqwen3_omni_moeÚqwen3_vlÚqwen3_vl_moeÚ	qwen4_expÚrealmÚrecurrent_gemmaÚreformerÚReformerTokenizerÚrembertÚRemBertTokenizerÚ	retribertÚroformerÚRoFormerTokenizerÚrwkvÚsam3Ú
sam3_videoÚseamless_m4tÚSeamlessM4TTokenizerÚseamless_m4t_v2Úshieldgemma2ÚsiglipÚSiglipTokenizerÚsiglip2ÚSiglip2TokenizerÚspeech_to_textÚSpeech2TextTokenizerÚspeecht5ÚSpeechT5TokenizerÚsqueezebertÚstablelmÚ
starcoder2Úswitch_transformersÚt5Út5gemmaÚtipsv2ÚTipsv2TokenizerÚtrocrÚtvpÚudopÚUdopTokenizerÚumt5Ú	vibevoiceÚ
videoprismÚVideoPrismTokenizerÚviltÚvisual_bertÚvoxtralÚvoxtral_realtimeÚwhisperÚWhisperTokenizerÚxclipÚxglmÚXGLMTokenizerzxlm-robertazxlm-roberta-xlÚxlnetÚXLNetTokenizerÚxlstmÚxmodÚyosoÚzayaÚarcticÚ	chameleonÚchatlmÚdeepseek_v2Údeepseek_v3Údeepseek_v32Údeepseek_v4Údeepseek_vlÚdeepseek_vl_hybridÚdeepseek_vl_v2Údeepseek_ocrÚdeepseek_ocr2Úernie4_5Úernie4_5_moeÚfuyuÚ
h2ovl_chatÚhyperclovaxÚhyperclovax_vlmÚ	internlm2ÚjambaÚjanusÚllavaÚ
llava_nextÚminicpm3ÚminicpmvÚ
minimax_m2Ú
modernbertÚmolmoÚmolmo2ÚnemotronÚnvfp4ÚopencuaÚopenvlaÚpaddleocr_vlÚphi3Úphi3_vÚphimoeÚstep3p5Ústep3_vlÚvipllavaÚ
cohere_asrzcamembertv2-baseÚsmolvlmzvision-encoder-decoderc                ó˜   € \        V RRR7      ;_uu_ 4       p\        P                  ! V4      uuRRR4       #   + '       g   i     R# ; i)z*Loads a vocabulary file into a dictionary.Úrúutf-8©ÚencodingN)ÚopenÚjsonÚload)Ú
vocab_fileÚreaders   & Úk/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/auto/tokenization_auto.pyÚ
load_vocabr`  »  s.   € ä	ˆj˜#¨×	0Õ	0°FÜ�yŠy˜Ó ÷ 
1×	0×	0Ó	0ús	   —8¸A		c           	     ó8  € . p\        V RRR7      ;_uu_ 4       pV F^  pVP                  4       pV'       g   K  VP                  R4      '       d   K6  VP                  \	        VP                  4       4      4       K`  	  RRR4       V#   + '       g   i     T# ; i)z Loads a merges file into a list.rV  rW  rX  Ú#N)rZ  ÚstripÚ
startswithÚappendÚtupleÚsplit)Úmerges_fileÚmergesr^  Úlines   &   r_  Úload_mergesrk  Á  ss   € à€FÜ	ˆk˜3¨×	1Õ	1°VÛˆDØ—:‘:“<ˆDß‰t˜DŸO™O¨C×0Ô0Ø—‘œe D§J¡J£LÓ1Ö2ñ ÷ 
2ð
 €M÷ 
2Ö	1ð
 €Mús   ™B¹BÁ,BÂB	c                óŠ   € V ^8„  d   QhR\         \        P                  \         ,          ,          R\        R,          R\        /# )é   Úpretrained_model_name_or_pathÚmistral_formatNÚreturn)ÚstrÚosÚPathLikeÚbool)Úformats   "r_  Ú__annotate__rv  Ì  s=   € ÷ sñ sÜ#&¬¯©´SÕ)9Õ#9ðsä˜4•Kðsô 
ñ	só    c                óŒ   € ^RI Hp VRJ d   R# R Uu/ uF  qDV9   g   K  WBV,          bK  	  ppV! V 3RV/VB ^ ,          # u upi )u  Probe whether to use the MistralCommonBackend for *pretrained_model_name_or_path*.

Args:
    pretrained_model_name_or_path: Model identifier or local path.
    mistral_format: Tri-state flag forwarded to ``resolve_mistral_format``.
        - ``None``: auto-detect â€” returns ``True`` only when mistral-common is
          installed *and* ``tekken.json`` can be found.
        - ``True``: force mode â€” raises ``ImportError`` if mistral-common is not
          installed or ``OSError`` if ``tekken.json`` is absent.
        - ``False``: short-circuits immediately and returns ``False`` without
          any network/file probe.
    **kwargs: Forwarded verbatim; only hub-probe keys
        (``subfolder``, ``revision``, ``token``, ``cache_dir``,
        ``local_files_only``) are passed through to ``resolve_mistral_format``.

Returns:
    ``True`` if the MistralCommonBackend should be used.
)Úresolve_mistral_formatFro  )Ú	subfolderÚrevisionÚtokenÚ	cache_dirÚlocal_files_only)Úintegrations.mistral.tokenizerry  )rn  ro  Úkwargsry  ÚkÚprobe_kwargss   &&,   r_  Ú_use_mistral_formatrƒ  Ì  sf   € õ. Ià˜ÓÙñ bóÙa˜ÐjpÑepŒˆ�!�9ŠÑað ð ñ "Ð"?ÑoÐP^ÐoÐbnÑoÐpqÕrÐrùòs
   “A Ac                óT   € V ^8„  d   QhR\         R\        \        ,          R,          /# )rm  Ú
class_namerp  N)rq  Útyper   )ru  s   "r_  rv  rv  î  s#   € ÷ 3ñ 3¬#ð 3´$´sµ)¸dÕ2Bñ 3rw  c                 óê  € V R9   d   \         # V \        9   d   \        V ,          # V \        9   d   \        V ,          # V R8X  d   \         # \        P	                  4        Fë  w  rW 8X  g   K  \        V4      pVR9   d    V R8X  d   \        P                  ! RR4      pM\        P                  ! RV 2R4      p \        W04      p\        VRR4      ;p'       dq   V\        P                  9   d\   \        P                  V,          p\        WdP                  R	,           V4       \        P                  P                  VR
,           V4       Vu # 	  \        P                   P#                  4        F  p\        VRR4      V 8X  g   K  Vu # 	  \        P                  ! R4      p\%        W€4      '       d   \        W€4      # V P'                  R	4      '       d   \)        V RR 4      # R#   \         d     EK’  i ; i)ÚBloomTokenizerr   r­   z.tokenization_mistral_commonÚtransformersÚ.ztransformers.modelsÚ
__module__NÚFastÚ_fastÚ__name__>   rˆ  ÚBloomTokenizerFast)r¯   r°   r±   r¬   r®   rÜ   r  éüÿÿÿ)r   ÚREGISTERED_FAST_ALIASESÚREGISTERED_TOKENIZER_CLASSESÚTOKENIZER_MAPPING_NAMESÚitemsr   Ú	importlibÚimport_moduleÚgetattrÚsysÚmodulesÚsetattrrŽ  Ú
setdefaultÚAttributeErrorÚTOKENIZER_MAPPINGÚ_extra_contentÚvaluesÚhasattrÚendswithÚtokenizer_class_from_name)	r…  Úmodule_nameÚtokenizer_classÚmoduleÚresultÚsubmodÚbase_modÚ	tokenizerÚmain_modules	   &        r_  r¢  r¢  î  s·  € àÐ=Ô=Ü Ð àÔ,Ô,Ü& zÕ2Ð2àÔ1Ô1Ü+¨JÕ7Ð7àÐ(Ô(Ü Ð ô )@×(EÑ(EÖ(GÑ$ˆØÖ(Ü3°KÓ@ˆKàÐrÔrØÐ"8Ô8ä"×0Ò0Ð1OÐQ_Ó`‘ä"×0Ò0°1°[°MÐ1BÐDYÓZ�ð	Ü  Ó4�ä% f¨l¸DÓAÐA�FÖAÀvÔQT×Q\ÑQ\ÔG\Ü"Ÿ{™{¨6Õ2�HÜ˜H§o¡o¸Õ&>ÀÔGÜ—K‘K×*Ñ*¨6°GÕ+;¸XÔFØ’ñ# )Hô* '×5Ñ5×<Ñ<Ö>ˆ	Ü�9˜j¨$Ó/°:Ö=ØÒñ ?ô ×)Ò)¨.Ó9€KÜˆ{×'Ò'Ü�{Ó/Ð/ð ×Ñ˜6×"Ò"Ü(¨°C°R¨Ó9Ð9áøô% "ô Ûðús   Â8BG"Ç"G2Ç1G2c                ó–  € V ^8„  d   QhR\         \        P                  \         ,          ,          R\         \        P                  \         ,          ,          R,          R\        R\        \         \         3,          R,          R\        \         ,          R,          R\         R,          R\        R	\         R
\        \         \
        3,          /	# )rm  rn  r}  NÚforce_downloadÚproxiesr|  r{  r~  rz  rp  )rq  rr  rs  rt  Údictr   )ru  s   "r_  rv  rv  $  s°   € ÷ ]ñ ]Ü#&¬¯©´SÕ)9Õ#9ð]ä”R—[‘[¤Õ%Õ%¨Õ,ð]ô ð]ô ”#”s�(�^˜dÕ"ð	]ô
 ”#�:˜Õð]ô �D�jð]ô ð]ô ð]ô 
Œ#Œsˆ(…^ñ]rw  c                óD  € VP                  R4      p	\        V \        VVVVVVVRRRV	R7      p
V
f   \        P	                  R4       / # \        W©4      p	\        V
RR7      ;_uu_ 4       p\        P                  ! V4      pRRR4       V	XR&   V#   + '       g   i     L; i)a¡
  
Loads the tokenizer configuration from a pretrained model tokenizer configuration.

Args:
    pretrained_model_name_or_path (`str` or `os.PathLike`):
        This can be either:

        - a string, the *model id* of a pretrained model configuration hosted inside a model repo on
          huggingface.co.
        - a path to a *directory* containing a configuration file saved using the
          [`~PreTrainedTokenizer.save_pretrained`] method, e.g., `./my_model_directory/`.

    cache_dir (`str` or `os.PathLike`, *optional*):
        Path to a directory in which a downloaded pretrained model configuration should be cached if the standard
        cache should not be used.
    force_download (`bool`, *optional*, defaults to `False`):
        Whether or not to force to (re-)download the configuration files and override the cached versions if they
        exist.
    proxies (`dict[str, str]`, *optional*):
        A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
        'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request.
    token (`str` or *bool*, *optional*):
        The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated
        when running `hf auth login` (stored in `~/.huggingface`).
    revision (`str`, *optional*, defaults to `"main"`):
        The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a
        git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any
        identifier allowed by git.
    local_files_only (`bool`, *optional*, defaults to `False`):
        If `True`, will only try to load the tokenizer configuration from local files.
    subfolder (`str`, *optional*, defaults to `""`):
        In case the tokenizer config is located inside a subfolder of the model repo on huggingface.co, you can
        specify the folder name here.

<Tip>

Passing `token=True` is required when you want to use a private model.

</Tip>

Returns:
    `dict`: The configuration of the tokenizer.

Examples:

```python
# Download configuration from huggingface.co and cache.
tokenizer_config = get_tokenizer_config("google-bert/bert-base-uncased")
# This model does not have a tokenizer config so the result will be an empty dict.
tokenizer_config = get_tokenizer_config("FacebookAI/xlm-roberta-base")

# Save a pretrained tokenizer locally and you can reload its config
from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-cased")
tokenizer.save_pretrained("tokenizer-test")
tokenizer_config = get_tokenizer_config("tokenizer-test")
```Ú_commit_hashF)r}  r¬  r­  r|  r{  r~  rz  Ú _raise_exceptions_for_gated_repoÚ%_raise_exceptions_for_missing_entriesÚ'_raise_exceptions_for_connection_errorsr°  Nz\Could not locate the tokenizer configuration file, will try to use the model config instead.rW  rX  )	Úgetr   r	   ÚloggerÚinfor
   rZ  r[  r\  )rn  r}  r¬  r­  r|  r{  r~  rz  r€  Úcommit_hashÚresolved_config_filer^  r¦  s   &&&&&&&&,    r_  Úget_tokenizer_configr¹  $  s¥   € ðJ —*‘*˜^Ó,€KÜ&Ø%ÜØØ%ØØØØ)ØØ).Ø.3Ø05Ø ôÐð Ò#Ü�‰ÐrÔsØˆ	Ü%Ð&:ÓH€Kä	Ð"¨W×	5Õ	5¸Ü—’˜6Ó"ˆ÷ 
6à(€Fˆ>ÑØ€M÷ 
6×	5ús   Á)BÂB	c                   óp   a € ] tR tRt o RtR t]]! ]4      V 3R lR l4       4       t	]
R	R l4       tRtV tR# )
ÚAutoTokenizeri„  a  
This is a generic tokenizer class that will be instantiated as one of the tokenizer classes of the library when
created with the [`AutoTokenizer.from_pretrained`] class method.

This class cannot be instantiated directly using `__init__()` (throws an error).
c                ó   € \        R 4      h)z}AutoTokenizer is designed to be instantiated using the `AutoTokenizer.from_pretrained(pretrained_model_name_or_path)` method.)ÚOSError)Úselfs   &r_  Ú__init__ÚAutoTokenizer.__init__Œ  s   € Üð_ó
ð 	
rw  c                ó0   <€ V ^8„  d   QhRS[ S[,          /# )rm  rp  )r   r   )ru  Ú__classdict__s   "€r_  rv  ÚAutoTokenizer.__annotate__”  s   ø€ ÷ J
ñ J
á	Ñ1Õ	1ñJ
rw  c           	     óö  a$€ VP                  RR4      pRVR&   VP                  RR4      pVP                  RR4      pVP                  RR4      pVP                  R4      pVP                  R	R4      p	V	RJ d0   \        V3R	R/VB  \        R
4      p
V
P                  ! V.VO5/ VB # Ve{   \
        P                  VR4      pVf-   \        RV RRP                  R \
         4       4       R24      h\        V4      p
V
f   \        RV R24      hV
P                  ! V.VO5/ VB # V'       de   ^RIH	pH
pHp \        W3/ VB pV! V4      w  ppVR,          V9   d   V! VV4      pM\        VRR7      R,          p\        P                  ! R0/ VB pMVf    \        P                  ! V3RV/VB pVP$                  p\'        VR4      '       d   VP(                  MRp\+        V3/ VB pVP                  RR4      pRpRV9   dG   \-        VR,          \.        \0        34      '       d   VR,          pMVR,          P                  RR4      p\-        \3        VRR4      ;p\4        4      '       d   VP7                  4       MRo$Vfk   \8        ec   \:        ;QJ d#    V$3R l\<         4       F  '       g   K   RM	  RM! V$3R l\<         4       4      '       d   \8        P                  ! V.VO5/ VB # T;'       g    \3        VRR4      pVEfV   VEeQ   VEeL   VR8w  EdD   \
        P                  V4      Ee,   \
        P                  V4      P?                  R4      VP?                  R4      8w  dô   \
        P                  V4      P?                  R4      pVR19  dQ   V\@        9   g   V\@        9   d   TMTp\        V4      p
V
e(   V
PB                  R29  d   V
P                  ! V.VO5/ VB # VR
8X  dA   R!V9  d:   \        V3R	V	/VB '       d&   \        R
4      p
V
e   V
P                  ! V.VO5/ VB # \8        e   \8        P                  ! V.VO5/ VB # \        R"V R#24      hR$V9   d   VR$,          VR$&   V'       d   VPE                  R4      '       d   VRR3 pVRJp\G        V4      \H        9   ;'       g6    VRJ;'       d*    \        V4      RJ;'       g    \        VR,           4      RJpT;'       dj    \G        V4      \H        9  ;'       dP    VRJ;'       dD    \        V4      ;'       g    \        VR,           4      PJ                  PM                  R%4      '       * pV'       d   V\@        9   d   VRJd   RpRpV'       dW   V'       gO   V^,          e   V^,          pM	V^ ,          pR&V9   d   VPO                  R&4      ^ ,          p MRp \Q        WqVVV 4      pV'       dz   V'       dr   V'       gj   V'       d   \        VP?                  R4      4       \S        XV3/ VB p
VP                  R'R4      pV
PU                  4        V
P                  ! V.VO5RV/VB # Vev   Tp!\        V!4      p
V
f*   V!PE                  R4      '       g   \        V!R,           4      p
V
e   V
PB                  R8X  d   \8        p
V
f   \8        p
V
P                  ! V.VO5/ VB # \3        VRR4      '       dQ   VPV                  p"R V"9  d   V"PE                  R4      '       d   V"RR3 p"\        V"4      p
V
P                  ! V.VO5/ VB # \-        V\X        4      '       d}   \G        VPZ                  4      \G        VP\                  4      JdE   \^        Pa                  R(VP\                  Pb                   R)VPZ                  Pb                   R*24       VP\                  p\e        \G        V4      PB                  4      ;'       g    \3        VR+R4      p#V#eq   \H        P                  \G        V4      \8        4      p
V
eJ   \3        V
R,R4      R
8X  d"   R!V9   g   \        V3R	V	/VB '       g   \8        p
V
P                  ! V.VO5/ VB # VP                  RR4      pVe—   VR8w  d   VPE                  R4      '       d   VRR3 p\        V4      p
V
f*   VPE                  R4      '       g   \        VR,           4      p
V
e   V
PB                  R8X  d   \8        p
V
f   \8        p
V
P                  ! V.VO5/ VB # \        R-VPb                   R.RP                  R/ \H         4       4       R24      h  \        \         3 d    \"        P                  ! T3/ TB p ELÈi ; i)4aÍ  
Instantiate one of the tokenizer classes of the library from a pretrained model vocabulary.

The tokenizer class to instantiate is selected based on the `model_type` property of the config object (either
passed as an argument or loaded from `pretrained_model_name_or_path` if possible), or when it's missing, by
falling back to using pattern matching on `pretrained_model_name_or_path`:

List options

Params:
    pretrained_model_name_or_path (`str` or `os.PathLike`):
        Can be either:

            - A string, the *model id* of a predefined tokenizer hosted inside a model repo on huggingface.co.
            - A path to a *directory* containing vocabulary files required by the tokenizer, for instance saved
              using the [`~PreTrainedTokenizer.save_pretrained`] method, e.g., `./my_model_directory/`.
            - a path to a single saved vocabulary file if and only if the tokenizer only requires a
              single vocabulary file (like Bert or XLNet), e.g.: `./my_model_directory/vocab.txt`. (Not
              applicable to all derived classes)
    inputs (additional positional arguments, *optional*):
        Will be passed along to the Tokenizer `__init__()` method.
    config ([`PreTrainedConfig`], *optional*)
        The configuration object used to determine the tokenizer class to instantiate.
    cache_dir (`str` or `os.PathLike`, *optional*):
        Path to a directory in which a downloaded pretrained model configuration should be cached if the
        standard cache should not be used.
    force_download (`bool`, *optional*, defaults to `False`):
        Whether or not to force the (re-)download the model weights and configuration files and override the
        cached versions if they exist.
    proxies (`dict[str, str]`, *optional*):
        A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
        'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request.
    revision (`str`, *optional*, defaults to `"main"`):
        The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a
        git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any
        identifier allowed by git.
    subfolder (`str`, *optional*):
        In case the relevant files are located inside a subfolder of the model repo on huggingface.co (e.g. for
        facebook/rag-token-base), specify it here.
    tokenizer_type (`str`, *optional*):
        Tokenizer type to be loaded.
    backend (`str`, *optional*, defaults to `"tokenizers"`):
        Backend to use for tokenization. Valid options are:
        - `"tokenizers"`: Use the HuggingFace tokenizers library backend (default)
        - `"sentencepiece"`: Use the SentencePiece backend
    trust_remote_code (`bool`, *optional*, defaults to `False`):
        Whether or not to allow for custom models defined on the Hub in their own modeling files. This option
        should only be set to `True` for repositories you trust and in which you have read the code, as it will
        execute code present on the Hub on your local machine.
    kwargs (additional keyword arguments, *optional*):
        Will be passed to the Tokenizer `__init__()` method. Can be used to set special tokens like
        `bos_token`, `eos_token`, `unk_token`, `sep_token`, `pad_token`, `cls_token`, `mask_token`,
        `additional_special_tokens`. See parameters in the `__init__()` for more details.

Examples:

```python
>>> from transformers import AutoTokenizer

>>> # Download vocabulary from huggingface.co and cache.
>>> tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-uncased")

>>> # Download vocabulary from huggingface.co (user-uploaded) and cache.
>>> tokenizer = AutoTokenizer.from_pretrained("dbmdz/bert-base-german-cased")

>>> # If vocabulary files are in a directory (e.g. tokenizer was saved using *save_pretrained('./test/saved_model/')*)
>>> # tokenizer = AutoTokenizer.from_pretrained("./test/bert_saved_model/")

>>> # Download vocabulary from huggingface.co and define model-specific arguments
>>> tokenizer = AutoTokenizer.from_pretrained("FacebookAI/roberta-base", add_prefix_space=True)

>>> # Explicitly use the tokenizers backend
>>> tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/llama-tokenizer", backend="tokenizers")

>>> # Explicitly use the sentencepiece backend
>>> tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/llama-tokenizer", backend="sentencepiece")
```ÚconfigNTÚ
_from_autoÚuse_fastÚtokenizer_typeÚtrust_remote_codeÚ	gguf_filero  r­   zPassed `tokenizer_type` z3 does not exist. `tokenizer_type` should be one of z, c              3   ó$   "  € T F  qx € K  	  R # 5i©N© ©Ú.0Úcs   & r_  Ú	<genexpr>Ú0AutoTokenizer.from_pretrained.<locals>.<genexpr>ú  s   é € Ð DÑ,C q¤Ó,Cùs   ‚rŠ  zTokenizer class z is not currently imported.)ÚGGUF_CONFIG_ARCHSÚget_gguf_configÚread_gguf_metadatazgeneral.architectureF)Úreturn_tensorsÚ
model_namer¤  Úauto_mapr»  Ú_name_or_pathÚ c              3   óR   <"  € T F  p\         P                   ! SV4      x € K  	  R # 5irÌ  )Úfnmatch)rÏ  ÚpÚ_config_name_or_paths   & €r_  rÑ  rÒ  .  s"   øé € ÐfÑFeÀ”G—O’OÐ$8¸!×<Ð<ÓFeùs   ƒ$'rŒ  r   ÚPythonBackendÚPreTrainedTokenizerFastÚfix_mistral_regexzTokenizer class 'zf' specified in the tokenizer config was not found. The tokenizer may need to be converted or re-saved.r°  ztransformers.z--Úcode_revisionz The encoder model config class: z3 is different from the decoder model config class: z˜. It is not recommended to use the `AutoTokenizer.from_pretrained()` method in this case. Please use the encoder and decoder specific tokenizer classes.Ú
model_typerŽ  z!Unrecognized configuration class z8 to build an AutoTokenizer.
Model type should be one of c              3   ó8   "  € T F  qP                   x € K  	  R # 5irÌ  )rŽ  rÎ  s   & r_  rÑ  rÒ  Ý  s   é € Ð4[ÑIZÀA·Z¶ZÓIZùs   ‚rÍ  )r   rß  rà  r­   )r   rß  rà  r�  )3Úpopr´  rƒ  r¢  Úfrom_pretrainedr“  Ú
ValueErrorÚjoinÚintegrations.ggufrÓ  rÔ  rÕ  r   r   r   Ú	for_modelr½  r   rã  r   r×  r¹  Ú
isinstancerf  Úlistr—  rq  Úlowerr   ÚanyÚMODEL_IDS_TO_TOKENIZERS_BACKENDÚremovesuffixÚ)MODELS_WITH_INCORRECT_HUB_TOKENIZER_CLASSrŽ  r¡  r†  r�  r‹  rd  rg  r   r   Úregister_for_auto_classr¤  r   ÚdecoderÚencoderrµ  ÚwarningÚ	__class__r   )%Úclsrn  Úinputsr€  rÅ  Ú_rÈ  rÉ  rÊ  ro  r¤  Útokenizer_class_namerÓ  rÔ  rÕ  Ú	gguf_pathÚmetadataÚtensor_namesÚconfig_dictÚconfig_model_typeÚconfig_model_nameÚtokenizer_configÚtokenizer_config_classÚtokenizer_auto_mapÚnameÚ
_hub_classÚregistered_class_namer…  Úhas_remote_codeÚhas_local_codeÚexplicit_local_codeÚ	class_refÚupstream_repoÚtokenizer_class_candidateÚ_classrã  rÞ  s%   &&*,                                @r_  ræ  ÚAutoTokenizer.from_pretrained’  sä	  ø€ ðd —‘˜H dÓ+ˆØ#ˆˆ|Ñð �J‰J�z 4Ó(ˆØŸ™Ð$4°dÓ;ˆØ"ŸJ™JÐ':¸DÓAÐØ—J‘J˜{Ó+ˆ	ØŸ™Ð$4°dÓ;ˆà˜TÓ!ÜÐ =Ñ]ÈdÐ]ÐV\Ò]Ü7Ð8NÓOˆOØ"×2Ò2Ð3PÐdÐSYÒdÐ]cÑdÐdð Ò%Ü#:×#>Ñ#>¸~ÈtÓ#TÐ à#Ò+Ü Ø.¨~Ð.>Ð>qØ—y‘yÑ DÕ,CÓ DÓDÐEÀQðHóð ô
 8Ð8LÓMˆOàÒ&Ü Ð#3Ð4HÐ3IÐIdÐ!eÓfÐfà"×2Ò2Ð3PÐdÐSYÒdÐ]cÑdÐdççcÑcä#Ð$AÑWÐPVÑWˆIÙ%7¸	Ó%BÑ"ˆH�lØÐ.Õ/Ð3DÔDÙ-¨h¸ÓE‘ä2°9ÈUÔSÐT\Õ]�Ü×)Ò)Ñ8¨KÑ8‰FØŠ^ðcÜ#×3Ò3Ø1ñØEVðØZ`ñ�ð #×-Ñ-ÐÜ18¸À×1NÒ1N˜F×-Ò-ÐTXÐô 0Ð0MÑXÐQWÑXÐØ!1×!5Ñ!5Ð6GÈÓ!NÐð "ÐØÐ)Ô)ÜÐ*¨:Õ6¼Ä¸×FÒFà%5°jÕ%AÑ"à%5°jÕ%A×%EÑ%EÀoÐW[Ó%\Ð"ô '´¸ÀÐQUÓ0VÐ(V¨ÔY\×]Ò]ˆD�J‰JŒLÐceð 	ð Ò&Ü!Ò-ß“ÔfÕFeÓf——’ÔfÕFeÓf×fÒfä$×4Ò4Ð5RÐfÐU[ÒfÐ_eÑfÐfð
 ,×WÐW¬w°vÐ?PÐRVÓ/Wˆ
àÓ&ØÓ&Ø!Ó-Ø! RÕ'Ü'×+Ñ+Ð,=Ó>ÓJÜ(×,Ñ,Ð->Ó?×LÑLÈVÓTØ×'Ñ'¨Ó/ô1ô %<×$?Ñ$?Ð@QÓ$R×$_Ñ$_Ð`fÓ$gÐ!Ø$ð -ô ð *Ô-VÔVØ,Ô0YÔYñ *ð
 $ð ô #<¸JÓ"G�Ø"Ò.°?×3KÑ3Kð Tô 4ð
 +×:Ò:Ð;XÐlÐ[aÒlÐekÑlÐlð &Ð)?Ô?Ø'¨vÔ5Ü'Ð(EÑoÐVdÐoÐhn×oÐoä";Ð<RÓ"S�Ø"Ò.Ø*×:Ò:Ð;XÐlÐ[aÒlÐekÑlÐlä Ò,Ü(×8Ò8Ð9VÐjÐY_ÒjÐciÑjÐjäØ# J <ð 0Fð Góð ð
 Ð-Ô-Ø%5°nÕ%EˆF�>Ñ"ç!Ð&<×&EÑ&EÀf×&MÒ&MØ%;¸C¸RÐ%@Ð"à,°DÐ8ˆÜ˜f›Ô):Ñ:÷ 
ð 
Ø"¨$Ð.÷ ð ä)Ð*@ÓAÈÐM÷ Zð ZÜ,Ð-CÀfÕ-LÓMÐUYÐYð	 	ð ÷ ð Ü�V“Ô$5Ñ5÷ð ð '¨dÐ2÷ 9ð 9ä-Ð.DÓE÷ Rð RÜ0Ð1GÈ&Õ1PÓQß‘*ŸZ™Z¨Ó8ô9ð 	÷ Ø!Ô%NÔNØ!¨Ó-à#ˆOØ!%Ðç×#6à! !Õ$Ò0Ø.¨qÕ1‘	à.¨qÕ1�	Ø�yÔ Ø )§¡°Ó 5°aÕ 8‘à $�Ü 9Ø!À.ÐRaÐcpó!Ð÷ ×0×9Lç%Ü)Ð*@×*MÑ*MÈfÓ*UÔVÜ;¸IÐGdÑoÐhnÑoˆOØ—
‘
˜?¨DÓ1ˆAØ×3Ñ3Ô5Ø"×2Ò2Ø-ðØ06òØJ[ðØ_eñð ð $Ò/Ø(>Ð%Ü7Ð8QÓRˆOØÒ&Ð/H×/QÑ/QÐRX×/YÒ/YÜ";Ð<UÐX^Õ<^Ó"_�ØÒ*¨×/GÑ/GÈ?Ô/ZÜ"3�àÒ&Ü"3�à"×2Ò2Ð3PÐdÐSYÒdÐ]cÑdÐdÜ�VÐ.°×5Ò5Ø×+Ñ+ˆFØ(°Ô6¸6¿?¹?È6×;RÒ;RØ  ˜�Ü7¸Ó?ˆOØ"×2Ò2Ð3PÐdÐSYÒdÐ]cÑdÐdô �fÔ2×3Ò3Ü�F—N‘NÓ#¬4°·±Ó+?Ó?Ü—‘Ø6°v·~±~×7OÑ7OÐ6Pð Q%Ø%+§^¡^×%=Ñ%=Ð$>ð ?2ð2ôð —^‘^ˆFä/´°V³×0EÑ0EÓF×mÐmÌ'ÐRXÐZfÐhlÓJmˆ
ØÒ!Ü/×3Ñ3´D¸³LÔBSÓTˆOØÒ*Ü˜?¨J¸Ó=ÐAWÔWØ'¨6Ô1Ü.Ð/LÑvÐ]kÐvÐou×vÐvä&7�OØ&×6Ò6Ð7TÐhÐW]ÒhÐagÑhÐhð "2×!5Ñ!5Ð6GÈÓ!NÐØ!Ò-Ø%Ð)<Ô<ÐAW×A`ÑA`Ðag×AhÒAhØ)?ÀÀÐ)DÐ&Ü7Ð8NÓOˆOØÒ&Ð/E×/NÑ/NÈv×/VÒ/VÜ";Ð<RÐU[Õ<[Ó"\�ØÒ*¨×/GÑ/GÈ?Ô/ZÜ"3�ØÒ&Ü"3�Ø"×2Ò2Ð3PÐdÐSYÒdÐ]cÑdÐdäØ/°×0@Ñ0@Ð/Að B+Ø+/¯9©9Ñ4[ÕIZÓ4[Ó+[Ð*\Ð\]ð_ó
ð 	
øôO ¤Ð(ô cÜ)×9Ò9Ð:WÑbÐ[aÑb“ðcús   Æe å(e8å7e8Nc                óì   € Vf   Ve   TpMVe   TpM\        R4      hW#V3 F  pVf   K	  V\        VP                  &   K  	  Ve   Ve   V\        VP                  &   \        P                  WVR7       R# )a|  
Register a new tokenizer in this mapping.

Args:
    config_class ([`PreTrainedConfig`]):
        The configuration corresponding to the model to register.
    tokenizer_class: The tokenizer class to register (V5 - preferred parameter).
    slow_tokenizer_class: (Deprecated) The slow tokenizer to register.
    fast_tokenizer_class: (Deprecated) The fast tokenizer to register.
Nz$You need to pass a `tokenizer_class`)Úexist_ok)rç  r’  rŽ  r‘  r�  Úregister)Úconfig_classr¤  Úslow_tokenizer_classÚfast_tokenizer_classr  Ú	candidates   &&&&& r_  r  ÚAutoTokenizer.registerà  s†   € ð Ò"à#Ò/Ø"6‘Ø%Ò1Ø"6‘ä Ð!GÓHÐHà.ÀoÓVˆIØÔ$ØCLÔ,¨Y×-?Ñ-?Ó@ñ Wð  Ò+Ð0DÒ0PØEYÔ#Ð$8×$AÑ$AÑBä×"Ñ" <È8Ð"ÖTrw  rÍ  )NNNF)rŽ  r‹  Ú__qualname__Ú__firstlineno__Ú__doc__r¿  Úclassmethodr   r“  ræ  Ústaticmethodr  Ú__static_attributes__Ú__classdictcell__)rÂ  s   @r_  r»  r»  „  sK   ø‡ € ñò
ð Ù&Ð'>Ó?÷J
ó @ó ðJ
ðX
 óUó öUrw  r»  r�  c                ó  € V ^8„  d   Qh/ ^ \         9   d&   \        \        \        \        ,          3,          ;R&   ^\         9   d&   \        \        \        \        ,          3,          ;R&   ^\         9   d   \
        \        ,          ;R&   # )rm  r’  r‘  rñ  )Ú__conditional_annotations__r®  rq  r†  r   Úset)ru  s   "r_  rv  rv     sf   € × Ñ ÷` 8Ò 7œd¤3¬¬S­	 >Õ2Ñ 7ña ÷b 3Ò 2œœc¤4¬¥9˜nÕ-Ñ 2ñc ÷R.ò .¬3¬s­8ñ .òS rw  )ÚbartphoÚBartphoTokenizer)zbert-japaneseÚBertJapaneseTokenizer)ÚbertweetÚBertweetTokenizer)ÚbiogptÚBioGptTokenizer)zblenderbot-smallÚBlenderbotSmallTokenizer)Úbridgetowerr'   )Úbyt5ÚByT5Tokenizer)ÚcanineÚCanineTokenizer)Úclapr'   )ÚclvpÚClvpTokenizer)ÚcpmantÚCpmAntTokenizer)ÚctrlÚCTRLTokenizer)zdata2vec-audioÚWav2Vec2CTCTokenizer)zdata2vec-textr'   )ÚdiaÚDiaTokenizer)ÚesmÚEsmTokenizer)ÚflaubertÚFlaubertTokenizer)ÚfsmtÚFSMTTokenizer)Úgpt_neox_japaneseÚGPTNeoXJapaneseTokenizer)Úhubertr5  )Úibertr'   )ÚlukeÚLukeTokenizer)Úmegar'   )zmgp-strÚMgpstrTokenizer)Úmrar'   )Úmyt5ÚMyT5Tokenizer)Ú	perceiverÚPerceiverTokenizer)ÚphobertÚPhobertTokenizer)Ú
prophetnetÚProphetNetTokenizer)ÚragÚRagTokenizer)Úrobertar'   )zroberta-prelayernormr'   )Úroc_bertÚRoCBertTokenizer)ÚsplinterÚSplinterTokenizer)ÚtapasÚTapasTokenizer)Ú	unispeechr5  )zunispeech-satr5  )ÚvitsÚVitsTokenizer)Úwav2vec2r5  )zwav2vec2-bertr5  )zwav2vec2-conformerr5  )Úwav2vec2_phonemeÚWav2Vec2PhonemeCTCTokenizer)ÚxlmÚXLMTokenizer)z'deepseek-ai/deepseek-r1-distill-llama-*zdeepseek-ai/deepseek-coder-*zallenai/dolma2-tokenizerzgoogle/umt5-smallzsalesforce/blip2-opt-*zsalesforce/blip2-flan-t5-*z!salesforce/instructblip-flan-t5-*zstepfun-ai/step-3.7-*rÌ  )NFNNNFrÚ  )Ar  r  rÜ  r•  r[  rr  r˜  Úcollectionsr   Útypingr   Útransformers.utils.import_utilsr   Úconfiguration_utilsr   Údynamic_module_utilsr   r   Úmodeling_gguf_pytorch_utilsr   Útokenization_utils_baser	   Úutilsr
   r   r   r   r   Ú	utils.hubr   Úencoder_decoderr   Úauto_factoryr   Úconfiguration_autor   r   r   r   r   Útokenization_utils_tokenizersr   Ú tokenization_utils_sentencepiecer   Ú
get_loggerrŽ  rµ  r’  r‘  rq  r“  rñ  rã  r�  r”  ÚCONFIG_TO_TYPErï  r`  rk  rƒ  r¢  r¹  r»  Ú__all__rv  )r�  Úvr  s   00@r_  Ú<module>rr     s\  øð÷ ã Û Û Û 	Û 
Ý #Ý å Gå 3ß \Ý ?Ý <÷õ õ %Ý 2Ý *÷õ ñ ×ÒÞBàÐá×ÒÞHàÐà	×	Ò	˜HÓ	%€ð 68Ð Ó 7Ø02Ð Ó 2à% c¨3°­: oÖ6ðpØ	Ñ/F×/HÒ/HÑ+ÈdÐSðpà	Ñ%<×%>Ò%>‘/ÀDÐIðpð 
Ñ(?×(AÒ(AÑ$ÀtÐLðpð 
Ñ%<×%>Ò%>‘/ÀDÐIð	pð
 
Ñ(?×(AÒ(AÑ$ÀtÐLðpð 
Ñ/F×/HÒ/HÑ+ÈdÐSðpð 
Ñ,C×,EÒ,EÑ(È4ÐPðpð 
Ñ$;×$=Ò$=‘À4ÐHðpð 
Ñ'>×'@Ò'@Ñ#ÀdÐKðpð 
Ñ*A×*CÒ*CÑ&ÈÐNðpñ 	(ðpð 
Ñ$;×$=Ò$=‘À4ÐHðpð 
Ñ9S×9UÒ9UÑ5Ð[_Ð`ðpñ 	3ðpñ 	*ðpð  
Ñ+B×+DÒ+DÑ'È$ÐOð!pð" 
Ñ3J×3LÒ3LÑ/ÐRVÐWð#pñ$ 	&ð%pð& 
Ñ0G×0IÒ0IÑ,ÈtÐTð'pñ( 	9ð)pð* 
Ñ$;×$=Ò$=‘À4ÐHð+pð, 
Ñ&=×&?Ò&?‘?ÀTÐJð-pñ. 	,ð/pð0 
Ñ$;×$=Ò$=‘À4ÐHð1pñ2 	"ð3pð4 
Ñ.E×.GÒ.GÑ*ÈTÐRð5pð6 
Ñ*A×*CÒ*CÑ&ÈÐNð7pñ8 	&ð9pð: 
Ñ,C×,EÒ,E™È4ÐPð;pñ< 	%ð=pð> 
Ñ$;×$=Ò$=‘À4ÐHð?pð@ 
Ñ'>×'@Ò'@‘OÀdÐKðApñB 	"ðCpðD 
Ñ/F×/HÒ/HÑ+ÈdÐSðEpðF 
Ñ'>×'@Ò'@‘OÀdÐKðGpðH 
Ñ(?×(AÒ(AÑ$ÀtÐLðIpðJ 
Ñ)@×)BÒ)BÑ%ÈÐMðKpðL 
Ñ)@×)BÒ)BÑ%ÈÐMðMpðN 
Ñ(?×(AÒ(A‘_ÀtÐLðOpðP 
Ñ-D×-FÒ-FÑ)ÈDÐQðQpðR 
Ñ"9×";Ò";‘ÀÐFðSpñT 	&ðUpñV 	"ðWpñX 	3ðYpñZ 	.ð[pð\ 
Ñ$;×$=Ò$=‘À4ÐHð]pð^ 
Ñ*A×*CÒ*CÑ&ÈÐNð_pð` 
Ñ/F×/HÒ/HÑ+ÈdÐSðapñb 	 ðcpðd 
Ñ0G×0IÒ0IÑ,ÈtÐTðepðf 
Ñ*A×*CÒ*C‘ÈÐNðgpðh 
Ñ1H×1JÒ1JÑ-ÐPTÐUðipðj 
Ñ'>×'@Ò'@‘OÀdÐKðkpðl 
Ñ$;×$=Ò$=‘À4ÐHðmpðn 
Ñ%<×%>Ò%>‘/ÀDÐIðopñp 	 ðqpðr 
Ñ$;×$=Ò$=‘À4ÐHðspðt 
Ñ(?×(AÒ(A‘_ÀtÐLðupðv 
Ñ/F×/HÒ/HÑ+ÈdÐSðwpðx 
!ÑEX×EZÒEZÑ"AÐ`dÐeðypñz 	*ð{pð| 
Ñ%<×%>Ò%>‘/ÀDÐIð}pð~ 
Ñ-D×-FÒ-FÑ)ÈDÐQðpð@ 
Ñ)@×)BÒ)B‘oÈÐMðApðB 
Ñ$;×$=Ò$=‘À4ÐHðCpñD 	"ðEpðF 
Ñ-D×-FÒ-FÑ)ÈDÐQðGpðH 
Ñ(?×(AÒ(AÑ$ÀtÐLðIpðJ 
Ñ&=×&?Ò&?Ñ"ÀTÐJðKpðL 
Ñ'>×'@Ò'@Ñ#ÀdÐKðMpðN 
Ñ'>×'@Ò'@Ñ#ÀdÐKðOpðP 
Ñ,C×,EÒ,EÑ(È4ÐPðQpðR 
Ñ(?×(AÒ(AÑ$ÀtÐLðSpðT 
Ñ-D×-FÒ-FÑ)ÈDÐQðUpðV 
Ñ#:×#<Ò#<‘À$ÐGðWpðX 
Ñ'>×'@Ò'@Ñ#ÀdÐKðYpðZ 
Ñ(?×(AÒ(AÑ$ÀtÐLð[pð\ 
Ñ,C×,EÒ,EÑ(È4ÐPð]pð^ 
Ñ1H×1JÒ1JÑ-ÐPTÐUð_pð` 
Ñ)@×)BÒ)BÑ%ÈÐMðapðb 
Ñ-D×-FÒ-FÑ)ÈDÐQðcpðd 
Ñ-D×-FÒ-FÑ)ÈDÐQðepðf 
Ñ*A×*CÒ*CÑ&ÈÐNðgpðh 
Ñ,C×,EÒ,EÑ(È4ÐPðipðj 
Ñ)C×)EÒ)EÑ%È4ÐPðkpðl 
Ñ$;×$=Ò$=‘À4ÐHðmpðn 
Ñ/F×/HÒ/HÑ+ÈdÐSðopðp 
Ñ'>×'@Ò'@‘OÀdÐKðqpðr 
Ñ+B×+DÒ+DÑ'È$ÐOðspñt 	:ðupðv 
Ñ$;×$=Ò$=‘À4ÐHðwpðx 
Ñ+B×+DÒ+DÑ'È$ÐOðypðz 
Ñ7N×7PÒ7PÑ 3ÐVZÐ[ð{pð| 
Ñ.E×.GÒ.GÑ*ÈTÐRð}pð~ 
Ñ4K×4MÒ4MÑ0ÐSWÐXðpð@ 
Ñ4K×4MÒ4MÑ0ÐSWÐXðApðB 
Ñ.E×.GÒ.G™?ÈTÐRðCpðD 
Ñ(?×(AÒ(A‘_ÀtÐLðEpðF 
Ñ*A×*CÒ*CÑ&ÈÐNðGpñH 	+ðIpðJ 
Ñ+B×+DÒ+DÑ'È$ÐOðKpñL 	&ðMpðN 
Ñ(?×(AÒ(AÑ$ÀtÐLðOpðP 
Ñ)@×)BÒ)BÑ%ÈÐMðQpðR 
Ñ,C×,EÒ,E™È4ÐPðSpðT 
Ñ1H×1JÒ1J™oÐPTÐUðUpðV 
Ñ)@×)BÒ)BÑ%ÈÐMðWpðX 
Ñ%<×%>Ò%>‘/ÀDÐIðYpðZ 
Ñ8O×8QÒ8QÑ4ÐW[Ð\ð[pð\ 
Ñ,C×,EÒ,EÑ(È4ÐPð]pð^ 
Ñ/F×/HÒ/HÑ+ÈdÐSð_pð` 
Ñ,C×,EÒ,EÑ(È4ÐPðapðb 
Ñ(?×(AÒ(A‘_ÀtÐLðcpðd 
Ñ,C×,EÒ,E™È4ÐPðepðf 
Ñ(?×(AÒ(A‘_ÀtÐLðgpðh 
Ñ0G×0IÒ0IÑ,ÈtÐTðipðj 
Ñ0G×0IÒ0IÑ,ÈtÐTðkpðl 
Ñ.E×.GÒ.GÑ*ÈTÐRðmpðn 
Ñ"9×";Ò";‘ÀÐFðopðp 
Ñ0G×0IÒ0IÑ,ÈtÐTðqpðr 
Ñ'>×'@Ò'@Ñ#ÀdÐKðspðt 
Ñ-D×-FÒ-FÑ)ÈDÐQðupñv 	"ðwpðx 
Ñ(?×(AÒ(AÑ$ÀtÐLðypðz 
Ñ)C×)EÒ)EÑ%È4ÐPð{pð| 
Ñ(?×(AÒ(AÑ$ÀtÐLð}pð~ 
Ñ)@×)BÒ)BÑ%ÈÐMðpð@ 
Ñ(B×(DÒ(DÑ$È$ÐOðApðB 
Ñ,C×,EÒ,EÑ(È4ÐPðCpðD 
Ñ&=×&?Ò&?Ñ"ÀTÐJðEpðF 
Ñ*A×*CÒ*CÑ&ÈÐNðGpñH 	%ðIpðJ 
Ñ-D×-FÒ-F™/ÈDÐQðKpðL 
Ñ0G×0IÒ0IÑ,ÈtÐTðMpñN 	'ðOpðP 
Ñ1H×1JÒ1JÑ-ÐPTÐUðQpðR 
Ñ/F×/HÒ/HÑ+ÈdÐSðSpðV á*×,Ò,ñ #á)@×)BÒ)BÑ%Èð		
ðUpðb á*×,Ò,ñ #á)@×)BÒ)BÑ%Èð		
ðapðn á*×,Ò,ñ #á)@×)BÒ)BÑ%Èð		
ðmpðz á*×,Ò,ñ #á)@×)BÒ)BÑ%Èð		
ðypðF á*×,Ò,ñ #á)@×)BÒ)BÑ%Èð		
ðEpðP 
Ñ&@×&BÒ&BÑ"ÈÐMðQpðR 
Ñ1H×1JÒ1J™oÐPTÐUðSpðT 
Ñ0G×0IÒ0IÑ,ÈtÐTðUpðV 
Ñ&=×&?Ò&?Ñ"ÀTÐJðWpðX 
Ñ&=×&?Ò&?Ñ"ÀTÐJðYpñZ 	$ð[pð\ 
Ñ!8×!:Ò!:‘ÀÐEð]pð^ 
Ñ&=×&?Ò&?‘]ÀTÐJð_pð` 
Ñ-D×-FÒ-F™MÈDÐQðapðb 
Ñ"9×";Ò";‘ÀÐFðcpñd 	"ðepðf 
Ñ3J×3LÒ3LÑ/ÐRVÐWðgpðh 
"Ñ:Q×:SÒ:SÑ#6ÐY]Ð^ðipðj 
Ñ%<×%>Ò%>‘/ÀDÐIðkpðl 
Ñ$;×$=Ò$=‘À4ÐHðmpðn 
Ñ(?×(AÒ(A‘_ÀtÐLðopðp 
Ñ*A×*CÒ*C‘ÈÐNðqpðr 
Ñ(?×(AÒ(AÑ$ÀtÐLðspðt 
Ñ1H×1JÒ1JÑ-ÐPTÐUðupðv 
Ñ'>×'@Ò'@Ñ#ÀdÐKðwpðx 
Ñ)@×)BÒ)BÑ%ÈÐMðypðz 
Ñ)@×)BÒ)BÑ%ÈÐMð{pð| 
Ñ/F×/HÒ/HÑ+ÈdÐSð}pð~ 
Ñ(?×(AÒ(AÑ$ÀtÐLðpð@ 
Ñ+B×+DÒ+D™È$ÐOðApðB 
Ñ)@×)BÒ)B‘oÈÐMðCpðD 
Ñ/F×/HÒ/HÑ+ÈdÐSðEpðF 
Ñ#:×#<Ò#<‘À$ÐGðGpðH 
Ñ&=×&?Ò&?Ñ"ÀTÐJðIpðJ 
Ñ%<×%>Ò%>‘/ÀDÐIðKpðL 
Ñ&=×&?Ò&?‘?ÀTÐJðMpðN 
Ñ0G×0IÒ0IÑ,ÈtÐTðOpðP 
Ñ1H×1JÒ1JÑ-ÐPTÐUðQpðR 
Ñ0G×0IÒ0IÑ,ÈtÐTðSpðT 
Ñ*A×*CÒ*CÑ&ÈÐNðUpðV 
Ñ-D×-FÒ-FÑ)ÈDÐQðWpñX 	,ðYpðZ 
Ñ-D×-FÒ-FÑ)ÈDÐQð[pð\ 
Ñ#:×#<Ò#<‘À$ÐGð]pñ^ 	(ð_pð` 
Ñ.E×.GÒ.GÑ*ÈTÐRðapðd á*×,Ò,ñ #á)@×)BÒ)BÑ%Èð		
ðcpðn 
Ñ(?×(AÒ(AÑ$ÀtÐLðopðp 
Ñ/F×/HÒ/HÑ+ÈdÐSðqpñr 	.ðspðt 
Ñ'>×'@Ò'@‘OÀdÐKðupðv 
Ñ,C×,EÒ,EÑ(È4ÐPðwpðx 
Ñ&=×&?Ò&?Ñ"ÀTÐJðypðz 
Ñ-D×-FÒ-FÑ)ÈDÐQð{pð| 
Ñ+B×+DÒ+DÑ'È$ÐOð}pð~ 
Ñ,C×,EÒ,EÑ(È4ÐPðpð@ 
Ñ*A×*CÒ*CÑ&ÈÐNðApðB 
Ñ)@×)BÒ)BÑ%ÈÐMðCpðD 
Ñ&=×&?Ò&?Ñ"ÀTÐJðEpðF 
Ñ*A×*CÒ*CÑ&ÈÐNðGpðH 
Ñ.E×.GÒ.GÑ*ÈTÐRðIpðJ 
Ñ/F×/HÒ/HÑ+ÈdÐSðKpðL 
Ñ*A×*CÒ*CÑ&ÈÐNðMpðN 
Ñ*A×*CÒ*CÑ&ÈÐNðOpðP 
Ñ+B×+DÒ+DÑ'È$ÐOðQpðR 
Ñ/F×/HÒ/HÑ+ÈdÐSðSpðT 
Ñ)@×)BÒ)BÑ%ÈÐMðUpðV 
Ñ-D×-FÒ-FÑ)ÈDÐQðWpðX 
Ñ,C×,EÒ,EÑ(È4ÐPðYpñZ 	 ð[pð\ 
Ñ%<×%>Ò%>‘/ÀDÐIð]pð^ 
Ñ0G×0IÒ0IÑ,ÈtÐTð_pð` 
Ñ,C×,EÒ,EÑ(È4ÐPðapðb 
Ñ*A×*CÒ*CÑ&ÈÐNðcpðd 
Ñ)@×)BÒ)B‘oÈÐMðepñf 	(ðgpñh 	5ðipñj 	)ðkpðl 
Ñ,C×,EÒ,EÑ(È4ÐPðmpñn 
Ñ'>×'@Ò'@Ñ#ÀdÐKðopñp 
Ñ$;×$=Ò$=‘À4ÐHðqpñr 
Ñ*A×*CÒ*C‘ÈÐNðspñt 
Ñ3J×3LÒ3LÒ/ÐRVÐWðupñv 
Ñ6M×6OÒ6OÒ2ÐUYÐZðwpñx 
Ñ-D×-FÒ-FÑ)ÈDÐQðypñz 
Ñ(B×(DÒ(DÒ$È$ÐOð{pñ| 
Ñ*A×*CÒ*CÒ&ÈÐNð}pñ~ 
Ñ5O×5QÒ5QÒ1ÐW[Ð\ðpñ@ 
Ñ,F×,HÒ,HÒ(ÈdÐSðApñB 	*ðCpñD 
Ñ+B×+DÒ+D™È$ÐOðEpñF 
Ñ,C×,EÒ,EÑ(È4ÐPðGpñH 
Ñ*A×*CÒ*C‘ÈÐNðIpñJ 
Ñ1H×1JÒ1J¡ÐPTÐUðKpñL 
Ñ 7× 9Ò 9‰}¸tÐDðMpñN 
Ñ(?×(AÒ(AÑ$ÀtÐLðOpñP 	$ðQpñR 
Ñ(B×(DÒ(DÒ$È$ÐOðSpñT 
Ñ+B×+DÒ+DÑ'È$ÐOðUpñV 
Ñ#:×#<Ò#<‘À$ÐGðWpñX 
Ñ$;×$=Ò$=’À4ÐHðYpñZ 
Ñ(?×(AÒ(AÑ$ÀtÐLð[pñ\ 	.ð]pñ^ 	2ð_pñ` 
Ñ.E×.GÒ.GÑ*ÈTÐRðapñb 
Ñ0G×0IÒ0IÒ,ÈtÐTðcpñd 
Ñ$;×$=Ò$=‘À4ÐHðepñf 
Ñ+B×+DÒ+D™È$ÐOðgpñh 	"ðipñl á*×,Ò,ñ #á)@×)BÒ)BÑ%Èð		
ðkpñx á*×,Ò,ñ #á)@×)BÒ)BÑ%Èð		
ðwpñB	 	-ðC	pñD	 	2ðE	pñF	 	7ðG	pñH	 	<ðI	pñJ	 
Ñ*A×*CÒ*CÒ&ÈÐNðK	pñL	 
Ñ%<×%>Ò%>‘/ÀDÐIðM	pñN	 
Ñ$;×$=Ò$=’À4ÐHðO	pñP	 	 ðQ	pñR	 
Ñ1H×1JÒ1JÑ-ÐPTÐUðS	pñT	 
Ñ2I×2KÒ2KÑ.ÐQUÐVðU	pñV	 
Ñ&=×&?Ò&?Ò"ÀTÐJðW	pñX	 
Ñ(?×(AÒ(AÑ$ÀtÐLðY	pñZ	 
Ñ*A×*CÒ*CÑ&ÈÐNð[	pñ\	 
Ñ&=×&?Ò&?Ñ"ÀTÐJð]	pñ^	 
Ñ%<×%>Ò%>Ñ!ÀDÐIð_	pórÐ ðl	.7Ùð.7áð.7ñ ð.7ñ ð	.7ñ
 ð.7ñ ð.7ñ ð.7ñ ð.7ñ ð.7ñ ð.7ñ ð.7ñ ð.7ñ ð.7ñ ð.7ñ ð.7ñ  ð!.7ñ" ð#.7ñ$ ð%.7ñ& ð'.7ñ( ð).7ñ* ð+.7ñ, ð-.7ñ. ð/.7ñ0 ð1.7ñ2 ð3.7ñ4 ð5.7ñ6 ð7.7ñ8 ð9.7ñ: ð;.7ñ< ð=.7ñ> ð?.7ñ@ ðA.7ñB ðC.7ñD ðE.7ñF ðG.7ñH ðI.7ñJ ðK.7ðL ðM.7ñN ðO.7ñP ðQ.7ñR ðS.7ñT ðU.7ñV ðW.7ñX ðY.7ñZ ð[.7Ð )ó .ó` <€JØÐ0Ö0ÙE\×E^ÒE^Ñ.AÐdhÐ 
Ó+ñ <ñ %Ð%9Ð;RÓSÐ à#7×#=Ò#=Ô#?Ô@Ñ#?™4˜1�!’$Ñ#?Ò@€ó	#Ð ó!ó÷sò s÷D3÷l]ò ]÷@zUó zUñz ¡Ð
0‚ùói As   ÁL'AM7