+
    œ°³j�' ã                   óh  € R t ^ RIt^ RIt^ RIHt ^ RIHt ^ RIHtH	t	H
t
HtHtHtHt ^ RIHtHtHt ^RIHt ^RIHtHtHtHt ^RIHt ]P8                  ! ]4      t. R§Ot]. R¨O,           t R©R	 lt!R
 R lt"RªR R llt# ! R R4      t$ ! R R]$4      t%R R lt& ! R R4      t' ! R R]'4      t( ! R R]'4      t) ! R R]'4      t* ! R R]'4      t+ ! R R]'4      t, ! R  R!]'4      t- ! R" R#]'4      t. ! R$ R%]'4      t/ ! R& R']'4      t0 ! R( R)]'4      t1 ! R* R+]'4      t2 ! R, R-]'4      t3 ! R. R/]34      t4 ! R0 R1]34      t5 ! R2 R3]34      t6 ! R4 R5]34      t7 ! R6 R7]34      t8 ! R8 R9]34      t9 ! R: R;]34      t: ! R< R=]34      t; ! R> R?]34      t< ! R@ RA]34      t= ! RB RC]34      t> ! RD RE]34      t? ! RF RG]34      t@ ! RH RI]34      tA ! RJ RK]34      tB ! RL RM]34      tC ! RN RO]B4      tD ! RP RQ]'4      tE ! RR RS]34      tF ! RT RU]'4      tG ! RV RW]'4      tH ! RX RY]'4      tI ! RZ R[]34      tJ ! R\ R]]34      tK ! R^ R_]34      tL ! R` Ra]'4      tM ! Rb Rc]34      tN ! Rd Re]34      tO ! Rf Rg]34      tP ! Rh Ri]P4      tQRj tR ! Rk Rl4      tS/ Rm]4bRn]0bRo]5bRp](bRq]FbRr]IbRs]6bRt]GbRu]-bRv](bRw]2bRx]7bRy](bRz](bR{](bR|](bR}](b/ R~]4bR]*bR€]-bR�].bR‚](bRƒ](bR„]0bR…]<bR†]0bR‡]0bRˆ](bR‰]MbRŠ]8bR‹]9bRŒ]+bR�](bRŽ]0bC/ R�]:bR�],bR‘]AbR’]/bR“]>bR”]?bR•]0bR–]1bR—];bR˜](bR™]BbRš]CbR›]DbRœ]EbR�]<bRž]=bRŸ])bCR ]JR¡]LR¢]LR£]KR¤]L/CtTR«R¥ R¦ lltUR# )¬zù
Utilities to convert slow tokenizers in their fast tokenizers counterparts.

All the conversions are grouped here to gather SentencePiece dependencies outside of the fast tokenizers files and
allow to make our dependency on SentencePiece optional.
N)Ú
Collection)Úversion)Ú
AddedTokenÚRegexÚ	TokenizerÚdecodersÚnormalizersÚpre_tokenizersÚ
processors)ÚBPEÚUnigramÚ	WordPiece)Úis_tekken_vocab_filename)Úis_protobuf_availableÚis_sentencepiece_availableÚloggingÚrequires_backends)ÚPROTOBUF_IMPORT_ERRORc                 ó>  € \        4       '       d	   ^ RIHp V# \        4       '       dX   ^ RIp\
        P                  ! VP                  P                  4      \
        P                  ! R4      8  d	   ^ RI	Hp V# ^ RI	H
p V# \        \        P                  ! V 4      4      h)é    )Úsentencepiece_model_pb2Nz4.0.0)Úsentencepiece_model_pb2_new)r   Úsentencepiecer   r   Úgoogle.protobufr   ÚparseÚprotobufÚ__version__Útransformers.utilsr   ÚImportErrorr   Úformat)Úerror_messager   Úgoogles   &  Úd/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/convert_slow_tokenizer.pyÚimport_protobufr#   _   sr   € Ü!×#Ò#Ý9à&Ð&Ü×ÒÛä�=Š=˜Ÿ™×4Ñ4Ó5¼¿ºÀgÓ8NÔNÝBð 'Ð&õ bØ&Ð&äÔ/×6Ò6°}ÓEÓFÐFó    c                ó0   € V ^8„  d   QhR\         R\        /# )é   Úadd_prefix_spaceÚreturn)ÚboolÚstr)r   s   "r"   Ú__annotate__r+   p   s   € ÷ ñ ¬$ð Äsñ r$   c                 óL   € V '       d   R p\        VRR4      '       g   RpV# RpV# )ÚalwaysÚlegacyTÚfirstÚnever)Úgetattr)r'   Úoriginal_tokenizerÚprepend_schemes   && r"   Ú_get_prepend_schemer4   p   s4   € ßØ!ˆÜÐ)¨8°T×:Ò:Ø$ˆNð Ðð !ˆØÐr$   c                óH   € V ^8„  d   QhR\         \        ,          R,          /# )r&   Úskip_tokensN)r   r*   )r   s   "r"   r+   r+   z   s   € ÷ ñ ´jÄµoÈÕ6Lñ r$   c                 ó  a € Ve   \        V4      M	\        4       pVR JpV'       d   \        V4      MS p. pVP                  4        Fˆ  w  rVWR9   d   K  . p\        ^\	        V4      4       F>  pVR V WXR  r©W’9   g   W¢9   d   K  V	S 9   g   K"  V
S 9   g   K+  VP                  WšV34       K@  	  \        VV 3R lR7      pVP                  V4       KŠ  	  \        VR VR7      pV Uu. uF  q»^ ,          V^,          3NK  	  ppV# u upi )Nc                 óD   <€ SV ^ ,          ,          SV ^,          ,          3# ©r   © )ÚxÚvocabs   &€r"   Ú<lambda>Ú!generate_merges.<locals>.<lambda>Š   s   ø€ ¨U°1°Qµ4­[¸%ÀÀ!Å½+Ñ,Fr$   ©Úkeyc                 óZ   € V ^,          \        V ^ ,          4      \        V ^,          4      3# ©r&   )Úlen©Úvals   &r"   r=   r>   �   s   € ¨S°­V´S¸¸Q½³[Ä#ÀcÈ!ÅfÃ+Ñ,Nr$   ©r@   Úreverse)ÚsetÚdictÚitemsÚrangerC   ÚappendÚsortedÚextend)r<   Úvocab_scoresr6   rG   ÚmergesÚmergeÚpiece_scoreÚlocalÚindexÚpiece_lÚpiece_rrE   s   f&&         r"   Úgenerate_mergesrW   z   s  ø€ Ø&1Ò&=”#�kÔ"Ä3Ã5€KØ $Ð&€Gß)0”4˜Ô%°e€Là€FØ*×0Ñ0Ö2ÑˆØÔÙØˆÜ˜1œc %›jÖ)ˆEØ$ V e˜}¨e°F¨m�WØÔ%¨Ô)?ÙØ˜%Ö G¨uÖ$4Ø—‘˜g°Ð<Ö=ñ *ô �uÔ"FÔGˆØ�‰�eÖñ 3ô �FÑ NÐX_Ô`€FÙ*0Ó1©& 3�1�v�s˜1•vÓ©&€FÐ1Ø€Mùò 2s   Ã%Dc                   óH   a € ] tR t^’t o RtV 3R lR ltV 3R lR ltRtV tR# )ÚSentencePieceExtractorzd
Extractor implementation for SentencePiece trained models. https://github.com/google/sentencepiece
c                ó    <€ V ^8„  d   QhRS[ /# )r&   Úmodel©r*   )r   Ú__classdict__s   "€r"   r+   Ú#SentencePieceExtractor.__annotate__—   s   ø€ ÷ 
ñ 
™cñ 
r$   c                ó  € \        V R 4       \        V R4       \        4       pVP                  4       p\        VR4      ;_uu_ 4       pVP	                  VP                  4       4       RRR4       W0n        R#   + '       g   i     L; i)r   r   ÚrbN)r   r#   Ú
ModelProtoÚopenÚParseFromStringÚreadÚproto)Úselfr[   Ú	model_pb2ÚmÚfs   &&   r"   Ú__init__ÚSentencePieceExtractor.__init__—   sd   € Ü˜$ Ô0Ü˜$ 
Ô+ô $Ó%ˆ	à× Ñ Ó"ˆÜ�%˜×Ô !Ø×Ñ˜aŸf™f›hÔ'÷ àŽ
÷ ×ús   Á A7Á7B	c                ó\   <€ V ^8„  d   QhRS[ S[S[S[3,          S[S[ ,          3,          /# ©r&   r(   ©ÚtuplerI   r*   ÚintÚlist)r   r]   s   "€r"   r+   r^   £   s*   ø€ ÷ !ñ !©u±T¹#¹s¸(µ^ÁTÉ%Å[Ð5PÕ/Qñ !r$   c                ó¦  € V P                   P                  P                   Vf2   ^ RIHpHp V P                   P                  P                  ^8X  d   TMTpV P                   P                   Uu. uF  qUP                  VP                  3NK  	  ppVP                  R8w  d)   V P                   P                  P                  VR&   WbR&   M9^RIHp \        V4       UU	U
u/ uF
  w  pw  ršW˜bK  	  pp	pp
V! V4      pWbR&   W²R&   \        V P                   P                  4       UUu. uF3  w  rÍVP                  R9   g   K  WÍP                  VP                  ^8H  3NK5  	  ppp\        VR R	7       UUUu. uF  w  rÏp\!        VR
VR7      NK  	  upppVR&   \#        V P                   P$                  RR4      VR&   V# u upi u up
p	pi u uppi u upppi )úª
By default will return vocab and merges with respect to their order, by sending `vocab_scores` we're going to
order the merges with respect to the piece scores instead.
N)r   r   r   Úunk_idr<   )rW   rP   c                 ó   € V ^ ,          # r9   r:   ©r;   s   &r"   r=   Ú0SentencePieceExtractor.extract.<locals>.<lambda>Á   s   € ÈQÈqÎTr$   r?   F©Ú
normalizedÚspecialÚadditional_special_tokensÚprecompiled_charsmapÚ_spm_precompiled_charsmap©é   é   )re   Útrainer_specrt   Útokenizers.modelsr   r   Ú
model_typeÚpiecesÚpieceÚscoreÚ__name__Útokenization_utils_baserW   Ú	enumerateÚtyperM   r   r1   Únormalizer_spec)rf   rƒ   Úkwargsr   r   r…   r<   rW   ÚiÚwordr†   rP   ÚidÚpÚspm_added_tokensÚtokenrz   s   &&,              r"   ÚextractÚSentencePieceExtractor.extract£   s¡  € ð
 	�
‰
×Ñ×&Ò&ØÒß6à$(§J¡J×$;Ñ$;×$FÑ$FÈ!Ô$K™ÐQTˆJØ9=¿¹×9JÒ9JÓKÑ9J°—+‘+˜uŸ{™{Ó+Ñ9JˆÐKà×Ñ %Ô'Ø#Ÿz™z×6Ñ6×=Ñ=ˆF�8ÑØ#�7ŠOå@ä5>¸uÔ5EÕFÑ5EÑ!1 ¡M T�T’WÑ5EˆEÒFÙ$ UÓ+ˆFØ#�7‰OØ%�8Ñô ENÈdÏjÉj×N_ÑN_ÔD`ÔuÑD`¹5¸2Ðde×djÑdjÐntÑdtÔ6˜R§¡¨!¯&©&°A©+Ó6ÑD`ÐÑuô '-Ð-=Á>Õ&Rõ/
á&RÑ"�˜7ô �u¨¸×@Ù&Ró/
ˆÐ*Ñ+ô /6°d·j±j×6PÑ6PÐRhÐjnÓ.oˆÐ*Ñ+Øˆùò/ Lùô Gùó vùô/
s   Á. F:ÃF?Ä$GÄ>GÅ2G©re   N)	r‡   Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__rj   r“   Ú__static_attributes__Ú__classdictcell__©r]   s   @r"   rY   rY   ’   s   ø‡ € ñ÷
ð 
÷!ö !r$   rY   c                   ó6   a € ] tR t^Çt o RV 3R lR lltRtV tR# )ÚGemmaSentencePieceExtractorNc                ó\   <€ V ^8„  d   QhRS[ S[S[S[3,          S[S[ ,          3,          /# rm   rn   )r   r]   s   "€r"   r+   Ú(GemmaSentencePieceExtractor.__annotate__È   s*   ø€ ÷ ñ ©E±$±s¹C°xµ.Á$ÁuÅ+Ð2MÕ,Nñ r$   c                óä   € V P                   p\        VP                  4       4       Uu/ uF  q2P                  V4      VbK  	  ppRV9  d   VP	                  R4      VR&   \        WA4      pWE3# u upi )rs   Ú	ú<0x09>)ÚsprK   ÚGetPieceSizeÚid_to_pieceÚgetrW   )rf   rO   r¤   rT   r<   rP   s   &&    r"   r“   Ú#GemmaSentencePieceExtractor.extractÈ   so   € ð
 �W‰WˆÜ;@ÀÇÁÓARÔ;SÓTÑ;S°%—‘ Ó&¨Ò-Ñ;SˆÐTð �uÔØŸ)™) HÓ-ˆE�$‰KÜ  Ó5ˆØˆ}Ðùò Us   ¨A-r:   ©N)r‡   r–   r—   r˜   r“   rš   r›   rœ   s   @r"   rž   rž   Ç   s   ø‡ € ÷÷ ð r$   rž   c                ó0   € V ^8„  d   QhR\         R\        /# )r&   r…   r(   )r*   r)   )r   s   "r"   r+   r+   Ø   s   € ÷ Iñ Iœcð I¤dñ Ir$   c                 óŽ   € \        V 4      ^8  ;'       g1    V R,          R8g  ;'       g    V R,          P                  4       '       * # )r&   Ú,éÿÿÿÿéþÿÿÿ)rC   Úisdigit)r…   s   &r"   Úcheck_number_commar°   Ø   s9   € Üˆu‹:˜‰>×HÐH˜U 2�Y¨#Ñ-×HÐH°U¸2µY×5FÑ5FÓ5HÔ1HÐHr$   c                   ó8   a € ] tR t^Üt o R tV 3R lR ltRtV tR# )Ú	Converterc                ó   € Wn         R # r©   ©r2   )rf   r2   s   &&r"   rj   ÚConverter.__init__Ý   s   € Ø"4Ör$   c                ó    <€ V ^8„  d   QhRS[ /# rm   ©r   )r   r]   s   "€r"   r+   ÚConverter.__annotate__à   s   ø€ ÷ $ñ $™9ñ $r$   c                ó   € \        4       hr©   )ÚNotImplementedError©rf   s   &r"   Ú	convertedÚConverter.convertedà   s   € Ü!Ó#Ð#r$   r´   N)r‡   r–   r—   r˜   rj   r¼   rš   r›   rœ   s   @r"   r²   r²   Ü   s   ø‡ € ò5÷$ö $r$   r²   c                   ó2   a € ] tR t^ät o V 3R lR ltRtV tR# )ÚBertConverterc                ó    <€ V ^8„  d   QhRS[ /# rm   r·   )r   r]   s   "€r"   r+   ÚBertConverter.__annotate__å   ó   ø€ ÷ #ñ #™9ñ #r$   c           	     ó|  € V P                   P                  p\        \        V\	        V P                   P
                  4      R 7      4      pRpRpRp\        V P                   R4      '       da   V P                   P                  P                  pV P                   P                  P                  pV P                   P                  P                  p\        P                  ! RVVVR7      Vn        \        P                  ! 4       Vn        \	        V P                   P"                  4      p\	        V P                   P$                  4      pV P                   P&                  pV P                   P(                  p	\*        P,                  ! V RV R2V RV RV R2Wh3Wy3.R	7      Vn        \0        P                  ! R
R7      Vn        V# )©Ú	unk_tokenFÚbasic_tokenizerT©Ú
clean_textÚhandle_chinese_charsÚstrip_accentsÚ	lowercaseú:0 $A:0 ú:0ú:0 $B:1 ú:1©ÚsingleÚpairÚspecial_tokensú##©Úprefix©r2   r<   r   r   r*   rÅ   ÚhasattrrÆ   Útokenize_chinese_charsrÊ   Údo_lower_caser   ÚBertNormalizerÚ
normalizerr	   ÚBertPreTokenizerÚpre_tokenizerÚ	cls_tokenÚ	sep_tokenÚcls_token_idÚsep_token_idr
   ÚTemplateProcessingÚpost_processorr   Údecoder©
rf   r<   Ú	tokenizerrÙ   rÊ   rÚ   ÚclsÚseprá   râ   s
   &         r"   r¼   ÚBertConverter.convertedå   ó„  € Ø×'Ñ'×-Ñ-ˆÜœi¨¼¸T×=TÑ=T×=^Ñ=^Ó9_Ô`Óaˆ	à!&ÐØˆØˆÜ�4×*Ñ*Ð,=×>Ò>Ø%)×%<Ñ%<×%LÑ%L×%cÑ%cÐ"Ø ×3Ñ3×CÑC×QÑQˆMØ ×3Ñ3×CÑC×QÑQˆMä*×9Ò9ØØ!7Ø'Ø#ô	 
ˆ	Ôô #1×"AÒ"AÓ"Cˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆä#-×#@Ò#@Ø�U˜( 3 % rÐ*Ø�5˜   X¨c¨U°"Ð5àÐ#ØÐ#ðô$
ˆ	Ô ô %×.Ò.°dÔ;ˆ	ÔàÐr$   r:   N©r‡   r–   r—   r˜   r¼   rš   r›   rœ   s   @r"   r¿   r¿   ä   ó   ø‡ € ÷#ö #r$   r¿   c                   ó2   a € ] tR tRt o V 3R lR ltRtV tR# )ÚSplinterConverteri  c                ó    <€ V ^8„  d   QhRS[ /# rm   r·   )r   r]   s   "€r"   r+   ÚSplinterConverter.__annotate__  s   ø€ ÷ .ñ .™9ñ .r$   c           
     ó’  € V P                   P                  p\        \        V\	        V P                   P
                  4      R 7      4      pRpRpRp\        V P                   R4      '       da   V P                   P                  P                  pV P                   P                  P                  pV P                   P                  P                  p\        P                  ! RVVVR7      Vn        \        P                  ! 4       Vn        \	        V P                   P"                  4      p\	        V P                   P$                  4      p\	        V P                   P&                  4      pRp	V P                   P(                  p
V P                   P*                  pV P                   P,                  pV P                   P/                  R4      pV P                   P0                  R8X  d   V RV RV	 RV R	V R
2
pMV RV R	V RV	 RV R
2
p\2        P4                  ! V RV R2VWj3W{3WŒ3W�3.R7      Vn        \8        P                  ! RR7      Vn        V# )rÄ   FrÆ   TrÇ   Ú.ÚrightrÌ   Ú rÎ   rÏ   rÍ   rÐ   rÔ   rÕ   )r2   r<   r   r   r*   rÅ   rØ   rÆ   rÙ   rÊ   rÚ   r   rÛ   rÜ   r	   rÝ   rÞ   rß   rà   Úquestion_tokenrá   râ   Úquestion_token_idÚconvert_tokens_to_idsÚpadding_sider
   rã   rä   r   rå   )rf   r<   rç   rÙ   rÊ   rÚ   rè   ré   ÚquestionÚdotrá   râ   r÷   Údot_token_idrÒ   s   &              r"   r¼   ÚSplinterConverter.converted  s  € Ø×'Ñ'×-Ñ-ˆÜœi¨¼¸T×=TÑ=T×=^Ñ=^Ó9_Ô`Óaˆ	à!&ÐØˆØˆÜ�4×*Ñ*Ð,=×>Ò>Ø%)×%<Ñ%<×%LÑ%L×%cÑ%cÐ"Ø ×3Ñ3×CÑC×QÑQˆMØ ×3Ñ3×CÑC×QÑQˆMä*×9Ò9ØØ!7Ø'Ø#ô	 
ˆ	Ôô #1×"AÒ"AÓ"Cˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆÜ�t×.Ñ.×=Ñ=Ó>ˆØˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆØ ×3Ñ3×EÑEÐØ×.Ñ.×DÑDÀSÓIˆà×"Ñ"×/Ñ/°7Ô:Ø�U˜( 8 *¨A¨c¨U°!°C°5¸ÀÀÀRÐH‰Dà�U˜( 3 % x°¨z¸¸3¸%¸qÀÀÀRÐHˆDä#-×#@Ò#@Ø�U˜( 3 % rÐ*ØàÐ#ØÐ#ØÐ-ØÐ#ð	ô	$
ˆ	Ô ô %×.Ò.°dÔ;ˆ	ÔàÐr$   r:   Nrì   rœ   s   @r"   rï   rï     s   ø‡ € ÷.ö .r$   rï   c                   ó2   a € ] tR tRt o V 3R lR ltRtV tR# )ÚFunnelConverteri=  c                ó    <€ V ^8„  d   QhRS[ /# rm   r·   )r   r]   s   "€r"   r+   ÚFunnelConverter.__annotate__>  rÂ   r$   c           	     ó|  € V P                   P                  p\        \        V\	        V P                   P
                  4      R 7      4      pRpRpRp\        V P                   R4      '       da   V P                   P                  P                  pV P                   P                  P                  pV P                   P                  P                  p\        P                  ! RVVVR7      Vn        \        P                  ! 4       Vn        \	        V P                   P"                  4      p\	        V P                   P$                  4      pV P                   P&                  pV P                   P(                  p	\*        P,                  ! V RV R2V RV RV R2Wh3Wy3.R	7      Vn        \0        P                  ! R
R7      Vn        V# )rÄ   FrÆ   TrÇ   z:2 $A:0 rÍ   rÎ   rÏ   rÐ   rÔ   rÕ   r×   ræ   s
   &         r"   r¼   ÚFunnelConverter.converted>  rë   r$   r:   Nrì   rœ   s   @r"   rÿ   rÿ   =  rí   r$   rÿ   c                   ó2   a € ] tR tRt o V 3R lR ltRtV tR# )ÚMPNetConverterid  c                ó    <€ V ^8„  d   QhRS[ /# rm   r·   )r   r]   s   "€r"   r+   ÚMPNetConverter.__annotate__e  rÂ   r$   c                ó‚  € V P                   P                  p\        \        V\	        V P                   P
                  4      R 7      4      pRpRpRp\        V P                   R4      '       da   V P                   P                  P                  pV P                   P                  P                  pV P                   P                  P                  p\        P                  ! RVVVR7      Vn        \        P                  ! 4       Vn        \	        V P                   P"                  4      p\	        V P                   P$                  4      pV P                   P&                  pV P                   P(                  p	\*        P,                  ! V RV R2V RV RV RV R	2Wh3Wy3.R
7      Vn        \0        P                  ! RR7      Vn        V# )rÄ   FrÆ   TrÇ   rÌ   rÍ   z:0 rÎ   rÏ   rÐ   rÔ   rÕ   r×   ræ   s
   &         r"   r¼   ÚMPNetConverter.convertede  sŠ  € Ø×'Ñ'×-Ñ-ˆÜœi¨¼¸T×=TÑ=T×=^Ñ=^Ó9_Ô`Óaˆ	à!&ÐØˆØˆÜ�4×*Ñ*Ð,=×>Ò>Ø%)×%<Ñ%<×%LÑ%L×%cÑ%cÐ"Ø ×3Ñ3×CÑC×QÑQˆMØ ×3Ñ3×CÑC×QÑQˆMä*×9Ò9ØØ!7Ø'Ø#ô	 
ˆ	Ôô #1×"AÒ"AÓ"Cˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆä#-×#@Ò#@Ø�U˜( 3 % rÐ*Ø�5˜   S¨¨¨X°c°U¸"Ð=àÐ#ØÐ#ðô$
ˆ	Ô ô %×.Ò.°dÔ;ˆ	ÔàÐr$   r:   Nrì   rœ   s   @r"   r  r  d  rí   r$   r  c                   ó2   a € ] tR tRt o V 3R lR ltRtV tR# )ÚOpenAIGPTConverteri‹  c                ó    <€ V ^8„  d   QhRS[ /# rm   r·   )r   r]   s   "€r"   r+   ÚOpenAIGPTConverter.__annotate__Œ  s   ø€ ÷ ñ ™9ñ r$   c                ó  € V P                   P                  p\        V P                   P                  P	                  4       4      pV P                   P
                  p\        \        VVR \        V4      RRR7      4      pVP                  \        V4      4      e   VP                  \        V4      .4       \        P                  ! RR7      Vn        \        P                  ! 4       Vn        \"        P$                  ! RR7      Vn        V# )Nú</w>F)r<   rP   ÚdropoutrÅ   Úend_of_word_suffixÚfuse_unkT)rË   ©Úsuffix)r2   Úencoderrq   Ú	bpe_ranksÚkeysrÅ   r   r   r*   Útoken_to_idÚadd_special_tokensr   rÛ   rÜ   r	   rÝ   rÞ   r   Ú
BPEDecoderrå   ©rf   r<   rP   rÅ   rç   s   &    r"   r¼   ÚOpenAIGPTConverter.convertedŒ  sÔ   € Ø×'Ñ'×/Ñ/ˆÜ�d×-Ñ-×7Ñ7×<Ñ<Ó>Ó?ˆØ×+Ñ+×5Ñ5ˆ	äÜØØØÜ˜i›.Ø#)Øôó	
ˆ	ð × Ñ ¤ Y£Ó0Ò<Ø×(Ñ(¬#¨i«.Ð)9Ô:ä*×9Ò9ÀDÔIˆ	ÔÜ"0×"AÒ"AÓ"Cˆ	ÔÜ$×/Ò/°vÔ>ˆ	ÔàÐr$   r:   Nrì   rœ   s   @r"   r  r  ‹  s   ø‡ € ÷ö r$   r  c                   ó6   a € ] tR tRt o RV 3R lR lltRtV tR# )ÚGPT2Converteri¦  Nc                ó„   <€ V ^8„  d   QhRS[ S[S[3,          R,          RS[S[S[S[3,          ,          R,          RS[/# ©r&   r<   NrP   r(   ©rI   r*   rp   rq   ro   r   )r   r]   s   "€r"   r+   ÚGPT2Converter.__annotate__§  sF   ø€ ÷ "ñ "™t¡C© H�~°Õ4ð "ÁTÉ%ÑPSÑUXÐPXÍ/ÕEZÐ]aÕEað "Ñmvñ "r$   c                óx  € V'       g   V P                   P                  pV'       g    \        V P                   P                  4      p\	        \        VVR RRRR7      4      p\        V P                   RR4      p\        P                  ! VR7      Vn	        \        P                  ! 4       Vn        \        V P                   RR4      '       dU   V P                   P                  pV P                   P                  p\        P                  ! V R2V R2WV3.R	7      Vn        V# \        P                  ! RR
7      Vn        V# )NÚ F©r<   rP   r  Úcontinuing_subword_prefixr  r  r'   ©r'   Úadd_bos_tokenz:0 $A:0z:0 $A:0 $B:1rÐ   ©Útrim_offsets)r2   r  rq   r  r   r   r1   r	   Ú	ByteLevelrÞ   r   rå   Ú	bos_tokenÚbos_token_idr
   rã   rä   )rf   r<   rP   rç   r'   Úbosr-  s   &&&    r"   r¼   ÚGPT2Converter.converted§  s  € ßØ×+Ñ+×3Ñ3ˆEßÜ˜$×1Ñ1×;Ñ;Ó<ˆFäÜØØØØ*,Ø#%Øôó	
ˆ	ô # 4×#:Ñ#:Ð<NÐPUÓVÐÜ"0×":Ò":ÐL\Ô"]ˆ	ÔÜ$×.Ò.Ó0ˆ	ÔÜ�4×*Ñ*¨O¸U×CÒCØ×)Ñ)×3Ñ3ˆCØ×2Ñ2×?Ñ?ˆLÜ'1×'DÒ'DØ˜˜g�Ø�u˜LÐ)àÐ'ð ô(ˆIÔ$ð Ðô (2×';Ò';ÈÔ'OˆIÔ$ØÐr$   r:   ©NNrì   rœ   s   @r"   r  r  ¦  s   ø‡ € ÷"÷ "ð "r$   r  c                   ó2   a € ] tR tRt o V 3R lR ltRtV tR# )ÚHerbertConverteriÌ  c                ó    <€ V ^8„  d   QhRS[ /# rm   r·   )r   r]   s   "€r"   r+   ÚHerbertConverter.__annotate__Í  ó   ø€ ÷ ñ ™9ñ r$   c           
     ó®  € R pRpV P                   P                  p\        V P                   P                  P	                  4       4      pW^ ,          ^ ,          9   d
   VR,          p\        \        VVRV P                   P                  VR7      4      p\        P                  ! RRR7      Vn
        \        P                  ! 4       Vn        \        P                  ! VR7      Vn        \"        P$                  ! V P                   P&                  V P                   P(                  3V P                   P*                  V P                   P,                  34      Vn        V# )z	#version:r  ºé   NNN)r  rÅ   r  F)rË   rÊ   r  )r2   r  rq   r  r  r   r   rÅ   r   rÛ   rÜ   r	   rÝ   rÞ   r   r  rå   r
   ÚBertProcessingrà   râ   rß   rá   rä   )rf   Útokenizer_info_strÚtoken_suffixr<   rP   rç   s   &     r"   r¼   ÚHerbertConverter.convertedÍ  s  € Ø(ÐØˆà×'Ñ'×/Ñ/ˆÜ�d×-Ñ-×7Ñ7×<Ñ<Ó>Ó?ˆØ¨¥¨1¥Ô-Ø˜B•ZˆFäÜØØØØ×1Ñ1×;Ñ;Ø#/ôó
ˆ	ô  +×9Ò9ÀEÐY^Ô_ˆ	ÔÜ"0×"AÒ"AÓ"Cˆ	ÔÜ$×/Ò/°|ÔDˆ	ÔÜ#-×#<Ò#<Ø×$Ñ$×.Ñ.°×0GÑ0G×0TÑ0TÐUØ×$Ñ$×.Ñ.°×0GÑ0G×0TÑ0TÐUó$
ˆ	Ô ð
 Ðr$   r:   Nrì   rœ   s   @r"   r2  r2  Ì  ó   ø‡ € ÷ö r$   r2  c                   ó6   a € ] tR tRt o RV 3R lR lltRtV tR# )ÚQwen2Converterië  Nc                ó„   <€ V ^8„  d   QhRS[ S[S[3,          R,          RS[S[S[S[3,          ,          R,          RS[/# r   r!  )r   r]   s   "€r"   r+   ÚQwen2Converter.__annotate__ì  sF   ø€ ÷ (ñ (™t¡C© H�~°Õ4ð (ÁTÉ%ÑPSÑUXÐPXÍ/ÕEZÐ]aÕEað (Ñmvñ (r$   c                óT  € V'       g   V P                   P                  pV'       g.   \        V P                   P                  P	                  4       4      p\        \        VVR R RRRRR7      4      p\        P                  ! 4       Vn	        \        P                  ! \        P                  ! \        R4      RRR7      \        P                  ! \        V P                   RR4      RR7      .4      Vn        \"        P                  ! 4       Vn        \&        P                  ! RR	7      Vn        V# )
Nr$  F)r<   rP   r  rÅ   r&  r  r  Úbyte_fallbackzn(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+Úisolated©ÚbehaviorÚinvertr'   ©r'   Ú	use_regexr)  )r2   r  rq   r  r  r   r   r   ÚNFCrÜ   r	   ÚSequenceÚSplitr   r+  r1   rÞ   r   rå   r
   rä   )rf   r<   rP   rç   s   &&& r"   r¼   ÚQwen2Converter.convertedì  s  € ßØ×+Ñ+×3Ñ3ˆEßÜ˜$×1Ñ1×;Ñ;×@Ñ@ÓBÓCˆFäÜØØØØØ*,Ø#%ØØ#ô	ó
ˆ	ô  +ŸšÓ0ˆ	Ôä"0×"9Ò"9ä×$Ò$Üð Nóð (Ø ôô ×(Ò(Ü%,¨T×-DÑ-DÐFXÐZ_Ó%`Ø#ôðó#
ˆ	Ôô  %×.Ò.Ó0ˆ	ÔÜ#-×#7Ò#7ÀUÔ#Kˆ	Ô àÐr$   r:   r0  rì   rœ   s   @r"   r?  r?  ë  s   ø‡ € ÷(÷ (ð (r$   r?  c                   ó2   a € ] tR tRt o V 3R lR ltRtV tR# )ÚRobertaConverteri  c                ó    <€ V ^8„  d   QhRS[ /# rm   r·   )r   r]   s   "€r"   r+   ÚRobertaConverter.__annotate__  ó   ø€ ÷ ñ ™9ñ r$   c                óØ  € V P                   pVP                  p\        VP                  P	                  4       4      p\        \        VVR RRRR7      4      p\        P                  ! VP                  R7      Vn
        \        P                  ! 4       Vn        \        P                  ! VP                  VP                   3VP"                  VP$                  3RVP                  4      Vn        V# )Nr$  Fr%  r'  T)r2   r  rq   r  r  r   r   r	   r+  r'   rÞ   r   rå   r
   ÚRobertaProcessingrà   râ   rß   rá   rä   ©rf   Úotr<   rP   rç   s   &    r"   r¼   ÚRobertaConverter.converted  sÄ   € Ø×$Ñ$ˆØ—
‘
ˆÜ�b—l‘l×'Ñ'Ó)Ó*ˆäÜØØØØ*,Ø#%Øôó	
ˆ	ô #1×":Ò":ÈB×L_ÑL_Ô"`ˆ	ÔÜ$×.Ò.Ó0ˆ	ÔÜ#-×#?Ò#?Ø�\‰\˜2Ÿ?™?Ð+Ø�\‰\˜2Ÿ?™?Ð+ØØ×Ñó	$
ˆ	Ô ð Ðr$   r:   Nrì   rœ   s   @r"   rO  rO    ó   ø‡ € ÷ö r$   rO  c                   ó2   a € ] tR tRt o V 3R lR ltRtV tR# )ÚRoFormerConverteri4  c                ó    <€ V ^8„  d   QhRS[ /# rm   r·   )r   r]   s   "€r"   r+   ÚRoFormerConverter.__annotate__5  rÂ   r$   c           	     ód  € ^RI Hp V P                  P                  p\	        \        V\        V P                  P                  4      R7      4      pRpRp\        V P                  R4      '       dA   V P                  P                  P                  pV P                  P                  P                  p\        P                  ! RRVVR7      Vn        \        P                   P#                  V! V4      4      Vn        \        V P                  P&                  4      p\        V P                  P(                  4      pV P                  P*                  pV P                  P,                  p	\.        P0                  ! V RV R2V RV R	V R
2Wh3Wy3.R7      Vn        \4        P
                  ! RR7      Vn        V# )r8  )ÚJiebaPreTokenizerrÄ   FrÆ   TrÇ   rÌ   rÍ   rÎ   rÏ   rÐ   rÔ   rÕ   )Ú"models.roformer.tokenization_utilsr^  r2   r<   r   r   r*   rÅ   rØ   rÆ   rÊ   rÚ   r   rÛ   rÜ   r	   ÚPreTokenizerÚcustomrÞ   rß   rà   rá   râ   r
   rã   rä   r   rå   )
rf   r^  r<   rç   rÊ   rÚ   rè   ré   rá   râ   s
   &         r"   r¼   ÚRoFormerConverter.converted5  sx  € ÝIà×'Ñ'×-Ñ-ˆÜœi¨¼¸T×=TÑ=T×=^Ñ=^Ó9_Ô`Óaˆ	àˆØˆÜ�4×*Ñ*Ð,=×>Ò>Ø ×3Ñ3×CÑC×QÑQˆMØ ×3Ñ3×CÑC×QÑQˆMä*×9Ò9ØØ!&Ø'Ø#ô	 
ˆ	Ôô #1×"=Ñ"=×"DÑ"DÑEVÐW\ÓE]Ó"^ˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆä#-×#@Ò#@Ø�U˜( 3 % rÐ*Ø�5˜   X¨c¨U°"Ð5àÐ#ØÐ#ðô$
ˆ	Ô ô %×.Ò.°dÔ;ˆ	ÔàÐr$   r:   Nrì   rœ   s   @r"   rZ  rZ  4  rí   r$   rZ  c                   ó2   a € ] tR tRt o V 3R lR ltRtV tR# )ÚDebertaConverteri[  c                ó    <€ V ^8„  d   QhRS[ /# rm   r·   )r   r]   s   "€r"   r+   ÚDebertaConverter.__annotate__\  r5  r$   c                óÜ  € V P                   pVP                  p\        VP                  P	                  4       4      p\        \        VVR RRRR7      4      p\        P                  ! VP                  R7      Vn
        \        P                  ! 4       Vn        \        P                  ! RRRV P                   P                  R4      3RV P                   P                  R4      3.R	7      Vn        V# )
Nr$  Fr%  r'  ú[CLS]:0 $A:0 [SEP]:0ú![CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1ú[CLS]ú[SEP]rÐ   )r2   r  rq   r  r  r   r   r	   r+  r'   rÞ   r   rå   r
   rã   rø   rä   rU  s   &    r"   r¼   ÚDebertaConverter.converted\  sÕ   € Ø×$Ñ$ˆØ—
‘
ˆÜ�b—l‘l×'Ñ'Ó)Ó*ˆäÜØØØØ*,Ø#%Øôó	
ˆ	ô #1×":Ò":ÈB×L_ÑL_Ô"`ˆ	ÔÜ$×.Ò.Ó0ˆ	ÔÜ#-×#@Ò#@Ø)Ø4à˜$×1Ñ1×GÑGÈÓPÐQØ˜$×1Ñ1×GÑGÈÓPÐQðô$
ˆ	Ô ð Ðr$   r:   Nrì   rœ   s   @r"   rd  rd  [  r=  r$   rd  c                   ó¢   a a€ ] tR tRt oRt]t/ t]RR l4       t	]
RR l4       tV 3R ltR tR tR tR	 tR
 tR tR tV3R lR ltRtVtV ;t# )ÚSpmConverteriz  Fc                óÒ  € V P                   P                  pV P                   P                  pV P                  P                  p\        V\        4      '       d&   \        \        TT;'       g    . VRVRR7      4      pMt\        V\        4      '       d]   V'       dU   \        V^ ,          \        \        ,          4      '       d-   \        \        VV P                   P                  VR7      4      pMR# \        P                  ! RR4      .pV'       d'   VP                  ^ \        P                   ! V4      4       \        P"                  ! V4      Vn        V'       d\   \&        P"                  ! \&        P                  ! RR4      \&        P(                  ! 4       \&        P*                  ! 4       .4      Vn        V# \&        P"                  ! \&        P                  ! RR4      .4      Vn        V# )zç
Similar to convert_from_spm method, but used only when there is no `model_type` class, i.e. there is no matching class in `TOKENIZERS_MAPPING` and we just create a tokenizer instead of extracting stuff from the sentencepiece file
TN)r<   rP   rÅ   r  rC  r  )r<   rt   rC  rõ   õ   â–�)r�   rC  Ú	unk_piecer‹   r|   Ú
isinstancerI   r   r   rq   ro   r   rt   r   ÚReplaceÚinsertÚPrecompiledrK  rÜ   r   ÚByteFallbackÚFuserå   )re   r<   rP   rC  rq  r|   rç   Ú_normalizerss   &&&     r"   Úbuild_tokenizer_from_spm_protoÚ+SpmConverter.build_tokenizer_from_spm_proto  s}  € ð
 ×*Ñ*×8Ñ8ˆØ×&Ñ&×0Ñ0ˆ	Ø$×4Ñ4×IÑIÐô �eœT×"Ò"Ü!ÜØØ!Ÿ<˜< RØ'Ø!Ø"/Ø ôó	‰Iô ˜œt×$Ò$¯´:¸eÀA½hÌÔPTÍ×3UÒ3UÜ!ÜØØ ×-Ñ-×4Ñ4Ø"/ôó‰Iñ ô $×+Ò+¨C°Ó7Ð8ˆßØ×Ñ ¤;×#:Ò#:Ð;OÓ#PÔQÜ*×3Ò3°LÓAˆ	Ô÷ Ü (× 1Ò 1Ü×!Ò! %¨Ó-¬x×/DÒ/DÓ/FÌÏÊËÐXó!ˆIÔð Ðô !)× 1Ò 1´8×3CÒ3CÀEÈ3Ó3OÐ2PÓ QˆIÔàÐr$   c                ó   € Ve   WR&   V# )z‡
Hook used when converting directly from a SentencePiece model without a slow tokenizer instance.
By default, return kwargs unchanged.
r<   r:   )rè   r<   rŒ   s   &&,r"   Úconvert_from_spmÚSpmConverter.convert_from_spm¯  s   € ð ÒØ#�7‰OØˆr$   c                óæ  <€ \        V R 4       \        SV `  ! V!   \        4       pVP	                  4       p\        V P                  P                  R4      ;_uu_ 4       pVP                  VP                  4       4       RRR4       W0n
        V P                  P                  P                  '       d-   V P                  '       g   \        P                  ! R4       R# R# R#   + '       g   i     Lj; i)r   r`   Na  The sentencepiece tokenizer that you are converting to a fast tokenizer uses the byte fallback option which is not implemented in the fast tokenizers. In practice this means that the fast version of the tokenizer can produce unknown tokens whereas the sentencepiece version would have converted these unknown tokens into a sequence of byte tokens matching the original piece of text.)r   Úsuperrj   r#   ra   rb   r2   Ú
vocab_filerc   rd   re   r�   rC  Úhandle_byte_fallbackÚwarningsÚwarn)rf   Úargsrg   rh   ri   Ú	__class__s   &*   €r"   rj   ÚSpmConverter.__init__¹  s°   ø€ Ü˜$ 
Ô+ä‰Ò˜$Òô $Ó%ˆ	à× Ñ Ó"ˆÜ�$×)Ñ)×4Ñ4°d×;Ô;¸qØ×Ñ˜aŸf™f›hÔ'÷ <àŒ
à�:‰:×"Ñ"×0×0Ð0¸×9R×9RÐ9RÜ�MŠMðeöñ :SÑ0÷	 <×;ús   Á C Ã C0	c                ón   € VP                    Uu. uF  q"P                  VP                  3NK  	  up# u upi r©   ©r„   r…   r†   ©rf   re   r…   s   && r"   r<   ÚSpmConverter.vocabÎ  s)   € Ø8=¿ºÓE¹¨u—‘˜eŸk™kÓ*¹ÑEÐEùÒEs   � 2c                ó.   € VP                   P                  # r©   )r�   rt   ©rf   re   s   &&r"   rt   ÚSpmConverter.unk_idÑ  s   € Ø×!Ñ!×(Ñ(Ð(r$   c                óž  € VP                   P                  pV P                  V4      pV^8X  d2   \        \	        VV P                  V4      V P                  R7      4      pM¤V^8X  d“   V P                  V P                  P                  4      P                  V4      w  rV\        V4       UUU	u/ uF
  w  pw  r‰W‡bK  	  p
ppp	\        \        V
VVP                   P                  RV P                  RR7      4      pM\        R4      h\        VP                  4       UUu. uFT  w  r¼VP                   R
9   g   K  Y¼P"                  VP                   ^8H  ;'       g    VP"                  V P$                  9   3NKV  	  pppTP'                  \)        VR R7       UUUu. uF  w  r¾p\+        VRVR	7      NK  	  uppp4       V# u up	ppi u uppi u upppi )r8  ©rt   rC  TN©rÅ   r  rC  r  z]You're trying to run a `Unigram` model but you're file was trained with a different algorithmc                 ó   € V ^ ,          # r9   r:   rv   s   &r"   r=   Ú(SpmConverter.tokenizer.<locals>.<lambda>   ó   € ÐQRÐSTÖQUr$   r?   Frx   r~   )r�   rƒ   r<   r   r   rt   r�  ÚSpmExtractorr2   r€  r“   r‰   r   rq  Ú	Exceptionr„   rŠ   r…   rÓ   Ú
add_tokensrM   r   )rf   re   rƒ   rO   rç   Ú_rP   r�   rŽ   r†   Ú	bpe_vocabr�   r�   r‘   r’   rz   s   &&              r"   rç   ÚSpmConverter.tokenizerÔ  s±  € Ø×'Ñ'×2Ñ2ˆ
Ø—z‘z %Ó(ˆà˜Œ?Ü!ÜØ ØŸ;™; uÓ-Ø"&×";Ñ";ôó‰Ið ˜1Œ_Ø×)Ñ)¨$×*AÑ*A×*LÑ*LÓM×UÑUÐVbÓc‰IˆAÜ9BÀ<Ô9PÕQÑ9PÑ%5 Q©¨˜šÑ9PˆIÒQÜ!ÜØØØ#×0Ñ0×:Ñ:Ø!Ø"&×";Ñ";Ø ôó	‰Iô Øoóð ô # 5§<¡<Ô0ô
á0‘�Ø�v‰v˜Ñô IˆR—‘˜!Ÿ&™& A™+×GÐG¨¯©°D×4GÑ4GÑ)GÓHÙ0ð 	ñ 
ð
 	×Ñô +1Ð1AÁ~Õ*Võá*VÑ&�B˜wô ˜5¨U¸G×DÙ*Vóô	
ð ÐùôC Rùó*
ùôs   Â,F;ÄGÄ6 GÅGÆGc                ó>  € VP                   P                  p\        P                  ! R RR7      \        P                  ! \        R4      R4      .pV'       g   \        P                  ! V4      # \        P                  ! \        P                  ! V4      .V,           4      # )FT)Úleftrô   ú {2,}rp  )r‹   r|   r   ÚStriprs  r   rK  ru  ©rf   re   r|   rx  s   &&  r"   rÜ   ÚSpmConverter.normalizer  s{   € Ø$×4Ñ4×IÑIÐä×Ò 5°Ô5Ü×Ò¤ g£°Ó6ð
ˆ÷ $Ü×'Ò'¨Ó5Ð5ä×'Ò'¬×)@Ò)@ÐAUÓ)VÐ(WÐZfÕ(fÓgÐgr$   c                óZ   € \        W P                  4      p\        P                  ! WR 7      # ©©Úreplacementr3   )r4   r2   r	   Ú	Metaspace©rf   r£  r'   r3   s   &&& r"   rÞ   ÚSpmConverter.pre_tokenizer  s$   € Ü,Ð-=×?VÑ?VÓWˆÜ×'Ò'°KÔ_Ð_r$   c                ó   € R # r©   r:   r»   s   &r"   rä   ÚSpmConverter.post_processor  ó   € Ùr$   c                óZ   € \        W P                  4      p\        P                  ! WR 7      # r¡  )r4   r2   r   r¤  r¥  s   &&& r"   rå   ÚSpmConverter.decoder  s$   € Ü,Ð-=×?VÑ?VÓWˆÜ×!Ò!¨kÔYÐYr$   c                ó    <€ V ^8„  d   QhRS[ /# rm   r·   )r   r]   s   "€r"   r+   ÚSpmConverter.__annotate__  s   ø€ ÷ ñ ™9ñ r$   c                ó�  € V P                  V P                  4      pV P                  V P                  4      pVe   W!n        RpRp\        V P                  R4      '       d   V P                  P
                  pV P                  W44      pVe   WQn        V P                  W44      Vn        V P                  4       pV'       d   Wan        V# )Nrp  Tr'   )	rç   re   rÜ   rØ   r2   r'   rÞ   rå   rä   )rf   rç   rÜ   r£  r'   rÞ   rä   s   &      r"   r¼   ÚSpmConverter.converted  s¯   € Ø—N‘N 4§:¡:Ó.ˆ	ð —_‘_ T§Z¡ZÓ0ˆ
ØÒ!Ø#-Ô àˆØÐÜ�4×*Ñ*Ð,>×?Ò?Ø#×6Ñ6×GÑGÐà×*Ñ*¨;ÓIˆØÒ$Ø&3Ô#à ŸL™L¨ÓGˆ	ÔØ×,Ñ,Ó.ˆßØ'5Ô$àÐr$   r•   r©   )r‡   r–   r—   r˜   r�  rY   r”  rÓ   Ústaticmethodry  Úclassmethodr|  rj   r<   rt   rç   rÜ   rÞ   rä   rå   r¼   rš   r›   Ú__classcell__©r…  r]   s   @@r"   rn  rn  z  sx   ù‡ € Ø ÐØ)€LØ€Nàó-ó ð-ð^ óó ðõò*Fò)ò0òd	hò`òòZ÷÷ ð r$   rn  c                   ó2   a € ] tR tRt o R tR tR tRtV tR# )ÚAlbertConverteri5  c                óä   € VP                    Uu. uFU  p\        VP                  4      '       d   VP                  VP                  3MVP                  VP                  ^d,
          3NKW  	  up# u upi ©éd   ©r„   r°   r…   r†   r‰  s   && r"   r<   ÚAlbertConverter.vocab6  óf   € ð Ÿšó
á%�ô +=¸U¿[¹[×*IÒ*IˆU�[‰[˜%Ÿ+™+Ñ&ÐPU×P[ÑP[Ð]b×]hÑ]hÐknÕ]nÐOoÒoÙ%ñ
ð 	
ùò 
ó   �AA-c                ó¶  € \         P                  ! R R4      \         P                  ! RR4      .pV P                  P                  '       gI   VP	                  \         P
                  ! 4       4       VP	                  \         P                  ! 4       4       V P                  P                  '       d%   VP	                  \         P                  ! 4       4       VP                  P                  pV'       d&   VP	                  \         P                  ! V4      4       VP	                  \         P                  ! \        R4      R4      4       \         P                  ! V4      # ©z``Ú"z''rœ  rõ   ©r   rs  r2   Úkeep_accentsrL   ÚNFKDÚStripAccentsrÚ   Ú	Lowercaser‹   r|   ru  r   rK  ©rf   re   Úlist_normalizersr|   s   &&  r"   rÜ   ÚAlbertConverter.normalizer<  óõ   € ä×Ò  cÓ*Ü×Ò  cÓ*ð
Ðð ×&Ñ&×3×3Ð3Ø×#Ñ#¤K×$4Ò$4Ó$6Ô7Ø×#Ñ#¤K×$<Ò$<Ó$>Ô?Ø×"Ñ"×0×0Ð0Ø×#Ñ#¤K×$9Ò$9Ó$;Ô<à$×4Ñ4×IÑIÐçØ×#Ñ#¤K×$;Ò$;Ð<PÓ$QÔRà×Ñ¤× 3Ò 3´E¸'³NÀCÓ HÔIÜ×#Ò#Ð$4Ó5Ð5r$   c           	     ó¤   € \         P                  ! R RRV P                  P                  R4      3RV P                  P                  R4      3.R7      # ©rh  ri  rj  rk  rÐ   ©r
   rã   r2   rø   r»   s   &r"   rä   ÚAlbertConverter.post_processorO  óR   € Ü×,Ò,Ø)Ø4à˜$×1Ñ1×GÑGÈÓPÐQØ˜$×1Ñ1×GÑGÈÓPÐQðô
ð 	
r$   r:   N©	r‡   r–   r—   r˜   r<   rÜ   rä   rš   r›   rœ   s   @r"   rµ  rµ  5  ó   ø‡ € ò
ò6÷&
ð 
r$   rµ  c                   ó,   a € ] tR tRt o R tR tRtV tR# )ÚBarthezConverteriZ  c                ó
   € ^pV# ©r   r:   ©rf   re   rt   s   && r"   rt   ÚBarthezConverter.unk_id[  ó   € ØˆØˆr$   c           	     ó¤   € \         P                  ! R RRV P                  P                  R4      3RV P                  P                  R4      3.R7      # ©z<s> $A </s>z<s> $A </s> </s> $B </s>ú<s>ú</s>rÐ   rË  r»   s   &r"   rä   ÚBarthezConverter.post_processor_  óR   € Ü×,Ò,Ø Ø+à˜×/Ñ/×EÑEÀeÓLÐMØ˜×0Ñ0×FÑFÀvÓNÐOðô
ð 	
r$   r:   N)r‡   r–   r—   r˜   rt   rä   rš   r›   rœ   s   @r"   rÑ  rÑ  Z  s   ø‡ € ò÷
ð 
r$   rÑ  c                   óF   a € ] tR tRt o R tR tR t]RR l4       tRt	V t
R# )	ÚCamembertConverterij  c                ó¨   € . ROpY!P                   R,           Uu. uF  q3P                  VP                  3NK  	  up,          pVR.,          pV# u upi )ú
<s>NOTUSEDr7  )©rà  ç        ©ú<pad>râ  ©z</s>NOTUSEDrâ  ©ú<unk>râ  )ú<unk>NOTUSEDiœÿÿÿ©ú<mask>râ  rˆ  ©rf   re   r<   r…   s   &&  r"   r<   ÚCamembertConverter.vocabk  sP   € ò
ˆð 	¿,¹,ÀrÖ:JÓKÑ:J°—;‘; §¡Ó,Ñ:JÑKÕKˆØ�/Ð"Õ"ˆØˆùò Ló   š Ac                ó   € ^# rÓ  r:   rŒ  s   &&r"   rt   ÚCamembertConverter.unk_idx  s   € ár$   c           	     ó¤   € \         P                  ! R RRV P                  P                  R4      3RV P                  P                  R4      3.R7      # rØ  rË  r»   s   &r"   rä   Ú!CamembertConverter.post_processor|  rÜ  r$   Nc                ó6  € \        VP                  R R4      4      p\        VP                  RR4      4      p\        VP                  RR4      4      pR	VR3R
VR3R.pVe"   VP                  \        V4      R,          4       VP	                  VR34       WbR&   V# )Ú	pad_tokenrä  rÅ   rç  Ú
mask_tokenrê  râ  r7  r<   rá  rå  )rè  ç      YÀ©r*   r§   rN   rq   rL   )rè   r<   rŒ   ró  rÅ   rô  Ú
vocab_lists   &&,    r"   r|  Ú#CamembertConverter.convert_from_spm†  s    € ä˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ™ L°(Ó;Ó<ˆ
ð  Ø˜ÐØ Ø˜ÐØ$ð
ˆ
ð ÒØ×Ñœd 5›k¨"�oÔ.Ø×Ñ˜: sÐ+Ô,Ø$ˆw‰Øˆr$   r:   r©   ©r‡   r–   r—   r˜   r<   rt   rä   r±  r|  rš   r›   rœ   s   @r"   rÞ  rÞ  j  s(   ø‡ € òòò
ð óó ör$   rÞ  c                   ó2   a € ] tR tRt o R tR tR tRtV tR# )ÚDebertaV2Converteriš  c                ó,  € . pV P                   P                  '       d'   VP                  \        P                  ! R R7      4       \        W P                   4      pVP                  \        P                  ! WR7      4       \        P                  ! V4      # )rD  )rF  r¢  )r2   Úsplit_by_punctrL   r	   ÚPunctuationr4   r¤  rK  )rf   r£  r'   Úlist_pretokenizersr3   s   &&&  r"   rÞ   Ú DebertaV2Converter.pre_tokenizer›  so   € ØÐØ×"Ñ"×1×1Ð1Ø×%Ñ%¤n×&@Ò&@È*Ô&UÔVÜ,Ð-=×?VÑ?VÓWˆØ×!Ñ!¤.×":Ò":À{Ô"rÔsÜ×&Ò&Ð'9Ó:Ð:r$   c                óÞ  € . pV P                   P                  '       d%   VP                  \        P                  ! 4       4       VP                  \        P
                  ! 4       4       VP                  P                  pV'       d&   VP                  \        P                  ! V4      4       VP                  \        P                  ! \        R 4      R4      4       \        P                  ! V4      # )rœ  rõ   )r2   rÚ   rL   r   rÄ  r�  r‹   r|   ru  rs  r   rK  rÅ  s   &&  r"   rÜ   ÚDebertaV2Converter.normalizer£  s¨   € ØÐØ×"Ñ"×0×0Ð0Ø×#Ñ#¤K×$9Ò$9Ó$;Ô<Ø×Ñ¤× 1Ò 1Ó 3Ô4à$×4Ñ4×IÑIÐßØ×#Ñ#¤K×$;Ò$;Ð<PÓ$QÔRØ×Ñ¤× 3Ò 3´E¸'³NÀCÓ HÔIä×#Ò#Ð$4Ó5Ð5r$   c           	     ó¤   € \         P                  ! R RRV P                  P                  R4      3RV P                  P                  R4      3.R7      # rÊ  rË  r»   s   &r"   rä   Ú!DebertaV2Converter.post_processor°  rÍ  r$   r:   N)	r‡   r–   r—   r˜   rÞ   rÜ   rä   rš   r›   rœ   s   @r"   rû  rû  š  s   ø‡ € ò;ò6÷
ð 
r$   rû  c                   óF   a € ] tR tRt o R tR tR t]RR l4       tRt	V t
R# )	ÚMBartConverteri»  c                ó¾   € . ROpY!P                   R,           Uu. uF  q3P                  VP                  3NK  	  up,          pV. RO,          pVR.,          pV# u upi )rÙ  ºr   NN©©rÙ  râ  rã  ©rÚ  râ  ræ  )©Úar_ARrâ  ©Úcs_CZrâ  ©Úde_DErâ  ©Úen_XXrâ  ©Úes_XXrâ  ©Úet_EErâ  ©Úfi_FIrâ  ©Úfr_XXrâ  ©Úgu_INrâ  ©Úhi_INrâ  ©Úit_ITrâ  ©Úja_XXrâ  ©Úkk_KZrâ  ©Úko_KRrâ  ©Últ_LTrâ  ©Úlv_LVrâ  ©Úmy_MMrâ  ©Úne_NPrâ  ©Únl_XXrâ  ©Úro_ROrâ  ©Úru_RUrâ  ©Úsi_LKrâ  ©Útr_TRrâ  ©Úvi_VNrâ  ©Úzh_CNrâ  ré  rˆ  rë  s   &&  r"   r<   ÚMBartConverter.vocab¼  sa   € ò
ˆð 	¿,¹,ÀrÖ:JÓKÑ:J°—;‘; §¡Ó,Ñ:JÑKÕKˆØò 
õ 	
ˆð6 	�/Ð"Õ"ˆØˆùò; Ls   š Ac                ó   € ^# rÓ  r:   rŒ  s   &&r"   rt   ÚMBartConverter.unk_idâ  ó   € Ùr$   c           	     ó¤   € \         P                  ! R RRV P                  P                  R4      3RV P                  P                  R4      3.R7      # )z$A </s> en_XXz$A $B </s> en_XXr  rÚ  rÐ   rË  r»   s   &r"   rä   ÚMBartConverter.post_processorå  óR   € Ü×,Ò,Ø"Ø#à˜$×1Ñ1×GÑGÈÓPÐQØ˜×0Ñ0×FÑFÀvÓNÐOðô
ð 	
r$   Nc                óà  € \        VP                  R R4      4      p\        VP                  RR4      4      p\        VP                  RR4      4      p\        VP                  RR4      4      p\        VP                  RR	4      4      pVR
3VR
3VR
3VR
3.pVe"   VP                  \        V4      R,          4       VP                  R \         4       4       VP                  VR
34       W‚R&   V# )r,  rÙ  ró  rä  Ú	eos_tokenrÚ  rÅ   rç  rô  rê  râ  r  c              3   ó(   "  € T F  qR 3x € K
  	  R# 5i©râ  Nr:   ©Ú.0Ú	lang_codes   & r"   Ú	<genexpr>Ú2MBartConverter.convert_from_spm.<locals>.<genexpr>ÿ  s   é € ÐL¹O¨y cÕ*»Oùó   ‚r<   )r*   r§   rN   rq   ÚMBART_LANGUAGESrL   ©	rè   r<   rŒ   r,  ró  rF  rÅ   rô  r÷  s	   &&,      r"   r|  ÚMBartConverter.convert_from_spmï  så   € ä˜Ÿ
™
 ;°Ó6Ó7ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ
™
 ;°Ó7Ó8ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ™ L°(Ó;Ó<ˆ
ð ˜ÐØ˜ÐØ˜ÐØ˜Ðð	
ˆ
ð ÒØ×Ñœd 5›k¨"�oÔ.Ø×ÑÑL½OÓLÔLØ×Ñ˜: sÐ+Ô,Ø$ˆw‰Øˆr$   r:   r©   rù  rœ   s   @r"   r  r  »  s)   ø‡ € ò$òLò
ð óó ör$   r  c                   óF   a € ] tR tRt o R tR tR t]RR l4       tRt	V t
R# )	ÚMBart50Converteri  c                óŠ  € . ROpY!P                   R,           Uu. uF  q3P                  VP                  3NK  	  up,          pV. RNRNRNRNRNRNR	NR
NRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNR NR!NR"NR#NR$NR%NR&NR'NR(NR)NR*NR+NR,NR-NR.NR/NR0NR1NR2NR3NR4NR5NR6N,          pVR7.,          pV# u upi )8rÙ  r  r	  r  r  r  r  r  r  r  r  r  r  r   r"  r$  r&  r(  r*  r,  r.  r0  r2  r4  r6  r8  r:  r<  )Úaf_ZArâ  )Úaz_AZrâ  )Úbn_INrâ  )Úfa_IRrâ  )Úhe_ILrâ  )Úhr_HRrâ  )Úid_IDrâ  )Úka_GErâ  )Úkm_KHrâ  )Úmk_MKrâ  )Úml_INrâ  )Úmn_MNrâ  )Úmr_INrâ  )Úpl_PLrâ  )Úps_AFrâ  )Úpt_XXrâ  )Úsv_SErâ  )Úsw_KErâ  )Úta_INrâ  )Úte_INrâ  )Úth_THrâ  )Útl_XXrâ  )Úuk_UArâ  )Úur_PKrâ  )Úxh_ZArâ  )Úgl_ESrâ  )Úsl_SIrâ  ré  rˆ  rë  s   &&  r"   r<   ÚMBart50Converter.vocab  sé  € ò
ˆð 	¿,¹,ÀrÖ:JÓKÑ:J°—;‘; §¡Ó,Ñ:JÑKÕKˆØð  R�.ð  R .ð  R°.ð  RÀ.ð  RÐR`ð  RÐbpð  Rð  sAð  Rð  CQð  Rð  Sað  Rð  cqð  Rð  sAð  Rð  CQð  Rð  Sað  Rð  cqð  Rð  sAð  Rð  CQð  Rð  Sað  Rð  cqð  Rð  sAð  Rð  CQð  Rð  Sað  Rð  cqð  Rð  sAð  Rð  CQð  Rð  Sað  Rð  cqð  Rð  sAð  Rð  CQð  Rð  Sað  Rð  cqð  Rð  sAð  Rð  CQð  Rð  Sað  Rð  cqð  Rð  sA	ð  Rð  C	Q	ð  Rð  S	a	ð  Rð  c	q	ð  Rð  s	A
ð  Rð  C
Q
ð  Rð  S
a
ð  Rð  c
q
ð  Rð  s
Að  Rð  CQð  Rð  Sað  Rð  cqð  Rð  sAð  Rð  CQð  Rð  Sað  Rð  cqð  Rð  sAð  Rð  CQð  Rõ  	RˆØ�/Ð"Õ"ˆØˆùò Ls   š C c                ó   € ^# rÓ  r:   rŒ  s   &&r"   rt   ÚMBart50Converter.unk_id  rA  r$   c           	     ó¤   € \         P                  ! R RRV P                  P                  R4      3RV P                  P                  R4      3.R7      # )zen_XX $A </s>zen_XX $A $B </s>r  rÚ  rÐ   rË  r»   s   &r"   rä   ÚMBart50Converter.post_processor  rD  r$   Nc                óà  € \        VP                  R R4      4      p\        VP                  RR4      4      p\        VP                  RR4      4      p\        VP                  RR4      4      p\        VP                  RR	4      4      pVR
3VR
3VR
3VR
3.pVe"   VP                  \        V4      R,          4       VP                  R \         4       4       VP                  VR
34       W‚R&   V# )rß   rÙ  ró  rä  rF  rÚ  rÅ   rç  rô  rê  râ  r  c              3   ó(   "  € T F  qR 3x € K
  	  R# 5irH  r:   rI  s   & r"   rL  Ú4MBart50Converter.convert_from_spm.<locals>.<genexpr>/  s   é € ÐNÑ<M¨y cÕ*Ó<MùrN  r<   )r*   r§   rN   rq   ÚMBART50_LANGUAGESrL   )	rè   r<   rŒ   rß   ró  rF  rÅ   rô  r÷  s	   &&,      r"   r|  Ú!MBart50Converter.convert_from_spm  sæ   € ä˜Ÿ
™
 ;°Ó6Ó7ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ
™
 ;°Ó7Ó8ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ™ L°(Ó;Ó<ˆ
ð ˜ÐØ˜ÐØ˜ÐØ˜Ðð	
ˆ
ð ÒØ×Ñœd 5›k¨"�oÔ.Ø×ÑÑNÕ<MÓNÔNØ×Ñ˜: sÐ+Ô,Ø$ˆw‰Øˆr$   r:   r©   rù  rœ   s   @r"   rS  rS    s(   ø‡ € ò
òò
ð óó ör$   rS  c                   óF   a € ] tR tRt o R tR tR t]RR l4       tRt	V t
R# )	ÚNllbConverteri5  c                ó”   € . ROpY!P                   R,           Uu. uF  q3P                  VP                  3NK  	  up,          pV# u upi )rÙ  r  r	  rˆ  rë  s   &&  r"   r<   ÚNllbConverter.vocab6  óC   € ò
ˆð 	¿,¹,ÀrÖ:JÓKÑ:J°—;‘; §¡Ó,Ñ:JÑKÕKˆØˆùò Ló   š Ac                ó   € ^# rÓ  r:   rŒ  s   &&r"   rt   ÚNllbConverter.unk_id@  rA  r$   c           	     ó¤   € \         P                  ! R RRV P                  P                  R4      3RV P                  P                  R4      3.R7      # )zeng_Latn $A </s>zeng_Latn $A $B </s>Úeng_LatnrÚ  rÐ   rË  r»   s   &r"   rä   ÚNllbConverter.post_processorC  sR   € Ü×,Ò,Ø%Ø&à˜T×4Ñ4×JÑJÈ:ÓVÐWØ˜×0Ñ0×FÑFÀvÓNÐOðô
ð 	
r$   Nc                óÂ  € \        VP                  R R4      4      p\        VP                  RR4      4      p\        VP                  RR4      4      p\        VP                  RR4      4      pV^ V^V^V^/pVe[   \        V\        4      '       d   VP	                  4       MV UU	u. uF  w  r‰VNK	  	  up	pp
V
 F  pW·9   d   K  \        V4      W{&   K  	  WrR&   V# u up	pi )	r,  rÙ  ró  rä  rF  rÚ  rÅ   rç  r<   )r*   r§   rr  rI   r  rC   )rè   r<   rŒ   r,  ró  rF  rÅ   Úreordered_vocabÚtokr—  Útokensr’   s   &&,         r"   r|  ÚNllbConverter.convert_from_spmM  sÝ   € ä˜Ÿ
™
 ;°Ó6Ó7ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ
™
 ;°Ó7Ó8ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	ð �qØ�qØ�qØ�qð	
ˆð ÒÜ%/°´t×%<Ò%<�U—Z‘Z”\ÑUZÔB[ÑUZÉ6È3Ã3ÑUZÒB[ˆFÛ�ØÔ+ÙÜ),¨_Ó)=�Ó&ñ  ð *ˆw‰Øˆùó C\s   Â&Cr:   r©   rù  rœ   s   @r"   r{  r{  5  s(   ø‡ € òòò
ð óó ör$   r{  c                   ó2   a € ] tR tRt o R tR tR tRtV tR# )ÚSeamlessM4TConverterid  c                ó”   € . ROpY!P                   R,           Uu. uF  q3P                  VP                  3NK  	  up,          pV# u upi )rä  r  )rã  ræ  r
  r  rˆ  rë  s   &&  r"   r<   ÚSeamlessM4TConverter.vocabe  r~  r  c                ó.   € V P                   P                  # r©   )r2   Úunk_token_idrŒ  s   &&r"   rt   ÚSeamlessM4TConverter.unk_ido  s   € Ø×&Ñ&×3Ñ3Ð3r$   c           	     ó¤   € \         P                  ! R RRV P                  P                  R4      3RV P                  P                  R4      3.R7      # )z__eng__ $A </s>z__eng__ $A $B </s>Ú__eng__rÚ  rÐ   rË  r»   s   &r"   rä   Ú#SeamlessM4TConverter.post_processorr  sR   € Ü×,Ò,Ø$Ø%à˜D×3Ñ3×IÑIÈ)ÓTÐUØ˜×0Ñ0×FÑFÀvÓNÐOðô
ð 	
r$   r:   N©	r‡   r–   r—   r˜   r<   rt   rä   rš   r›   rœ   s   @r"   r‹  r‹  d  s   ø‡ € òò4÷
ð 
r$   r‹  c                   óF   a € ] tR tRt o R tR tR t]RR l4       tRt	V t
R# )	ÚXLMRobertaConverteri}  c                ó¨   € . ROpY!P                   R,           Uu. uF  q3P                  VP                  3NK  	  up,          pVR.,          pV# u upi )rÙ  r  r	  ré  rˆ  rë  s   &&  r"   r<   ÚXLMRobertaConverter.vocab~  sP   € ò
ˆð 	¿,¹,ÀrÖ:JÓKÑ:J°—;‘; §¡Ó,Ñ:JÑKÕKˆØ�/Ð"Õ"ˆØˆùò Lrí  c                ó
   € ^pV# rÓ  r:   rÔ  s   && r"   rt   ÚXLMRobertaConverter.unk_id‰  rÖ  r$   c           	     ó¤   € \         P                  ! R RRV P                  P                  R4      3RV P                  P                  R4      3.R7      # rØ  rË  r»   s   &r"   rä   Ú"XLMRobertaConverter.post_processor�  rÜ  r$   Nc                ó¨  € \        VP                  R R4      4      p\        VP                  RR4      4      p\        VP                  RR4      4      p\        VP                  RR4      4      p\        VP                  RR	4      4      pVR
3VR
3VR
3VR
3.pVe"   VP                  \        V4      R,          4       VP	                  VR
34       W‚R&   V# )r,  rÙ  ró  rä  rF  rÚ  rÅ   rç  rô  rê  râ  r  r<   rö  rP  s	   &&,      r"   r|  Ú$XLMRobertaConverter.convert_from_spm—  sÑ   € ä˜Ÿ
™
 ;°Ó6Ó7ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ
™
 ;°Ó7Ó8ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ™ L°(Ó;Ó<ˆ
ð ˜ÐØ˜ÐØ˜ÐØ˜Ðð	
ˆ
ð ÒØ×Ñœd 5›k¨"�oÔ.Ø×Ñ˜: sÐ+Ô,Ø$ˆw‰Øˆr$   r:   r©   rù  rœ   s   @r"   r–  r–  }  s(   ø‡ € ò	òò
ð óó ör$   r–  c                   ó2   a € ] tR tRt o R tR tR tRtV tR# )ÚXLNetConverteri¬  c                óä   € VP                    Uu. uFU  p\        VP                  4      '       d   VP                  VP                  3MVP                  VP                  ^d,
          3NKW  	  up# u upi r·  r¹  r‰  s   && r"   r<   ÚXLNetConverter.vocab­  r»  r¼  c                ó¶  € \         P                  ! R R4      \         P                  ! RR4      .pV P                  P                  '       gI   VP	                  \         P
                  ! 4       4       VP	                  \         P                  ! 4       4       V P                  P                  '       d%   VP	                  \         P                  ! 4       4       VP                  P                  pV'       d&   VP	                  \         P                  ! V4      4       VP	                  \         P                  ! \        R4      R4      4       \         P                  ! V4      # r¾  rÀ  rÅ  s   &&  r"   rÜ   ÚXLNetConverter.normalizer³  rÈ  r$   c           	     ó¤   € \         P                  ! R RRV P                  P                  R4      3RV P                  P                  R4      3.R7      # )z$A:0 <sep>:0 <cls>:2z!$A:0 <sep>:0 $B:1 <sep>:1 <cls>:2z<sep>z<cls>rÐ   rË  r»   s   &r"   rä   ÚXLNetConverter.post_processorÆ  rÍ  r$   r:   NrÎ  rœ   s   @r"   r   r   ¬  rÏ  r$   r   c                   ó   € ] tR tRtRtR# )ÚReformerConverteriÑ  r:   N©r‡   r–   r—   r˜   rš   r:   r$   r"   r¨  r¨  Ñ  ó   † Ûr$   r¨  c                   ó,   a € ] tR tRt o R tR tRtV tR# )ÚRemBertConverteriÕ  c                ó–  € \         P                  ! R R4      \         P                  ! RR4      \         P                  ! \        R4      R4      .pV P                  P                  '       gI   VP                  \         P                  ! 4       4       VP                  \         P                  ! 4       4       V P                  P                  '       d%   VP                  \         P                  ! 4       4       VP                  P                  pV'       d&   VP                  \         P                  ! V4      4       \         P                  ! V4      # r¾  )r   rs  r   r2   rÁ  rL   rÂ  rÃ  rÚ   rÄ  r‹   r|   ru  rK  rÅ  s   &&  r"   rÜ   ÚRemBertConverter.normalizer×  sê   € ä×Ò  cÓ*Ü×Ò  cÓ*Ü×Ò¤ g£°Ó4ð
Ðð
 ×&Ñ&×3×3Ð3Ø×#Ñ#¤K×$4Ò$4Ó$6Ô7Ø×#Ñ#¤K×$<Ò$<Ó$>Ô?Ø×"Ñ"×0×0Ð0Ø×#Ñ#¤K×$9Ò$9Ó$;Ô<à$×4Ñ4×IÑIÐçØ×#Ñ#¤K×$;Ò$;Ð<PÓ$QÔRä×#Ò#Ð$4Ó5Ð5r$   c           	     ó¤   € \         P                  ! R RRV P                  P                  R4      3RV P                  P                  R4      3.R7      # rÊ  rË  r»   s   &r"   rä   ÚRemBertConverter.post_processorê  rÍ  r$   r:   N)r‡   r–   r—   r˜   rÜ   rä   rš   r›   rœ   s   @r"   r¬  r¬  Õ  s   ø‡ € ò6÷&
ð 
r$   r¬  c                   ó   € ] tR tRtRtR# )ÚBertGenerationConverteriõ  r:   Nr©  r:   r$   r"   r²  r²  õ  rª  r$   r²  c                   óL   a € ] tR tRt o R t]R	R l4       tR tR tR t	Rt
V tR# )
ÚPegasusConverteriù  c                óª  € V P                   P                  R 3V P                   P                  R 3.pV P                   P                  e    W P                   P                  R 3.,          pV P                   P                  eO   V P                   P
                  V P                   P                  8  d    W P                   P                  R 3.,          pT\        ^V P                   P                  4       Uu. uF  pRV R2R3NK  	  up,          pY!P                  R,           Uu. uF  qDP                  VP                  3NK  	  up,          pV# u upi u upi )râ  ú<unk_Ú>ºr&   NNrõ  )r2   ró  rF  Úmask_token_sentrô  Úmask_token_idÚoffsetrK   r„   r…   r†   )rf   re   r<   r�   r…   s   &&   r"   r<   ÚPegasusConverter.vocabú  s'  € à×$Ñ$×.Ñ.°Ð4Ø×$Ñ$×.Ñ.°Ð4ð
ˆð
 ×"Ñ"×2Ñ2Ò>Ø×.Ñ.×>Ñ>ÀÐDÐEÕEˆEð ×#Ñ#×.Ñ.Ò:Ø×'Ñ'×5Ñ5¸×8OÑ8O×8VÑ8VÔVà×.Ñ.×9Ñ9¸3Ð?Ð@Õ@ˆEà´%¸¸4×;RÑ;R×;YÑ;YÔ2ZÓ[Ñ2Z¨Q�U˜1˜#˜Q�< Ó(Ñ2ZÑ[Õ[ˆØ¿,¹,ÀrÖ:JÓKÑ:J°—;‘; §¡Ó,Ñ:JÑKÕKˆØˆùò \ùÚKs   Ã1EÄ  ENc           	     ó@  € \        VP                  R R4      4      p\        VP                  RR4      4      p\        VP                  RR4      4      p\        VP                  RR4      4      pVR3VR3.pVR	8w  d   VP                  VR34       VR	8w  d   We8w  d   VP                  VR34       TP                  \	        ^VP                  R
^g4      4       Uu. uF  pRV R2R3NK  	  up4       Ve"   VP                  \        V4      R,          4       WrR&   V# u upi )ró  rä  rF  rÚ  rô  z<mask_1>r¹  z<mask_2>râ  ÚNoner»  r¶  r·  r¸  r<   rõ  )r*   r§   rL   rN   rK   rq   )	rè   r<   rŒ   ró  rF  rô  r¹  r÷  r�   s	   &&,      r"   r|  Ú!PegasusConverter.convert_from_spm  s  € ä˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ
™
 ;°Ó7Ó8ˆ	Ü˜Ÿ™ L°*Ó=Ó>ˆ
Ü˜fŸj™jÐ):¸JÓGÓHˆð ˜ÐØ˜Ðð
ˆ
ð ˜ÔØ×Ñ˜z¨3Ð/Ô0Ø˜fÔ$¨Ô)FØ×Ñ˜°Ð4Ô5à×Ñ¼5ÀÀFÇJÁJÈxÐY\ÓD]Ô;^Ó_Ñ;^°a˜e A 3 a˜L¨&Ó1Ñ;^Ñ_Ô`ØÒØ×Ñœd 5›k¨"�oÔ.Ø$ˆw‰Øˆùò	 `s   ÃDc                ód   € VP                   P                  V P                  P                  ,           # r©   )r�   rt   r2   r»  rŒ  s   &&r"   rt   ÚPegasusConverter.unk_id#  s%   € Ø×!Ñ!×(Ñ(¨4×+BÑ+B×+IÑ+IÕIÐIr$   c                ó¬   € \        W P                  4      p\        P                  ! \        P                  ! 4       \        P
                  ! WR 7      .4      # r¡  )r4   r2   r	   rK  ÚWhitespaceSplitr¤  r¥  s   &&& r"   rÞ   ÚPegasusConverter.pre_tokenizer&  sE   € Ü,Ð-=×?VÑ?VÓWˆÜ×&Ò&ä×.Ò.Ó0Ü×(Ò(°[Ô`ðó
ð 	
r$   c                óš   € V P                   P                  pWP                   P                  3.p\        P                  ! R V.R RV.VR7      # )ú$Aú$BrÐ   )r2   rF  Úeos_token_idr
   rã   )rf   ÚeosrÓ   s   &  r"   rä   ÚPegasusConverter.post_processor/  sP   € Ø×%Ñ%×/Ñ/ˆà×)Ñ)×6Ñ6Ð7ð
ˆô ×,Ò,°T¸3°KÀtÈTÐSVÐFWÐhvÔwÐwr$   r:   r©   )r‡   r–   r—   r˜   r<   r±  r|  rt   rÞ   rä   rš   r›   rœ   s   @r"   r´  r´  ù  s5   ø‡ € òð& óó ðò*Jò
÷xð xr$   r´  c                   ó@   a € ] tR tRt o R tR t]RR l4       tRtV t	R# )ÚT5Converteri7  c                ó  € V P                   P                  pVP                   Uu. uF  q3P                  VP                  3NK  	  ppT\        V^,
          RR4       Uu. uF  pRV R2R3NK  	  up,          pV# u upi u upi )r8  ú
<extra_id_r·  râ  r­   )r2   Ú
_extra_idsr„   r…   r†   rK   )rf   re   Únum_extra_idsr…   r<   r�   s   &&    r"   r<   ÚT5Converter.vocab8  s|   € Ø×/Ñ/×:Ñ:ˆØ9>¿ºÓF¹°—+‘+˜uŸ{™{Ó+¹ˆÐFØ´E¸-È!Õ:KÈRÐQSÔ4TÓUÑ4T¨q�Z ˜s !Ð$ cÓ*Ñ4TÑUÕUˆØˆùò GùÚUs   ¥ A:ÁA?c                ót   € \         P                  ! R R.. RORV P                  P                  R4      3.R7      # ©rÆ  rÚ  rÐ   )rÆ  rÚ  rÇ  rÚ  rË  r»   s   &r"   rä   ÚT5Converter.post_processor>  ó=   € Ü×,Ò,Ø˜&�>Ú-à˜×0Ñ0×FÑFÀvÓNÐOðô
ð 	
r$   Nc                ó   € VP                  R ^d4      p\        V^,
          RR4       Uu. uF	  pRV R2NK  	  ppVe   \        V4      M. pVP                  R V 4       4       VP	                  RV4       WbR&   V# u upi )Ú	extra_idsrÎ  r·  c              3   ó(   "  € T F  qR 3x € K
  	  R# 5irH  r:   )rJ  r’   s   & r"   rL  Ú/T5Converter.convert_from_spm.<locals>.<genexpr>L  s   é € ÐA±L¨5 #�,³LùrN  r{   r<   r­   )r§   rK   rq   rN   Ú
setdefault)rè   r<   rŒ   r×  r�   Úextra_tokensr÷  s   &&,    r"   r|  ÚT5Converter.convert_from_spmG  s…   € à—J‘J˜{¨CÓ0ˆ	Ü38¸ÀQ½ÈÈBÔ3OÓPÑ3O¨a˜* Q C qÓ)Ñ3OˆÐPØ$)Ò$5”T˜%”[¸2ˆ
Ø×ÑÑA±LÓAÔAà×ÑÐ5°|ÔDØ$ˆw‰Øˆùò Qs   ©A;r:   r©   )
r‡   r–   r—   r˜   r<   rä   r±  r|  rš   r›   rœ   s   @r"   rÌ  rÌ  7  s#   ø‡ € òò
ð óó ör$   rÌ  c                   ó&   a € ] tR tRt o R tRtV tR# )ÚUdopConverteriS  c                ót   € \         P                  ! R R.. RORV P                  P                  R4      3.R7      # rÓ  rË  r»   s   &r"   rä   ÚUdopConverter.post_processorT  rÕ  r$   r:   N©r‡   r–   r—   r˜   rä   rš   r›   rœ   s   @r"   rÞ  rÞ  S  s   ø‡ € ÷
ð 
r$   rÞ  c                   ó&   a € ] tR tRt o R tRtV tR# )ÚVideoPrismConverteri^  c                ó   € R # r©   r:   r»   s   &r"   rä   Ú"VideoPrismConverter.post_processor_  r©  r$   r:   Nrá  rœ   s   @r"   rã  rã  ^  s   ø‡ € ÷ð r$   rã  c                   ó2   a € ] tR tRt o V 3R lR ltRtV tR# )ÚWhisperConverteric  c                ó    <€ V ^8„  d   QhRS[ /# rm   r·   )r   r]   s   "€r"   r+   ÚWhisperConverter.__annotate__d  s   ø€ ÷  ñ  ™9ñ  r$   c                óÌ  € V P                   P                  p\        V P                   P                  P	                  4       4      p\        \        VVR RRRR7      4      p\        P                  ! V P                   P                  R7      Vn
        \        P                  ! 4       Vn        V P                   P                  pV P                   P                  V4      pV P                   P                  pV P                   P                   pRP#                  V Uu. uF  qˆ R2NK	  	  up4      p	\$        P&                  ! V	 RV R2V	 RV R	2Wg3.\)        WT4      OR
7      Vn        V# u upi )Nr$  Fr%  r'  rõ   rÍ   z $A:0 z $A:0 $B:1 rÏ   rÐ   )r2   r  rq   r  r  r   r   r	   r+  r'   rÞ   r   rå   Úprefix_tokensÚconvert_ids_to_tokensrF  rÈ  Újoinr
   rã   Úziprä   )
rf   r<   rP   rç   Úprefix_token_idsÚprefixesrÉ  rÈ  r’   Úprefix_templates
   &         r"   r¼   ÚWhisperConverter.convertedd  sO  € Ø×'Ñ'×/Ñ/ˆÜ�d×-Ñ-×7Ñ7×<Ñ<Ó>Ó?ˆäÜØØØØ*,Ø#%Øôó	
ˆ	ô #1×":Ò":ÈD×LcÑLc×LtÑLtÔ"uˆ	ÔÜ$×.Ò.Ó0ˆ	Ôà×2Ñ2×@Ñ@ÐØ×*Ñ*×@Ñ@ÐAQÓRˆØ×%Ñ%×/Ñ/ˆØ×.Ñ.×;Ñ;ˆØŸ(™(¹hÓ#G¹h°U g¨R£L¹hÑ#GÓHˆÜ#-×#@Ò#@Ø%Ð& f¨S¨E°Ð4Ø#Ð$ K°¨u°BÐ7àÐ#ðä�XÓ0ðô$
ˆ	Ô ð Ðùò $Hs   ÄE!r:   Nrì   rœ   s   @r"   rç  rç  c  s   ø‡ € ÷ ö  r$   rç  c                   ó&   a € ] tR tRt o R tRtV tR# )ÚBigBirdConverteri‡  c           	     ó¤   € \         P                  ! R RRV P                  P                  R4      3RV P                  P                  R4      3.R7      # rÊ  rË  r»   s   &r"   rä   ÚBigBirdConverter.post_processorˆ  rÍ  r$   r:   Nrá  rœ   s   @r"   rô  rô  ‡  s   ø‡ € ÷
ð 
r$   rô  c                   ó2   a € ] tR tRt o V 3R lR ltRtV tR# )ÚCLIPConverteri“  c                ó    <€ V ^8„  d   QhRS[ /# rm   r·   )r   r]   s   "€r"   r+   ÚCLIPConverter.__annotate__”  s   ø€ ÷ 'ñ '™9ñ 'r$   c                ó€  € V P                   P                  p\        V P                   P                  P	                  4       4      pV P                   P
                  p\        \        VVR RRR\        V4      R7      4      p\        P                  ! \        P                  ! 4       \        P                  ! \        R4      R4      \        P                  ! 4       .4      Vn        \         P                  ! \         P"                  ! \        R4      RR	R
7      \         P$                  ! RR7      .4      Vn        \(        P$                  ! 4       Vn        \,        P.                  ! V P                   P0                  V P                   P2                  3V P                   P4                  V P                   P6                  3RR4      Vn        V# )Nr$  r  F©r<   rP   r  r&  r  r  rÅ   z\s+rõ   z9's|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+ÚremovedTrE  r'  )r2   r  rq   r  r  rÅ   r   r   r*   r   rK  rJ  rs  r   rÄ  rÜ   r	   rL  r+  rÞ   r   rå   r
   rT  rF  rÈ  r,  r-  rä   r  s   &    r"   r¼   ÚCLIPConverter.converted”  sk  € Ø×'Ñ'×/Ñ/ˆÜ�d×-Ñ-×7Ñ7×<Ñ<Ó>Ó?ˆØ×+Ñ+×5Ñ5ˆ	äÜØØØØ*,Ø#)ØÜ˜i›.ôó

ˆ	ô  +×3Ò3Ü�_Š_Ó¤× 3Ò 3´E¸&³MÀ3Ó GÌ×I^ÒI^ÓI`Ðaó 
ˆ	Ôô #1×"9Ò"9ä×$Ò$ÜÐZÓ[Ø&Øôô
 ×(Ò(¸%Ô@ðó	#
ˆ	Ôô %×.Ò.Ó0ˆ	Ôô $.×#?Ò#?Ø×$Ñ$×.Ñ.°×0GÑ0G×0TÑ0TÐUØ×$Ñ$×.Ñ.°×0GÑ0G×0TÑ0TÐUØØó	$
ˆ	Ô ð Ðr$   r:   Nrì   rœ   s   @r"   rø  rø  “  s   ø‡ € ÷'ö 'r$   rø  c                   ó2   a € ] tR tRt o V 3R lR ltRtV tR# )ÚLayoutLMv2Converteri¾  c                ó    <€ V ^8„  d   QhRS[ /# rm   r·   )r   r]   s   "€r"   r+   Ú LayoutLMv2Converter.__annotate__¿  rÂ   r$   c           	     ó|  € V P                   P                  p\        \        V\	        V P                   P
                  4      R 7      4      pRpRpRp\        V P                   R4      '       da   V P                   P                  P                  pV P                   P                  P                  pV P                   P                  P                  p\        P                  ! RVVVR7      Vn        \        P                  ! 4       Vn        \	        V P                   P"                  4      p\	        V P                   P$                  4      pV P                   P&                  pV P                   P(                  p	\*        P,                  ! V RV R2V RV RV R2Wh3Wy3.R	7      Vn        \0        P                  ! R
R7      Vn        V# )rÄ   FTrÆ   rÇ   rÌ   rÍ   rÎ   rÏ   rÐ   rÔ   rÕ   r×   ræ   s
   &         r"   r¼   ÚLayoutLMv2Converter.converted¿  s„  € Ø×'Ñ'×-Ñ-ˆÜœi¨¼¸T×=TÑ=T×=^Ñ=^Ó9_Ô`Óaˆ	à!&ÐØˆØˆÜ�4×*Ñ*Ð,=×>Ò>Ø%)×%<Ñ%<×%LÑ%L×%cÑ%cÐ"Ø ×3Ñ3×CÑC×QÑQˆMØ ×3Ñ3×CÑC×QÑQˆMä*×9Ò9ØØ!7Ø'Ø#ô	 
ˆ	Ôô #1×"AÒ"AÓ"Cˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆä#-×#@Ò#@Ø�U˜( 3 % rÐ*Ø�5˜   X¨c¨U°"Ð5àÐ#ØÐ#ðô$
ˆ	Ô ô %×.Ò.°dÔ;ˆ	ÔàÐr$   r:   Nrì   rœ   s   @r"   r   r   ¾  rí   r$   r   c                   ó2   a € ] tR tRt o V 3R lR ltRtV tR# )ÚBlenderbotConverteriå  c                ó    <€ V ^8„  d   QhRS[ /# rm   r·   )r   r]   s   "€r"   r+   Ú BlenderbotConverter.__annotate__æ  rR  r$   c                ó´  € V P                   pVP                  p\        VP                  P	                  4       4      p\        \        VVR RRRR7      4      p\        P                  ! VP                  R7      Vn
        \        P                  ! 4       Vn        \        P                  ! RVP                   R2VP                  VP                   3.R7      Vn        V# )Nr$  Fr%  r'  z$A:0 rÍ   )rÑ   rÓ   )r2   r  rq   r  r  r   r   r	   r+  r'   rÞ   r   rå   r
   rã   rF  rÈ  rä   rU  s   &    r"   r¼   ÚBlenderbotConverter.convertedæ  s½   € Ø×$Ñ$ˆØ—
‘
ˆÜ�b—l‘l×'Ñ'Ó)Ó*ˆäÜØØØØ*,Ø#%Øôó	
ˆ	ô #1×":Ò":ÈB×L_ÑL_Ô"`ˆ	ÔÜ$×.Ò.Ó0ˆ	ÔÜ#-×#@Ò#@Ø˜2Ÿ<™<˜.¨Ð+à—‘˜rŸ™Ð/ðô$
ˆ	Ô ð Ðr$   r:   Nrì   rœ   s   @r"   r  r  å  rX  r$   r  c                   ó2   a € ] tR tRt o R tR tR tRtV tR# )ÚXGLMConverteri  c                óª   € . ROpY!P                   R,           Uu. uF  q3P                  VP                  3NK  	  up,          pV. RO,          pV# u upi )rÙ  r  r	  ))z<madeupword0>râ  )z<madeupword1>râ  )z<madeupword2>râ  )z<madeupword3>râ  )z<madeupword4>râ  )z<madeupword5>râ  )z<madeupword6>râ  rˆ  rë  s   &&  r"   r<   ÚXGLMConverter.vocab  sT   € ò
ˆð 	¿,¹,ÀrÖ:JÓKÑ:J°—;‘; §¡Ó,Ñ:JÑKÕKˆØò  zõ  	zˆØˆùò Ls   š Ac                ó
   € ^pV# rÓ  r:   rÔ  s   && r"   rt   ÚXGLMConverter.unk_id  rÖ  r$   c           	     ó¤   € \         P                  ! R RRV P                  P                  R4      3RV P                  P                  R4      3.R7      # )z</s> $Az</s> $A </s> </s> $BrÙ  rÚ  rÐ   rË  r»   s   &r"   rä   ÚXGLMConverter.post_processor  sR   € Ü×,Ò,ØØ'à˜×/Ñ/×EÑEÀeÓLÐMØ˜×0Ñ0×FÑFÀvÓNÐOðô
ð 	
r$   r:   Nr”  rœ   s   @r"   r  r    s   ø‡ € ò	ò÷
ð 
r$   r  c                   óP   a € ] tR tRt o Rt]tRR0t R tR t	R t
R tR	 tR
tV tR# )ÚGemmaConverteri  Tz<start_of_turn>z<end_of_turn>c                ó0   € \         P                  ! R R4      # ©rõ   rp  )r   rs  rŒ  s   &&r"   rÜ   ÚGemmaConverter.normalizer-  s   € Ü×"Ò" 3¨Ó.Ð.r$   c                óÖ  € V P                   P                  R 3V P                   P                  R 3V P                   P                  R 3.pY!P                  R,           Uu. uF  q3P
                  VP                  3NK  	  up,          p\        ;QJ d    R V 4       F  '       g   K   RM	  RM! R V 4       4      '       g%   \        R \        V4       4       R4      pVe   RW$&   V# u upi )râ  r  c              3   ó8   "  € T F  q^ ,          R8H  x € K  	  R# 5i)r   r¢   Nr:   )rJ  r;   s   & r"   rL  Ú'GemmaConverter.vocab.<locals>.<genexpr>9  s   é € Ð/© A�Q•4˜4–<«ùs   ‚TFc              3   óJ   "  € T F  w  rV^ ,          R8X  g   K  Vx € K  	  R# 5i)r   r£   Nr:   )rJ  r�   r;   s   &  r"   rL  r  :  s!   é € Ð"VÑ1A©¨ÀQÀqÅTÈXÑEU§1¢1Ó1Aùs   ‚#™
#N)r¢   râ  )
r2   ró  rF  r,  r„   r…   r†   ÚanyÚnextr‰   )rf   re   r<   r…   Úoverride_indexs   &&   r"   r<   ÚGemmaConverter.vocab0  sÈ   € à×$Ñ$×.Ñ.°Ð4Ø×$Ñ$×.Ñ.°Ð4Ø×$Ñ$×.Ñ.°Ð4ð
ˆð
 	¿,¹,ÀrÖ:JÓKÑ:J°—;‘; §¡Ó,Ñ:JÑKÕKˆ÷ ‹sÑ/©Ó/�s�sŠsÑ/©Ó/×/Ò/Ü!Ñ"V´¸5Ô1AÓ"VÐX\Ó]ˆNØÒ)Ø(3�Ñ%àˆùò Ls   Á C&c                ó0   € \         P                  ! R R4      # )rõ   Úmerged_with_previous)r	   rL  ©rf   r£  r'   s   &&&r"   rÞ   ÚGemmaConverter.pre_tokenizer@  s   € Ü×#Ò# CÐ)?Ó@Ð@r$   c                ó
   € ^pV# rÓ  r:   rÔ  s   && r"   rt   ÚGemmaConverter.unk_idC  rÖ  r$   c                óª   € \         P                  ! \         P                  ! R R4      \         P                  ! 4       \         P                  ! 4       .4      # )rp  rõ   )r   rK  rs  rv  rw  r"  s   &&&r"   rå   ÚGemmaConverter.decoderG  s?   € Ü× Ò ä× Ò  ¨Ó,Ü×%Ò%Ó'Ü—’“ðó
ð 	
r$   r:   N)r‡   r–   r—   r˜   r�  rž   r”  rÓ   rÜ   r<   rÞ   rt   rå   rš   r›   rœ   s   @r"   r  r    s>   ø‡ € ØÐØ.€Là'¨Ð9€Nðò/òò Aò÷
ð 
r$   r  c                   óH   a € ] tR tRt o RtR tR tR tR tR t	R t
R	tV tR
# )ÚLlamaConverteriQ  Tc                ó8  € V P                   P                  ^ 4      R3V P                   P                  ^4      R3V P                   P                  ^4      R3.pY!P                  R,           Uu. uF  q3P                  VP                  3NK  	  up,          pV# u upi )r   râ  r  )r2   rì  r„   r…   r†   rë  s   &&  r"   r<   ÚLlamaConverter.vocabT  sŽ   € à×$Ñ$×:Ñ:¸1Ó=¸sÐCØ×$Ñ$×:Ñ:¸1Ó=¸sÐCØ×$Ñ$×:Ñ:¸1Ó=¸sÐCð
ˆð
 	¿,¹,ÀrÖ:JÓKÑ:J°—;‘; §¡Ó,Ñ:JÑKÕKˆØˆùò Ls   Á, Bc                ó
   € ^ pV# r9   r:   rÔ  s   && r"   rt   ÚLlamaConverter.unk_id]  rÖ  r$   c                óþ   € \         P                  ! R R4      \         P                  ! 4       \         P                  ! 4       .pV'       d!   V\         P                  ! R^R7      .,          p\         P
                  ! V4      # ©rp  rõ   )Úcontentr›  ©r   rs  rv  rw  r�  rK  ©rf   r£  r'   Úsequences   &&& r"   rå   ÚLlamaConverter.decodera  ó\   € ä×Ò˜U CÓ(Ü×!Ò!Ó#Ü�MŠM‹Oð
ˆ÷
 ØœŸš°¸!Ô<Ð=Õ=ˆHÜ× Ò  Ó*Ð*r$   c                ó(  € \        V P                  R R4      '       du   . p\        V P                  RR4      '       d    V\        P                  ! RR7      .,          pV\        P                  ! RRR7      .,          p\        P
                  ! V4      # R# )r.   Tr'   rp  )Úprependrõ   )Úpatternr0  N)r1   r2   r   ÚPrependrs  rK  )rf   re   r3  s   && r"   rÜ   ÚLlamaConverter.normalizerk  sx   € Ü�4×*Ñ*¨H°d×;Ò;ØˆHÜ�t×.Ñ.Ð0BÀD×IÒIØœ[×0Ò0¸Ô?Ð@Õ@�Øœ×,Ò,°SÀ%ÔHÐIÕIˆHÜ×'Ò'¨Ó1Ð1Ùr$   c                óš   € \        V P                  R R4      '       g.   \        W P                  4      p\        P                  ! WRR7      # R# )r.   TF©r£  r3   ÚsplitN)r1   r2   r4   r	   r¤  r¥  s   &&& r"   rÞ   ÚLlamaConverter.pre_tokenizert  s?   € Ü�t×.Ñ.°¸$×?Ò?Ü0Ð1A×CZÑCZÓ[ˆNÜ!×+Ò+¸ÐjoÔpÐpÙr$   c                ó   € R # r©   r:   r»   s   &r"   rä   ÚLlamaConverter.post_processorz  s   € ár$   r:   N)r‡   r–   r—   r˜   r�  r<   rt   rå   rÜ   rÞ   rä   rš   r›   rœ   s   @r"   r)  r)  Q  s.   ø‡ € ØÐòòò+òò÷ð r$   r)  c                   ó2   a € ] tR tRt o V 3R lR ltRtV tR# )ÚMarkupLMConverteri  c                ó    <€ V ^8„  d   QhRS[ /# rm   r·   )r   r]   s   "€r"   r+   ÚMarkupLMConverter.__annotate__€  s   ø€ ÷ "ñ "™9ñ "r$   c                óŒ  € V P                   pVP                  p\        VP                  P	                  4       4      p\        \        VVR RRRV P                   P                  R7      4      p\        P                  ! VP                  R7      Vn        \        P                  ! 4       Vn        \        V P                   P                  4      p\        V P                   P                   4      pV P                   P"                  pV P                   P$                  p\&        P(                  ! V RV 2V RV RV 2WW3Wh3.R7      Vn        V# )Nr$  Frü  r'  z $A z $B rÐ   )r2   r  rq   r  r  r   r   rÅ   r	   r+  r'   rÞ   r   rå   r*   rß   rà   rá   râ   r
   rã   rä   )	rf   rV  r<   rP   rç   rè   ré   rá   râ   s	   &        r"   r¼   ÚMarkupLMConverter.converted€  s(  € Ø×$Ñ$ˆØ—
‘
ˆÜ�b—l‘l×'Ñ'Ó)Ó*ˆäÜØØØØ*,Ø#%ØØ×1Ñ1×;Ñ;ôó

ˆ	ô #1×":Ò":ÈB×L_ÑL_Ô"`ˆ	ÔÜ$×.Ò.Ó0ˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆä#-×#@Ò#@Ø�U˜$˜s˜eÐ$Ø�5˜˜S˜E  c UÐ+àÐ#ØÐ#ðô$
ˆ	Ô ð Ðr$   r:   Nrì   rœ   s   @r"   rB  rB    s   ø‡ € ÷"ö "r$   rB  c                   ó<   a € ] tR tRt o RtR tR tR tR tRt	V t
R# )	ÚMoshiConverteri¥  Tc                ó&  € \        V R 4       \        P                  W4       \        4       pVP	                  4       p\        VR4      ;_uu_ 4       pVP                  VP                  4       4       RRR4       W@n        R#   + '       g   i     L; i©r   r`   N©	r   r²   rj   r#   ra   rb   rc   rd   re   ©rf   r€  rŒ   rg   rh   ri   s   &&,   r"   rj   ÚMoshiConverter.__init__¨  sh   € Ü˜$ 
Ô+ä×Ñ˜4Ô,ô $Ó%ˆ	à× Ñ Ó"ˆÜ�*˜d×#Ô# qØ×Ñ˜aŸf™f›hÔ'÷ $àŽ
÷ $×#úó   Á B Â B	c                óþ   € VP                   P                  p\        P                  ! R R4      .pV'       g   \        P                  ! V4      # \        P                  ! \        P
                  ! V4      .V,           4      # r  )r‹   r|   r   rs  rK  ru  rž  s   &&  r"   rÜ   ÚMoshiConverter.normalizerµ  sg   € Ø$×4Ñ4×IÑIÐä×Ò  UÓ+ð
ˆ÷ $Ü×'Ò'¨Ó5Ð5ä×'Ò'¬×)@Ò)@ÐAUÓ)VÐ(WÐZfÕ(fÓgÐgr$   c                óþ   € \         P                  ! R R4      \         P                  ! 4       \         P                  ! 4       .pV'       d!   V\         P                  ! R^R7      .,          p\         P
                  ! V4      # r/  r1  r2  s   &&& r"   rå   ÚMoshiConverter.decoder¿  r5  r$   c                ó6   € R p\         P                  ! WRR7      # )r/   Fr<  )r	   r¤  r¥  s   &&& r"   rÞ   ÚMoshiConverter.pre_tokenizerÉ  s   € Ø ˆÜ×'Ò'°KÐfkÔlÐlr$   r•   N)r‡   r–   r—   r˜   r�  rj   rÜ   rå   rÞ   rš   r›   rœ   s   @r"   rH  rH  ¥  s'   ø‡ € ØÐòòhò+÷mð mr$   rH  c                   óX   a € ] tR tRt o RtRR ltR tR tR tR t	R	 t
R
 tR tRtV tR# )ÚHeliumConverteriÎ  TNc                ó&  € \        V R 4       \        P                  W4       \        4       pVP	                  4       p\        VR4      ;_uu_ 4       pVP                  VP                  4       4       RRR4       W@n        R#   + '       g   i     L; irJ  rK  rL  s   &&,   r"   rj   ÚHeliumConverter.__init__Ñ  sf   € Ü˜$ 
Ô+ä×Ñ˜4Ô,ä#Ó%ˆ	à× Ñ Ó"ˆÜ�*˜d×#Ô# qØ×Ñ˜aŸf™f›hÔ'÷ $àŽ
÷ $×#úrN  c                óp  € V P                  V4      p\        \        VV P                  V4      V P                  R 7      4      p\        VP                  4       UUu. uFT  w  rEVP                  R
9   g   K  YEP                  VP                  ^8H  ;'       g    VP                  V P                  9   3NKV  	  pppTP                  \        VR R7       UUUu. uF  w  rGp\        VRVRR7      NK  	  uppp4       VP                  \        RRRR7      .4       VP                  R^R	7       V# u uppi u upppi )r�  c                 ó   € V ^ ,          # r9   r:   rv   s   &r"   r=   Ú+HeliumConverter.tokenizer.<locals>.<lambda>ò  r“  r$   r?   FT)ry   rz   Úsingle_wordÚ
rx   rä  )ró  Úpad_idr~   )r<   r   r   rt   r�  r‰   r„   rŠ   r…   rÓ   r–  rM   r   Úenable_padding)	rf   re   rO   rç   r�   r�   r‘   r’   rz   s	   &&       r"   rç   ÚHeliumConverter.tokenizerÝ  s#  € Ø—z‘z %Ó(ˆÜÜØØ—{‘{ 5Ó)Ø"×7Ñ7ôó
ˆ	ô # 5§<¡<Ô0ô
á0‘�Ø�v‰v˜Ñô IˆR—‘˜!Ÿ&™& A™+×GÐG¨¯©°D×4GÑ4GÑ)GÓHÙ0ð 	ñ 
ð
 	×Ñô +1Ð1AÁ~Õ*Võá*VÑ&�B˜wô ˜5¨U¸GÐQU×VÙ*Vóô	
ð 	×Ñœj¨¸%ÈÔOÐPÔQØ× Ñ ¨7¸1Ð Ô=ØÐùó
ùôs   ÁD+Á4 D+ÂD+ÃD1c                óÀ   € . pVP                    FK  pVP                  R 8X  d   VRVP                  3.,          pK,  W#P                  VP                  3.,          pKM  	  V# )z<0x0A>r]  rˆ  rë  s   &&  r"   r<   ÚHeliumConverter.vocabù  sV   € ØˆØ—\”\ˆEØ�{‰{˜hÔ&Ø˜4 §¡Ð-Ð.Õ.’àŸ;™;¨¯©Ð4Ð5Õ5’ñ	 "ð
 ˆr$   c                ó
   € ^ pV# r9   r:   rÔ  s   && r"   rt   ÚHeliumConverter.unk_id  rÖ  r$   c                óî   € \         P                  ! R R4      \         P                  ! 4       \         P                  ! 4       .pV\         P                  ! R^R7      .,          p\         P
                  ! V4      # r/  r1  r2  s   &&& r"   rå   ÚHeliumConverter.decoder  sY   € ä×Ò˜U CÓ(Ü×!Ò!Ó#Ü�MŠM‹Oð
ˆð
 	”X—^’^¨C°aÔ8Ð9Õ9ˆÜ× Ò  Ó*Ð*r$   c                ó„   € \         P                  ! \         P                  ! R 4      \         P                  ! R R4      .4      # r  )r   rK  r9  rs  rŒ  s   &&r"   rÜ   ÚHeliumConverter.normalizer  s2   € Ü×#Ò#¤[×%8Ò%8¸Ó%=¼{×?RÒ?RÐSWÐY^Ó?_Ð$`ÓaÐar$   c                óZ   € \         P                  ! \         P                  ! R R4      .4      # )r]  Ú
contiguous)r	   rK  rL  r"  s   &&&r"   rÞ   ÚHeliumConverter.pre_tokenizer  s#   € Ü×&Ò&¬×(<Ò(<¸TÀ<Ó(PÐ'QÓRÐRr$   c                ó>   € \         P                  ! R R.. ROR.R7      # )rÙ  rÆ  rÐ   )rÙ  rÆ  rÙ  rÇ  )rÙ  r8  )r
   rã   r»   s   &r"   rä   ÚHeliumConverter.post_processor  s/   € Ü×,Ò,àØðòð ðô
ð 	
r$   r•   r©   )r‡   r–   r—   r˜   r�  rj   rç   r<   rt   rå   rÜ   rÞ   rä   rš   r›   rœ   s   @r"   rV  rV  Î  s:   ø‡ € ØÐô
òò8òò+òbòS÷
ð 
r$   rV  c                   ó4   a € ] tR tRt o RtRR ltR tRtV tR# )ÚParakeetConverteri'  TNc                ó2  € Wn         \        V R 4       \        P                  W4       \	        4       pVP                  4       p\        VR4      ;_uu_ 4       pVP                  VP                  4       4       RRR4       W@n	        R#   + '       g   i     L; irJ  )
r€  r   r²   rj   r#   ra   rb   rc   rd   re   )rf   r€  r„  rg   rh   ri   s   &&*   r"   rj   ÚParakeetConverter.__init__*  sk   € Ø$Œä˜$ 
Ô+ä×Ñ˜4Ô,ä#Ó%ˆ	Ø× Ñ Ó"ˆÜ�*˜d×#Ô# qØ×Ñ˜aŸf™f›hÔ'÷ $àŽ
÷ $×#ús   Á BÂB	c                ó†  € V P                  V4      p\        V4       UUUu/ uF
  w  pw  rEWCbK  	  pppp\        Wb4      p\        \	        VVVP
                  P                  R V P                  RR7      4      p\        VP                  4       U	U
u. uFT  w  ršV
P                  R9   g   K  YšP                  V
P                  ^8H  ;'       g    V
P                  V P                  9   3NKV  	  pp	p
TP                  \        VR R7       U	UUu. uF  w  rœp\        VRVR7      NK  	  uppp	4       V# u upppi u up
p	i u uppp	i )TNr�  c                 ó   € V ^ ,          # r9   r:   rv   s   &r"   r=   Ú-ParakeetConverter.tokenizer.<locals>.<lambda>P  r“  r$   r?   Frx   r~   )r<   r‰   rW   r   r   r�   rq  r�  r„   rŠ   r…   rÓ   r–  rM   r   )rf   re   rO   r�   rŽ   r†   r˜  rP   rç   r�   r�   r‘   r’   rz   s   &&            r"   rç   ÚParakeetConverter.tokenizer7  s3  € Ø—z‘z %Ó(ˆä5>¸|Ô5LÕMÑ5LÑ!1 ¡M T�T’WÑ5Lˆ	ÒMÜ  Ó9ˆÜÜØØØ×,Ñ,×6Ñ6ØØ"×7Ñ7Øôó	
ˆ	ô # 5§<¡<Ô0ô
á0‘�Ø�v‰v˜Ñô IˆR—‘˜!Ÿ&™& A™+×GÐG¨¯©°D×4GÑ4GÑ)GÓHÙ0ð 	ñ 
ð
 	×Ñô +1Ð1AÁ~Õ*Võá*VÑ&�B˜wô ˜5¨U¸G×DÙ*Vóô	
ð Ðùô5 Nùó
ùôs   ¡D/ÂD6Â* D6ÃD6Ä
D<)re   r€  r©   )	r‡   r–   r—   r˜   r�  rj   rç   rš   r›   rœ   s   @r"   ro  ro  '  s   ø‡ € ØÐô÷ð r$   ro  c                   ó6   a a€ ] tR tRt oRV 3R lltRtVtV ;t# )ÚCanaryConverteriW  c                óR  <€ \         SV `  ! V.VO5!   V P                  P                   Uu0 uFh  pVP                  ^8X  g   K  VP
                  P                  R4      '       g   K9  VP
                  P                  R4      '       g   K\  VP
                  kKj  	  upV n        R# u upi )r€   z<|z|>N)	r  rj   re   r„   rŠ   r…   Ú
startswithÚendswithrÓ   )rf   r€  r„  r…   r…  s   &&* €r"   rj   ÚCanaryConverter.__init__X  s†   ø€ Ü‰Ò˜Ð+ dÔ+ð Ÿ™×*Ò*ó
á*�Ø�z‰z˜Q‰ô à#(§;¡;×#9Ñ#9¸$×#?ô àDIÇKÁK×DXÑDXÐY]×D^ô ˆE�KŒKÙ*ñ
ˆÖùò 
s   ¬B$ÁB$Á'B$Â
B$)rÓ   r©   )r‡   r–   r—   r˜   rj   rš   r›   r²  r³  s   @@r"   rw  rw  W  s   ù‡ € ÷
÷ 
r$   rw  c            	     ó>  € \        \        \        R4      \        R4      ^,           4      4      \        \        \        R4      \        R4      ^,           4      4      ,           \        \        \        R4      \        R4      ^,           4      4      ,           p V R,          p^ p\        R4       F=  pW09  g   K  V P                  V4       VP                  RV,           4       V^,          pK?  	  V Uu. uF  p\	        V4      NK  	  pp\        \        W4      4      # u upi )	a  
Returns list of utf-8 byte and a mapping to unicode strings. We specifically avoids mapping to whitespace/control
characters the bpe code barfs on.

The reversible bpe codes work on unicode strings. This means you need a large # of unicode characters in your vocab
if you want to avoid UNKs. When you're at something like a 10B token dataset you end up needing around 5K for
decent coverage. This is a significant percentage of your normal, say, 32K bpe vocab. To avoid that, we want lookup
tables between utf-8 bytes and unicode strings.
Ú!Ú~ô   Â¡ô   Â¬ô   Â®ô   Ã¿:NNNé   )rq   rK   ÚordrL   ÚchrrI   rî  )ÚbsÚcsÚnÚbs       r"   Úbytes_to_unicoderŠ  b  sÞ   € ô 	ŒU”3�s“8œS ›X¨�\Ó*Ó+¬d´5¼¸T»ÄCÈÃIÐPQÅMÓ3RÓ.SÕSÔVZÔ[`ÔadÐeiÓajÔloÐptÓluÐxyÕlyÓ[zÓV{Õ{ð ð 
ˆA�€BØ	€AÜ�4Ž[ˆØŽ;Ø�I‰I�aŒLØ�I‰I�d˜Q•hÔØ��FŠAñ	 ñ
 Ó	™"�QŒ#ˆaŽ&™"€BÐ	Ü”�B“ÓÐùò 
s   Ã0Dc                   ót   a € ] tR tRt o RtRR lt]V 3R lR l4       tV 3R lR ltR	 t	V 3R
 lR lt
RtV tR# )ÚTikTokenConverteriz  z
A general tiktoken converter.
Nc                ó’   € Wn         W n        W0n        \        V\        4      '       d   VP                  4       V n        R # TV n        R # r©   )r€  r8  r'   rr  rI   r  Úextra_special_tokens)rf   r€  r8  r'   rŽ  rŒ   s   &&&&&,r"   rj   ÚTikTokenConverter.__init__  sA   € ð %ŒØŒØ 0Ôä+5Ð6JÌD×+QÒ+QÐ ×%Ñ%Ó'ð 	Ö!ØWkð 	Ö!r$   c                ó<   <€ V ^8„  d   QhRS[ RS[S[S[3,          /# )r&   Útiktoken_urlr(   )r*   rI   Úbytesrp   )r   r]   s   "€r"   r+   ÚTikTokenConverter.__annotate__�  s$   ø€ ÷ /ñ /©ð /±±U¹C°ZÕ0@ñ /r$   c                óT   €  ^ RI Hp T! T 4      #   \         d    \        R4      hi ; i)r   )Úload_tiktoken_bpezY`tiktoken` is required to read a `tiktoken` file. Install it with `pip install tiktoken`.)Útiktoken.loadr•  r•  Ú
ValueError)r‘  r•  s   & r"   r•  Ú#TikTokenConverter.load_tiktoken_bpeŽ  s8   € ð	Ý7ñ
 ! Ó.Ð.øô	 ô 	ÜØkóð ð	ús   ‚ �'c                ó    <€ V ^8„  d   QhRS[ /# )r&   r‘  r\   )r   r]   s   "€r"   r+   r“  ˜  s   ø€ ÷ ñ ¹Cñ r$   c                ó:  aa€ V P                  V4      o\        4       oV3R  lp. p/ pSP                  4        Fž  w  rVWdV! V4      &   \        V4      ^8X  d   K!  . p\	        ^\        V4      4       F?  pVRV WXR r©V	S9   g   K  V
S9   g   K  Wš,           S9   g   K,  VP                  WšV34       KA  	  \        VV3R lRR7      pVP                  V4       K   	  \        VR RR7      pV Uu. uF   q²! V^ ,          4      V! V^,          4      3NK"  	  ppWC3# u upi )c           	      óŽ   <€ R P                  V P                  R4       Uu. uF  pS\        V4      ,          NK  	  up4      # u upi )r$  zlatin-1)rí  Údecoder„  )r‰  ÚcharÚbyte_encoders   & €r"   Útoken_bytes_to_stringÚPTikTokenConverter.extract_vocab_merges_from_model.<locals>.token_bytes_to_stringœ  s;   ø€ Ø—7‘7ÀÇÁÈÔ@SÓTÑ@S¸˜L¬¨T«×3Ð3Ñ@SÑTÓUÐUùÒTs    ANc                 óD   <€ SV ^ ,          ,          SV ^,          ,          3# r9   r:   )r;   r  s   &€r"   r=   ÚCTikTokenConverter.extract_vocab_merges_from_model.<locals>.<lambda>ª  s   ø€ °¸1¸Q½4µÀ)ÈAÈaÍDÅ/Ñ0Rr$   FrF   c                 ó   € V ^,          # rB   r:   rD   s   &r"   r=   r¢  ¬  s   € °°A¶r$   )r•  rŠ  rJ   rC   rK   rL   rM   rN   )rf   r‘  rŸ  rP   r<   r’   ÚrankrS   rT   rU   rV   rE   r  rž  s   &&          @@r"   Úextract_vocab_merges_from_modelÚ1TikTokenConverter.extract_vocab_merges_from_model˜  s  ù€ Ø×*Ñ*¨<Ó8ˆ	Ü'Ó)ˆõ	Vð ˆØˆØ$Ÿ?™?Ö,‰KˆEØ26Ñ'¨Ó.Ñ/Ü�5‹z˜QŒÙØˆEÜ˜q¤# e£*Ö-�Ø#(¨¨% =°%¸°-˜Ø˜iÖ'¨G°yÖ,@ÀgÕFWÐ\eÖEeØ—L‘L '°DÐ!9Ö:ñ .ô ˜5Ô&RÐ\aÔbˆEØ�M‰M˜%Ö ñ -ô ˜Ñ$6ÀÔFˆÙ\bÓcÑ\bÐUXÐ(¨¨Q­Ó0Ñ2GÈÈAÍÓ2OÓPÑ\bˆÐcØˆ}Ðùò ds   Ã-&Dc                óÆ   € V P                  V P                  4      w  r\        \        WR R7      4      p\	        VP
                  R4      '       d   RVP
                  n        V# )F)r  Úignore_mergesT)r¥  r€  r   r   rØ   r[   r¨  )rf   rO   rP   rç   s   &   r"   rç   ÚTikTokenConverter.tokenizer°  sM   € Ø#×CÑCÀDÇOÁOÓTÑˆÜœc ,ÀÔGÓHˆ	Ü�9—?‘? O×4Ò4Ø,0ˆI�O‰OÔ)ØÐr$   c                ó    <€ V ^8„  d   QhRS[ /# rm   r·   )r   r]   s   "€r"   r+   r“  ·  s   ø€ ÷ ñ ™9ñ r$   c                óô  € V P                  4       p\        P                  ! \        P                  ! \	        V P
                  4      R RR7      \        P                  ! V P                  RR7      .4      Vn        \        P                  ! 4       Vn
        V P                  e8   TP                  V P                   Uu. uF  p\        VRRR7      NK  	  up4       \        P                  ! RR7      Vn        V# u upi )rD  FrE  rH  Trx   r)  )rç   r	   rK  rL  r   r8  r+  r'   rÞ   r   rå   rŽ  r  r   r
   rä   )rf   rç   r’   s   &  r"   r¼   ÚTikTokenConverter.converted·  sÊ   € Ø—N‘NÓ$ˆ	Ü"0×"9Ò"9ä×$Ò$¤U¨4¯<©<Ó%8À:ÐV[Ô\Ü×(Ò(¸$×:OÑ:OÐ[`Ôaðó#
ˆ	Ôô %×.Ò.Ó0ˆ	Ôà×$Ñ$Ò0Ø×(Ñ(ØPT×PiÒPiÓjÑPiÀu”˜E¨e¸T×BÑPiÑjôô $.×#7Ò#7ÀUÔ#Kˆ	Ô àÐùò ks   Â9C5)r'   rŽ  r8  r€  )Nzs(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+FN)r‡   r–   r—   r˜   r™   rj   r°  r•  r¥  rç   r¼   rš   r›   rœ   s   @r"   rŒ  rŒ  z  s<   ø‡ € ñô
ð ÷/ó ð/÷ð ò0÷ö r$   rŒ  ÚAlbertTokenizerÚBartTokenizerÚBarthezTokenizerÚBertTokenizerÚBigBirdTokenizerÚBlenderbotTokenizerÚCamembertTokenizerÚCLIPTokenizerÚCodeGenTokenizerÚConvBertTokenizerÚDebertaTokenizerÚDebertaV2TokenizerÚDistilBertTokenizerÚDPRReaderTokenizerÚDPRQuestionEncoderTokenizerÚDPRContextEncoderTokenizerÚElectraTokenizerÚFNetTokenizerÚFunnelTokenizerÚGPT2TokenizerÚHerbertTokenizerÚLayoutLMTokenizerÚLayoutLMv2TokenizerÚLayoutLMv3TokenizerÚLayoutXLMTokenizerÚLongformerTokenizerÚLEDTokenizerÚLxmertTokenizerÚMarkupLMTokenizerÚMBartTokenizerÚMBart50TokenizerÚMPNetTokenizerÚMobileBertTokenizerÚMvpTokenizerÚNllbTokenizerÚOpenAIGPTTokenizerÚPegasusTokenizerÚQwen2TokenizerÚReformerTokenizerÚRemBertTokenizerÚRobertaTokenizerÚRoFormerTokenizerÚSeamlessM4TTokenizerÚSqueezeBertTokenizerÚT5TokenizerÚUdopTokenizerÚVideoPrismTokenizerÚWhisperTokenizerÚXLMRobertaTokenizerÚXLNetTokenizerÚSplinterTokenizerÚXGLMTokenizerÚLlamaTokenizerÚCodeLlamaTokenizerÚGemmaTokenizerÚPhi3Tokenizerc                ó$   € V ^8„  d   QhR\         /# rm   r·   )r   s   "r"   r+   r+     s   € ÷ (ñ (Ì)ñ (r$   c                ó²  € V P                   P                  pV\        9   d,   V'       g$   \        V,          pV! V 4      P                  4       # V P                  p\        V\        4      '       dn   \        P                  P                  V4      '       dI   \        V4      '       d8   ^RIHp W n        \        P                  R4       V! V4      P                  4       #  \        P                  R4       \!        V P                  V P"                  R7      P                  4       #   \$         d+    \'        R\)        \        P+                  4       4       24      hi ; i)a\  
Utilities to convert a slow tokenizer instance in a fast tokenizer instance.

Args:
    transformer_tokenizer ([`~tokenization_utils_base.PreTrainedTokenizer`]):
        Instance of a slow tokenizer to convert in the backend tokenizer for
        [`~tokenization_utils_base.PreTrainedTokenizerFast`].
   from_tiktoken (bool, optional): Whether to use the `tiktoken` library to convert the tokenizer instead of sentencepiece.
        Defaults to False.

Return:
    A instance of [`~tokenizers.Tokenizer`] to be used as the backend tokenizer of a
    [`~tokenization_utils_base.PreTrainedTokenizerFast`]
)ÚMistralConverterz#Converting from Mistral tekken.jsonzConverting from Tiktoken)r€  rŽ  zÌConverting from SentencePiece and Tiktoken failed, if a converter for SentencePiece is available, provide a model path with a SentencePiece tokenizer.model file.Currently available slow->fast converters: )r…  r‡   ÚSLOW_TO_FAST_CONVERTERSr¼   r€  rr  r*   ÚosÚpathÚisfiler   Úintegrations.mistral.tokenizerrç  r2   ÚloggerÚinforŒ  rŽ  r•  r—  rq   r  )Útransformer_tokenizerÚfrom_tiktokenÚtokenizer_class_nameÚconverter_classr€  rç  s   &&    r"   Úconvert_slow_tokenizerró    s  € ð  1×:Ñ:×CÑCÐØÔ6Ô6¿}Ü1Ð2FÕGˆÙÐ4Ó5×?Ñ?ÓAÐAà&×1Ñ1€JÜ�*œc×"Ò"¤r§w¡w§~¡~°j×'AÒ'AÔF^Ð_i×FjÒFjÝDà3HÔ0Ü�‰Ð9Ô:Ù 
Ó+×5Ñ5Ó7Ð7ð	Ü�K‰KÐ2Ô3Ü$Ø0×;Ñ;Ø%:×%OÑ%Oô÷ ‰i‹kðøô ô 	Üð>ä>BÔCZ×C_ÑC_ÓCaÓ>bÐ=cðeóð ð	ús   ÃAD! Ä!5E)r  r  r  r  r  r  r  r  r  r  r!  r#  r%  r'  r)  r+  r-  r/  r1  r3  r5  r7  r9  r;  r=  )rU  rV  rW  rX  rY  rZ  r[  r\  r]  r^  r_  r`  ra  rb  rc  rd  re  rf  rg  rh  ri  rj  rk  rl  rm  rn  ro  )r$  r©   )F)Vr™   ré  r‚  Úcollections.abcr   Ú	packagingr   Ú
tokenizersr   r   r   r   r   r	   r
   r‚   r   r   r   Úintegrations.mistral.constantsr   Úutilsr   r   r   r   Úutils.import_utilsr   Ú
get_loggerr‡   rí  rO  rx  r#   r4   rW   rY   rž   r°   r²   r¿   rï   rÿ   r  r  r  r2  r?  rO  rZ  rd  rn  rµ  rÑ  rÞ  rû  r  rS  r{  r‹  r–  r   r¨  r¬  r²  r´  rÌ  rÞ  rã  rç  rô  rø  r   r  r  r  r)  rB  rH  rV  ro  rw  rŠ  rŒ  rè  ró  r:   r$   r"   Ú<module>rû     s�  ðñó 
Û Ý &å ß f× fÑ fß 5Ñ 5å Dß `Ó `Ý 5ð 
×	Ò	˜HÓ	%€ò€ð8 $ò 'õ Ð ô>Gõ"÷÷02ñ 2ôjÐ"8ô õ"I÷$ñ $ô$�Iô $ôN/˜	ô /ôd$�iô $ôN$�Yô $ôN˜ô ô6#�Iô #ôL�yô ô>)�Yô )ôX�yô ô:$˜	ô $ôN�yô ô>x�9ô xôv"
�lô "
ôJ
�|ô 
ô -˜ô -ô`
˜ô 
ôBG�\ô GôT-�|ô -ô`,�Lô ,ô^
˜<ô 
ô2,˜,ô ,ô^"
�\ô "
ôJ	˜ô 	ô
�|ô 
ô@	˜lô 	ô;x�|ô ;xô|�,ô ô8
�Lô 
ô˜+ô ô
!�yô !ôH	
�|ô 	
ô(�Iô (ôV$˜)ô $ôN˜)ô ô:
�Lô 
ô61
�\ô 1
ôh+�\ô +ô\#˜	ô #ôL&m�\ô &môRV
�lô V
ôr-˜ô -ô`
Ð'ô 
ò÷0Nñ Nðb9Ø�ð9àÐ%ð9ð Ð(ð9ð �]ð	9ð
 Ð(ð9ð Ð.ð9ð Ð,ð9ð �]ð9ð ˜ð9ð ˜ð9ð Ð(ð9ð Ð,ð9ð ˜=ð9ð ˜-ð9ð " =ð9ð  ! -ð!9ð" ˜ñ#9ð$ �_ð%9ð& �ð'9ð( �]ð)9ð* Ð(ð+9ð, ˜ð-9ð. ˜=ð/9ð0 Ð+ð19ð2 Ð-ð39ð4 Ð+ð59ð6 Ð$ð79ð8 �}ð99ð: Ð*ð;9ð< �nð=9ð> Ð(ð?9ð@ �nðA9ðB ˜=ðC9ðD Ð$òE9ðF �]ðG9ðH Ð,ðI9ðJ Ð(ðK9ðL �nðM9ðN Ð*ðO9ðP Ð(ðQ9ðR Ð(ðS9ðT Ð*ðU9ðV Ð0ðW9ðX ˜MðY9ðZ �;ð[9ð\ �]ð]9ð^ Ð.ð_9ð` Ð(ða9ðb Ð.ðc9ðd �nðe9ðf Ð*ñg9ðh �]Ø�nØ˜.Ø�nØ�^ñq9Ð ÷x(ñ (r$   