+
    œ°³jÇ  ã            	       óÖ  € R t ^ RIt^ RIt^RIHt ^RIHt ]P                  ! ]	4      t
RRRR/t/ RR	bR
RbRRbRRbRRbRRbRRbRRbRRbRRbRRbRRbR R!bR"R#bR$R%bR&R'bR(R)b/ R*R+bR,R-bR.R/bR0R1bR2R3bR4R5bR6R7bR8R9bR:R;bR<R=bR>R?bR@RAbRBRCbRDREbRFRGbRHRIbRJRKbC/ RLRMbRNRObRPRQbRRRSbRTRUbRVRWbRXRYbRZR[bR\R]bR^R_bR`RabRbRcbRdRebRfRgbRhRibRjRkbRlRmbCRnRoRpRqRrRsRtRu/CtRv t ! Rw Rx]4      tRx.tR# )yz)Tokenization classes for Salesforce CTRL.N)ÚPreTrainedTokenizer)ÚloggingÚ
vocab_filez
vocab.jsonÚmerges_filez
merges.txtÚ	Pregnancyiµ’ ÚChristianityiû  ÚExplaini·Ÿ ÚFitnessiÐ÷  ÚSavingi»ö  ÚAski#j  ÚAssiñv ÚJokeiµ~ Ú	Questionsi6²  ÚThoughtsiÅÁ  ÚRetailivÌ  ÚFeminismiò� ÚWritingiØ.  ÚAtheismiï ÚNetflixiè½  Ú	Computingi×š  ÚOpinioniÍ¨  ÚAlonei§¯  ÚFunnyi%æ  ÚGamingi¦�  ÚHumaniø  ÚIndiai3  ÚJokeriR- ÚDietin�  ÚLegaliS.  ÚNormaniK  ÚTipiñ ÚWeightiwÌ  ÚMoviesiÁ´  ÚRunningi�[  ÚSciencei*  ÚHorrori¡“  Ú
Confessioniœì  ÚFinanceiÚ/  ÚPoliticsiè?  ÚScaryiñí ÚSupportin1  ÚTechnologiesi  ÚTeenageip ÚEventi€  ÚLearnedi„ ÚNotioniòÉ Ú	WikipediaiÏ’  ÚBooksi	  ÚExtracti) ÚConfessionsi-‘ Ú
Conspiracyiœ( ÚLinksiºø  Ú	Narcissusi™K ÚRelationshipiîÕ  ÚRelationshipsiŒ ÚReviewsiÇ¢  ÚNewsi   ÚTranslationiÄh  Úmultilinguali–õ c                óŠ   € \        4       pV ^ ,          pV R,           F  pVP                  W#34       TpK  	  \        V4      pV# )zy
Return set of symbol pairs in a word.

Word is represented as tuple of symbols (symbols being variable-length strings).
:é   NN)ÚsetÚadd)ÚwordÚpairsÚ	prev_charÚchars   &   Úk/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/ctrl/tokenization_ctrl.pyÚ	get_pairsrF   [   sH   € ô ‹E€EØ�Q•€IØ�R—�ˆØ�	‰	�9Ð#Ô$ØŠ	ñ ô �‹J€EØ€Ló    c                   óv   a a€ ] tR t^kt oRt]t]tRV 3R llt	]
R 4       tR tR tR tR tR tR	 tR
tVtV ;t# )ÚCTRLTokenizera0  
Construct a CTRL tokenizer. Based on Byte-Pair-Encoding.

This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
this superclass for more information regarding those methods.

Args:
    vocab_file (`str`):
        Path to the vocabulary file.
    merges_file (`str`):
        Path to the merges file.
    unk_token (`str`, *optional*, defaults to `"<unk>"`):
        The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
        token instead.
c                ó¶  <€ \        VR R7      ;_uu_ 4       p\        P                  ! V4      V n        RRR4       V P                  P	                  4        UUu/ uF  w  rgWvbK	  	  uppV n        \        VR R7      ;_uu_ 4       pVP                  4       P                  R4      ^R p	RRR4       X	 U
u. uF  p
\        V
P                  4       4      NK  	  p	p
\        \        V	\        \        V	4      4      4      4      V n        / V n        RV n        \         SV `D  ! RRVRRRRR	R
/VB  R#   + '       g   i     Lÿ; iu uppi   + '       g   i     L¥; iu up
i )zutf-8)ÚencodingNÚ
TÚ	unk_tokenÚtoken_type_ids_patternÚ	all_zerosÚ%token_type_ids_include_special_tokensÚspecial_tokens_patternÚnoneéÿÿÿÿ© )ÚopenÚjsonÚloadÚencoderÚitemsÚdecoderÚreadÚsplitÚtupleÚdictÚzipÚrangeÚlenÚ	bpe_ranksÚcacheÚadd_bpe_version_headerÚsuperÚ__init__)Úselfr   r   rM   ÚkwargsÚvocab_handleÚkÚvÚmerges_handleÚmergesÚmergeÚ	__class__s   &&&&,      €rE   rf   ÚCTRLTokenizer.__init__   s  ø€ Ü�* w×/Õ/°<ÜŸ9š9 \Ó2ˆDŒL÷ 0à)-¯©×);Ñ);Ô)=Ô>Ñ)=¡ ˜šÑ)=Ò>ˆŒÜ�+¨×0Õ0°MØ"×'Ñ'Ó)×/Ñ/°Ó5°a¸Ð;ˆF÷ 1á4:Ó;±F¨5”%˜Ÿ™›Ö&±FˆÐ;Üœc &¬%´°F³Ó*<Ó=Ó>ˆŒØˆŒ
Ø&*ˆÔ#Ü‰Òñ 	
Øð	
à#.ð	
ð 37ð	
ð $*ð		
ð
 ô	
÷ 0×/üã>ß0×0üâ;s#   —D*ÁD=Â#EÂ2"EÄ*D:	ÅE	c                ó,   € \        V P                  4      # ©N)ra   rX   ©rg   s   &rE   Ú
vocab_sizeÚCTRLTokenizer.vocab_size‘   s   € ä�4—<‘<Ó Ð rG   c                óB   € \        V P                  3/ V P                  B # rr   )r^   rX   Úadded_tokens_encoderrs   s   &rE   Ú	get_vocabÚCTRLTokenizer.get_vocab•   s   € Ü�D—L‘LÑ> D×$=Ñ$=Ñ>Ð>rG   c                ó”  a € VS P                   9   d   S P                   V,          # \        V4      p\        \        VR R 4      VR,          R,           .,           4      p\        V4      pV'       g   V#  \	        VV 3R lR7      pVS P
                  9  d   MçVw  rV. p^ pV\        V4      8  d¤    VP                  WX4      p	VP                  W(V	 4       T	pY(,          T8X  dN   T\        T4      ^,
          8  d7   Y(^,           ,          T8X  d#   TP                  YV,           4       T^,          pK‘  TP                  Y(,          4       T^,          pK³  \        V4      pTp\        V4      ^8X  d   M\        V4      pEK  RP                  V4      pVR R pVS P                   V&   V#   \         d    TP                  Y(R  4        Kt  i ; i)Nz</w>c                 óN   <€ SP                   P                  V \        R 4      4      # )Úinf)rb   ÚgetÚfloat)Úpairrg   s   &€rE   Ú<lambda>Ú#CTRLTokenizer.bpe.<locals>.<lambda>£   s   ø€ °·±×1CÑ1CÀDÌ%ÐPUË,Ô1WrG   )Úkeyú@@ rS   éüÿÿÿ)rc   r]   ÚlistrF   Úminrb   ra   ÚindexÚextendÚ
ValueErrorÚappendÚjoin)
rg   ÚtokenrA   rB   ÚbigramÚfirstÚsecondÚnew_wordÚiÚjs
   f&        rE   ÚbpeÚCTRLTokenizer.bpe˜   s‹  ø€ Ø�D—J‘JÔØ—:‘:˜eÕ$Ð$Ü�U‹|ˆÜ”T˜$˜s ˜)“_¨¨R­°6Õ(9Ð':Õ:Ó;ˆÜ˜$“ˆçØˆLàÜ˜Ô$WÔXˆFØ˜TŸ^™^Ô+ØØ"‰MˆEØˆHØˆAØ”c˜$“i”-ðØŸ
™
 5Ó,�Að
 —O‘O D¨1 IÔ.Ø�Aà•7˜eÔ#¨¬C°«I¸­MÔ(9¸dÀqÅ5½kÈVÔ>SØ—O‘O E¥NÔ3Ø˜•F’Aà—O‘O D¥GÔ,Ø˜•F’AÜ˜X“ˆHØˆDÜ�4‹y˜AŒ~Øä! $›“Ø�z‰z˜$ÓˆØ�C�RˆyˆØ ˆ�
‰
�5ÑØˆøô- "ô Ø—O‘O D¨ HÔ-Úðús   Â0F% Æ%GÇGc                óº   € . p\         P                  ! RV4      pV F;  pVP                  \        V P	                  V4      P                  R4      4      4       K=  	  V# )zTokenize a string.z\S+\n?Ú )ÚreÚfindallrˆ   r…   r“   r\   )rg   ÚtextÚsplit_tokensÚwordsrŒ   s   &&   rE   Ú	_tokenizeÚCTRLTokenizer._tokenizeÄ   sM   € àˆä—
’
˜9 dÓ+ˆãˆEØ×Ñ¤ T§X¡X¨e£_×%:Ñ%:¸3Ó%?Ó @ÖAñ àÐrG   c                ó~   € V P                   P                  WP                   P                  V P                  4      4      # )z0Converts a token (str) in an id using the vocab.)rX   r}   rM   )rg   rŒ   s   &&rE   Ú_convert_token_to_idÚ"CTRLTokenizer._convert_token_to_idÎ   s*   € à�|‰|×Ñ §|¡|×'7Ñ'7¸¿¹Ó'GÓHÐHrG   c                óL   € V P                   P                  WP                  4      # )z=Converts an index (integer) in a token (str) using the vocab.)rZ   r}   rM   )rg   r‡   s   &&rE   Ú_convert_id_to_tokenÚ"CTRLTokenizer._convert_id_to_tokenÒ   s   € à�|‰|×Ñ §~¡~Ó6Ð6rG   c                ód   € RP                  V4      P                  RR4      P                  4       pV# )z:Converts a sequence of tokens (string) in a single string.r–   rƒ   Ú )r‹   ÚreplaceÚstrip)rg   ÚtokensÚ
out_strings   && rE   Úconvert_tokens_to_stringÚ&CTRLTokenizer.convert_tokens_to_stringÖ   s,   € à—X‘X˜fÓ%×-Ñ-¨e°RÓ8×>Ñ>Ó@ˆ
ØÐrG   )rd   rb   rc   rZ   rX   )z<unk>)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__ÚVOCAB_FILES_NAMESÚvocab_files_namesÚCONTROL_CODESÚcontrol_codesrf   Úpropertyrt   rx   r“   rœ   rŸ   r¢   rª   Ú__static_attributes__Ú__classdictcell__Ú__classcell__)ro   Ú__classdict__s   @@rE   rI   rI   k   sU   ù‡ € ñð  *ÐØ!€M÷
ð$ ñ!ó ð!ò?ò*òXòIò7÷ò rG   rI   )r°   rV   Úregexr—   Útokenization_pythonr   Úutilsr   Ú
get_loggerr¬   Úloggerr±   r³   rF   rI   Ú__all__rT   rG   rE   Ú<module>rÀ      s  ðñ 0ã ã å 6Ý ð 
×	Ò	˜HÓ	%€ð �,Ø�<ðÐ ð8Ø�ð8à�Dð8ð ˆvð8ð ˆuð	8ð
 ˆeð8ð 
ˆ5ð8ð 
ˆ5ð8ð ˆFð8ð �ð8ð �ð8ð ˆeð8ð �ð8ð ˆuð8ð ˆvð8ð ˆuð8ð  �ð!8ð" ˆuñ#8ð$ ˆUð%8ð& ˆUð'8ð( ˆeð)8ð* ˆTð+8ð, ˆTð-8ð. ˆUð/8ð0 ˆEð18ð2 ˆUð38ð4 ˆdð58ð6 
ˆ5ð78ð8 ˆeð98ð: ˆeð;8ð< ˆuð=8ð> ˆtð?8ð@ ˆeðA8ðB �%ðC8ðD ˆuòE8ðF �ðG8ðH ˆVðI8ðJ ˆuðK8ðL �EðM8ðN ˆuðO8ðP ˆUðQ8ðR ˆuðS8ðT ˆfðU8ðV �ðW8ðX ˆTðY8ðZ ˆuð[8ð\ �6ð]8ð^ �%ð_8ð` ˆUða8ðb �ðc8ðd �Eðe8ðf �Vñg8ðh ˆuØ
ˆDØ�5Ø�Fño8€òvô nÐ'ô nðn Ð
‚rG   