+
    œ°³j,  ã                   ó  a€ R t0 t R t^RIHtHt ^RIHt ]P                  ! ]4      t	Rt
^ tRtRtRtRtRt]R	]R
]R]R]R]R/t] ^ k ]P%                  4        U Uu/ uF  w  rWbK	  	  upp t] ^k  ! R R]4      tR.tR# u upp i )z Tokenization classes for CANINE.)Ú
AddedTokenÚPreTrainedTokenizer)Úloggingi   i à  ià  ià  ià  ià  z[CLS]z[SEP]z[BOS]z[MASK]z[PAD]z
[RESERVED]c                   óø   a a€ ] tR t^7t oRt. ROt]! ]4      ]! ]4      ]! ]4      ]! ]4      ]! ]	4      ]! ]
4      RR3V 3R llt]V3R lR l4       tR tV3R lR	 ltV3R
 lR ltV3R lR ltR tRtVtV ;t# )ÚCanineTokenizeraÉ  
Construct a CANINE tokenizer (i.e. a character splitter). It turns text into a sequence of characters, and then
converts each character into its Unicode code point.

[`CanineTokenizer`] inherits from [`PreTrainedTokenizer`].

Refer to superclass [`PreTrainedTokenizer`] for usage examples and documentation concerning parameters.

Args:
    model_max_length (`int`, *optional*, defaults to 2048):
            The maximum sentence length the model accepts.
Fi   c	                ó<  <€ \        V\        4      '       d   \        VR R R7      MTp\        V\        4      '       d   \        VR R R7      MTp\        V\        4      '       d   \        VR R R7      MTp\        V\        4      '       d   \        VR R R7      MTp\        V\        4      '       d   \        VR R R7      MTp\        V\        4      '       d   \        VRR R7      MTp/ V n        \        P                  4        F  w  r«W P                  V&   K  	  V P                  P                  4        UU
u/ uF  w  rºW«bK	  	  up
pV n        \        V n        \        V P                  4      V n
        \        SV `0  ! RRVRVRVRVRVRVR	VR
VRRRRRR/V	B  R# u up
pi )F)ÚlstripÚrstripTÚ	bos_tokenÚ	eos_tokenÚ	sep_tokenÚ	cls_tokenÚ	pad_tokenÚ
mask_tokenÚadd_prefix_spaceÚmodel_max_lengthÚtoken_type_ids_patternÚ	all_zerosÚ%token_type_ids_include_special_tokensÚspecial_tokens_patternÚcls_sepN© )Ú
isinstanceÚstrr   Ú_special_codepointsÚSPECIAL_CODEPOINTSÚitemsÚ_special_codepoint_stringsÚUNICODE_VOCAB_SIZEÚ_unicode_vocab_sizeÚlenÚ_num_special_tokensÚsuperÚ__init__)Úselfr
   r   r   r   r   r   r   r   ÚkwargsÚ	codepointÚnameÚ	__class__s   &&&&&&&&&,  €Úo/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/canine/tokenization_canine.pyr#   ÚCanineTokenizer.__init__G   sÇ  ø€ ô JTÐT]Ô_b×IcÒIc”J˜y°¸uÕEÐirˆ	ÜISÐT]Ô_b×IcÒIc”J˜y°¸uÕEÐirˆ	ÜISÐT]Ô_b×IcÒIc”J˜y°¸uÕEÐirˆ	ÜISÐT]Ô_b×IcÒIc”J˜y°¸uÕEÐirˆ	ÜISÐT]Ô_b×IcÒIc”J˜y°¸uÕEÐirˆ	ô KUÐU_Ôad×JeÒJe”Z 
°4ÀÕFÐkuˆ
ð 46ˆÔ Ü1×7Ñ7Ö9‰OˆIØ-6×$Ñ$ TÓ*ñ  :ð
 48×3KÑ3K×3QÑ3QÔ3Sô;
Ù3S¡ ˆIŠOÑ3Sò;
ˆÔ'ô $6ˆÔ Ü#& t×'?Ñ'?Ó#@ˆÔ ä‰Òñ 	
Øð	
àð	
ð  ð	
ð  ð		
ð
  ð	
ð "ð	
ð .ð	
ð .ð	
ð $/ð	
ð 37ð	
ð $-ð	
ð ô	
ùó;
s   Ä5Fc                ó    <€ V ^8„  d   QhRS[ /# )é   Úreturn)Úint)ÚformatÚ__classdict__s   "€r)   Ú__annotate__ÚCanineTokenizer.__annotate__y   s   ø€ ÷ (ñ (™Cñ (ó    c                ó   € V P                   # ©N)r   )r$   s   &r)   Ú
vocab_sizeÚCanineTokenizer.vocab_sizex   s   € à×'Ñ'Ð'r3   c                ó¢   € \        V P                  4       Uu/ uF  p\        V4      VbK  	  ppVP                  V P                  4       V# u upi r5   )Úranger6   ÚchrÚupdateÚadded_tokens_encoder)r$   ÚiÚvocabs   &  r)   Ú	get_vocabÚCanineTokenizer.get_vocab|   sE   € Ü$)¨$¯/©/Ô$:Ó;Ñ$:˜q”�Q“˜’Ñ$:ˆÐ;Ø�‰�T×.Ñ.Ô/Øˆùò <s   ˜Ac                ó6   <€ V ^8„  d   QhRS[ RS[S[ ,          /# )r,   Útextr-   )r   Úlist)r/   r0   s   "€r)   r1   r2   �   s   ø€ ÷ ñ ™cð ¡d©3¥iñ r3   c                ó   € \        V4      # )z5Tokenize a string (i.e. perform character splitting).)rC   )r$   rB   s   &&r)   Ú	_tokenizeÚCanineTokenizer._tokenize�   s   € ä�D‹zÐr3   c                ó&   <€ V ^8„  d   QhRS[ RS[/# )r,   Útokenr-   )r   r.   )r/   r0   s   "€r)   r1   r2   …   s   ø€ ÷ :ñ :©#ð :±#ñ :r3   c                óV   €  \        V4      #   \         d    \        RT R24      hi ; i)zaConverts a token (i.e. a Unicode character) in an id (i.e. its integer Unicode code point value).zinvalid token: 'Ú')ÚordÚ	TypeErrorÚ
ValueError)r$   rH   s   &&r)   Ú_convert_token_to_idÚ$CanineTokenizer._convert_token_to_id…   s5   € ð	:Ü�u“:ÐøÜô 	:ÜÐ/°¨w°aÐ8Ó9Ð9ð	:ús   ‚
 �(c                ó&   <€ V ^8„  d   QhRS[ RS[/# )r,   Úindexr-   )r.   r   )r/   r0   s   "€r)   r1   r2   Œ   s   ø€ ÷ 
5ñ 
5©#ð 
5±#ñ 
5r3   c                ó„   €  V\         9   d   \         V,          # \        V4      #   \         d    \        RT 24      hi ; i)z€
Converts a Unicode code point (integer) in a token (str). In case it's a special code point, convert to
human-readable format.
zinvalid id: )r   r:   rL   rM   )r$   rQ   s   &&r)   Ú_convert_id_to_tokenÚ$CanineTokenizer._convert_id_to_tokenŒ   sF   € ð
	5ØÔ*Ô*Ü)¨%Õ0Ð0Ü�u“:ÐøÜô 	5Ü˜|¨E¨7Ð3Ó4Ð4ð	5ús   ‚% š
% ¥?c                ó$   € R P                  V4      # )Ú )Újoin)r$   Útokenss   &&r)   Úconvert_tokens_to_stringÚ(CanineTokenizer.convert_tokens_to_string˜   s   € Ø�w‰w�v‹Ðr3   )r!   r   r   r   )Ú	input_idsÚattention_maskÚtoken_type_ids)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Úmodel_input_namesr:   ÚCLSÚSEPÚPADÚMASKr#   Úpropertyr6   r?   rE   rN   rS   rY   Ú__static_attributes__Ú__classdictcell__Ú__classcell__)r(   r0   s   @@r)   r   r   7   sŒ   ù‡ € ñò JÐñ �c“(Ù�c“(Ù�c“(Ù�c“(Ù�c“(Ù�t“9ØØ÷/
ðb ÷(ó ð(ò÷
ð ÷:ð :÷
5ð 
5÷ò r3   r   c                ó¬   € V ^8„  d   Qh/ ^ \         9   d   \        \        \        3,          ;R&   ^\         9   d   \        \        \        3,          ;R&   # )r,   r   ÚSPECIAL_CODEPOINTS_BY_NAME)Ú__conditional_annotations__Údictr.   r   )r/   s   "r)   r1   r1      sC   € × &Ñ &÷,ò ”Dœœc˜•Nñ ñ- '÷L qÒ pœD¤¤c �NÑ pòM 'r3   N)rn   rb   Útokenization_pythonr   r   Úutilsr   Ú
get_loggerr^   Úloggerr   rf   rd   re   ÚBOSrg   ÚRESERVEDr   r   rm   r   Ú__all__r1   )r&   r'   rn   s   00@r)   Ú<module>rw      sÄ   øðö 'ç BÝ ð 
×	Ò	˜HÓ	%€ð Ð ð €Ø€Ø€Ø€Ø€Ø€ð ˆØˆØˆØˆ(ØˆØˆlð&Ð ó ð  Vh×UmÑUmÔUoÔ-pÑUoÁ/À)¨dªoÑUoÒ-pÐ Ó pôbÐ)ô bðJ Ð
‚ùóQ .qs   ÁB