+
    œ°³j  ã                   óŠ   € ^ RI HtHtHtHt ^ RIHt ^RIHt ^RI	H
t
 ]
P                  ! ]4      tRR/t ! R R]4      tR.tR	# )
é    )Ú	TokenizerÚdecodersÚnormalizersÚpre_tokenizers)ÚBPE)ÚTokenizersBackend)ÚloggingÚtokenizer_fileztokenizer.jsonc                   óZ   a a€ ] tR t^t oRt]tRtRR.t]	t
RV3R lV 3R llltRtVtV ;t# )	ÚGemmaTokenizeru…  
Construct a fast Gemma tokenizer (backed by HuggingFace's tokenizers library).

This tokenizer uses a BPE model with byte fallback, no prefix space, and a normalizer that replaces
spaces with "â–�".

Args:
    tokenizer_file (`str`, optional):
        A tokenizers JSON file containing the serialization of a tokenizer.
    unk_token (`str`, optional, defaults to "<unk>"):
        The unknown token.
    bos_token (`str`, optional, defaults to "<bos>"):
        The beginning of sequence token.
    eos_token (`str`, optional, defaults to "<eos>"):
        The end of sequence token.
    pad_token (`str`, optional, defaults to "<pad>"):
        The padding token.
    mask_token (`str`, optional, defaults to "<mask>"):
        The mask token.
    add_bos_token (`bool`, optional, defaults to True):
        Whether or not to add a `bos_token` at the start of sequences.
    add_eos_token (`bool`, optional, defaults to False):
        Whether or not to add an `eos_token` at the end of sequences.
    vocab (`str` or `dict[str, int]`, optional):
        Custom vocabulary dict. If not provided, a minimal vocabulary is created using the special tokens.
ÚleftÚ	input_idsÚattention_maskc                ó¦   <€ V ^8„  d   QhRS[ S[S[ S[3,          ,          R,          RS[ S[S[ ,          ,          R,          RS[ RS[ RS[ RS[ RS[ /# )	é   ÚvocabNÚmergesÚ	unk_tokenÚ	bos_tokenÚ	eos_tokenÚ	pad_tokenÚ
mask_token)ÚstrÚdictÚintÚlist)ÚformatÚ__classdict__s   "€Úm/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/gemma/tokenization_gemma.pyÚ__annotate__ÚGemmaTokenizer.__annotate__;   sp   ø€ ÷ /
ñ /
á‘T™#™s˜(•^Õ# dÕ*ð/
ñ ‘d™3•i• $Õ&ð/
ñ ð	/
ñ
 ð/
ñ ð/
ñ ð/
ñ ñ/
ó    c                ó¾  <€ Vf:   \        V4      ^ \        V4      ^\        V4      ^\        V4      ^\        V4      ^/pWn        T;'       g    . V n        \        \	        V P                  V P                  R\        V4      R RR7      4      V n        \        P                  ! RRRR7      V P
                  n        \        P                  ! \        P                  ! RR4      \        P                  ! 4       \        P                  ! 4       .4      V P
                  n        \        P                  ! RR4      V P
                  n        \"        S	V `H  ! RRVR	VR
VRVRV/VB  R # )NT)r   r   Úfuse_unkr   ÚdropoutÚbyte_fallbackÚ Úmerged_with_previousF)ÚpatternÚbehaviorÚinvertu   â–�r   r   r   r   r   © )r   Ú_vocabÚ_mergesr   r   Ú
_tokenizerr   ÚSplitÚpre_tokenizerr   ÚSequenceÚReplaceÚByteFallbackÚFuseÚdecoderr   Ú
normalizerÚsuperÚ__init__)
Úselfr   r   r   r   r   r   r   ÚkwargsÚ	__class__s
   &&&&&&&&,€r   r9   ÚGemmaTokenizer.__init__;   sD  ø€ ð Š=ä�I“ Ü�I“ Ü�I“ Ü�I“ Ü�J“ ðˆEð ŒØ—|�| ˆŒä#ÜØ—k‘kØ—|‘|ØÜ˜i›.ØØ"ôó	
ˆŒô )7×(<Ò(<ØÐ"8Àô)
ˆ�‰Ô%ô #+×"3Ò"3Ü×Ò˜e SÓ)¬8×+@Ò+@Ó+BÄHÇMÂMÃOÐTó#
ˆ�‰Ôô &1×%8Ò%8¸¸eÓ%Dˆ�‰Ô"Ü‰Òñ 	
Øð	
àð	
ð  ð	
ð  ð		
ð
 "ð	
ð ô	
r"   )r.   r/   r-   )NNz<unk>z<bos>z<eos>z<pad>z<mask>)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__ÚVOCAB_FILES_NAMESÚvocab_files_namesÚpadding_sideÚmodel_input_namesr   Úmodelr9   Ú__static_attributes__Ú__classdictcell__Ú__classcell__)r<   r   s   @@r   r   r      s7   ù‡ € ñð6 *ÐØ€LØ$Ð&6Ð7ÐØ€E÷/
÷ /
õ /
r"   r   N)Ú
tokenizersr   r   r   r   Útokenizers.modelsr   Útokenization_utils_tokenizersr   Úutilsr	   Ú
get_loggerr>   ÚloggerrC   r   Ú__all__r,   r"   r   Ú<module>rR      sO   ð÷ HÓ GÝ !å >Ý ð 
×	Ò	˜HÓ	%€Ø%Ð'7Ð8Ð ôP
Ð&ô P
ðf Ð
‚r"   