+
    œ°³jP  ã                   óž   € R t ^ RIHtHtHtHtHtHt ^ RIH	t	 ^RI
Ht ^RIHt ]P                  ! ]4      tRRRRR	R
/t ! R R]4      tR.tR# )zTokenization classes for CLIP.)ÚRegexÚ	TokenizerÚdecodersÚnormalizersÚpre_tokenizersÚ
processors)ÚBPE)ÚTokenizersBackend)ÚloggingÚ
vocab_filez
vocab.jsonÚmerges_filez
merges.txtÚtokenizer_fileztokenizer.jsonc                   ó\   a a€ ] tR t^t oRt]tRR.t]t	RV3R lV 3R lllt
R tRtVtV ;t# )	ÚCLIPTokenizera*  
Construct a CLIP tokenizer (backed by HuggingFace's *tokenizers* library). Based on byte-level
Byte-Pair-Encoding.

This tokenizer inherits from [`TokenizersBackend`] which contains most of the main methods. Users should
refer to this superclass for more information regarding those methods.

Args:
    vocab (`str`, `dict` or `list`, *optional*):
        Vocabulary dict to use for the tokenizer.
    merges (`str` or `list`, *optional*):
        Merges list to use for the BPE tokenizer.
    unk_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
        The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
        token instead.
    bos_token (`str`, *optional*, defaults to `"<|startoftext|>"`):
        The beginning of sequence token.
    eos_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
        The end of sequence token.
    pad_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
        The token used for padding, for example when batching sequences of different lengths.
Ú	input_idsÚattention_maskc                ó    <€ V ^8„  d   QhRS[ S[S[ S[3,          ,          R,          RS[ S[S[ ,          ,          R,          RS[ RS[ RS[ RS[ /# )é   ÚvocabNÚmergesÚ	unk_tokenÚ	bos_tokenÚ	eos_tokenÚ	pad_token)ÚstrÚdictÚintÚlist)ÚformatÚ__classdict__s   "€Úk/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/clip/tokenization_clip.pyÚ__annotate__ÚCLIPTokenizer.__annotate__8   sn   ø€ ÷ E5ñ E5á‘T™#™s˜(•^Õ# dÕ*ðE5ñ ‘d™3•i• $Õ&ðE5ñ ð	E5ñ
 ðE5ñ ðE5ñ ñE5ó    c                ó®  <€ Ve   TM"\        V4      ^ \        V4      ^\        V4      ^/pT;'       g    . V n        \        \        VV P                  R RRR\        V4      R7      4      V n        \
        P                  ! \
        P                  ! 4       \
        P                  ! \        R4      R4      \
        P                  ! 4       .4      V P                  n        \        P                  ! \        P                  ! \        R4      RR	R
7      \        P                  ! RR7      .4      V P                  n        \         P                  ! 4       V P                  n        \$        S	V `L  ! RRVRVRVRV/VB  \(        P*                  ! \        V4      V P,                  3\        V4      V P.                  3RR4      V P                  n        V P3                  4        R # )NÚ z</w>F)r   r   ÚdropoutÚcontinuing_subword_prefixÚend_of_word_suffixÚfuse_unkr   z\s+Ú z[<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+ÚremovedT)ÚbehaviorÚinvert)Úadd_prefix_spacer   r   r   r   © )r   Ú_mergesr   r   Ú
_tokenizerr   ÚSequenceÚNFCÚReplacer   Ú	LowercaseÚ
normalizerr   ÚSplitÚ	ByteLevelÚpre_tokenizerr   ÚdecoderÚsuperÚ__init__r   ÚRobertaProcessingÚeos_token_idÚbos_token_idÚpost_processorÚ%_wrap_decode_method_backend_tokenizer)
Úselfr   r   r   r   r   r   ÚkwargsÚ_vocabÚ	__class__s
   &&&&&&&, €r    r<   ÚCLIPTokenizer.__init__8   s­  ø€ ð Ò ñ ô �I“ Ü�I“ Ü�I“ ðð 	ð —|�| ˆŒä#ÜØØ—|‘|ØØ*,Ø#)ØÜ˜i›.ôó

ˆŒô &1×%9Ò%9Ü�_Š_Ó¤× 3Ò 3´E¸&³MÀ3Ó GÌ×I^ÒI^ÓI`Ðaó&
ˆ�‰Ô"ô )7×(?Ò(?ä×$Ò$ÜØzóð 'Øôô ×(Ò(¸%Ô@ð	ó)
ˆ�‰Ô%ô #+×"4Ò"4Ó"6ˆ�‰Ôä‰Òñ 	
Øð	
àð	
ð  ð	
ð  ð		
ð
 ò	
ô *4×)EÒ)EÜ�‹^˜T×.Ñ.Ð/Ü�‹^˜T×.Ñ.Ð/ØØó	*
ˆ�‰Ô&ð 	×2Ñ2Ö4r#   c                ó¤   aa€ V P                   P                  oV P                   P                  P                  oVV3R  lpWP                   n        R# )c                  óX   <€ S! V / VB pVP                  SR 4      P                  4       pV# )r*   )ÚreplaceÚstrip)ÚargsrC   Útextr(   Úorig_decode_methods   *, €€r    Únew_decode_methodÚNCLIPTokenizer._wrap_decode_method_backend_tokenizer.<locals>.new_decode_method†   s1   ø€ Ù% tÐ6¨vÑ6ˆDØ—<‘<Ð 2°CÓ8×>Ñ>Ó@ˆDØˆKr#   N)Úbackend_tokenizerÚdecodeÚmodelr(   )rB   rN   r(   rM   s   & @@r    rA   Ú3CLIPTokenizer._wrap_decode_method_backend_tokenizer   sB   ù€ Ø!×3Ñ3×:Ñ:Ðð "×3Ñ3×9Ñ9×LÑLÐö	ð
 ):×ÑÖ%r#   )r0   r1   )NNú<|endoftext|>z<|startoftext|>rT   rT   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__ÚVOCAB_FILES_NAMESÚvocab_files_namesÚmodel_input_namesr   rR   r<   rA   Ú__static_attributes__Ú__classdictcell__Ú__classcell__)rE   r   s   @@r    r   r      s:   ù‡ € ñð. *ÐØ$Ð&6Ð7ÐØ€E÷E5õ E5÷N:ò :r#   r   N)rY   Ú
tokenizersr   r   r   r   r   r   Útokenizers.modelsr   Útokenization_utils_tokenizersr	   Úutilsr
   Ú
get_loggerrU   ÚloggerrZ   r   Ú__all__r/   r#   r    Ú<module>rg      sZ   ðñ %ç Z× ZÝ !å >Ý ð 
×	Ò	˜HÓ	%€à! <°ÀÐN^Ð`pÐqÐ ôo:Ð%ô o:ðd Ð
‚r#   