+
    œ°³j	  ã                   ó–   € R t ^ RIHtHtHtHt ^ RIHt ^RIH	t	 ^RI
Ht ]P                  ! ]4      tRRRRR	R
/t ! R R]	4      tR.tR# )z$Tokenization classes for OpenAI GPT.)Ú	TokenizerÚdecodersÚnormalizersÚpre_tokenizers)ÚBPE)ÚTokenizersBackend)ÚloggingÚ
vocab_filez
vocab.jsonÚmerges_filez
merges.txtÚtokenizer_fileztokenizer.jsonc                   óf   a a€ ] tR t^t oRt]tRR.t]t	RV3R lV 3R lllt
]R 4       tRtVtV ;t# )	ÚOpenAIGPTTokenizera`  
Construct a GPT Tokenizer (backed by HuggingFace's *tokenizers* library). Based on Byte-Pair-Encoding with
the following peculiarities:

- lower case all inputs
- uses BERT's BasicTokenizer for pre-BPE tokenization

This tokenizer inherits from [`TokenizersBackend`] which contains most of the main methods. Users should
refer to this superclass for more information regarding those methods.

Args:
    vocab_file (`str`, *optional*):
        Path to the vocabulary file.
    merges_file (`str`, *optional*):
        Path to the merges file.
    tokenizer_file (`str`, *optional*):
        Path to a tokenizers JSON file containing the serialization of a tokenizer.
    unk_token (`str`, *optional*, defaults to `"<unk>"`):
        The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
        token instead.
    vocab (`str` or `dict[str, int]`, *optional*):
        Custom vocabulary dictionary. If not provided, a blank vocabulary is initialized.
    merges (`str` or `list[str]`, *optional*):
        Custom merges list. If not provided, an empty list is used.
Ú	input_idsÚattention_maskc                óŽ   <€ V ^8„  d   QhRS[ S[S[ S[3,          ,          R,          RS[ S[S[ ,          ,          R,          RS[ /# )é   ÚvocabNÚmergesÚ	unk_token)ÚstrÚdictÚintÚlist)ÚformatÚ__classdict__s   "€Úo/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/openai/tokenization_openai.pyÚ__annotate__ÚOpenAIGPTTokenizer.__annotate__;   sH   ø€ ÷  
ñ  
á‘T™#™s˜(•^Õ# dÕ*ð 
ñ ‘d™3•i• $Õ&ð 
ñ ñ	 
ó    c                óØ  <€ Ve   TM\        V4      ^ /V n        T;'       g    . V n        \        \	        V P                  V P                  R RRR\        V4      R7      4      V n        \        P                  ! RR7      V P
                  n        \        P                  ! 4       V P
                  n        \        P                  ! RR7      V P
                  n        \        SV `@  ! R	RV/VB  R # )
NÚ z</w>F)r   r   ÚdropoutÚcontinuing_subword_prefixÚend_of_word_suffixÚfuse_unkr   T)Ú	lowercase)Úsuffixr   © )r   Ú_vocabÚ_mergesr   r   Ú
_tokenizerr   ÚBertNormalizerÚ
normalizerr   ÚBertPreTokenizerÚpre_tokenizerr   Ú
BPEDecoderÚdecoderÚsuperÚ__init__)Úselfr   r   r   ÚkwargsÚ	__class__s   &&&&,€r   r2   ÚOpenAIGPTTokenizer.__init__;   sÀ   ø€ ð  %Ò0‘e´s¸9³~ÀqÐ6IˆŒØ—|�| ˆŒä#ÜØ—k‘kØ—|‘|ØØ*,Ø#)ØÜ˜i›.ôó

ˆŒô &1×%?Ò%?È$Ô%Oˆ�‰Ô"ä(6×(GÒ(GÓ(Iˆ�‰Ô%Ü"*×"5Ò"5¸VÔ"Dˆ�‰Ôä‰Òñ 	
Øð	
àô	
r   c                ó   € R # )Tr'   )r3   s   &r   Údo_lower_caseÚ OpenAIGPTTokenizer.do_lower_case]   s   € ár   )r)   r*   r(   )NNz<unk>)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__ÚVOCAB_FILES_NAMESÚvocab_files_namesÚmodel_input_namesr   Úmodelr2   Úpropertyr8   Ú__static_attributes__Ú__classdictcell__Ú__classcell__)r5   r   s   @@r   r   r      sG   ù‡ € ñð4 *ÐØ$Ð&6Ð7ÐØ€E÷ 
õ  
ðD ñó ÷ð r   r   N)r>   Ú
tokenizersr   r   r   r   Útokenizers.modelsr   Útokenization_utils_tokenizersr   Úutilsr   Ú
get_loggerr:   Úloggerr?   r   Ú__all__r'   r   r   Ú<module>rN      sZ   ðñ +ç GÓ GÝ !å >Ý ð 
×	Ò	˜HÓ	%€à! <°ÀÐN^Ð`pÐqÐ ôCÐ*ô CðL  Ð
 ‚r   