+
    œ°³j$  ã                   ó    € R t ^ RIt^ RIHtHtHtHtHt ^ RIH	t	 ^RI
Ht ^RIHt ]P                  ! ]4      tRR/tR t ! R	 R
]4      tR
.tR# )z"Tokenization classes for Splinter.N)Ú	TokenizerÚdecodersÚnormalizersÚpre_tokenizersÚ
processors)Ú	WordPiece)ÚTokenizersBackend)ÚloggingÚ
vocab_filez	vocab.txtc                 ó  € \         P                  ! 4       p\        V R RR7      ;_uu_ 4       pVP                  4       pRRR4       \	        X4       F  w  rEVP                  R4      pWAV&   K  	  V#   + '       g   i     L;; i)Úrzutf-8)ÚencodingNÚ
)ÚcollectionsÚOrderedDictÚopenÚ	readlinesÚ	enumerateÚrstrip)r
   ÚvocabÚreaderÚtokensÚindexÚtokens   &     Ús/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/splinter/tokenization_splinter.pyÚ
load_vocabr      sj   € Ü×#Ò#Ó%€EÜ	ˆj˜#¨×	0Õ	0°FØ×!Ñ!Ó#ˆ÷ 
1ä! &Ö)‰ˆØ—‘˜TÓ"ˆØˆe‹ñ *ð €L÷ 
1×	0ús   ¬A0Á0B 	c                   ól   a a€ ] tR t^)t oRt]tRR.t]t	R	V3R lV 3R lllt
]R 4       tR tRtVtV ;t# )
ÚSplinterTokenizera~  
Construct a Splinter tokenizer (backed by HuggingFace's tokenizers library). Based on WordPiece.

This tokenizer inherits from [`TokenizersBackend`] which contains most of the main methods. Users should
refer to this superclass for more information regarding those methods.

Args:
    vocab_file (`str`, *optional*):
        Path to a vocabulary file.
    tokenizer_file (`str`, *optional*):
        Path to a tokenizers JSON file containing the serialization of a tokenizer.
    do_lower_case (`bool`, *optional*, defaults to `True`):
        Whether or not to lowercase the input when tokenizing.
    unk_token (`str`, *optional*, defaults to `"[UNK]"`):
        The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
        token instead.
    sep_token (`str`, *optional*, defaults to `"[SEP]"`):
        The separator token, which is used when building a sequence from multiple sequences.
    pad_token (`str`, *optional*, defaults to `"[PAD]"`):
        The token used for padding, for example when batching sequences of different lengths.
    cls_token (`str`, *optional*, defaults to `"[CLS]"`):
        The classifier token which is used when doing sequence classification.
    mask_token (`str`, *optional*, defaults to `"[MASK]"`):
        The token used for masking values.
    question_token (`str`, *optional*, defaults to `"[QUESTION]"`):
        The token used for constructing question representations.
    tokenize_chinese_chars (`bool`, *optional*, defaults to `True`):
        Whether or not to tokenize Chinese characters.
    strip_accents (`bool`, *optional*):
        Whether or not to strip all accents. If this option is not specified, then it will be determined by the
        value for `lowercase`.
    vocab (`str`, `dict` or `list`, *optional*):
        Custom vocabulary dictionary. If not provided, a minimal vocabulary is created.
Ú	input_idsÚattention_maskc                ó˜   <€ V ^8„  d   QhRS[ S[S[ S[3,          ,          R,          RS[RS[ RS[ RS[ RS[ RS[ R	S[ R
S[RS[R,          /
# )é   r   NÚdo_lower_caseÚ	unk_tokenÚ	sep_tokenÚ	pad_tokenÚ	cls_tokenÚ
mask_tokenÚquestion_tokenÚtokenize_chinese_charsÚstrip_accents)ÚstrÚdictÚintÚbool)ÚformatÚ__classdict__s   "€r   Ú__annotate__ÚSplinterTokenizer.__annotate__Q   s…   ø€ ÷ :%ñ :%á‘T™#™s˜(•^Õ# dÕ*ð:%ñ ð:%ñ ð	:%ñ
 ð:%ñ ð:%ñ ð:%ñ ð:%ñ ð:%ñ !%ð:%ñ ˜d•{ñ:%ó    c                óò  <€ Ve   TME\        V4      ^ \        V4      ^\        V4      ^\        V4      ^\        V4      ^\        V4      ^R^/V n        \        \        V P                  \        V4      R7      4      V n        \
        P                  ! RV	V
VR7      V P                  n        \        P                  ! 4       V P                  n
        \        P                  ! RR7      V P                  n        \        SV `8  ! RRVRVR	VR
VRVRVRVRV	RV
/	VB  W n        W�n        W n        W€n        V P$                  V P&                  9  d   V P)                  V P$                  .RR7       V P+                  4        R # )NÚ.)r#   T)Ú
clean_textÚhandle_chinese_charsr*   Ú	lowercasez##)Úprefixr#   r$   r%   r&   r'   r(   r"   r)   r*   )Úspecial_tokens© )r+   Ú_vocabr   r   Ú
_tokenizerr   ÚBertNormalizerÚ
normalizerr   ÚBertPreTokenizerÚpre_tokenizerr   ÚdecoderÚsuperÚ__init__r"   r)   r*   r(   Úall_special_tokensÚ
add_tokensÚupdate_post_processor)Úselfr   r"   r#   r$   r%   r&   r'   r(   r)   r*   ÚkwargsÚ	__class__s   &&&&&&&&&&&,€r   rD   ÚSplinterTokenizer.__init__Q   sˆ  ø€ ð  Ò ñ ô �I“ Ü�I“ Ü�I“ Ü�I“ Ü�J“ Ü�NÓ# QØ�Qðð 	Œô $¤I¨d¯k©kÄSÈÃ^Ô$TÓUˆŒä%0×%?Ò%?ØØ!7Ø'Ø#ô	&
ˆ�‰Ô"ô )7×(GÒ(GÓ(Iˆ�‰Ô%Ü"*×"4Ò"4¸DÔ"Aˆ�‰Ôä‰Òñ 	
Øð	
àð	
ð  ð	
ð  ð		
ð
 "ð	
ð *ð	
ð (ð	
ð $:ð	
ð (ð	
ð ò	
ð +ÔØ&<Ô#Ø*ÔØ,ÔØ×Ñ d×&=Ñ&=Ô=Ø�O‰O˜T×0Ñ0Ð1À$ˆOÔGØ×"Ñ"Ö$r3   c                ó8   € V P                  V P                  4      # )N)Úconvert_tokens_to_idsr(   )rH   s   &r   Úquestion_token_idÚ#SplinterTokenizer.question_token_id�   s   € à×)Ñ)¨$×*=Ñ*=Ó>Ð>r3   c           
     ó¤  € V P                   pV P                  pV P                  pR pV P                  pV P                  pV P
                  pV P                  R 4      pVe   Vf   R# V P                  R8X  d   V RV RV RV RV R2
p	MV RV RV RV RV R2
p	\        P                  ! V RV R2V	W3W&3W73WH3.R7      V P                  n        R# )	r5   NÚrightz:0 $A:0 Ú z:0 $B:1 z:1z:0)ÚsingleÚpairr:   )r&   r$   r(   Úcls_token_idÚsep_token_idrN   rM   Úpadding_sider   ÚTemplateProcessingr=   Úpost_processor)
rH   ÚclsÚsepÚquestionÚdotrU   rV   rN   Údot_token_idrT   s
   &         r   rG   Ú'SplinterTokenizer.update_post_processor‘   sÿ   € Ø�n‰nˆØ�n‰nˆØ×&Ñ&ˆØˆØ×(Ñ(ˆØ×(Ñ(ˆØ ×2Ñ2ÐØ×1Ñ1°#Ó6ˆàŠ;˜#š+Ùà×Ñ Ô'Ø�U˜( 8 *¨A¨c¨U°!°C°5¸ÀÀÀRÐH‰Dà�U˜( 3 % x°¨z¸¸3¸%¸qÀÀÀRÐHˆDä)3×)FÒ)FØ�U˜( 3 % rÐ*ØàÐ#ØÐ#ØÐ-ØÐ#ð	ô	*
ˆ�‰Ö&r3   )r=   r<   r"   r(   r*   r)   )
NTz[UNK]z[SEP]z[PAD]z[CLS]z[MASK]z
[QUESTION]TN)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__ÚVOCAB_FILES_NAMESÚvocab_files_namesÚmodel_input_namesr   ÚmodelrD   ÚpropertyrN   rG   Ú__static_attributes__Ú__classdictcell__Ú__classcell__)rJ   r0   s   @@r   r   r   )   sM   ù‡ € ñ!ðF *ÐØ$Ð&6Ð7ÐØ€E÷:%õ :%ðx ñ?ó ð?÷
ò 
r3   r   )rd   r   Ú
tokenizersr   r   r   r   r   Útokenizers.modelsr   Útokenization_utils_tokenizersr   Úutilsr	   Ú
get_loggerr`   Úloggerre   r   r   Ú__all__r;   r3   r   Ú<module>rt      sX   ðñ )ã ç SÕ SÝ 'å >Ý ð 
×	Ò	˜HÓ	%€à! ;Ð/Ð òôC
Ð)ô C
ðL Ð
‚r3   