+
    œ°³jŒ  ã                   óf   € R t ^RIHtHt ^RIHt ]P                  ! ]4      t ! R R]4      t	R.t
R# )zTokenization class for Dia.)Ú
AddedTokenÚPreTrainedTokenizer)Úloggingc                   ó”   a a€ ] tR t^t oRtRR.tRV3R lV 3R lllt]R 4       tR t	V3R lR	 lt
R
 tR tV3R lR ltRtVtV ;t# )ÚDiaTokenizera€  
Construct a Dia tokenizer. Dia simply uses raw bytes utf-8 encoding except for special tokens `[S1]` and `[S2]`.

This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
refer to this superclass for more information regarding those methods.

Args:
    pad_token (`str`, *optional*, defaults to `"<pad>"`):
        The token used for padding, for example when batching sequences of different lengths.
    unk_token (`str`, *optional*, defaults to `"<pad>"`):
        The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
        token instead.
    max_length (`int`, *optional*, defaults to 1024):
        The maximum length of the sequences when encoding. Sequences longer than this will be truncated.
    offset (`int`, *optional*, defaults to 0):
        The offset of the tokenizer.
Ú	input_idsÚattention_maskc                ó\   <€ V ^8„  d   QhRS[ R,          RS[ R,          RS[R,          RS[/# )é   Ú	pad_tokenNÚ	unk_tokenÚ
max_lengthÚoffset)ÚstrÚint)ÚformatÚ__classdict__s   "€Úi/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/dia/tokenization_dia.pyÚ__annotate__ÚDiaTokenizer.__annotate__,   s?   ø€ ÷ 
ñ 
á˜•:ð
ñ ˜•:ð
ñ ˜$•Jð	
ñ
 ñ
ó    c                ó(  <€ \        V\        4      '       d   \        V4      MTp\        V\        4      '       d   \        V4      MTpRV n        ^ V^\        R4      ^\        R4      /V n        W@n        \        SV `  ! RRVRVRVRVRRR	R
RR/VB  R# )r
   z[S1]z[S2]r   r   r   r   Útoken_type_ids_patternÚ	all_zerosÚ%token_type_ids_include_special_tokensTÚspecial_tokens_patternÚnoneNé   © )Ú
isinstancer   r   Ú_utf_vocab_sizeÚ_added_tokens_decoderr   ÚsuperÚ__init__)Úselfr   r   r   r   ÚkwargsÚ	__class__s   &&&&&,€r   r#   ÚDiaTokenizer.__init__,   s»   ø€ ô .8¸	Ä3×-GÒ-G”J˜yÔ)ÈYˆ	Ü-7¸	Ä3×-GÒ-G”J˜yÔ)ÈYˆ	à#ˆÔØ&'¨°A´zÀ&Ó7IÈ1ÌjÐY_ÓN`Ð%aˆÔ"ØŒÜ‰Òñ 		
Øð		
àð		
ð "ð		
ð ð			
ð
 $/ð		
ð 37ð		
ð $*ð		
ð ô		
r   c                ó   € V P                   # ©N)r    )r$   s   &r   Ú
vocab_sizeÚDiaTokenizer.vocab_sizeF   s   € à×#Ñ#Ð#r   c                óÎ   € \        V P                  V P                  ,           4       Uu/ uF  qP                  V4      VbK  	  ppVP	                  V P
                  4       V# u upi r)   )Úranger*   r   Úconvert_ids_to_tokensÚupdateÚadded_tokens_encoder)r$   ÚiÚvocabs   &  r   Ú	get_vocabÚDiaTokenizer.get_vocabJ   sX   € Ü;@ÀÇÁÐSW×S^ÑS^ÕA^Ô;_Ó`Ñ;_°a×+Ñ+¨AÓ.°Ò1Ñ;_ˆÐ`Ø�‰�T×.Ñ.Ô/Øˆùò as   ©A"c                ó6   <€ V ^8„  d   QhRS[ RS[S[ ,          /# )r
   ÚtextÚreturn)r   Úlist)r   r   s   "€r   r   r   O   s   ø€ ÷ ñ ™cð ¡d©3¥iñ r   c                ób   € VP                  R4       Uu. uF  p\        V4      NK  	  ppV# u upi )zPTake as input a string and return a list of strings (tokens) for words/sub-wordsúutf-8)ÚencodeÚchr)r$   r6   r1   Útokenss   &&  r   Ú	_tokenizeÚDiaTokenizer._tokenizeO   s/   € à"&§+¡+¨gÔ"6Ó7Ñ"6˜Q”#�a–&Ñ"6ˆÐ7Øˆùò 8s   ”,c                óf   € \        V4      ^8w  d   RpV# \        V4      V P                  ,           pV# )z0Converts a token (str) in an id using the vocab.N)ÚlenÚordr   )r$   ÚtokenÚtoken_ids   && r   Ú_convert_token_to_idÚ!DiaTokenizer._convert_token_to_idT   s4   € ô ˆu‹:˜Œ?ØˆHð ˆô ˜5“z D§K¡KÕ/ˆHàˆr   c                ó<   € \        WP                  ,
          4      pV# )z=Converts an index (integer) in a token (str) using the vocab.)r<   r   )r$   ÚindexrC   s   && r   Ú_convert_id_to_tokenÚ!DiaTokenizer._convert_id_to_token^   s   € ä�EŸK™KÕ'Ó(ˆØˆr   c                ó6   <€ V ^8„  d   QhRS[ S[,          RS[/# )r
   r=   r7   )r8   r   )r   r   s   "€r   r   r   c   s   ø€ ÷ ñ ©t±C­yð ¹Sñ r   c                ó4  € RpV F|  pW0P                   9   d/   V P                   V,          p\        V4      P                  R4      pM3W0P                  9   d   VP                  R4      pMVP                  R4      pW%,          pK~  	  VP	                  RRR7      pV# )z:Converts a sequence of tokens (string) in a single string.r   r:   Úignore)Úerrors)r!   r   r;   Ú_added_tokens_encoderÚdecode)r$   r=   ÚbstringrC   Úadded_token_objÚ
tok_stringÚstrings   &&     r   Úconvert_tokens_to_stringÚ%DiaTokenizer.convert_tokens_to_stringc   s‹   € àˆÛˆEØ×2Ñ2Ô2Ø"&×"<Ñ"<¸UÕ"C�Ü  Ó1×8Ñ8¸ÓA‘
Ø×4Ñ4Ô4Ø"Ÿ\™\¨'Ó2‘
à"Ÿ\™\¨'Ó2�
ØÕ!ŠGñ ð —‘ °�Ó9ˆØˆr   )r!   r    r   )ú<pad>rW   i   é    )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Úmodel_input_namesr#   Úpropertyr*   r3   r>   rE   rI   rU   Ú__static_attributes__Ú__classdictcell__Ú__classcell__)r&   r   s   @@r   r   r      s^   ù‡ € ñð$ %Ð&6Ð7Ð÷
õ 
ð4 ñ$ó ð$ò÷
ð ò
ò÷
÷ ð r   r   N)r]   Útokenization_pythonr   r   Úutilsr   Ú
get_loggerrY   Úloggerr   Ú__all__r   r   r   Ú<module>rh      s<   ðñ "ç BÝ ð 
×	Ò	˜HÓ	%€ôYÐ&ô Yðx Ð
‚r   