+
    œ°³j2
  ã                   ó2   € R t ^ RIHtHtHt  ! R R4      tR# )z Tokenization utils for RoFormer.)ÚNormalizedStringÚPreTokenizedStringÚnormalizersc                   óV   a € ] tR t^t o V 3R lR ltV 3R lR ltV 3R lR ltRtV tR# )	ÚJiebaPreTokenizerc                ó   <€ V ^8„  d   QhRR/# )é   ÚreturnN© )ÚformatÚ__classdict__s   "€Úp/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/roformer/tokenization_utils.pyÚ__annotate__ÚJiebaPreTokenizer.__annotate__   s   ø€ ÷ ñ  ñ ó    c                óš   € Wn         \        P                  ! R RR R R7      V n         ^ RIpY n        R#   \         d    \	        R4      hi ; i)FT)Ú
clean_textÚhandle_chinese_charsÚstrip_accentsÚ	lowercaseNzkYou need to install rjieba to use RoFormerTokenizer. See https://pypi.org/project/rjieba/ for installation.)Úvocabr   ÚBertNormalizerÚrjiebaÚImportErrorÚjieba)Úselfr   r   s   && r   Ú__init__ÚJiebaPreTokenizer.__init__   sY   € ØŒ
Ü&×5Ò5ØØ!%ØØô	
ˆÔð	Ûð Ž
øô ô 	ÜðIóð ð	ús	   §3 ³A
c                ó<   <€ V ^8„  d   QhRS[ RS[RS[S[,          /# )r   ÚiÚnormalized_stringr	   )Úintr   Úlist)r   r   s   "€r   r   r   %   s)   ø€ ÷ ñ ™Sð Ñ5Eð É$ÑO_ÕJ`ñ r   c                óz  € . pV P                   P                  \        V4      R R7       FŽ  w  rEpW@P                  9   d   VP	                  W%V 4       K+  V P
                  P                  V4      P                  4       pV F4  pV'       g   K  V\        V4      ,           pVP	                  W%V 4       TpK6  	  K�  	  V# )F)Úhmm)	r   ÚtokenizeÚstrr   Úappendr   Únormalize_strÚsplitÚlen)r   r   r    ÚsplitsÚtokenÚstartÚendÚ
token_lists   &&&     r   Újieba_splitÚJiebaPreTokenizer.jieba_split%   sª   € Øˆð "&§¡×!4Ñ!4´SÐ9JÓ5KÐQVÐ!4Ö!WÑˆE˜#ØŸ
™
Ô"Ø—‘Ð/°cÐ:Ö;à!×-Ñ-×;Ñ;¸EÓB×HÑHÓJ�
Û'�Eß‘uØ#¤c¨%£jÕ0˜ØŸ™Ð&7¸cÐ&BÔCØ #šó	 (ñ "Xð* ˆr   c                ó    <€ V ^8„  d   QhRS[ /# )r   Úpretok)r   )r   r   s   "€r   r   r   @   s   ø€ ÷ 'ñ 'Ñ#5ñ 'r   c                ó<   € VP                  V P                  4       R # )N)r)   r0   )r   r3   s   &&r   Úpre_tokenizeÚJiebaPreTokenizer.pre_tokenize@   s   € Ø�‰�T×%Ñ%Ö&r   )r   r   r   N)	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r0   r5   Ú__static_attributes__Ú__classdictcell__)r   s   @r   r   r      s#   ø‡ € ÷ð ÷"ð ÷6'ö 'r   r   N)Ú__doc__Ú
tokenizersr   r   r   r   r
   r   r   Ú<module>r?      s   ðñ 'ç HÑ H÷.'ó .'r   