+
    3S·j=•  ã                   óú  € ^ RI t ^ RIt^ RIt^ RIt^ RIt^RIHt R tRs	Rs
R tRsRsR tR tR tR]R	]R
]R
RRRRRRRR/R]R]R]RRRRRRRRR//tR2]R]RRRRR/R3]R]RRRRR/R4]R ]R!RR"RR/R5]R#]R$RR%RR//t]P,                  t]P.                  t]P0                  t]P2                  t]P4                  t]t]t]t ! R& R'4      t ! R( R)4      t ! R* R+4      t  ! R, R-4      t!R. t"]PF                  ]n$        ]PJ                  ]n&        ]"! ]4       ]"! ] 4       ]"! ]!4       ^R/I'H(t(  ]! ] PR                  PU                  ]! ]PV                  PY                  R04      4      R14      4       R#   ]- d     R# i ; i)6é    N)Ú_sentencepiecec                 ó`   € \        V R 4      ;'       d    \        V \        \        34      '       * # )Ú__len__)ÚhasattrÚ
isinstanceÚstrÚbytes)Úxs   &ÚW/home/luca/screen-checking/.venv/lib/python3.14/site-packages/sentencepiece/__init__.pyÚ_is_sequencer      s%   € Ü�1�iÓ ×DÐD¬°A¼¼U°|Ó)DÔ%DÐDó    Fc                  óv   € \         '       g    ^ RIHp  V sRs R# R#   \         d   p\	        R4      ThRp?ii ; i)r   )Úsentencepiece_pb2z_protobuf is required when using out_type='proto'. Please install protobuf: pip install protobufNT)Ú_protobuf_loadedÚsentencepiecer   Ú_sentencepiece_pb2ÚImportError)r   Úes     r   Ú_load_protobufr      sM   € çÓð	Ý7Ø!2Ðð  Òñ øô ô 	Üð@óð ðûð	ús   Ž œ8§3³8c                  óv   € \         '       g
    ^ RIp V sRs \        #   \         d   p\        R4      ThRp?ii ; i)r   NzSnumpy is required when return_type='numpy'. Please install numpy: pip install numpyT)Ú_numpy_loadedÚnumpyÚ_numpyr   )Únpr   s     r   Ú_load_numpyr      sI   € ç‹=ð	ÛØˆFð ˆÜ€Møô ô 	Üð:óð ðûð	ús   Ž œ8§3³8c                 ód   € \        4        \        P                  4       pVP                  V 4       V# ©N)r   r   ÚSentencePieceTextÚParseFromString)Ú
serializedÚspts   & r   Ú_to_sptr"   ,   s)   € ÜÔÜ
×
.Ñ
.Ó
0€CØ×Ñ˜
Ô#Ø€Jr   c                 ód   € \        4        \        P                  4       pVP                  V 4       V# r   )r   r   ÚNBestSentencePieceTextr   )r    Ú	nbest_spts   & r   Ú_to_nbest_sptr&   2   s*   € ÜÔÜ"×9Ñ9Ó;€IØ×Ñ˜jÔ)ØÐr   Ú_EncodeAsIdsÚ_EncodeAsPiecesÚserialized_protoÚ_EncodeAsSerializedProtoÚprotoÚoffset_mappingÚ_EncodeAsOffsetMappingr   Ú_EncodeAsBufferTÚ_EncodeAsIdsBatchÚ_EncodeAsPiecesBatchÚ_EncodeAsSerializedProtoBatchÚ_EncodeAsOffsetMappingBatchÚ_EncodeAsBufferBatchÚ
_DecodeIdsÚ_DecodeIdsAsBytesÚ_DecodeIdsAsSerializedProtoÚ_DecodeAsOffsetMappingÚ_DecodeIdsBatchÚ_DecodeIdsAsBytesBatchÚ _DecodeIdsAsSerializedProtoBatchÚ_DecodeAsOffsetMappingBatchÚ_DecodePiecesÚ_DecodePiecesAsBytesÚ_DecodePiecesAsSerializedProtoÚ_DecodePiecesBatchÚ_DecodePiecesAsBytesBatchÚ#_DecodePiecesAsSerializedProtoBatchc                   ór  a € ] tR t^ˆt o RLR ltR tRMR lt]R 4       t]R 4       t	RNR lt
R tR	 tROR
 ltR tR tR tR tR tR tR tRNR ltRNR ltRNR ltRNR ltRNR ltRNR ltRPR ltRMR ltRMR ltRMR ltRMR ltRMR lt RMR lt!RQR lt"RNR  lt#RNR! lt$RNR" lt%RNR# lt&RNR$ lt'RRR% lt(R& t)R' t*R( t+R) t,R* t-R+ t.RSR, lt/RTR- lt0]13R. lt2]13R/ lt3RUR0 lt4RUR1 lt5R2 t6R3 t7R4 t8R5 t9R6 t:RMR7 lt;R8 t<R9 t=R: t>R; t?R< t@R= tAR> tBR? tCR@ tDRA tERB tFRC tGRD tHRE tI]=tJ]=tKRF tLRG tMRH tNRI tORJ tPRKtQV tRR# )VÚSentencePieceProcessorNc                ó  € \         P                  ! 4       V n        V P                  WÃ\        R 7      V n        W@n        WPn        W`n        Wpn	        W€n
        W�n        W n        W°n        V'       g	   V'       d   V P                  WR7       R# R# )©Údefault)Ú
model_fileÚmodel_protoN)r   rC   Ú
_processorÚ_resolve_return_typeÚintÚ_return_typeÚ_add_bosÚ_add_eosÚ_reverseÚ_emit_unk_pieceÚ_enable_samplingÚ_nbest_sizeÚ_alphaÚ_num_threadsÚLoad)ÚselfrG   rH   Úout_typeÚadd_bosÚadd_eosÚreverseÚemit_unk_pieceÚenable_samplingÚ
nbest_sizeÚalphaÚnum_threadsÚreturn_types   &&&&&&&&&&&&&r   Ú__init__ÚSentencePieceProcessor.__init__‰   ss   € ô )×?Ò?ÓAˆŒð !×5Ñ5°kÔUXÐ5ÓYˆÔØŒØŒØŒØ-ÔØ /ÔØ%ÔØŒØ'ÔßŸØ�I‰I ˆIÖEñ %r   c                ó´   € Vf   V P                   pVR8X  d   \        R4      h\        \        \        RRRR3pW9  d   \        RP                  W4      4      hV# )NÚimmutable_protoúWimmutable_proto is deprecated and no longer supported. Use return_type='proto' instead.r+   r)   r   r,   z*Invalid return_type: {}. Must be one of {})rL   Ú
ValueErrorrK   r   r	   Úformat)rV   r`   Úvalid_typess   && r   Ú!_resolve_and_validate_return_typeÚ8SentencePieceProcessor._resolve_and_validate_return_type¥   sb   € ØÒØ×+Ñ+ˆKØÐ+Ô+ÜÐvÓwÐwäœC¤¨Ð1CÀWÐN^Ð_ˆØÔ)ÜÐI×PÑPÐQ\ÓjÓkÐkØÐr   c                óz   € Ve   Ve   \        R4      hTpVf   Ve   TMV P                  pV P                  V4      # )NzFCannot specify both out_type and return_type. Use return_type instead.)rf   rL   ri   )rV   r`   rW   rF   s   &&&&r   rJ   Ú+SentencePieceProcessor._resolve_return_type°   sH   € ØÒØÒ&Ü Ð!iÓjÐjØ"ˆKØÒØ%,Ò%8™'¸d×>OÑ>OˆKØ×5Ñ5°kÓBÐBr   c                ój   € RV9   g   RV9   d   \        R4      hV ! R/ VB pVP                  V4       V# )zCCreates a SentencePieceProcessor by loading a model file from disk.rG   rH   zXmodel_file and model_proto cannot be passed via kwargs when using from_file classmethod.© )rf   ÚLoadFromFile)ÚclsrG   ÚkwargsÚ	processors   &&, r   Ú	from_fileÚ SentencePieceProcessor.from_file¹   s>   € ð ˜6Ô! ]°fÔ%<ÜÐwÓxÐxÙ‘M˜&‘Mˆ	Ø×Ñ˜zÔ*ØÐr   c                ój   € RV9   g   RV9   d   \        R4      hV ! R/ VB pVP                  V4       V# )zQCreates a SentencePieceProcessor by loading a serialized model proto from memory.rG   rH   zYmodel_file and model_proto cannot be passed via kwargs when using from_proto classmethod.rn   )rf   ÚLoadFromSerializedProto)rp   rH   rq   rr   s   &&, r   Ú
from_protoÚ!SentencePieceProcessor.from_protoÂ   s>   € ð ˜6Ô! ]°fÔ%<ÜÐxÓyÐyÙ‘M˜&‘Mˆ	Ø×)Ñ)¨+Ô6ØÐr   c                ó²   € V'       d   V'       d   \        R 4      hV'       d   V P                  V4      # V'       d   V P                  V4      # \        R4      h)z-model_file and model_proto must be exclusive.z3Either model_file or model_proto must be specified.)rf   rv   ro   )rV   rG   rH   s   &&&r   rU   ÚSentencePieceProcessor.LoadË   sG   € ßŸ+ÜÐLÓMÐMßØ×/Ñ/°Ó<Ð<ßØ×$Ñ$ ZÓ0Ð0ÜÐNÓOÐOr   c                ó8   € V P                   P                  V4      # r   )rI   ro   )rV   Úfilenames   &&r   ro   Ú#SentencePieceProcessor.LoadFromFileÔ   s   € Ø�‰×+Ñ+¨HÓ5Ð5r   c                ó|   € \        VR 4      '       d   VP                  4       pV P                  P                  V4      # )ÚSerializeToString)r   r   rI   rv   )rV   r    s   &&r   rv   Ú.SentencePieceProcessor.LoadFromSerializedProto×   s3   € Ü�:Ð2×3Ò3Ø#×5Ñ5Ó7ˆJØ�‰×6Ñ6°zÓBÐBr   c           	     óÐ  € V P                  W¼4      pVe   VR8w  d   \        R4      hVf   V P                  pVf   V P                  pVf   V P                  pVf   V P
                  pVf   V P                  pVf   V P                  pVf   V P                  pV	f   V P                  p	VRJ d"   Ve   V^ 8X  g   V^8X  g   Vf   \        R4      hV	e   \        V	\        4      '       g   \        R4      h\        V\        4      p\        P                  V4      pV'       d   W¿9  d   \        RP!                  V4      4      hWû,          p\#        V P$                  V4      pV\&        \(        39   pT;'       g    VR
9   pV'       d	   WW¦WxW#V.	pMWWxW#V.pV'       d   VP+                  V4       V'       d   V\(        J pVP+                  V4       MpVR8X  dj   VfS   \        V\(        4      ;'       g:    \        V\        4      ;'       d"    T;'       d    \        V^ ,          \(        4      pMTpVP+                  V4       V! V!  pVR8X  d]   \-        4       pV'       d.   V Uu. uF   pVP/                  VVP0                  R	7      NK"  	  up# VP/                  VVP0                  R	7      # VR8X  d/   V'       d   V Uu. uF  p\3        V4      NK  	  up# \3        V4      # V# u upi u upi )Nr,   ú@return_bytes is only supported when return_type='offset_mapping'Ta  When enable_sampling is True, We must specify "nbest_size > 1" or "nbest_size = -1", and "alpha". "nbest_size" is enabled only on unigram mode ignored in BPE-dropout. when "nbest_size = -1" , this method samples from all candidates on the lattice instead of nbest segmentations.únum_threads must be intúunknown return_type={}r+   r   ©Údtype)r)   r+   r,   )rJ   rf   rM   rN   rO   rP   rQ   rR   rS   rT   r   rK   Ú	TypeErrorÚlistÚ_ENCODE_DISPATCHÚgetrg   ÚgetattrrI   r   r	   Úappendr   Ú
frombufferÚint32r"   )rV   ÚinputrX   rY   rZ   r[   r\   r]   r^   r_   Úthread_poolr`   rW   Úreturn_bytesÚis_batchÚmethodsÚmethod_nameÚmethodÚis_pieces_methodÚhas_unkÚargsÚreturn_bytes_valÚraw_resr   ÚbufÚss   &&&&&&&&&&&&&&            r   ÚEncodeÚSentencePieceProcessor.EncodeÜ   sØ  € ð ×/Ñ/°ÓFˆØÒ#¨Ð7GÔ(GÜÐ_Ó`Ð`ØŠ?Ø—m‘mˆGØŠ?Ø—m‘mˆGØŠ?Ø—m‘mˆGØÒ!Ø!×1Ñ1ˆNØÒ"Ø"×3Ñ3ˆOØÒØ×)Ñ)ˆJØŠ=Ø—K‘KˆEØÒØ×+Ñ+ˆKà˜dÓ"¨
Ò(:¸jÈA¼oØ)3°q¬¸EºMÜð2óð ð Ò¤j°¼c×&BÒ&BÜÐ5Ó6Ð6ä˜e¤TÓ*ˆô #×&Ñ& xÓ0ˆß˜+Ô4ÜÐ5×<Ñ<¸[ÓIÓJÐJàÕ*ˆÜ˜Ÿ™¨+Ó6ˆð
 '¬3´¨,Ñ6ÐØ"×dÐd kÐ5dÑ&dˆßØ¨ÀjÐY`ÐkrÐs‰Dà¨J¸wÐQXÐYˆDßØ�K‰K˜Ô'ßØ +¬uÐ 4ÐØ�K‰KÐ(Õ)ØÐ,Ô,ØÒ#ä#-¨e´UÓ#;÷  $Cð  $CÄ
È5ÔRVÓ@W÷  ABð  ABÐ\a÷  ABð  ABÔfpÐqvÐwxÕqyô  |Aó  gBÑ à#/Ð Ø�K‰KÐ(Ô)á˜$‘-ˆð ˜'Ô!ä“ˆBßÙFMÓNÁg¸s˜Ÿ™ c°·±˜Ö:ÁgÑNÐNØ—=‘= °·±�=Ó9Ð9Ø˜GÔ#çÙ,3Ó4©G qœ ž
©GÑ4Ð4Ü˜7Ó#Ð#àˆùò Oùò
 5s   É!&KÊ:K#c                ó6   € V P                   ! RR VR\        /VB # ©r�   r`   rn   ©r�   r   ©rV   r�   rq   s   &&,r   ÚEncodeAsPiecesÚ%SentencePieceProcessor.EncodeAsPieces>  ó   € Ø�{Š{ÑB ÐB´CÐB¸6ÑBÐBr   c                ó6   € V P                   ! RR VR\        /VB # r    ©r�   rK   r¢   s   &&,r   ÚEncodeAsIdsÚ"SentencePieceProcessor.EncodeAsIdsA  r¥   r   c                ó.   € V P                   ! RR VRR/VB # ©r�   r`   r   rn   ©r�   r¢   s   &&,r   ÚEncodeAsNumpyÚ$SentencePieceProcessor.EncodeAsNumpyD  ó   € Ø�{Š{ÑF ÐF°GÐF¸vÑFÐFr   c                ó.   € V P                   ! RR VRR/VB # ©r�   r`   r)   rn   r¬   r¢   s   &&,r   ÚEncodeAsSerializedProtoÚ.SentencePieceProcessor.EncodeAsSerializedProtoG  s   € Ø�{Š{ÑQ ÐQÐ4FÐQÈ&ÑQÐQr   c                ó.   € V P                   ! RR VRR/VB # ©r�   r`   r+   rn   r¬   r¢   s   &&,r   ÚEncodeAsProtoÚ$SentencePieceProcessor.EncodeAsProtoJ  r¯   r   c                ó.   € V P                   ! RR VRR/VB # )r�   r`   r,   rn   r¬   r¢   s   &&,r   ÚEncodeAsOffsetMappingÚ,SentencePieceProcessor.EncodeAsOffsetMappingM  s   € Ø�{Š{ÑO ÐOÐ4DÐOÈÑOÐOr   c                ó   € \        R 4      h©re   ©rf   r¢   s   &&,r   ÚEncodeAsImmutableProtoÚ-SentencePieceProcessor.EncodeAsImmutableProtoP  ó   € ÜÐrÓsÐsr   c                óB   € V P                   ! RR VRVRVR\        RR/VB # ©r�   r]   r^   r`   r\   Trn   r¡   ©rV   r�   r]   r^   rq   s   &&&&,r   ÚSampleEncodeAsPiecesÚ+SentencePieceProcessor.SampleEncodeAsPiecesS  óH   € Ø�{Š{ñ L ð L°:ð LÀUð LÜ'*ðLØ<@ðLØDJñLð 	Lr   c                óB   € V P                   ! RR VRVRVR\        RR/VB # rÂ   r§   rÃ   s   &&&&,r   ÚSampleEncodeAsIdsÚ(SentencePieceProcessor.SampleEncodeAsIdsW  rÆ   r   c                ó:   € V P                   ! RR VRVRVRRRR/VB # )r�   r]   r^   r`   r   r\   Trn   r¬   rÃ   s   &&&&,r   ÚSampleEncodeAsNumpyÚ*SentencePieceProcessor.SampleEncodeAsNumpy[  óH   € Ø�{Š{ñ P ð P°:ð PÀUð PØ'.ðPØ@DðPØHNñPð 	Pr   c                ó:   € V P                   ! RR VRVRVRRRR/VB # )r�   r]   r^   r`   r)   r\   Trn   r¬   rÃ   s   &&&&,r   ÚSampleEncodeAsSerializedProtoÚ4SentencePieceProcessor.SampleEncodeAsSerializedProto_  sH   € Ø�{Š{ñ [ ð [°:ð [ÀUð [Ø'9ð[ØKOð[ØSYñ[ð 	[r   c                ó:   € V P                   ! RR VRVRVRRRR/VB # )r�   r]   r^   r`   r+   r\   Trn   r¬   rÃ   s   &&&&,r   ÚSampleEncodeAsProtoÚ*SentencePieceProcessor.SampleEncodeAsProtoc  rÍ   r   c                ó   € \        R 4      hr¼   r½   rÃ   s   &&&&,r   ÚSampleEncodeAsImmutableProtoÚ3SentencePieceProcessor.SampleEncodeAsImmutableProtog  rÀ   r   c	                ó   a aaaaaaa€ S P                  SV4      oSf   S P                  oSf   S P                  oSf   S P                  oSf   S P                  oSf   S P
                  oS^ 8:  d   ^oSR8X  dK   \        4       oVVVVVV 3R lp	\        V\        4      '       d   V U
u. uF
  q©! V
4      NK  	  up
# V	! V4      # VVVVVVV 3R lp\        V\        4      '       d   V U
u. uF
  q«! V
4      NK  	  up
# V! V4      # u up
i u up
i )Nr   c                 ó¦   <€ SP                   P                  V SSSS4      pV Uu. uF   pSP                  VSP                  R 7      NK"  	  up# u upi ©r…   )rI   Ú_NBestEncodeAsBufferr�   rŽ   )	ÚtextÚraw_buffersr›   rX   rY   r]   r   rZ   rV   s	   &  €€€€€€r   Ú
_encode_npÚ6SentencePieceProcessor.NBestEncode.<locals>._encode_np…  sL   ø€ Ø"Ÿo™o×BÑBØ˜* g¨w¸óA�áFQÓRÁk¸s˜Ÿ™ c°·±˜Ö:ÁkÑRÐRùÒRs   ¥&Ac           	      ó¶  <€ S\         J d    S	P                  P                  V SSSS4      # S\        J g   S\        J d*   S\        J pS	P                  P                  V SSSSSV4      # SR 8X  d!   S	P                  P                  V SSSSS4      # SR8X  d,   S	P                  P                  V SSSSS4      p\        V4      # \        RP                  S4      4      h©r)   r+   r„   )
rK   rI   Ú_NBestEncodeAsIdsr   r	   Ú_NBestEncodeAsPiecesÚ_NBestEncodeAsSerializedProtor&   rf   rg   )
rÛ   r‘   r    rX   rY   r[   r]   r`   rZ   rV   s
   &  €€€€€€€r   Ú_encodeÚ3SentencePieceProcessor.NBestEncode.<locals>._encodeŽ  sì   ø€ ØœcÓ!Ø—‘×8Ñ8Ø˜* g¨w¸óAð AàœcÓ! [´EÓ%9Ø +¬uÐ 4�Ø—‘×;Ñ;Ø˜* g¨w¸ÀÐQ]ó_ð _àÐ0Ô0Ø—‘×DÑDØ˜* g¨w¸ÀóQð Qà˜gÔ%Ø!Ÿ_™_×JÑJØ˜* g¨w¸ÀóQ�
ä$ ZÓ0Ð0ÜÐ5×<Ñ<¸[ÓIÓJÐJr   )	rJ   rM   rN   rO   rP   rR   r   r   rˆ   )rV   r�   rX   rY   rZ   r[   r]   r`   rW   rÝ   Únrä   r   s   f&ffffff&   @r   ÚNBestEncodeÚ"SentencePieceProcessor.NBestEncodej  s  ÿ€ ð ×/Ñ/°¸XÓFˆØŠ?Ø—m‘mˆGØŠ?Ø—m‘mˆGØŠ?Ø—m‘mˆGØÒ!Ø!×1Ñ1ˆNØÒØ×)Ñ)ˆJà˜Œ?ØˆJà˜'Ô!Ü“ˆB÷Sò Sô
 ˜%¤×&Ò&Ù/4Ó5©u¨!˜
 1ž©uÑ5Ð5Ù˜eÓ$Ð$÷	Kó 	Kô" �eœT×"Ò"Ù(-Ó.© 1�G˜A–J©Ñ.Ð.Ù�u‹~Ðùò- 6ùò* /s   Â*DÃ+Dc           	     ó:   € V P                   ! RR VRVR\        /VB # ©r�   r]   r`   rn   )rç   r   ©rV   r�   r]   rq   s   &&&,r   ÚNBestEncodeAsPiecesÚ*SentencePieceProcessor.NBestEncodeAsPieces£  ó'   € Ø×ÒÑ^ eÐ^¸
Ð^ÔPSÐ^ÐW]Ñ^Ð^r   c           	     ó:   € V P                   ! RR VRVR\        /VB # rê   )rç   rK   rë   s   &&&,r   ÚNBestEncodeAsIdsÚ'SentencePieceProcessor.NBestEncodeAsIds¦  rî   r   c           	     ó2   € V P                   ! RR VRVRR/VB # )r�   r]   r`   r   rn   ©rç   rë   s   &&&,r   ÚNBestEncodeAsNumpyÚ)SentencePieceProcessor.NBestEncodeAsNumpy©  ó'   € Ø×ÒÑb eÐb¸
ÐbÐPWÐbÐ[aÑbÐbr   c           	     ó2   € V P                   ! RR VRVRR/VB # )r�   r]   r`   r)   rn   ró   rë   s   &&&,r   ÚNBestEncodeAsSerializedProtoÚ3SentencePieceProcessor.NBestEncodeAsSerializedProto¬  s'   € Ø×ÒÑm eÐm¸
ÐmÐPbÐmÐflÑmÐmr   c           	     ó2   € V P                   ! RR VRVRR/VB # )r�   r]   r`   r+   rn   ró   rë   s   &&&,r   ÚNBestEncodeAsProtoÚ)SentencePieceProcessor.NBestEncodeAsProto¯  rö   r   c                ó   € \        R 4      hr¼   r½   rë   s   &&&,r   ÚNBestEncodeAsImmutableProtoÚ2SentencePieceProcessor.NBestEncodeAsImmutableProto²  rÀ   r   c           
     óú  a aaaaaaaa	a
€ S P                  S
V4      o
Sf   S P                  oSf   S P                  oSf   S P                  oSf   S P                  oSf   ^oSf   RoSf   RoS	f   Ro	S^ 8:  d   \        R4      hS	'       d   S'       g   \        R4      hS
R8X  d   \        R4      hVVVVV	VV
VV V3
R lp\        V\        4      '       d   V Uu. uF
  qÜ! V4      NK  	  up# V! V4      # u upi )Ng      ð?Fznum_samples must be positivez8When include_best is True, We must specify "wor = True".r   z:numpy return_type is not supported in SampleEncodeAndScorec                 óÎ  <
€ S	\         J d#   SP                  P                  V SSSSSSS
4      # S	\        J g   S	\        J d-   S	\        J pSP                  P                  V SSSSSSS
SV4
      # S	R 8X  d$   SP                  P                  V SSSSSSS
S4	      # S	R8X  d/   SP                  P                  V SSSSSSS
S4	      p\        V4      # \        RP                  S	4      4      hrà   )
rK   rI   Ú_SampleEncodeAndScoreAsIdsr   r	   Ú_SampleEncodeAndScoreAsPiecesÚ&_SampleEncodeAndScoreAsSerializedProtor&   rf   rg   )rÛ   r‘   r    rX   rY   r^   r[   Úinclude_bestÚnum_samplesr`   rZ   rV   Úwors   &  €€€€€€€€€€r   rä   Ú<SentencePieceProcessor.SampleEncodeAndScore.<locals>._encodeÛ  s  ø€ ØœcÓ!Ø—‘×AÑAØ˜+ u¨c°<ÀÈ'ÐSZó\ð \àœcÓ! [´EÓ%9Ø +¬uÐ 4�Ø—‘×DÑDØ˜+ u¨c°<ÀÈ'ÐSZÐ\jÐlxózð zàÐ0Ô0Ø—‘×MÑMØ˜+ u¨c°<ÀÈ'ÐSZÐ\jólð là˜gÔ%Ø!Ÿ_™_×SÑSØ˜+ u¨c°<ÀÈ'ÐSZÐ\jól�
ä$ ZÓ0Ð0ÜÐ5×<Ñ<¸[ÓIÓJÐJr   )rJ   rM   rN   rO   rP   rf   r   rˆ   )rV   r�   rX   rY   rZ   r[   r  r^   r  r  r`   rW   rä   ræ   s   f&fffffffff&  r   ÚSampleEncodeAndScoreÚ+SentencePieceProcessor.SampleEncodeAndScoreµ  sù   ÿù€ ð ×/Ñ/°¸XÓFˆØŠ?Ø—m‘mˆGØŠ?Ø—m‘mˆGØŠ?Ø—m‘mˆGØÒ!Ø!×1Ñ1ˆNØÒØˆKØŠ=ØˆEØŠ;ØˆCØÒØ ˆLà˜!ÔÜÐ;Ó<Ð<ß§ÜÐWÓXÐXà˜'Ô!ÜÐYÓZÐZ÷	Kö 	Kô" �eœT×"Ò"Ù(-Ó.© 1�G˜A–J©Ñ.Ð.Ù�u‹~Ðùò /s   ÃC8c                ó>   € V P                   ! RR VRVRVR\        /VB # ©r�   r  r^   r`   rn   )r	  r   ©rV   r�   r  r^   rq   s   &&&&,r   ÚSampleEncodeAndScoreAsPiecesÚ3SentencePieceProcessor.SampleEncodeAndScoreAsPiecesð  ó-   € Ø×(Ò(Ñv¨uÐvÀ+ÐvÐUZÐvÔhkÐvÐouÑvÐvr   c                ó>   € V P                   ! RR VRVRVR\        /VB # r  )r	  rK   r  s   &&&&,r   ÚSampleEncodeAndScoreAsIdsÚ0SentencePieceProcessor.SampleEncodeAndScoreAsIdsó  r  r   c                ó6   € V P                   ! RR VRVRVRR/VB # )r�   r  r^   r`   r)   rn   ©r	  r  s   &&&&,r   Ú%SampleEncodeAndScoreAsSerializedProtoÚ<SentencePieceProcessor.SampleEncodeAndScoreAsSerializedProtoö  sF   € Ø×(Ò(ñ  F¨uð  FÀ+ð  FÐUZð  FÐhzð  Fð  Eñ  Fð  	Fr   c                ó6   € V P                   ! RR VRVRVRR/VB # )r�   r  r^   r`   r+   rn   r  r  s   &&&&,r   ÚSampleEncodeAndScoreAsProtoÚ2SentencePieceProcessor.SampleEncodeAndScoreAsProtoù  s-   € Ø×(Ò(Ñz¨uÐzÀ+ÐzÐUZÐzÐhoÐzÐsyÑzÐzr   c                ó   € \        R 4      hr¼   r½   r  s   &&&&,r   Ú$SampleEncodeAndScoreAsImmutableProtoÚ;SentencePieceProcessor.SampleEncodeAndScoreAsImmutableProtoü  rÀ   r   c           	     ó¼  a aaaaaaaa	a€ S P                  S	V
4      o	Sf   S P                  oSf   S P                  oSf   S P                  oSf   S P                  oSf   S P
                  oSe   \        S\        4      '       g   \        R4      hSe   \        S\        4      '       d   S^ 8:  d   \        R4      hS	R8X  dM   \        4       oVVVVVVV V3R lp\        V\        4      '       d   V Uu. uF
  qË! V4      NK  	  up# V! V4      # VVVVVV	VV V3	R lp\        V\        4      '       d   V Uu. uF
  qÍ! V4      NK  	  up# V! V4      # u upi u upi )Nrƒ   zchunk_len must be positive.r   c           	      ó€   <€ SP                   P                  V SSS	SSS4      pSP                  VSP                  R 7      # rÙ   )rI   Ú_ParallelEncodeAsBufferr�   rŽ   )
rÛ   r›   rX   rY   Ú	chunk_lenr   r_   rZ   rV   r�   s
   & €€€€€€€€r   rÝ   Ú9SentencePieceProcessor.ParallelEncode.<locals>._encode_np  s?   ø€ Ø—o‘o×=Ñ=Ø˜) [°+¸wÈÐQXóZ�à—}‘} S°·±�}Ó9Ð9r   c                 óÆ  <	€ S\         J d"   S
P                  P                  V SSSSSS	4      # S\        J g   S\        J d,   S\        J pS
P                  P                  V SSSSSS	SV4	      # SR 8X  d#   S
P                  P                  V SSSSSS	S4      # SR8X  d.   S
P                  P                  V SSSSSS	S4      p\        V4      # \        RP                  S4      4      hrà   )
rK   rI   Ú_ParallelEncodeAsIdsr   r	   Ú_ParallelEncodeAsPiecesÚ _ParallelEncodeAsSerializedProtor"   rf   rg   )rÛ   r‘   r    rX   rY   r!  r[   r_   r`   rZ   rV   r�   s   &  €€€€€€€€€r   rä   Ú6SentencePieceProcessor.ParallelEncode.<locals>._encode'  s  ø€ ØœcÓ!Ø—‘×;Ñ;Ø˜) [°+¸wÈÐQXóZð ZàœcÓ! [´EÓ%9Ø +¬uÐ 4�Ø—‘×>Ñ>Ø˜) [°+¸wÈÐQXÐZhÐjvóxð xàÐ0Ô0Ø—‘×GÑGØ˜) [°+¸wÈÐQXÐZhójð jà˜gÔ%Ø!Ÿ_™_×MÑMØ˜) [°+¸wÈÐQXÐZhój�
ä˜zÓ*Ð*ÜÐ5×<Ñ<¸[ÓIÓJÐJr   )rJ   rM   rN   rO   rP   rT   r   rK   r‡   rf   r   rˆ   )rV   r�   rX   rY   rZ   r[   r!  r_   r�   r`   rW   rÝ   ræ   rä   r   s   f&ffffffff&   @r   ÚParallelEncodeÚ%SentencePieceProcessor.ParallelEncodeÿ  s:  ÿù€ ð ×/Ñ/°¸XÓFˆØŠ?Ø—m‘mˆGØŠ?Ø—m‘mˆGØŠ?Ø—m‘mˆGØÒ!Ø!×1Ñ1ˆNØÒØ×+Ñ+ˆKàÒ¤j°¼c×&BÒ&BÜÐ5Ó6Ð6ØÒ¤J¨y¼#×$>Ò$>À)ÈqÄ.ÜÐ:Ó;Ð;à˜'Ô!Ü“ˆB÷:ô :ô
 ˜%¤×&Ò&Ù/4Ó5©u¨!˜
 1ž©uÑ5Ð5Ù˜eÓ$Ð$÷	Kõ 	Kô" �eœT×"Ò"Ù(-Ó.© 1�G˜A–J©Ñ.Ð.Ù�u‹~Ðùò- 6ùò* /s   Ã6EÄ9Ec                ó6   € V P                   ! RR VR\        /VB # r    )r(  r   r¢   s   &&,r   ÚParallelEncodeAsPiecesÚ-SentencePieceProcessor.ParallelEncodeAsPieces<  ó    € Ø×"Ò"ÑJ¨ÐJ¼CÐJÀ6ÑJÐJr   c                ó6   € V P                   ! RR VR\        /VB # r    )r(  rK   r¢   s   &&,r   ÚParallelEncodeAsIdsÚ*SentencePieceProcessor.ParallelEncodeAsIds?  r-  r   c                ó.   € V P                   ! RR VRR/VB # r«   ©r(  r¢   s   &&,r   ÚParallelEncodeAsNumpyÚ,SentencePieceProcessor.ParallelEncodeAsNumpyB  ó    € Ø×"Ò"ÑN¨ÐN¸GÐNÀvÑNÐNr   c                ó.   € V P                   ! RR VRR/VB # r±   r2  r¢   s   &&,r   ÚParallelEncodeAsSerializedProtoÚ6SentencePieceProcessor.ParallelEncodeAsSerializedProtoE  s"   € Ø×"Ò"ÑY¨ÐYÐ<NÐYÐRXÑYÐYr   c                ó.   € V P                   ! RR VRR/VB # rµ   r2  r¢   s   &&,r   ÚParallelEncodeAsProtoÚ,SentencePieceProcessor.ParallelEncodeAsProtoH  r5  r   c                ó   € \        R 4      hr¼   r½   r¢   s   &&,r   ÚParallelEncodeAsImmutableProtoÚ5SentencePieceProcessor.ParallelEncodeAsImmutableProtoK  rÀ   r   c                óÀ  € V P                  W#\        R 7      pVe   VR8w  d   \        R4      hVf   V P                  pVe   \	        V\
        4      '       g   \        R4      hVR8X  d   \        V4      MRpVR8X  d   RMTpV P                  WWEVR7      p	VR8X  d=   \	        V	\        4      '       d   V	 U
u. uF  p
\        V
4      NK  	  up
# \        V	4      # V	# u up
i )rE   r,   r‚   rƒ   Fr+   r)   )r‘   )rJ   r   rf   rT   r   rK   r‡   ÚboolÚ_decode_rawrˆ   r"   )rV   r�   r`   rW   r_   r�   r‘   r™   Úcpp_return_typeÚraw_valrœ   s   &&&&&&&    r   ÚDecodeÚSentencePieceProcessor.DecodeN  sà   € Ø×/Ñ/°ÌsÐ/ÓSˆØÒ#¨Ð7GÔ(GÜÐ_Ó`Ð`ØÒØ×+Ñ+ˆKØÒ¤j°¼c×&BÒ&BÜÐ5Ó6Ð6à1<Ð@PÔ1Pœ4 Ô-ÐV[ÐØ0;¸wÔ0FÑ,ÈKˆØ×"Ñ" 5¸;ÐbrÐ"Ósˆà˜'Ô!Ü˜'¤4×(Ò(Ù,3Ó4©G qœ ž
©GÑ4Ð4Ü˜7Ó#Ð#Øˆùò 5s   Â7Cc                óÀ  € Vf   R# \        V4      '       d   \        V4      ^ 8X  d   R# M.\        V\        \        34      '       d   \        V4      ^ 8X  d   R# R p\        V4      '       * pV'       d   V.p\        VR4      '       d   VP                  ^8„  pM\        V^ ,          4      pV'       d   V! V^ ,          4      p	MV! V4      p	\        P                  W˜34      p
V
'       d   W*9  d   \        RP                  V4      4      hW¢,          p\        V P                  V4      pVR8X  d   V'       d
   V! WWE4      # V! W4      # V'       d
   V! WV4      # V! V4      # )NÚ c                 óˆ   € \        V 4      ^ 8X  d   R# \        V R4      '       d   R# \        V ^ ,          \        \        34      # )r   Fr†   )Úlenr   r   r   r	   )Úseqs   &r   Ú
_is_piecesÚ6SentencePieceProcessor._decode_raw.<locals>._is_piecesk  s5   € Ü�3‹x˜1Œ}ÙÜ�s˜G×$Ò$ÙÜ˜c !�f¤s¬E lÓ3Ð3r   Úndimz$unknown return_type={} or input typer,   )r   rI  r   r   r	   r   rM  Ú_DECODE_DISPATCHrŠ   rf   rg   r‹   rI   )rV   r�   r`   r_   r�   r‘   rK  Ú	is_singler’   Úinput_is_piecesr“   r”   r•   s   &&&&&&       r   rA  Ú"SentencePieceProcessor._decode_rawa  s5  € ØŠ=ÙÜ˜×ÒÜ�5‹z˜QŒÙð ä˜¤¤U˜|×,Ò,´°U³¸q´Ùò	4ô % UÓ+Ô+ˆ	ßØ�GˆEô �5˜&×!Ò!Ø—z‘z A‘~‰Hä# E¨!¥HÓ-ˆH÷ Ù(¨¨q­Ó2‰Oá(¨Ó/ˆOô #×&Ñ&¨Ð'BÓCˆß˜+Ô4ÜÐC×JÑJÈ;ÓWÓXÐXàÕ*ˆÜ˜Ÿ™¨+Ó6ˆàÐ*Ô*ßÙ˜e°+ÓLÐLá˜eÓ2Ð2çÙ˜e°+Ó>Ð>á˜e“}Ð$r   c                ó.   € V P                   ! RR VRV/VB # r    ©rD  ©rV   r�   r`   rq   s   &&&,r   ÚDecodePiecesÚ#SentencePieceProcessor.DecodePieces–  ó   € Ø�{Š{ÑJ ÐJ°KÐJÀ6ÑJÐJr   c                ó.   € V P                   ! RR VRV/VB # r    rS  rT  s   &&&,r   Ú	DecodeIdsÚ SentencePieceProcessor.DecodeIds™  rW  r   c                ó.   € V P                   ! RR VRV/VB # r    rS  rT  s   &&&,r   ÚDecodePiecesAsSerializedProtoÚ4SentencePieceProcessor.DecodePiecesAsSerializedProtoœ  rW  r   c                ó.   € V P                   ! RR VRV/VB # r    rS  rT  s   &&&,r   ÚDecodeIdsAsSerializedProtoÚ1SentencePieceProcessor.DecodeIdsAsSerializedProtoŸ  rW  r   c                ó.   € V P                   ! RR VRR/VB # rµ   rS  r¢   s   &&,r   ÚDecodePiecesAsProtoÚ*SentencePieceProcessor.DecodePiecesAsProto¢  r¯   r   c                ó.   € V P                   ! RR VRR/VB # rµ   rS  r¢   s   &&,r   ÚDecodeIdsAsProtoÚ'SentencePieceProcessor.DecodeIdsAsProto¥  r¯   r   c                ó   € \        R 4      hr¼   r½   r¢   s   &&,r   ÚDecodePiecesAsImmutableProtoÚ3SentencePieceProcessor.DecodePiecesAsImmutableProto¨  rÀ   r   c                ó   € \        R 4      hr¼   r½   r¢   s   &&,r   ÚDecodeIdsAsImmutableProtoÚ0SentencePieceProcessor.DecodeIdsAsImmutableProto«  rÀ   r   c                óÂ   € \        V\        4      '       d+   V Uu. uF  q0P                  P                  W24      NK  	  up# V P                  P                  W4      # u upi r   )r   rˆ   rI   Ú_CalculateEntropy)rV   r�   r^   r
   s   &&& r   ÚCalculateEntropyÚ'SentencePieceProcessor.CalculateEntropy®  sL   € Ü�eœT×"Ò"ÙINÓOÉÀA—O‘O×5Ñ5°aÖ?ÉÑOÐOØ�‰×0Ñ0°Ó>Ð>ùò Ps   ›#Ac                óv  € \        V\        4      '       d]   V'       d+   V Uu. uF  q0P                  P                  V4      NK  	  up# V Uu. uF  q0P                  P	                  V4      NK  	  up# V'       d   V P                  P                  V4      # V P                  P	                  V4      # u upi u upi r   )r   rˆ   rI   Ú_NormalizeWithOffsetsÚ
_Normalize©rV   r�   Úwith_offsetsr
   s   &&& r   Ú	NormalizeÚ SentencePieceProcessor.Normalize³  sŒ   € Ü�eœT×"Ò"ßÙJOÓPÉ%ÀQŸ™×=Ñ=¸aÖ@É%ÑPÐPÙ;@ÓA¹5°a—O‘O×.Ñ.¨qÖ1¹5ÑAÐAßØ—?‘?×8Ñ8¸Ó?Ð?Ø�‰×)Ñ)¨%Ó0Ð0ùò	 QùÚAó   £#B1Á#B6c                óž   € VP                  4        UUu/ uF  w  r#V\        V4      bK  	  pppV P                  P                  V4      # u uppi r   )Úitemsr   rI   Ú_OverrideNormalizerSpec)rV   rq   ÚkeyÚvalueÚ
new_kwargss   &,   r   ÚOverrideNormalizerSpecÚ-SentencePieceProcessor.OverrideNormalizerSpec¼  sA   € Ø8>¿¹¼ÔG¹©*¨#�cœ3˜u›:’o¹ˆ
ÑGØ�‰×6Ñ6°zÓBÐBùó Hs   ”A	c                ó6   € V P                   P                  4       # r   )rI   ÚGetPieceSize©rV   s   &r   r‚  Ú#SentencePieceProcessor.GetPieceSizeÀ  s   € Ø�‰×+Ñ+Ó-Ð-r   c                ó8   € V P                   P                  V4      # r   )rI   Ú	PieceToId©rV   Úpieces   &&r   r†  Ú SentencePieceProcessor.PieceToIdÃ  s   € Ø�‰×(Ñ(¨Ó/Ð/r   c                ó8   € V P                   P                  V4      # r   )rI   Ú	IdToPiece©rV   Úids   &&r   r‹  Ú SentencePieceProcessor.IdToPieceÆ  ó   € Ø�‰×(Ñ(¨Ó,Ð,r   c                ó8   € V P                   P                  V4      # r   )rI   ÚGetScorerŒ  s   &&r   r‘  ÚSentencePieceProcessor.GetScoreÉ  ó   € Ø�‰×'Ñ'¨Ó+Ð+r   c                ó8   € V P                   P                  V4      # r   )rI   Ú	IsUnknownrŒ  s   &&r   r•  Ú SentencePieceProcessor.IsUnknownÌ  r�  r   c                ó8   € V P                   P                  V4      # r   )rI   Ú	IsControlrŒ  s   &&r   r˜  Ú SentencePieceProcessor.IsControlÏ  r�  r   c                ó8   € V P                   P                  V4      # r   )rI   ÚIsUnusedrŒ  s   &&r   r›  ÚSentencePieceProcessor.IsUnusedÒ  r“  r   c                ó8   € V P                   P                  V4      # r   )rI   ÚIsByterŒ  s   &&r   rž  ÚSentencePieceProcessor.IsByteÕ  s   € Ø�‰×%Ñ% bÓ)Ð)r   c                ó6   € V P                   P                  4       # r   )rI   Úunk_idrƒ  s   &r   r¡  ÚSentencePieceProcessor.unk_idØ  ó   € Ø�‰×%Ñ%Ó'Ð'r   c                ó6   € V P                   P                  4       # r   )rI   Úbos_idrƒ  s   &r   r¥  ÚSentencePieceProcessor.bos_idÛ  r£  r   c                ó6   € V P                   P                  4       # r   )rI   Úeos_idrƒ  s   &r   r¨  ÚSentencePieceProcessor.eos_idÞ  r£  r   c                ó6   € V P                   P                  4       # r   )rI   Úpad_idrƒ  s   &r   r«  ÚSentencePieceProcessor.pad_idá  r£  r   c                ó6   € V P                   P                  4       # r   )rI   Úserialized_model_protorƒ  s   &r   r®  Ú-SentencePieceProcessor.serialized_model_protoä  s   € Ø�‰×5Ñ5Ó7Ð7r   c                ó"   € V P                  4       # r   ©r®  rƒ  s   &r   Ú__getstate__Ú#SentencePieceProcessor.__getstate__ê  ó   € Ø×*Ñ*Ó,Ð,r   c                ó"   € V P                  4       # r   r±  rƒ  s   &r   Ú__getstate_for_pickle__Ú.SentencePieceProcessor.__getstate_for_pickle__í  r´  r   c                óH   € V P                  4        V P                  V4       R # r   )ra   rv   ©rV   r®  s   &&r   Ú__setstate__Ú#SentencePieceProcessor.__setstate__ð  s   € Ø�‰ŒØ×$Ñ$Ð%;Ö<r   c                ó"   € V P                  4       # r   )r‚  rƒ  s   &r   r   ÚSentencePieceProcessor.__len__ô  s   € Ø× Ñ Ó"Ð"r   c                ó$   € V P                  V4      # r   )r†  r‡  s   &&r   Ú__getitem__Ú"SentencePieceProcessor.__getitem__÷  s   € Ø�~‰~˜eÓ$Ð$r   )
rM   rN   rS   rP   rQ   rR   rT   rI   rL   rO   )NNNFFFFFéÿÿÿÿgš™™™™™¹?rÁ  Nr   ©NN)NNNNNNNNNNNN)NNNNNNN)
NNNNNNNNNN)	NNNNNNNNN)NNNNN)F)r)   )SÚ__name__Ú
__module__Ú__qualname__Ú__firstlineno__ra   ri   rJ   Úclassmethodrs   rw   rU   ro   rv   r�   r£   r¨   r­   r²   r¶   r¹   r¾   rÄ   rÈ   rË   rÏ   rÒ   rÕ   rç   rì   rð   rô   rø   rû   rþ   r	  r  r  r  r  r  r(  r+  r/  r3  r7  r:  r=  rD  rA  r   rU  rY  r\  r_  rb  re  rh  rk  ro  rv  r  r‚  r†  r‹  r‘  r•  r˜  r›  rž  r¡  r¥  r¨  r«  r®  Ú
piece_sizeÚ
vocab_sizer²  r¶  rº  r   r¿  Ú__static_attributes__Ú__classdictcell__©Ú__classdict__s   @r   rC   rC   ˆ   sÜ  ø‡ € ôFò8	ôCð ñó ðð ñó ðôPò6òCô
`òDCòCòGòRòGòPòtôLôLôPô[ôPôtô7ôr_ô_ôcônôcôtô9ôvwôwôFô{ôtô;òzKòKòOòZòOòtôô&3%ðj /2ô Kð ,/ô KôKôKòGòGòtòtò?ô
1òCò.ò0ò-ò,ò-ò-ò,ò*ò(ò(ò(ò(ò8ð €JØ€Jò-ò-ò=ò#÷%ð %r   rC   c                   ó6   a € ] tR tRt o RR ltR tR tRtV tR# )Ú
_LogStreamiû  Nc                óx   € Wn         V P                   e&   \        P                  P                  4       V n        R # R # r   )ÚostreamÚsysÚstderrÚfilenoÚorig_stream_fileno)rV   rÑ  s   &&r   ra   Ú_LogStream.__init__ü  s+   € ØŒØ�<‰<Ò#Ü&)§j¡j×&7Ñ&7Ó&9ˆDÖ#ñ $r   c                óâ   € V P                   ea   \        P                  ! V P                  4      V n        \        P
                  ! V P                   P                  4       V P                  4       R # R # r   )rÑ  ÚosÚduprÕ  Úorig_stream_dupÚdup2rÔ  rƒ  s   &r   Ú	__enter__Ú_LogStream.__enter__  sI   € Ø�<‰<Ò#Ü#%§6¢6¨$×*AÑ*AÓ#BˆDÔ Ü�GŠG�D—L‘L×'Ñ'Ó)¨4×+BÑ+BÖCñ $r   c                ó0  € V P                   eˆ   \        P                  ! V P                  4       \        P                  ! V P
                  V P                  4       \        P                  ! V P
                  4       V P                   P                  4        R # R # r   )rÑ  rØ  ÚcloserÕ  rÛ  rÚ  )rV   Útyper}  Ú	tracebacks   &&&&r   Ú__exit__Ú_LogStream.__exit__  s_   € Ø�<‰<Ò#Ü�HŠH�T×,Ñ,Ô-Ü�GŠG�D×(Ñ(¨$×*AÑ*AÔBÜ�HŠH�T×)Ñ)Ô*Ø�L‰L×ÑÖ ñ	 $r   )rÚ  rÕ  rÑ  r   )	rÃ  rÄ  rÅ  rÆ  ra   rÜ  râ  rÊ  rË  rÌ  s   @r   rÏ  rÏ  û  s   ø‡ € ô:ò
D÷
!ð !r   rÏ  c                   óH   a € ] tR tRt o ]RR l4       t]RR l4       tRtV tR# )ÚSentencePieceTraineri  Nc                ó˜  € V e6   \        V \        4      '       d    \        P                  P	                  V 4      # R pR pR pR p/ pVP                  4        F0  w  rxVR9   d   TpK  VR9   d   TpK  VR9   d   TpK&  V! V4      Wg&   K2  	  V'       d   VP                  4       VR&   V'       d[   V'       d!   \        P                  P                  Wc4      p	M\        P                  P                  V4      p	VP                  V	4       R # V'       d    \        P                  P                  Wc4      # \        P                  P                  V4      # )Nc                 ó  € \        V \        4      '       dh   \        P                  ! 4       p\        P
                  ! VR R7      pTP                  V  Uu. uF  p\        V4      NK  	  up4       VP                  4       # \        V 4      # u upi )rG  )Úlineterminator)	r   rˆ   ÚioÚStringIOÚcsvÚwriterÚwriterowr   Úgetvalue)r}  Úfrì  Úvs   &   r   rä   Ú,SentencePieceTrainer._Train.<locals>._encode  sd   € Ü˜%¤×&Ò&Ü—K’K“M�ÜŸš A°bÔ9�Ø—‘±Ó 7±¨A¤ Q¦±Ñ 7Ô8Ø—z‘z“|Ð#ä˜5“zÐ!ùò !8s   ÁB	Ú_serialized_normalizer_spec)Úsentence_iteratorÚsentence_reader)Úmodel_writer)Ú
normalizer)r   r   r   rå  Ú_TrainFromStringrz  Úserialized_normalizer_specÚ_TrainFromMap4Ú_TrainFromMap3ÚwriteÚ_TrainFromMap2Ú_TrainFromMap)
Úargrq   rä   ró  rõ  rö  r~  r|  r}  rH   s
   &,        r   Ú_TrainÚSentencePieceTrainer._Train  s   € àŠ?œz¨#¬s×3Ò3Ü!×6Ñ6×GÑGÈÓLÐLò	"ð !ÐØˆØˆ
Øˆ
Ø Ÿ,™,ž.‰JˆCØÐ>Ô>Ø$)Ò!ØÐ(Ô(Ø$’Ø˜Ô&Ø"’
á")¨%£.�
“ñ )÷ Ø8B×8]Ñ8]Ó8_ˆJÐ4Ñ5çß Ü,×AÑA×PÑPÐQ[Óo‘ä,×AÑA×PÑPÐQ[Ó\�Ø×Ñ˜{Ô+ñ ÷	 !Ü%×:Ñ:×IÑIÈ*ÓhÐhä%×:Ñ:×HÑHÈÓTÐTr   c                ó˜   € \        VR 7      ;_uu_ 4        \        P                  ! RRV /VB uuRRR4       #   + '       g   i     R# ; i))rÑ  rþ  Nrn   )rÏ  rå  rÿ  )rþ  Ú	logstreamrq   s   &&,r   ÚTrainÚSentencePieceTrainer.Train;  s2   € ä 	×*Ö*Ü'×.Ò.ÑA°3ÐA¸&ÑA÷ +×*×*Ó*ús	   •8¸A		rn   r   rÂ  )	rÃ  rÄ  rÅ  rÆ  Ústaticmethodrÿ  r  rÊ  rË  rÌ  s   @r   rå  rå    s.   ø‡ € Øó)ó ð)ðV óBó öBr   rå  c                   óR   a € ] tR tRt o RR ltR tR tR tRR ltR t	R	 t
R
tV tR# )ÚSentencePieceNormalizeriA  Nc
                ó2  € \         P                  ! 4       V n        V'       d   V P                  P                  V4       EMV'       d?   \	        VR 4      '       d   VP                  4       pV P                  P                  V4       M½V'       d?   \	        VR 4      '       d   VP                  4       pV P                  P                  V4       MwV'       d   V P                  P                  V4       MSV'       d   V P                  P                  V4       M/V'       d   V P                  P                  V4       M\        R4      hV P                  P                  RV4       V P                  P                  RV4       V P                  P                  RV	4       R# )r   zno model is specifiedÚadd_dummy_prefixÚescape_whitespacesÚremove_extra_whitespacesN)r   r  Ú_normalizerro   r   r   rv   Ú LoadFromSerializedNormalizerSpecÚLoadFromRuleTSVÚLoadFromRuleNameÚLoadFromMaprf   Ú_SetProtoField)
rV   rG   rH   Únormalizer_specÚrule_tsvÚ	rule_nameÚnorm_mapr	  r
  r  s
   &&&&&&&&&&r   ra   Ú SentencePieceNormalizer.__init__B  s+  € ô *×AÒAÓCˆÔçØ×Ñ×)Ñ)¨*Ö5ßÜ�{Ð$7×8Ò8Ø)×;Ñ;Ó=�Ø×Ñ×4Ñ4°[ÕAßÜ�Ð(;×<Ò<Ø"1×"CÑ"CÓ"E�Ø×Ñ×=Ñ=¸oÕNßØ×Ñ×,Ñ,¨XÕ6ßØ×Ñ×-Ñ-¨iÕ8ßØ×Ñ×(Ñ(¨Õ2äÐ4Ó5Ð5à×Ñ×'Ñ'Ð(:Ð<LÔMØ×Ñ×'Ñ'Ð(<Ð>PÔQØ×Ñ×'Ñ'Ð(BÐD\Ö]r   c                ó6   € V P                   P                  4       # r   )r  Ú	Decompilerƒ  s   &r   r  Ú!SentencePieceNormalizer.Decompilee  s   € Ø×Ñ×)Ñ)Ó+Ð+r   c                ó6   € V P                   P                  4       # r   ©r  r®  rƒ  s   &r   r®  Ú.SentencePieceNormalizer.serialized_model_protoh  ó   € Ø×Ñ×6Ñ6Ó8Ð8r   c                ó6   € V P                   P                  4       # r   )r  rø  rƒ  s   &r   rø  Ú2SentencePieceNormalizer.serialized_normalizer_speck  s   € Ø×Ñ×:Ñ:Ó<Ð<r   c                óv  € \        V\        4      '       d]   V'       d+   V Uu. uF  q0P                  P                  V4      NK  	  up# V Uu. uF  q0P                  P	                  V4      NK  	  up# V'       d   V P                  P                  V4      # V P                  P	                  V4      # u upi u upi r   )r   rˆ   r  rr  rs  rt  s   &&& r   rv  Ú!SentencePieceNormalizer.Normalizen  s”   € Ü�eœT×"Ò"ßÙKPÓQÉ5Àa×(Ñ(×>Ñ>¸qÖAÉ5ÑQÐQÙ<AÓB¹E°q×$Ñ$×/Ñ/°Ö2¹EÑBÐBßØ×#Ñ#×9Ñ9¸%Ó@Ð@Ø×Ñ×*Ñ*¨5Ó1Ð1ùò	 RùÚBrx  c                ó6   € V P                   P                  4       # r   r  rƒ  s   &r   r²  Ú$SentencePieceNormalizer.__getstate__w  r  r   c                óp   € \         P                  ! 4       V n        V P                  P                  V4       R # r   )r   r  r  rv   r¹  s   &&r   rº  Ú$SentencePieceNormalizer.__setstate__z  s(   € Ü)×AÒAÓCˆÔØ×Ñ×0Ñ0Ð1GÖHr   )r  )	NNNNNNFFFr   )rÃ  rÄ  rÅ  rÆ  ra   r  r®  rø  rv  r²  rº  rÊ  rË  rÌ  s   @r   r  r  A  s1   ø‡ € ô!^òF,ò9ò=ô2ò9÷Ið Ir   r  c                 ó@  € ^ RI p/ pV P                  P                  4        FT  w  r4VP                  RV4      '       g   K  VP	                  RRV4      P                  4       P                  RR4      pWBV&   KV  	  VP                  4        F  w  r4\        WV4       K  	  R# )r   Nz^[A-Z]+z(?<!^)(?=[A-Z])Ú_Ún_bestÚnbest)ÚreÚ__dict__rz  ÚmatchÚsubÚlowerÚreplaceÚsetattr)Ú	classnamer*  Ú	snake_mapÚkrð  Úsnakes   &     r   Ú_add_snake_caser5  €  s‡   € ÛØ€IØ×"Ñ"×(Ñ(Ö*‰ˆØ�8‰8�J ×"Ô"Ø—F‘FÐ-¨s°AÓ6×<Ñ<Ó>×FÑFÀxÐQXÓYˆEØ �eÓñ +ð —‘Ö!‰ˆÜ�	˜aÖ ó "r   )Ú__version__r   Úpackage_data)FF)FT)TF)TT).rØ  rÒ  rë  ré  Úimportlib.resourcesÚ	importlibrG  r   r   r   r   r   r   r   r   r"   r&   rK   r   r	   r‰   rN  Ú
ThreadPoolÚSetRandomGeneratorSeedÚSetMinLogLevelÚSetNBestTimeoutÚ
SetDataDirÚset_random_generator_seedÚset_min_log_levelÚset_nbest_timeoutrC   rÏ  rå  r  r5  r�   ÚTokenizerD  Ú
DetokenizeÚ_versionr6  ÚpathÚjoinÚ	resourcesÚfilesÚ	Exceptionrn   r   r   Ú<module>rJ     sR  ðÛ 	Û 
Û 
Û 	Û Ý òEð Ð ØÐ ò ð €Ø	€òòòð 
Øˆ^ØÐØÐ ØÐ6ØÐ+ØÐ2ØÐ"ðð 	ØÐ ØÐ#ØÐ%ØÐ;ØÐ0ØÐ7ØÐ'ððÐ ð@ Øˆ\ØÐ"ØÐ9ØÐ2ð	ð ØÐØÐ'ØÐ>ØÐ7ð	ð Øˆ_ØÐ%ØÐ<ØÐ2ð	ð ØÐ!ØÐ*ØÐAØÐ7ð	ð/Ð ð@ ×&Ñ&€
à'×>Ñ>Ð Ø×.Ñ.€Ø ×0Ñ0€Ø×&Ñ&€
à2Ð Ø"Ð Ø#Ð ÷p	%ñ p	%÷f!ñ !÷&0Bñ 0B÷f;Iñ ;Iò~!ð #9×"?Ñ"?Ð Ô Ø$:×$AÑ$AÐ Ô !ñ Ð&Ô 'Ù Ð$Ô %Ù Ð'Ô (õ "ð	Ùˆr�w‰w�|‰|™C 	× 3Ñ 3× 9Ñ 9¸/Ó JÓKÈ^Ó\Ö]øØô 	Úð	ús   Ä,AE/ Å/E:Å9E: