+
    œ°³jP  ã                   ó¼  € R t ^ RIHt ^ RIHtHtHt ]]]3,          t]	t
 ]]]	,          ]]	,          3,          t ]]
]]
]
3,          ]]
,          3,          t ]]]]]3,          ]],          3,          t ]]
]3,          t ]]]3,          t  ! R R]4      t ! R R]4      t ! R R]4      t^R	IHtHtHtHtHtHtHtHtHtHtHtHtH t H!t! ^R
I"H#t#H$t$H%t%H&t&H't' R# )u  Tokenizers â€” fast, batteries-included tokenization library.

Free-threaded Python (3.14t) note:
    Wheels built against free-threaded CPython declare ``Py_MOD_GIL_NOT_USED``
    and use ``RwLock``-guarded interior mutability so component setters are
    safe to call from multiple threads. Compound mutations
    (``tokenizer.post_processor.special_tokens = â€¦``) are still not atomic â€”
    use a Python lock if you need the read-then-write to be serialized.
    See ``docs/free-threading-audit.md`` for the full analysis.
)ÚEnum)ÚListÚTupleÚUnionc                   ó   € ] tR t^GtRtRtRtR# )ÚOffsetReferentialÚoriginalÚ
normalized© N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__ÚORIGINALÚ
NORMALIZEDÚ__static_attributes__r
   ó    ÚT/home/luca/screen-checking/.venv/lib/python3.14/site-packages/tokenizers/__init__.pyr   r   G   s   † Ø€HØ„Jr   r   c                   ó   € ] tR t^LtRtRtRtR# )Ú
OffsetTypeÚbyteÚcharr
   N)r   r   r   r   ÚBYTEÚCHARr   r
   r   r   r   r   L   s   † Ø€DØ„Dr   r   c                   ó*   € ] tR t^QtRtRtRtRtRtRt	R# )ÚSplitDelimiterBehaviorÚremovedÚisolatedÚmerged_with_previousÚmerged_with_nextÚ
contiguousr
   N)
r   r   r   r   ÚREMOVEDÚISOLATEDÚMERGED_WITH_PREVIOUSÚMERGED_WITH_NEXTÚ
CONTIGUOUSr   r
   r   r   r   r   Q   s   † Ø€GØ€HØ1ÐØ)ÐØ„Jr   r   )Ú
AddedTokenÚEncodingÚNormalizedStringÚPreTokenizedStringÚRegexÚTokenÚ	TokenizerÚdecodersÚmodelsÚnormalizersÚpre_tokenizersÚ
processorsÚtrainersÚ__version__)ÚBertWordPieceTokenizerÚByteLevelBPETokenizerÚCharBPETokenizerÚSentencePieceBPETokenizerÚSentencePieceUnigramTokenizerN)(Ú__doc__Úenumr   Útypingr   r   r   ÚintÚOffsetsÚstrÚTextInputSequenceÚPreTokenizedInputSequenceÚTextEncodeInputÚPreTokenizedEncodeInputÚInputSequenceÚEncodeInputr   r   r   Ú
tokenizersr&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   Úimplementationsr4   r5   r6   r7   r8   r
   r   r   Ú<module>rG      s  ðñ	õ ß %Ñ %ð ��S��/€àÐ Ø 5à! $ s¥)¨U°3­ZÐ"7Õ8Ð ðð ØØ	Ð
Ð.Ð
.Õ/ØÐ	Õðõ€ð
ð  ØØ	Ð
#Ð%>Ð
>Õ?ØÐ	"Õ#ð%õÐ ð
ð Ð'Ð)BÐBÕC€ðð �OÐ%<Ð<Õ=€ðô˜ô ô
�ô ô
˜Tô ÷÷ ÷ ÷ ÷ ÷ r   