+ j<^RIt^RIt^RIt^RIt^RIHt^RIHtHtH t H t H t H t H t ^RIHtHt]P"P%]4t]P"P)]R4t]!]4;_uu_4t]P14tRRR4!RR4t] ],t] ] ],,t!RR ] 4t!R R 4t!R R 4t !RR4t!]3RRllt"R# +'giLg;i)N)Template)AnyCallableDictList NamedTupleOptionalTuple)Encoding Tokenizerzvisualizer-styles.cssc>a]tRt^toV3RlRltV3RltRtVtR#) Annotationc,<V^8dQhRS[RS[RS[/#startendlabelintstr)format __classdict__s"\/home/luca/screen-checking/.venv/lib/python3.14/site-packages/tokenizers/tools/visualizer.py __annotate__Annotation.__annotate__s"cCc*WnW nW0nR#N)rrr)selfrrrs&&&&r__init__Annotation.__init__s  rc><V^8dQh/S[;R&S[;R&S[;R&#rr)rrs"rrrs% J H Jr)rrrN)__name__ __module__ __qualname____firstlineno__r!__annotate_func____static_attributes____classdictcell__rs@rrrs  rrc,a]tRt^toV3RltRtVtR#) CharStateKeycR<V^8dQh/S[S[,;R&S[S[,;R&#)rtoken_ixanno_ixr r)rrs"rrCharStateKey.__annotate__s#sm c]rN)r$r%r&r'r(r)r*r+s@rr-r-srr-cda]tRt^$toRt]R4t]R4tV3RlRltV3Rlt Rt Vt R#) CharStatec.WnRVn.VnR#r)char_ixr0tokens)r r7s&&rr!CharState.__init__'s &* !# rc`\VP4^8dVP^,#R#)rNlenr8r s&rr/CharState.token_ix-s%!$T[[!1A!5t{{1~?4?rc 2\VP4^8#)z: BPE tokenizers can output more than one token for a char r;r=s&r is_multitokenCharState.is_multitoken1s 4;;!##rc <V^8dQhRS[/#)rreturn)r-)rrs"rrCharState.__annotate__8s  | rcD\VPVPR7#))r/r0)r-r/r0r=s&r partition_keyCharState.partition_key8s]]LL  rc6<V^8dQh/S[S[,;R&#)rr7r1)rrs"rrrD$s c]r)r0r7r8N) r$r%r&r'r!propertyr/r@rFr(r)r*r+s@rr5r5$sE$ @@$$   )rr5c]tRt^?tRtR#)Alignedr3N)r$r%r&r'r)r3rrrKrK?srrKca]tRt^CtoRt]P !R]PR7tRV3RlRllt RV3RlRllt ] V3R lR l4t ] V3R lR l4t ] V3R lRl4t] V3RlRl4t] V3RlRl4tRtVtR#)EncodingVisualizera Build an EncodingVisualizer Args: tokenizer (:class:`~tokenizers.Tokenizer`): A tokenizer instance default_to_notebook (:obj:`bool`): Whether to render html output in a notebook by default annotation_converter (:obj:`Callable`, `optional`): An optional (lambda) function that takes an annotation in any format and returns an Annotation object z(.{1})?(unk|oov)(.{1})?)flagsNc T<V^8dQhRS[RS[RS[S[S[.S[3,,/#)r tokenizerdefault_to_notebookannotation_converter)r boolr rrr)rrs"rrEncodingVisualizer.__annotate__Vs:   " 'xz0A'BC  rcV'd ^RIHpHpWnW nW0nR# \d*^RIHpHpL* \dRp\T4Rhi;ii;i)rHTMLdisplayWe couldn't import IPython utils for html display. Are you running in a notebook? You can also pass `default_to_notebook=False` to get back raw HTML. N)IPython.displayrWrX ImportErrorIPython.core.displayrPrQannotation_coverter)r rPrQrRrWrXmsgs&&&& rr!EncodingVisualizer.__init__Vsj  59##6 #7   55BB"5` &c*4 5 5s& A<AAc r<V^8dQhRS[RS[S[S[,,RS[S[,RS[S[,/#)rtext annotationsrQrC)rr rrrS)rrs"rrrTnsD222d3i(2&d^ 2 # 2rc VPpVeTpV'd ^RIHpHpVf.pVP e \\VP V44pVPPV4p\PWV4p V'dX!X!V 44R#V # \d*^RIHpHpL \dRp\ T4Rhi;ii;i)aw Build a visualization of the given text Args: text (:obj:`str`): The text to tokenize annotations (:obj:`List[Annotation]`, `optional`): An optional list of annotations of the text. The can either be an annotation class or anything else if you instantiated the visualizer with a converter function default_to_notebook (:obj:`bool`, `optional`, defaults to `False`): If True, will render the html in a notebook. Otherwise returns an html string. Returns: The HTML string if default_to_notebook is False, otherwise (default) returns None and renders the HTML in the notebook NrVrY) rQrZrWrXr[r\r]listmaprPencoderM_EncodingVisualizer__make_html) r rarbrQfinal_default_to_notebookrWrXr^encodinghtmls &&&& r__call__EncodingVisualizer.__call__ns2%)$<$<!  *(; % $ 59  K  # # /s4#;#;[IJK>>((.!--dkJ $ DJ K' 55BB"5` &c*4 5 5sB"" C.B88CCc<<V^8dQhRS[RS[S[S[3,/#)rrbrC)AnnotationListrr)rrs"rrrTs#NtCH~rc \V4^8Xd/#\\RV44p\V4p\^V, 4pV^8d^p^ p^@p^ p/p\ V4FpRV RV RV R2Wx&Wc, pK V#)z Generates a color palette for all the labels in a given set of annotations Args: annotations (:obj:`Annotation`): A list of annotations Returns: :obj:`dict`: A dictionary mapping labels to colors in HSL format cVP#r)r)xs&r;EncodingVisualizer.calculate_label_colors..s177rzhsl(,z%,z%))r<setrersorted) rblabels num_labelsh_stepslhcolorsrs & rcalculate_label_colors)EncodingVisualizer.calculate_label_colorss { q IS*K89[ S:%& B;F   F^E"1#QqcA3b1FM KA$ rc<<V^8dQhRS[S[,RS[RS[/#)rconsecutive_chars_listrari)rr5rr )rrs"rrrTs0D9D9 $YD9D9D9rc V^,pVPf$VPVP,pRV R2#VR,pVPfQhVPfQhVPpVP^,pWVp.p /p VPeV PR4VP'dV PR4VP^,'dV PR4MV PR4\ P PVPVP,4e2V PR4VPVP,V R&MV PR 4R R PV 4 R 2p R p V P4FwrV RV RV R 2, p K \P!V4pRV R V RV R2#)as Converts a list of "consecutive chars" into a single HTML element. Chars are consecutive if they fall under the same word, token and annotation. The CharState class is a named tuple with a "partition_key" method that makes it easy to compare if two chars are consecutive. Args: consecutive_chars_list (:obj:`List[CharState]`): A list of CharStates that have been grouped together text (:obj:`str`): The original text being processed encoding (:class:`~tokenizers.Encoding`): The encoding returned from the tokenizer Returns: :obj:`str`: The HTML span for a set of consecutive chars z(tokenz multi-tokenz odd-tokenz even-tokenz special-tokenstokz non-tokenzclass=" "z data-z="z) r7r8r/appendr@rMunk_token_regexsearchjoinitemsrjescape)rrarifirststokenlastrr span_text css_classes data_itemscssdatakeyvals&&& rconsecutive_chars_to_html,EncodingVisualizer.consecutive_chars_to_htmls2'q) == __U^^4F>fXXN N%b)}}(((||''' llQsO   >> %   w '"""""=1~~!! "";/""<0!11889XYe""?3%-__U^^%D 6"   { +#((;/04"((*HC fSEC5* *D+KK * uAdV2i[88rc2<V^8dQhRS[RS[RS[RS[/#rrarirbrC)rr rn)rrs"rrrTs0DD#DDDSVDrc \PWV4pV^,.pV^,Pp.p\PV4pV^,PpVe5W(,p V Pp Wz,p VP RV RV R24VR,Fp V PpW8wdxVP \P VVVR74V .pVeVP R4Ve5W(,p V Pp Wz,p VP RV RV R24TpV P4V^,P48XdVP V 4KVP \P VVVR74V .pK VP \P VVVR74VeVP R4\V4p V #)rz&:NN)rarir) rM%_EncodingVisualizer__make_char_statesr0r~rrrrFHTMLBody)rarirb char_statescurrent_consecutive_chars prev_anno_ixspanslabel_colors_dict cur_anno_ixannorcolorcsress&&& r __make_htmlEncodingVisualizer.__make_htmls(;;DKX %0^$4!"1~-- .EEkR!!n,,  "+DJJE%,E LLA%W\V]]_` ab//B**K* &@@1!!)A.0D)+LL+*&3D JJE-4ELL#I%P^_d^eeg!hi&L!%>q%A%O%O%QQ)004 &@@1!!)A.0D)M"R   8 8)! 9    " LL #uo rc,<V^8dQhRS[RS[RS[/#)rrarbrC)rrnPartialIntList)rrs"rrrTNs"c>rc R.\V4,p\V4F0wr4\VPVP4FpW2V&K K2 V#)a Args: text (:obj:`str`): The raw text we want to align to annotations (:obj:`AnnotationList`): A (possibly empty) list of annotations Returns: A list of length len(text) whose entry at index i is None if there is no annotation on character i or k, the index of the annotation that covers index i where k is with respect to the list of annotations N)r< enumeraterangerr)rarbannotation_mapr0ais&& r__make_anno_map"EncodingVisualizer.__make_anno_mapMsN#d)+#K0JG177AEE*$+q!+1rc B<V^8dQhRS[RS[RS[RS[S[,/#r)rr rnrr5)rrs"rrrTcs0""""~"Z^_hZi"rc \PW4p\\V44Uu.uFp\ V4NK pp\ VP 4FSwrgVPV4pVfKVwr\W4F$p W[,P PV4K& KU \ V4FwrLWV,n K V#uupi)aJ For each character in the original text, we emit a tuple representing it's "state": * which token_ix it corresponds to * which word_ix it corresponds to * which annotation_ix it corresponds to Args: text (:obj:`str`): The raw text we want to align to annotations (:obj:`List[Annotation]`): A (possibly empty) list of annotations encoding: (:class:`~tokenizers.Encoding`): The encoding returned from the tokenizer Returns: :obj:`List[CharState]`: A list of CharStates, indicating for each char in the text what it's state is ) rM"_EncodingVisualizer__make_anno_maprr<r5rr8token_to_charsrr0) rarirbrr7rr/roffsetsrrrr0s &&& r__make_char_states%EncodingVisualizer.__make_char_statesbs.,;;DNJOPSTXPYJZ'[JZw '(:JZ '[(9OH--h7G"$ u*AN))00:+ : !*. 9 G+2 (!:(\sC)r]rQrP)TN)NN)r$r%r&r'__doc__recompile IGNORECASErr!rk staticmethodr~rrgrrr)r*r+s@rrMrMCs jj!>bmmTO  022h8D9D9LDDL(""rrMcFV^8dQhR\\,R\/#)rchildrenrC)rr)rs"rrrstCySrc6RPV4pRV RV R2#)a7 Generates the full html with css from a list of html spans Args: children (:obj:`List[str]`): A list of strings, assumed to be html elements css_styles (:obj:`str`, `optional`): Optional alternative implementation of the css Returns: :obj:`str`: An HTML string with style markup rz?
z4
)r)r css_styles children_texts&& rrrs9GGH%M  O  r)#rj itertoolsosrstringrtypingrrrrrr r tokenizersr r pathdirname__file__r css_filenameopenfreadrrrnrrr-r5rKrMrr3rrrs  III* ''//( #ww||G%<=  ,1 &&(Cj!hsm$:   6  BBJ .1y s 8C(( C8