+
    œ°³j!  ã                   ó¢   € ^ RI HtHtHtHt ^ RIHt ^RIHt ^RI	H
t
 ^RIHt ]P                  ! ]4      tRRRR	/t. ROt ! R
 R]
4      tR.tR# )é    )Ú	TokenizerÚdecodersÚpre_tokenizersÚ
processors)ÚUnigram)Ú
AddedToken)ÚTokenizersBackend)ÚloggingÚ
vocab_filezsentencepiece.bpe.modelÚtokenizer_fileztokenizer.jsonc                   óø   a a€ ] tR t^!t oRt]tRR.t]t	. t
. tRV3R lV 3R lllt]V3R lR l4       t]P                  V3R lR	 l4       tV3R
 lR ltR tR tV3R lR ltV3R lR ltV3R ltRtVtV ;t# )ÚMBartTokenizeru  
Construct an MBART tokenizer (backed by HuggingFace's *tokenizers* library). Based on
[Unigram](https://huggingface.co/docs/tokenizers/python/latest/components.html?highlight=unigram#models).

This tokenizer inherits from [`TokenizersBackend`] which contains most of the main methods. Users should
refer to this superclass for more information regarding those methods.

The tokenization method is `<tokens> <eos> <language code>` for source language documents, and `<language code>
<tokens> <eos>` for target language documents.

Examples:

```python
>>> from transformers import MBartTokenizer

>>> tokenizer = MBartTokenizer.from_pretrained(
...     "facebook/mbart-large-en-ro", src_lang="en_XX", tgt_lang="ro_RO"
... )
>>> example_english_phrase = " UN Chief Says There Is No Military Solution in Syria"
>>> expected_translation_romanian = "Åžeful ONU declarÄƒ cÄƒ nu existÄƒ o soluÅ£ie militarÄƒ Ã®n Siria"
>>> inputs = tokenizer(example_english_phrase, text_target=expected_translation_romanian, return_tensors="pt")
```Ú	input_idsÚattention_maskc                óN   <€ V ^8„  d   QhRS[ S[,          S[,          R,          /# )é   ÚvocabN)ÚstrÚdictÚlist)ÚformatÚ__classdict__s   "€Úm/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/mbart/tokenization_mbart.pyÚ__annotate__ÚMBartTokenizer.__annotate__@   s&   ø€ ÷ R9ñ R9á‘T�z™DÕ  4Õ'ñR9ó    c                óÀ  <€ \        V\        4      '       d   \        VR RR7      MTp\        P	                  4       pVe*   TP                  V Uu. uF  qîV9  g   K  VNK  	  up4       Vfy   \        V4      R3\        V4      R3\        V4      R3\        V4      R3.pVR.,          p\         F  pVP                  VR34       K  	  VP                  \        V4      R34       Wn        \        \        V P                  ^RR7      4      V n
        RV P                  n        \        P                  ! \        P                  ! 4       \        P                  ! RRR R7      .4      V P                  n        \"        P                  ! RRR R7      V P                  n        \&        SV `P  ! RR	VR
VRVRVRVRVRVRV	RV
RV/
VB  \         Uu/ uF  qÿV P+                  V4      bK  	  upV n        ^V n        R^ R^R^R^/V n        V P0                  P3                  V P,                  4       V P+                  \        V4      4      V P0                  R&   V P0                  P5                  4        UUu/ uF	  w  ppVVbK  	  uppV n        V	e   T	MRV n        V P+                  V P8                  4      V n        W n        V P?                  V P8                  4       R# u upi u upi u uppi )TF)ÚlstripÚrstripNg        õ   â–�)Úunk_idÚbyte_fallbackÚalways)ÚreplacementÚprepend_schemeÚsplitÚ	bos_tokenÚ	eos_tokenÚ	sep_tokenÚ	cls_tokenÚ	unk_tokenÚ	pad_tokenÚ
mask_tokenÚsrc_langÚtgt_langÚadditional_special_tokensú<s>ú<pad>ú</s>ú<unk>ú<mask>Úen_XX)r    g       À© ) Ú
isinstancer   r   ÚFAIRSEQ_LANGUAGE_CODESÚcopyÚextendÚappendÚ_vocabr   r   Ú
_tokenizerÚ
normalizerr   ÚSequenceÚWhitespaceSplitÚ	MetaspaceÚpre_tokenizerr   ÚdecoderÚsuperÚ__init__Úconvert_tokens_to_idsÚlang_code_to_idÚfairseq_offsetÚfairseq_tokens_to_idsÚupdateÚitemsÚfairseq_ids_to_tokensÚ	_src_langÚcur_lang_coder/   Úset_src_lang_special_tokens)Úselfr   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   ÚkwargsÚ_additional_special_tokensÚtÚ	lang_codeÚkÚvÚ	__class__s   &&&&&&&&&&&&,     €r   rF   ÚMBartTokenizer.__init__@   sñ  ø€ ô KUÐU_Ôad×JeÒJe”Z 
°4ÀÕFÐkuˆ
ä%;×%@Ñ%@Ó%BÐ"Ø$Ò0Ø&×-Ñ-Ù5Ó]Ñ5�qÐB\Ñ9\—�Ñ5Ñ]ôð Š=ä�Y“ Ð%Ü�Y“ Ð%Ü�Y“ Ð%Ü�Y“ Ð%ð	ˆEð �m�_Õ$ˆEß3�	Ø—‘˜i¨Ð-Ö.ñ 4à�L‰Lœ#˜j›/¨3Ð/Ô0àŒÜ#¤G¨D¯K©KÀÐQVÔ$WÓXˆŒà%)ˆ�‰Ô"ä(6×(?Ò(?ä×.Ò.Ó0Ü×(Ò(°UÈ8Ð[_Ô`ðó)
ˆ�‰Ô%ô #+×"4Ò"4ÀÐW_ÐgkÔ"lˆ�‰Ôä‰Òñ 	
Øð	
àð	
ð  ð	
ð  ð		
ð
  ð	
ð  ð	
ð "ð	
ð ð	
ð ð	
ð 'Að	
ð ò	
õ Oeó 
ÙNdÀ�t×1Ñ1°)Ó<Ò<ÑNdñ 
ˆÔð  ˆÔð �1Ø�QØ�AØ�Qð	&
ˆÔ"ð 	×"Ñ"×)Ñ)¨$×*>Ñ*>Ô?Ø/3×/IÑ/IÌ#ÈjË/Ó/Zˆ×"Ñ" 8Ñ,Ø7;×7QÑ7Q×7WÑ7WÔ7YÔ%ZÑ7Y©t¨q°! a¨¢dÑ7YÒ%ZˆÔ"à%-Ò%9™¸wˆŒØ!×7Ñ7¸¿¹ÓGˆÔØ ŒØ×(Ñ(¨¯©Ö8ùò} ^ùòV 
ùó &[s   ÁKÁKÇKÉ(Kc                ó    <€ V ^8„  d   QhRS[ /# )r   Úreturn©r   )r   r   s   "€r   r   r   •   s   ø€ ÷ ñ ™#ñ r   c                ó   € V P                   # ©N)rN   ©rQ   s   &r   r.   ÚMBartTokenizer.src_lang”   s   € à�~‰~Ðr   c                ó$   <€ V ^8„  d   QhRS[ RR/# )r   Únew_src_langr[   Nr\   )r   r   s   "€r   r   r   ™   s   ø€ ÷ 9ñ 9¡Sð 9¨Tñ 9r   c                óH   € Wn         V P                  V P                   4       R # r^   )rN   rP   )rQ   rb   s   &&r   r.   r`   ˜   s   € à%ŒØ×(Ñ(¨¯©Ö8r   c                óH   <€ V ^8„  d   QhRS[ RS[ R,          RS[ R,          /# )r   Úreturn_tensorsr.   Nr/   r\   )r   r   s   "€r   r   r   �   s.   ø€ ÷ 
ñ 
Ù*-ð
Ù9<¸t½ð
ÙORÐUYÍzñ
r   c                ó|   € Ve   Vf   \        R4      hW0n        V ! V3RRRV/VB pV P                  V4      pWvR&   V# )zIUsed by translation pipeline, to prepare inputs for the generate functionzATranslation requires a `src_lang` and a `tgt_lang` for this modelÚadd_special_tokensTre   Úforced_bos_token_id)Ú
ValueErrorr.   rG   )rQ   Ú
raw_inputsre   r.   r/   Úextra_kwargsÚinputsÚtgt_lang_ids   &&&&&,  r   Ú_build_translation_inputsÚ(MBartTokenizer._build_translation_inputs�   sX   € ð Ò˜xÒ/ÜÐ`ÓaÐaØ ŒÙ�jÑi°TÐiÈ.ÐiÐ\hÑiˆØ×0Ñ0°Ó:ˆØ(3Ð$Ñ%Øˆr   c                ó8   € V P                  V P                  4      # r^   )rP   r.   r_   s   &r   Ú_switch_to_input_modeÚ$MBartTokenizer._switch_to_input_mode©   s   € Ø×/Ñ/°·±Ó>Ð>r   c                óv   € V P                   f   V P                  V n         V P                  V P                   4      # r^   )r/   rN   Úset_tgt_lang_special_tokensr_   s   &r   Ú_switch_to_target_modeÚ%MBartTokenizer._switch_to_target_mode¬   s,   € Ø�=‰=Ò Ø ŸN™NˆDŒMØ×/Ñ/°·±Ó>Ð>r   c                ó   <€ V ^8„  d   QhRR/# )r   r[   Nr7   )r   r   s   "€r   r   r   ±   s   ø€ ÷ 
ñ 
°tñ 
r   c                óÜ  € V P                  V4      V n        . V n        V P                  V P                  .V n        V P                  V P                  4      pV P                  V P                  4      p\        P                  ! VR.,           V,           VRR.,           V,           \        \        W#,           V P                  V P                  ,           4      4      R7      V P                  n        R# )z_Reset the special tokens to the source lang setting. No prefix and suffix=[eos, src_lang_code].ú$Aú$B©ÚsingleÚpairÚspecial_tokensN©rG   rO   Úprefix_tokensÚeos_token_idÚsuffix_tokensÚconvert_ids_to_tokensr   ÚTemplateProcessingr   Úzipr>   Úpost_processor)rQ   r.   Úprefix_tokens_strÚsuffix_tokens_strs   &&  r   rP   Ú*MBartTokenizer.set_src_lang_special_tokens±   sÂ   € à!×7Ñ7¸ÓAˆÔØˆÔØ"×/Ñ/°×1CÑ1CÐDˆÔà ×6Ñ6°t×7IÑ7IÓJÐØ ×6Ñ6°t×7IÑ7IÓJÐä)3×)FÒ)FØ$¨ vÕ-Ð0AÕAØ" d¨D \Õ1Ð4EÕEÜ¤Ð$5Õ$IÈ4×K]ÑK]Ð`d×`rÑ`rÕKrÓ sÓtô*
ˆ�‰Ö&r   c                ó$   <€ V ^8„  d   QhRS[ RR/# )r   Úlangr[   Nr\   )r   r   s   "€r   r   r   À   s   ø€ ÷ 
ñ 
±ð 
¸ñ 
r   c                óÜ  € V P                  V4      V n        . V n        V P                  V P                  .V n        V P                  V P                  4      pV P                  V P                  4      p\        P                  ! VR.,           V,           VRR.,           V,           \        \        W#,           V P                  V P                  ,           4      4      R7      V P                  n        R# )zcReset the special tokens to the target language setting. No prefix and suffix=[eos, tgt_lang_code].ry   rz   r{   Nr   )rQ   r‹   r‡   rˆ   s   &&  r   rt   Ú*MBartTokenizer.set_tgt_lang_special_tokensÀ   sÂ   € à!×7Ñ7¸Ó=ˆÔØˆÔØ"×/Ñ/°×1CÑ1CÐDˆÔà ×6Ñ6°t×7IÑ7IÓJÐØ ×6Ñ6°t×7IÑ7IÓJÐä)3×)FÒ)FØ$¨ vÕ-Ð0AÕAØ" d¨D \Õ1Ð4EÕEÜ¤Ð$5Õ$IÈ4×K]ÑK]Ð`d×`rÑ`rÕKrÓ sÓtô*
ˆ�‰Ö&r   c                óR   <€ V ^8„  d   Qh/ S[ S[,          ;R&   S[ S[,          ;R&   # )r   r€   r‚   )r   Úint)r   r   s   "€r   r   r   !   s'   ø‡ ‚ ñ8 ™•9Ñ!ñ9 ñ: ™•9Ñ!ò; r   )rN   r>   r=   rO   rM   rI   rJ   rH   r€   r.   r‚   r/   )Nr1   r3   r3   r1   r4   r2   r5   NNN)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__ÚVOCAB_FILES_NAMESÚvocab_files_namesÚmodel_input_namesr   Úmodelr€   r‚   rF   Úpropertyr.   Úsetterrn   rq   ru   rP   rt   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__Ú__classcell__)rX   r   s   @@r   r   r   !   s–   ù‡ € ñð. *ÐØ$Ð&6Ð7ÐØ€Eà!€MØ!€M÷R9õ R9ðh ÷ó ðð ‡_�_÷9ó ð9÷
ð 
ò?ò?÷

ð 
÷
ð 
÷ … r   r   N)Úar_ARÚcs_CZÚde_DEr6   Úes_XXÚet_EEÚfi_FIÚfr_XXÚgu_INÚhi_INÚit_ITÚja_XXÚkk_KZÚko_KRÚlt_LTÚlv_LVÚmy_MMÚne_NPÚnl_XXÚro_ROÚru_RUÚsi_LKÚtr_TRÚvi_VNÚzh_CN)Ú
tokenizersr   r   r   r   Útokenizers.modelsr   Útokenization_pythonr   Útokenization_utils_tokenizersr	   Úutilsr
   Ú
get_loggerr�   Úloggerr•   r9   r   Ú__all__r7   r   r   Ú<module>r¿      sc   ð÷  GÓ FÝ %å -Ý >Ý ð 
×	Ò	˜HÓ	%€ð "Ð#<Ð>NÐP`ÐaÐ ò {Ð ôl
Ð&ô l
ð^ Ð
‚r   