+
    œ°³jR=  ã                   óê   € ^ RI Ht ^RIHt ^RIHt ^RIHt ^RIH	t	 ^RI
Ht ]	P                  ! ]4      tRtRR	R
R/tR. R OR. R!O/tRRRRRRRRRRRRRR/t]! R"R7       ! R R]4      4       tR.tR# )#é    )ÚAny)ÚBatchEncoding)Ú
AddedToken)ÚSentencePieceBackend)Úlogging)Úrequiresu   â–�Ú
vocab_filezsentencepiece.bpe.modelÚtokenizer_fileztokenizer.jsonÚbaseÚ__java__Ú
__python__Ú	__en_XX__ÚmultiÚ__javascript__Ú__php__Ú__ruby__Ú__go__ÚjavaÚpythonÚen_XXÚ
javascriptÚphpÚrubyÚgo)Úbackendsc                   óT  a a€ ] tR t^/t oRt]tRR.t. t. t	RV3R lV 3R lllt
]R 4       tR t]V3R lR	 l4       t]P                  V3R
 lR l4       tV3R lR ltR tR tRV3R lV 3R llltR tR tV3R lR ltV3R lR ltV3R lR ltRV 3R lltV3R ltRtVtV ;t# ) ÚPLBartTokenizera  
Construct an PLBART tokenizer.

Adapted from [`RobertaTokenizer`] and [`XLNetTokenizer`]. Based on
[SentencePiece](https://github.com/google/sentencepiece).

The tokenization method is `<tokens> <eos> <language code>` for source language documents, and `<language code>
<tokens> <eos>` for target language documents.

Args:
    vocab_file (`str`):
        Path to the vocabulary file.
    src_lang (`str`, *optional*):
        A string representing the source language.
    tgt_lang (`str`, *optional*):
        A string representing the target language.
    bos_token (`str`, *optional*, defaults to `"<s>"`):
        The start of sequence token.
    eos_token (`str`, *optional*, defaults to `"</s>"`):
        The end of sequence token.
    sep_token (`str`, *optional*, defaults to `"</s>"`):
        The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for
        sequence classification or for a text and a question for question answering. It is also used as the last
        token of a sequence built with special tokens.
    cls_token (`str`, *optional*, defaults to `"<s>"`):
        The cls token, which is a special token used as the first token for all tasks.
    unk_token (`str`, *optional*, defaults to `"<unk>"`):
        The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
        token instead.
    pad_token (`str`, *optional*, defaults to `"<pad>"`):
        The token used for padding, for example when batching sequences of different lengths.
    mask_token(`str`, *optional*, defaults to `"<mask>"`):
        The token used for masking values. This is the token used when training this model with masking tasks. This
        is only used in the `"base"` tokenizer type. For `"multi"` tokenizer, masking is never done for the
        downstream tasks.
    language_codes (`str`, *optional*, defaults to `"base"`):
        What language codes to use. Should be one of `"base"` or `"multi"`.
    sp_model_kwargs (`dict`, *optional*):
        Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for
        SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,
        to set:
        - `enable_sampling`: Enable subword regularization.
        - `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.
          - `nbest_size = {0,1}`: No sampling is performed.
          - `nbest_size > 1`: samples from the nbest_size results.
          - `nbest_size < 0`: assuming that nbest_size is infinite and samples from the all hypothesis (lattice)
            using forward-filtering-and-backward-sampling algorithm.
        - `alpha`: Smoothing parameter for unigram sampling, and dropout probability of merge operations for
          BPE-dropout.

Examples:

```python
>>> from transformers import PLBartTokenizer

>>> tokenizer = PLBartTokenizer.from_pretrained("uclanlp/plbart-python-en_XX", src_lang="python", tgt_lang="en_XX")
>>> example_python_phrase = "def maximum(a,b,c):NEW_LINE_INDENTreturn max([a,b,c])"
>>> expected_translation_english = "Returns the maximum value of a b c."
>>> inputs = tokenizer(example_python_phrase, text_target=expected_translation_english, return_tensors="pt")
```Ú	input_idsÚattention_maskc                óD   <€ V ^8„  d   QhRS[ S[S[3,          R,          /# )é   Úsp_model_kwargsN)ÚdictÚstrr   )ÚformatÚ__classdict__s   "€Úo/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/plbart/tokenization_plbart.pyÚ__annotate__ÚPLBartTokenizer.__annotate__t   s'   ø€ ÷ s9ñ s9ñ ™c¡3˜h�¨$Õ.ñs9ó    c           	     ó(	  <€ \        V\        4      '       d   \        VR RR7      MTpVf   / MTV n        V P	                  V
4      p
V P	                  V4      pW�n        \        V P
                  ,          pWn        / V n        / V n	        R^ R^R^R^/V n
        V P                  P                  4        UUu/ uF	  w  ppVVbK  	  uppV n        ^V n        \        V4      pVe+   TP                  V Uu. uF  pVV9  g   K  VNK  	  up4       \         SV `D  ! R/ RVbR	VbR
VbRVbRVbRVbRVbRVbRV
bRVbRVbRV P                  bRVbRV	bRRbRRbVB  \%        V P&                  4      V n        \+        V4       UUu/ uF+  w  ppVV P(                  V,           V P                  ,           bK-  	  uppV n        V P                  P                  4        UUu/ uF	  w  ppVVbK  	  uppV n	        R^ R^R^R^/V n
        V P
                  R8X  dN   \%        V P&                  4      \%        V P                  4      ,           V P                  ,           V P                  R&   V P                  P-                  V P                  4       V P                  P                  4        UUu/ uF	  w  ppVVbK  	  uppV n        0 RmpVP-                  \        V P
                  ,          4       RpV FC  pV P.                  P1                  VR4      pVf   K%  V P2                  P1                  VR4       R pKE  	  V'       d!   V P5                  4        V P7                  4        RpV P.                  P                  4        F8  w  ppVV P2                  9   d   K  \        VR RRRR7      V P2                  V&   R pK:  	  V'       d!   V P5                  4        V P7                  4        V P
                  R8X  dD   W n        V P8                  e   V P                  V P8                  ,          MV P8                  V n        M/V
e   T
MRV n        V P                  V P8                  ,          V n        W°n        V P?                  V P8                  4       R# u uppi u upi u uppi u uppi u uppi ) TF)ÚlstripÚrstripNú<s>ú<pad>ú</s>ú<unk>r	   Ú	bos_tokenÚ	eos_tokenÚ	unk_tokenÚ	sep_tokenÚ	cls_tokenÚ	pad_tokenÚ
mask_tokenÚsrc_langÚtgt_langÚadditional_special_tokensr"   Úclean_up_tokenization_spacesÚlanguage_codesÚspecial_tokens_patternÚprefix_suffixÚtoken_type_ids_patternÚ	all_zerosr   ú<mask>)ÚspecialÚ
normalizedr,   r-   r   © >   r.   r0   r/   r1   rB   ) Ú
isinstancer$   r   r"   Ú!_convert_lang_code_special_formatr=   ÚFAIRSEQ_LANGUAGE_CODESr	   Úlang_code_to_idÚid_to_lang_codeÚfairseq_tokens_to_idsÚitemsÚfairseq_ids_to_tokensÚfairseq_offsetÚlistÚextendÚsuperÚ__init__ÚlenÚsp_modelÚsp_model_sizeÚ	enumerateÚupdateÚ_added_tokens_encoderÚpopÚ_added_tokens_decoderÚ_update_trieÚ_update_total_vocab_sizeÚ	_src_langÚcur_lang_code_idr:   Úset_src_lang_special_tokens)Úselfr	   r2   r3   r5   r6   r4   r7   r8   r=   r9   r:   r"   r;   r<   ÚkwargsÚfairseq_language_codesÚkÚvÚ_additional_special_tokensÚtÚiÚcodeÚreserved_tokensÚremovedÚtokenÚidxÚsyncedÚ	__class__s   &&&&&&&&&&&&&&&,            €r'   rR   ÚPLBartTokenizer.__init__t   sŒ  ø€ ô& KUÐU_Ôad×JeÒJe”Z 
°4ÀÕFÐkuˆ
à%4Ò%<™rÀ/ˆÔØ×9Ñ9¸(ÓCˆØ×9Ñ9¸(ÓCˆØ,ÔÜ!7¸×8KÑ8KÕ!LÐð %ŒØ!ˆÔØ!ˆÔØ&+¨Q°¸¸FÀAÀwÐPQÐ%RˆÔ"Ø7;×7QÑ7Q×7WÑ7WÔ7YÔ%ZÑ7Y©t¨q°! a¨¢dÑ7YÒ%ZˆÔ"ØˆÔÜ%)Ð*@Ó%AÐ"à$Ò0Ø&×-Ñ-Ù5Ó]Ñ5�q¸ÐB\Ñ9\—�Ñ5Ñ]ôô 	‰Òò 	
Ù!ð	
áð	
ñ  ð	
ñ  ð		
ñ
  ð	
ñ  ð	
ñ  ð	
ñ "ð	
ñ ð	
ñ ð	
ñ 'Að	
ð !×0Ò0ð	
ñ *Fð	
ñ *ð	
ñ $3ð	
ñ  $/Øò#	
ô* ! §¡Ó/ˆÔäNWÐXnÔNoô 
ÙNoÁ7À1ÀdˆD�$×$Ñ$ qÕ(¨4×+>Ñ+>Õ>Ò>ÑNoò 
ˆÔð 26×1EÑ1E×1KÑ1KÔ1MÔNÑ1M©¨¨A  1¢Ñ1MÒNˆÔØ&+¨Q°¸¸FÀAÀwÐPQÐ%RˆÔ"à×Ñ &Ô(Ü36°t·}±}Ó3EÌÈD×L`ÑL`ÓHaÕ3aÐdh×dwÑdwÕ3wˆD×&Ñ& xÑ0à×"Ñ"×)Ñ)¨$×*>Ñ*>Ô?Ø7;×7QÑ7Q×7WÑ7WÔ7YÔ%ZÑ7Y©t¨q°! a¨¢dÑ7YÒ%ZˆÔ"ÚEˆØ×ÑÔ5°d×6IÑ6IÕJÔKàˆÛ$ˆEØ×,Ñ,×0Ñ0°¸Ó=ˆCØŒØ×*Ñ*×.Ñ.¨s°DÔ9Ø’ñ	 %÷
 Ø×ÑÔØ×)Ñ)Ô+àˆØ×4Ñ4×:Ñ:Ö<‰JˆE�3Ø�d×0Ñ0Ô0ÙÜ.8Ø˜t°¸eÈEô/ˆD×&Ñ& sÑ+ð ŠFñ =÷ Ø×ÑÔØ×)Ñ)Ô+à×Ñ &Ô(Ø%ŒNà8<¿¹Ò8R�×$Ñ$ T§^¡^Ö4ÐX\×XfÑXfð Õ!ð *2Ò)=™XÀ;ˆDŒNØ$(×$8Ñ$8¸¿¹Õ$HˆDÔ!à ŒØ×(Ñ(¨¯©Ö8ùó[ &[ùò ^ùó2 
ùó  Oùó &[s$   Â4Q7Ã1	Q=Ã?Q=Å>1RÇRÊRc                ó  € \        \        V R / 4      4      p\        V R^4      p\        V R4      '       d   \        V P                  4      M^ p\        V RR4      R8X  d   W1,           V,           ^,           # W1,           V,           # )rI   rN   rT   r=   r   )rS   ÚgetattrÚhasattrrT   )r`   Úlang_code_countrN   Ú
base_vocabs   &   r'   Ú
vocab_sizeÚPLBartTokenizer.vocab_sizeé   st   € äœg dÐ,=¸rÓBÓCˆÜ  Ð'7¸Ó;ˆÜ+2°4¸×+DÒ+D”S˜Ÿ™Ô'È!ˆ
Ü�4Ð)¨6Ó2°fÔ<ØÕ/°.Õ@À1ÕDÐDØÕ+¨nÕ<Ð<r*   c                ó¼  € V P                   P                  4       p\        V P                  P	                  4       4       FO  pV P                  P                  V4      pV^ 8X  d   V P                  MW P                  ,           pW19  g   KK  WAV&   KQ  	  TP                  V P                  P                  4        UUu/ uF  w  rVWQ9  g   K  WVbK  	  upp4       V# u uppi )z,Override to use fairseq vocabulary structure)rK   ÚcopyÚrangerT   Úget_piece_sizeÚ	IdToPieceÚunk_token_idrN   rW   rX   rL   )r`   Úvocabrg   Úsp_tokenÚvocab_idrk   rl   s   &      r'   Ú	get_vocabÚPLBartTokenizer.get_vocabò   s´   € à×*Ñ*×/Ñ/Ó1ˆÜ�t—}‘}×3Ñ3Ó5Ö6ˆAØ—}‘}×.Ñ.¨qÓ1ˆHà,-°¬F�t×(Ò(¸×=PÑ=PÕ9PˆHØÖ$Ø"*�h“ñ 7ð 	�‰°4×3MÑ3M×3SÑ3SÔ3UÔlÑ3U¡Z UÐY^ÑYk”j�e’jÑ3UÒlÔmØˆùó ms   Â9
C
ÃC
c                ó    <€ V ^8„  d   QhRS[ /# )r!   Úreturn©r$   )r%   r&   s   "€r'   r(   r)      s   ø€ ÷ ñ ™#ñ r*   c                ó   € V P                   # ©N)r]   ©r`   s   &r'   r9   ÚPLBartTokenizer.src_langÿ   s   € à�~‰~Ðr*   c                ó$   <€ V ^8„  d   QhRS[ RR/# )r!   Únew_src_langrƒ   Nr„   )r%   r&   s   "€r'   r(   r)     s   ø€ ÷ 9ñ 9¡Sð 9¨Tñ 9r*   c                ój   € V P                  V4      pWn        V P                  V P                  4       R # r†   )rG   r]   r_   )r`   rŠ   s   &&r'   r9   rˆ     s)   € à×=Ñ=¸lÓKˆØ%ŒØ×(Ñ(¨¯©Ö8r*   c                óH   <€ V ^8„  d   QhRS[ RS[ R,          RS[ R,          /# )r!   Úreturn_tensorsr9   Nr:   r„   )r%   r&   s   "€r'   r(   r)   	  s.   ø€ ÷ ñ Ù*-ðÙ9<¸t½ðÙORÐUYÍzñr*   c                óÜ   € Ve   Vf   \        R4      hV P                  V4      V n        V P                  V4      V n        V ! V3RRRV/VB pV P	                  V P                  4      pWvR&   V# )zIUsed by translation pipeline, to prepare inputs for the generate functionzATranslation requires a `src_lang` and a `tgt_lang` for this modelÚadd_special_tokensTr�   Úforced_bos_token_id)Ú
ValueErrorrG   r9   r:   Úconvert_tokens_to_ids)r`   Ú
raw_inputsr�   r9   r:   Úextra_kwargsÚinputsÚtgt_lang_ids   &&&&&,  r'   Ú_build_translation_inputsÚ)PLBartTokenizer._build_translation_inputs	  s{   € ð Ò˜xÒ/ÜÐ`ÓaÐaØ×>Ñ>¸xÓHˆŒØ×>Ñ>¸xÓHˆŒÙ�jÑi°TÐiÈ.ÐiÐ\hÑiˆØ×0Ñ0°·±Ó?ˆØ(3Ð$Ñ%Øˆr*   c                óÊ   € WP                   9   d   V P                   V,          # V P                  P                  V4      pV'       d   W P                  ,           # V P                  # )z0Converts a token (str) in an id using the vocab.)rK   rT   Ú	PieceToIdrN   r|   )r`   rk   Úspm_ids   && r'   Ú_convert_token_to_idÚ$PLBartTokenizer._convert_token_to_id  sQ   € à×.Ñ.Ô.Ø×-Ñ-¨eÕ4Ð4Ø—‘×(Ñ(¨Ó/ˆ÷ 06ˆv×+Ñ+Õ+ÐL¸4×;LÑ;LÐLr*   c                óž   € WP                   9   d   V P                   V,          # V P                  P                  WP                  ,
          4      # )z=Converts an index (integer) in a token (str) using the vocab.)rM   rT   r{   rN   )r`   Úindexs   &&r'   Ú_convert_id_to_tokenÚ$PLBartTokenizer._convert_id_to_token  s=   € à×.Ñ.Ô.Ø×-Ñ-¨eÕ4Ð4Ø�}‰}×&Ñ& u×/BÑ/BÕ'BÓCÐCr*   c          
      óf   <€ V ^8„  d   QhRS[ S[,          RS[RS[ S[,          R,          RS[RS[/# )r!   Ú	src_textsr9   Ú	tgt_textsNr:   rƒ   )rO   r$   r   )r%   r&   s   "€r'   r(   r)   %  sQ   ø€ ÷ 
Mñ 
Má™•9ð
Mñ ð
Mñ ™•9˜tÕ#ð	
Mñ
 ð
Mñ 
ñ
Mr*   c                ó~   <€ V P                  V4      V n        V P                  V4      V n        \        SV `  ! W3/ VB # r†   )rG   r9   r:   rQ   Úprepare_seq2seq_batch)r`   r£   r9   r¤   r:   ra   rn   s   &&&&&,€r'   r¦   Ú%PLBartTokenizer.prepare_seq2seq_batch%  s>   ø€ ð ×>Ñ>¸xÓHˆŒØ×>Ñ>¸xÓHˆŒÜ‰wÒ,¨YÑLÀVÑLÐLr*   c                ó8   € V P                  V P                  4      # r†   )r_   r9   r‡   s   &r'   Ú_switch_to_input_modeÚ%PLBartTokenizer._switch_to_input_mode1  ó   € Ø×/Ñ/°·±Ó>Ð>r*   c                ó8   € V P                  V P                  4      # r†   )Úset_tgt_lang_special_tokensr:   r‡   s   &r'   Ú_switch_to_target_modeÚ&PLBartTokenizer._switch_to_target_mode4  r«   r*   c                ó   <€ V ^8„  d   QhRR/# )r!   rƒ   NrE   )r%   r&   s   "€r'   r(   r)   7  s   ø€ ÷ 5ñ 5°tñ 5r*   c                óð   € V P                  V4      pVe   V P                  V,          MRV n        . V n        V P                  e    V P                  V P                  .V n        R# V P                  .V n        R# )z_Reset the special tokens to the source lang setting. No prefix and suffix=[eos, src_lang_code].N©rG   rI   Úcur_lang_codeÚprefix_tokensÚeos_token_idÚsuffix_tokens)r`   r9   s   &&r'   r_   Ú+PLBartTokenizer.set_src_lang_special_tokens7  sk   € à×9Ñ9¸(ÓCˆØ?GÒ?S˜T×1Ñ1°(Ö;ÐY]ˆÔØˆÔØ×ÑÒ)Ø"&×"3Ñ"3°T×5GÑ5GÐ!HˆDÖà"&×"3Ñ"3Ð!4ˆDÖr*   c                ó$   <€ V ^8„  d   QhRS[ RR/# )r!   Úlangrƒ   Nr„   )r%   r&   s   "€r'   r(   r)   A  s   ø€ ÷ 	5ñ 	5±ð 	5¸ñ 	5r*   c                óð   € V P                  V4      pVe   V P                  V,          MRV n        . V n        V P                  e    V P                  V P                  .V n        R# V P                  .V n        R# )zcReset the special tokens to the target language setting. No prefix and suffix=[eos, tgt_lang_code].Nr²   ©r`   r¹   s   &&r'   r­   Ú+PLBartTokenizer.set_tgt_lang_special_tokensA  sk   € à×5Ñ5°dÓ;ˆà;?Ò;K˜T×1Ñ1°$Ö7ÐQUˆÔØˆÔØ×ÑÒ)Ø"&×"3Ñ"3°T×5GÑ5GÐ!HˆDÖà"&×"3Ñ"3Ð!4ˆDÖr*   c                ó&   <€ V ^8„  d   QhRS[ RS[ /# )r!   r¹   rƒ   r„   )r%   r&   s   "€r'   r(   r)   L  s   ø€ ÷ ñ ±cð ¹cñ r*   c                ó0   € \         P                  W4      pV# )z;Convert Language Codes to format tokenizer uses if required)ÚFAIRSEQ_LANGUAGE_CODES_MAPÚgetr»   s   &&r'   rG   Ú1PLBartTokenizer._convert_lang_code_special_formatL  s   € ä)×-Ñ-¨dÓ9ˆØˆr*   c           	     óD   <€ \         SV `  ! RRVRVRV P                  /VB # )zOOverride to use self.clean_up_tokenization_spaces as default for batched input.Ú	token_idsÚskip_special_tokensr<   rE   )rQ   Údecoder<   )r`   rÃ   rÄ   r<   ra   rn   s   &&&&,€r'   rÅ   ÚPLBartTokenizer.decodeQ  s>   ø€ ä‰wŠ~ñ 
Øð
à 3ð
ð *.×)JÑ)Jð
ð ñ	
ð 	
r*   c                óR   <€ V ^8„  d   Qh/ S[ S[,          ;R&   S[ S[,          ;R&   # )r!   r´   r¶   )rO   Úint)r%   r&   s   "€r'   r(   r)   /   s+   ø‡ ‚ ñD ™•9Ñ!ñE ñF ™•9Ñ!òG r*   )r]   r³   r^   rM   rN   rK   rJ   rI   r=   r´   r"   rU   r9   r¶   r:   r	   )r.   r0   r0   r.   r1   r/   rB   r   NNNNT)r   Nr   )FN)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__ÚVOCAB_FILES_NAMESÚvocab_files_namesÚmodel_input_namesr´   r¶   rR   Úpropertyru   r€   r9   Úsetterr—   rœ   r    r¦   r©   r®   r_   r­   rG   rÅ   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__Ú__classcell__)rn   r&   s   @@r'   r   r   /   sÒ   ù‡ € ñ;ðz *ÐØ$Ð&6Ð7Ðà!€MØ!€M÷s9õ s9ðj ñ=ó ð=òð ÷ó ðð ‡_�_÷9ó ð9÷
ð òMòD÷
Mõ 
Mò?ò?÷5ð 5÷	5ð 	5÷ð ÷

÷E	 … r*   r   N)r   r   r   )r   r   r   r   r   r   r   )Úsentencepiece)Útypingr   Útokenization_pythonr   Útokenization_utils_baser   Ú tokenization_utils_sentencepiecer   Úutilsr   Úutils.import_utilsr   Ú
get_loggerrÉ   ÚloggerÚSPIECE_UNDERLINErÎ   rH   r¿   r   Ú__all__rE   r*   r'   Ú<module>râ      s·   ðõ å 0Ý 1Ý DÝ Ý *ð 
×	Ò	˜HÓ	%€àÐ à!Ð#<Ð>NÐP`ÐaÐ ð Ò3ØÒgðÐ ð ˆJØˆlØˆ[ØÐ"Ø	ˆ9Ø
ˆJØˆ(ðÐ ñ 
Ð%Ô&ôh
Ð*ó h
ó 'ðh
ðV	 Ð
‚r*   