+
    œ°³jÕ²  ã                   óº  € R t ^ RIt^ RIHt ^ RIt^ RIHtHt ^ RIHtH	t	 ^RI
Ht ^RIHt ^RIHtHtHt ^RIHt ^R	IHtHtHtHt ^R
IHt ^RIHtHt ^RIH t  ]PB                  ! ]"4      t# R t$R+R lt%RR]PL                  3R lt'] ! R R]4      4       t(R t)R t*R t+R,R lt, ! R R]PZ                  4      t. ! R R]PZ                  4      t/ ! R R]PZ                  4      t0 ! R R]PZ                  4      t1R t2 ! R R ]PZ                  4      t3R! t4R" t5] ! R# R$](4      4       t6]! R%R&7       ! R' R(](]4      4       t7 ! R) R*]Pp                  4      t9. R-Ot:R# ).z`PyTorch Fairseq model, ported from https://github.com/pytorch/fairseq/tree/master/examples/wmt19N)ÚAny)ÚTensorÚnn)ÚCrossEntropyLossÚ	LayerNorm)Úinitialization)ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)ÚBaseModelOutputÚ)BaseModelOutputWithPastAndCrossAttentionsÚSeq2SeqLMOutputÚSeq2SeqModelOutput)ÚPreTrainedModel)Úauto_docstringÚlogging©Ú
FSMTConfigc                óR   € V P                  4       ^8X  g   Q hV P                  ^ 4      # )z+Turns 1->0, 0->1, False->True, True-> False)ÚdimÚeq)Úattention_masks   &Úg/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/fsmt/modeling_fsmt.pyÚinvert_maskr   ®   s*   € à×ÑÓ 1Ô$Ð$Ð$Ø×Ñ˜QÓÐó    c                 ó   € V P                   ^ ,          p\        P                  ! W P                  R7      pVP	                  W"4      pVP                  R4      pV'       d	   W1,           pWC8¬  pV P                  V^ 8H  ^ 4      # )é    ©Údeviceéÿÿÿÿ)ÚshapeÚtorchÚaranger    ÚexpandÚ	unsqueezeÚmasked_fill)ÚxÚdiagonalÚlr$   Úmasks   &&   r   Ú	triu_onnxr,   ´   sf   € Ø	�‰��
€AÜ�\Š\˜!§H¡HÔ-€FØ�=‰=˜Ó€DØ×Ñ˜bÓ!€FßØÕ"ˆØ‰>€DØ�=‰=˜ ™ AÓ&Ð&r   c           
     ó$  € V P                   pVf   \        W4      pVP                  4       w  rgVf   \        W%4      pM\	        V4      p\        \        \        P                  ! WwVR7      4      ^4      P                  VP                  R7      pW#V3# )zç
Prepare masks that ignore padding tokens in the decoder and a causal mask for the decoder if none are provided.
This mimics the default behavior in fairseq. To override it pass in masks. Note: this is not called during
generation
©Údtyper   )Úpad_token_idÚshift_tokens_rightÚsizeÚmake_padding_maskr   r,   Úfill_with_neg_infr#   ÚzerosÚtor    )	ÚconfigÚ	input_idsÚdecoder_input_idsÚdecoder_padding_maskÚcausal_mask_dtyper0   ÚbszÚtgt_lenÚcausal_masks	   &&&&&    r   Ú_prepare_fsmt_decoder_inputsr?   ¿   s”   € ð ×&Ñ&€LØÒ Ü.¨yÓGÐØ$×)Ñ)Ó+�L€CØÒ#Ü0Ð1BÓQÑä*Ð+?Ó@ÐÜÔ-¬e¯kªk¸'ÐRcÔ.dÓeÐghÓi×lÑlØ ×'Ñ'ð mó €Kð °KÐ?Ð?r   c                   óz   a a€ ] tR t^Ùt oRt]P                  ! 4       V 3R l4       t]R 4       t	V3R lt
RtVtV ;t# )ÚPretrainedFSMTModelÚmodelc                óþ   <€ \         SV `  V4       \        V\        4      '       dW   VP                  ! . VP
                  P                  OVP                  N5!  p\        P                  ! VP
                  V4       R # R # ©N)
ÚsuperÚ_init_weightsÚ
isinstanceÚSinusoidalPositionalEmbeddingÚget_embeddingÚweightr"   Úpadding_idxÚinitÚcopy_)ÚselfÚmodulerJ   Ú	__class__s   && €r   rF   Ú!PretrainedFSMTModel._init_weightsÞ   s]   ø€ ä‰Ñ˜fÔ%Ü�fÔ;×<Ò<Ø×)Ò)ÐS¨6¯=©=×+>Ñ+>ÐSÀ×@RÑ@RÓSˆFÜ�JŠJ�v—}‘} fÖ-ñ =r   c                ó²   € V P                   P                  p\        P                  ! . RO^ ^^^V..V P                  R7      pRVP                  V4      RV/pV# )r   r   r   r8   )r   é   é
   é   é   )r7   r0   r#   Útensorr    Úne)rN   Ú	pad_tokenr8   Údummy_inputss   &   r   rZ   Ú PretrainedFSMTModel.dummy_inputså   s[   € à—K‘K×,Ñ,ˆ	Ü—L’LÒ"2°Q¸¸2¸qÀ)Ð4LÐ!MÐVZ×VaÑVaÔbˆ	à˜iŸl™l¨9Ó5Ø˜ð
ˆð Ðr   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # ©rV   r7   r   )ÚformatÚ__classdict__s   "€r   Ú__annotate__Ú PretrainedFSMTModel.__annotate__Ù   s   ø‡ ‚ áÑò r   © )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Úbase_model_prefixr#   Úno_gradrF   ÚpropertyrZ   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__Ú__classcell__©rP   r_   s   @@r   rA   rA   Ù   s@   ù‡ € ð  Ðà
‡]‚]ƒ_ô.ó ð.ð ñó ð÷ … r   rA   c                 ó°   € V P                   P                  w  r\        P                  ! WR R7      pV P                   P                  VP                   n        V# )F©Úbias)rJ   r"   r   ÚLinearÚdata)ÚembÚ
vocab_sizeÚemb_sizeÚ	lin_layers   &   r   Ú_make_linear_from_embrx   ð   s@   € ØŸ:™:×+Ñ+Ñ€JÜ—	’	˜*°UÔ;€IØŸJ™JŸO™O€I×ÑÔØÐr   c                 ó4   € W8w  d   \        R V  RV 24      hR# )zshape mismatch: z != N)ÚAssertionError)Úshape_1Úshape2s   &&r   Ú_check_shapesr}   ø   s&   € ØÔÜÐ/°¨y¸¸V¸HÐEÓFÐFñ r   c                ó$  € V P                  V R8H  V4       V P                  4       pV P                  V4      P                  ^R7      ^,
          P	                  R4      pV P                  ^V4      P                  4       VR&   V RRR13,          VR&   V# )zXShift input ids one token to the right, and wrap the last non pad token (usually <eos>).©r   ºNNNNiœÿÿÿr!   )r€   r   )r€   ºé   NN)Úmasked_fill_ÚclonerX   Úsumr&   ÚgatherÚsqueeze)r8   r0   Úprev_output_tokensÚindex_of_eoss   &&  r   r1   r1   ý   s‘   € ð ×Ñ˜9¨Ñ,¨lÔ;à"Ÿ™Ó*ÐØ—L‘L Ó.×2Ñ2°qÐ2Ó9¸AÕ=×HÑHÈÓL€LØ(×/Ñ/°°<Ó@×HÑHÓJÐ�tÑØ )¨!¨S¨b¨S¨&Õ 1Ð�uÑØÐr   c                óX   € V P                  V4      pVP                  4       '       g   RpV# )zTrue for pad tokensN)r   Úany)r8   rK   Úpadding_masks   && r   r3   r3   
  s*   € à—<‘< Ó,€LØ×Ñ×ÒØˆØÐr   c                   óH   a a€ ] tR tRt oV3R lV 3R lltRR ltRtVtV ;t# )ÚEncoderLayeri  c                ó    <€ V ^8„  d   QhRS[ /# r]   r   )r^   r_   s   "€r   r`   ÚEncoderLayer.__annotate__  s   ø€ ÷ 
:ñ 
:™zñ 
:r   c                óL  <€ \         SV `  4        VP                  V n        \	        V P                  VP
                  VP                  R 7      V n        \        V P                  4      V n	        VP                  V n
        \        VP                  ,          V n        VP                  V n        \        P                   ! V P                  VP"                  4      V n        \        P                   ! VP"                  V P                  4      V n        \        V P                  4      V n        R# ))ÚdropoutN)rE   Ú__init__Úd_modelÚ	embed_dimÚ	AttentionÚencoder_attention_headsÚattention_dropoutÚ	self_attnr   Úself_attn_layer_normr’   r   Úactivation_functionÚactivation_fnÚactivation_dropoutr   rr   Úencoder_ffn_dimÚfc1Úfc2Úfinal_layer_norm©rN   r7   rP   s   &&€r   r“   ÚEncoderLayer.__init__  sÀ   ø€ Ü‰ÑÔØŸ™ˆŒÜ" 4§>¡>°6×3QÑ3QÐ[a×[sÑ[sÔtˆŒÜ$-¨d¯n©nÓ$=ˆÔ!Ø—~‘~ˆŒÜ# F×$>Ñ$>Õ?ˆÔØ"(×";Ñ";ˆÔÜ—9’9˜TŸ^™^¨V×-CÑ-CÓDˆŒÜ—9’9˜V×3Ñ3°T·^±^ÓDˆŒÜ )¨$¯.©.Ó 9ˆÖr   c                óB  € TpV P                  VVVVR7      w  r\        P                  P                  WP                  V P                  R7      pWA,           pV P                  V4      pTpV P                  V P                  V4      4      p\        P                  P                  WP                  V P                  R7      pV P                  V4      p\        P                  P                  WP                  V P                  R7      pWA,           pV P                  V4      pW3# )aŠ  
Args:
    x (`torch.Tensor`): input to the layer of shape *(seq_len, batch, embed_dim)*
    encoder_padding_mask (`torch.ByteTensor`): binary ByteTensor of shape
        *(batch, src_len)* where padding elements are indicated by `1`.
    for t_tgt, t_src is excluded (or masked out), =0 means it is
    included in attention

Returns:
    encoded output of shape *(seq_len, batch, embed_dim)*
)ÚqueryÚkeyÚkey_padding_maskÚoutput_attentions©ÚpÚtraining)r™   r   Ú
functionalr’   r«   rš   rœ   rŸ   r�   r    r¡   )rN   r(   Úencoder_padding_maskr¨   ÚresidualÚattn_weightss   &&&&  r   ÚforwardÚEncoderLayer.forward"  sñ   € ð ˆØŸ.™.ØØØ1Ø/ð	 )ó 
‰ˆô �M‰M×!Ñ! !§|¡|¸d¿m¹mÐ!ÓLˆØ�LˆØ×%Ñ% aÓ(ˆàˆØ×Ñ˜tŸx™x¨›{Ó+ˆÜ�M‰M×!Ñ! !×'>Ñ'>ÈÏÉÐ!ÓWˆØ�H‰H�Q‹KˆÜ�M‰M×!Ñ! !§|¡|¸d¿m¹mÐ!ÓLˆØ�LˆØ×!Ñ! !Ó$ˆØˆÐr   )	r�   rœ   r’   r•   rŸ   r    r¡   r™   rš   )F©	rc   rd   re   rf   r“   r°   rk   rl   rm   rn   s   @@r   rŽ   rŽ     s   ù‡ € ÷
:ó 
:÷ô r   rŽ   c                   óX   a a€ ] tR tRt oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )	ÚFSMTEncoderiC  z’
Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a [`EncoderLayer`].

Args:
    config: FSMTConfig
c                ó    <€ V ^8„  d   QhRS[ /# r]   r   )r^   r_   s   "€r   r`   ÚFSMTEncoder.__annotate__K  s   ø€ ÷ bñ b™zñ br   c                óœ  <€ \         SV `  4        VP                  V n        VP                  V n        VP
                  V n        \        P                  ! VP                  VP                  VP
                  4      V n        V P                  P                  pVP                  '       d   \        P                  ! V4      MR V n        \#        VP$                  V P                  ,           ^,           W P                  4      V n        \        P(                  ! \+        VP,                  4       Uu. uF  p\/        V4      NK  	  up4      V n        R# u upi )ç      ð?N)rE   r“   r’   Úencoder_layerdropÚ	layerdropr0   rK   r   Ú	EmbeddingÚsrc_vocab_sizer”   Úembed_tokensÚembedding_dimÚscale_embeddingÚmathÚsqrtÚembed_scalerH   Úmax_position_embeddingsÚembed_positionsÚ
ModuleListÚrangeÚencoder_layersrŽ   Úlayers)rN   r7   r•   Ú_rP   s   &&  €r   r“   ÚFSMTEncoder.__init__K  sí   ø€ Ü‰ÑÔØ—~‘~ˆŒØ×1Ñ1ˆŒØ!×.Ñ.ˆÔÜŸLšL¨×)>Ñ)>ÀÇÁÐPV×PcÑPcÓdˆÔØ×%Ñ%×3Ñ3ˆ	Ø39×3I×3IÐ3Iœ4Ÿ9š9 YÔ/ÈsˆÔÜ<Ø×*Ñ*¨T×-=Ñ-=Õ=ÀÕAÀ9×N^ÑN^ó 
ˆÔô —m’mÄ5È×I^ÑI^ÔC_Ó$`ÑC_¸a¤\°&Ö%9ÑC_Ñ$`ÓaˆŽùÒ$`s   Ä'E	c                ó–   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[RS[RS[/# )rV   r8   r   NÚinputs_embedsr¨   Úoutput_hidden_statesÚreturn_dict)r#   r   Úbool)r^   r_   s   "€r   r`   r¶   X  sm   ø€ ÷ Smñ Smá—<‘<ðSmñ Ÿ™ tÕ+ðSmñ —|‘| dÕ*ð	Smñ
  ðSmñ #ðSmñ ñSmr   c                óœ  € Ve   \        V4      pVe   Ve   \        R4      hVe5   V P                  V4      V P                  ,          pV P	                  V4      pMvVeh   W0P                  ,          pVR,          P                  VR,          P                  ^ 4      V P                  P                  4      pV P	                  V4      pM\        R4      hW7,           p	\        P                  P                  W�P                  V P                  R7      p	V	P                  ^ ^4      p	V'       d   R	MRp
V'       d   R	MRp\        V P                  4       F•  w  rÍV'       d.   V	P                  ^ ^4      p	W©3,          p
V	P                  ^ ^4      p	\        P                   ! . 4      pV P                  '       d   WàP"                  8  d   RpMV! V	VVR7      w  rŸV'       g   KŒ  W¿3,           pK—  	  V	P                  ^ ^4      p	V'       d
   W©3,          p
V'       g1   \$        ;QJ d    . R WšV3 4       F  NK  	  5# ! R WšV3 4       4      # \'        WšVR7      # )
a9  
Args:
    input_ids (`torch.LongTensor`): tokens in the source language of shape
        *(batch, src_len)*
    attention_mask (`torch.LongTensor`): indicating which indices are padding tokens
    inputs_embeds (`torch.FloatTensor`):
        embedding vectors of shape *(batch, src_len, embed_dim)*

Returns:
    BaseModelOutput or Tuple comprised of:

        - **x** (`torch.Tensor`): the last encoder layer's output of shape *(src_len, batch, embed_dim)*
        - **encoder_states** (`Tuple(torch.FloatTensor)`): all intermediate hidden states of shape *(src_len,
          batch, embed_dim)*. Only populated if *output_hidden_states:* is True.
        - **all_attentions** (`Tuple(torch.FloatTensor)`): Attention weights for each layer.
        During training might not be of length n_layers because of layer dropout.
NzDYou cannot specify both input_ids and inputs_embeds at the same timez5You have to specify either input_ids or inputs_embedsr©   )r¨   c              3   ó0   "  € T F  qf   K  Vx € K  	  R # 5irD   rb   ©Ú.0Úvs   & r   Ú	<genexpr>Ú&FSMTEncoder.forward.<locals>.<genexpr>ª  s   é € ÐYÑ$G˜qŸšÓ$Gùó   ‚Œ
©Úlast_hidden_stateÚhidden_statesÚ
attentions©r€   r€   r   rb   )r   Ú
ValueErrorr½   rÂ   rÄ   r'   r   rK   r   r¬   r’   r«   Ú	transposeÚ	enumeraterÈ   r#   Úrandrº   Útupler   )rN   r8   r   rÌ   r¨   rÍ   rÎ   Ú	embed_posÚposition_idsr(   Úencoder_statesÚall_attentionsÚidxÚencoder_layerÚdropout_probabilityÚattns   &&&&&&&         r   r°   ÚFSMTEncoder.forwardX  sÿ  € ð6 Ò%Ü(¨Ó8ˆNàÒ  ]Ò%>ÜÐcÓdÐdØÒ"Ø ×-Ñ-¨iÓ8¸4×;KÑ;KÕKˆMØ×,Ñ,¨YÓ7‰IØÒ&Ø)×,<Ñ,<Õ<ˆMð )¨Õ1×=Ñ=Ø˜gÕ&×)Ñ)¨!Ó,¨d×.BÑ.B×.NÑ.NóˆLð ×,Ñ,¨\Ó:‰IäÐTÓUÐUàÕ%ˆÜ�M‰M×!Ñ! !§|¡|¸d¿m¹mÐ!ÓLˆð �K‰K˜˜1Óˆç3™¸ˆß0™°dˆÜ"+¨D¯K©KÖ"8ÑˆCß#Ø—K‘K  1Ó%�Ø $Õ&�Ø—K‘K  1Ó%�ä"'§*¢*¨R£.ÐØ�}�}ˆ}Ð"5¿¹Ô"FØ‘á'ØØ"Ø&7ô‘�÷ !Ñ Ø!/°'Õ!9’ñ# #9ð( �K‰K˜˜1ÓˆçØ˜dÕ"ˆNçß”5ÑY Q¸Ñ$GÓY—5ÐY�5ÑY Q¸Ñ$GÓYÓYÐYÜ°Ð]kÔlÐlr   )r’   rÄ   rÂ   r½   rº   rÈ   rK   )NNFFT©
rc   rd   re   rf   Ú__doc__r“   r°   rk   rl   rm   rn   s   @@r   r´   r´   C  s+   ù‡ € ñ÷bó b÷Sm÷ Smò Smr   r´   c                   óL   a a€ ] tR tRt oRV3R lV 3R llltRR ltRtVtV ;t# )ÚDecoderLayeri®  c                ó    <€ V ^8„  d   QhRS[ /# r]   r   )r^   r_   s   "€r   r`   ÚDecoderLayer.__annotate__¯  s   ø€ ÷ :ñ :™zñ :r   c                óè  <€ \         SV `  4        VP                  V n        \	        V P                  VP
                  VP                  VR 7      V n        VP                  V n        \        VP                  ,          V n        VP                  V n        \        V P                  4      V n        \	        V P                  VP
                  VP                  RVR7      V n        \        V P                  4      V n        \"        P$                  ! V P                  VP&                  4      V n        \"        P$                  ! VP&                  V P                  4      V n        \        V P                  4      V n        R# ))r•   Ú	num_headsr’   Ú	layer_idxT)r’   Úencoder_decoder_attentionró   N)rE   r“   r”   r•   r–   Údecoder_attention_headsr˜   r™   r’   r   r›   rœ   r�   r   rš   Úencoder_attnÚencoder_attn_layer_normr   rr   Údecoder_ffn_dimrŸ   r    r¡   )rN   r7   ró   rP   s   &&&€r   r“   ÚDecoderLayer.__init__¯  s  ø€ Ü‰ÑÔØŸ™ˆŒä"Ø—n‘nØ×4Ñ4Ø×,Ñ,Øô	
ˆŒð —~‘~ˆŒÜ# F×$>Ñ$>Õ?ˆÔØ"(×";Ñ";ˆÔä$-¨d¯n©nÓ$=ˆÔ!Ü%Ø�N‰NØ×*Ñ*Ø×,Ñ,Ø&*Øô
ˆÔô (1°·±Ó'@ˆÔ$Ü—9’9˜TŸ^™^¨V×-CÑ-CÓDˆŒÜ—9’9˜V×3Ñ3°T·^±^ÓDˆŒÜ )¨$¯.©.Ó 9ˆÖr   c           	     ó|  € Tp	V P                  VVVVVVR 7      w  r\        P                  P                  WP                  V P                  R7      pW‘,           pV P                  V4      pTp	V P                  P                  V P                   P                  8w  g   Q hV P                  VVVVVR7      w  r\        P                  P                  WP                  V P                  R7      pW‘,           pV P                  V4      pTp	V P                  V P                  V4      4      p\        P                  P                  WP                  V P                  R7      pV P                  V4      p\        P                  P                  WP                  V P                  R7      pW‘,           pV P                  V4      pVV
V3# ))r¥   r¦   Úlayer_stater§   Ú	attn_maskr¨   r©   )r¥   r¦   r§   rû   r¨   )r™   r   r¬   r’   r«   rš   rö   Ú	cache_keyr÷   rœ   rŸ   r�   r    r¡   )rN   r(   Úencoder_hidden_statesÚencoder_attn_maskrû   r>   r:   r¨   Úkwargsr®   Úself_attn_weightsÚcross_attn_weightss   &&&&&&&&,   r   r°   ÚDecoderLayer.forwardÊ  s“  € ð ˆð  $Ÿ~™~ØØØ#Ø1Ø!Ø/ð  .ó  
Ñˆô �M‰M×!Ñ! !§|¡|¸d¿m¹mÐ!ÓLˆØ�LˆØ×%Ñ% aÓ(ˆð ˆØ× Ñ ×*Ñ*¨d¯n©n×.FÑ.FÔFÐFÐFØ $× 1Ñ 1ØØ%Ø.Ø#Ø/ð !2ó !
Ñˆô �M‰M×!Ñ! !§|¡|¸d¿m¹mÐ!ÓLˆØ�LˆØ×(Ñ(¨Ó+ˆð ˆØ×Ñ˜tŸx™x¨›{Ó+ˆÜ�M‰M×!Ñ! !×'>Ñ'>ÈÏÉÐ!ÓWˆØ�H‰H�Q‹KˆÜ�M‰M×!Ñ! !§|¡|¸d¿m¹mÐ!ÓLˆØ�LˆØ×!Ñ! !Ó$ˆàØØð
ð 	
r   )r�   rœ   r’   r•   rö   r÷   rŸ   r    r¡   r™   rš   rD   )NNNNFr²   rn   s   @@r   rî   rî   ®  s   ù‡ € ÷:õ :÷64
ô 4
r   rî   c                   óX   a a€ ] tR tRt oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )	ÚFSMTDecoderi  z´
Transformer decoder consisting of *config.decoder_layers* layers. Each layer is a [`DecoderLayer`]

Args:
    config: FSMTConfig
    embed_tokens (nn.Embedding): output embedding
c                ó    <€ V ^8„  d   QhRS[ /# r]   r   )r^   r_   s   "€r   r`   ÚFSMTDecoder.__annotate__
  s   ø€ ÷ ^ñ ^™zñ ^r   c           
     ó  <€ \         SV `  4        VP                  V n        VP                  V n        VP
                  V n        VP                  '       d!   \        P                  ! VP                  4      MR V n        \        P                  ! VP                  VP                  V P                  4      V n        V P                  P                   p\#        VP$                  V P                  ,           ^,           W P                  4      V n        \        P(                  ! \+        VP,                  4       Uu. uF  p\/        WR7      NK  	  up4      V n        \        P2                  ! VP                  VP                  RR7      V n        R# u upi )r¸   )ró   Frp   N)rE   r“   r’   Údecoder_layerdroprº   r0   rK   r¿   rÀ   rÁ   r”   rÂ   r   r»   Útgt_vocab_sizer½   r¾   rH   rÃ   rÄ   rÅ   rÆ   Údecoder_layersrî   rÈ   rr   Úoutput_projection)rN   r7   r•   ÚirP   s   &&  €r   r“   ÚFSMTDecoder.__init__
  s  ø€ Ü‰ÑÔØ—~‘~ˆŒØ×1Ñ1ˆŒØ!×.Ñ.ˆÔØ8>×8N×8NÐ8Nœ4Ÿ9š9 V§^¡^Ô4ÐTWˆÔÜŸLšL¨×)>Ñ)>ÀÇÁÐPT×P`ÑP`ÓaˆÔØ×%Ñ%×3Ñ3ˆ	Ü<Ø×*Ñ*¨T×-=Ñ-=Õ=ÀÕAÀ9×N^ÑN^ó 
ˆÔô —m’mÔPUÐV\×VkÑVkÔPlÓ$mÑPlÈ1¤\°&×%FÑPlÑ$mÓnˆŒÜ!#§¢¨6¯>©>¸6×;PÑ;PÐW\Ô!]ˆÖùò %ns   Ä1Fc                ó(  <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[ P                  RS[ P                  RS[ P                  R,          RS[R,          R	S[R,          R
S[R,          RS[R,          RS[R,          /# )rV   r8   rþ   r­   r:   Údecoder_causal_maskrÌ   NÚpast_key_valuesÚ	use_cacher¨   rÍ   rÎ   )r#   r   r	   rÏ   )r^   r_   s   "€r   r`   r    s·   ø€ ÷ v
ñ v
á—<‘<ðv
ñ  %Ÿ|™|ðv
ñ $Ÿl™lð	v
ñ
 $Ÿl™lðv
ñ #Ÿ\™\ðv
ñ —|‘| dÕ*ðv
ñ  �ðv
ñ ˜$•;ðv
ñ   $�;ðv
ñ # T�kðv
ñ ˜D•[ñv
r   c                óà  € Ve   \        V4      pVe   Ve   \        R4      hVeW   V P                  V4      pV'       d   VRR	R13,          pVRR	R13,          pV P                  V4      V P                  ,          pMvVeh   VR
,          P                  VR
,          P                  ^ 4      V P                  P                  4      pV P                  V4      pW`P                  ,          pM\        R4      hWí,          p\        P                  P                  WàP                  V P                  R7      pVP                  ^ ^4      pVP                  ^ ^4      pV
'       d   RMRpV	'       d   RMRpV	'       d   RMRp\        V P                  4       F¨  w  ppV
'       d/   VP                  ^ ^4      pVV3,          pVP                  ^ ^4      pV P                  '       d*   \        P                   ! . 4      pVV P"                  8  d   Kw  V! VVVVVVV	R7      w  pppV	'       g   K”  VV3,          pVV3,          pKª  	  V
'       d/   VP                  ^ ^4      pVV3,          pVP                  ^ ^4      pVP                  ^ ^4      pVP                  ^ ^4      pV P%                  V4      pV'       g5   \&        ;QJ d    . R WçVVV3 4       F  NK  	  5# ! R WçVVV3 4       4      # \)        VVVVVR7      # )a™  
Includes several features from "Jointly Learning to Align and Translate with Transformer Models" (Garg et al.,
EMNLP 2019).

Args:
    input_ids (`torch.LongTensor` of shape `(batch, tgt_len)`):
        previous decoder outputs for teacher forcing
    encoder_hidden_states: output from the encoder, used for
        encoder-side attention
    encoder_padding_mask: for ignoring pad tokens
    past_key_values (dict or None): dictionary used for storing state during generation

Returns:
    BaseModelOutputWithPast or tuple:

        - the decoder's features of shape *(batch, tgt_len, embed_dim)*
        - the cache
        - hidden states
        - attentions
NzTYou cannot specify both decoder_input_ids and decoder_inputs_embeds at the same timer€   zEYou have to specify either decoder_input_ids or decoder_inputs_embedsr©   )rÿ   r:   rû   r>   r¨   c              3   ó0   "  € T F  qf   K  Vx € K  	  R # 5irD   rb   rÒ   s   & r   rÕ   Ú&FSMTDecoder.forward.<locals>.<genexpr>…  s   é € ð Ùc�a—’Ócùr×   )rÙ   r  rÚ   rÛ   Úcross_attentionsr!   rÜ   rb   )r   rÝ   rÄ   r½   rÂ   r'   r   rK   r   r¬   r’   r«   rÞ   rß   rÈ   r#   rà   rº   r  rá   r   )rN   r8   rþ   r­   r:   r  rÌ   r  r  r¨   rÍ   rÎ   r   Ú	positionsr(   rã   Úall_hidden_statesÚall_self_attnsÚall_cross_attnsræ   Údecoder_layerrè   Úlayer_self_attnÚlayer_cross_attns   &&&&&&&&&&&&,           r   r°   ÚFSMTDecoder.forward  sß  € ðH  Ò+Ü#.Ð/CÓ#DÐ àÒ  ]Ò%>ÜÐsÓtÐtØÒ"à×,Ñ,¨YÓ7ˆIßØ% a¨© fÕ-�	Ø% a¨© fÕ-�	Ø×!Ñ! )Ó,¨t×/?Ñ/?Õ?‰AØÒ&ð )¨Õ1×=Ñ=Ø˜gÕ&×)Ñ)¨!Ó,¨d×.BÑ.B×.NÑ.NóˆLð ×,Ñ,¨\Ó:ˆIØ× 0Ñ 0Õ0‰AäÐdÓeÐeà	�ˆÜ�M‰M×!Ñ! !§|¡|¸d¿m¹mÐ!ÓLˆð �K‰K˜˜1ÓˆØ 5× ?Ñ ?ÀÀ1Ó EÐ÷ #7™B¸DÐß0™°dˆß 1™"°tˆä"+¨D¯K©KÖ"8ÑˆC�ç#Ø—K‘K  1Ó%�Ø! a TÕ)Ð!Ø—K‘K  1Ó%�Ø�}�}ˆ}Ü&+§j¢j°£nÐ#Ø&¨¯©Ô7Ùá3@ØØ%Ø"6Ø%9Ø+Ø/Ø"3ô4Ñ0ˆAˆÐ 0÷ !Ñ Ø ?Ð"4Õ4�ØÐ$4Ð#6Õ6’ñ/ #9÷4  Ø—‘˜A˜qÓ!ˆAØ ! Õ%ÐØ—‘˜A˜qÓ!ˆAð �K‰K˜˜1ÓˆØ 5× ?Ñ ?ÀÀ1Ó EÐà×"Ñ" 1Ó%ˆçß”5ñ ØÐ0AÀ>ÐSbÑcó—5ð �5ñ ØÐ0AÀ>ÐSbÑcóó ð ô 9ØØ+Ø+Ø%Ø,ô
ð 	
r   )r’   rÄ   rÂ   r½   rº   rÈ   r  rK   )NNFFFTrë   rn   s   @@r   r  r    s(   ù‡ € ñ÷^ó ^÷v
÷ v
ò v
r   r  c                 ól   € V P                  4        F  w  r#Vf   K  VP                  ^ V4      W&   K!  	  V # rD   )ÚitemsÚindex_select)Ú
attn_cacheÚ	new_orderÚkÚinput_buffer_ks   &&  r   Ú_reorder_bufferr&  ‘  s:   € Ø'×-Ñ-Ö/ÑˆØÔ%Ø*×7Ñ7¸¸9ÓEˆJ‹Mñ 0ð Ðr   c                   óP   a a€ ] tR tRt oRtRV 3R lltRV3R lR lltRtVtV ;t	# )	r–   i˜  z=Multi-headed attention from 'Attention Is All You Need' paperc                ó(  <€ \         SV `  4        Wn        W n        W0n        W,          V n        V P
                  V,          V P                  8X  g   Q R 4       hV P
                  R,          V n        W`n        WPn        \        P                  ! WVR7      V n        \        P                  ! WVR7      V n        \        P                  ! WVR7      V n        \        P                  ! WVR7      V n        V P                  '       d
   RV n        R# RV n        R# )z(embed_dim must be divisible by num_headsrp   Úencoder_decoderrN   Ng      à¿)rE   r“   r•   rò   r’   Úhead_dimÚscalingró   rô   r   rr   Úk_projÚv_projÚq_projÚout_projrý   )rN   r•   rò   r’   rq   rô   ró   rP   s   &&&&&&&€r   r“   ÚAttention.__init__›  sÇ   ø€ ô 	‰ÑÔØ"ŒØ"ŒØŒØ!Õ.ˆŒØ�}‰}˜yÕ(¨D¯N©NÔ:ÐfÐ<fÓfÐ:Ø—}‘} dÕ*ˆŒØ"Œà)BÔ&Ü—i’i 	¸4Ô@ˆŒÜ—i’i 	¸4Ô@ˆŒÜ—i’i 	¸4Ô@ˆŒÜŸ	š	 )¸TÔBˆŒØ.2×.L×.LÐ.LÐ*ˆŽÐRXˆŽr   c                ó¨   <€ V ^8„  d   QhRS[ R,          RS[ R,          RS[R,          RS[ R,          RS[R,          RS[S[ S[ R,          3,          /# )rV   r¦   Nr§   rû   rü   r¨   Úreturn)r   r	   rÏ   rá   )r^   r_   s   "€r   r`   ÚAttention.__annotate__´  ss   ø€ ÷ `2ñ `2ñ �d�]ð`2ñ ! 4�-ð	`2ñ
 ˜T•\ð`2ñ ˜D•=ð`2ñ   $�;ð`2ñ 
‰v‘v •}Ð$Õ	%ñ`2r   c                ó2  € VP                  4       w  r‰p
W P                  8X  g   Q h\        VP                  4       4      W‰V
.8X  g   Q hVej   \        V\        4      '       dR   VP
                  P                  V P                  4      pV P                  '       d   VP                  pMVP                  pMTpV P                  '       d   TMTpV P                  '       d\   VeX   X'       dP   XP                  V P                  ,          P                  pVP                  V P                  ,          P                  pMáV P                  V4      pV P                  V4      pVP!                  RW�P"                  V P$                  4      P'                  ^^^ ^4      pVP!                  RW�P"                  V P$                  4      P'                  ^^^ ^4      pVeJ   XP)                  WïV P                  4      w  rïV P                  '       d   RVP
                  V P                  &   V P+                  V4      V P,                  ,          pVP!                  RW�P"                  ,          V P$                  4      P/                  ^ ^4      pVP1                  W�P"                  ,          RV P$                  4      pVP1                  W�P"                  ,          RV P$                  4      pVf   Q hVP                  ^4      p\2        P4                  ! VVP/                  ^^4      4      pVP                  4       W�P"                  ,          VV38X  g   Q hVeH   VP!                  W�P"                  VV4      V,           pVP!                  W�P"                  ,          VV4      pVe   VP7                  4       ^ 8X  d   RpVe!   VP                  4       R,          V	V38X  g   Q hVe›   VP!                  W�P"                  VV4      pVP9                  ^4      P9                  ^4      pVP;                  V\2        P<                  ! VP>                  4      P@                  4      pVP!                  W�P"                  ,          VV4      p\B        PD                  PG                  VRR7      pV'       dB   VP!                  W�P"                  VV4      pVP!                  W�P"                  ,          VV4      pMRp\B        PD                  PI                  VV PH                  V PJ                  R7      pVf   Q h\2        P4                  ! VV4      pVP                  4       W�P"                  ,          W€P$                  38X  g   Q hVP/                  ^ ^4      PM                  4       P!                  W‰V
4      pV PO                  V4      pVV3# )z+Input shape: Time(SeqLen) x Batch x ChannelNTºNrV   Nr   r©   r!   )(r2   r•   ÚlistrG   r   Ú
is_updatedÚgetró   rô   Úcross_attention_cacheÚself_attention_cacherÈ   ÚkeysÚvaluesr,  r-  Úviewrò   r*  ÚpermuteÚupdater.  r+  rÞ   Úreshaper#   Úbmmr   r&   r'   Úfinfor/   Úminr   r¬   Úsoftmaxr’   r«   Ú
contiguousr/  )rN   r¥   r¦   r§   rû   rü   r¨   r   r=   r<   r•   r7  Úcurr_past_key_valuesÚcurrent_statesÚ
key_statesÚvalue_statesÚquery_statesÚsrc_lenr¯   ÚreshapedÚattn_weights_reshapedÚ
attn_probsÚattn_outputs   &&&&&&&,               r   r°   ÚAttention.forward´  s«  € ð #(§*¡*£,Ñˆ�iØŸN™NÔ*Ð*Ð*Ü�E—J‘J“LÓ! g°IÐ%>Ô>Ð>Ð>àÒ"Ü˜+Ô':×;Ò;Ø(×3Ñ3×7Ñ7¸¿¹ÓG�
Ø×1×1Ð1à+6×+LÑ+LÑ(à+6×+KÑ+KÑ(à'2Ð$ð !%× >× >Ð >™ÀEˆØ×)×)Ð)¨kÒ.EÏ*à-×4Ñ4°T·^±^ÕD×IÑIˆJØ/×6Ñ6°t·~±~ÕF×MÑM‰LàŸ™ ^Ó4ˆJØŸ;™; ~Ó6ˆLØ#Ÿ™¨¨S·.±.À$Ç-Á-ÓP×XÑXÐYZÐ\]Ð_`ÐbcÓdˆJØ'×,Ñ,¨R°·n±nÀdÇmÁmÓT×\Ñ\Ð]^Ð`aÐcdÐfgÓhˆLàÒ&à+?×+FÑ+FÀzÐae×aoÑaoÓ+pÑ(�
à×1×1Ð1Ø=A�K×*Ñ*¨4¯>©>Ñ:à—{‘{ 5Ó)¨D¯L©LÕ8ˆð $×(Ñ(¨¨S·>±>Õ-AÀ4Ç=Á=ÓQ×[Ñ[Ð\]Ð_`ÓaˆØ×'Ñ'¨¯n©nÕ(<¸bÀ$Ç-Á-ÓPˆ
Ø#×+Ñ+¨C·.±.Õ,@À"ÀdÇmÁmÓTˆàÒ%Ð%Ð%Ø—/‘/ !Ó$ˆÜ—y’y ¨z×/CÑ/CÀAÀqÓ/IÓJˆØ× Ñ Ó" s¯^©^Õ';¸WÀgÐ&NÔNÐNÐNàÒ Ø'×,Ñ,¨S·.±.À'È7ÓSÐV_Õ_ˆLØ'×,Ñ,¨S·>±>Õ-AÀ7ÈGÓTˆLð Ò'Ð,<×,@Ñ,@Ó,BÀaÔ,GØ#ÐØÒ'Ð+;×+@Ñ+@Ó+BÀ2Õ+FØØðK
ô ,
ð 	
ð 
ð
 Ò'Ø'×,Ñ,¨S·.±.À'È7ÓSˆLØ'×1Ñ1°!Ó4×>Ñ>¸qÓAˆHØ'×3Ñ3°H¼e¿kºkÈ,×J\ÑJ\Ó>]×>aÑ>aÓbˆLØ'×,Ñ,¨S·>±>Õ-AÀ7ÈGÓTˆLä—}‘}×,Ñ,¨\¸rÐ,ÓBˆçà$0×$5Ñ$5°c¿>¹>È7ÐT[Ó$\Ð!Ø0×5Ñ5°c¿N¹NÕ6JÈGÐU\Ó]‰Là$(Ð!ä—]‘]×*Ñ*ØØ�l‰lØ—]‘]ð +ó 
ˆ
ð Ò'Ð'Ð'Ü—i’i 
¨LÓ9ˆØ×ÑÓ! c¯N©NÕ&:¸GÇ]Á]Ð%SÔSÐSÐSØ!×+Ñ+¨A¨qÓ1×<Ñ<Ó>×CÑCÀGÐR[Ó\ˆØ—m‘m KÓ0ˆàÐ1Ð1Ð1r   )rý   r’   r•   rô   r*  r,  ró   rò   r/  r.  r+  r-  )g        TFN)NNNFrë   rn   s   @@r   r–   r–   ˜  s    ù‡ € ÙG÷Y÷2`2÷ `2ò `2r   r–   c                ó®   € V P                  4       P                  \        P                  ! V P                  4      P
                  4      P                  V 4      # )z;FP16-compatible function that fills an input_ids with -inf.)ÚfloatÚfill_r#   rB  r/   rC  Útype_as©Úts   &r   r4   r4     s5   € à�7‰7‹9�?‰?œ5Ÿ;š; q§w¡wÓ/×3Ñ3Ó4×<Ñ<¸QÓ?Ð?r   c                 ó   € \        V R R4      # )r"   N)ÚgetattrrU  s   &r   Ú
_get_shaperY    s   € Ü�1�g˜tÓ$Ð$r   c                   ó‚   a a€ ] tR tRt oRRRR/tV3R lV 3R llt]RV3R lR ll4       tR	 tR
 t	R t
R tRtVtV ;t# )Ú	FSMTModeli!  zencoder.embed_tokens.weightzdecoder.embed_tokens.weightz decoder.output_projection.weightc                ó    <€ V ^8„  d   QhRS[ /# r]   r   )r^   r_   s   "€r   r`   ÚFSMTModel.__annotate__(  s   ø€ ÷ ñ ™zñ r   c                ó†   <€ \         SV `  V4       \        V4      V n        \	        V4      V n        V P                  4        R # rD   )rE   r“   r´   Úencoderr  ÚdecoderÚ	post_initr¢   s   &&€r   r“   ÚFSMTModel.__init__(  s2   ø€ Ü‰Ñ˜Ô Ü" 6Ó*ˆŒÜ" 6Ó*ˆŒØ�‰Ör   c                óÒ  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[ P
                  ,          R,          RS[R,          RS[R,          R	S[R,          R
S[R,          RS[ P
                  R,          RS[ P
                  R,          RS[R,          RS[S[ P                  ,          S[,          /# )rV   r8   r   Nr9   Údecoder_attention_maskÚencoder_outputsr  r  r¨   rÍ   rÌ   Údecoder_inputs_embedsrÎ   r2  )	r#   Ú
LongTensorr   Ú
BoolTensorrá   ÚFloatTensorr	   rÏ   r   )r^   r_   s   "€r   r`   r]  /  s
  ø€ ÷ g
ñ g
á×#Ñ#ðg
ñ Ÿ™ tÕ+ðg
ñ !×+Ñ+¨dÕ2ð	g
ñ
 !&× 0Ñ 0°4Õ 7ðg
ñ ™u×0Ñ0Õ1°DÕ8ðg
ñ  �ðg
ñ ˜$•;ðg
ñ   $�;ðg
ñ # T�kðg
ñ ×(Ñ(¨4Õ/ðg
ñ  %×0Ñ0°4Õ7ðg
ñ ˜D•[ðg
ñ 
‰u�|‰|Õ	Ñ1Õ	1ñg
r   c                ón  € Vf   RpVe   TMV P                   P                  pV	e   T	MV P                   P                  p	Ve   TMV P                   P                  pVe   TMV P                   P                  pV'       gK   VeG   \        V P                   VVVV P                  P                  P                  P                  R7      w  r>pMRRrþVf   Vf   \        R4      hV'       d9   Vf5   \        \        V P                   R7      \        V P                   R7      4      pVf   V P                  VVV
VV	VR7      pMeV'       d^   \        V\        4      '       gH   \        V^ ,          \!        V4      ^8”  d
   V^,          MR\!        V4      ^8”  d
   V^,          MRR7      pV P                  VV^ ,          VVVVVVVV	VR7      pV'       g
   VV,           # \#        VP$                  VP&                  VP(                  VP*                  VP,                  VP$                  VP(                  VP*                  R	7      # )
a:  
decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Indices of decoder input sequence tokens in the vocabulary.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are decoder input IDs?](../glossary#decoder-input-ids)

    FSMT uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
    is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).
decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
    be used by default.
NF)r9   r:   r;   zIMake sure that `decoder_input_ids` or `decoder_inputs_embeds` are passed.)r7   )r8   r   rÌ   r¨   rÍ   rÎ   rØ   )r  rÌ   r  r  r¨   rÍ   rÎ   )rÙ   r  Údecoder_hidden_statesÚdecoder_attentionsr  Úencoder_last_hidden_staterþ   Úencoder_attentions)r7   r¨   rÍ   r  rÎ   r?   r`  r½   rJ   r/   rÝ   r   r
   r_  rG   r   Úlenr   rÙ   r  rÚ   rÛ   r  )rN   r8   r   r9   rd  re  r  r  r¨   rÍ   rÌ   rf  rÎ   r   r:   r>   Údecoder_outputss   &&&&&&&&&&&&&,   r   r°   ÚFSMTModel.forward.  s  € ð@ Ò$ØˆIà1BÒ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ò$DÑ È$Ï+É+×JjÑJjð 	ð "+Ò!6‘I¸D¿K¹K×<QÑ<Qˆ	Ø%0Ò%<‘kÀ$Ç+Á+×BYÑBYˆ÷ ˜YÒ2ÜC_Ø—‘ØØ"3Ø%;Ø"&§,¡,×";Ñ";×"BÑ"B×"HÑ"HôDÑ@Ð±[ð 15°d +àÒ$Ð)>Ò)FÜÐhÓiÐiç˜Ò0Ü1´,ÀdÇkÁkÔ2RÔT`Ðhl×hsÑhsÔTtÓuˆOàÒ"Ø"Ÿl™lØ#Ø-Ø+Ø"3Ø%9Ø'ð +ó ‰O÷ ¤¨O¼_×!MÒ!MÜ-Ø"1°!Õ"4Ü47¸Ó4HÈ1Ô4L˜o¨aÖ0ÐRVÜ14°_Ó1EÈÔ1I˜?¨1Ö-ÈtôˆOð Ÿ,™,ØØ˜AÕØØ Ø +Ø/Ø+ØØ/Ø!5Ø#ð 'ó 
ˆ÷ Ø" _Õ4Ð4ä!Ø-×?Ñ?Ø+×;Ñ;Ø"1×"?Ñ"?Ø.×9Ñ9Ø,×=Ñ=Ø&5×&GÑ&GØ"1×"?Ñ"?Ø.×9Ñ9ô	
ð 		
r   c                ó.   € V P                   P                  # rD   ©r_  r½   ©rN   s   &r   Úget_input_embeddingsÚFSMTModel.get_input_embeddings˜  ó   € Ø�|‰|×(Ñ(Ð(r   c                ó&   € WP                   n        R # rD   rs  ©rN   Úvalues   &&r   Úset_input_embeddingsÚFSMTModel.set_input_embeddings›  ó   € Ø$)�‰Ö!r   c                ó.   € V P                   P                  # rD   ©r`  r½   rt  s   &r   Úget_output_embeddingsÚFSMTModel.get_output_embeddingsž  rw  r   c                ó&   € WP                   n        R # rD   r  ry  s   &&r   Úset_output_embeddingsÚFSMTModel.set_output_embeddings¡  r}  r   )r`  r_  )NNNNNNNNNNN)rc   rd   re   rf   Ú_tied_weights_keysr“   r   r°   ru  r{  r€  rƒ  rk   rl   rm   rn   s   @@r   r[  r[  !  s[   ù‡ € ð 	&Ð'DØ*Ð,IðÐ÷
ó ð ÷g
ñ g
ó ðg
òR)ò*ò)÷*ò *r   r[  zV
    The FSMT Model with a language modeling head. Can be used for summarization.
    )Úcustom_introc                   ó€   a a€ ] tR tRt oRtV3R lV 3R llt]RV3R lR ll4       tV3R lR ltR	 t	R
 t
RtVtV ;t# )ÚFSMTForConditionalGenerationi¥  rB   c                ó    <€ V ^8„  d   QhRS[ /# r]   r   )r^   r_   s   "€r   r`   Ú)FSMTForConditionalGeneration.__annotate__­  s   ø€ ÷ ñ ™zñ r   c                óh   <€ \         SV `  V4       \        V4      pW n        V P	                  4        R # rD   )rE   r“   r[  rB   ra  )rN   r7   Ú
base_modelrP   s   && €r   r“   Ú%FSMTForConditionalGeneration.__init__­  s+   ø€ Ü‰Ñ˜Ô Ü˜vÓ&ˆ
ØŒ
ð 	�‰Ör   c                ó  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[ P
                  ,          R,          RS[R,          RS[ P                  R,          R	S[ P                  R,          R
S[ P                  R,          RS[R,          RS[R,          RS[R,          RS[R,          RS[S[ P                  ,          S[,          /# )rV   r8   Nr   r9   rd  re  r  rÌ   rf  Úlabelsr  r¨   rÍ   rÎ   r2  )	r#   rg  r   rh  rá   ri  r	   rÏ   r   )r^   r_   s   "€r   r`   rŠ  ¶  s   ø€ ÷ ]
ñ ]
á×#Ñ# dÕ*ð]
ñ Ÿ™ tÕ+ð]
ñ !×+Ñ+¨dÕ2ð	]
ñ
 !&× 0Ñ 0°4Õ 7ð]
ñ ™u×0Ñ0Õ1°DÕ8ð]
ñ  �ð]
ñ —|‘| dÕ*ð]
ñ  %Ÿ|™|¨dÕ2ð]
ñ × Ñ  4Õ'ð]
ñ ˜$•;ð]
ñ   $�;ð]
ñ # T�kð]
ñ ˜D•[ð]
ñ  
‰u�|‰|Õ	™Õ	.ñ!]
r   c                ó0  € Ve   TMV P                   P                  pV	e   Rp
V P                  VVVVVVVVV
VVVR7      pV^ ,          pRpV	eG   \        4       pV! VP	                  RV P                   P
                  4      V	P	                  R4      4      pV'       g"   V3VR,          ,           pVe   V3V,           # T# \        VVVP                  VP                  VP                  VP                  VP                  VP                  VP                  R7	      # )u  
decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Indices of decoder input sequence tokens in the vocabulary.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are decoder input IDs?](../glossary#decoder-input-ids)

    FSMT uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
    is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).
decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
    be used by default.
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

Example Translation:

```python
>>> from transformers import AutoTokenizer, FSMTForConditionalGeneration

>>> mname = "facebook/wmt19-ru-en"
>>> model = FSMTForConditionalGeneration.from_pretrained(mname)
>>> tokenizer = AutoTokenizer.from_pretrained(mname)

>>> src_text = "ÐœÐ°ÑˆÐ¸Ð½Ð½Ð¾Ðµ Ð¾Ð±ÑƒÑ‡ÐµÐ½Ð¸Ðµ - Ñ�Ñ‚Ð¾ Ð·Ð´Ð¾Ñ€Ð¾Ð²Ð¾, Ð½Ðµ Ñ‚Ð°Ðº Ð»Ð¸?"
>>> input_ids = tokenizer(src_text, return_tensors="pt").input_ids
>>> outputs = model.generate(input_ids, num_beams=5, num_return_sequences=3)
>>> tokenizer.decode(outputs[0], skip_special_tokens=True)
"Machine learning is great, isn't it?"
```
NF)rÌ   r   r9   rf  re  rd  r  r  r¨   rÍ   rÎ   r�   )	ÚlossÚlogitsr  rk  rl  r  rm  rþ   rn  r!   )r7   rÎ   rB   r   r=  r
  r   r  rk  rl  r  rm  rþ   rn  )rN   r8   r   r9   rd  re  r  rÌ   rf  r�  r  r¨   rÍ   rÎ   r   ÚoutputsÚ	lm_logitsÚmasked_lm_lossÚloss_fctÚoutputs   &&&&&&&&&&&&&&,     r   r°   Ú$FSMTForConditionalGeneration.forwardµ  s'  € ðj &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆàÒØˆIà—*‘*ØØ'Ø)Ø/Ø"7Ø+Ø#9Ø+ØØ/Ø!5Ø#ð ó 
ˆð ˜A•Jˆ	àˆØÒÜ'Ó)ˆHá% i§n¡n°R¸¿¹×9SÑ9SÓ&TÐV\×VaÑVaÐbdÓVeÓfˆNçØ�\ G¨B¥KÕ/ˆFØ3AÒ3M�^Ð%¨Õ.ÐYÐSYÐYäØØØ#×3Ñ3Ø")×"?Ñ"?Ø&×9Ñ9Ø$×5Ñ5Ø&-×&GÑ&GØ")×"?Ñ"?Ø&×9Ñ9ô

ð 
	
r   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )rV   r�  )r#   r   )r^   r_   s   "€r   r`   rŠ    s   ø€ ÷ Dñ D¹E¿L¹Lñ Dr   c                ó@   € \        WP                  P                  4      # rD   )r1   r7   r0   )rN   r�  s   &&r   Ú%prepare_decoder_input_ids_from_labelsÚBFSMTForConditionalGeneration.prepare_decoder_input_ids_from_labels  s   € Ü! &¯+©+×*BÑ*BÓCÐCr   c                óB   € V P                   P                  P                  # rD   ©rB   r`  r½   rt  s   &r   r€  Ú2FSMTForConditionalGeneration.get_output_embeddings  s   € Ø�z‰z×!Ñ!×.Ñ.Ð.r   c                ó:   € WP                   P                  n        R # rD   rž  ry  s   &&r   rƒ  Ú2FSMTForConditionalGeneration.set_output_embeddings  s   € Ø*/�
‰
×ÑÖ'r   )rB   )NNNNNNNNNNNNN)rc   rd   re   rf   rg   r“   r   r°   r›  r€  rƒ  rk   rl   rm   rn   s   @@r   rˆ  rˆ  ¥  sO   ù‡ € ð  Ð÷ó ð ÷]
ñ ]
ó ð]
÷~Dð Dò/÷0ò 0r   rˆ  c                   ó„   a a€ ] tR tRt oRtV 3R ltR t]R 4       t]V3R lR l4       t	RV3R lV 3R	 lllt
R
tVtV ;t# )rH   i  a(  
This module produces sinusoidal positional embeddings of any length.

We don't want to save the weight of this embedding since it's not trained (deterministic) and it can be huge.

Padding symbols are ignored.

These embeddings get automatically extended in forward if more positions is needed.
c                ó(   <€ \         SV `  WV4       R # rD   )rE   r“   )rN   Únum_positionsr¾   rK   rP   s   &&&&€r   r“   Ú&SinusoidalPositionalEmbedding.__init__*  s   ø€ Ü‰Ñ˜°{ÖCr   c                ó,  € V P                  WV4      pVP                  V P                  P                  V P                  P                  R 7      p\
        P                  ! V4      V n        V P                  P                  4        RV P                  n        R# ))r/   r    FN)	rI   r6   rJ   r/   r    r   Ú	ParameterÚdetach_Úrequires_grad)rN   r¤  r¾   rK   rJ   s   &&&& r   Úmake_weightÚ)SinusoidalPositionalEmbedding.make_weight-  sf   € Ø×#Ñ# MÀ+ÓNˆà—‘ §¡×!2Ñ!2¸4¿;¹;×;MÑ;M�ÓNˆÜ—l’l 6Ó*ˆŒØ�‰×ÑÔØ$)ˆ�‰Ö!r   c                óä  € V^,          p\         P                  ! R4      V^,
          ,          p\        P                  ! \        P                  ! V\        P
                  R7      P                  4       V) ,          4      p\        P                  ! V \        P
                  R7      P                  4       P                  ^4      VP                  ^ 4      ,          p\        P                  ! \        P                  ! V4      \        P                  ! V4      .^R7      P                  V R4      pV^,          ^8X  d0   \        P                  ! V\        P                  ! V ^4      .^R7      pVe   ^ WBR3&   V# )zª
Build sinusoidal embeddings.

This matches the implementation in tensor2tensor, but differs slightly from the description in Section 3.5 of
"Attention Is All You Need".
i'  r.   r   r€   r!   )rÀ   Úlogr#   Úexpr$   Úint64rR  r&   ÚcatÚsinÚcosr=  r5   )Únum_embeddingsr¾   rK   Úhalf_dimrt   s   &&&  r   rI   Ú+SinusoidalPositionalEmbedding.get_embedding5  s  € ð ! AÕ%ˆÜ�hŠh�u‹o ¨A¥Õ.ˆÜ�iŠiœŸš X´U·[±[ÔA×GÑGÓIÈSÈDÕPÓQˆÜ�lŠl˜>´·±Ô=×CÑCÓE×OÑOÐPQÓRÐUX×UbÑUbÐcdÓUeÕeˆÜ�iŠiœŸš 3›¬¯ª°3«Ð8¸aÔ@×EÑEÀnÐVXÓYˆØ˜1Õ Ô!ä—)’)˜S¤%§+¢+¨n¸aÓ"@ÐAÀqÔIˆCØÒ"Ø"#ˆC˜Q�ÑØˆ
r   c                ó    <€ V ^8„  d   QhRS[ /# )rV   rK   )Úint)r^   r_   s   "€r   r`   Ú*SinusoidalPositionalEmbedding.__annotate__J  s   ø€ ÷ Uñ U©Cñ Ur   c                óÆ   € V P                  V4      P                  4       p\        P                  ! V^R7      P	                  V4      V,          P                  4       V,           # )z�
Replace non-padding symbols with their position numbers.

Position numbers begin at padding_idx+1. Padding symbols are ignored.
r   )rX   r·  r#   ÚcumsumrT  Úlong)rW   rK   r+   s   && r   Úmake_positionsÚ,SinusoidalPositionalEmbedding.make_positionsI  sJ   € ð �y‰y˜Ó%×)Ñ)Ó+ˆÜ—’˜T qÔ)×1Ñ1°$Ó7¸$Õ>×DÑDÓFÈÕTÐTr   c                óB   <€ V ^8„  d   QhRS[ R,          RS[R,          /# )rV   Úincremental_stateNÚtimestep)r   r   )r^   r_   s   "€r   r`   r¸  W  s)   ø€ ÷ *ñ *ñ  �:ð*ñ ˜4•-ñ	*r   c                ó@  <€ VP                   R,          w  rEV P                  ^,           V,           pW`P                  P                  ^ 4      8”  d'   V P	                  W`P
                  V P                  4       V P                  WP                  4      p\        SV `!  V4      # )z/Input is expected to be of size [bsz x seqlen].r5  )	r"   rK   rJ   r2   rª  r¾   r¼  rE   r°   )	rN   Úinputr¿  rÀ  r<   Úseq_lenÚmax_posr  rP   s	   &&&&    €r   r°   Ú%SinusoidalPositionalEmbedding.forwardW  s   ø€ ð —{‘{ 2•‰ˆØ×"Ñ" QÕ&¨Õ0ˆØ—[‘[×%Ñ% aÓ(Ô(à×Ñ˜W×&8Ñ&8¸$×:JÑ:JÔKØ×'Ñ'¨×/?Ñ/?Ó@ˆ	Ü‰w‰˜yÓ)Ð)r   )rJ   )NN)rc   rd   re   rf   rì   r“   rª  ÚstaticmethodrI   r¼  r°   rk   rl   rm   rn   s   @@r   rH   rH     sN   ù‡ € ñõDò*ð ñó ðð& ÷Uó ðU÷*÷ *õ *r   rH   )r   )r‚   )rˆ  r[  rA   );rì   rÀ   Útypingr   r#   r   r   Útorch.nnr   r   Ú r   rL   Úactivationsr   Úcache_utilsr	   r
   r   Ú
generationr   Úmodeling_outputsr   r   r   r   Úmodeling_utilsr   Úutilsr   r   Úconfiguration_fsmtr   Ú
get_loggerrc   Úloggerr   r,   Úfloat32r?   rA   rx   r}   r1   r3   ÚModulerŽ   r´   rî   r  r&  r–   r4   rY  r[  rˆ  r»   rH   Ú__all__rb   r   r   Ú<module>rÖ     s}  ðñ6 gã Ý ã ß ß 0å &Ý !ß CÑ CÝ )÷ó õ .ß ,Ý *ð 
×	Ò	˜HÓ	%€ðhDòN ô'ð ØØ—m‘mô@ð4 ô˜/ó ó ðò,òGò

ôô+�2—9‘9ô +ô\hm�"—)‘)ô hmôVP
�2—9‘9ô P
ôfM
�"—)‘)ô M
ò`ô|2�—	‘	ô |2ò~@ò%ð ô@*Ð#ó @*ó ð@*ñF ðôô
r0Ð#6¸ó r0óð
r0ôjE* B§L¡Lô E*òP O‚r   