+
    œ°³jü ã                   ó  € R t ^ RIt^ RIt^ RIHt ^ RIHtHtHt ^RIH	t
 ^RIHt ^RIHtHtHt ^RIHt ^RIHtHt ^R	IHt ^R
IHtHtHtHtHtHtHt ^RI H!t! ^RI"H#t#H$t$H%t% ^RI&H't' ]$PP                  ! ])4      t*R R lt+ ! R R]PX                  4      t- ! R R]P\                  4      t/ ! R R]4      t0 ! R R]4      t1 ! R R]P\                  4      t2 ! R R]P\                  4      t3]# ! R R]!4      4       t4 ! R R]44      t5 ! R  R!]44      t6]# ! R" R#]44      4       t7]#! R$R%7       ! R& R']4]4      4       t8]#! R(R%7       ! R) R*]44      4       t9]# ! R+ R,]44      4       t: ! R- R.]44      t; ! R/ R0]4]4      t<. R1Ot=R# )2zPyTorch MVP model.N)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELoss)Úinitialization)ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚ)BaseModelOutputWithPastAndCrossAttentionsÚ!CausalLMOutputWithCrossAttentionsÚSeq2SeqLMOutputÚSeq2SeqModelOutputÚ#Seq2SeqQuestionAnsweringModelOutputÚSeq2SeqSequenceClassifierOutput)ÚPreTrainedModel)Úauto_docstringÚloggingÚtorch_compilable_check©Ú	MvpConfigc                óP   € V ^8„  d   QhR\         P                  R\        R\        /# )é   Ú	input_idsÚpad_token_idÚdecoder_start_token_id©ÚtorchÚTensorÚint)Úformats   "Úe/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/mvp/modeling_mvp.pyÚ__annotate__r'   .   s&   € ÷ ñ ¤%§,¡,ð ¼cð Ô[^ñ ó    c                óÈ   € V P                  V P                  4      pV RRR13,          P                  4       VR&   W#R&   Vf   \        R4      hVP	                  VR8H  V4       V# )z)
Shift input ids one token to the right.
ºNNNNz1self.model.config.pad_token_id has to be defined.éÿÿÿÿ)r*   ºé   NN)r*   é    iœÿÿÿ)Ú	new_zerosÚshapeÚcloneÚ
ValueErrorÚmasked_fill_)r   r   r    Úshifted_input_idss   &&& r&   Úshift_tokens_rightr5   .   sp   € ð "×+Ñ+¨I¯O©OÓ<ÐØ(¨¨C¨R¨C¨Õ0×6Ñ6Ó8Ð�eÑØ4�dÑàÒÜÐLÓMÐMà×"Ñ"Ð#4¸Ñ#<¸lÔKàÐr(   c                   ó^   a a€ ] tR t^?t oRtV3R lV 3R lltRV3R lV 3R llltRtVtV ;t	# )ÚMvpLearnedPositionalEmbeddingzF
This module learns positional embeddings up to a fixed maximum size.
c                ó&   <€ V ^8„  d   QhRS[ RS[ /# )r   Únum_embeddingsÚembedding_dim©r$   )r%   Ú__classdict__s   "€r&   r'   Ú*MvpLearnedPositionalEmbedding.__annotate__D   s   ø€ ÷ Fñ F¡sð F¹3ñ Fr(   c                óV   <€ ^V n         \        SV `	  WP                   ,           V4       R# ©r   N)ÚoffsetÚsuperÚ__init__)Úselfr9   r:   Ú	__class__s   &&&€r&   rB   Ú&MvpLearnedPositionalEmbedding.__init__D   s"   ø€ ð ˆŒÜ‰Ñ˜¯+©+Õ5°}ÖEr(   c                ób   <€ V ^8„  d   QhRS[ P                  RS[RS[ P                  R,          /# )r   r   Úpast_key_values_lengthÚposition_idsNr!   )r%   r<   s   "€r&   r'   r=   J   s5   ø€ ÷ ;ñ ;ÙŸ™ð;Ù?Bð;ÙV[×VbÑVbÐeiÕViñ;r(   c                ó<  <€ Vfi   VP                   R,          w  rE\        P                  ! W"V,           \        P                  V P                  P
                  R7      P                  VR4      pMVP                  ^ 4      p\        SV `%  W0P                  ,           4      # )z3`input_ids' shape is expected to be [bsz x seqlen].:Nr   N)ÚdtypeÚdevicer+   )r0   r"   ÚarangeÚlongÚweightrK   ÚexpandÚ	unsqueezerA   Úforwardr@   )rC   r   rG   rH   ÚbszÚseq_lenrD   s   &&&&  €r&   rQ   Ú%MvpLearnedPositionalEmbedding.forwardJ   s   ø€ ð
 ÒØ$Ÿ?™?¨2Õ.‰LˆCÜ Ÿ<š<Ø&ÀÕ(HÔPU×PZÑPZÐcg×cnÑcn×cuÑcuôç‰f�S˜"‹oñ ð (×1Ñ1°!Ó4ˆLä‰w‰˜|¯k©kÕ9Ó:Ð:r(   )r@   )r.   N©
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__rB   rQ   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©rD   r<   s   @@r&   r7   r7   ?   s%   ù‡ € ñ÷Fó F÷;÷ ;õ ;r(   r7   c                   ó\   a a€ ] tR t^Zt oRtRV3R lV 3R llltRV3R lR lltRtVtV ;t	# )	ÚMvpAttentionz=Multi-headed attention from 'Attention Is All You Need' paperc                óv   <€ V ^8„  d   QhRS[ RS[ RS[R,          RS[R,          RS[R,          RS[R,          /# )r   Ú	embed_dimÚ	num_headsÚdropoutNÚ
is_decoderÚbiasÚ	layer_idx)r$   ÚfloatÚbool)r%   r<   s   "€r&   r'   ÚMvpAttention.__annotate__]   s_   ø€ ÷ Cñ CáðCñ ðCñ ˜•ð	Cñ
 ˜4•KðCñ �T�kðCñ ˜$•;ñCr(   c                ó  <€ \         SV `  4        Wn        W n        W0n        W,          V n        V P
                  V,          V P                  8w  d   \        R V P                   RV R24      hV P
                  R,          V n        W@n        W`n	        \        P                  ! WVR7      V n        \        P                  ! WVR7      V n        \        P                  ! WVR7      V n        \        P                  ! WVR7      V n        R# )z;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).©rf   Ng      à¿)rA   rB   rb   rc   rd   Úhead_dimr2   Úscalingre   rg   r   ÚLinearÚk_projÚv_projÚq_projÚout_proj)rC   rb   rc   rd   re   rf   rg   rD   s   &&&&&&&€r&   rB   ÚMvpAttention.__init__]   sÉ   ø€ ô 	‰ÑÔØ"ŒØ"ŒØŒØ!Õ.ˆŒà�M‰M˜IÕ%¨$¯.©.Ô8ÜØMÈdÏnÉnÐM]Ø$ Y K¨rð3óð ð —}‘} dÕ*ˆŒØ$ŒØ"Œä—i’i 	¸4Ô@ˆŒÜ—i’i 	¸4Ô@ˆŒÜ—i’i 	¸4Ô@ˆŒÜŸ	š	 )¸TÔBˆŽr(   c                óN  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[RS[S[ P                  S[ P                  R,          S[S[ P                  ,          R,          3,          /# )	r   Úhidden_statesÚkey_value_statesNÚpast_key_valuesÚattention_maskÚattn_promptÚoutput_attentionsÚreturn)r"   r#   r   ri   Útuple)r%   r<   s   "€r&   r'   rj   z   s¤   ø€ ÷ p2ñ p2á—|‘|ðp2ñ  Ÿ,™,¨Õ-ðp2ñ  �ð	p2ñ
 Ÿ™ tÕ+ðp2ñ —\‘\ DÕ(ðp2ñ  ðp2ñ 
‰u�|‰|™UŸ\™\¨DÕ0±%¹¿¹Õ2EÈÕ2LÐLÕ	Mñp2r(   c                óâ  € VRJpVP                  4       w  ršpV P                  V4      V P                  ,          pRpVe`   \        V\        4      '       dH   VP
                  P                  V P                  4      pV'       d   VP                  pMVP                  pMTpV'       d   TMTpV'       d\   VeX   V'       dP   XP                  V P                  ,          P                  pVP                  V P                  ,          P                  pMíV P                  V4      pV P                  V4      pVP                  V	R
V P                   V P"                  4      P%                  ^^4      pVP                  V	R
V P                   V P"                  4      P%                  ^^4      pVeX   XP'                  VVV P                  4      w  ppV'       d0   \        V\        4      '       d   RVP
                  V P                  &   VeÌ   \(        P*                  ! V^ ,          P-                  V	R
R
R
4      V.^R7      p\(        P*                  ! V^,          P-                  V	R
R
R
4      V.^R7      pVeb   \(        P.                  ! V	^W¥^ ,          P                  ^4      4      P1                  VP2                  4      p\(        P*                  ! VV.R
R7      pW�P                   ,          R
V P"                  3pVP                  WšV P                   V P"                  4      P%                  ^^4      pVP4                  ! V!  pVP4                  ! V!  pVP4                  ! V!  pVP                  ^4      p\(        P6                  ! VVP%                  ^^4      4      pVP                  4       W�P                   ,          V
V38w  d3   \9        RW�P                   ,          V
V3 RVP                  4        24      hVe„   VP                  4       V	^V
V38w  d$   \9        RV	^V
V3 RVP                  4        24      hVP                  W�P                   V
V4      V,           pVP                  W�P                   ,          V
V4      p\:        P<                  P?                  VR
R7      pV'       dB   VP                  W�P                   V
V4      pVP                  W�P                   ,          V
V4      pMRp\:        P<                  PA                  VV P@                  V PB                  R7      p\(        P6                  ! VV4      pVP                  4       W�P                   ,          W P"                  38w  d6   \9        R	W�P                   W P"                  3 RVP                  4        24      hVP                  W�P                   W P"                  4      pVP%                  ^^4      pVP5                  WšV PD                  4      pV PG                  V4      pVV3# )z#Input shape: Batch x Time x ChannelNFT©Údimz$Attention weights should be of size z	, but is z!Attention mask should be of size ©ÚpÚtrainingz `attn_output` should be of size r+   )$Úsizerr   rn   Ú
isinstancer
   Ú
is_updatedÚgetrg   Úcross_attention_cacheÚself_attention_cacheÚlayersÚkeysÚvaluesrp   rq   Úviewrc   rm   Ú	transposeÚupdater"   ÚcatrO   ÚzerosÚtorK   ÚreshapeÚbmmr2   r   Ú
functionalÚsoftmaxrd   rƒ   rb   rs   )rC   rv   rw   rx   ry   rz   r{   ÚkwargsÚis_cross_attentionrR   Útgt_lenÚ_Úquery_statesr†   Úcurr_past_key_valuesÚcurrent_statesÚ
key_statesÚvalue_statesÚprompt_maskÚ
proj_shapeÚsrc_lenÚattn_weightsÚattn_weights_reshapedÚ
attn_probsÚattn_outputs   &&&&&&&,                 r&   rQ   ÚMvpAttention.forwardz   s  € ð .°TÐ9Ðà'×,Ñ,Ó.‰ˆ�að —{‘{ =Ó1°D·L±LÕ@ˆàˆ
ØÒ&Ü˜/Ô+>×?Ò?Ø,×7Ñ7×;Ñ;¸D¿N¹NÓK�
ß%à+:×+PÑ+PÑ(à+:×+OÑ+OÑ(à'6Ð$ç-?Ñ)À]ˆß /Ò"=Ç*à-×4Ñ4°T·^±^ÕD×IÑIˆJØ/×6Ñ6°t·~±~ÕF×MÑM‰LàŸ™ ^Ó4ˆJØŸ;™; ~Ó6ˆLØ#Ÿ™¨¨b°$·.±.À$Ç-Á-ÓP×ZÑZÐ[\Ð^_Ó`ˆJØ'×,Ñ,¨S°"°d·n±nÀdÇmÁmÓT×^Ñ^Ð_`ÐbcÓdˆLàÒ*à+?×+FÑ+FÀzÐS_Ðae×aoÑaoÓ+pÑ(�
˜Lç%¬*°_ÔFY×*ZÒ*ZØAE�O×.Ñ.¨t¯~©~Ñ>àÒ"ÜŸš K°¥N×$9Ñ$9¸#¸rÀ2ÀrÓ$JÈJÐ#WÐ]^Ô_ˆJÜ Ÿ9š9 k°!¥n×&;Ñ&;¸CÀÀRÈÓ&LÈlÐ%[ÐabÔcˆLØÒ)Ü#Ÿkšk¨#¨q°'Àq½>×;NÑ;NÈqÓ;QÓR×UÑUÐVd×VkÑVkÓl�Ü!&§¢¨K¸Ð+HÈrÔ!S�àŸN™NÕ*¨B°·±Ð>ˆ
Ø#×(Ñ(¨°t·~±~ÀtÇ}Á}ÓU×_Ñ_Ð`aÐcdÓeˆØ#×+Ò+¨ZÑ8ˆØ×'Ò'¨Ñ4ˆ
Ø#×+Ò+¨ZÑ8ˆà—/‘/ !Ó$ˆÜ—y’y ¨z×/CÑ/CÀAÀqÓ/IÓJˆà×ÑÓ 3¯©Õ#7¸À'Ð"JÔJÜØ6¸¿n¹nÕ8LÈgÐW^Ð7_Ð6`ð aØ ×%Ñ%Ó'Ð(ð*óð ð
 Ò%Ø×"Ñ"Ó$¨¨a°¸'Ð(BÔBÜ Ø7¸¸aÀÈ'Ð8RÐ7SÐS\Ð]k×]pÑ]pÓ]rÐ\sÐtóð ð (×,Ñ,¨S·.±.À'È7ÓSÐVdÕdˆLØ'×,Ñ,¨S·>±>Õ-AÀ7ÈGÓTˆLä—}‘}×,Ñ,¨\¸rÐ,ÓBˆçð
 %1×$5Ñ$5°c¿>¹>È7ÐT[Ó$\Ð!Ø0×5Ñ5°c¿N¹NÕ6JÈGÐU\Ó]‰Là$(Ð!ä—]‘]×*Ñ*¨<¸4¿<¹<ÐRV×R_ÑR_Ð*Ó`ˆ
ä—i’i 
¨LÓ9ˆà×ÑÓ #¯©Õ"6¸ÇÁÐ!OÔOÜØ2°C¿¹È×R_ÑR_Ð3`Ð2að bØ×$Ñ$Ó&Ð'ð)óð ð
 "×&Ñ& s¯N©N¸GÇ]Á]ÓSˆØ!×+Ñ+¨A¨qÓ1ˆð "×)Ñ)¨#¸¿¹ÓGˆà—m‘m KÓ0ˆàÐ1Ð1Ð1r(   )rd   rb   rm   re   rp   rg   rc   rs   rr   rn   rq   )g        FTN)NNNNFrU   r^   s   @@r&   r`   r`   Z   s&   ù‡ € ÙG÷Cõ C÷:p2÷ p2ò p2r(   r`   c                   óT   a a€ ] tR t^ít oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚMvpEncoderLayerc                ó    <€ V ^8„  d   QhRS[ /# ©r   Úconfigr   )r%   r<   s   "€r&   r'   ÚMvpEncoderLayer.__annotate__î   s   ø€ ÷ =ñ =™yñ =r(   c                óx  <€ \         SV `  4        VP                  V n        \	        V P                  VP
                  VP                  R 7      V n        \        P                  ! V P                  4      V n
        VP                  V n        \        VP                  ,          V n        VP                  V n        \        P                   ! V P                  VP"                  4      V n        \        P                   ! VP"                  V P                  4      V n        \        P                  ! V P                  4      V n        R# ))rb   rc   rd   N)rA   rB   Úd_modelrb   r`   Úencoder_attention_headsÚattention_dropoutÚ	self_attnr   Ú	LayerNormÚself_attn_layer_normrd   r   Úactivation_functionÚactivation_fnÚactivation_dropoutro   Úencoder_ffn_dimÚfc1Úfc2Úfinal_layer_norm©rC   r¬   rD   s   &&€r&   rB   ÚMvpEncoderLayer.__init__î   sÎ   ø€ Ü‰ÑÔØŸ™ˆŒÜ%Ø—n‘nØ×4Ñ4Ø×,Ñ,ô
ˆŒô
 %'§L¢L°·±Ó$@ˆÔ!Ø—~‘~ˆŒÜ# F×$>Ñ$>Õ?ˆÔØ"(×";Ñ";ˆÔÜ—9’9˜TŸ^™^¨V×-CÑ-CÓDˆŒÜ—9’9˜V×3Ñ3°T·^±^ÓDˆŒÜ "§¢¨T¯^©^Ó <ˆÖr(   c                óÎ   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[R,          RS[S[ P                  S[ P                  R,          3,          /# )r   rv   ry   Úself_attn_promptr{   Nr|   )r"   ÚFloatTensorri   r}   )r%   r<   s   "€r&   r'   r­   þ   sn   ø€ ÷ )+ñ )+á×(Ñ(ð)+ñ ×)Ñ)ð)+ñ  ×+Ñ+ð	)+ñ
   $�;ð)+ñ 
‰u× Ñ ¡%×"3Ñ"3°dÕ":Ð:Õ	;ñ)+r(   c                óh  € TpV P                  VVVVR7      w  r\        P                  P                  WP                  V P                  R7      pWQ,           pV P                  V4      pTpV P                  V P                  V4      4      p\        P                  P                  WP                  V P                  R7      pV P                  V4      p\        P                  P                  WP                  V P                  R7      pWQ,           pV P                  V4      pVP                  \        P                  8X  du   \        P                  ! V4      P                  4       '       gK   \        P                   ! VP                  4      P"                  R,
          p\        P$                  ! W) VR7      pW3# )a]  
Args:
    hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
    attention_mask (`torch.FloatTensor`): attention mask of size
        `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
    self_attn_prompt (`torch.FloatTensor`): prompt of self attention of shape
        `(2, encoder_attention_heads, pro_len, head_dim)`.
    output_attentions (`bool`, *optional*):
        Whether or not to return the attentions tensors of all attention layers. See `attentions` under
        returned tensors for more detail.
)rv   ry   rz   r{   r�   iè  )ÚminÚmax)r²   r   r•   rd   rƒ   r´   r¶   r¹   r·   rº   r»   rJ   r"   Úfloat16ÚisfiniteÚallÚfinforÃ   Úclamp)rC   rv   ry   r¿   r{   Úresidualr£   Úclamp_values   &&&&&   r&   rQ   ÚMvpEncoderLayer.forwardþ   sZ  € ð$ !ˆØ&*§n¡nØ'Ø)Ø(Ø/ð	 '5ó '
Ñ#ˆô Ÿ™×-Ñ-¨m¿|¹|ÐVZ×VcÑVcÐ-ÓdˆØ Õ0ˆØ×1Ñ1°-Ó@ˆà ˆØ×*Ñ*¨4¯8©8°MÓ+BÓCˆÜŸ™×-Ñ-¨m×?VÑ?VÐae×anÑanÐ-ÓoˆØŸ™ Ó/ˆÜŸ™×-Ñ-¨m¿|¹|ÐVZ×VcÑVcÐ-ÓdˆØ Õ0ˆØ×-Ñ-¨mÓ<ˆà×Ñ¤%§-¡-Ô/¼¿ºÀ}Ó8U×8YÑ8Y×8[Ò8[ÜŸ+š+ m×&9Ñ&9Ó:×>Ñ>ÀÕEˆKÜ!ŸKšK¨¸<È[ÔYˆMàÐ*Ð*r(   )	r·   r¶   rd   rb   r¹   rº   r»   r²   r´   ©F©	rV   rW   rX   rY   rB   rQ   r[   r\   r]   r^   s   @@r&   r©   r©   í   s   ù‡ € ÷=ó =÷ )+÷ )+ò )+r(   r©   c                   óX   a a€ ] tR tRt oRV3R lV 3R llltRV3R lR lltRtVtV ;t# )	ÚMvpDecoderLayeri*  c                ó    <€ V ^8„  d   QhRS[ /# r«   r   )r%   r<   s   "€r&   r'   ÚMvpDecoderLayer.__annotate__+  s   ø€ ÷ =ñ =™yñ =r(   c                ó,  <€ \         SV `  4        VP                  V n        \	        V P                  VP
                  VP                  R VR7      V n        VP                  V n        \        VP                  ,          V n        VP                  V n        \        P                  ! V P                  4      V n        \	        V P                  VP
                  VP                  R VR7      V n        \        P                  ! V P                  4      V n        \        P$                  ! V P                  VP&                  4      V n        \        P$                  ! VP&                  V P                  4      V n        \        P                  ! V P                  4      V n        R# )T)rb   rc   rd   re   rg   )rd   re   rg   N)rA   rB   r¯   rb   r`   Údecoder_attention_headsr±   r²   rd   r   rµ   r¶   r·   r   r³   r´   Úencoder_attnÚencoder_attn_layer_normro   Údecoder_ffn_dimr¹   rº   r»   )rC   r¬   rg   rD   s   &&&€r&   rB   ÚMvpDecoderLayer.__init__+  s  ø€ Ü‰ÑÔØŸ™ˆŒä%Ø—n‘nØ×4Ñ4Ø×,Ñ,ØØô
ˆŒð —~‘~ˆŒÜ# F×$>Ñ$>Õ?ˆÔØ"(×";Ñ";ˆÔä$&§L¢L°·±Ó$@ˆÔ!Ü(Ø�N‰NØ×*Ñ*Ø×,Ñ,ØØô
ˆÔô (*§|¢|°D·N±NÓ'CˆÔ$Ü—9’9˜TŸ^™^¨V×-CÑ-CÓDˆŒÜ—9’9˜V×3Ñ3°T·^±^ÓDˆŒÜ "§¢¨T¯^©^Ó <ˆÖr(   c                ó´  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[R,          R
S[R,          RS[S[ P
                  S[S[ P
                  S[ P
                  3,          R,          3,          /
# )r   rv   ry   NÚencoder_hidden_statesÚencoder_attention_maskr¿   Úcross_attn_promptrx   r{   Ú	use_cacher|   )r"   r#   r   ri   r}   rÀ   )r%   r<   s   "€r&   r'   rÑ   G  sä   ø€ ÷ Lñ Lá—|‘|ðLñ Ÿ™ tÕ+ðLñ  %Ÿ|™|¨dÕ2ð	Lñ
 !&§¡¨tÕ 3ðLñ  Ÿ,™,¨Õ-ðLñ !Ÿ<™<¨$Õ.ðLñ  �ðLñ   $�;ðLñ ˜$•;ðLñ 
‰u× Ñ ¡%©×(9Ñ(9¹5×;LÑ;LÐ(LÕ"MÐPTÕ"TÐTÕ	UñLr(   c
           	     óJ  € TpV P                  VVVVVR7      w  r\        P                  P                  WP                  V P                  R7      pW±,           pV P                  V4      pRpVej   TpV P                  VVVVVVR7      w  r\        P                  P                  WP                  V P                  R7      pW±,           pV P                  V4      pTpV P                  V P                  V4      4      p\        P                  P                  WP                  V P                  R7      pV P                  V4      p\        P                  P                  WP                  V P                  R7      pW±,           pV P                  V4      pV3pV'       d   WìV3,          pV# )ar  
Args:
    hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
    attention_mask (`torch.FloatTensor`): attention mask of size
        `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
    encoder_hidden_states (`torch.FloatTensor`):
        cross attention input to the layer of shape `(batch, seq_len, embed_dim)`
    encoder_attention_mask (`torch.FloatTensor`): encoder attention mask of size
        `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
    self_attn_prompt (`torch.FloatTensor`): prompt of self attention of shape
        `(2, decoder_attention_heads, pro_len, head_dim)`.
    cross_attn_prompt (`torch.FloatTensor`): prompt of cross attention of shape
        `(2, decoder_attention_heads, pro_len, head_dim)`.
    past_key_values (`Cache`): cached past key and value projection states
    output_attentions (`bool`, *optional*):
        Whether or not to return the attentions tensors of all attention layers. See `attentions` under
        returned tensors for more detail.
)rv   rx   ry   rz   r{   r�   N)rv   rw   ry   rz   rx   r{   )r²   r   r•   rd   rƒ   r´   rÔ   rÕ   r¶   r¹   r·   rº   r»   )rC   rv   ry   rÙ   rÚ   r¿   rÛ   rx   r{   rÜ   r—   rÉ   Úself_attn_weightsÚcross_attn_weightsÚoutputss   &&&&&&&&&&,    r&   rQ   ÚMvpDecoderLayer.forwardG  s˜  € ð> !ˆð ,0¯>©>Ø'Ø+Ø)Ø(Ø/ð ,:ó ,
Ñ(ˆô Ÿ™×-Ñ-¨m¿|¹|ÐVZ×VcÑVcÐ-ÓdˆØ Õ0ˆØ×1Ñ1°-Ó@ˆð "ÐØ Ò,Ø$ˆHà04×0AÑ0AØ+Ø!6Ø5Ø-Ø /Ø"3ð 1Bó 1Ñ-ˆMô ŸM™M×1Ñ1°-Ç<Á<ÐZ^×ZgÑZgÐ1ÓhˆMØ$Õ4ˆMØ ×8Ñ8¸ÓGˆMð !ˆØ×*Ñ*¨4¯8©8°MÓ+BÓCˆÜŸ™×-Ñ-¨m×?VÑ?VÐae×anÑanÐ-ÓoˆØŸ™ Ó/ˆÜŸ™×-Ñ-¨m¿|¹|ÐVZ×VcÑVcÐ-ÓdˆØ Õ0ˆØ×-Ñ-¨mÓ<ˆà Ð"ˆçØÐ+=Ð>Õ>ˆGàˆr(   )r·   r¶   rd   rb   rÔ   rÕ   r¹   rº   r»   r²   r´   ©N)NNNNNNFTrÍ   r^   s   @@r&   rÏ   rÏ   *  s!   ù‡ € ÷=õ =÷8L÷ Lò Lr(   rÏ   c                   óT   a a€ ] tR tRt oRtV3R lV 3R lltV3R lR ltRtVtV ;t	# )ÚMvpClassificationHeadi—  z-Head for sentence-level classification tasks.c                ó2   <€ V ^8„  d   QhRS[ RS[ RS[ RS[/# )r   Ú	input_dimÚ	inner_dimÚnum_classesÚpooler_dropout)r$   rh   )r%   r<   s   "€r&   r'   Ú"MvpClassificationHead.__annotate__š  s3   ø€ ÷ 
:ñ 
:áð
:ñ ð
:ñ ð	
:ñ
 ñ
:r(   c                óÈ   <€ \         SV `  4        \        P                  ! W4      V n        \        P
                  ! VR 7      V n        \        P                  ! W#4      V n        R# ©)r‚   N)rA   rB   r   ro   ÚdenseÚDropoutrd   rs   )rC   ræ   rç   rè   ré   rD   s   &&&&&€r&   rB   ÚMvpClassificationHead.__init__š  s@   ø€ ô 	‰ÑÔÜ—Y’Y˜yÓ4ˆŒ
Ü—z’z NÔ3ˆŒÜŸ	š	 )Ó9ˆŽr(   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r   rv   r|   ©r"   r#   )r%   r<   s   "€r&   r'   rê   ¦  s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r(   c                óº   € V P                  V4      pV P                  V4      p\        P                  ! V4      pV P                  V4      pV P	                  V4      pV# râ   )rd   rí   r"   Útanhrs   )rC   rv   s   &&r&   rQ   ÚMvpClassificationHead.forward¦  sN   € ØŸ™ ]Ó3ˆØŸ
™
 =Ó1ˆÜŸ
š
 =Ó1ˆØŸ™ ]Ó3ˆØŸ™ mÓ4ˆØÐr(   )rí   rd   rs   rU   r^   s   @@r&   rä   rä   —  s!   ù‡ € Ù7÷
:ó 
:÷÷ ð r(   rä   c                   óH   a a€ ] tR tRt oRtV 3R ltV3R lR ltRtVtV ;t	# )Ú	MvpPrompti¯  z)Layer-wise prompt for encoder or decoder.c           	     óZ  <€ \         SV `  4        VP                  V n        W n        W0n        VP
                  V,          V n        \        P                  ! VP                  R 7      V n	        \        P                  ! VP                  VP
                  4      V n        \        P                  ! \        P                  ! VP
                  VP                  4      \        P                  ! 4       \        P                  ! VP                  V^,          VP
                  ,          4      4      V n        R# rì   )rA   rB   Úprompt_lengthÚ
num_layersrc   r¯   rm   r   rî   rd   Ú	EmbeddingÚprompt_embeddingÚ
Sequentialro   Úprompt_mid_dimÚGELUÚprompt_trans)rC   r¬   rù   rc   rD   s   &&&&€r&   rB   ÚMvpPrompt.__init__²  sº   ø€ Ü‰ÑÔØ#×1Ñ1ˆÔØ$ŒØ"ŒØŸ™¨)Õ3ˆŒÜ—z’z F§N¡NÔ3ˆŒÜ "§¢¨V×-AÑ-AÀ6Ç>Á>Ó RˆÔÜŸMšMÜ�IŠI�f—n‘n f×&;Ñ&;Ó<Ü�GŠG‹IÜ�IŠI�f×+Ñ+¨Z¸!­^¸f¿n¹nÕ-LÓMó
ˆÖr(   c                ó^   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          /# )r   Ú
prompt_idsr|   )r"   r#   r}   )r%   r<   s   "€r&   r'   ÚMvpPrompt.__annotate__À  s(   ø€ ÷ ñ ¡%§,¡,ð ±5¹¿¹Õ3Fñ r(   c                ó2  € V P                  V P                  V4      4      pVP                  V P                  V P                  ^,          V P
                  V P                  4      pV P                  V4      pVP                  . RO4      P                  ^4      pV# )r   )r-   r   r.   é   )
rÿ   rû   r�   rø   rù   rc   rm   rd   ÚpermuteÚsplit)rC   r  Úprompts   && r&   rQ   ÚMvpPrompt.forwardÀ  sw   € Ø×"Ñ" 4×#8Ñ#8¸Ó#DÓEˆØ—‘˜T×/Ñ/°·±À1Õ1DÀdÇnÁnÐVZ×VcÑVcÓdˆØ—‘˜fÓ%ˆØ—‘¢Ó-×3Ñ3°AÓ6ˆØˆr(   )rd   rm   rc   rù   rû   rø   rÿ   rU   r^   s   @@r&   rö   rö   ¯  s   ù‡ € Ù3õ
÷÷ ð r(   rö   c                   óV   a a€ ] tR tRt oRtRtV 3R lt]R 4       tV3R lt	Rt
VtV ;t# )ÚMvpPreTrainedModeliÈ  ÚmodelTc                ó–   <€ \         SV `  V4       \        V\        4      '       d#   \        P
                  ! VP                  4       R # R # râ   )rA   Ú_init_weightsr…   ÚMvpForConditionalGenerationÚinitÚzeros_Úfinal_logits_bias)rC   ÚmodulerD   s   &&€r&   r  Ú MvpPreTrainedModel._init_weightsÎ  s5   ø€ Ü‰Ñ˜fÔ%Ü�fÔ9×:Ò:Ü�KŠK˜×0Ñ0Ö1ñ ;r(   c                ó²   € V P                   P                  p\        P                  ! . RO^ ^^^V..V P                  R7      pRVP                  V4      RV/pV# )r.   ©rK   ry   r   )r.   é   é
   é   r   )r¬   r   r"   ÚtensorrK   Úne)rC   Ú	pad_tokenr   Údummy_inputss   &   r&   r  ÚMvpPreTrainedModel.dummy_inputsÓ  s[   € à—K‘K×,Ñ,ˆ	Ü—L’LÒ"2°Q¸¸2¸qÀ)Ð4LÐ!MÐVZ×VaÑVaÔbˆ	à˜iŸl™l¨9Ó5Ø˜ð
ˆð Ðr(   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r«   r   )r%   r<   s   "€r&   r'   ÚMvpPreTrainedModel.__annotate__È  s   ø‡ ‚ áÑò r(   © )rV   rW   rX   rY   Úbase_model_prefixÚsupports_gradient_checkpointingr  Úpropertyr  Ú__annotate_func__r[   r\   r]   r^   s   @@r&   r  r  È  s3   ù‡ € ð  ÐØ&*Ð#õ2ð
 ñó ð÷ … r(   r  c                   ó\   a a€ ] tR tRt oRtRV3R lV 3R llltR	V3R lR lltRtVtV ;t	# )
Ú
MvpEncoderiÞ  zó
Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a
[`MvpEncoderLayer`].

Args:
    config: MvpConfig
    embed_tokens (nn.Embedding): output embedding
    use_prompt (bool): whether to use prompt
c                ó\   <€ V ^8„  d   QhRS[ RS[P                  R,          RS[R,          /# )r   r¬   Úembed_tokensNÚ
use_prompt)r   r   rú   ri   )r%   r<   s   "€r&   r'   ÚMvpEncoder.__annotate__é  s2   ø€ ÷ ñ ™yð ¹¿¹ÀtÕ8Kð Ñ`dÐgkÕ`kñ r(   c                ó@  <€ \         SV `  V4       VP                  V n        VP                  V n        VP
                  pVP                  V n        VP                  V n	        VP                  '       d   \        P                  ! V4      MR V n        \        P                  ! VP                   W@P                  4      V n        \%        VP                  V4      V n        \        P(                  ! \+        VP,                  4       Uu. uF  p\/        V4      NK  	  up4      V n        \        P2                  ! V4      V n        W0n        V'       d8   VP8                  V n        \;        VVP,                  VP<                  4      V n        RV n         V PC                  4        R# u upi )ç      ð?FN)"rA   rB   rd   Úencoder_layerdropÚ	layerdropr¯   r   Úpadding_idxÚmax_position_embeddingsÚmax_source_positionsÚscale_embeddingÚmathÚsqrtÚembed_scaler   rú   Ú
vocab_sizer)  r7   Úembed_positionsÚ
ModuleListÚrangeÚencoder_layersr©   rŠ   r³   Úlayernorm_embeddingr*  rø   rö   r°   r¿   Úgradient_checkpointingÚ	post_init)rC   r¬   r)  r*  rb   rš   rD   s   &&&&  €r&   rB   ÚMvpEncoder.__init__é  s9  ø€ Ü‰Ñ˜Ô à—~‘~ˆŒØ×1Ñ1ˆŒà—N‘Nˆ	Ø!×.Ñ.ˆÔØ$*×$BÑ$BˆÔ!Ø39×3I×3IÐ3Iœ4Ÿ9š9 YÔ/ÈsˆÔäŸLšL¨×):Ñ):¸I×GWÑGWÓXˆÔä<Ø×*Ñ*Øó 
ˆÔô —m’mÄeÈF×LaÑLaÔFbÓ$cÑFbÀ¤_°VÖ%<ÑFbÑ$cÓdˆŒÜ#%§<¢<°	Ó#:ˆÔ à$ŒßØ!'×!5Ñ!5ˆDÔÜ$-ØØ×%Ñ%Ø×.Ñ.ó%ˆDÔ!ð ',ˆÔ#à�‰Öùò %ds   ÄFc                óä   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[R,          RS[S[,          /# )	r   r   Nry   Úinputs_embedsr{   Úoutput_hidden_statesÚreturn_dictr|   )r"   Ú
LongTensorr#   rÀ   ri   r}   r   )r%   r<   s   "€r&   r'   r+  
  sŠ   ø€ ÷ t
ñ t
á×#Ñ# dÕ*ðt
ñ Ÿ™ tÕ+ðt
ñ ×(Ñ(¨4Õ/ð	t
ñ
   $�;ðt
ñ # T�kðt
ñ ˜D•[ðt
ñ 
‘Õ	 ñt
r(   c                óÜ  € Ve   TMV P                   P                  pVe   TMV P                   P                  pVe   TMV P                   P                  pVe   Ve   \	        R4      hVe)   TpVP
                  p	VP                  RV	R,          4      pM,Ve   VP                  4       RR p	VR,          pM\	        R4      hVf#   V P                  V4      V P                  ,          pV P                  V4      p
W:,           pV P                  V4      p\        P                  P                  W°P                  V P                  R7      pV P                   '       dK   \"        P$                  ! V P&                  4      P)                  V P*                  4      pV P-                  V4      pVe   \/        V P                   VVR7      pV'       d   RMRpV'       d   RMRp\1        V P2                  4       Fª  w  ppV'       d
   Wë3,           pRpV P                  '       d*   \"        P4                  ! . 4      pVV P6                  8  d   RpV'       d   RpM0T! TTV P                   '       d
   XV,          MRVR7      pV^ ,          pV'       g   K™  VV^,          3,           pK¬  	  V'       d
   Wë3,           pV'       g1   \8        ;QJ d    . R	 W¾V3 4       F  NK  	  5# ! R	 W¾V3 4       4      # \;        W¾VR
7      # )ap  
Args:
    input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
        Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
        provide it.

        Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
        [`PreTrainedTokenizer.__call__`] for details.

        [What are input IDs?](../glossary#input-ids)
    attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
        Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

        - 1 for tokens that are **not masked**,
        - 0 for tokens that are **masked**.

        [What are attention masks?](../glossary#attention-mask)
    inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
        Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
        This is useful if you want more control over how to convert `input_ids` indices into associated vectors
        than the model's internal embedding lookup matrix.
    output_attentions (`bool`, *optional*):
        Whether or not to return the attentions tensors of all attention layers. See `attentions` under
        returned tensors for more detail.
    output_hidden_states (`bool`, *optional*):
        Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
        for more detail.
    return_dict (`bool`, *optional*):
        Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
NzDYou cannot specify both input_ids and inputs_embeds at the same timez5You have to specify either input_ids or inputs_embedsr�   )r¬   rA  ry   FT)r¿   r{   c              3   ó0   "  € T F  qf   K  Vx € K  	  R # 5irâ   r!  ©Ú.0Úvs   & r&   Ú	<genexpr>Ú%MvpEncoder.forward.<locals>.<genexpr>{  s   é € ÐeÑ$S˜qŸšÓ$Sùs   ‚Œ
©Úlast_hidden_staterv   Ú
attentionsr+   ©r*   r*   r+   r!  )NN)r¬   r{   rB  rC  r2   r0   r�   r„   r)  r6  r8  r<  r   r•   rd   rƒ   r*  r"   rL   rø   r’   rK   r¿   r   Ú	enumeraterŠ   Úrandr/  r}   r   )rC   r   ry   rA  r{   rB  rC  r—   ÚinputÚinput_shapeÚ	embed_posrv   r  r¿   Úencoder_statesÚall_attentionsÚidxÚencoder_layerÚto_dropÚdropout_probabilityÚlayer_outputss   &&&&&&&,             r&   rQ   ÚMvpEncoder.forward
  sœ  € ðP 2CÒ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ò$DÑ È$Ï+É+×JjÑJjð 	ð &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆð Ò  ]Ò%>ÜÐcÓdÐdØÒ"ØˆEØŸ+™+ˆKØ!Ÿ™ r¨;°r­?Ó;‰IØÒ&Ø'×,Ñ,Ó.¨s°Ð3ˆKØ! (Õ+‰EäÐTÓUÐUàÒ Ø ×-Ñ-¨iÓ8¸4×;KÑ;KÕKˆMà×(Ñ(¨Ó/ˆ	à%Õ1ˆØ×0Ñ0°Ó?ˆÜŸ™×-Ñ-¨m¿|¹|ÐVZ×VcÑVcÐ-Ódˆð �?�?ˆ?ÜŸš d×&8Ñ&8Ó9×<Ñ<¸T¿[¹[ÓIˆJØ#×4Ñ4°ZÓ@Ðð Ò%Ü6Ø—{‘{Ø+Ø-ôˆN÷  4™¸ˆß0™°dˆä"+¨D¯K©KÖ"8ÑˆC�ß#Ø!/Ð2BÕ!B�àˆGØ�}�}ˆ}Ü&+§j¢j°£nÐ#Ø&¨¯©Ô7Ø"�GçØ ,‘á -Ø!Ø"Ø?C¿¿¸Ð&6°sÖ&;ÐTXØ&7ô	!�ð !.¨aÕ 0�ç Ñ Ø!/°=ÀÕ3CÐ2EÕ!E’ñ/ #9÷2  Ø+Ð.>Õ>ˆNçß”5Ñe ]ÀNÑ$SÓe—5Ðe�5Ñe ]ÀNÑ$SÓeÓeÐeÜØ+ÐVdô
ð 	
r(   )rd   r8  r6  r)  r=  r/  r<  rŠ   r2  r0  rø   r¿   r*  )NF)NNNNNNrU   r^   s   @@r&   r'  r'  Þ  s'   ù‡ € ñ÷õ ÷Bt
÷ t
ò t
r(   r'  c                   ó\   a a€ ] tR tRt oRtRV3R lV 3R llltR	V3R lR lltRtVtV ;t	# )
Ú
MvpDecoderi�  zã
Transformer decoder consisting of *config.decoder_layers* layers. Each layer is a [`MvpDecoderLayer`]

Args:
    config: MvpConfig
    embed_tokens (nn.Embedding): output embedding
    use_prompt (bool): whether to use prompt
c                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# )r   r¬   r*  N)r   ri   )r%   r<   s   "€r&   r'   ÚMvpDecoder.__annotate__‹  s   ø€ ÷  ñ  ™yð  ±d¸Tµkñ  r(   c           
     óÈ  <€ \         SV `  V4       VP                  V n        VP                  V n        VP
                  V n        VP                  V n        VP                  '       d!   \        P                  ! VP                  4      MR V n        \        P                  ! VP                   VP                  V P                  4      V n        \%        VP                  VP                  4      V n        \        P(                  ! \+        VP,                  4       Uu. uF  p\/        WR7      NK  	  up4      V n        \        P2                  ! VP                  4      V n        W n        V'       d^   VP8                  V n        \;        VVP,                  VP<                  4      V n        \;        VVP,                  VP<                  4      V n         RV n!        V PE                  4        R# u upi )r-  )rg   FN)#rA   rB   rd   Údecoder_layerdropr/  r   r0  r1  Úmax_target_positionsr3  r4  r5  r¯   r6  r   rú   r7  r)  r7   r8  r9  r:  Údecoder_layersrÏ   rŠ   r³   r<  r*  rø   rö   rÓ   r¿   rÛ   r=  r>  )rC   r¬   r*  ÚirD   s   &&& €r&   rB   ÚMvpDecoder.__init__‹  si  ø€ Ü‰Ñ˜Ô Ø—~‘~ˆŒØ×1Ñ1ˆŒØ!×.Ñ.ˆÔØ$*×$BÑ$BˆÔ!Ø8>×8N×8NÐ8Nœ4Ÿ9š9 V§^¡^Ô4ÐTWˆÔäŸLšL¨×):Ñ):¸F¿N¹NÈD×L\ÑL\Ó]ˆÔÜ<Ø×*Ñ*Ø�N‰Nó 
ˆÔô —m’mÔSXÐY_×YnÑYnÔSoÓ$pÑSoÈa¤_°V×%IÑSoÑ$pÓqˆŒÜ#%§<¢<°·±Ó#?ˆÔ à$ŒßØ!'×!5Ñ!5ˆDÔÜ$-ØØ×%Ñ%Ø×.Ñ.ó%ˆDÔ!ô
 &/ØØ×%Ñ%Ø×.Ñ.ó&ˆDÔ"ð ',ˆÔ#à�‰Öùò' %qs   ÄGc                ó\  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          R	S[R,          R
S[R,          RS[R,          RS[S[,          /# )r   r   Nry   rÙ   rÚ   rx   rA  rÜ   r{   rB  rC  r|   )r"   rD  r#   rÀ   r   ri   r}   r   )r%   r<   s   "€r&   r'   r`  ­  sÕ   ø€ ÷ u
ñ u
á×#Ñ# dÕ*ðu
ñ Ÿ™ tÕ+ðu
ñ  %×0Ñ0°4Õ7ð	u
ñ
 !&× 0Ñ 0°4Õ 7ðu
ñ  �ðu
ñ ×(Ñ(¨4Õ/ðu
ñ ˜$•;ðu
ñ   $�;ðu
ñ # T�kðu
ñ ˜D•[ðu
ñ 
Ñ:Õ	:ñu
r(   c                óœ  € Ve   TMV P                   P                  pV	e   T	MV P                   P                  p	Ve   TMV P                   P                  pV
e   T
MV P                   P                  p
Ve   Ve   \        R4      hVe)   TpVP                  pVP                  RVR,          4      pM,Ve   VP                  4       RR pVR,          pM\        R4      hVf#   V P                  V4      V P                  ,          pV P                  '       d2   V P                  '       d    V'       d   \        P                  R4       RpV'       do   Vfk   Vf   V P                   P                  '       d5   \!        \#        V P                   R7      \#        V P                   R7      4      M\#        V P                   R7      pVe   VP%                  4       M^ p\'        V P                   VVVR7      pVe   Ve   \)        V P                   VVVR7      pV P+                  WÎ4      pWo,           pV P-                  V4      p\.        P0                  P3                  VV P2                  V P                  R	7      pV P4                  '       d\   \6        P8                  ! V P:                  4      P=                  V P>                  4      pV PA                  V4      pV PC                  V4      pV	'       d   RMRpV'       d   RMRpV'       d   Ve   RMRp\E        V PF                  4       FÕ  w  ppV	'       d   VV3,          pV P                  '       d*   \6        PH                  ! . 4      pVV PJ                  8  d   KS  T! TTTTV P4                  '       d
   XV,          MRV P4                  '       d
   XV,          MRVVVR
7	      pV^ ,          pV'       g   K­  VV^,          3,          pVf   KÄ  VV^,          3,          pK×  	  V	'       d   VV3,          pV
'       g7   \L        ;QJ d    . R VVVVV3 4       F  NK  	  5# ! R VVVVV3 4       4      # \O        VVVVVR7      # )a  
Args:
    input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
        Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
        provide it.

        Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
        [`PreTrainedTokenizer.__call__`] for details.

        [What are input IDs?](../glossary#input-ids)
    attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
        Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

        - 1 for tokens that are **not masked**,
        - 0 for tokens that are **masked**.

        [What are attention masks?](../glossary#attention-mask)
    encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*):
        Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention
        of the decoder.
    encoder_attention_mask (`torch.LongTensor` of shape `(batch_size, encoder_sequence_length)`, *optional*):
        Mask to avoid performing cross-attention on padding tokens indices of encoder input_ids. Mask values
        selected in `[0, 1]`:

        - 1 for tokens that are **not masked**,
        - 0 for tokens that are **masked**.

        [What are attention masks?](../glossary#attention-mask)
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks and in the
        cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.

        If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those
        that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of
        all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
    inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
        Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
        This is useful if you want more control over how to convert `input_ids` indices into associated vectors
        than the model's internal embedding lookup matrix.
    output_attentions (`bool`, *optional*):
        Whether or not to return the attentions tensors of all attention layers. See `attentions` under
        returned tensors for more detail.
    output_hidden_states (`bool`, *optional*):
        Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
        for more detail.
    return_dict (`bool`, *optional*):
        Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
NzTYou cannot specify both decoder_input_ids and decoder_inputs_embeds at the same timezEYou have to specify either decoder_input_ids or decoder_inputs_embedszZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...F)r¬   )r¬   rA  ry   rx   )r¬   rA  ry   rÙ   r�   )rÚ   r¿   rÛ   rx   r{   rÜ   c              3   ó2   "  € T F  pVf   K	  Vx € K  	  R # 5irâ   r!  rG  s   & r&   rJ  Ú%MvpDecoder.forward.<locals>.<genexpr>W  s   é € ð ár�AØ÷ ’Ûrùs   ‚�
)rM  rx   rv   rN  Úcross_attentionsr+   rO  r!  )(r¬   r{   rB  rÜ   rC  r2   r0   r�   r„   r)  r6  r=  rƒ   ÚloggerÚwarning_onceÚis_encoder_decoderr
   r	   Úget_seq_lengthr   r   r8  r<  r   r•   rd   r*  r"   rL   rø   r’   rK   r¿   rÛ   rP  rŠ   rQ  r/  r}   r   )rC   r   ry   rÙ   rÚ   rx   rA  rÜ   r{   rB  rC  r—   rR  rS  rG   Ú	positionsrv   r  r¿   rÛ   Úall_hidden_statesÚall_self_attnsÚall_cross_attentionsrW  Údecoder_layerrZ  r[  s   &&&&&&&&&&&,               r&   rQ   ÚMvpDecoder.forward­  só  € ð@ 2CÒ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ò$DÑ È$Ï+É+×JjÑJjð 	ð "+Ò!6‘I¸D¿K¹K×<QÑ<Qˆ	Ø%0Ò%<‘kÀ$Ç+Á+×BYÑBYˆð Ò  ]Ò%>ÜÐsÓtÐtØÒ"ØˆEØ#Ÿ/™/ˆKØ!Ÿ™ r¨;°r­?Ó;‰IØÒ&Ø'×,Ñ,Ó.¨s°Ð3ˆKØ! (Õ+‰EäÐdÓeÐeàÒ Ø ×-Ñ-¨iÓ8¸4×;KÑ;KÕKˆMà×&×&Ð&¨4¯=¯=¨=ßÜ×#Ñ#Øpôð "�	ç˜Ò0ð )Ò4¸¿¹×8V×8VÐ8Vô $¤L¸¿¹Ô$DÄlÐZ^×ZeÑZeÔFfÔgä!¨¯©Ô5ð ð FUÒE` ×!?Ñ!?Ô!AÐfgÐä+Ø—;‘;Ø'Ø)Ø+ô	
ˆð !Ò,Ð1GÒ1SÜ%>Ø—{‘{Ø+Ø5Ø&;ô	&Ð"ð ×(Ñ(¨ÓGˆ	à%Õ1ˆØ×0Ñ0°Ó?ˆäŸ™×-Ñ-¨m¸t¿|¹|ÐVZ×VcÑVcÐ-Ódˆð �?�?ˆ?ÜŸš d×&8Ñ&8Ó9×<Ñ<¸T¿[¹[ÓIˆJØ#×4Ñ4°ZÓ@ÐØ $× 6Ñ 6°zÓ BÐ÷ #7™B¸DÐß0™°dˆß&7Ð<QÒ<]™rÐdhÐä"+¨D¯K©KÖ"8ÑˆC�ç#Ø! mÐ%5Õ5Ð!Ø�}�}ˆ}Ü&+§j¢j°£nÐ#Ø&¨¯©Ô7Ùá)ØØØ%Ø'=Ø;?¿?¿?¸?Ð"2°3Ö"7ÐPTØ=A¿_¿_¸_Ð#4°SÖ#9ÐRVØ /Ø"3Ø#ô
ˆMð *¨!Õ,ˆMß Ñ Ø =°Õ#3Ð"5Õ5�à(Ô4Ø(¨]¸1Õ-=Ð,?Õ?Ò(ñ3 #9÷8  Ø -Ð!1Õ1Ðçß”5ñ à'¨Ð:KÈ^Ð]qÑró—5ð �5ñ à'¨Ð:KÈ^Ð]qÑróó ð ô
 9Ø+Ø+Ø+Ø%Ø1ô
ð 	
r(   )rÛ   rd   r8  r6  r)  r=  r/  r<  rŠ   rc  r0  rø   r¿   r*  rÌ   )
NNNNNNNNNNrU   r^   s   @@r&   r^  r^  �  s'   ù‡ € ñ÷ õ  ÷Du
÷ u
ò u
r(   r^  c                   ó‚   a a€ ] tR tRt oR.tRRRR/tV3R lV 3R lltR tR	 tR
 t	]
RV3R lR ll4       tRtVtV ;t# )ÚMvpModelie  r  zencoder.embed_tokens.weightzshared.weightzdecoder.embed_tokens.weightc                ó    <€ V ^8„  d   QhRS[ /# r«   r   )r%   r<   s   "€r&   r'   ÚMvpModel.__annotate__m  ó   ø€ ÷ ñ ™yñ r(   c                óJ  <€ \         SV `  V4       VP                  VP                  r2VP                  V n        \
        P                  ! W1P                  V4      V n        \        WP                  4      V n
        \        WP                  4      V n        V P                  4        R # râ   )rA   rB   r   r7  r*  r   rú   r¯   Úsharedr'  Úencoderr^  Údecoderr>  )rC   r¬   r0  r7  rD   s   &&  €r&   rB   ÚMvpModel.__init__m  sv   ø€ Ü‰Ñ˜Ô à"(×"5Ñ"5°v×7HÑ7H�ZØ ×+Ñ+ˆŒÜ—l’l :¯~©~¸{ÓKˆŒä! &×*;Ñ*;Ó<ˆŒÜ! &×*;Ñ*;Ó<ˆŒð 	�‰Ör(   c                ó   € V P                   # râ   )r|  ©rC   s   &r&   Úget_input_embeddingsÚMvpModel.get_input_embeddingsz  s   € Ø�{‰{Ðr(   c                ó~   € Wn         V P                   V P                  n        V P                   V P                  n        R # râ   )r|  r}  r)  r~  ©rC   Úvalues   &&r&   Úset_input_embeddingsÚMvpModel.set_input_embeddings}  s'   € ØŒØ$(§K¡Kˆ�‰Ô!Ø$(§K¡Kˆ�‰Ö!r(   c                ó8  € V P                   '       g   Q R 4       hV P                  R4       V P                  P                  P                  R4       V P                  P                  P                  R4       V P                  P
                  P                  R4       R# )zHIf you want to use lightweight tuning, make sure that `use_prompt=True`.FTN)r*  Úrequires_grad_r}  r¿   r~  rÛ   r�  s   &r&   Úset_lightweight_tuningÚMvpModel.set_lightweight_tuning‚  sl   € Ø��ˆÐjÐ jÓjˆà×Ñ˜EÔ"Ø�‰×%Ñ%×4Ñ4°TÔ:Ø�‰×%Ñ%×4Ñ4°TÔ:Ø�‰×&Ñ&×5Ñ5°dÖ;r(   c                ó¼  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[ P                  ,          R,          RS[R,          RS[ P                  R,          R	S[ P                  R,          R
S[R,          RS[R,          RS[R,          RS[R,          RS[S[,          /# )r   r   Nry   Údecoder_input_idsÚdecoder_attention_maskÚencoder_outputsrx   rA  Údecoder_inputs_embedsrÜ   r{   rB  rC  r|   )	r"   rD  r#   ÚlistrÀ   r   ri   r}   r   )r%   r<   s   "€r&   r'   ry  ‹  s  ø€ ÷ g
ñ g
á×#Ñ# dÕ*ðg
ñ Ÿ™ tÕ+ðg
ñ !×+Ñ+¨dÕ2ð	g
ñ
 !&× 0Ñ 0°4Õ 7ðg
ñ ™e×/Ñ/Õ0°4Õ7ðg
ñ  �ðg
ñ ×(Ñ(¨4Õ/ðg
ñ  %×0Ñ0°4Õ7ðg
ñ ˜$•;ðg
ñ   $�;ðg
ñ # T�kðg
ñ ˜D•[ðg
ñ 
Ñ#Õ	#ñg
r(   c                ó¤  € VfH   VfD   Vf   \        R4      h\        WP                  P                  V P                  P                  4      pV
e   T
MV P                  P
                  p
Ve   TMV P                  P                  pV	e   T	MV P                  P                  p	Ve   TMV P                  P                  pVf   V P                  VVVV
VVR7      pMeV'       d^   \        V\        4      '       gH   \        V^ ,          \        V4      ^8”  d
   V^,          MR\        V4      ^8”  d
   V^,          MRR7      pV P                  VVV^ ,          VVVV	V
VVR7
      pV'       g	   Wå,           # \        VP                  VP                   VP"                  VP$                  VP&                  VP                  VP"                  VP$                  R7      # )aA  
decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Indices of decoder input sequence tokens in the vocabulary.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are decoder input IDs?](../glossary#decoder-input-ids)

    Mvp uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
    is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).

    For translation and summarization training, `decoder_input_ids` should be provided. If no
    `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right
    for denoising pre-training following the paper.
decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
    be used by default.

    If you want to change padding behavior, you should read [`modeling_mvp._prepare_decoder_attention_mask`]
    and modify to your needs. See diagram 1 in [the paper](https://huggingface.co/papers/1910.13461) for more
    information on the default strategy.
Nz°If no `decoder_input_ids` or `decoder_inputs_embeds` are passed, `input_ids` cannot be `None`. Please pass either `input_ids` or `decoder_input_ids` or `decoder_inputs_embeds`.)r   ry   rA  r{   rB  rC  rL  ©
r   ry   rÙ   rÚ   rx   rA  rÜ   r{   rB  rC  )rM  rx   Údecoder_hidden_statesÚdecoder_attentionsrk  Úencoder_last_hidden_staterÙ   Úencoder_attentions)r2   r5   r¬   r   r    r{   rB  rÜ   rC  r}  r…   r   Úlenr~  r   rM  rx   rv   rN  rk  )rC   r   ry   rŽ  r�  r�  rx   rA  r‘  rÜ   r{   rB  rC  r—   Údecoder_outputss   &&&&&&&&&&&&&, r&   rQ   ÚMvpModel.forwardŠ  sÌ  € ðT Ò$Ð)>Ò)FØÒ Ü ðUóð ô !3ØŸ;™;×3Ñ3°T·[±[×5WÑ5Wó!Ðð 2CÒ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ò$DÑ È$Ï+É+×JjÑJjð 	ð "+Ò!6‘I¸D¿K¹K×<QÑ<Qˆ	Ø%0Ò%<‘kÀ$Ç+Á+×BYÑBYˆàÒ"Ø"Ÿl™lØ#Ø-Ø+Ø"3Ø%9Ø'ð +ó ‰O÷ ¤¨O¼_×!MÒ!MÜ-Ø"1°!Õ"4Ü47¸Ó4HÈ1Ô4L˜o¨aÖ0ÐRVÜ14°_Ó1EÈÔ1I˜?¨1Ö-ÈtôˆOð Ÿ,™,Ø'Ø1Ø"1°!Õ"4Ø#1Ø+Ø/ØØ/Ø!5Ø#ð 'ó 
ˆ÷ Ø"Õ4Ð4ä!Ø-×?Ñ?Ø+×;Ñ;Ø"1×"?Ñ"?Ø.×9Ñ9Ø,×=Ñ=Ø&5×&GÑ&GØ"1×"?Ñ"?Ø.×9Ñ9ô	
ð 		
r(   )r~  r}  r|  r*  ©NNNNNNNNNNNN)rV   rW   rX   rY   Ú"_keys_to_ignore_on_load_unexpectedÚ_tied_weights_keysrB   r‚  r‡  r‹  r   rQ   r[   r\   r]   r^   s   @@r&   rw  rw  e  s[   ù‡ € à*=Ð)>Ð&à% Ø% ðÐ÷
ó òò0ò
<ð ÷g
ñ g
ó ÷g
ð g
r(   rw  ze
    The MVP Model with a language modeling head. Can be used for various text generation tasks.
    )Úcustom_introc                   ó¬   a a€ ] tR tRt oRR/tV3R lV 3R lltRV3R lV 3R llltV3R lR	 ltR
 t]	RV3R lR ll4       t
V3R lR ltRtVtV ;t# )r  iõ  úlm_head.weightzmodel.shared.weightc                ó    <€ V ^8„  d   QhRS[ /# r«   r   )r%   r<   s   "€r&   r'   Ú(MvpForConditionalGeneration.__annotate__ÿ  s   ø€ ÷ ñ ™yñ r(   c                ó�  <€ \         SV `  V4       \        V4      V n        \        P
                  ! \        P                  ! ^V P                  P                  P                  34      4      V n
        \        P                  ! VP                  V P                  P                  P                  RR7      V n        V P                  4        R# )r-   Frl   N)rA   rB   rw  r  r   ÚBufferr"   r‘   r|  r9   r  ro   r¯   Úlm_headr>  r¼   s   &&€r&   rB   Ú$MvpForConditionalGeneration.__init__ÿ  s€   ø€ Ü‰Ñ˜Ô Ü˜fÓ%ˆŒ
Ü!#§¢¬5¯;ª;¸¸4¿:¹:×;LÑ;L×;[Ñ;[Ð7\Ó+]Ó!^ˆÔÜ—y’y §¡°·±×1BÑ1B×1QÑ1QÐX]Ô^ˆŒð 	�‰Ör(   c                óT   <€ V ^8„  d   QhRS[ RS[ R,          RS[RS[P                  /# )r   Únew_num_tokensÚpad_to_multiple_ofNÚmean_resizingr|   )r$   ri   r   rú   )r%   r<   s   "€r&   r'   r£    s5   ø€ ÷ ñ Ù!ðÙ7:¸TµzðÙY]ðá	�‰ñr(   c                óJ   <€ \         SV `  WV4      pV P                  V4       V# râ   )rA   Úresize_token_embeddingsÚ_resize_final_logits_bias)rC   r©  rª  r«  Únew_embeddingsrD   s   &&&& €r&   r­  Ú3MvpForConditionalGeneration.resize_token_embeddings  s+   ø€ ô ™Ñ8¸Ð]jÓkˆØ×&Ñ& ~Ô6ØÐr(   c                ó$   <€ V ^8„  d   QhRS[ RR/# )r   r©  r|   Nr;   )r%   r<   s   "€r&   r'   r£    s   ø€ ÷ 5ñ 5¹ð 5Àñ 5r(   c                ób  € V P                   P                  R,          pW8:  d   V P                   RRV13,          pMX\        P                  ! ^W,
          3V P                   P                  R7      p\        P
                  ! V P                   V.^R7      p\        P                  ! V4      V n         R# )r-   r*   Nr  r   r+   )r  r0   r"   r‘   rK   r�   r   r¥  )rC   r©  Úold_num_tokensÚnew_biasÚ
extra_biass   &&   r&   r®  Ú5MvpForConditionalGeneration._resize_final_logits_bias  sˆ   € Ø×/Ñ/×5Ñ5°bÕ9ˆØÔ+Ø×-Ñ-¨a°°.°Ð.@ÕA‰HäŸš a¨Õ)HÐ%IÐRV×RhÑRh×RoÑRoÔpˆJÜ—y’y $×"8Ñ"8¸*Ð!EÈ1ÔMˆHÜ!#§¢¨8Ó!4ˆÖr(   c                óp   € V P                   P                  4        V P                  P                  R 4       R# ©FN©r  r‹  r¦  rŠ  r�  s   &r&   r‹  Ú2MvpForConditionalGeneration.set_lightweight_tuning  ó$   € Ø�
‰
×)Ñ)Ô+Ø�‰×#Ñ# EÖ*r(   c                óä  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[ P                  ,          R,          RS[R,          RS[ P                  R,          R	S[ P                  R,          R
S[ P                  R,          RS[R,          RS[R,          RS[R,          RS[R,          RS[S[,          /# )r   r   Nry   rŽ  r�  r�  rx   rA  r‘  ÚlabelsrÜ   r{   rB  rC  r|   )	r"   rD  r#   r’  rÀ   r   ri   r}   r   )r%   r<   s   "€r&   r'   r£    s  ø€ ÷ v
ñ v
á×#Ñ# dÕ*ðv
ñ Ÿ™ tÕ+ðv
ñ !×+Ñ+¨dÕ2ð	v
ñ
 !&× 0Ñ 0°4Õ 7ðv
ñ ™e×/Ñ/Õ0°4Õ7ðv
ñ  �ðv
ñ ×(Ñ(¨4Õ/ðv
ñ  %×0Ñ0°4Õ7ðv
ñ × Ñ  4Õ'ðv
ñ ˜$•;ðv
ñ   $�;ðv
ñ # T�kðv
ñ ˜D•[ðv
ñ  
‘Õ	 ñ!v
r(   c                ó"  € Ve   TMV P                   P                  pV	e\   V
'       d   \        P                  R4       Rp
Vf9   Vf5   \	        W�P                   P
                  V P                   P                  4      pV P                  VVVVVVVVV
VVVR7      pV P                  V^ ,          4      V P                  ,           pRpV	eG   \        4       pV! VP                  RV P                   P                  4      V	P                  R4      4      pV'       g"   V3VR,          ,           pVe   V3V,           # T# \        VVVP                  VP                  VP                   VP"                  VP$                  VP&                  VP(                  R7	      # )a
  
decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Indices of decoder input sequence tokens in the vocabulary.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are decoder input IDs?](../glossary#decoder-input-ids)

    Mvp uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
    is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).

    For translation and summarization training, `decoder_input_ids` should be provided. If no
    `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right
    for denoising pre-training following the paper.
decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
    be used by default.

    If you want to change padding behavior, you should read [`modeling_mvp._prepare_decoder_attention_mask`]
    and modify to your needs. See diagram 1 in [the paper](https://huggingface.co/papers/1910.13461) for more
    information on the default strategy.
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

Example of summarization:

Fine-tuning a model
```python
>>> import torch
>>> from transformers import AutoTokenizer, MvpForConditionalGeneration

>>> tokenizer = AutoTokenizer.from_pretrained("RUCAIBox/mvp")
>>> model = MvpForConditionalGeneration.from_pretrained("RUCAIBox/mvp")

>>> inputs = tokenizer(
...     "Summarize: You may want to stick it to your boss and leave your job, but don't do it if these are your reasons.",
...     return_tensors="pt",
... )
>>> labels = tokenizer("Bad Reasons To Quit Your Job", return_tensors="pt")["input_ids"]

>>> loss = model(**inputs, labels=labels).loss
>>> loss.backward()
```

Inference after the model fine-tuned
```python
>>> with torch.no_grad():
...     generated_ids = model.generate(**inputs)

>>> generated_text = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)
```
NzJThe `use_cache` argument is changed to `False` since `labels` is provided.F)ry   rŽ  r�  r�  rx   rA  r‘  rÜ   r{   rB  rC  r,   ©	ÚlossÚlogitsrx   r•  r–  rk  r—  rÙ   r˜  r+   )r¬   rC  rl  Úwarningr5   r   r    r  r¦  r  r   r�   r7  r   rx   r•  r–  rk  r—  rÙ   r˜  )rC   r   ry   rŽ  r�  r�  rx   rA  r‘  r½  rÜ   r{   rB  rC  r—   rà   Ú	lm_logitsÚmasked_lm_lossÚloss_fctÚoutputs   &&&&&&&&&&&&&&,     r&   rQ   Ú#MvpForConditionalGeneration.forward  s}  € ðR &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆàÒßÜ—‘ÐkÔlØˆIØ Ò(Ð-BÒ-JÜ$6ØŸK™K×4Ñ4°d·k±k×6XÑ6Xó%Ð!ð —*‘*ØØ)Ø/Ø+Ø#9Ø+Ø'Ø"7ØØ/Ø!5Ø#ð ó 
ˆð —L‘L ¨¥Ó,¨t×/EÑ/EÕEˆ	àˆØÒÜ'Ó)ˆHÙ% i§n¡n°R¸¿¹×9OÑ9OÓ&PÐRX×R]ÑR]Ð^`ÓRaÓbˆNçØ�\ G¨B¥KÕ/ˆFØ3AÒ3M�^Ð%¨Õ.ÐYÐSYÐYäØØØ#×3Ñ3Ø")×"?Ñ"?Ø&×9Ñ9Ø$×5Ñ5Ø&-×&GÑ&GØ")×"?Ñ"?Ø&×9Ñ9ô

ð 
	
r(   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r   r½  rñ   )r%   r<   s   "€r&   r'   r£  •  s   ø€ ÷ hñ h¹E¿L¹Lñ hr(   c                ój   € \        WP                  P                  V P                  P                  4      # râ   )r5   r¬   r   r    )rC   r½  s   &&r&   Ú%prepare_decoder_input_ids_from_labelsÚAMvpForConditionalGeneration.prepare_decoder_input_ids_from_labels•  s#   € Ü! &¯+©+×*BÑ*BÀDÇKÁK×DfÑDfÓgÐgr(   )r  r¦  r  )NT©NNNNNNNNNNNNN)rV   rW   rX   rY   rž  rB   r­  r®  r‹  r   rQ   rÊ  r[   r\   r]   r^   s   @@r&   r  r  õ  sg   ù‡ € ð 	Ð/ðÐ÷ó ÷õ ÷5ð 5ò+ð ÷v
ñ v
ó ðv
÷ph÷ hð hr(   r  z„
    Mvp model with a sequence classification/head on top (a linear layer on top of the pooled output) e.g. for GLUE
    tasks.
    c                   ód   a a€ ] tR tRt oV3R lV 3R lltR t]RV3R lR ll4       tRtVt	V ;t
# )	ÚMvpForSequenceClassificationi™  c                ó    <€ V ^8„  d   QhRS[ /# r«   r   )r%   r<   s   "€r&   r'   Ú)MvpForSequenceClassification.__annotate__   rz  r(   c                óà   <€ \         SV `  ! V3/ VB  \        V4      V n        \	        VP
                  VP
                  VP                  VP                  4      V n        V P                  4        R # râ   )
rA   rB   rw  r  rä   r¯   Ú
num_labelsÚclassifier_dropoutÚclassification_headr>  )rC   r¬   r—   rD   s   &&,€r&   rB   Ú%MvpForSequenceClassification.__init__   sZ   ø€ Ü‰Ò˜Ñ* 6Ò*Ü˜fÓ%ˆŒ
Ü#8Ø�N‰NØ�N‰NØ×ÑØ×%Ñ%ó	$
ˆÔ ð 	�‰Ör(   c                óp   € V P                   P                  4        V P                  P                  R 4       R# r¸  )r  r‹  rÔ  rŠ  r�  s   &r&   r‹  Ú3MvpForSequenceClassification.set_lightweight_tuning­  s&   € Ø�
‰
×)Ñ)Ô+Ø× Ñ ×/Ñ/°Ö6r(   c                óÐ  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[ P                  ,          R,          RS[ P                  R,          RS[ P                  R,          R	S[ P                  R,          R
S[R,          RS[R,          RS[R,          RS[R,          RS[S[,          /# )r   r   Nry   rŽ  r�  r�  rA  r‘  r½  rÜ   r{   rB  rC  r|   )r"   rD  r#   r’  rÀ   ri   r}   r   )r%   r<   s   "€r&   r'   rÐ  ²  s  ø€ ÷ N
ñ N
á×#Ñ# dÕ*ðN
ñ Ÿ™ tÕ+ðN
ñ !×+Ñ+¨dÕ2ð	N
ñ
 !&× 0Ñ 0°4Õ 7ðN
ñ ™e×/Ñ/Õ0°4Õ7ðN
ñ ×(Ñ(¨4Õ/ðN
ñ  %×0Ñ0°4Õ7ðN
ñ × Ñ  4Õ'ðN
ñ ˜$•;ðN
ñ   $�;ðN
ñ # T�kðN
ñ ˜D•[ðN
ñ 
Ñ0Õ	0ñN
r(   c                óŽ  € Ve   TMV P                   P                  pVe   Rp	Vf'   Ve#   \        RV P                  P                   24      hV P                  VVVVVVVV	V
VVR7      pV^ ,          pVP                  V P                   P                  4      P                  VP                  4      p\        \        P                  ! VP                  ^4      4      P                  4       ^8H  R4       VVR3,          p\        VP                  ^ ,          VP                  ^ ,          ,          ^8¬  R4       VP!                  VP#                  ^ 4      RVP#                  R4      4      R,          pV P%                  V4      pRpVEeÂ   V P                   P&                  fª   V P                   P(                  ^8X  d   RV P                   n        M|V P                   P(                  ^8”  dQ   VP*                  \        P,                  8X  g    VP*                  \        P.                  8X  d   R	V P                   n        MR
V P                   n        V P                   P&                  R8X  dV   \1        4       pV P                   P(                  ^8X  d'   V! VP3                  4       VP3                  4       4      pMšV! VV4      pM�V P                   P&                  R	8X  dH   \5        4       pV! VP!                  RV P                   P(                  4      VP!                  R4      4      pM.V P                   P&                  R
8X  d   \7        4       pV! VV4      pV'       g"   V3VR,          ,           pVe   V3V,           # T# \9        VVVP:                  VP<                  VP>                  VP@                  VPB                  VPD                  VPF                  R7	      # )af	  
decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Indices of decoder input sequence tokens in the vocabulary.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are decoder input IDs?](../glossary#decoder-input-ids)

    Mvp uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
    is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).

    For translation and summarization training, `decoder_input_ids` should be provided. If no
    `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right
    for denoising pre-training following the paper.
decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
    be used by default.

    If you want to change padding behavior, you should read [`modeling_mvp._prepare_decoder_attention_mask`]
    and modify to your needs. See diagram 1 in [the paper](https://huggingface.co/papers/1910.13461) for more
    information on the default strategy.
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).

Example of single-label classification:

Fine-tuning a model on `num_labels` classes
```python
>>> import torch
>>> from transformers import AutoTokenizer, MvpForSequenceClassification

>>> num_labels = 2  # for example, this is a binary classification task
>>> tokenizer = AutoTokenizer.from_pretrained("RUCAIBox/mvp")
>>> model = MvpForSequenceClassification.from_pretrained("RUCAIBox/mvp", num_labels=num_labels)

>>> inputs = tokenizer("Classify: Hello, my dog is cute", return_tensors="pt")
>>> labels = torch.tensor(1)  # the real label for inputs

>>> loss = model(**inputs, labels=labels).loss
>>> loss.backward()
```

Inference after the model fine-tuned
```python
>>> with torch.no_grad():
...     logits = model(**inputs).logits

>>> predicted_class_id = logits.argmax()
```
NFz8Passing input embeddings is currently not supported for ©
ry   rŽ  r�  r�  rA  r‘  rÜ   r{   rB  rC  z7All examples must have the same number of <eos> tokens.r*   z3Each example must contain at least one <eos> token.Ú
regressionÚsingle_label_classificationÚmulti_label_classificationr,   r¿  r+   )r*   r+   r*   )$r¬   rC  ÚNotImplementedErrorrD   rV   r  ÚeqÚeos_token_idr’   rK   r   r"   Úunique_consecutiveÚsumÚnumelr0   r�   r„   rÔ  Úproblem_typerÒ  rJ   rM   r$   r   Úsqueezer   r   r   rx   r•  r–  rk  r—  rÙ   r˜  )rC   r   ry   rŽ  r�  r�  rA  r‘  r½  rÜ   r{   rB  rC  r—   rà   rv   Úeos_maskÚselectedÚsentence_representationrÁ  rÀ  rÅ  rÆ  s   &&&&&&&&&&&&&,         r&   rQ   Ú$MvpForSequenceClassification.forward±  s+  € ðJ &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆØÒØˆIàÒ Ò!:Ü%ØJÈ4Ï>É>×KbÑKbÐJcÐdóð ð —*‘*ØØ)Ø/Ø#9Ø+Ø'Ø"7ØØ/Ø!5Ø#ð ó 
ˆð   �
ˆà—<‘< §¡× 8Ñ 8Ó9×<Ñ<¸]×=QÑ=QÓRˆäÜ×$Ò$ X§\¡\°!£_Ó5×;Ñ;Ó=ÀÑBØEô	
ð ! ¨1 Õ-ˆÜØ�N‰N˜1Õ ×!4Ñ!4°QÕ!7Õ7¸1Ñ<ØAô	
ð #+§-¡-°×0BÑ0BÀ1Ó0EÀrÈ=×K]ÑK]Ð^`ÓKaÓ"bÐckÕ"lÐØ×)Ñ)Ð*AÓBˆàˆØÓØ�{‰{×'Ñ'Ò/Ø—;‘;×)Ñ)¨QÔ.Ø/;�D—K‘KÕ,Ø—[‘[×+Ñ+¨aÔ/°V·\±\ÄUÇZÁZÔ5OÐSY×S_ÑS_Ôch×clÑclÔSlØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ô7Ü"›9�Ø—;‘;×)Ñ)¨QÔ.Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# F¨FÓ3‘DØ—‘×)Ñ)Ð-JÔJÜ+Ó-�Ù §¡¨B°·±×0FÑ0FÓ GÈÏÉÐUWËÓY‘Ø—‘×)Ñ)Ð-IÔIÜ,Ó.�Ù ¨Ó/�ßØ�Y ¨¥Õ,ˆFØ)-Ò)9�T�G˜fÕ$ÐE¸vÐEä.ØØØ#×3Ñ3Ø")×"?Ñ"?Ø&×9Ñ9Ø$×5Ñ5Ø&-×&GÑ&GØ")×"?Ñ"?Ø&×9Ñ9ô

ð 
	
r(   )rÔ  r  rœ  ©rV   rW   rX   rY   rB   r‹  r   rQ   r[   r\   r]   r^   s   @@r&   rÎ  rÎ  ™  s6   ù‡ € ÷ó ò7ð ÷N
ñ N
ó ÷N
ð N
r(   rÎ  c                   óX   a a€ ] tR tRt oV 3R ltR t]RV3R lR ll4       tRtVt	V ;t
# )ÚMvpForQuestionAnsweringiC  c                óö   <€ \         SV `  V4       ^Vn        VP                  V n        \        V4      V n        \
        P                  ! VP                  VP                  4      V n        V P                  4        R# r?   )
rA   rB   rÒ  rw  r  r   ro   Úhidden_sizeÚ
qa_outputsr>  r¼   s   &&€r&   rB   Ú MvpForQuestionAnswering.__init__E  s[   ø€ Ü‰Ñ˜Ô àˆÔØ ×+Ñ+ˆŒä˜fÓ%ˆŒ
ÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Ör(   c                óp   € V P                   P                  4        V P                  P                  R 4       R# r¸  )r  r‹  rï  rŠ  r�  s   &r&   r‹  Ú.MvpForQuestionAnswering.set_lightweight_tuningQ  s$   € Ø�
‰
×)Ñ)Ô+Ø�‰×&Ñ& uÖ-r(   c                óø  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[ P                  ,          R,          RS[ P                  R,          RS[ P                  R,          R	S[ P                  R,          R
S[ P                  R,          RS[R,          RS[R,          RS[R,          RS[R,          RS[S[,          /# )r   r   Nry   rŽ  r�  r�  Ústart_positionsÚend_positionsrA  r‘  rÜ   r{   rB  rC  r|   )r"   r#   rD  r’  rÀ   ri   r}   r   )r%   r<   s   "€r&   r'   Ú$MvpForQuestionAnswering.__annotate__V  s!  ø€ ÷ F
ñ F
á—<‘< $Õ&ðF
ñ Ÿ™ tÕ+ðF
ñ !×+Ñ+¨dÕ2ð	F
ñ
 !&× 0Ñ 0°4Õ 7ðF
ñ ™e×/Ñ/Õ0°4Õ7ðF
ñ ×)Ñ)¨DÕ0ðF
ñ ×'Ñ'¨$Õ.ðF
ñ ×(Ñ(¨4Õ/ðF
ñ  %×0Ñ0°4Õ7ðF
ñ ˜$•;ðF
ñ   $�;ðF
ñ # T�kðF
ñ ˜D•[ðF
ñ  
Ñ4Õ	4ñ!F
r(   c                ó  € Ve   TMV P                   P                  pVe   Ve   Rp
V P                  VVVVVVV	V
VVVR7      pV^ ,          pV P                  V4      pVP	                  ^RR7      w  ppVP                  R4      P                  4       pVP                  R4      P                  4       pRpVeÆ   VeÂ   \        VP                  4       4      ^8”  d   VP                  R4      p\        VP                  4       4      ^8”  d   VP                  R4      pVP                  ^4      pVP                  ^ V4      pVP                  ^ V4      p\        VR7      pV! VV4      pV! VV4      pVV,           ^,          pV'       g#   VV3VR,          ,           pVe   V3V,           # T# \        VVVVP                  VP                  VP                  VP                  VP                   VP"                  VP$                  R7
      # )	a¨	  
decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Indices of decoder input sequence tokens in the vocabulary.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are decoder input IDs?](../glossary#decoder-input-ids)

    Mvp uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
    is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).

    For translation and summarization training, `decoder_input_ids` should be provided. If no
    `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right
    for denoising pre-training following the paper.
decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
    be used by default.

    If you want to change padding behavior, you should read [`modeling_mvp._prepare_decoder_attention_mask`]
    and modify to your needs. See diagram 1 in [the paper](https://huggingface.co/papers/1910.13461) for more
    information on the default strategy.

Example:

Fine-tuning a model for extractive question answering, and our model also supports generative question answering
using `BartForConditionalGeneration`
```python
>>> import torch
>>> from transformers import AutoTokenizer, MvpForQuestionAnswering

>>> tokenizer = AutoTokenizer.from_pretrained("RUCAIBox/mvp")
>>> model = MvpForQuestionAnswering.from_pretrained("RUCAIBox/mvp")

>>> inputs = tokenizer(
...     "Answer the following question: Who was Jim Henson? [SEP] Jim Henson was a nice puppet",
...     return_tensors="pt",
... )
>>> target_start_index = torch.tensor([18])
>>> target_end_index = torch.tensor([19])

>>> loss = model(**inputs, start_positions=target_start_index, end_positions=target_end_index).loss
>>> loss.backward()
```

Inference after the model fine-tuned
```python
>>> with torch.no_grad():
...     outputs = model(**inputs)

>>> answer_start_index = outputs.start_logits.argmax()
>>> answer_end_index = outputs.end_logits.argmax()

>>> predict_answer_tokens = inputs.input_ids[0, answer_start_index : answer_end_index + 1]
>>> predict_answer = tokenizer.decode(predict_answer_tokens)
```
NFrÚ  r   )Úignore_indexr,   )
rÀ  Ústart_logitsÚ
end_logitsrx   r•  r–  rk  r—  rÙ   r˜  r+   )r¬   rC  r  rï  r  rå  Ú
contiguousr™  r„   rÈ   r   r   rx   r•  r–  rk  r—  rÙ   r˜  )rC   r   ry   rŽ  r�  r�  rô  rõ  rA  r‘  rÜ   r{   rB  rC  r—   rà   Úsequence_outputrÁ  rù  rú  Ú
total_lossÚignored_indexrÅ  Ú
start_lossÚend_lossrÆ  s   &&&&&&&&&&&&&&,           r&   rQ   ÚMvpForQuestionAnswering.forwardU  s  € ðV &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆØÒ&¨=Ò+DØˆIà—*‘*ØØ)Ø/Ø#9Ø+Ø'Ø"7ØØ/Ø!5Ø#ð ó 
ˆð " !�*ˆà—‘ Ó1ˆØ#)§<¡<°°r <Ó#:Ñ ˆ�jØ#×+Ñ+¨BÓ/×:Ñ:Ó<ˆØ×'Ñ'¨Ó+×6Ñ6Ó8ˆ
àˆ
ØÒ&¨=Ò+Dä�?×'Ñ'Ó)Ó*¨QÔ.Ø"1×"9Ñ"9¸"Ó"=�Ü�=×%Ñ%Ó'Ó(¨1Ô,Ø -× 5Ñ 5°bÓ 9�à(×-Ñ-¨aÓ0ˆMØ-×3Ñ3°A°}ÓEˆOØ)×/Ñ/°°=ÓAˆMä'°]ÔCˆHÙ! ,°Ó@ˆJÙ 
¨MÓ:ˆHØ$ xÕ/°1Õ4ˆJçàØðð ˜•õˆFð 0:Ò/E�Z�M FÕ*ÐQÈ6ÐQä2ØØ%Ø!Ø#×3Ñ3Ø")×"?Ñ"?Ø&×9Ñ9Ø$×5Ñ5Ø&-×&GÑ&GØ")×"?Ñ"?Ø&×9Ñ9ô
ð 	
r(   )r  rÒ  rï  rÌ  rê  r^   s   @@r&   rì  rì  C  s1   ù‡ € õ
ò.ð ÷F
ñ F
ó ÷F
ð F
r(   rì  c                   ó<   a a€ ] tR tRt oRtV 3R ltR tRtVtV ;t	# )ÚMvpDecoderWrapperià  z±
This wrapper class is a helper class to correctly load pretrained checkpoints when the causal language model is
used in combination with the [`EncoderDecoderModel`] framework.
c                óf   <€ \         SV `  V4       \        V4      V n        V P	                  4        R # râ   )rA   rB   r^  r~  r>  r¼   s   &&€r&   rB   ÚMvpDecoderWrapper.__init__æ  s&   ø€ Ü‰Ñ˜Ô Ü! &Ó)ˆŒØ�‰Ör(   c                ó&   € V P                   ! V/ VB # râ   ©r~  )rC   Úargsr—   s   &*,r&   rQ   ÚMvpDecoderWrapper.forwardë  s   € Ø�|Š|˜TÐ, VÑ,Ð,r(   r  rU   r^   s   @@r&   r  r  à  s   ù‡ € ñõ
÷
-ò -r(   r  c                   ól   a a€ ] tR tRt oRR/tV 3R ltR tR tR t]	RV3R lR	 ll4       t
R
tVtV ;t# )ÚMvpForCausalLMiï  r¡  z!model.decoder.embed_tokens.weightc                óæ   <€ R Vn         RVn        \        SV `  V4       \	        V4      V n        \        P                  ! VP                  VP                  RR7      V n
        V P                  4        R# )TFrl   N)re   rn  rA   rB   r  r  r   ro   rî  r7  r¦  r>  r¼   s   &&€r&   rB   ÚMvpForCausalLM.__init__ò  sX   ø€ Ø ˆÔØ$)ˆÔ!Ü‰Ñ˜Ô Ü& vÓ.ˆŒ
ä—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒð 	�‰Ör(   c                óB   € V P                   P                  P                  # râ   ©r  r~  r)  r�  s   &r&   r‚  Ú#MvpForCausalLM.get_input_embeddingsý  s   € Ø�z‰z×!Ñ!×.Ñ.Ð.r(   c                ó:   € WP                   P                  n        R # râ   r  r…  s   &&r&   r‡  Ú#MvpForCausalLM.set_input_embeddings   s   € Ø*/�
‰
×ÑÖ'r(   c                óp   € V P                   P                  4        V P                  P                  R 4       R# r¸  r¹  r�  s   &r&   r‹  Ú%MvpForCausalLM.set_lightweight_tuning  r»  r(   c                ó®  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          R	S[R,          R
S[R,          RS[R,          RS[R,          RS[S[ P                  ,          RS[S[,          /# )r   r   Nry   rÙ   rÚ   rx   rA  r½  rÜ   r{   rB  rC  Úlogits_to_keepr|   )	r"   rD  r#   rÀ   r   ri   r$   r}   r   )r%   r<   s   "€r&   r'   ÚMvpForCausalLM.__annotate__  sÿ   ø€ ÷ O
ñ O
á×#Ñ# dÕ*ðO
ñ Ÿ™ tÕ+ðO
ñ  %×0Ñ0°4Õ7ð	O
ñ
 !&× 1Ñ 1°DÕ 8ðO
ñ  �ðO
ñ ×(Ñ(¨4Õ/ðO
ñ × Ñ  4Õ'ðO
ñ ˜$•;ðO
ñ   $�;ðO
ñ # T�kðO
ñ ˜D•[ðO
ñ ™eŸl™lÕ*ðO
ñ 
Ñ2Õ	2ñO
r(   c                óâ  € V	e   T	MV P                   P                  p	V
e   T
MV P                   P                  p
Ve   TMV P                   P                  pV P                  P                  VVVVVVVV	V
VR7
      pV^ ,          p\        V\        4      '       d   \        V) R4      MTpV P                  VRVR3,          4      pRpVeG   \        4       pV! VP                  RV P                   P                  4      VP                  R4      4      pV'       g"   V3VR,          ,           pVe   V3V,           # T# \        VVVP                  VP                  VP                   VP"                  R7      # )aø  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

Example:

```python
>>> from transformers import AutoTokenizer, MvpForCausalLM

>>> tokenizer = AutoTokenizer.from_pretrained("RUCAIBox/mvp")
>>> model = MvpForCausalLM.from_pretrained("RUCAIBox/mvp")

>>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
>>> outputs = model(**inputs)

>>> logits = outputs.logits
>>> list(logits.shape)
[1, 8, 50267]
```Nr”  r*   r,   )rÀ  rÁ  rx   rv   rN  rk  r+   )r¬   r{   rB  rC  r  r~  r…   r$   Úslicer¦  r   r�   r7  r   rx   rv   rN  rk  )rC   r   ry   rÙ   rÚ   rx   rA  r½  rÜ   r{   rB  rC  r  r—   rà   rv   Úslice_indicesrÁ  rÀ  rÅ  rÆ  s   &&&&&&&&&&&&&,       r&   rQ   ÚMvpForCausalLM.forward  sm  € ðN 2CÒ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ò$DÑ È$Ï+É+×JjÑJjð 	ð &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆð —*‘*×$Ñ$ØØ)Ø"7Ø#9Ø+Ø'ØØ/Ø!5Ø#ð %ó 
ˆð   �
ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒÜ'Ó)ˆHÙ˜FŸK™K¨¨D¯K©K×,BÑ,BÓCÀVÇ[Á[ÐQSÃ_ÓUˆDçØ�Y ¨¥Õ,ˆFØ'+Ò'7�D�7˜VÕ#ÐC¸VÐCä0ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø$×5Ñ5ô
ð 	
r(   )r¦  r  )NNNNNNNNNNNr.   )rV   rW   rX   rY   rž  rB   r‚  r‡  r‹  r   rQ   r[   r\   r]   r^   s   @@r&   r  r  ï  sG   ù‡ € Ø*Ð,OÐPÐõ	ò/ò0ò+ð ÷O
ñ O
ó ÷O
ð O
r(   r  )r  r  rì  rÎ  rw  r  )>rZ   r4  r"   r   Útorch.nnr   r   r   Ú r   r  Úactivationsr   Úcache_utilsr   r	   r
   Ú
generationr   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   r   Úmodeling_utilsr   Úutilsr   r   r   Úconfiguration_mvpr   Ú
get_loggerrV   rl  r5   rú   r7   ÚModuler`   r©   rÏ   rä   rö   r  r'  r^  rw  r  rÎ  rì  r  r  Ú__all__r!  r(   r&   Ú<module>r*     s½  ðñ ã ã Ý ß AÑ Aå &Ý !ß CÑ CÝ )ß JÝ 9÷÷ ñ õ .ß DÑ DÝ (ð 
×	Ò	˜HÓ	%€õô"; B§L¡Lô ;ô6P2�2—9‘9ô P2ôf:+Ð0ô :+ôziÐ0ô iôZ˜BŸI™Iô ô0�—	‘	ô ð2 ô˜ó ó ðô*`
Ð#ô `
ôFa
Ð#ô a
ðH ôL
Ð!ó L
ó ðL
ñ^ ðôô
\hÐ"4°oó \hóð
\hñ~ ðôôa
Ð#5ó a
óða
ðH ôX
Ð0ó X
ó ðX
ôx-Ð*ô -ôh
Ð'¨ô h
òV‚r(   