+
    œ°³jŸ  ã                   óÒ  € ^ RI Ht ^ RIt^ RIHu Ht ^ RIHt ^RIHt	 ^RI
Ht ^RIHtHt ^RIHt ^RIHtHtHt ^R	IHtHtHt ^R
IHt ^RIHtHtHtHt ^RI H!t!H"t" ^RI#H$t$H%t% ^RI&H't'H(t( ^RI)H*t* ^RI+H,t,H-t-H.t. ^RI/H0t0 ^RI1H2t2H3t3 ^RI4H5t5H6t6 ^RI7H8t8 ]! R4       ! R R]Pr                  4      4       t: ! R R]4      t;R t< ! R R]Pr                  4      t= ! R R]Pr                  4      t>R t?]! R 4      R>R! l4       t@R" R# ltAR?R$ R% lltB]! ]@4       ! R& R']Pr                  4      4       tC ! R( R)]Pr                  4      tD] ! R* R+]Pr                  4      4       tE ! R, R-]Pr                  4      tF ! R. R/]4      tG]- ! R0 R1](4      4       tH]- ! R2 R3]H4      4       tIR@R4 R5 lltJ]- ! R6 R7]H]4      4       tK ! R8 R9]]H4      tL ! R: R;]]H4      tM ! R< R=]]H4      tN. RAOtOR# )Bé    )ÚCallableN)Únn)Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_experts_implementationÚuse_kernel_forward_from_hubÚuse_kernelized_func)Úcreate_causal_maskÚcreate_recurrent_attention_maskÚ!create_sliding_window_causal_mask)ÚFlashAttentionKwargs)ÚGenericForQuestionAnsweringÚ GenericForSequenceClassificationÚGenericForTokenClassificationÚGradientCheckpointingLayer)ÚMoeCausalLMOutputWithPastÚMoeModelOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputs©ÚMiniMaxConfigÚRMSNormc                   óZ   a a€ ] tR t^4t oRV3R lV 3R llltV3R lR ltR tRtVtV ;t	# )ÚMiniMaxRMSNormc                ó$   <€ V ^8„  d   QhRS[ RR/# )é   ÚepsÚreturnN)Úfloat)ÚformatÚ__classdict__s   "€Úm/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/minimax/modeling_minimax.pyÚ__annotate__ÚMiniMaxRMSNorm.__annotate__6   s   ø€ ÷ $ñ $©ð $¸$ñ $ó    c                óŽ   <€ \         SV `  4        \        P                  ! \        P
                  ! V4      4      V n        W n        R# )z-
MiniMaxRMSNorm is equivalent to T5LayerNorm
N)ÚsuperÚ__init__r   Ú	ParameterÚtorchÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizer+   Ú	__class__s   &&&€r0   r6   ÚMiniMaxRMSNorm.__init__6   s/   ø€ ô 	‰ÑÔÜ—l’l¤5§:¢:¨kÓ#:Ó;ˆŒØ #Ör3   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r*   Úhidden_statesr,   ©r8   ÚTensor)r.   r/   s   "€r0   r1   r2   >   s#   ø€ ÷ ;ñ ;¡U§\¡\ð ;±e·l±lñ ;r3   c                ó:  € VP                   pVP                  \        P                  4      pVP	                  ^4      P                  RRR7      pV\        P                  ! W0P                  ,           4      ,          pV P                  VP                  V4      ,          # )r*   T)Úkeepdiméÿÿÿÿ)	ÚdtypeÚtor8   Úfloat32ÚpowÚmeanÚrsqrtr;   r:   )r<   rB   Úinput_dtypeÚvariances   &&  r0   ÚforwardÚMiniMaxRMSNorm.forward>   sw   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯ª°H×?TÑ?TÕ4TÓ(UÕUˆØ�{‰{˜]×-Ñ-¨kÓ:Õ:Ð:r3   c                ó^   € \        V P                  P                  4       R V P                   2# )z, eps=)Útupler:   Úshaper;   )r<   s   &r0   Ú
extra_reprÚMiniMaxRMSNorm.extra_reprE   s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr3   )r;   r:   )g�íµ ÷Æ°>)
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r6   rP   rU   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r>   r/   s   @@r0   r(   r(   4   s%   ù‡ € ÷$õ $÷;ð ;÷Jò Jr3   r(   c                   ó�   a a€ ] tR t^It oRtV 3R ltR tV3R lR ltV 3R ltV3R lR lt	V3R	 lR
 lt
V3R lR ltRtVtV ;t# )ÚMiniMaxCacheFc                ó2   <€ \         SV `  4        . V n        R # ©N)r5   r6   Úlinear_cache©r<   r>   s   &€r0   r6   ÚMiniMaxCache.__init__N   s   ø€ Ü‰ÑÔØ02ˆÖr3   c                ó²   € \        \        V P                  4      V^,           4       F  pV P                  P                  . 4       K   	  W P                  V&   R# )é   N)ÚrangeÚlenrc   Úappend)r<   Ú	layer_idxrc   Ú_s   &&& r0   Úset_linear_cacheÚMiniMaxCache.set_linear_cacheR   sD   € ä”s˜4×,Ñ,Ó-¨y¸1­}Ö=ˆAØ×Ñ×$Ñ$ RÖ(ñ >à'3×Ñ˜)Ó$r3   c                ó    <€ V ^8„  d   QhRS[ /# )r*   rk   ©Úint)r.   r/   s   "€r0   r1   ÚMiniMaxCache.__annotate__X   s   ø€ ÷ ñ ©#ñ r3   c                óL   € V\        V 4      8  d   V P                  V,          # R # rb   )ri   rc   )r<   rk   s   &&r0   Úget_linear_cacheÚMiniMaxCache.get_linear_cacheX   s"   € Ø”s˜4“yÔ Ø×$Ñ$ YÕ/Ð/Ùr3   c                óZ   <€ \        \        SV `	  4       \        V P                  4      4      # rb   )Úmaxr5   Ú__len__ri   rc   rd   s   &€r0   rx   ÚMiniMaxCache.__len__]   s"   ø€ Ü”5‘7‘?Ó$¤c¨$×*;Ñ*;Ó&<Ó=Ð=r3   c                ó    <€ V ^8„  d   QhRS[ /# )r*   Úrepeatsrp   )r.   r/   s   "€r0   r1   rr   `   s   ø€ ÷ Hñ H©sñ Hr3   c                ó  € \        \        V 4      4       Fp  pV P                  V,          . 8w  d4   V P                  V,          P                  V^ R7      V P                  V&   KN  V P                  V,          P                  V4       Kr  	  R# )r   ©ÚdimN)rh   ri   rc   Úrepeat_interleaveÚlayersÚbatch_repeat_interleave)r<   r{   rk   s   && r0   r�   Ú$MiniMaxCache.batch_repeat_interleave`   sl   € Üœs 4›yÖ)ˆIØ× Ñ  Õ+¨rÔ1Ø/3×/@Ñ/@ÀÕ/K×/]Ñ/]Ð^eÐklÐ/]Ó/m�×!Ñ! )Ó,à—‘˜IÕ&×>Ñ>¸wÖGó	 *r3   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r*   ÚindicesrC   )r.   r/   s   "€r0   r1   rr   g   s   ø€ ÷ Eñ E©E¯L©Lñ Er3   c                ó  € \        \        V 4      4       Fh  pV P                  V,          . 8w  d,   V P                  V,          VR 3,          V P                  V&   KF  V P                  V,          P	                  V4       Kj  	  R# ).N)rh   ri   rc   r€   Úbatch_select_indices)r<   r„   rk   s   && r0   r†   Ú!MiniMaxCache.batch_select_indicesg   sd   € Üœs 4›yÖ)ˆIØ× Ñ  Õ+¨rÔ1Ø/3×/@Ñ/@ÀÕ/KÈGÐUXÈLÕ/Y�×!Ñ! )Ó,à—‘˜IÕ&×;Ñ;¸GÖDó	 *r3   c                ó$   <€ V ^8„  d   QhRS[ RR/# )r*   Útokens_to_remover,   Nrp   )r.   r/   s   "€r0   r1   rr   n   s   ø€ ÷ Jñ J¡Sð J¨Tñ Jr3   c                ó   € \        R 4      h)z+MiniMaxCache does not support `crop` method)ÚRuntimeError)r<   r‰   s   &&r0   ÚcropÚMiniMaxCache.cropn   s   € ÜÐHÓIÐIr3   )rc   )rW   rX   rY   rZ   Úis_croppabler6   rm   rt   rx   r�   r†   rŒ   r[   r\   r]   r^   s   @@r0   r`   r`   I   sO   ù‡ € ð €Lõ3ò4÷ð õ
>÷Hð H÷Eð E÷J÷ Jð Jr3   r`   c                ób   € Ve+   V P                   pWR,          ,          P                  V4      p V # )ze
Tunes out the hidden states for padding tokens, see https://github.com/state-spaces/mamba/issues/66
)ºNNNr�   N)rH   rI   )rB   Úattention_maskrH   s   && r0   Úapply_mask_to_padding_statesr’   r   s3   € ð
 Ò!Ø×#Ñ#ˆØ&¸
Õ)CÕC×GÑGÈÓNˆàÐr3   c                   ó`   a a€ ] tR t^~t oV3R lV 3R lltR tR tRV3R lR lltRtVt	V ;t
# )	ÚMiniMaxLightningAttentionc                ó&   <€ V ^8„  d   QhRS[ RS[/# ©r*   Úconfigrk   ©r%   rq   )r.   r/   s   "€r0   r1   Ú&MiniMaxLightningAttention.__annotate__   s   ø€ ÷ 8ñ 8™}ð 8¹ñ 8r3   c                ó‚  <€ \         SV `  4        W n        \        VR R4      ;'       g    VP                  VP
                  ,          V n        VP
                  V n        VP                  V n        VP                  V n        \        VP                  ,          V n        \        V P                  V P
                  ,          4      V n        \        P                  ! VP                  V P
                  V P                  ,          ^,          RR7      V n        \        P                  ! V P
                  V P                  ,          VP                  RR7      V n        \        P                  ! VP                  V P
                  V P                  ,          RR7      V n        V P'                  4       pV P)                  V4      w  rEp\        P*                  ! V4      V n        \        P*                  ! V4      V n        \        P*                  ! V4      V n        \        P*                  ! V4      V n        VP4                  V,          V n        R# )Úhead_dimNF©Úbias)r5   r6   rk   Úgetattrr=   Únum_attention_headsr›   Únum_hidden_layersÚ
block_sizer   Ú
hidden_actÚact_fnr(   Únormr   ÚLinearÚqkv_projÚout_projÚoutput_gateÚget_slope_rateÚdecay_factorsÚBufferÚ
slope_rateÚquery_decayÚ	key_decayÚdiagonal_decayÚlayer_typesÚ
layer_type)r<   r—   rk   r¬   r­   r®   r¯   r>   s   &&&    €r0   r6   Ú"MiniMaxLightningAttention.__init__   s•  ø€ Ü‰ÑÔØ"ŒÜ ¨
°DÓ9×mÐm¸V×=OÑ=OÐSY×SmÑSmÕ=mˆŒØ#)×#=Ñ#=ˆÔ Ø!'×!9Ñ!9ˆÔØ ×+Ñ+ˆŒä˜V×.Ñ.Õ/ˆŒÜ" 4§=¡=°4×3KÑ3KÕ#KÓLˆŒ	ÜŸ	š	 &×"4Ñ"4°d×6NÑ6NÐQU×Q^ÑQ^Õ6^ÐabÕ6bÐinÔoˆŒÜŸ	š	 $×":Ñ":¸T¿]¹]Õ"JÈF×L^ÑL^ÐejÔkˆŒÜŸ9š9 V×%7Ñ%7¸×9QÑ9QÐTX×TaÑTaÕ9aÐhmÔnˆÔà×(Ñ(Ó*ˆ
Ø15×1CÑ1CÀJÓ1OÑ.ˆ äŸ)š) JÓ/ˆŒÜŸ9š9 [Ó1ˆÔÜŸš 9Ó-ˆŒÜ Ÿiši¨Ó7ˆÔà ×,Ñ,¨YÕ7ˆŽr3   c                ó:  € ^^^V P                   ,          ,          ,          p\        P                  ! V P                   4      ^,           p^V P                  V P                  ^,
          R,           ,          ,
          R,           pW,          pWC,          pVR,          pV# )rg   gñhãˆµøä>)r�   NN)rŸ   r8   Úarangerk   r    )r<   ÚbaseÚexponentÚfactorÚrates   &    r0   r©   Ú(MiniMaxLightningAttention.get_slope_rate—   sz   € Ø�A˜!˜d×6Ñ6Õ6Õ7Õ8ˆÜ—<’< × 8Ñ 8Ó9¸AÕ=ˆØ�T—^‘^ t×'=Ñ'=ÀÕ'AÀDÕ'HÕIÕIÈDÕPˆà�~ˆØ�}ˆØ�MÕ"ˆàˆr3   c                óÔ  € \         P                  ! V P                  4      ^,           p\         P                  ! V) VR,          ,          4      p\         P                  ! V) V P                  VR,          ,
          ,          4      pVR,          VR,          ,
          pVR,          pW,          p\         P                  ! V^ 8¬  V) \        R4      4      p\         P                  ! V4      pW4V3# )rg   z-inf)r�   N)Nr�   )NNr�   r�   )r8   r´   r¡   ÚexpÚwherer-   )r<   r¬   Úblock_size_ranger­   r®   r¯   s   &&    r0   rª   Ú'MiniMaxLightningAttention.decay_factors¢   s¹   € Ü Ÿ<š<¨¯©Ó8¸1Õ<Ðä—i’i  Ð.>¸wÕ.GÕ GÓHˆÜ—I’I˜z˜k¨T¯_©_Ð?OÐPWÕ?XÕ-XÕYÓZˆ	à)¨'Õ2Ð5EÀgÕ5NÕNˆØ'Ð(8Õ9ˆØ#Õ4ˆÜŸš ^°qÑ%8¸>¸/Ì5ÐQWË=ÓYˆÜŸš >Ó2ˆà ~Ð5Ð5r3   c                óR  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  S[ P                  R,          S[S[ P                  ,          R,          3,          /# ©r*   rB   Úposition_embeddingsr‘   NÚpast_key_valuesÚkwargsr,   ©r8   rD   rS   r   r   r   )r.   r/   s   "€r0   r1   r™   °   sŸ   ø€ ÷ [/ñ [/á—|‘|ð[/ñ #¡5§<¡<±·±Ð#=Õ>ð[/ñ Ÿ™ tÕ+ð	[/ñ
  �ð[/ñ Ñ-Õ.ð[/ñ 
‰u�|‰|™UŸ\™\¨DÕ0±%¹¿¹Õ2EÈÕ2LÐLÕ	Mñ[/r3   c                ó6	  € VP                   w  rgpWpP                  ,           ^,
          V P                  ,          p	V P                  V P                  V4      4      p
\	        W£4      p
V
P                  WgV P                  ^V P                  ,          4      p
\        P                  ! W P                  ^R7      w  r¼pVP                  ^^4      pVP                  ^^4      pVP                  ^^4      pRpVe   VP                  V P                  4      pVEfí   \        P                  ! W`P                  V P                  V P                  4      P                  V4      p. p\        V	4       EF”  pVV P                  ,          p\!        VV P                  ,           V4      pVV,
          pVRRVV13,          pVRRVV13,          pVRRVV13,          pV P"                  RRV13,          pV P$                  RV) R13,          pV P&                  RRRV1RV13,          p\        P(                  ! V P*                  ) V,          4      p\        P,                  ! VVP                  RR4      4      p\        P,                  ! VV,          V4      p\        P,                  ! VV,          V4      pVV,           pVP/                  V4       \        P,                  ! VV,          P                  RR4      V4      pVV,          V,           pEK—  	  MÓ\        P(                  ! V P*                  ) 4      p . p\        V4       F¡  pVRRVV^,           13,          pVRRVV^,           13,          pVRRVV^,           13,          p\        P,                  ! VP                  RR4      V4      p!V V,          V!,           p\        P,                  ! VV4      pVP/                  V4       K£  	  \        P0                  ! VRR7      pVP                  ^^4      pVP                  WgV P                  V P                  ,          4      pV P3                  V4      p\4        P6                  ! V P9                  V4      4      V,          pV P;                  V4      pVe   VP=                  V P                  V4       Wþ3# )rg   r}   Nr�   rG   éþÿÿÿ)rT   r¡   r£   r¦   r’   ÚreshaperŸ   r›   r8   ÚsplitÚ	transposert   rk   ÚzerosrI   rh   Úminr­   r®   r¯   r»   r¬   Úmatmulrj   Úcatr¤   ÚFÚsigmoidr¨   r§   rm   )"r<   rB   rÁ   r‘   rÂ   rÃ   Ú
batch_sizeÚseq_lenr=   Ú
num_blocksÚ
qkv_statesÚquery_statesÚ
key_statesÚvalue_statesÚattn_weights_interÚattn_outputÚiÚ	start_idxÚend_idxÚcurrent_block_sizeÚcurrent_query_statesÚcurrent_key_statesÚcurrent_value_statesÚcurrent_query_decayÚcurrent_key_decayÚcurrent_diagonal_decayÚblock_decayÚattn_weights_intraÚattn_output_intraÚattn_output_interÚcurrent_attn_outputÚnext_attn_weights_interÚratioÚcurrent_attn_weights_inters"   &&&&&,                            r0   rP   Ú!MiniMaxLightningAttention.forward°   sP  € ð ,9×+>Ñ+>Ñ(ˆ
˜[Ø§¡Õ/°!Õ3¸¿¹ÕGˆ
à—[‘[ §¡¨}Ó!=Ó>ˆ
Ü1°*ÓMˆ
Ø×'Ñ'¨
¸T×=UÑ=UÐWXÐ[_×[hÑ[hÕWhÓiˆ
ä16·²¸ZÏÉÐ\]Ô1^Ñ.ˆ ,à#×-Ñ-¨a°Ó3ˆØ×)Ñ)¨!¨QÓ/ˆ
Ø#×-Ñ-¨a°Ó3ˆð "ÐØÒ&Ø!0×!AÑ!AÀ$Ç.Á.Ó!QÐàÓ%Ü!&§¢¨Z×9QÑ9QÐSW×S`ÑS`Ðbf×boÑboÓ!p×!sÑ!sØó"Ðð ˆKÜ˜:×&�Ø §¡Õ/�	Ü˜i¨$¯/©/Õ9¸7ÓC�Ø%,¨yÕ%8Ð"à'3°A°q¸)ÀGÐ:KÐ4KÕ'LÐ$Ø%/°°1°iÀÐ6GÐ0GÕ%HÐ"Ø'3°A°q¸)ÀGÐ:KÐ4KÕ'LÐ$à&*×&6Ñ&6°qÐ:MÐ;MÐ:MÐ7MÕ&NÐ#Ø$(§N¡N°1Ð7IÐ6IÑ6JÐ3JÕ$KÐ!Ø)-×)<Ñ)<¸QÀÐCVÐDVÐCVÐXkÐYkÐXkÐ=kÕ)lÐ&Ü#Ÿiši¨¯©Ð(8Ð;MÕ(MÓN�ô &+§\¢\Ð2FÐHZ×HdÑHdÐegÐikÓHlÓ%mÐ"Ü$)§L¢LÐ1CÐF\Õ1\Ð^rÓ$sÐ!ô %*§L¢LÐ1EÐH[Õ1[Ð]oÓ$pÐ!ð '8Ð:KÕ&KÐ#Ø×"Ñ"Ð#6Ô7ô +0¯,ª,Ø'Ð*;Õ;×FÑFÀrÈ2ÓNÐPdó+Ð'ð &8¸+Õ%EÐH_Õ%_Ó"ò; 'ô@ —I’I˜tŸ™Ð.Ó/ˆEØˆKÜ˜7–^�Ø'3°A°q¸!¸aÀ!½e¸)°OÕ'DÐ$Ø%/°°1°a¸!¸a½%°i°Õ%@Ð"Ø'3°A°q¸!¸aÀ!½e¸)°OÕ'DÐ$ä-2¯\ª\Ð:L×:VÑ:VÐWYÐ[]Ó:^Ð`tÓ-uÐ*Ø%*Ð-?Õ%?ÐB\Õ%\Ð"Ü&+§l¢lÐ3GÐI[Ó&\Ð#à×"Ñ"Ð#6Ö7ñ $ô —i’i °Ô4ˆð "×+Ñ+¨A¨qÓ1ˆØ!×)Ñ)¨*¸t×?WÑ?WÐZ^×ZgÑZgÕ?gÓhˆØ—i‘i Ó,ˆÜ—i’i × 0Ñ 0°Ó ?Ó@À;ÕNˆØ—m‘m KÓ0ˆð Ò&Ø×,Ñ,¨T¯^©^Ð=OÔPàÐ.Ð.r3   )r£   r¡   r¯   r›   r®   rk   r±   r¤   rŸ   r    r§   r¨   r¦   r­   r¬   rb   )rW   rX   rY   rZ   r6   r©   rª   rP   r[   r\   r]   r^   s   @@r0   r”   r”   ~   s+   ù‡ € ÷8ó 8ò0	ò6÷[/÷ [/ò [/r3   r”   c                   óÎ   a a€ ] tR tRt o]! RRR7      RV3R lV 3R lll4       t]]! RRR7      RV3R lR ll4       4       t]P                  ! 4       ]
R	 4       4       tR
tVtV ;t# )ÚMiniMaxRotaryEmbeddingi  Údevicez5.18)Úversionc                ó    <€ V ^8„  d   QhRS[ /# ©r*   r—   r$   )r.   r/   s   "€r0   r1   Ú#MiniMaxRotaryEmbedding.__annotate__  s   ø€ ÷ Oñ O™}ñ Or3   c                óä  <€ \         SV `  4        VP                  V n        VP                  V n        Wn        V P
                  P                  R ,          V n        V P                  pV P                  R8w  d   \        V P                  ,          pV! V P
                  V4      w  q@n
        \        P                  ! VRR7      V n        \        P                  ! VP                  4       RR7      V n        R# )Ú	rope_typeÚdefaultF)Ú
persistentN)r5   r6   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr—   Úrope_parametersrô   Úcompute_default_rope_parametersr   Úattention_scalingr   r«   Úinv_freqÚcloneÚoriginal_inv_freq)r<   r—   rî   Úrope_init_fnrý   r>   s   &&&  €r0   r6   ÚMiniMaxRotaryEmbedding.__init__  s¬   ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àŒàŸ™×4Ñ4°[ÕAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÔ&Ü.¨t¯~©~Õ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÔ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÔ!NˆÖr3   c                óP   <€ V ^8„  d   QhRS[ RS[S[P                  S[3,          /# )r*   r—   r,   )r%   rS   r8   rD   r-   )r.   r/   s   "€r0   r1   rò   "  s.   ø€ ÷ 5ñ 5±ð 5ÑY^Ñ_d×_kÑ_kÑmrÐ_rÕYsñ 5r3   c           	     ó0  € V P                   R,          p\        V RR4      ;'       g    V P                  V P                  ,          pRpRV\        P
                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetar›   Nç      ð?©rH   )rú   rž   r=   rŸ   r8   r´   r-   rI   )r—   rî   rÃ   rµ   r~   Úattention_factorrý   s   &&,    r0   rû   Ú6MiniMaxRotaryEmbedding.compute_default_rope_parameters   s~   € ð ×%Ñ% lÕ3ˆÜ�f˜j¨$Ó/×cÐc°6×3EÑ3EÈ×IcÑIcÕ3cˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r3   c                ó”  € V P                   R,          P                  4       P                  VP                  ^ ,          R^4      P	                  VP
                  4      pVR	,          P                  4       p\        VP
                  P                  \        4      '       d2   VP
                  P                  R8w  d   VP
                  P                  MRp\        VRR7      ;_uu_ 4        VP                  4       VP                  4       ,          P                  ^^4      p\        P                  ! Wf3RR7      pVP                  4       V P                  ,          pVP                  4       V P                  ,          p	R R R 4       XP	                  VP                   R7      X	P	                  VP                   R7      3#   + '       g   i     LH; i)
NÚmpsÚcpuF)Údevice_typeÚenabledr}   r  )Nr�   NrG   )r�   Nr�   )rý   r-   ÚexpandrT   rI   rî   Ú
isinstanceÚtypeÚstrr    rÉ   r8   rÍ   Úcosrü   ÚsinrH   )
r<   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedr  ÚfreqsÚembr  r  s
   &&&       r0   rP   ÚMiniMaxRotaryEmbedding.forward4  sE  € ð !ŸM™M¨-Õ8×>Ñ>Ó@×GÑGÈ×HZÑHZÐ[\ÕH]Ð_aÐcdÓe×hÑhÐij×iqÑiqÓrÐØ ,¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆÜ¨¸U×CÖCØ&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÕN×YÑYÐZ[Ð]^Ó_ˆEÜ—)’)˜U˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆC÷	 Dð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷ D×Cús   Ã&BF7Æ7G	)rü   r—   rý   rø   rÿ   rù   rô   rb   )rW   rX   rY   rZ   r   r6   Ústaticmethodrû   r8   Úno_gradr   rP   r[   r\   r]   r^   s   @@r0   rí   rí     su   ù‡ € Ù�X vÔ.÷Oô Oó /ðOð  Ù�X vÔ.÷5ñ 5ó /ó ð5ð$ ‡]‚]ƒ_Øñ<ó ó ÷<ð <r3   rí   c                óÌ   € V RRV P                   R,          ^,          13,          pV RV P                   R,          ^,          R13,          p\        P                  ! V) V3RR7      # )z*Rotates half the hidden dims of the input..Nr}   rG   )rT   r8   rÍ   )r  Úx1Úx2s   &  r0   Úrotate_halfr   D  sZ   € à	
ˆ3Ð"�!—'‘'˜"•+ Õ"Ð"Ð"Õ	#€BØ	
ˆ3�—‘˜•˜qÕ Ñ"Ð"Õ	#€BÜ�9Š9�r�c˜2�Y BÔ'Ð'r3   Úrotary_pos_embc                óÈ   € VP                  V4      pVP                  V4      pW,          \        V 4      V,          ,           pW,          \        V4      V,          ,           pWV3# )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)Ú	unsqueezer   )ÚqÚkr  r  Úunsqueeze_dimÚq_embedÚk_embeds   &&&&&  r0   Úapply_rotary_pos_embr)  K  sS   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�wœ; q›>¨CÕ/Õ0€GØ�wœ; q›>¨CÕ/Õ0€GØÐÐr3   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r*   rB   Ún_repr,   )r8   rD   rq   )r.   s   "r0   r1   r1   e  s.   € ÷ 	Uñ 	UœUŸ\™\ð 	U´#ð 	U¼%¿,¹,ñ 	Ur3   c                ó˜   € V P                   w  r#rEV^8X  d   V # V R,          P                  W#WV4      p V P                  W#V,          WE4      # )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
)r�   r�   Nr�   r�   )rT   r  rÇ   )rB   r+  ÚbatchÚnum_key_value_headsÚslenr›   s   &&    r0   Ú	repeat_kvr0  e  sU   € ð
 2?×1DÑ1DÑ.€E Ø�„zØÐØ!Ð"2Õ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÕ(CÀTÓTÐTr3   c                ó   € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R\        R	\
        \        ,          /# )
r*   ÚmoduleÚqueryÚkeyÚvaluer‘   NÚscalingÚdropoutrÃ   )r   ÚModuler8   rD   r-   r   r   )r.   s   "r0   r1   r1   q  sw   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ð%ô ð%ô Ô'Õ(ñ%r3   c                 ó"  € \        W P                  4      p\        W0P                  4      p	\        P                  ! WP	                  ^^4      4      V,          p
Ve	   W¤,           p
\
        P                  P                  V
R\        P                  R7      P                  VP                  4      p
\
        P                  P                  W¦V P                  R7      p
\        P                  ! W©4      pVP	                  ^^4      P                  4       pWº3# )r*   )r~   rH   )ÚpÚtrainingrG   )r0  Únum_key_value_groupsr8   rÌ   rÉ   r   Ú
functionalÚsoftmaxrJ   rI   rH   r7  r;  Ú
contiguous)r2  r3  r4  r5  r‘   r6  r7  rÃ   rÕ   rÖ   Úattn_weightsrØ   s   &&&&&&&,    r0   Úeager_attention_forwardrA  q  sÒ   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÕL€LØÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r3   c                   óX   a a€ ] tR tRt oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )	ÚMiniMaxAttentioniŠ  z=Multi-headed attention from 'Attention Is All You Need' paperc                ó&   <€ V ^8„  d   QhRS[ RS[/# r–   r˜   )r.   r/   s   "€r0   r1   ÚMiniMaxAttention.__annotate__Ž  s   ø€ ÷ lñ l™}ð l¹ñ lr3   c                óf  <€ \         SV `  4        Wn        W n        \	        VR R4      ;'       g    VP
                  VP                  ,          V n        VP                  VP                  ,          V n	        V P                  R,          V n
        VP                  V n        RV n        \        P                  ! VP
                  VP                  V P                  ,          RR7      V n        \        P                  ! VP
                  VP                  V P                  ,          RR7      V n        \        P                  ! VP
                  VP                  V P                  ,          RR7      V n        \        P                  ! VP                  V P                  ,          VP
                  RR7      V n        R# )r›   NTFrœ   g      à¿)r5   r6   r—   rk   rž   r=   rŸ   r›   r.  r<  r6  Úattention_dropoutÚ	is_causalr   r¥   Úq_projÚk_projÚv_projÚo_proj©r<   r—   rk   r>   s   &&&€r0   r6   ÚMiniMaxAttention.__init__Ž  s1  ø€ Ü‰ÑÔØŒØ"ŒÜ ¨
°DÓ9×mÐm¸V×=OÑ=OÐSY×SmÑSmÕ=mˆŒØ$*×$>Ñ$>À&×B\ÑB\Õ$\ˆÔ!Ø—}‘} dÕ*ˆŒØ!'×!9Ñ!9ˆÔØˆŒÜ—i’i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Õ4^ÐejÔkˆŒÜ—i’i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Õ4^ÐejÔkˆŒÜ—i’i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Õ4^ÐejÔkˆŒÜ—i’i × :Ñ :¸T¿]¹]Õ JÈF×L^ÑL^ÐejÔkˆŽr3   c                ó  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  S[ P                  R,          3,          /# rÀ   rÄ   )r.   r/   s   "€r0   r1   rE  œ  s‡   ø€ ÷ ')ñ ')á—|‘|ð')ñ #¡5§<¡<±·±Ð#=Õ>ð')ñ Ÿ™ tÕ+ð	')ñ
  �ð')ñ Ñ-Õ.ð')ñ 
‰u�|‰|™UŸ\™\¨DÕ0Ð0Õ	1ñ')r3   c                ó>  € VP                   R R p. VORNV P                  N5pV P                  V4      P                  V4      P	                  ^^4      pV P                  V4      P                  V4      P	                  ^^4      p	V P                  V4      P                  V4      P	                  ^^4      p
Vw  r¼\        W‰W¼4      w  r‰Ve   VP                  WšV P                  4      w  rš\        P                  ! V P                  P                  \        4      pT! V VV	V
V3RV P                  '       g   RMV P                   RV P"                  R\%        V P                  RR 4      /VB w  rïVP&                  ! . VORN5!  P)                  4       pV P+                  V4      pWï3# )Nr7  ç        r6  Úsliding_windowrG   )rT   r›   rI  ÚviewrÉ   rJ  rK  r)  Úupdaterk   r   Úget_interfacer—   Ú_attn_implementationrA  r;  rG  r6  rž   rÇ   r?  rL  )r<   rB   rÁ   r‘   rÂ   rÃ   Úinput_shapeÚhidden_shaperÔ   rÕ   rÖ   r  r  Úattention_interfacerØ   r@  s   &&&&&,          r0   rP   ÚMiniMaxAttention.forwardœ  sŸ  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ
%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð
%
ð —L‘Lð
%
ô # 4§;¡;Ð0@À$ÓGð
%
ð ñ
%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—k‘k +Ó.ˆØÐ(Ð(r3   )rG  r—   r›   rH  rJ  rk   r<  rL  rI  r6  rK  rb   ©
rW   rX   rY   rZ   Ú__doc__r6   rP   r[   r\   r]   r^   s   @@r0   rC  rC  Š  s#   ù‡ € áG÷ló l÷')÷ ')ò ')r3   rC  c                   ó8   a a€ ] tR tRt oV 3R ltR tRtVtV ;t# )ÚMiniMaxTopKRouteriÆ  c                ó  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  V n        \        P                  ! \        P                  ! V P
                  V P                  4      4      V n        R # rb   )r5   r6   Únum_experts_per_tokÚtop_kÚnum_local_expertsÚnum_expertsr=   Ú
hidden_dimr   r7   r8   Úemptyr:   ©r<   r—   r>   s   &&€r0   r6   ÚMiniMaxTopKRouter.__init__Ç  s[   ø€ Ü‰ÑÔØ×/Ñ/ˆŒ
Ø!×3Ñ3ˆÔØ ×,Ñ,ˆŒÜ—l’l¤5§;¢;¨t×/?Ñ/?ÀÇÁÓ#QÓRˆŽr3   c                ór  € VP                  RV P                  4      p\        P                  ! WP                  4      p\
        P                  P                  P                  VP                  4       RR7      p\
        P                  ! W0P                  RR7      w  rEWDP                  RRR7      ,          pTpW&V3# )rg   r}   T)r~   rF   rG   )rÇ   rd  rÎ   Úlinearr:   r8   r   r=  r>  r-   Útopkra  Úsum)r<   rB   Úrouter_logitsÚrouter_probsÚrouter_top_valueÚrouter_indicesÚrouter_scoress   &&     r0   rP   ÚMiniMaxTopKRouter.forwardÎ  s”   € Ø%×-Ñ-¨b°$·/±/ÓBˆÜŸš ·±Ó<ˆÜ—x‘x×*Ñ*×2Ñ2°=×3FÑ3FÓ3HÈbÐ2ÓQˆÜ+0¯:ª:°lÇJÁJÐTVÔ+WÑ(ÐØ×0Ñ0°RÀÐ0ÓFÕFÐØ(ˆØ¨^Ð;Ð;r3   )rd  rc  ra  r:   ©	rW   rX   rY   rZ   r6   rP   r[   r\   r]   r^   s   @@r0   r^  r^  Æ  s   ù‡ € õS÷<ò <r3   r^  c                   óT   a a€ ] tR tRt oRtV3R lV 3R lltV3R lR ltRtVtV ;t	# )ÚMiniMaxExpertsiØ  z2Collection of expert weights stored as 3D tensors.c                ó    <€ V ^8„  d   QhRS[ /# rñ   r$   )r.   r/   s   "€r0   r1   ÚMiniMaxExperts.__annotate__Ü  s   ø€ ÷ 0ñ 0™}ñ 0r3   c                ó  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  V n        \        P                  ! \        P                  ! V P                  ^V P                  ,          V P
                  4      4      V n        \        P                  ! \        P                  ! V P                  V P
                  V P                  4      4      V n        \        VP                  ,          V n        R# )r*   N)r5   r6   rb  rc  r=   rd  Úintermediate_sizeÚintermediate_dimr   r7   r8   re  Úgate_up_projÚ	down_projr   r¢   r£   rf  s   &&€r0   r6   ÚMiniMaxExperts.__init__Ü  s­   ø€ Ü‰ÑÔØ!×3Ñ3ˆÔØ ×,Ñ,ˆŒØ &× 8Ñ 8ˆÔÜŸLšL¬¯ª°T×5EÑ5EÀqÈ4×K`ÑK`ÕG`Ðbf×bqÑbqÓ)rÓsˆÔÜŸš¤e§k¢k°$×2BÑ2BÀDÇOÁOÐUY×UjÑUjÓ&kÓlˆŒÜ˜V×.Ñ.Õ/ˆŽr3   c                ó‚   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[ P                  /# )r*   rB   Útop_k_indexÚtop_k_weightsr,   rC   )r.   r/   s   "€r0   r1   rv  å  sC   ø€ ÷ #ñ #á—|‘|ð#ñ —\‘\ð#ñ —|‘|ð	#ñ
 
�‰ñ#r3   c                óÆ  € \         P                  ! V4      p\         P                  ! 4       ;_uu_ 4        \         P                  P                  P                  W P                  R 7      pVP                  ^^^ 4      p\         P                  ! VP                  RR7      ^ 4      P                  4       pRRR4       X EF  pV^ ,          pWpP                  8X  d   K  \         P                  ! XV,          4      w  r‰W,          p
\        P                  P                  W P                  V,          4      P                  ^RR7      w  r¼V P                  V4      V,          p\        P                  P                  WÐP                   V,          4      pWÓW˜R3,          ,          pVP#                  ^ W�P%                  VP&                  4      4       EK  	  V#   + '       g   i     EL'; i))Únum_classesr}   NrG   )rG   rÆ   )r8   Ú
zeros_liker  r   r=  Úone_hotrc  ÚpermuteÚgreaterrk  Únonzeror¼   ri  rz  Úchunkr£   r{  Ú
index_add_rI   rH   )r<   rB   r~  r  Úfinal_hidden_statesÚexpert_maskÚ
expert_hitÚ
expert_idxÚ	top_k_posÚ	token_idxÚcurrent_stateÚgateÚupÚcurrent_hidden_statess   &&&&          r0   rP   ÚMiniMaxExperts.forwardå  ss  € ô $×.Ò.¨}Ó=ÐÜ�]Š]�_�_ÜŸ(™(×-Ñ-×5Ñ5°k×O_ÑO_Ð5Ó`ˆKØ%×-Ñ-¨a°°AÓ6ˆKÜŸš {§¡¸8 Ó'DÀaÓH×PÑPÓRˆJ÷ ô
 %ˆJØ# A�ˆJØ×-Ñ-Ô-ÙÜ#(§;¢;¨{¸:Õ/FÓ#GÑ ˆIØ)Õ4ˆMÜ—}‘}×+Ñ+¨M×;LÑ;LÈZÕ;XÓY×_Ñ_Ð`aÐgiÐ_Ój‰HˆDØ$(§K¡K°Ó$5¸Õ$:Ð!Ü$&§M¡M×$8Ñ$8Ð9N×P^ÑP^Ð_iÕPjÓ$kÐ!Ø$9È)Ð`dÐJdÕ<eÕ$eÐ!Ø×*Ñ*¨1¨i×9QÑ9QÐRe×RkÑRkÓ9l×mñ %ð #Ð"÷# �_ˆ_ús   ´A=GÇG 	)r£   r{  rz  rd  ry  rc  r[  r^   s   @@r0   rt  rt  Ø  s!   ù‡ € á<÷0ó 0÷#÷ #ð #r3   rt  c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚMiniMaxSparseMoeBlocki   c                ó¨   <€ \         SV `  4        VP                  V n        VP                  V n        \        V4      V n        \        V4      V n	        R # rb   )
r5   r6   r`  ra  Úrouter_jitter_noiseÚjitter_noiser^  r�  rt  Úexpertsrf  s   &&€r0   r6   ÚMiniMaxSparseMoeBlock.__init__  sA   ø€ Ü‰ÑÔØ×/Ñ/ˆŒ
Ø"×6Ñ6ˆÔÜ% fÓ-ˆŒ	Ü% fÓ-ˆŽr3   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rA   rC   )r.   r/   s   "€r0   r1   Ú"MiniMaxSparseMoeBlock.__annotate__  s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r3   c                ó¾  € VP                   w  r#pV P                  '       da   V P                  ^ 8”  dP   V\        P                  ! V4      P                  RV P                  ,
          RV P                  ,           4      ,          pVP                  RVP                   R,          4      pV P                  V4      w  rVpV P                  WV4      pVP                  W#V4      pV# )r   r  rG   )
rT   r;  r˜  r8   Ú
empty_likeÚuniform_rS  r�  r™  rÇ   )r<   rB   rÐ   Úsequence_lengthrd  rl   r  r~  s   &&      r0   rP   ÚMiniMaxSparseMoeBlock.forward  sº   € Ø2?×2EÑ2EÑ/ˆ
 ZØ�=�=ˆ=˜T×.Ñ.°Ô2ØœU×-Ò-¨mÓ<×EÑEÀcÈD×L]ÑL]ÕF]Ð_bÐei×evÑevÕ_vÓwÕwˆMØ%×*Ñ*¨2¨}×/BÑ/BÀ2Õ/FÓGˆØ(,¯	©	°-Ó(@Ñ%ˆ˜+ØŸ™ ]ÀÓOˆØ%×-Ñ-¨jÈ:ÓVˆØÐr3   )r™  r�  r˜  ra  rr  r^   s   @@r0   r•  r•     s   ù‡ € õ.÷÷ ð r3   r•  c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚMiniMaxDecoderLayeri  c                ó&   <€ V ^8„  d   QhRS[ RS[/# r–   r˜   )r.   r/   s   "€r0   r1   Ú MiniMaxDecoderLayer.__annotate__  s   ø€ ÷ Añ A™}ð A¹ñ Ar3   c                ó´  <€ \         SV `  4        VP                  V n        \        W4      V n        \        VP                  VP                  R 7      V n        \        VP                  VP                  R 7      V n        W n	        \        VR4      '       d   VP                  V,          MRV n        VP                  V n        VP                  V n        \        V4      V n        V P                  R8X  d5   \#        W4      V n        VP$                  V n        VP(                  V n        R# \        W4      V n        VP,                  V n        VP.                  V n        R# )©r+   r°   NÚlinear_attention)r5   r6   r=   rC  Ú	self_attnr(   Úrms_norm_epsÚinput_layernormÚpost_attention_layernormrk   Úhasattrr°   Ú
block_typeÚmlp_alpha_factorÚmlp_beta_factorr•  Úmlpr”   Úlinear_attn_alpha_factorÚattn_alpha_factorÚlinear_attn_beta_factorÚattn_beta_factorÚfull_attn_alpha_factorÚfull_attn_beta_factorrM  s   &&&€r0   r6   ÚMiniMaxDecoderLayer.__init__  s	  ø€ Ü‰ÑÔØ!×-Ñ-ˆÔä)¨&Ó<ˆŒÜ-¨f×.@Ñ.@Àf×FYÑFYÔZˆÔÜ(6°v×7IÑ7IÈv×ObÑObÔ(cˆÔ%à"ŒÜ;BÀ6È=×;YÒ;Y˜&×,Ñ,¨YÖ7Ð_cˆŒØ &× 7Ñ 7ˆÔØ%×5Ñ5ˆÔÜ(¨Ó0ˆŒØ�?‰?Ð0Ô0Ü6°vÓIˆDŒNØ%+×%DÑ%DˆDÔ"Ø$*×$BÑ$BˆDÖ!ä-¨fÓ@ˆDŒNØ%+×%BÑ%BˆDÔ"Ø$*×$@Ñ$@ˆDÖ!r3   c                ó�  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[,          R	S[S[ P                  S[S[ P                  S[ P                  3,          R,          3,          /# )
r*   rB   rÁ   Nr‘   r  rÂ   Ú	use_cacherÃ   r,   )	r8   rD   rS   Ú
LongTensorr   Úboolr   r   ÚFloatTensor)r.   r/   s   "€r0   r1   r¥  *  sÄ   ø€ ÷ ñ á—|‘|ðñ #¡5§<¡<±·±Ð#=Õ>ÀÕEðñ Ÿ™ tÕ+ð	ñ
 ×&Ñ&¨Õ-ðñ  �ðñ ˜$•;ðñ Ñ-Õ.ðñ 
‰u× Ñ ¡%©×(9Ñ(9¹5×;LÑ;LÐ(LÕ"MÐPTÕ"TÐTÕ	Uñr3   c                óX  € V P                  V4      pTpV P                  ! RR VRVRVRVRVRV/VB w  rW€P                  ,          WP                  ,          ,           pV P	                  V4      pTpV P                  V4      pW€P                  ,          WP                  ,          ,           pV# )rB   rÁ   r‘   r  rÂ   rº  © )r«  r©  r³  rµ  r¬  r±  r¯  r°  )
r<   rB   rÁ   r‘   r  rÂ   rº  rÃ   Úresidualrl   s
   &&&&&&&,  r0   rP   ÚMiniMaxDecoderLayer.forward*  sÏ   € ð ×,Ñ,¨]Ó;ˆØ ˆØŸ>š>ñ 
Ø'ð
à 3ð
ð *ð
ð &ð	
ð
 ,ð
ð  ð
ð ñ
Ñˆð !×#9Ñ#9Õ9¸M×LaÑLaÕ<aÕaˆØ×5Ñ5°mÓDˆØ ˆØŸ™ Ó/ˆØ ×#8Ñ#8Õ8¸=×K_ÑK_Õ;_Õ_ˆàÐr3   )r³  rµ  r®  r=   r«  rk   r±  r¯  r°  r¬  r©  )NNNNFrr  r^   s   @@r0   r£  r£    s    ù‡ € ÷Aó A÷,÷ ò r3   r£  c                   ó´   a a€ ] tR tRt oRtRtR.tR.tRtRt	Rt
RtRtR]! ]R^ R	7      R
]R]]./t]P(                  ! 4       V 3R l4       tV3R ltRtVtV ;t# )ÚMiniMaxPreTrainedModeliH  ÚmodelTr£  rÂ   Frl  zmlp.gate)Ú
layer_nameÚindexrB   Ú
attentionsc                óþ  <€ \         SV `  V4       V P                  P                  p\	        V\
        4      '       dH   \        P                  ! VP                  R VR7       \        P                  ! VP                  R VR7       M9\	        V\        4      '       d$   \        P                  ! VP                  R VR7       \	        V\        4      '       d«   VP                  4       pVP                  V4      w  rEp\        P                  ! VP                   V4       \        P                  ! VP"                  V4       \        P                  ! VP$                  V4       \        P                  ! VP&                  V4       R# R# )rQ  )rL   ÚstdN)r5   Ú_init_weightsr—   Úinitializer_ranger  rt  ÚinitÚnormal_rz  r{  r^  r:   r”   r©   rª   Úcopy_r¬   r­   r®   r¯   )r<   r2  rÉ  r¬   r­   r®   r¯   r>   s   &&     €r0   rÊ  Ú$MiniMaxPreTrainedModel._init_weightsZ  s   ø€ ä‰Ñ˜fÔ%Ø�k‰k×+Ñ+ˆÜ�fœn×-Ò-Ü�LŠL˜×,Ñ,°3¸CÕ@Ü�LŠL˜×)Ñ)°¸Ö=Ü˜Ô 1×2Ò2Ü�LŠL˜Ÿ™¨S°cÕ:Ü�fÔ7×8Ò8Ø×.Ñ.Ó0ˆJØ5;×5IÑ5IÈ*Ó5UÑ2ˆK NÜ�JŠJ�v×(Ñ(¨*Ô5Ü�JŠJ�v×)Ñ)¨;Ô7Ü�JŠJ�v×'Ñ'¨Ô3Ü�JŠJ�v×,Ñ,¨nÖ=ñ 9r3   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # rñ   r$   )r.   r/   s   "€r0   r1   Ú#MiniMaxPreTrainedModel.__annotate__H  s   ø‡ ‚ áÑò r3   r¿  )rW   rX   rY   rZ   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendr"   r^  r£  rC  r”   Ú_can_record_outputsr8   r  rÊ  Ú__annotate_func__r[   r\   r]   r^   s   @@r0   rÃ  rÃ  H  s�   ù‡ € ð  ÐØ&*Ð#Ø.Ð/ÐØ#4Ð"5ÐØÐØ€NØÐØ"ÐØ"&Ðà™Ð(9ÀjÐXYÔZØÐ,ØÐ'Ð)BÐCðÐð ‡]‚]ƒ_ô>ó ð>÷' … r3   rÃ  c                   óh   a a€ ] tR tRt oV3R lV 3R llt]]RV3R lR ll4       4       tRtVt	V ;t
# )ÚMiniMaxModelil  c                ó    <€ V ^8„  d   QhRS[ /# rñ   r$   )r.   r/   s   "€r0   r1   ÚMiniMaxModel.__annotate__n  s   ø€ ÷ ñ ™}ñ r3   c           	     ó  <€ \         SV `  V4       VP                  V n        VP                  V n        \
        P                  ! VP                  VP                  V P                  4      V n        \
        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n        \        VP                  VP                  R 7      V n        \#        VR7      V n        RV n        V P)                  4        R# u upi )r§  )r—   FN)r5   r6   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingr=   Úembed_tokensÚ
ModuleListrh   r    r£  r€   r(   rª  r¤   rí   Ú
rotary_embÚgradient_checkpointingÚ	post_initrM  s   && €r0   r6   ÚMiniMaxModel.__init__n  sÊ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸLšL¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m’mÜEJÈ6×KcÑKcÔEdÓeÑEd¸	Ô  Ö3ÑEdÑeó
ˆŒô # 6×#5Ñ#5¸6×;NÑ;NÔOˆŒ	Ü0¸Ô?ˆŒØ&+ˆÔ#ð 	�‰Öùò fs   ÂDc                ó  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[S[,          R	S[S[	,          /# )
r*   Ú	input_idsNr‘   r  rÂ   Úinputs_embedsrº  rÃ   r,   )
r8   r»  rD   r`   r½  r¼  r   r   rS   r   )r.   r/   s   "€r0   r1   rà  €  sš   ø€ ÷ ?
ñ ?
á×#Ñ# dÕ*ð?
ñ Ÿ™ tÕ+ð?
ñ ×&Ñ&¨Õ-ð	?
ñ
 &¨Õ,ð?
ñ ×(Ñ(¨4Õ/ð?
ñ ˜$•;ð?
ñ Ñ+Õ,ð?
ñ 
Ñ'Õ	'ñ?
r3   c                ó   € VR J VR J,          '       d   \        R4      hV'       d   Vf   \        4       pM6V'       d/   \        V\        4      '       g   \        R\        V4       R24      hVf   V P	                  V4      pVfb   Ve   VP                  4       M^ p\        P                  ! VP                  ^,          VP                  R7      V,           pVP                  ^ 4      p\        T;p	\        4      '       gP   V P                  P                  f   \        M\        p
RV P                  RVRVRVR	V/pR
V
! R/ VB R\!        R/ VB /p	TpV P#                  WÃ4      p\%        V P&                  4       F9  w  rïV! V3RW�P                  P(                  V,          ,          RVR	VRVRV/VB pK;  	  V P+                  V4      p\-        VVR7      # )Nz:You must specify exactly one of input_ids or inputs_embedszSMiniMax uses cache of its own and is not compatible with `past_key_values` of type Ú.)rî   r—   rî  r‘   rÂ   r  Úfull_attentionr¨  rÁ   rº  )Úlast_hidden_staterÂ   r¿  )Ú
ValueErrorr`   r  r  ræ  Úget_seq_lengthr8   r´   rT   rî   r#  Údictr—   rR  r   r   r   rè  Ú	enumerater€   r°   r¤   r   )r<   rí  r‘   r  rÂ   rî  rº  rÃ   Úpast_seen_tokensÚcausal_mask_mappingÚmask_functionÚmask_kwargsrB   rÁ   rÙ   Údecoder_layers   &&&&&&&,        r0   rP   ÚMiniMaxModel.forward~  sò  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZç˜Ò0Ü*›n‰Oßœz¨/¼<×HÒHÜØeÔfjÐkzÓf{Ðe|Ð|}Ð~óð ð Ò Ø ×-Ñ-¨iÓ8ˆMàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLä°Ð?Ð-Ä×FÒFà&*§k¡k×&@Ñ&@Ò&HÕ"ÔNoð ð ˜$Ÿ+™+Ø Ø  .Ø! ?Ø ðˆKð !¡-Ñ">°+Ñ">Ø"Ô$CÑ$RÀkÑ$Rð#Ðð
 &ˆØ"Ÿo™o¨mÓJÐä )¨$¯+©+Ö 6ÑˆAÙ)Øñà2·;±;×3JÑ3JÈ1Õ3MÕNðð %8ðð *ð	ð
 !0ðð $ðð ñŠMñ !7ð Ÿ	™	 -Ó0ˆä%Ø+Ø+ô
ð 	
r3   )ræ  ré  r€   r¤   rã  rè  rä  )NNNNNN)rW   rX   rY   rZ   r6   r!   r#   rP   r[   r\   r]   r^   s   @@r0   rÞ  rÞ  l  s5   ù‡ € ÷ó ð   Ø÷?
ñ ?
ó ó  ÷?
ð ?
r3   rÞ  c          	      ó  € V ^8„  d   QhR\         P                  \        \         P                  ,          ,          R,          R\        R,          R\         P                  R,          R\         P                  \        ,          /# )r*   Úgate_logitsNrc  r‘   r,   )r8   rD   rS   rq   )r.   s   "r0   r1   r1   Â  sc   € ÷ B&ñ B&Ü—‘¤¤e§l¡lÕ 3Õ3°dÕ:ðB&ä�t•ðB&ô —L‘L 4Õ'ð	B&ô
 ‡\�\”CÕñB&r3   c           	     ó  € V e   \        V \        4      '       g   ^ # V ^ ,          P                  p\        P                  ! V\        P
                  VR7      p\        P                  ! V\        P
                  VR7      pRpVe1   VP                  R4      P                  V\        P
                  R7      pV  EFˆ  p	\        P                  P                  P                  V	P                  V4      RR7      p
\        P                  ! W¢RR7      w  r¼Vf~   V\        P                  ! VP                  R4      VR7      P                  4       ,           pWjP                  4       P                  ^ R7      ,           pWzP                  ^ ,          ,           pKÙ  V\        P                  ! V\        P
                  VR7      P!                  ^ VP                  R4      XP#                  V4      4      ,           pWjP                  4       VP%                  R4      ,          P                  ^ R7      ,           pWxP                  4       ,           pEK‹  	  WW,          pWg,          p\        P                  ! WÞP%                  ^ 4      ,          4      pWñ,          # )ax  
Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch.

See Switch Transformer (https://huggingface.co/papers/2101.03961) for more details. This function implements the loss
function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between
experts is too unbalanced.

Args:
    gate_logits:
        Logits from the `gate`, should be a tuple of model.config.num_hidden_layers tensors of
        shape [batch_size X sequence_length, num_experts].
    num_experts:
        Number of experts
    top_k:
        The number of experts to route per-token, can be also interpreted as the `top-k` routing
        parameter.
    attention_mask (`torch.Tensor`, *optional*):
        The attention_mask used in forward function
        shape [batch_size X sequence_length] if not None.

Returns:
    The auxiliary loss.
)rH   rî   rQ  )rî   rH   r}   )Ú	minlengthrG   )r  rS   rî   r8   rÊ   rJ   rÇ   rI   r   r=  r>  rj  Úbincountr-   rk  rT   Úscatter_add_r   r#  )rþ  rc  ra  r‘   Úcompute_deviceÚtokens_per_expert_sumÚrouter_prob_sumÚ
total_rowsÚ	flat_maskÚ
layer_gateÚrouting_weightsrl   Úselected_expertsÚtokens_per_expertÚrouter_prob_per_expertÚoverall_losss   &&&&            r0   Úload_balancing_loss_funcr  Â  sÿ  € ð: Ò¤*¨[¼%×"@Ò"@Ùð ! •^×*Ñ*€NÜ!ŸKšK¨¼5¿=¹=ÐQ_Ô`ÐÜ—k’k +´U·]±]È>ÔZ€OØ€JàÒ!à"×*Ñ*¨2Ó.×1Ñ1¸ÌuÏ}É}Ð1Ó]ˆ	ä!ˆ
ÜŸ(™(×-Ñ-×5Ñ5°j·m±mÀNÓ6SÐY[Ð5Ó\ˆÜ#Ÿjšj¨ÀRÔHÑˆØÒ!ð &¬¯ªÐ7G×7OÑ7OÐPRÓ7SÐ_jÔ(k×(qÑ(qÓ(sÕsð "ð .×0EÑ0EÓ0G×0KÑ0KÐPQÐ0KÓ0RÕRˆOØ#×&;Ñ&;¸AÕ&>Õ>ŠJð %:¼E¿KºKØ¤5§=¡=¸ô=ç‰l˜1Ð.×6Ñ6°rÓ:¸I×<WÑ<WÐX]Ó<^Ó_õ%`Ð!ð .×1FÑ1FÓ1HÈ9×K^ÑK^Ð_aÓKbÕ1b×0gÑ0gÐlmÐ0gÓ0nÕnˆOØ#§m¡m£oÕ5‹Jñ# "ð& .Õ:ÐØ,Õ9Ðä—9’9Ð.×1QÑ1QÐRSÓ1TÕTÓU€LØÕ%Ð%r3   c                   ó„   a a€ ] tR tRt oRR/tRR/tRR.R.3/tRR/tV 3R	 lt]	]
RV3R
 lR ll4       4       tRtVtV ;t# )ÚMiniMaxForCausalLMi  zlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputrB   ÚlogitsÚkeep_full_weightc                óR  <€ \         SV `  V4       \        V4      V n        VP                  V n        \
        P                  ! VP                  VP                  R R7      V n        VP                  V n	        VP                  V n        VP                  V n        V P                  4        R# )Frœ   N)r5   r6   rÞ  rÄ  rä  r   r¥   r=   r  Úrouter_aux_loss_coefrb  rc  r`  rê  rf  s   &&€r0   r6   ÚMiniMaxForCausalLM.__init__  s   ø€ Ü‰Ñ˜Ô Ü! &Ó)ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒØ$*×$?Ñ$?ˆÔ!Ø!×3Ñ3ˆÔØ#)×#=Ñ#=ˆÔ ð 	�‰Ör3   c                ód  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[R,          R
S[S[ P                  ,          RS[S[,          RS[	/# )r*   rí  Nr‘   r  rÂ   rî  Úlabelsrº  Úoutput_router_logitsÚlogits_to_keeprÃ   r,   )
r8   r»  rD   r   r½  r¼  rq   r   r   r   )r.   r/   s   "€r0   r1   ÚMiniMaxForCausalLM.__annotate__  sÖ   ø€ ÷ P
ñ P
á×#Ñ# dÕ*ðP
ñ Ÿ™ tÕ+ðP
ñ ×&Ñ&¨Õ-ð	P
ñ
  �ðP
ñ ×(Ñ(¨4Õ/ðP
ñ × Ñ  4Õ'ðP
ñ ˜$•;ðP
ñ # T�kðP
ñ ™eŸl™lÕ*ðP
ñ Ñ+Õ,ðP
ñ 
#ñP
r3   c
                ó¦  € Ve   TMV P                   P                  pV P                  ! RRVRVRVRVRVRVRV/V
B pVP                  p\	        V	\
        4      '       d   \        V	) R4      MT	pV P                  VR	VR	3,          4      pRpVe   V P                  ! WæV P                  3/ V
B pRpV'       dc   \        VP                  V P                  V P                  V4      pVe3   WðP                  VP                  VP                   4      ,          ,          p\#        VVVVP$                  VP&                  VP(                  VP                  R
7      # )a   
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

Example:

```python
>>> from transformers import AutoTokenizer, MiniMaxForCausalLM

>>> model = MiniMaxForCausalLM.from_pretrained("MiniMaxAI/MiniMax-Text-01-hf")
>>> tokenizer = AutoTokenizer.from_pretrained("MiniMaxAI/MiniMax-Text-01-hf")

>>> prompt = "Hey, are you conscious? Can you talk to me?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
```Nrí  r‘   r  rÂ   rî  rº  r  r�   )ÚlossÚaux_lossr  rÂ   rB   rÇ  rl  r¿  )r—   r  rÄ  rò  r  rq   Úslicer  Úloss_functionrä  r  rl  rc  r`  r  rI   rî   r   rÂ   rB   rÇ  )r<   rí  r‘   r  rÂ   rî  r  rº  r  r  rÃ   ÚoutputsrB   Úslice_indicesr  r  r  s   &&&&&&&&&&,      r0   rP   ÚMiniMaxForCausalLM.forward  sx  € ðN %9Ò$DÑ È$Ï+É+×JjÑJjð 	ð
 +/¯*ª*ñ 	+
Øð	+
à)ð	+
ð &ð	+
ð ,ð		+
ð
 (ð	+
ð  ð	+
ð "6ð	+
ð ñ	+
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒØ×%Ò% f°d·o±oÑPÈÑPˆDàˆßÜ/Ø×%Ñ%Ø× Ñ Ø×(Ñ(Øó	ˆHð Ò!Ø×1Ñ1°H·K±KÀÇÁÓ4LÕLÕL�ä(ØØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø!×/Ñ/ô
ð 	
r3   )r  rÄ  rc  r`  r  rä  )	NNNNNNNNr   )rW   rX   rY   rZ   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr6   r   r   rP   r[   r\   r]   r^   s   @@r0   r  r    sj   ù‡ € à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÐ/Ð0€Jõ
ð Ø÷P
ñ P
ó ó ÷P
ð P
r3   r  c                   ó   € ] tR tRtRtR# )Ú MiniMaxForSequenceClassificationio  r¿  N©rW   rX   rY   rZ   r[   r¿  r3   r0   r*  r*  o  ó   † Ûr3   r*  c                   ó   € ] tR tRtRtR# )ÚMiniMaxForTokenClassificationis  r¿  Nr+  r¿  r3   r0   r.  r.  s  r,  r3   r.  c                   ó   € ] tR tRtRtR# )ÚMiniMaxForQuestionAnsweringiw  r¿  Nr+  r¿  r3   r0   r0  r0  w  r,  r3   r0  )rg   )rQ  )Nr*   N)rÃ  rÞ  r  r*  r.  r0  )PÚcollections.abcr   r8   Útorch.nn.functionalr   r=  rÎ   Ú r   rÌ  Úactivationsr   Úcache_utilsr   r   Ú
generationr	   Úintegrationsr
   r   r   Úmasking_utilsr   r   r   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   r   r   r   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.deprecationr   Úutils.genericr    r!   Úutils.output_capturingr"   r#   Úconfiguration_minimaxr%   r8  r(   r`   r’   r”   rí   r   r)  r0  rA  rC  r^  rt  r•  r£  rÃ  rÞ  r  r  r*  r.  r0  Ú__all__r¿  r3   r0   Ú<module>rE     sõ  ðõ, %ã ß Ð Ý å &Ý !ß .Ý )ß hÑ hß sÑ sÝ B÷ó ÷ Rß Kß FÝ &ß IÑ IÝ 0ß Gß EÝ 0ñ ˜YÓ'ôJ�R—Y‘Yó Jó (ðJô(&J�<ô &JòR	ôM/ §	¡	ô M/ô`3<˜RŸY™Yô 3<òl(ñ Ð-Ó.óó /ðõ2	U÷%ñ2 Ð)Ó*ô8)�r—y‘yó 8)ó +ð8)ôv<˜Ÿ	™	ô <ð$ ô$#�R—Y‘Yó $#ó ð$#ôN˜BŸI™Iô ô&2Ð4ô 2ðj ô >˜_ó  >ó ð >ðF ôR
Ð)ó R
ó ðR
÷jB&ðJ ôd
Ð/°ó d
ó ðd
ôN	Ð'GÐI_ô 	ô	Ð$AÐCYô 	ô	Ð"=Ð?Uô 	ò‚r3   