+
    œ°³j§X  ã                   óê  € R t ^ RIt^ RIHu Ht ^ RIHt ^ RIHt ^RIH	t
 ^RIHt ^RIHtHt ^RIHt ^RIHtHtHt ^R	IHt ^R
IHt ^RIHt ^RIHt ^RIHt ^RI H!t!H"t"H#t# ^RI$H%t% ^RI&H't'H(t( ^RI)H*t* ^RI+H,t, ^RI-H.t.H/t/H0t0H1t1H2t2H3t3H4t4H5t5H6t6H7t7H8t8 ]#Pr                  ! ]:4      t;]"! RR7      ] ! R R]4      4       4       t< ! R R]64      t= ! R R]4      t> ! R R]P~                  4      t@ ! R R]*4      tA ! R  R!].4      tB ! R" R#]84      tC ! R$ R%]74      tD ! R& R']/]4      tE ! R( R)]54      tF ! R* R+]44      tG ! R, R-]04      tH ! R. R/]24      tI ! R0 R1]34      tJ ! R2 R3]14      tK. R4OtLR# )5zPyTorch MiniMax model.N)Ústrict)Únn)Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚPreTrainedConfig)Úcreate_causal_maskÚcreate_recurrent_attention_maskÚ!create_sliding_window_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚMoeModelOutputWithPast)ÚRopeParameters)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚlogging)Úmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputs)ÚGemma2RotaryEmbedding)Úapply_mask_to_padding_states)ÚMixtralAttentionÚMixtralDecoderLayerÚMixtralForCausalLMÚMixtralForQuestionAnsweringÚ MixtralForSequenceClassificationÚMixtralForTokenClassificationÚMixtralModelÚMixtralPreTrainedModelÚMixtralRMSNormÚMixtralSparseMoeBlockÚMixtralTopKRouterzMiniMaxAI/MiniMax-Text-01-hf)Ú
checkpointc                   ó6  a a€ ] tR t^7t oRtRtR.tRtRRRRRRR	R
RRRR
RR/tRR.R.3RRR.R.3RR.R.3/t	RRRRRRRR/t
RR/tRtRtRt^ t^ t^tRtR tR+tR!tR"tR#tRt^t^tR$tRtR%t^t^tR$t R&t!R%t"Rt#Rt$R't%^t&^t'^t(^t)^t*^t+V 3R( lt,V3R) lt-R*t.Vt/V ;t0# ),ÚMiniMaxConfigaA  
block_size (`int`, *optional*, defaults to 256):
    The length of each attention block, determining how queries, keys, and values
    are grouped and processed for intra- and inter-block attention.
full_attn_alpha_factor (`float`, *optional*, defaults to 1):
    Weight for residual value in residual connection after normal attention.
full_attn_beta_factor (`float`, *optional*, defaults to 1):
    Weight for hidden state value in residual connection after normal attention.
linear_attn_alpha_factor (`float`, *optional*, defaults to 1):
    Weight for residual value in residual connection after lightning attention.
linear_attn_beta_factor (`float`, *optional*, defaults to 1):
    Weight for hidden state value in residual connection after lightning attention.
mlp_alpha_factor (`float`, *optional*, defaults to 1):
    Weight for residual value in residual connection after MLP.
mlp_beta_factor (`float`, *optional*, defaults to 1):
    Weight for hidden state value in residual connection after MLP.

```python
>>> from transformers import MiniMaxModel, MiniMaxConfig

>>> # Initializing a MiniMax style configuration
>>> configuration = MiniMaxConfig()

>>> # Initializing a model from the MiniMax style configuration
>>> model = MiniMaxModel(configuration)

>>> # Accessing the model configuration
>>> configuration = model.config
```ÚminimaxÚpast_key_valuesg    €„.Azlayers.*.self_attn.q_projÚcolwisezlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_projÚrowwisez!layers.*.mlp.experts.gate_up_projÚpacked_colwisezlayers.*.mlp.experts.down_projzlayers.*.mlp.expertsÚmoe_tp_expertsÚembed_tokensÚ	input_idsÚinputs_embedsÚlayersÚhidden_statesÚattention_maskÚnormzlayers.*.mlp.gateÚ	ep_routerÚgrouped_gemmÚnum_expertsÚnum_local_expertsi }  i   i 8  NÚsilug{®Gáz”?çñhãˆµøä>TFg        gü©ñÒMbP?é   c                ó"  <€ V P                   f   V P                  V n         V P                  fL   \        V P                  4       Uu. uF&  p\        V^,           ^,          4      '       d   RMRNK(  	  upV n        \        SV `  ! R/ VB  R # u upi )NÚfull_attentionÚlinear_attention© )Únum_key_value_headsÚnum_attention_headsÚlayer_typesÚrangeÚnum_hidden_layersÚboolÚsuperÚ__post_init__)ÚselfÚkwargsÚiÚ	__class__s   &, €Úl/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/minimax/modular_minimax.pyrF   ÚMiniMaxConfig.__post_init__“   sƒ   ø€ Ø×#Ñ#Ò+Ø'+×'?Ñ'?ˆDÔ$à×ÑÒ#äW\Ð]a×]sÑ]sÔWtó ÙWtÐRS¤D¨!¨a­%°1­×$5Ò$5Ñ Ð;MÒMÑWtñ ˆDÔô 	‰ÒÑ' Ô'ùò	 s   Á,Bc                ó¬  <€ V ^8„  d   Qh/ S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ R,          ;R&   S[;R	&   S[ ;R
&   S[;R&   S[;R&   S[;R&   S[ R,          ;R&   S[ R,          ;R&   S[ S[S[ ,          ,          R,          ;R&   S[;R&   S[ R,          ;R&   S[S[ ,          ;R&   S[ ;R&   S[ ;R&   S[;R&   S[;R&   S[;R&   S[S[,          R,          ;R&   S[S[,          R,          ;R&   S[ ;R&   S[ S[,          ;R&   S[ S[,          ;R&   S[ S[,          ;R&   S[ S[,          ;R&   S[ S[,          ;R &   S[ S[,          ;R!&   # )"é   Ú
vocab_sizeÚhidden_sizeÚintermediate_sizerC   r@   r?   NÚhead_dimÚ
hidden_actÚmax_position_embeddingsÚinitializer_rangeÚrms_norm_epsÚ	use_cacheÚpad_token_idÚbos_token_idÚeos_token_idÚtie_word_embeddingsÚsliding_windowÚattention_dropoutÚnum_experts_per_tokr7   Úoutput_router_logitsÚrouter_aux_loss_coefÚrouter_jitter_noiseÚrope_parametersrA   Ú
block_sizeÚfull_attn_alpha_factorÚfull_attn_beta_factorÚlinear_attn_alpha_factorÚlinear_attn_beta_factorÚmlp_alpha_factorÚmlp_beta_factor)ÚintÚstrÚfloatrD   Úlistr   Údict)ÚformatÚ__classdict__s   "€rK   Ú__annotate__ÚMiniMaxConfig.__annotate__7   s1  ø‡ ‚ ñv Ññw ñx Ññy ñz Ñ"ñ{ ñ| Ññ} ñ~ Ñ!ñ ñ@ Ñ ñA ñB �D�jÑñC ñD ÑñE ñF !Ñ,ñG ñH Ñ#ñI ñJ ÑñK ñL ÑñM ñN ˜•*Ñ#ñO ñP ˜•*Ñ ñQ ñR ™™S�	•/ DÕ(Ñ,ñS ñT Ñ%ñU ñV ˜$•JÑ%ñW ñX ™s•{Ñ(ñY ñZ Ñ ñ[ ñ\ Ññ] ñ^ Ñ&ñ_ ñ`  Ñ'ña ñb Ñ$ñc ñd $¡dÕ*¨TÕ1Ñ8ñe ñf ‘c•˜TÕ!Ñ(ñg ñh Ññi ñj  ¡%�KÑ+ñk ñl ¡�;Ñ*ñm ñn "¡E�kÑ-ño ñp !¡5�[Ñ,ñq ñr ™E•kÑ%ñs ñt ™5•[Ñ$òu ó    )rA   r?   i   )1Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferenceÚdefault_thetaÚbase_model_tp_planÚbase_model_pp_planÚbase_model_ep_planÚattribute_maprO   rP   rQ   rC   r@   r?   rR   rS   rT   rU   rV   rW   rX   rY   rZ   r[   r\   r]   r^   r7   r_   r`   ra   rb   rA   rc   rd   re   rf   rg   rh   ri   rF   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__Ú__classcell__©rJ   rp   s   @@rK   r&   r&   7   ss  ù‡ € ñð< €JØ#4Ð"5ÐØ€Mà# YØ# YØ# YØ# YØ+Ð-=Ø(¨)ØÐ 0ðÐð 	˜+˜¨Ð(9Ð:Ø�OÐ%5Ð6¸Ð8IÐJØ�/Ð" _Ð$5Ð6ðÐð 	˜[Ø+¨^Ø(¨.ØÐ 0ð	Ðð #Ð$7Ð8€Mà€JØ€KØ"ÐØÐØ!ÐØ ÐØ€HØ€JØ#,ÐØ#ÐØ€LØ€IØ#€LØ €LØ+,€LØ %ÐØ!%€NØ%(ÐØ ÐØÐØ!&ÐØ"'ÐØ!$ÐØ48€OØ$(€KØ€JØ*+ÐØ)*ÐØ,-ÐØ+,ÐØ$%ÐØ#$€Oõ	(÷y … rs   r&   c                   ó   € ] tR t^ŸtRtR# )ÚMiniMaxRMSNormr>   N©rt   ru   rv   rw   r�   r>   rs   rK   r†   r†   Ÿ   ó   † Ûrs   r†   c                   ó�   a a€ ] tR t^£t oRtV 3R ltR tV3R lR ltV 3R ltV3R lR lt	V3R	 lR
 lt
V3R lR ltRtVtV ;t# )ÚMiniMaxCacheFc                ó2   <€ \         SV `  4        . V n        R # ©N)rE   Ú__init__Úlinear_cache©rG   rJ   s   &€rK   r�   ÚMiniMaxCache.__init__¨   s   ø€ Ü‰ÑÔØ02ˆÖrs   c                ó²   € \        \        V P                  4      V^,           4       F  pV P                  P                  . 4       K   	  W P                  V&   R# )é   N)rB   ÚlenrŽ   Úappend)rG   Ú	layer_idxrŽ   Ú_s   &&& rK   Úset_linear_cacheÚMiniMaxCache.set_linear_cache¬   sD   € ä”s˜4×,Ñ,Ó-¨y¸1­}Ö=ˆAØ×Ñ×$Ñ$ RÖ(ñ >à'3×Ñ˜)Ó$rs   c                ó    <€ V ^8„  d   QhRS[ /# )rN   r•   ©rj   )ro   rp   s   "€rK   rq   ÚMiniMaxCache.__annotate__²   s   ø€ ÷ ñ ©#ñ rs   c                óL   € V\        V 4      8  d   V P                  V,          # R # rŒ   )r“   rŽ   )rG   r•   s   &&rK   Úget_linear_cacheÚMiniMaxCache.get_linear_cache²   s"   € Ø”s˜4“yÔ Ø×$Ñ$ YÕ/Ð/Ùrs   c                óZ   <€ \        \        SV `	  4       \        V P                  4      4      # rŒ   )ÚmaxrE   Ú__len__r“   rŽ   r�   s   &€rK   r¡   ÚMiniMaxCache.__len__·   s"   ø€ Ü”5‘7‘?Ó$¤c¨$×*;Ñ*;Ó&<Ó=Ð=rs   c                ó    <€ V ^8„  d   QhRS[ /# )rN   Úrepeatsrš   )ro   rp   s   "€rK   rq   r›   º   s   ø€ ÷ Hñ H©sñ Hrs   c                ó  € \        \        V 4      4       Fp  pV P                  V,          . 8w  d4   V P                  V,          P                  V^ R7      V P                  V&   KN  V P                  V,          P                  V4       Kr  	  R# )é    ©ÚdimN)rB   r“   rŽ   Úrepeat_interleaver0   Úbatch_repeat_interleave)rG   r¤   r•   s   && rK   rª   Ú$MiniMaxCache.batch_repeat_interleaveº   sl   € Üœs 4›yÖ)ˆIØ× Ñ  Õ+¨rÔ1Ø/3×/@Ñ/@ÀÕ/K×/]Ñ/]Ð^eÐklÐ/]Ó/m�×!Ñ! )Ó,à—‘˜IÕ&×>Ñ>¸wÖGó	 *rs   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )rN   Úindices)ÚtorchÚTensor)ro   rp   s   "€rK   rq   r›   Á   s   ø€ ÷ Eñ E©E¯L©Lñ Ers   c                ó  € \        \        V 4      4       Fh  pV P                  V,          . 8w  d,   V P                  V,          VR 3,          V P                  V&   KF  V P                  V,          P	                  V4       Kj  	  R# ).N)rB   r“   rŽ   r0   Úbatch_select_indices)rG   r­   r•   s   && rK   r±   Ú!MiniMaxCache.batch_select_indicesÁ   sd   € Üœs 4›yÖ)ˆIØ× Ñ  Õ+¨rÔ1Ø/3×/@Ñ/@ÀÕ/KÈGÐUXÈLÕ/Y�×!Ñ! )Ó,à—‘˜IÕ&×;Ñ;¸GÖDó	 *rs   c                ó$   <€ V ^8„  d   QhRS[ RR/# )rN   Útokens_to_removeÚreturnNrš   )ro   rp   s   "€rK   rq   r›   È   s   ø€ ÷ Jñ J¡Sð J¨Tñ Jrs   c                ó   € \        R 4      h)z+MiniMaxCache does not support `crop` method)ÚRuntimeError)rG   r´   s   &&rK   ÚcropÚMiniMaxCache.cropÈ   s   € ÜÐHÓIÐIrs   )rŽ   )rt   ru   rv   rw   Úis_croppabler�   r—   r�   r¡   rª   r±   r¸   r�   r‚   rƒ   r„   s   @@rK   rŠ   rŠ   £   sO   ù‡ € ð €Lõ3ò4÷ð õ
>÷Hð H÷Eð E÷J÷ Jð Jrs   rŠ   c                   ó`   a a€ ] tR t^Ìt oV3R lV 3R lltR tR tRV3R lR lltRtVt	V ;t
# )	ÚMiniMaxLightningAttentionc                ó&   <€ V ^8„  d   QhRS[ RS[/# ©rN   Úconfigr•   ©r&   rj   )ro   rp   s   "€rK   rq   Ú&MiniMaxLightningAttention.__annotate__Í   s   ø€ ÷ 8ñ 8™}ð 8¹ñ 8rs   c                ó‚  <€ \         SV `  4        W n        \        VR R4      ;'       g    VP                  VP
                  ,          V n        VP
                  V n        VP                  V n        VP                  V n        \        VP                  ,          V n        \        V P                  V P
                  ,          4      V n        \        P                  ! VP                  V P
                  V P                  ,          ^,          RR7      V n        \        P                  ! V P
                  V P                  ,          VP                  RR7      V n        \        P                  ! VP                  V P
                  V P                  ,          RR7      V n        V P'                  4       pV P)                  V4      w  rEp\        P*                  ! V4      V n        \        P*                  ! V4      V n        \        P*                  ! V4      V n        \        P*                  ! V4      V n        VP4                  V,          V n        R# )rR   NF)Úbias)rE   r�   r•   ÚgetattrrP   r@   rR   rC   rc   r   rS   Úact_fnr†   r3   r   ÚLinearÚqkv_projÚout_projÚoutput_gateÚget_slope_rateÚdecay_factorsÚBufferÚ
slope_rateÚquery_decayÚ	key_decayÚdiagonal_decayrA   Ú
layer_type)rG   r¿   r•   rÍ   rÎ   rÏ   rÐ   rJ   s   &&&    €rK   r�   Ú"MiniMaxLightningAttention.__init__Í   s•  ø€ Ü‰ÑÔØ"ŒÜ ¨
°DÓ9×mÐm¸V×=OÑ=OÐSY×SmÑSmÕ=mˆŒØ#)×#=Ñ#=ˆÔ Ø!'×!9Ñ!9ˆÔØ ×+Ñ+ˆŒä˜V×.Ñ.Õ/ˆŒÜ" 4§=¡=°4×3KÑ3KÕ#KÓLˆŒ	ÜŸ	š	 &×"4Ñ"4°d×6NÑ6NÐQU×Q^ÑQ^Õ6^ÐabÕ6bÐinÔoˆŒÜŸ	š	 $×":Ñ":¸T¿]¹]Õ"JÈF×L^ÑL^ÐejÔkˆŒÜŸ9š9 V×%7Ñ%7¸×9QÑ9QÐTX×TaÑTaÕ9aÐhmÔnˆÔà×(Ñ(Ó*ˆ
Ø15×1CÑ1CÀJÓ1OÑ.ˆ äŸ)š) JÓ/ˆŒÜŸ9š9 [Ó1ˆÔÜŸš 9Ó-ˆŒÜ Ÿiši¨Ó7ˆÔà ×,Ñ,¨YÕ7ˆŽrs   c                ó:  € ^^^V P                   ,          ,          ,          p\        P                  ! V P                   4      ^,           p^V P                  V P                  ^,
          R,           ,          ,
          R,           pW,          pWC,          pVR,          pV# )r’   r9   )ºNNNNN)r@   r®   Úaranger•   rC   )rG   ÚbaseÚexponentÚfactorÚrates   &    rK   rÊ   Ú(MiniMaxLightningAttention.get_slope_rateå   sz   € Ø�A˜!˜d×6Ñ6Õ6Õ7Õ8ˆÜ—<’< × 8Ñ 8Ó9¸AÕ=ˆØ�T—^‘^ t×'=Ñ'=ÀÕ'AÀDÕ'HÕIÕIÈDÕPˆà�~ˆØ�}ˆØ�MÕ"ˆàˆrs   c                óÔ  € \         P                  ! V P                  4      ^,           p\         P                  ! V) VR,          ,          4      p\         P                  ! V) V P                  VR,          ,
          ,          4      pVR,          VR,          ,
          pVR,          pW,          p\         P                  ! V^ 8¬  V) \        R4      4      p\         P                  ! V4      pW4V3# )r’   z-inf)rÔ   N)NrÔ   )NNrÔ   rÔ   )r®   rÕ   rc   ÚexpÚwhererl   )rG   rÍ   Úblock_size_rangerÎ   rÏ   rÐ   s   &&    rK   rË   Ú'MiniMaxLightningAttention.decay_factorsð   s¹   € Ü Ÿ<š<¨¯©Ó8¸1Õ<Ðä—i’i  Ð.>¸wÕ.GÕ GÓHˆÜ—I’I˜z˜k¨T¯_©_Ð?OÐPWÕ?XÕ-XÕYÓZˆ	à)¨'Õ2Ð5EÀgÕ5NÕNˆØ'Ð(8Õ9ˆØ#Õ4ˆÜŸš ^°qÑ%8¸>¸/Ì5ÐQWË=ÓYˆÜŸš >Ó2ˆà ~Ð5Ð5rs   c                óR  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  S[ P                  R,          S[S[ P                  ,          R,          3,          /# )rN   r1   Úposition_embeddingsr2   Nr(   rH   rµ   )r®   r¯   Útupler   r   r   )ro   rp   s   "€rK   rq   rÁ   þ   sŸ   ø€ ÷ [/ñ [/á—|‘|ð[/ñ #¡5§<¡<±·±Ð#=Õ>ð[/ñ Ÿ™ tÕ+ð	[/ñ
  �ð[/ñ Ñ-Õ.ð[/ñ 
‰u�|‰|™UŸ\™\¨DÕ0±%¹¿¹Õ2EÈÕ2LÐLÕ	Mñ[/rs   c                ó6	  € VP                   w  rgpWpP                  ,           ^,
          V P                  ,          p	V P                  V P                  V4      4      p
\	        W£4      p
V
P                  WgV P                  ^V P                  ,          4      p
\        P                  ! W P                  ^R7      w  r¼pVP                  ^^4      pVP                  ^^4      pVP                  ^^4      pRpVe   VP                  V P                  4      pVEfí   \        P                  ! W`P                  V P                  V P                  4      P                  V4      p. p\        V	4       EF”  pVV P                  ,          p\!        VV P                  ,           V4      pVV,
          pVRRVV13,          pVRRVV13,          pVRRVV13,          pV P"                  RRV13,          pV P$                  RV) R13,          pV P&                  RRRV1RV13,          p\        P(                  ! V P*                  ) V,          4      p\        P,                  ! VVP                  RR4      4      p\        P,                  ! VV,          V4      p\        P,                  ! VV,          V4      pVV,           pVP/                  V4       \        P,                  ! VV,          P                  RR4      V4      pVV,          V,           pEK—  	  MÓ\        P(                  ! V P*                  ) 4      p . p\        V4       F¡  pVRRVV^,           13,          pVRRVV^,           13,          pVRRVV^,           13,          p\        P,                  ! VP                  RR4      V4      p!V V,          V!,           p\        P,                  ! VV4      pVP/                  V4       K£  	  \        P0                  ! VRR7      pVP                  ^^4      pVP                  WgV P                  V P                  ,          4      pV P3                  V4      p\4        P6                  ! V P9                  V4      4      V,          pV P;                  V4      pVe   VP=                  V P                  V4       Wþ3# )r’   r§   NrÔ   éÿÿÿÿéþÿÿÿ)Úshaperc   rÅ   rÇ   r   Úreshaper@   rR   r®   ÚsplitÚ	transposer�   r•   ÚzerosÚtorB   ÚminrÎ   rÏ   rÐ   rÜ   rÍ   Úmatmulr”   Úcatr3   ÚFÚsigmoidrÉ   rÈ   r—   )"rG   r1   rá   r2   r(   rH   Ú
batch_sizeÚseq_lenrP   Ú
num_blocksÚ
qkv_statesÚquery_statesÚ
key_statesÚvalue_statesÚattn_weights_interÚattn_outputrI   Ú	start_idxÚend_idxÚcurrent_block_sizeÚcurrent_query_statesÚcurrent_key_statesÚcurrent_value_statesÚcurrent_query_decayÚcurrent_key_decayÚcurrent_diagonal_decayÚblock_decayÚattn_weights_intraÚattn_output_intraÚattn_output_interÚcurrent_attn_outputÚnext_attn_weights_interÚratioÚcurrent_attn_weights_inters"   &&&&&,                            rK   ÚforwardÚ!MiniMaxLightningAttention.forwardþ   sP  € ð ,9×+>Ñ+>Ñ(ˆ
˜[Ø§¡Õ/°!Õ3¸¿¹ÕGˆ
à—[‘[ §¡¨}Ó!=Ó>ˆ
Ü1°*ÓMˆ
Ø×'Ñ'¨
¸T×=UÑ=UÐWXÐ[_×[hÑ[hÕWhÓiˆ
ä16·²¸ZÏÉÐ\]Ô1^Ñ.ˆ ,à#×-Ñ-¨a°Ó3ˆØ×)Ñ)¨!¨QÓ/ˆ
Ø#×-Ñ-¨a°Ó3ˆð "ÐØÒ&Ø!0×!AÑ!AÀ$Ç.Á.Ó!QÐàÓ%Ü!&§¢¨Z×9QÑ9QÐSW×S`ÑS`Ðbf×boÑboÓ!p×!sÑ!sØó"Ðð ˆKÜ˜:×&�Ø §¡Õ/�	Ü˜i¨$¯/©/Õ9¸7ÓC�Ø%,¨yÕ%8Ð"à'3°A°q¸)ÀGÐ:KÐ4KÕ'LÐ$Ø%/°°1°iÀÐ6GÐ0GÕ%HÐ"Ø'3°A°q¸)ÀGÐ:KÐ4KÕ'LÐ$à&*×&6Ñ&6°qÐ:MÐ;MÐ:MÐ7MÕ&NÐ#Ø$(§N¡N°1Ð7IÐ6IÑ6JÐ3JÕ$KÐ!Ø)-×)<Ñ)<¸QÀÐCVÐDVÐCVÐXkÐYkÐXkÐ=kÕ)lÐ&Ü#Ÿiši¨¯©Ð(8Ð;MÕ(MÓN�ô &+§\¢\Ð2FÐHZ×HdÑHdÐegÐikÓHlÓ%mÐ"Ü$)§L¢LÐ1CÐF\Õ1\Ð^rÓ$sÐ!ô %*§L¢LÐ1EÐH[Õ1[Ð]oÓ$pÐ!ð '8Ð:KÕ&KÐ#Ø×"Ñ"Ð#6Ô7ô +0¯,ª,Ø'Ð*;Õ;×FÑFÀrÈ2ÓNÐPdó+Ð'ð &8¸+Õ%EÐH_Õ%_Ó"ò; 'ô@ —I’I˜tŸ™Ð.Ó/ˆEØˆKÜ˜7–^�Ø'3°A°q¸!¸aÀ!½e¸)°OÕ'DÐ$Ø%/°°1°a¸!¸a½%°i°Õ%@Ð"Ø'3°A°q¸!¸aÀ!½e¸)°OÕ'DÐ$ä-2¯\ª\Ð:L×:VÑ:VÐWYÐ[]Ó:^Ð`tÓ-uÐ*Ø%*Ð-?Õ%?ÐB\Õ%\Ð"Ü&+§l¢lÐ3GÐI[Ó&\Ð#à×"Ñ"Ð#6Ö7ñ $ô —i’i °Ô4ˆð "×+Ñ+¨A¨qÓ1ˆØ!×)Ñ)¨*¸t×?WÑ?WÐZ^×ZgÑZgÕ?gÓhˆØ—i‘i Ó,ˆÜ—i’i × 0Ñ 0°Ó ?Ó@À;ÕNˆØ—m‘m KÓ0ˆð Ò&Ø×,Ñ,¨T¯^©^Ð=OÔPàÐ.Ð.rs   )rÅ   rc   rÐ   rR   rÏ   r•   rÑ   r3   r@   rC   rÈ   rÉ   rÇ   rÎ   rÍ   rŒ   )rt   ru   rv   rw   r�   rÊ   rË   r  r�   r‚   rƒ   r„   s   @@rK   r¼   r¼   Ì   s+   ù‡ € ÷8ó 8ò0	ò6÷[/÷ [/ò [/rs   r¼   c                   ó   € ] tR tRtRtR# )ÚMiniMaxRotaryEmbeddingi\  r>   Nr‡   r>   rs   rK   r  r  \  rˆ   rs   r  c                   ó   € ] tR tRtRtR# )ÚMiniMaxAttentioni`  r>   Nr‡   r>   rs   rK   r  r  `  rˆ   rs   r  c                   ó   € ] tR tRtRtR# )ÚMiniMaxTopKRouterid  r>   Nr‡   r>   rs   rK   r  r  d  rˆ   rs   r  c                   ó   € ] tR tRtRtR# )ÚMiniMaxSparseMoeBlockih  r>   Nr‡   r>   rs   rK   r  r  h  rˆ   rs   r  c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚMiniMaxDecoderLayeril  c                ó&   <€ V ^8„  d   QhRS[ RS[/# r¾   rÀ   )ro   rp   s   "€rK   rq   Ú MiniMaxDecoderLayer.__annotate__m  s   ø€ ÷ Añ A™}ð A¹ñ Ars   c                óà  <€ \         SV `  W4       W n        \        VR 4      '       d   VP                  V,          MRV n        VP                  V n        VP                  V n        V =\        V4      V n        V P
                  R8X  d5   \        W4      V n        VP                  V n        VP                  V n        R# \!        W4      V n        VP"                  V n        VP$                  V n        R# )rA   Nr=   )rE   r�   r•   ÚhasattrrA   Ú
block_typerh   ri   Úmlpr  r¼   Ú	self_attnrf   Úattn_alpha_factorrg   Úattn_beta_factorr  rd   re   )rG   r¿   r•   rJ   s   &&&€rK   r�   ÚMiniMaxDecoderLayer.__init__m  sÀ   ø€ Ü‰Ñ˜Ô+à"ŒÜ;BÀ6È=×;YÒ;Y˜&×,Ñ,¨YÖ7Ð_cˆŒØ &× 7Ñ 7ˆÔØ%×5Ñ5ˆÔØˆHÜ(¨Ó0ˆŒØ�?‰?Ð0Ô0Ü6°vÓIˆDŒNØ%+×%DÑ%DˆDÔ"Ø$*×$BÑ$BˆDÖ!ä-¨fÓ@ˆDŒNØ%+×%BÑ%BˆDÔ"Ø$*×$@Ñ$@ˆDÖ!rs   c                ó�  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[,          R	S[S[ P                  S[S[ P                  S[ P                  3,          R,          3,          /# )
rN   r1   rá   Nr2   Úposition_idsr(   rW   rH   rµ   )	r®   r¯   râ   Ú
LongTensorr   rD   r   r   ÚFloatTensor)ro   rp   s   "€rK   rq   r    sÄ   ø€ ÷ ñ á—|‘|ðñ #¡5§<¡<±·±Ð#=Õ>ÀÕEðñ Ÿ™ tÕ+ð	ñ
 ×&Ñ&¨Õ-ðñ  �ðñ ˜$•;ðñ Ñ-Õ.ðñ 
‰u× Ñ ¡%©×(9Ñ(9¹5×;LÑ;LÐ(LÕ"MÐPTÕ"TÐTÕ	Uñrs   c                óX  € V P                  V4      pTpV P                  ! RR VRVRVRVRVRV/VB w  rW€P                  ,          WP                  ,          ,           pV P	                  V4      pTpV P                  V4      pW€P                  ,          WP                  ,          ,           pV# )r1   rá   r2   r"  r(   rW   r>   )Úinput_layernormr  r  r  Úpost_attention_layernormr  rh   ri   )
rG   r1   rá   r2   r"  r(   rW   rH   Úresidualr–   s
   &&&&&&&,  rK   r  ÚMiniMaxDecoderLayer.forward  sÏ   € ð ×,Ñ,¨]Ó;ˆØ ˆØŸ>š>ñ 
Ø'ð
à 3ð
ð *ð
ð &ð	
ð
 ,ð
ð  ð
ð ñ
Ñˆð !×#9Ñ#9Õ9¸M×LaÑLaÕ<aÕaˆØ×5Ñ5°mÓDˆØ ˆØŸ™ Ó/ˆØ ×#8Ñ#8Õ8¸=×K_ÑK_Õ;_Õ_ˆàÐrs   )r  r  r  r•   r  rh   ri   r  )NNNNF)	rt   ru   rv   rw   r�   r  r�   r‚   rƒ   r„   s   @@rK   r  r  l  s    ù‡ € ÷Aó A÷$÷ ò rs   r  c                   ó\   a a€ ] tR tRt oRtR]! ]R^ R7      R]R]]	./t
V 3R ltR	tVtV ;t# )
ÚMiniMaxPreTrainedModeli�  FÚrouter_logitszmlp.gate)Ú
layer_nameÚindexr1   Ú
attentionsc                ó¦  <€ \         SV `  V4       \        V\        4      '       d«   VP	                  4       pVP                  V4      w  r4p\        P                  ! VP                  V4       \        P                  ! VP                  V4       \        P                  ! VP                  V4       \        P                  ! VP                  V4       R # R # rŒ   )rE   Ú_init_weightsÚ
isinstancer¼   rÊ   rË   ÚinitÚcopy_rÍ   rÎ   rÏ   rÐ   )rG   ÚmodulerÍ   rÎ   rÏ   rÐ   rJ   s   &&    €rK   r1  Ú$MiniMaxPreTrainedModel._init_weights¥  s–   ø€ Ü‰Ñ˜fÔ%Ü�fÔ7×8Ò8Ø×.Ñ.Ó0ˆJØ5;×5IÑ5IÈ*Ó5UÑ2ˆK NÜ�JŠJ�v×(Ñ(¨*Ô5Ü�JŠJ�v×)Ñ)¨;Ô7Ü�JŠJ�v×'Ñ'¨Ô3Ü�JŠJ�v×,Ñ,¨nÖ=ñ 9rs   r>   )rt   ru   rv   rw   Ú_can_compile_fullgraphr   r  r  r  r¼   Ú_can_record_outputsr1  r�   r‚   rƒ   r„   s   @@rK   r+  r+  �  s?   ù‡ € Ø"Ðà™Ð(9ÀjÐXYÔZØÐ,ØÐ'Ð)BÐCðÐ÷>õ >rs   r+  c                   óJ   a € ] tR tRt o ]]RV 3R lR ll4       4       tRtV tR# )ÚMiniMaxModeli°  Nc                ó  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[S[,          R	S[S[	,          /# )
rN   r.   Nr2   r"  r(   r/   rW   rH   rµ   )
r®   r#  r¯   rŠ   r$  rD   r   r   râ   r   )ro   rp   s   "€rK   rq   ÚMiniMaxModel.__annotate__³  sš   ø€ ÷ ?
ñ ?
á×#Ñ# dÕ*ð?
ñ Ÿ™ tÕ+ð?
ñ ×&Ñ&¨Õ-ð	?
ñ
 &¨Õ,ð?
ñ ×(Ñ(¨4Õ/ð?
ñ ˜$•;ð?
ñ Ñ+Õ,ð?
ñ 
Ñ'Õ	'ñ?
rs   c                ó   € VR J VR J,          '       d   \        R4      hV'       d   Vf   \        4       pM6V'       d/   \        V\        4      '       g   \        R\        V4       R24      hVf   V P	                  V4      pVfb   Ve   VP                  4       M^ p\        P                  ! VP                  ^,          VP                  R7      V,           pVP                  ^ 4      p\        T;p	\        4      '       gP   V P                  P                  f   \        M\        p
RV P                  RVRVRVR	V/pR
V
! R/ VB R\!        R/ VB /p	TpV P#                  WÃ4      p\%        V P&                  4       F9  w  rïV! V3RW�P                  P(                  V,          ,          RVR	VRVRV/VB pK;  	  V P+                  V4      p\-        VVR7      # )Nz:You must specify exactly one of input_ids or inputs_embedszSMiniMax uses cache of its own and is not compatible with `past_key_values` of type Ú.)Údevicer¿   r/   r2   r(   r"  r<   r=   rá   rW   )Úlast_hidden_stater(   r>   )Ú
ValueErrorrŠ   r2  Útyper-   Úget_seq_lengthr®   rÕ   ræ   r?  Ú	unsqueezern   r¿   r\   r	   r   r
   Ú
rotary_embÚ	enumerater0   rA   r3   r   )rG   r.   r2   r"  r(   r/   rW   rH   Úpast_seen_tokensÚcausal_mask_mappingÚmask_functionÚmask_kwargsr1   rá   rI   Údecoder_layers   &&&&&&&,        rK   r  ÚMiniMaxModel.forward±  sò  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZç˜Ò0Ü*›n‰Oßœz¨/¼<×HÒHÜØeÔfjÐkzÓf{Ðe|Ð|}Ð~óð ð Ò Ø ×-Ñ-¨iÓ8ˆMàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLä°Ð?Ð-Ä×FÒFà&*§k¡k×&@Ñ&@Ò&HÕ"ÔNoð ð ˜$Ÿ+™+Ø Ø  .Ø! ?Ø ðˆKð !¡-Ñ">°+Ñ">Ø"Ô$CÑ$RÀkÑ$Rð#Ðð
 &ˆØ"Ÿo™o¨mÓJÐä )¨$¯+©+Ö 6ÑˆAÙ)Øñà2·;±;×3JÑ3JÈ1Õ3MÕNðð %8ðð *ð	ð
 !0ðð $ðð ñŠMñ !7ð Ÿ	™	 -Ó0ˆä%Ø+Ø+ô
ð 	
rs   r>   )NNNNNN)	rt   ru   rv   rw   r   r   r  r�   r‚   )rp   s   @rK   r:  r:  °  s$   ø‡ € ØØ÷?
ñ ?
ó ó  ö?
rs   r:  c                   ó2   a a€ ] tR tRt oV 3R ltRtVtV ;t# )ÚMiniMaxForCausalLMiõ  c                ó$   <€ \         SV `  ! R/ VB # )a   
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

Example:

```python
>>> from transformers import AutoTokenizer, MiniMaxForCausalLM

>>> model = MiniMaxForCausalLM.from_pretrained("MiniMaxAI/MiniMax-Text-01-hf")
>>> tokenizer = AutoTokenizer.from_pretrained("MiniMaxAI/MiniMax-Text-01-hf")

>>> prompt = "Hey, are you conscious? Can you talk to me?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
```r>   )rE   r  )rG   Úsuper_kwargsrJ   s   &,€rK   r  ÚMiniMaxForCausalLM.forwardö  s   ø€ ô. ‰wŠÑ. Ñ.Ð.rs   r>   )rt   ru   rv   rw   r  r�   r‚   rƒ   r„   s   @@rK   rN  rN  õ  s   ù‡ € ÷/õ /rs   rN  c                   ó   € ] tR tRtRtR# )Ú MiniMaxForSequenceClassificationi  r>   Nr‡   r>   rs   rK   rS  rS    rˆ   rs   rS  c                   ó   € ] tR tRtRtR# )ÚMiniMaxForTokenClassificationi  r>   Nr‡   r>   rs   rK   rU  rU    rˆ   rs   rU  c                   ó   € ] tR tRtRtR# )ÚMiniMaxForQuestionAnsweringi  r>   Nr‡   r>   rs   rK   rW  rW    rˆ   rs   rW  )r&   r+  r:  rN  rS  rU  rW  )Mrx   r®   Útorch.nn.functionalr   Ú
functionalrï   Úhuggingface_hub.dataclassesr   Ú r   r3  Úactivationsr   Úcache_utilsr   r   Úconfiguration_utilsr   Úmasking_utilsr	   r
   r   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   Úmodeling_rope_utilsr   Úprocessing_utilsr   Úutilsr   r   r   Úutils.genericr   Úutils.output_capturingr   r   Úgemma2.modeling_gemma2r   Úmamba2.modeling_mamba2r   Úmixtral.modeling_mixtralr   r   r   r   r   r   r   r    r!   r"   r#   Ú
get_loggerrt   Úloggerr&   r†   rŠ   ÚModuler¼   r  r  r  r  r  r+  r:  rN  rS  rU  rW  Ú__all__r>   rs   rK   Ú<module>ro     sj  ðñ ã ß Ð Ý .Ý å &Ý !ß .Ý 3ß sÑ sÝ BÝ 9Ý 6Ý 1Ý &ß @Ñ @Ý 7ß EÝ :Ý A÷÷ ÷ ñ ð 
×	Ò	˜HÓ	%€ñ Ð9Ô:Øôc(Ð$ó c(ó ó ;ðc(ôL	�^ô 	ô&J�<ô &JôRM/ §	¡	ô M/ô`	Ð2ô 	ô	Ð'ô 	ô	Ð)ô 	ô	Ð1ô 	ô.Ð-Ð/Iô .ôb>Ð3ô >ô&B
�<ô B
ôJ/Ð+ô /ô6	Ð'Gô 	ô	Ð$Aô 	ô	Ð"=ô 	ò‚rs   