+
    œ°³j™�  ã                   óÔ  € R t ^ RIt^ RIHt ^ RIt^ RIHu Ht ^ RIHt ^RI	H
t ^RIHt ^RIHtHt ^RIHt ^RIHtHt ^R	IHt ^R
IHt ^RIHt ^RIHt ^RIHtH t H!t! ^RI"H#t# ]!PH                  ! ]%4      t&R R lt'R t(R t)R t* ! R R]P                  PV                  4      t,]! RR4      RAR R ll4       t-]! RR4      RAR R ll4       t.]! RR4      RR]/! R 4      3R!R"RR#RRR^R$3R% R& ll4       t0]! R'R4      RBR( R) ll4       t1]! R*R4      RRRR!R]/! R 4      3R!3R+ R, ll4       t2]! ].]-]0]1]2.4       ! R- R.]PV                  4      4       t3 ! R/ R0]PV                  4      t4 ! R1 R2]4      t5]  ! R3 R4]4      4       t6] ! R5R67      ] ! R7 R8]4      4       4       t7] ! R9R67      ] ! R: R;]4      4       4       t8]  ! R< R=]64      4       t9] ! R>R67       ! R? R@]6]4      4       t:. RCOt;R# )DzPyTorch MAMBA2 model.N)Ú	dataclass)Únn)Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Ú&use_kernel_func_from_hub_with_fallbackÚuse_kernelized_func)Úforce_accelerate_hooks)Úcreate_recurrent_attention_mask)ÚGradientCheckpointingLayer)ÚPreTrainedModel)ÚModelOutputÚauto_docstringÚlogging©ÚMamba2Configc                óD   € V ^8„  d   QhR\         P                  R\        /# )é   Úinput_tensorÚpad_size)ÚtorchÚTensorÚint)Úformats   "Úk/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/mamba2/modeling_mamba2.pyÚ__annotate__r   *   s"   € ÷ Vñ V¤U§\¡\ð V¼Sñ Vó    c                ó²   € \        V P                  4      ^8X  d   ^ ^ ^ ^ ^ V^ ^ 3M^ ^ ^ V^ ^ 3p\        P                  P                  P                  WR^ R7      # )zv
Padding x tensor with `pad_size` on the seq_len dim (dim=1)

Assumes that we only have tensors of either size 4 or 3
Úconstant)ÚmodeÚvalue)ÚlenÚshaper   r   Ú
functionalÚpad)r   r   Ú	pad_shapes   && r   Úpad_tensor_by_sizer(   *   sd   € ô 47°|×7IÑ7IÓ3JÈaÔ3O��A�q˜!˜Q ¨!¨QÑ/ÐVWÐYZÐ\]Ð_gÐijÐlmÐUn€Iä�8‰8×Ñ×"Ñ" <ÀÐSTÐ"ÓUÐUr   c                óD  € \        W4      p \        V P                  4      ^8X  d6   V P                  V P                  ^ ,          RW P                  ^,          4      # V P                  V P                  ^ ,          RW P                  ^,          V P                  ^,          4      # )z°
Padding input_tensor with `pad_size` on the seq_len dim (dim=1) and
simultaneously splitting it into chunk sequences.

Assumes that we only have tensors of either size 4 or 3
éÿÿÿÿ)r(   r#   r$   Úreshape)r   r   Ú
chunk_sizes   &&&r   Úreshape_into_chunksr-   5   s�   € ô & lÓ=€Lä
ˆ<×ÑÓ !Ô#à×#Ñ# L×$6Ñ$6°qÕ$9¸2¸z×K]ÑK]Ð^_ÕK`ÓaÐað ×#Ñ#Ø×Ñ˜qÕ! 2 z×3EÑ3EÀaÕ3HÈ,×J\ÑJ\Ð]^ÕJ_ó
ð 	
r   c           	     ó2  € V P                  R4      pV R,          P                  ! . V P                  4       OVN5!  p \        P                  ! \        P                  ! WV P
                  \        P                  R7      RR7      pV P                  V( ^ 4      p \        P                  ! V RR7      p\        P                  ! \        P                  ! WV P
                  \        P                  R7      ^ R7      pVP                  V( \        P                  ) 4      pV# )zg
More stable segment sum calculation. Uses cumulative sums and masking instead of direct subtractions.
©ÚdeviceÚdtype)Údiagonal©Údimr*   ©.Néþÿÿÿ)
ÚsizeÚexpandr   ÚtrilÚonesr0   ÚboolÚmasked_fillÚcumsumÚinf)r   r,   ÚmaskÚtensor_segsums   &   r   Úsegment_sumrA   I   sØ   € ð ×"Ñ" 2Ó&€Jð   	Õ*×1Ò1ÐS°<×3DÑ3DÓ3FÐSÈ
ÓS€Lä�:Š:”e—j’j À×@SÑ@SÔ[`×[eÑ[eÔfÐqsÔt€DØ×+Ñ+¨T¨E°1Ó5€Lä—L’L °2Ô6€Mô �:Š:”e—j’j À×@SÑ@SÔ[`×[eÑ[eÔfÐqrÔs€DØ!×-Ñ-¨t¨e´e·i±i°ZÓ@€MØÐr   c                ób   € Ve+   V P                   pWR,          ,          P                  V4      p V # )ze
Tunes out the hidden states for padding tokens, see https://github.com/state-spaces/mamba/issues/66
©ºNNNrD   N)r1   Úto)Úhidden_statesÚattention_maskr1   s   && r   Úapply_mask_to_padding_statesrH   ]   s3   € ð
 Ò!Ø×#Ñ#ˆØ&¸
Õ)CÕC×GÑGÈÓNˆàÐr   c                   ó@   a a€ ] tR t^it oRV 3R lltRR ltRtVtV ;t# )ÚMambaRMSNormGatedc                óŽ   <€ \         SV `  4        \        P                  ! \        P
                  ! V4      4      V n        W n        R # ©N©ÚsuperÚ__init__r   Ú	Parameterr   r:   ÚweightÚvariance_epsilon©ÚselfÚhidden_sizeÚepsÚ	__class__s   &&&€r   rO   ÚMambaRMSNormGated.__init__j   s-   ø€ Ü‰ÑÔÜ—l’l¤5§:¢:¨kÓ#:Ó;ˆŒØ #Ör   c                óÈ  € VP                   pVP                  \        P                  4      pVeD   V\        P
                  P                  VP                  \        P                  4      4      ,          pVP                  ^4      P                  RRR7      pV\        P                  ! W@P                  ,           4      ,          pV P                  VP                  V4      ,          # )NT©Úkeepdimr*   )r1   rE   r   Úfloat32r   r%   ÚsiluÚpowÚmeanÚrsqrtrR   rQ   )rT   rF   ÚgateÚinput_dtypeÚvariances   &&&  r   ÚforwardÚMambaRMSNormGated.forwardo   s£   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆàÒØ)¬B¯M©M×,>Ñ,>¸t¿w¹wÄuÇ}Á}Ó?UÓ,VÕVˆMØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯ª°H×?TÑ?TÕ4TÓ(UÕUˆà�{‰{˜]×-Ñ-¨kÓ:Õ:Ð:r   ©rR   rQ   ©ç�íµ ÷Æ°>rL   ©	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__rO   rd   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©rW   Ú__classdict__s   @@r   rJ   rJ   i   s   ù‡ € ÷$÷
	;ô 	;r   rJ   Úcausal_conv1d_updateÚcausal_conv1dc                óÀ   € V ^8„  d   QhR\         P                  R\         P                  R\        P                  R\        P                  R,          R\        R,          /# )r   rF   Ú
conv_staterQ   ÚbiasNÚ
activation©r   r   r   rP   Ústr)r   s   "r   r   r   |   sS   € ÷ 'ñ 'Ü—<‘<ð'ä—‘ð'ô �L‰Lð'ô �,‰,˜Õ
ð	'ô
 �d•
ñ'r   c                 óº  € V P                   w  rVpVP                   R,          p\        P                  ! W.RR7      P                  VP                  4      p	VP                  V	RRV) R13,          4       \        P                  ! W’P                  ^4      V^ VR7      p
V
RRV) R13,          p
Ve   \        V,          ! V
4      p
V
P                  V P                  4      # )é   r3   rD   N)ÚpaddingÚgroupsr*   )
r$   r   ÚcatrE   r1   Úcopy_ÚFÚconv1dÚ	unsqueezer   )rF   rv   rQ   rw   rx   Ú_rU   Úseq_lenÚ	state_lenÚhidden_states_newÚouts   &&&&&      r   rs   rs   {   sÅ   € ð ,×1Ñ1Ñ€A�GØ× Ñ  Õ$€IäŸ	š	 :Ð"=À2ÔF×IÑIÈ&Ï,É,ÓWÐØ×ÑÐ& q¨!¨i¨Z©[Ð'8Õ9Ô:Ü
�(Š(Ð$×&6Ñ&6°qÓ&9¸4ÈÐS^Ô
_€CØ
ˆa��W�H‘IˆoÕ
€CØÒÜ�ZÖ  Ó%ˆØ�6‰6�-×%Ñ%Ó&Ð&r   Úcausal_conv1d_fnc          	      ó    € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R,          R\        R,          /# )r   rF   rQ   rw   Nrx   ry   )r   s   "r   r   r   �   sE   € ÷ 'ñ 'Ü—<‘<ð'ä�L‰Lð'ô �,‰,˜Õ
ð'ô �d•
ñ	'r   c                 óV  € V P                   w  rVpVP                   R,          ^,
          p\        P                  ! V P                  VP                  4      VP                  ^4      VVVR7      RRRV13,          p	Ve   \        V,          ! V	4      p	V	P                  V P                  4      # )r|   )rQ   rw   r}   r~   rD   Nr*   )r$   r�   r‚   rE   r1   rƒ   r   )
rF   rQ   rw   rx   Úkwargsr„   rU   r…   r}   rˆ   s
   &&&&,     r   r‰   r‰   �   sž   € ð ,×1Ñ1Ñ€A�GØ�l‰l˜2Õ Õ"€Gä
�(Š(Ø×Ñ˜Ÿ™Ó&Ø×Ñ Ó"ØØØôð ˆˆHˆWˆH€nõ€Cð ÒÜ�ZÖ  Ó%ˆØ�6‰6�-×%Ñ%Ó&Ð&r   Ú mamba_split_conv1d_scan_combinedÚ	mamba_ssmg        r>   Fr]   rh   Tc          $      ó.  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R,          R\         P                  R\         P                  R\         P                  R\        R	\         P                  R,          R
\        \        \        3,          R\
        R\        R\         P                  R,          R\        R\         P                  R,          R\         P                  R,          R\        R,          R\        R\
        /# )r   ÚzxbcdtÚconv1d_weightÚconv1d_biasNÚdt_biasÚAÚDr,   Úinitial_statesÚdt_limitÚreturn_final_statesrx   Úrmsnorm_weightÚrmsnorm_epsÚoutproj_weightÚoutproj_biasÚheaddimÚngroupsÚnorm_before_gate)r   r   r   ÚtupleÚfloatr;   rz   )r   s   "r   r   r   §   s
  € ÷ ñ Ü�L‰Lðä—<‘<ðô —‘ Õ$ðô �\‰\ð	ô
 ‡|�|ðô ‡|�|ðô ðô —L‘L 4Õ'ðô ”Eœ5�LÕ!ðô ðô ðô —L‘L 4Õ'ðô ðô —L‘L 4Õ'ðô —,‘, Õ%ðô  �4�Zð!ô" ð#ô$ ñ%r   c                 ó   € R # rL   © )r�   r‘   r’   r“   r”   r•   r,   r–   r—   r˜   rx   r™   rš   r›   rœ   r�   rž   rŸ   rŒ   s   &&&&&&&&&&&&&&&&&&,r   Ú!mamba2_split_conv1d_scan_combinedr¤   ¦   s   € ñ, r   Úselective_state_updatec                ón  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R,          R	\         P                  R,          R
\        R\         P                  R,          /
# )r   ÚstaterF   Údtr”   ÚBÚCr•   Nr“   Údt_softplusÚz)r   r   r;   )r   s   "r   r   r   À   sŸ   € ÷ :'ñ :'Ü�<‰<ð:'ä—<‘<ð:'ô 	�‰ð:'ô ‡|�|ð	:'ô
 ‡|�|ð:'ô ‡|�|ð:'ô ‡|�|�dÕð:'ô �\‰\˜DÕ ð:'ô ð:'ô ‡|�|�dÕñ:'r   c
                 ó†  € VP                   w  r¼pVP                   ^,          pVP                   R,          pVe"   W'P                  VP                  4      ,           pV'       d   \        P                  ! V4      pVR,          p\
        P                  ! VP                  4       VP                  4       ,          4      P                  V P                  R7      pVP                  W¾^V4      pVP                  W¾WÎ,          V4      P                  4       pVP                  W¼^V4      pW$,          pVVR,          ,          P                  V P                  R7      pV V,          V,           pV P                  VP                  V P                  4      4       VP                  W¾^V4      pVP                  W¾WÎ,          V4      P                  4       pVP                  W¼V4      pVP                  VP                  VP                  R7      pVP                  W¼,          Wß4      pVP                  W¼,          V^4      p\
        P                  ! VV4      pVP                  W¼V4      pVe)   VW,          ,           P                  VP                  4      pV	e   V\        P                  ! V	4      ,          pVP                  VP                  4      # )r|   )r0   r/   r*   r5   )r$   rE   r1   r�   Úsoftplusr   Úexpr¡   r0   r+   r8   Ú
contiguousr€   ÚviewÚbmmr]   )r§   rF   r¨   r”   r©   rª   r•   r“   r«   r¬   rŒ   Ú
batch_sizeÚ	num_headsÚhead_dimÚ
num_groupsÚ
state_sizeÚdAÚdBÚdBxÚ
ssm_statesÚssm_states_reshapedÚ
C_reshapedrˆ   s   &&&&&&&&&&,            r   Úmamba2_selective_state_updater¾   ¿   s"  € ð '4×&9Ñ&9Ñ#€J˜8Ø—‘˜•€JØ—‘˜•€JàÒØ—*‘*˜RŸX™XÓ&Õ&ˆßÜ�ZŠZ˜‹^ˆØ	ˆI�€Bô 
�Š�2—8‘8“: §¡£	Õ)Ó	*×	-Ñ	-°U·\±\Ð	-Ó	B€Bð 	
�	‰	�*¨!¨ZÓ8€AØ	�‰�¨Õ)@À*ÓM×XÑXÓZ€AØ	�	‰	�*¨¨JÓ7€AØ	�€Bð �˜iÕ(Õ(×
,Ñ
,°E·L±LÐ
,Ó
A€Cð ˜•˜cÕ!€JØ	‡K�K�
—‘˜eŸk™kÓ*Ô+ð 	
�	‰	�*¨!¨ZÓ8€AØ	�‰�¨Õ)@À*ÓM×XÑXÓZ€AØ	�	‰	�*¨Ó4€Að —‘ a§h¡h°a·g±g�Ó>€JØ$Ÿ/™/¨*Õ*@À(ÓWÐØ—‘˜
Õ.°
¸AÓ>€JÜ
�)Š)Ð'¨Ó
4€CØ
�(‰(�:¨(Ó
3€Cð 	‚}Ø�]Õ&Õ&×*Ñ*¨3¯9©9Ó5ˆà‚}Ø”A—F’F˜1“I�oˆà�6‰6�-×%Ñ%Ó&Ð&r   Úmamba_chunk_scan_combinedc                ó”  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R\        R\         P                  R,          R	\         P                  R,          R
\         P                  R,          R\        R\        \
        \
        3,          R\        /# )r   rF   r¨   r”   r©   rª   r,   r•   Nr“   r–   r«   r—   r˜   )r   r   r   r;   r    r¡   )r   s   "r   r   r   þ   sÆ   € ÷ ^ñ ^Ü—<‘<ð^ä�‰ð^ô ‡|�|ð^ô ‡|�|ð	^ô
 ‡|�|ð^ô ð^ô ‡|�|�dÕð^ô �\‰\˜DÕ ð^ô —L‘L 4Õ'ð^ô ð^ô ”Eœ5�LÕ!ð^ô ñ^r   c                 óô  € V P                   w  rÞppVP                   ^,          pVe"   WP                  VP                  4      ,           pV	'       d   \        P                  ! V4      p\
        P                  ! W^ ,          V
^,          R7      pV P                  4       p VP                  4       P                  VV,          ^VR7      pVP                  4       P                  VV,          ^VR7      pW^V,          ,
          V,          pRpVe   VR,          \        V V4      ,          pWR,          P                  4       ,          p VP                  V P                  4      VP                  4       ,          pWW43 Uu. uF  p\        VVV4      NK  	  upw  rr4VP                  ^ ^^^4      p\
        P                  ! VRR7      p\
        P                  ! \        V4      4      pVR	,          VR
,          ,          P                  RR7      pVR,          VP                  ^ ^^^^4      R,          ,          P                  RR7      pVR,          V R,          ,          P                  ^R7      p\
        P                  ! VRRRRR13,          V,
          4      pVVP                  ^ RR^4      R,          ,          pVR,          V R,          ,          P                  ^R7      pVe/   VR,          P                  VP                  VP                   R7      M\
        P"                  ! VR,          4      p\
        P$                  ! VV.^R7      p\
        P                  ! \        \        P&                  ! VR,          R4      4      4      P)                  ^^4      pVR,          VR,          ,          P                  ^R7      pVRRR13,          VR,          p p\
        P                  ! V4      p!VR,          VR,          ,          p"V"P                  R4      V!P                  ^ ^^^4      R,          ,          p#VV#,           p$V$P+                  VRVV4      p$Ve
   V$V,           p$V^ 8”  d   V$RRV13,          p$V'       d   V$V 3# V$# u upi )r   N)ÚminÚmax)r4   Úoutput_sizer3   rD   )r1   r0   r5   r*   )rD   rD   rD   NrD   rD   )rD   rD   NrD   rD   rD   rC   r6   ).NrD   ©rD   N)rD   :Nr|   N)rD   rD   rD   r*   )r|   é    ).NN)rD   rD   N.)rD   r*   )r$   rE   r1   r�   r®   r   Úclampr¡   Úrepeat_interleaver(   r-   Úpermuter=   r¯   rA   Úsumr0   Ú
zeros_liker   r&   Ú	transposer+   )%rF   r¨   r”   r©   rª   r,   r•   r“   r–   r«   r—   r˜   rŒ   r³   Úsequence_lengthr´   rµ   r¶   r   Ú
D_residualÚtensorÚA_cumsumÚLÚGÚMÚY_diagÚdecay_statesÚB_decayÚstatesÚprevious_statesÚdecay_chunkÚ
new_statesÚfinal_stateÚstate_decay_outÚC_times_statesÚY_offÚoutputs%   &&&&&&&&&&&&,                        r   Úmamba2_chunk_scanrà   ý   së  € ð  8E×7JÑ7JÑ4€J ¨HØ—‘˜•€JàÒØ—*‘*˜RŸX™XÓ&Õ&ˆßÜ�ZŠZ˜‹^ˆÜ	�Š�R a�[¨h°q­kÔ	:€Bà!×'Ñ'Ó)€MØ	�‰‹	×#Ñ# I°Õ$;ÀÐPYÐ#ÓZ€AØ	�‰‹	×#Ñ# I°Õ$;ÀÐPYÐ#ÓZ€Aà¨zÕ9Õ9¸ZÕG€HØ€JØ‚}Ø�y•\Ô$6°}ÀhÓ$OÕOˆ
ð " y¥M×$7Ñ$7Ó$9Õ9€MØ	�‰ˆ]× Ñ Ó! B§H¡H£JÕ.€Að `mÐqrÑ^vÓwÑ^vÐTZÔ1°&¸(ÀJÖOÑ^vÑwÑ€M�aà	�	‰	�!�Q˜˜1Ó€AÜ�|Š|˜A 2Ô&€Hô 	�	Š	”+˜a“.Ó!€Að 
ÐÕ	 !Ð$7Õ"8Õ	8×=Ñ=À"Ð=ÓE€Að 
ˆ9�˜Ÿ	™	 ! Q¨¨1¨aÓ0°Õ;Õ	;×@Ñ@ÀRÐ@ÓH€Að �	�l˜]¨:Õ6Õ6×;Ñ;ÀÐ;ÓB€Fô —9’9˜X a¨¨A¨r©s lÕ3°hÕ>Ó?€LØ�,×&Ñ& q¨"¨b°!Ó4°YÕ?Õ?€GØ�lÕ# m°IÕ&>Õ>×CÑCÈÐCÓJ€Fð Ò%ð 	�wÕ×"Ñ"¨¯©¸f¿m¹mÐ"ÔLä×Ò˜f U�mÓ,ð ô
 �YŠY˜¨Ð0°aÔ8€FÜ—)’)œK¬¯ª¨h°{Õ.CÀVÓ(LÓMÓN×XÑXÐYZÐ\]Ó^€KØ˜oÕ.°¸Õ1HÕH×MÑMÐRSÐMÓT€JØ$ Q¨¨¨ VÕ,¨j¸Õ.?ˆK€Fô —i’i Ó)€OØ�|•_ v¨oÕ'>Õ>€NØ×Ñ˜rÓ" _×%<Ñ%<¸QÀÀ1ÀaÓ%HÈÕ%SÕS€Eð �e�^€FØ�^‰^˜J¨¨I°xÓ@€FàÒØ˜*Õ$ˆð �!„|Ø˜Ð+˜OÐ+Ð+Õ,ˆçØ�{Ð"Ð"à€Mùòq xs   Å8Q5c                   ó    a a€ ] tR tRt oRtR
V3R lV 3R lllt]P                  ! 4       R 4       t]	! R4      RV3R lR ll4       t
R	tVtV ;t# )ÚMamba2Mixeri_  uo  
Compute âˆ†, A, B, C, and D the state space parameters and compute the `contextualized_states`.
A, D are input independent (see Mamba paper [1] Section 3.5.2 "Interpretation of A" for why A isn't selective)
âˆ†, B, C are input-dependent (this is a key difference between Mamba and the linear time invariant S4,
and is why Mamba is called **selective** state spaces)
c                ó,   <€ V ^8„  d   QhRS[ RS[RS[/# )r   ÚconfigÚ	layer_idxÚinitialize_mixer_weights)r   r   r;   )r   rr   s   "€r   r   ÚMamba2Mixer.__annotate__p  s#   ø€ ÷ 98ñ 98™|ð 98¹ð 98ÑW[ñ 98r   c           	     ó°  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  V n        VP                  V n        \        VP                  V P                  ,          4      V n
        \        VP                  4      V n        W n        VP                  V n        VP                  V n        \         VP                  ,          V n        VP$                  V n        VP&                  V n        VP(                  V n        VP*                  V n        VP,                  V n        VP.                  V n        VP0                  V n        VP2                  V n        V P                  ^V P&                  ,          V P
                  ,          ,           V n        \6        P8                  ! V P4                  V P4                  VP                  VP                  V P4                  VP                  ^,
          R7      V n        V P                  V P4                  ,           V P                  ,           p\6        P<                  ! V P                  VVP>                  R7      V n         \6        PB                  ! \D        PF                  ! V P                  4      4      V n$        \6        PB                  ! \D        PF                  ! V P                  4      4      V n%        \M        V P                  V P$                  R7      V n'        \6        PB                  ! \D        PF                  ! V P                  4      4      V n(        V'       d6   V PH                  PR                  PT                  R8w  d   V PW                  4        \6        P<                  ! V P                  V P                  VP>                  R7      V n,        VP>                  V n        VPZ                  V,          V n.        R# )r   )Úin_channelsÚout_channelsrw   Úkernel_sizer~   r}   ©rw   ©rV   ÚmetaN)/rN   rO   r´   rU   r·   Ússm_state_sizeÚconv_kernelÚconv_kernel_sizer   r8   Úintermediate_sizeÚtime_step_rankrå   Úuse_conv_biasÚ
hidden_actrx   r   ÚactÚlayer_norm_epsilonÚn_groupsrµ   r,   Útime_step_limitÚtime_step_minÚtime_step_maxÚtime_step_floorÚconv_dimr   ÚConv1dr‚   ÚLinearÚuse_biasÚin_projrP   r   Úemptyr“   ÚA_logrJ   Únormr•   r0   ÚtypeÚinit_mamba2_weightsÚout_projÚlayer_typesÚ
layer_type)rT   rä   rå   ræ   Úprojection_sizerW   s   &&&& €r   rO   ÚMamba2Mixer.__init__p  s›  ø€ Ü‰ÑÔØ×)Ñ)ˆŒØ!×-Ñ-ˆÔØ$×/Ñ/ˆÔØ &× 2Ñ 2ˆÔÜ!$ V§]¡]°T×5EÑ5EÕ%EÓ!FˆÔÜ! &×"7Ñ"7Ó8ˆÔØ"ŒØ#×1Ñ1ˆÔØ ×+Ñ+ˆŒÜ˜&×+Ñ+Õ,ˆŒà"(×";Ñ";ˆÔàŸ™ˆŒØŸ™ˆŒØ ×+Ñ+ˆŒà%×5Ñ5ˆÔØ#×1Ñ1ˆÔØ#×1Ñ1ˆÔØ%×5Ñ5ˆÔà×.Ñ.°°T·]±]Õ1BÀT×EXÑEXÕ1XÕXˆŒÜ—i’iØŸ™ØŸ™Ø×%Ñ%Ø×*Ñ*Ø—=‘=Ø×&Ñ&¨Õ*ô
ˆŒð ×0Ñ0°4·=±=Õ@À4Ç>Á>ÕQˆÜ—y’yØ×ÑØØ—‘ô
ˆŒô —|’|¤E§K¢K°·±Ó$?Ó@ˆŒô —\’\¤%§+¢+¨d¯n©nÓ"=Ó>ˆŒ
Ü% d×&<Ñ&<À$×BYÑBYÔZˆŒ	Ü—’œeŸkšk¨$¯.©.Ó9Ó:ˆŒß#¨¯©×(;Ñ(;×(@Ñ(@ÀFÔ(JØ×$Ñ$Ô&äŸ	š	 $×"8Ñ"8¸$×:JÑ:JÐQW×Q`ÑQ`ÔaˆŒØŸ™ˆŒà ×,Ñ,¨YÕ7ˆŽr   c                ó¶  € \         P                  ! ^V P                  ^,           V P                  P                  \         P
                  R7      p\        P                  ! V P                  \         P                  ! V4      4       \        P                  ! V P                  4       \         P                  ! \         P                  ! V P                  V P                  P                  \         P
                  R7      \        P                  ! V P                  4      \        P                  ! V P                   4      ,
          ,          \        P                  ! V P                   4      ,           4      P#                  V P$                  R7      pV\         P                  ! \         P&                  ! V) 4      ) 4      ,           p\        P                  ! V P                  V4       R# )r|   r/   )rÂ   N)r   Úaranger´   r  r0   r\   Úinitr€   ÚlogÚones_r•   r¯   Úrandr“   Úmathrû   rú   rÇ   rü   Úexpm1)rT   r”   r¨   Úinv_dts   &   r   r  ÚMamba2Mixer.init_mamba2_weights«  s  € ä�LŠL˜˜DŸN™N¨QÕ.°t·z±z×7HÑ7HÔPU×P]ÑP]Ô^ˆÜ�
Š
�4—:‘:œuŸyšy¨›|Ô,Ü�
Š
�4—6‘6Ôä�YŠYÜ�JŠJ�t—~‘~¨d¯l©l×.AÑ.AÌÏÉÔWÜ�xŠx˜×*Ñ*Ó+¬d¯hªh°t×7IÑ7IÓ.JÕJõLä�hŠh�t×)Ñ)Ó*õ+ó
÷ ‰%�D×(Ñ(ˆ%Ó
)ð	 	ð ”e—i’i¤§¢¨b¨SÓ!1Ð 1Ó2Õ2ˆÜ�
Š
�4—<‘< Ö(r   r‚   c                óp   <€ V ^8„  d   QhRS[ P                  RS[R,          RS[ P                  R,          /# )r   rF   Úcache_paramsNrG   )r   r   r   )r   rr   s   "€r   r   rç   ¼  s?   ø€ ÷ P%ñ P%á—|‘|ðP%ñ ˜d•lðP%ñ Ÿ™ tÕ+ñ	P%r   c                ó  € VP                   w  rVpVP                  pVR J;'       d    VP                  V P                  4      p	\	        W4      pV P                  V4      p
\        P                  ! V P                  P                  4       4      ) pVRV P                  /,          pV P                  '       dô   Vfð   \        V
V P                  P                  P                  ^4      V P                  P                   V P"                  V3RV P$                  RV P&                  RV P(                  RV P*                  P                  RV P*                  P,                  RV P.                  P                  RV P.                  P                   R	V P0                  R
V P2                  RRRR/VB pVe   V# V
P5                  V P6                  V P8                  V P:                  .RR7      w  rïpV	'       d]   VP<                  V P                  ,          P>                  ^ ,          pVP<                  V P                  ,          P@                  ^ ,          pVPC                  ^^4      pV	'       d‡   V^8X  d€   VP<                  V P                  ,          PD                  '       gS   \G        VXV P                  P                  P                  ^4      V P                  P                   V P(                  R7      pM�Ve)   VPI                  VV P                  V PJ                  R7      p\M        VV P                  P                  P                  ^4      V P                  P                   3RV P(                  /VB pVe   VRRV) R 13,          p\	        VPC                  ^^4      V4      p\        P4                  ! VV P6                  V P2                  V PN                  ,          V P2                  V PN                  ,          .RR7      w  pppV	'       Ed]   V^8X  EdU   VPQ                  WPP:                  V P0                  4      pVPC                  ^^4      PS                  RRV P0                  4      pVR,          PS                  RV P0                  V PN                  4      pVPQ                  WPP2                  V PN                  4      pVPQ                  WPP2                  V PN                  4      pV P$                  R,          PS                  RV P0                  4      pV P"                  R,          PS                  RV P0                  4      p\U        XVVVVVV3RR RVRR/VB pVPQ                  V^R4      pEMVR Jp\W        VPQ                  WVV P:                  V P0                  4      VVVPQ                  WVV P2                  V PN                  4      VPQ                  WVV P2                  V PN                  4      3RV P&                  RV P$                  RR RTRV P"                  RRRV	'       d   XMR RV P                  /VB pV'       d$   Vw  ppVPY                  VV P                  R7       MTpVP[                  WVR4      pV P+                  VV4      pV P/                  VP]                  V4      4      pV# )Nr—   r•   r,   rx   r™   rš   r›   rœ   r�   rž   rŸ   Fr˜   r3   )rx   )rñ   rD   r¬   r“   r«   Tr–   ©rå   r*   )rD   NNrÅ   )/r$   r1   Úhas_previous_staterå   rH   r  r   r¯   r  r¡   rù   Útrainingr¤   r‚   rQ   Úsqueezerw   r“   r•   r,   rx   r  rR   r  rµ   rø   Úsplitrò   rý   r´   ÚlayersÚconv_statesÚrecurrent_statesrÌ   Úrecord_pastrs   Úupdate_conv_staterñ   r‰   rï   r±   r8   r¾   rà   Úupdate_recurrent_stater+   rE   )rT   rF   r  rG   rŒ   r³   r…   r„   r1   Úuse_precomputed_statesÚprojected_statesr”   Úfused_kwargsÚfused_outputra   Úhidden_states_B_Cr¨   rv   Úrecurrent_stater©   rª   r•   r“   Úscan_outputÚoutput_final_stateÚscan_resultrÛ   Úcontextualized_statess   &&&&,                       r   rd   ÚMamba2Mixer.forward»  s  € ð "/×!4Ñ!4Ñˆ
˜QØ×#Ñ#ˆØ!-°TÐ!9×!mÐ!m¸l×>]Ñ>]Ð^b×^lÑ^lÓ>mÐô 5°]ÓSˆØŸ<™<¨Ó6Ðä�YŠY�t—z‘z×'Ñ'Ó)Ó*Ð*ˆØ ¨T×-AÑ-AÐ BÕBˆØ�=�=ˆ=˜\Ò1Ü<Ø Ø—‘×"Ñ"×*Ñ*¨1Ó-Ø—‘× Ñ Ø—‘Øñð —&‘&ðð  Ÿ?™?ðð  Ÿ?™?ðð  $Ÿy™y×/Ñ/ðð !ŸI™I×6Ñ6ðð  $Ÿ}™}×3Ñ3ðð "Ÿ]™]×/Ñ/ðð Ÿ™ðð Ÿ™ðð "'ðð  %*ð!ð" ñ#ˆLð* Ò'Ø#Ð#à&6×&<Ñ&<Ø×#Ñ# T§]¡]°D·N±NÐCÈð '=ó '
Ñ#ˆ ÷ "Ø%×,Ñ,¨T¯^©^Õ<×HÑHÈÕKˆJØ*×1Ñ1°$·.±.ÕA×RÑRÐSTÕUˆOð .×7Ñ7¸¸1Ó=Ðß! g°¤l¸<×;NÑ;NÈtÏ~É~Õ;^×;j×;jÐ;jÜ 4Ø!ØØ—‘×"Ñ"×*Ñ*¨1Ó-Ø—‘× Ñ ØŸ?™?ô!Ñð Ò'Ø$0×$BÑ$BØ%Ø—N‘NØ%)×%:Ñ%:ð %Có %Ð!ô !1Ø!Ø—‘×"Ñ"×*Ñ*¨1Ó-Ø—‘× Ñ ñ!ð  Ÿ?™?ð	!ð
 ñ!Ðð Ò'Ø$5°a¸¸W¸H¹I°oÕ$FÐ!ô 9Ð9J×9TÑ9TÐUVÐXYÓ9ZÐ\jÓkÐÜ#ŸkškØØ×#Ñ# T§]¡]°T×5HÑ5HÕ%HÈ$Ï-É-ÐZ^×ZmÑZmÕJmÐnØô
Ñˆ�q˜!÷ "Ð! g°¥lØ)×.Ñ.¨z¿>¹>È4Ï=É=ÓYˆMØ—‘˜a Ó#×*Ñ*¨2¨r°4·=±=ÓAˆBØ�-Õ ×'Ñ'¨¨D¯M©M¸4×;NÑ;NÓOˆAØ—‘�z§=¡=°$×2EÑ2EÓFˆAØ—‘�z§=¡=°$×2EÑ2EÓFˆAØ—‘�w•×&Ñ& r¨4¯=©=Ó9ˆAØ—l‘l 7Õ+×2Ñ2°2°t·}±}ÓEˆGä7ØØØØØØØñð ðð  ðð !ðð ñˆKð &×*Ñ*¨:°q¸"Ó=ŠKð ".°TÐ!9ÐÜ+Ø×"Ñ" :¸¿¹ÈÏÉÓVØØØ—‘�z¨D¯M©M¸4×;NÑ;NÓOØ—‘�z¨D¯M©M¸4×;NÑ;NÓOñð  Ÿ?™?ðð —&‘&ðð ðð %7ðð Ÿ™ðð !ð÷ 3I™Èdðð ×-Ñ-ðð ñˆK÷" "Ø+6Ñ(�˜[Ø×3Ñ3°KÈ4Ï>É>Ð3ÕZà)�à%×-Ñ-¨jÀ2ÓFˆKà—i‘i ¨TÓ2ˆð !%§¡¨k¯n©n¸UÓ.CÓ DÐØ$Ð$r   )r  r•   rö   rx   r,   r‚   rý   rñ   r“   rµ   rU   r  rò   rå   r÷   r	  rø   r  r´   r  rï   rü   rù   rû   rú   ró   r   rô   )T©NN)rj   rk   rl   rm   Ú__doc__rO   r   Úno_gradr  r   rd   rn   ro   rp   rq   s   @@r   râ   râ   _  sV   ù‡ € ñ÷98õ 98ðv ‡]‚]ƒ_ñ)ó ð)ñ ˜HÓ%÷P%ñ P%ó &÷P%ð P%r   râ   c                   ó<   a a€ ] tR tRt oRV 3R lltR tRtVtV ;t# )ÚMamba2RMSNormiO  c                óŽ   <€ \         SV `  4        \        P                  ! \        P
                  ! V4      4      V n        W n        R# )z=
Mamba2RMSNorm is equivalent to T5LayerNorm and LlamaRMSNorm
NrM   rS   s   &&&€r   rO   ÚMamba2RMSNorm.__init__P  s/   ø€ ô 	‰ÑÔÜ—l’l¤5§:¢:¨kÓ#:Ó;ˆŒØ #Ör   c                ó:  € VP                   pVP                  \        P                  4      pVP	                  ^4      P                  RRR7      pV\        P                  ! W0P                  ,           4      ,          pV P                  VP                  V4      ,          # )r   TrZ   r*   )	r1   rE   r   r\   r^   r_   r`   rR   rQ   )rT   rF   rb   rc   s   &&  r   rd   ÚMamba2RMSNorm.forwardX  sw   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯ª°H×?TÑ?TÕ4TÓ(UÕUˆØ�{‰{˜]×-Ñ-¨kÓ:Õ:Ð:r   rf   rg   ri   rq   s   @@r   r3  r3  O  s   ù‡ € ÷$÷;ò ;r   r3  c                   óH   a a€ ] tR tRt oV 3R ltRV3R lR lltRtVtV ;t# )ÚMamba2Blocki`  c                óÎ   <€ \         SV `  4        Wn        W n        VP                  V n        \        VP                  VP                  R 7      V n        \        WRR7      V n
        R# )rí   F)rå   ræ   N)rN   rO   rä   rå   Úresidual_in_fp32r3  rU   r÷   r  râ   Úmixer)rT   rä   rå   rW   s   &&&€r   rO   ÚMamba2Block.__init__a  sO   ø€ Ü‰ÑÔØŒØ"ŒØ &× 7Ñ 7ˆÔÜ! &×"4Ñ"4¸&×:SÑ:SÔTˆŒ	Ü  ÐW\Ô]ˆŽ
r   c                óV   <€ V ^8„  d   QhRS[ R,          RS[P                  R,          /# )r   r  NrG   )r   r   r   )r   rr   s   "€r   r   ÚMamba2Block.__annotate__i  s.   ø€ ÷ ñ ñ ˜d•lðñ Ÿ™ tÕ+ñ	r   c                ó(  € TpV P                  VP                  V P                   P                  P                  R 7      4      pV P                  '       d    VP                  \
        P                  4      pV P                  ! V3RVRV/VB pWQ,           pV# ))r1   r  rG   )r  rE   rQ   r1   r;  r   r\   r<  )rT   rF   r  rG   rŒ   Úresiduals   &&&&, r   rd   ÚMamba2Block.forwardi  s€   € ð !ˆØŸ	™	 -×"2Ñ"2¸¿¹×9IÑ9I×9OÑ9OÐ"2Ó"PÓQˆØ× × Ð Ø—{‘{¤5§=¡=Ó1ˆHàŸ
š
 =Ñu¸|ÐuÐ\jÐuÐntÑuˆØ Õ0ˆØÐr   )rä   rå   r<  r  r;  r/  ri   rq   s   @@r   r9  r9  `  s   ù‡ € õ^÷÷ ò r   r9  c                   ó|   a a€ ] tR tRt oRtR.tRtRtRt]	P                  ! 4       V 3R l4       tV3R ltRtVtV ;t# )ÚMamba2PreTrainedModeliz  Úbackboner9  Tc                óÆ  <€ \         SV `  V4       \        V\        4      '       Ed:   VP	                  4        \
        P                  ! VP                  P                  \        P                  ! ^4      R7       VP                  P                  e+   \
        P                  ! VP                  P                  4       \
        P                  ! VP                  P                  \        P                  ! ^4      R7       V P                  P                  '       dJ   VP                  P                  pV\        P                  ! V P                  P                   4      ,          pR# R# R# )zInitialize the weights.)ÚaN)rN   Ú_init_weightsÚ
isinstancerâ   r  r  Úkaiming_uniform_r‚   rQ   r  Úsqrtrw   Úzeros_r  rä   Úrescale_prenorm_residualÚnum_hidden_layers)rT   ÚmoduleÚprW   s   && €r   rH  Ú#Mamba2PreTrainedModel._init_weightsƒ  sß   ø€ ô 	‰Ñ˜fÔ%Ü�fœk×*Ó*ð ×&Ñ&Ô(ä×!Ò! &§-¡-×"6Ñ"6¼$¿)º)ÀA»,ÕGØ�}‰}×!Ñ!Ò-Ü—’˜FŸM™M×.Ñ.Ô/Ü×!Ò! &§/¡/×"8Ñ"8¼D¿IºIÀa»LÕIà�{‰{×3×3Ð3ð —O‘O×*Ñ*�Ø”T—Y’Y˜tŸ{™{×<Ñ<Ó=Õ=’ñ 4ñ +r   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # )r   rä   r   )r   rr   s   "€r   r   Ú"Mamba2PreTrainedModel.__annotate__z  s   ø‡ ‚ áÑò r   r£   )rj   rk   rl   rm   Úbase_model_prefixÚ_no_split_modulesÚsupports_gradient_checkpointingÚ_can_compile_fullgraphÚ_is_statefulr   r1  rH  Ú__annotate_func__rn   ro   rp   rq   s   @@r   rD  rD  z  sE   ù‡ € ð #ÐØ&˜ÐØ&*Ð#Ø!ÐØ€Là
‡]‚]ƒ_ô>ó ð>÷ … r   rD  z-
    Class for the MAMBA2 model outputs.
    )Úcustom_introc                   ó<   a € ] tR tRt o RtRtRtRtV 3R ltRt	V t
R# )ÚMamba2Outputi   a   
cache_params (`Cache`):
    The state of the model at the last time step. Can be used in a forward method with the next `input_ids` to
    avoid providing the old `input_ids`.

    Includes both the State space model state matrices after the selective scan, and the Convolutional states
Nc                ó    <€ V ^8„  d   Qh/ S[ P                  R,          ;R&   S[R,          ;R&   S[S[ P                  ,          R,          ;R&   # )r   NÚlast_hidden_stater  rF   ©r   ÚFloatTensorr   r    )r   rr   s   "€r   r   ÚMamba2Output.__annotate__   sK   ø‡ ‚ ñ  ×(Ñ(¨4Õ/Ñ6ñ! ñ" ˜$•,Ñ%ñ# ñ$ ™×*Ñ*Õ+¨dÕ2Ñ9ò% r   r£   )rj   rk   rl   rm   r0  r^  r  rF   rY  rn   ro   ©rr   s   @r   r\  r\     s#   ø‡ € ñð 37ÐØ!%€LØ59€M÷% ƒ r   r\  zK
    Base class for causal language model (or autoregressive) outputs.
    c                   ó@   a € ] tR tRt o RtRtRtRtRtV 3R lt	Rt
V tR# )ÚMamba2CausalLMOutputiµ  au  
loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
    Language modeling loss (for next-token prediction).
logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
    Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
cache_params (`Cache`):
    The state of the model at the last time step. Can be used in a forward method with the next `input_ids` to
    avoid providing the old `input_ids`.

    Includes both the State space model state matrices after the selective scan, and the Convolutional states
Nc                óÎ   <€ V ^8„  d   Qh/ S[ P                  R,          ;R&   S[ P                  R,          ;R&   S[R,          ;R&   S[S[ P                  ,          R,          ;R&   # )r   NÚlossÚlogitsr  rF   r_  )r   rr   s   "€r   r   Ú!Mamba2CausalLMOutput.__annotate__µ  sc   ø‡ ‚ ñ( ×
Ñ
˜dÕ
"Ñ)ñ) ñ* ×Ñ Õ$Ñ+ñ+ ñ, ˜$•,Ñ%ñ- ñ. ™×*Ñ*Õ+¨dÕ2Ñ9ò/ r   r£   )rj   rk   rl   rm   r0  rf  rg  r  rF   rY  rn   ro   rb  s   @r   rd  rd  µ  s'   ø‡ € ñ
ð &*€DØ'+€FØ!%€LØ59€M÷/ ƒ r   rd  c                   ód   a a€ ] tR tRt oV 3R ltR tR tR t]R	V3R lR ll4       t	Rt
VtV ;t# )
ÚMamba2ModeliÏ  c           
     óÐ  <€ \         SV `  V4       \        P                  ! VP                  VP
                  4      V n        \        P                  ! \        VP                  4       Uu. uF  p\        WR 7      NK  	  up4      V n        RV n        \        VP
                  VP                  R7      V n        V P!                  V P"                  4       V P%                  4        R# u upi )r  Frí   N)rN   rO   r   Ú	EmbeddingÚ
vocab_sizerU   Ú
embeddingsÚ
ModuleListÚrangerN  r9  r  Úgradient_checkpointingr3  r÷   Únorm_fÚ"_register_load_state_dict_pre_hookÚ	load_hookÚ	post_init)rT   rä   ÚidxrW   s   && €r   rO   ÚMamba2Model.__init__Ñ  s¥   ø€ Ü‰Ñ˜Ô äŸ,š, v×'8Ñ'8¸&×:LÑ:LÓMˆŒÜ—m’mÔSXÐY_×YqÑYqÔSrÓ$sÑSrÈC¤[°×%GÑSrÑ$sÓtˆŒà&+ˆÔ#Ü# F×$6Ñ$6¸F×<UÑ<UÔVˆŒà×/Ñ/°·±Ô?Ø�‰Öùò %ts   Á(C#c                ór   € V F0  pR V9   g   K  VP                  V4      WP                  R R4      &    R# 	  R# )z
embedding.zembeddings.N)ÚpopÚreplace)rT   Ú
state_dictÚprefixÚargsÚks   &&&* r   rt  ÚMamba2Model.load_hookÝ  s4   € ÛˆAØ˜qÖ ØEOÇ^Á^ÐTUÓEV�
Ÿ9™9 \°=ÓAÑBÚó r   c                ó   € V P                   # rL   ©rn  ©rT   s   &r   Úget_input_embeddingsÚ Mamba2Model.get_input_embeddingsã  s   € Ø�‰Ðr   c                ó   € Wn         R # rL   r�  ©rT   Únew_embeddingss   &&r   Úset_input_embeddingsÚ Mamba2Model.set_input_embeddingsæ  s   € Ø(Žr   c                óø   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[R,          RS[R,          RS[ P                  R,          R	S[S[,          /# )
r   Ú	input_idsNÚinputs_embedsr  Ú	use_cacheÚoutput_hidden_statesÚreturn_dictrG   Úreturn)r   Ú
LongTensorr   r;   r   r    r\  )r   rr   s   "€r   r   ÚMamba2Model.__annotate__ê  s™   ø€ ÷ D
ñ D
á×#Ñ# dÕ*ðD
ñ ×'Ñ'¨$Õ.ðD
ñ ˜d•lð	D
ñ
 ˜$•;ðD
ñ # T�kðD
ñ ˜D•[ðD
ñ Ÿ™ tÕ+ðD
ñ 
‘Õ	ñD
r   c                ó|  € Ve   TMV P                   P                  pVe   TM)V P                  '       g   V P                   P                  MRpVe   TMV P                   P                  pVRJ VRJ,          '       d   \        R4      hVf   V P                  V4      pV P                  '       d   V P                  '       d   V'       d   RpV'       d   Vf   \        V P                   R7      p\        V P                   VVVR7      pTp	V'       d   R
MRp
V P                   F#  pV! V	3RVRV/VB p	V'       g   K  W©3,           p
K%  	  V P                  V	4      p	V'       d
   W©3,           p
V'       g1   \        ;QJ d    . R W“V
3 4       F  NK  	  5# ! R W“V
3 4       4      # \        T	V'       d	   VV
R	7      # RV
R	7      # )at  
cache_params (`Cache`, *optional*):
    If passed along, the model uses the previous state in all the blocks (which will give the output for the
    `input_ids` provided as if the model add `state_input_ids + input_ids` as context).
use_cache (`bool`, *optional*):
    If set to `True`, the `cache_params` is returned and can be used to quickly generate the next logits.
NFz:You must specify exactly one of input_ids or inputs_embeds)rä   )rä   rŒ  rG   Úpast_key_valuesr  rG   c              3   ó0   "  € T F  qf   K  Vx € K  	  R # 5irL   r£   )Ú.0Úvs   & r   Ú	<genexpr>Ú&Mamba2Model.forward.<locals>.<genexpr>(  s   é € ÐfÑ$T˜qŸšÓ$Tùs   ‚Œ
)r^  r  rF   r£   )rä   rŽ  r  r�  r�  Ú
ValueErrorrn  rq  r   r   r  rr  r    r\  )rT   r‹  rŒ  r  r�  rŽ  r�  rG   rŒ   rF   Úall_hidden_statesÚmixer_blocks   &&&&&&&&,   r   rd   ÚMamba2Model.forwardé  s¯  € ð( %9Ò$DÑ È$Ï+É+×JjÑJjð 	ð "+Ò!6‘IÐZ^×Zg×ZgÐZg¸T¿[¹[×=RÒ=RÐmrˆ	Ø%0Ò%<‘kÀ$Ç+Á+×BYÑBYˆà˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ Ø ŸO™O¨IÓ6ˆMà×&×&Ð&¨4¯=¯=¨=¿YØˆIç˜Ò-Ü'¨t¯{©{Ô;ˆLä8Ø—;‘;Ø'Ø)Ø(ô	
ˆð &ˆß"6™B¸DÐØŸ;œ;ˆKÙ'Øñà)ðð  .ðð ñ	ˆM÷ $Ñ#Ø$5Ð8HÕ$HÒ!ñ 'ð Ÿ™ MÓ2ˆçØ 1Ð4DÕ DÐçß”5Ñf ]ÐBSÑ$TÓf—5Ðf�5Ñf ]ÐBSÑ$TÓfÓfÐfäØ+ß)2˜Ø+ô
ð 	
à8<Ø+ô
ð 	
r   )rn  rq  r  rr  )NNNNNNN)rj   rk   rl   rm   rO   rt  rƒ  rˆ  r   rd   rn   ro   rp   rq   s   @@r   rj  rj  Ï  s;   ù‡ € õ
òòò)ð ÷D
ñ D
ó ÷D
ð D
r   rj  z�
    The MAMBA2 Model transformer with a language modeling head on top (linear layer with weights not tied to the input
    embeddings).
    c                   ó‚   a a€ ] tR tRt oRR/tV 3R ltR tR tRV3R lV 3R lllt]	RV3R	 lR
 ll4       t
RtVtV ;t# )ÚMamba2ForCausalLMi1  zlm_head.weightzbackbone.embeddings.weightc                óÊ   <€ \         SV `  V4       \        V4      V n        \        P
                  ! VP                  VP                  R R7      V n        V P                  4        R# )Frì   N)
rN   rO   rj  rE  r   rÿ   rU   rm  Úlm_headru  )rT   rä   rW   s   &&€r   rO   ÚMamba2ForCausalLM.__init__:  sF   ø€ Ü‰Ñ˜Ô Ü# FÓ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒà�‰Ör   c                ó6   € V P                   P                  4       # rL   )rE  rƒ  r‚  s   &r   rƒ  Ú&Mamba2ForCausalLM.get_input_embeddingsA  s   € Ø�}‰}×1Ñ1Ó3Ð3r   c                ó8   € V P                   P                  V4      # rL   )rE  rˆ  r†  s   &&r   rˆ  Ú&Mamba2ForCausalLM.set_input_embeddingsD  s   € Ø�}‰}×1Ñ1°.ÓAÐAr   c                ój   <€ V ^8„  d   QhRS[ R,          RS[P                  R,          RS[R,          /# )r   r  NrG   Úis_first_iteration)r   r   r   r;   )r   rr   s   "€r   r   ÚMamba2ForCausalLM.__annotate__G  s<   ø€ ÷ ñ ñ
 ˜d•lðñ Ÿ™ tÕ+ðñ ! 4�Kñr   c                óh   <€ \         S	V `  ! V3R VRVRVRVRV/VB pV'       d   V'       g   RVR&   V# )rŒ  r�  r  rG   r¨  N)rN   Úprepare_inputs_for_generation)
rT   r‹  rŒ  r�  r  rG   r¨  rŒ   Úmodel_inputsrW   s
   &&&&&&&, €r   r«  Ú/Mamba2ForCausalLM.prepare_inputs_for_generationG  sj   ø€ ô ‘wÒ<Øñ
à'ð
ð  ð
ð &ð	
ð
 *ð
ð  2ð
ð ñ
ˆ÷ ×/Ø-1ˆLÐ)Ñ*àÐr   c                óJ  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[R,          RS[R,          R	S[ P
                  R,          R
S[S[ P
                  ,          RS[S[,          /
# )r   r‹  NrŒ  r  ÚlabelsrŽ  r�  r�  rG   Úlogits_to_keepr�  )	r   r‘  r`  r   r;   r   r   r    rd  )r   rr   s   "€r   r   r©  a  s¸   ø€ ÷ 6
ñ 6
á×#Ñ# dÕ*ð6
ñ ×(Ñ(¨4Õ/ð6
ñ ˜d•lð	6
ñ
 × Ñ  4Õ'ð6
ñ # T�kð6
ñ ˜D•[ð6
ñ ˜$•;ð6
ñ Ÿ™ tÕ+ð6
ñ ™eŸl™lÕ*ð6
ñ 
Ñ%Õ	%ñ6
r   c
           
     ón  € Ve   TMV P                   P                  pV P                  VVVVVVVR7      pV^ ,          p\        V	\        4      '       d   \        V	) R4      MT	pV P                  VRVR3,          P                  V P                  P                  P                  4      4      P                  4       pRpVe-   V P                  ! R	RVRVRV P                   P                  /V
B pV'       g"   V3VR,          ,           pVe   V3V,           # T# \        VVVP                  VP                  R7      # )
aþ  
cache_params (`Cache`, *optional*):
    If passed along, the model uses the previous state in all the blocks (which will give the output for the
    `input_ids` provided as if the model add `state_input_ids + input_ids` as context).
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
    `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
    are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
use_cache (`bool`, *optional*):
    If set to `True`, the `cache_params` is returned and can be used to quickly generate the next logits.
N)r  rŒ  rŽ  r�  r�  rG   rD   rg  r¯  rm  :r|   NN)rf  rg  r  rF   r£   )rä   r�  rE  rI  r   Úslicer¡  rE   rQ   r1   r¡   Úloss_functionrm  rd  r  rF   )rT   r‹  rŒ  r  r¯  rŽ  r�  r�  rG   r°  rŒ   Úmamba2_outputsrF   Úslice_indicesrg  rf  rß   s   &&&&&&&&&&,      r   rd   ÚMamba2ForCausalLM.forward`  s2  € ð2 &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆàŸ™ØØ%Ø'Ø!5Ø#ØØ)ð 'ó 
ˆð ' qÕ)ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@×CÑCÀDÇLÁL×DWÑDW×D]ÑD]Ó^Ó_×eÑeÓgˆàˆØÒØ×%Ò%Ñp¨VÐp¸FÐpÈtÏ{É{×OeÑOeÐpÐioÑpˆDçØ�Y °Õ!3Õ3ˆFØ)-Ò)9�T�G˜fÕ$ÐE¸vÐEä#ØØØ'×4Ñ4Ø(×6Ñ6ô	
ð 	
r   )rE  r¡  )NNNNF)	NNNNNNNNrÆ   )rj   rk   rl   rm   Ú_tied_weights_keysrO   rƒ  rˆ  r«  r   rd   rn   ro   rp   rq   s   @@r   rŸ  rŸ  1  sK   ù‡ € ð +Ð,HÐIÐõò4òB÷õ ð2 ÷6
ñ 6
ó ÷6
ð 6
r   rŸ  r/  )NNFN)rŸ  rj  rD  )<r0  r  Údataclassesr   r   Útorch.nn.functionalr   r%   r�   Ú r   r  Úactivationsr   Úcache_utilsr   r   Ú
generationr   Úintegrationsr	   r
   Úintegrations.accelerater   Úmasking_utilsr   Úmodeling_layersr   Úmodeling_utilsr   Úutilsr   r   r   Úconfiguration_mamba2r   Ú
get_loggerrj   Úloggerr(   r-   rA   rH   ÚModulerJ   rs   r‰   r¡   r¤   r¾   rà   râ   r3  r9  rD  r\  rd  rj  rŸ  Ú__all__r£   r   r   Ú<module>rÉ     s€  ðñ ã Ý !ã ß Ð Ý å &Ý !ß .Ý )ß WÝ =Ý <Ý 9Ý -ß 9Ñ 9Ý .ð 
×	Ò	˜HÓ	%€õVò
ò(ò(	ô;˜Ÿ™Ÿ™ô ;ñ$ (Ð(>ÀÓPö'ó Qð'ñ& (Ð(:¸OÓLö'ó Mð'ñ, (Ð(JÈKÓXð +/Ø%(©%°«,Ð$7Ø %ØØ*.ØØ*.Ø(,ØØØ!ö%ó Yðñ0 (Ð(@À+ÓNö:'ó Oð:'ñz (Ð(CÀ[ÓQð "Ø#'Ø*.ØØ%(©%°«,Ð$7Ø %ö^ó Rð^ñB àØØ)Ø%Øðóôd%�"—)‘)ó d%óðd%ôN;�B—I‘Iô ;ô"Ð,ô ð4 ô">˜Oó ">ó ð">ñJ ðôð
 ô:�;ó :ó óð:ñ ðôð
 ô:˜;ó :ó óð:ð& ô^
Ð'ó ^
ó ð^
ñB ðôô`
Ð-¨ó `
óð`
òF H‚r   