+
    œ°³j{¯  ã                   óF  € ^ RI Ht ^ RIt^ RIHu Ht ^ RIHt ^RIHt	 ^RI
Ht ^RIHtHt ^RIHt ^RIHtHtHt ^R	IHt ^R
IHtHt ^RIHt ^RIHtHt ^RIH t H!t! ^RI"H#t#H$t$ ^RI%H&t& ^RI'H(t(H)t) ^RI*H+t+ ^RI,H-t-H.t.H/t/ ^RI0H1t1 ^RI2H3t3  ! R R]Ph                  4      t5R t6R R lt7RLR R llt8RMR lt9 ! R R]Ph                  4      t: ! R  R!]P                  Ph                  4      t;R" R# lt<R$ t=R% t>R& t?]! R'R(4      RNR) R* ll4       t@]! R+R(4      RNR, R- ll4       tA]! R.R/4      RR]B! R04      3R1R2RR3RRR^R43R5 R6 ll4       tC]! R7R/4      ROR8 R9 ll4       tD]! R:R/4      RRRR1R]B! R04      3R13R; R< ll4       tE]! ]A]@]C]D]E.4       ! R= R>]Ph                  4      4       tF ! R? R@]Ph                  4      tG]! RA4       ! RB RC]Ph                  4      4       tH ! RD RE]4      tI]( ! RF RG]$4      4       tJ]( ! RH RI]J4      4       tK]( ! RJ RK]J]4      4       tL. RPOtMR# )Qé    )ÚCallableN)Únn)Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_kernel_forward_from_hubÚ&use_kernel_func_from_hub_with_fallbackÚuse_kernelized_func)Úforce_accelerate_hooks)Úcreate_causal_maskÚcreate_recurrent_attention_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)Úauto_docstringÚcan_return_tuple)Údeprecate_kwarg)ÚTransformersKwargsÚmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputs©ÚBambaConfigc                   óÎ   a a€ ] tR t^3t o]! RRR7      R
V3R lV 3R lll4       t]]! RRR7      R
V3R lR ll4       4       t]P                  ! 4       ]
R 4       4       tR	tVtV ;t# )ÚBambaRotaryEmbeddingÚdevicez5.18)Úversionc                ó    <€ V ^8„  d   QhRS[ /# ©é   Úconfigr   )ÚformatÚ__classdict__s   "€Úi/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/bamba/modeling_bamba.pyÚ__annotate__Ú!BambaRotaryEmbedding.__annotate__5   s   ø€ ÷ Oñ O™{ñ Oó    c                óä  <€ \         SV `  4        VP                  V n        VP                  V n        Wn        V P
                  P                  R ,          V n        V P                  pV P                  R8w  d   \        V P                  ,          pV! V P
                  V4      w  q@n
        \        P                  ! VRR7      V n        \        P                  ! VP                  4       RR7      V n        R# )Ú	rope_typeÚdefaultF)Ú
persistentN)ÚsuperÚ__init__Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr(   Úrope_parametersr0   Úcompute_default_rope_parametersr   Úattention_scalingr   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)Úselfr(   r#   Úrope_init_fnr<   Ú	__class__s   &&&  €r+   r4   ÚBambaRotaryEmbedding.__init__4   s¬   ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àŒàŸ™×4Ñ4°[ÕAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÔ&Ü.¨t¯~©~Õ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÔ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÔ!NˆÖr.   c                óP   <€ V ^8„  d   QhRS[ RS[S[P                  S[3,          /# )r'   r(   Úreturn)r    ÚtupleÚtorchÚTensorÚfloat)r)   r*   s   "€r+   r,   r-   G   s.   ø€ ÷ 5ñ 5±ð 5ÑW\Ñ]b×]iÑ]iÑkpÐ]pÕWqñ 5r.   c           	     óŠ  € V P                   R,          p\        V RR4      ;'       g    V P                  V P                  ,          pV P                   P	                  RR4      p\        WE,          4      pRpRV\        P                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚhead_dimNÚpartial_rotary_factorg      ð?©Údtype)
r8   ÚgetattrÚhidden_sizeÚnum_attention_headsÚgetÚintrF   ÚarangerH   Úto)r(   r#   ÚkwargsÚbaseÚdimrL   Úattention_factorr<   s   &&,     r+   r9   Ú4BambaRotaryEmbedding.compute_default_rope_parametersE   s¦   € ð ×%Ñ% lÕ3ˆÜ�f˜j¨$Ó/×cÐc°6×3EÑ3EÈ×IcÑIcÕ3cˆØ &× 6Ñ 6× :Ñ :Ð;RÐTWÓ XÐÜ�#Õ-Ó.ˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r.   c                ó^  € V P                   R,          P                  VP                  ^ ,          R	^4      P                  \        P
                  VP                  R7      pVR
,          P                  4       p\        VP                  P                  \        4      '       d2   VP                  P                  R8w  d   VP                  P                  MRp\        VRR7      ;_uu_ 4        W4,          P                  ^^4      p\        P                  ! Wf3R	R7      pVP                  4       V P                  ,          pVP                  4       V P                  ,          p	R R R 4       XP                  VP                   R7      X	P                  VP                   R7      3#   + '       g   i     LH; i)N©rN   r#   ÚmpsÚcpuF)Údevice_typeÚenabled©rX   rM   )NºNNNNéÿÿÿÿ)rb   Nrb   )r<   ÚexpandÚshaperU   rF   rH   r#   Ú
isinstanceÚtypeÚstrr   Ú	transposeÚcatÚcosr:   ÚsinrN   )
r?   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedr_   ÚfreqsÚembrk   rl   s
   &&&       r+   ÚforwardÚBambaRotaryEmbedding.forward[   s2  € ð �M‰M˜-Õ(×/Ñ/°×0BÑ0BÀ1Õ0EÀrÈ1ÓM×PÑPÔW\×WbÑWbÐkl×ksÑksÐPÓtð 	ð !-¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆä¨¸U×CÖCØ&Õ>×IÑIÈ!ÈQÓOˆEÜ—)’)˜U˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆC÷	 Dð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷ D×Cús   Ã(A4FÆF,	)r:   r(   r<   r6   r>   r7   r0   ©N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r4   Ústaticmethodr9   rF   Úno_gradr   rs   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©rA   r*   s   @@r+   r"   r"   3   su   ù‡ € Ù�X vÔ.÷Oô Oó /ðOð  Ù�X vÔ.÷5ñ 5ó /ó ð5ð( ‡]‚]ƒ_Øñ<ó ó ÷<ð <r.   r"   c                óÌ   € V RRV P                   R,          ^,          13,          pV RV P                   R,          ^,          R13,          p\        P                  ! V) V3RR7      # )z*Rotates half the hidden dims of the input..Nra   rc   )re   rF   rj   )rm   Úx1Úx2s   &  r+   Úrotate_halfrƒ   n   sZ   € à	
ˆ3Ð"�!—'‘'˜"•+ Õ"Ð"Ð"Õ	#€BØ	
ˆ3�—‘˜•˜qÕ Ñ"Ð"Õ	#€BÜ�9Š9�r�c˜2�Y BÔ'Ð'r.   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r'   Úhidden_statesÚn_reprD   ©rF   rG   rS   )r)   s   "r+   r,   r,   u   s.   € ÷ 	Uñ 	UœUŸ\™\ð 	U´#ð 	U¼%¿,¹,ñ 	Ur.   c                ó˜   € V P                   w  r#rEV^8X  d   V # V R,          P                  W#WV4      p V P                  W#V,          WE4      # )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
)rb   rb   Nrb   rb   )re   rd   Úreshape)r…   r†   ÚbatchÚnum_key_value_headsÚslenrK   s   &&    r+   Ú	repeat_kvr�   u   sU   € ð
 2?×1DÑ1DÑ.€E Ø�„zØÐØ!Ð"2Õ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÕ(CÀTÓTÐTr.   ç        c                ó   € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R\        R	\
        \        ,          /# )
r'   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNÚscalingÚdropoutrV   )r   ÚModulerF   rG   rH   r   r   )r)   s   "r+   r,   r,   �   sw   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ð%ô ð%ô Ô'Õ(ñ%r.   c                 ó"  € \        W P                  4      p\        W0P                  4      p	\        P                  ! WP	                  ^^4      4      V,          p
Ve	   W¤,           p
\
        P                  P                  V
R\        P                  R7      P                  VP                  4      p
\
        P                  P                  W¦V P                  R7      p
\        P                  ! W©4      pVP	                  ^^4      P                  4       pWº3# )r'   )rX   rN   )ÚpÚtrainingrc   )r�   Únum_key_value_groupsrF   Úmatmulri   r   Ú
functionalÚsoftmaxÚfloat32rU   rN   r–   rš   Ú
contiguous)r�   r‘   r’   r“   r”   r•   r–   rV   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs   &&&&&&&,    r+   Úeager_attention_forwardr¥   �   sÒ   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÕL€LØÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r.   c                ó¶  € VP                  V4      pVP                  V4      pVP                  R,          pV RRV13,          V RVR13,          rvVRRV13,          VRVR13,          r˜Wb,          \        V4      V,          ,           p
W‚,          \        V4      V,          ,           p\        P                  ! W§.RR7      p
\        P                  ! W¹.RR7      pW«3# )a{  Applies Rotary Position Embedding to the query and key tensors.

Removes the interleaving of cos and sin from GLM

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
.Nra   rc   )Ú	unsqueezere   rƒ   rF   rj   )ÚqÚkrk   rl   Úunsqueeze_dimÚ
rotary_dimÚq_rotÚq_passÚk_rotÚk_passÚq_embedÚk_embeds   &&&&&       r+   Úapply_rotary_pos_embr²   ›   sÐ   € ð( �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€Cð —‘˜2•€JØ�c˜;˜J˜;Ð&Õ'¨¨3°
±Ð+;Õ)<ˆ6Ø�c˜;˜J˜;Ð&Õ'¨¨3°
±Ð+;Õ)<ˆ6ð �{œ{¨5Ó1°CÕ7Õ8€GØ�{œ{¨5Ó1°CÕ7Õ8€Gô �iŠi˜Ð)¨rÔ2€GÜ�iŠi˜Ð)¨rÔ2€GØÐÐr.   c                   óX   a a€ ] tR t^Át oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )ÚBambaAttentionz=Multi-headed attention from 'Attention Is All You Need' paperc                ó&   <€ V ^8„  d   QhRS[ RS[/# )r'   r(   Ú	layer_idx)r    rS   )r)   r*   s   "€r+   r,   ÚBambaAttention.__annotate__Ä   s   ø€ ÷ 
ñ 
™{ð 
±sñ 
r.   c                ó¢  <€ \         SV `  4        Wn        W n        \	        VR VP
                  VP                  ,          4      V n        VP                  VP                  ,          V n	        V P                  R,          V n
        VP                  V n        RV n        \        P                  ! VP
                  VP                  V P                  ,          VP                  R7      V n        \        P                  ! VP
                  VP                  V P                  ,          VP                  R7      V n        \        P                  ! VP
                  VP                  V P                  ,          VP                  R7      V n        \        P                  ! VP                  V P                  ,          VP
                  VP                  R7      V n        R# )rK   T©ÚbiasNg      à¿)r3   r4   r(   r¶   rO   rP   rQ   rK   r‹   r›   r•   Úattention_dropoutÚ	is_causalr   ÚLinearÚattention_biasÚq_projÚk_projÚv_projÚo_proj)r?   r(   r¶   rA   s   &&&€r+   r4   ÚBambaAttention.__init__Ä   sI  ø€ Ü‰ÑÔØŒØ"ŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÕ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Õ$\ˆÔ!Ø—}‘} dÕ*ˆŒØ!'×!9Ñ!9ˆÔØˆŒä—i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×&Ñ&¨¯©Õ6¸×8JÑ8JÐQW×QfÑQfô
ˆŽr.   c                ó  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          R,          RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  S[ P                  3,          /# )r'   r…   Úposition_embeddingsNr”   Úpast_key_valuesrV   rD   )rF   rG   rE   r   r   r   )r)   r*   s   "€r+   r,   r·   Û   s‡   ø€ ÷ &)ñ &)á—|‘|ð&)ñ #¡5§<¡<±·±Ð#=Õ>ÀÕEð&)ñ Ÿ™ tÕ+ð	&)ñ
  �ð&)ñ Ñ+Õ,ð&)ñ 
‰u�|‰|™UŸ\™\Ð)Õ	*ñ&)r.   c                ó  € VP                   R R p. VORNV P                  N5pV P                  V4      P                  V4      P	                  ^^4      pV P                  V4      P                  V4      P	                  ^^4      p	V P                  V4      P                  V4      P	                  ^^4      p
Vw  r¼\        W‰W¼4      w  r‰Ve   VP                  WšV P                  4      w  rš\        P                  ! V P                  P                  \        4      pT! V VV	V
V3RV P                  '       g   RMV P                   RV P"                  /VB w  rïVP$                  ! . VORN5!  P'                  4       pV P)                  V4      pWï3# )Nr–   rŽ   r•   rc   )re   rK   r¿   Úviewri   rÀ   rÁ   r²   Úupdater¶   r   Úget_interfacer(   Ú_attn_implementationr¥   rš   r»   r•   r‰   r    rÂ   )r?   r…   rÅ   r”   rÆ   rV   Úinput_shapeÚhidden_shapeÚquery_statesr¡   r¢   rk   rl   Úattention_interfacer¤   r£   s   &&&&&,          r+   rs   ÚBambaAttention.forwardÛ   s‡  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—k‘k +Ó.ˆØÐ(Ð(r.   )r»   r(   rK   r¼   rÀ   r¶   r›   rÂ   r¿   r•   rÁ   )NNN)
rv   rw   rx   ry   Ú__doc__r4   rs   r|   r}   r~   r   s   @@r+   r´   r´   Á   s!   ù‡ € ÙG÷
ó 
÷.&)÷ &)ò &)r.   r´   c                   ó@   a a€ ] tR tRt oRV 3R lltRR ltRtVtV ;t# )ÚBambaRMSNormGatedi  c                óŽ   <€ \         SV `  4        \        P                  ! \        P
                  ! V4      4      V n        W n        R # ru   ©r3   r4   r   Ú	ParameterrF   ÚonesÚweightÚvariance_epsilon©r?   rP   ÚepsrA   s   &&&€r+   r4   ÚBambaRMSNormGated.__init__  s-   ø€ Ü‰ÑÔÜ—l’l¤5§:¢:¨kÓ#:Ó;ˆŒØ #Ör.   c                óÈ  € VP                   pVP                  \        P                  4      pVeD   V\        P
                  P                  VP                  \        P                  4      4      ,          pVP                  ^4      P                  RRR7      pV\        P                  ! W@P                  ,           4      ,          pV P                  VP                  V4      ,          # )NT©Úkeepdimrc   )rN   rU   rF   rŸ   r   r�   ÚsiluÚpowÚmeanÚrsqrtrÙ   rØ   )r?   r…   ÚgateÚinput_dtypeÚvariances   &&&  r+   rs   ÚBambaRMSNormGated.forward
  s£   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆàÒØ)¬B¯M©M×,>Ñ,>¸t¿w¹wÄuÇ}Á}Ó?UÓ,VÕVˆMØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯ª°H×?TÑ?TÕ4TÓ(UÕUˆà�{‰{˜]×-Ñ-¨kÓ:Õ:Ð:r.   ©rÙ   rØ   ©ç�íµ ÷Æ°>ru   ©	rv   rw   rx   ry   r4   rs   r|   r}   r~   r   s   @@r+   rÓ   rÓ     s   ù‡ € ÷$÷
	;ô 	;r.   rÓ   c                óD   € V ^8„  d   QhR\         P                  R\        /# )r'   Úinput_tensorÚpad_sizer‡   )r)   s   "r+   r,   r,     s"   € ÷ Vñ V¤U§\¡\ð V¼Sñ Vr.   c                ó²   € \        V P                  4      ^8X  d   ^ ^ ^ ^ ^ V^ ^ 3M^ ^ ^ V^ ^ 3p\        P                  P                  P                  WR^ R7      # )zv
Padding x tensor with `pad_size` on the seq_len dim (dim=1)

Assumes that we only have tensors of either size 4 or 3
Úconstant)Úmoder“   )Úlenre   rF   r   r�   Úpad)rí   rî   Ú	pad_shapes   && r+   Úpad_tensor_by_sizerõ     sd   € ô 47°|×7IÑ7IÓ3JÈaÔ3O��A�q˜!˜Q ¨!¨QÑ/ÐVWÐYZÐ\]Ð_gÐijÐlmÐUn€Iä�8‰8×Ñ×"Ñ" <ÀÐSTÐ"ÓUÐUr.   c                óD  € \        W4      p \        V P                  4      ^8X  d6   V P                  V P                  ^ ,          RW P                  ^,          4      # V P                  V P                  ^ ,          RW P                  ^,          V P                  ^,          4      # )z°
Padding input_tensor with `pad_size` on the seq_len dim (dim=1) and
simultaneously splitting it into chunk sequences.

Assumes that we only have tensors of either size 4 or 3
rc   )rõ   rò   re   r‰   )rí   rî   Ú
chunk_sizes   &&&r+   Úreshape_into_chunksrø   $  s�   € ô & lÓ=€Lä
ˆ<×ÑÓ !Ô#à×#Ñ# L×$6Ñ$6°qÕ$9¸2¸z×K]ÑK]Ð^_ÕK`ÓaÐað ×#Ñ#Ø×Ñ˜qÕ! 2 z×3EÑ3EÀaÕ3HÈ,×J\ÑJ\Ð]^ÕJ_ó
ð 	
r.   c           	     ó2  € V P                  R4      pV R,          P                  ! . V P                  4       OVN5!  p \        P                  ! \        P                  ! WV P
                  \        P                  R7      RR7      pV P                  V( ^ 4      p \        P                  ! V RR7      p\        P                  ! \        P                  ! WV P
                  \        P                  R7      ^ R7      pVP                  V( \        P                  ) 4      pV# )zg
More stable segment sum calculation. Uses cumulative sums and masking instead of direct subtractions.
©r#   rN   )Údiagonalra   rc   ©.Néþÿÿÿ)
Úsizerd   rF   Útrilr×   r#   ÚboolÚmasked_fillÚcumsumÚinf)rí   r÷   ÚmaskÚtensor_segsums   &   r+   Úsegment_sumr  8  sØ   € ð ×"Ñ" 2Ó&€Jð   	Õ*×1Ò1ÐS°<×3DÑ3DÓ3FÐSÈ
ÓS€Lä�:Š:”e—j’j À×@SÑ@SÔ[`×[eÑ[eÔfÐqsÔt€DØ×+Ñ+¨T¨E°1Ó5€Lä—L’L °2Ô6€Mô �:Š:”e—j’j À×@SÑ@SÔ[`×[eÑ[eÔfÐqrÔs€DØ!×-Ñ-¨t¨e´e·i±i°ZÓ@€MØÐr.   c                ób   € Ve+   V P                   pWR,          ,          P                  V4      p V # )ze
Tunes out the hidden states for padding tokens, see https://github.com/state-spaces/mamba/issues/66
©rb   rb   N)rN   rU   )r…   r”   rN   s   && r+   Úapply_mask_to_padding_statesr	  L  s3   € ð
 Ò!Ø×#Ñ#ˆØ&¸
Õ)CÕC×GÑGÈÓNˆàÐr.   Úcausal_conv1d_updateÚcausal_conv1dc                óÀ   € V ^8„  d   QhR\         P                  R\         P                  R\        P                  R\        P                  R,          R\        R,          /# )r'   r…   Ú
conv_staterØ   rº   NÚ
activation©rF   rG   r   rÖ   rh   )r)   s   "r+   r,   r,   Y  sS   € ÷ 'ñ 'Ü—<‘<ð'ä—‘ð'ô �L‰Lð'ô �,‰,˜Õ
ð	'ô
 �d•
ñ'r.   c                 óº  € V P                   w  rVpVP                   R,          p\        P                  ! W.RR7      P                  VP                  4      p	VP                  V	RRV) R13,          4       \        P                  ! W’P                  ^4      V^ VR7      p
V
RRV) R13,          p
Ve   \        V,          ! V
4      p
V
P                  V P                  4      # )é   ra   rb   N)ÚpaddingÚgroupsrc   )
re   rF   rj   rU   rN   Úcopy_ÚFÚconv1dr§   r   )r…   r  rØ   rº   r  Ú_rP   Úseq_lenÚ	state_lenÚhidden_states_newÚouts   &&&&&      r+   r
  r
  X  sÅ   € ð ,×1Ñ1Ñ€A�GØ× Ñ  Õ$€IäŸ	š	 :Ð"=À2ÔF×IÑIÈ&Ï,É,ÓWÐØ×ÑÐ& q¨!¨i¨Z©[Ð'8Õ9Ô:Ü
�(Š(Ð$×&6Ñ&6°qÓ&9¸4ÈÐS^Ô
_€CØ
ˆa��W�H‘IˆoÕ
€CØÒÜ�ZÖ  Ó%ˆØ�6‰6�-×%Ñ%Ó&Ð&r.   Úcausal_conv1d_fnc          	      ó    € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R,          R\        R,          /# )r'   r…   rØ   rº   Nr  r  )r)   s   "r+   r,   r,   m  sE   € ÷ 'ñ 'Ü—<‘<ð'ä�L‰Lð'ô �,‰,˜Õ
ð'ô �d•
ñ	'r.   c                 óV  € V P                   w  rVpVP                   R,          ^,
          p\        P                  ! V P                  VP                  4      VP                  ^4      VVVR7      RRRV13,          p	Ve   \        V,          ! V	4      p	V	P                  V P                  4      # )r  )rØ   rº   r  r  rb   Nrc   )re   r  r  rU   rN   r§   r   )
r…   rØ   rº   r  rV   r  rP   r  r  r  s
   &&&&,     r+   r  r  l  sž   € ð ,×1Ñ1Ñ€A�GØ�l‰l˜2Õ Õ"€Gä
�(Š(Ø×Ñ˜Ÿ™Ó&Ø×Ñ Ó"ØØØôð ˆˆHˆWˆH€nõ€Cð ÒÜ�ZÖ  Ó%ˆØ�6‰6�-×%Ñ%Ó&Ð&r.   Ú mamba_split_conv1d_scan_combinedÚ	mamba_ssmr  Frà   rê   Tc          $      ó.  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R,          R\         P                  R\         P                  R\         P                  R\        R	\         P                  R,          R
\        \        \        3,          R\
        R\        R\         P                  R,          R\        R\         P                  R,          R\         P                  R,          R\        R,          R\        R\
        /# )r'   ÚzxbcdtÚconv1d_weightÚconv1d_biasNÚdt_biasÚAÚDr÷   Úinitial_statesÚdt_limitÚreturn_final_statesr  Úrmsnorm_weightÚrmsnorm_epsÚoutproj_weightÚoutproj_biasÚheaddimÚngroupsÚnorm_before_gate)rF   rG   rS   rE   rH   r   rh   )r)   s   "r+   r,   r,   „  s
  € ÷ ñ Ü�L‰Lðä—<‘<ðô —‘ Õ$ðô �\‰\ð	ô
 ‡|�|ðô ‡|�|ðô ðô —L‘L 4Õ'ðô ”Eœ5�LÕ!ðô ðô ðô —L‘L 4Õ'ðô ðô —L‘L 4Õ'ðô —,‘, Õ%ðô  �4�Zð!ô" ð#ô$ ñ%r.   c                 ó   € R # ru   © )r"  r#  r$  r%  r&  r'  r÷   r(  r)  r*  r  r+  r,  r-  r.  r/  r0  r1  rV   s   &&&&&&&&&&&&&&&&&&,r+   Ú!mamba2_split_conv1d_scan_combinedr4  ƒ  s   € ñ, r.   Úselective_state_updatec                ón  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R,          R	\         P                  R,          R
\        R\         P                  R,          /
# )r'   Ústater…   Údtr&  ÚBÚCr'  Nr%  Údt_softplusÚz)rF   rG   r   )r)   s   "r+   r,   r,   �  sŸ   € ÷ :'ñ :'Ü�<‰<ð:'ä—<‘<ð:'ô 	�‰ð:'ô ‡|�|ð	:'ô
 ‡|�|ð:'ô ‡|�|ð:'ô ‡|�|�dÕð:'ô �\‰\˜DÕ ð:'ô ð:'ô ‡|�|�dÕñ:'r.   c
                 ó†  € VP                   w  r¼pVP                   ^,          pVP                   R,          pVe"   W'P                  VP                  4      ,           pV'       d   \        P                  ! V4      pVR,          p\
        P                  ! VP                  4       VP                  4       ,          4      P                  V P                  R7      pVP                  W¾^V4      pVP                  W¾WÎ,          V4      P                  4       pVP                  W¼^V4      pW$,          pVVR,          ,          P                  V P                  R7      pV V,          V,           pV P                  VP                  V P                  4      4       VP                  W¾^V4      pVP                  W¾WÎ,          V4      P                  4       pVP                  W¼V4      pVP                  VP                  VP                  R7      pVP                  W¼,          Wß4      pVP                  W¼,          V^4      p\
        P                  ! VV4      pVP                  W¼V4      pVe)   VW,          ,           P                  VP                  4      pV	e   V\        P                  ! V	4      ,          pVP                  VP                  4      # )r  ©r#   rú   rc   rü   )re   rU   rN   r  ÚsoftplusrF   ÚexprH   r#   r‰   rd   r    r  rÈ   Úbmmrà   )r7  r…   r8  r&  r9  r:  r'  r%  r;  r<  rV   Ú
batch_sizeÚ	num_headsrK   Ú
num_groupsÚ
state_sizeÚdAÚdBÚdBxÚ
ssm_statesÚssm_states_reshapedÚ
C_reshapedr  s   &&&&&&&&&&,            r+   Úmamba2_selective_state_updaterL  œ  s"  € ð '4×&9Ñ&9Ñ#€J˜8Ø—‘˜•€JØ—‘˜•€JàÒØ—*‘*˜RŸX™XÓ&Õ&ˆßÜ�ZŠZ˜‹^ˆØ	ˆI�€Bô 
�Š�2—8‘8“: §¡£	Õ)Ó	*×	-Ñ	-°U·\±\Ð	-Ó	B€Bð 	
�	‰	�*¨!¨ZÓ8€AØ	�‰�¨Õ)@À*ÓM×XÑXÓZ€AØ	�	‰	�*¨¨JÓ7€AØ	�€Bð �˜iÕ(Õ(×
,Ñ
,°E·L±LÐ
,Ó
A€Cð ˜•˜cÕ!€JØ	‡K�K�
—‘˜eŸk™kÓ*Ô+ð 	
�	‰	�*¨!¨ZÓ8€AØ	�‰�¨Õ)@À*ÓM×XÑXÓZ€AØ	�	‰	�*¨Ó4€Að —‘ a§h¡h°a·g±g�Ó>€JØ$Ÿ/™/¨*Õ*@À(ÓWÐØ—‘˜
Õ.°
¸AÓ>€JÜ
�)Š)Ð'¨Ó
4€CØ
�(‰(�:¨(Ó
3€Cð 	‚}Ø�]Õ&Õ&×*Ñ*¨3¯9©9Ó5ˆà‚}Ø”A—F’F˜1“I�oˆà�6‰6�-×%Ñ%Ó&Ð&r.   Úmamba_chunk_scan_combinedc                ó”  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R\        R\         P                  R,          R	\         P                  R,          R
\         P                  R,          R\        R\        \
        \
        3,          R\        /# )r'   r…   r8  r&  r9  r:  r÷   r'  Nr%  r(  r;  r)  r*  )rF   rG   rS   r   rE   rH   )r)   s   "r+   r,   r,   Û  sÆ   € ÷ ^ñ ^Ü—<‘<ð^ä�‰ð^ô ‡|�|ð^ô ‡|�|ð	^ô
 ‡|�|ð^ô ð^ô ‡|�|�dÕð^ô �\‰\˜DÕ ð^ô —L‘L 4Õ'ð^ô ð^ô ”Eœ5�LÕ!ð^ô ñ^r.   c                 óô  € V P                   w  rÞppVP                   ^,          pVe"   WP                  VP                  4      ,           pV	'       d   \        P                  ! V4      p\
        P                  ! W^ ,          V
^,          R7      pV P                  4       p VP                  4       P                  VV,          ^VR7      pVP                  4       P                  VV,          ^VR7      pW^V,          ,
          V,          pRpVe   VR,          \        V V4      ,          pWR,          P                  4       ,          p VP                  V P                  4      VP                  4       ,          pWW43 Uu. uF  p\        VVV4      NK  	  upw  rr4VP                  ^ ^^^4      p\
        P                  ! VRR7      p\
        P                  ! \        V4      4      pVR	,          VR
,          ,          P                  RR7      pVR,          VP                  ^ ^^^^4      R,          ,          P                  RR7      pVR,          V R,          ,          P                  ^R7      p\
        P                  ! VRRRRR13,          V,
          4      pVVP                  ^ RR^4      R,          ,          pVR,          V R,          ,          P                  ^R7      pVe/   VR,          P                  VP                  VP                   R7      M\
        P"                  ! VR,          4      p\
        P$                  ! VV.^R7      p\
        P                  ! \        \        P&                  ! VR,          R4      4      4      P)                  ^^4      pVR,          VR,          ,          P                  ^R7      pVRRR13,          VR,          p p\
        P                  ! V4      p!VR,          VR,          ,          p"V"P                  R4      V!P                  ^ ^^^4      R,          ,          p#VV#,           p$V$P+                  VRVV4      p$Ve
   V$V,           p$V^ 8”  d   V$RRV13,          p$V'       d   V$V 3# V$# u upi )r'   N)ÚminÚmax)rX   Úoutput_sizera   rb   r\   rü   rc   )rb   rb   rb   Nrb   rb   )rb   rb   Nrb   rb   rb   r  rý   ).Nrb   ©rb   N)rb   :Nr  N)rb   rb   rb   rc   )r  r   ).NN)rb   rb   N.)rb   rc   )re   rU   rN   r  r?  rF   ÚclamprH   Úrepeat_interleaverõ   rø   Úpermuter  r@  r  Úsumr#   Ú
zeros_likerj   ró   ri   r‰   )%r…   r8  r&  r9  r:  r÷   r'  r%  r(  r;  r)  r*  rV   rB  Úsequence_lengthrC  rK   rD  rî   Ú
D_residualÚtensorÚA_cumsumÚLÚGÚMÚY_diagÚdecay_statesÚB_decayÚstatesÚprevious_statesÚdecay_chunkÚ
new_statesÚfinal_stateÚstate_decay_outÚC_times_statesÚY_offÚoutputs%   &&&&&&&&&&&&,                        r+   Úmamba2_chunk_scanrl  Ú  së  € ð  8E×7JÑ7JÑ4€J ¨HØ—‘˜•€JàÒØ—*‘*˜RŸX™XÓ&Õ&ˆßÜ�ZŠZ˜‹^ˆÜ	�Š�R a�[¨h°q­kÔ	:€Bà!×'Ñ'Ó)€MØ	�‰‹	×#Ñ# I°Õ$;ÀÐPYÐ#ÓZ€AØ	�‰‹	×#Ñ# I°Õ$;ÀÐPYÐ#ÓZ€Aà¨zÕ9Õ9¸ZÕG€HØ€JØ‚}Ø�y•\Ô$6°}ÀhÓ$OÕOˆ
ð " y¥M×$7Ñ$7Ó$9Õ9€MØ	�‰ˆ]× Ñ Ó! B§H¡H£JÕ.€Að `mÐqrÑ^vÓwÑ^vÐTZÔ1°&¸(ÀJÖOÑ^vÑwÑ€M�aà	�	‰	�!�Q˜˜1Ó€AÜ�|Š|˜A 2Ô&€Hô 	�	Š	”+˜a“.Ó!€Að 
ÐÕ	 !Ð$7Õ"8Õ	8×=Ñ=À"Ð=ÓE€Að 
ˆ9�˜Ÿ	™	 ! Q¨¨1¨aÓ0°Õ;Õ	;×@Ñ@ÀRÐ@ÓH€Að �	�l˜]¨:Õ6Õ6×;Ñ;ÀÐ;ÓB€Fô —9’9˜X a¨¨A¨r©s lÕ3°hÕ>Ó?€LØ�,×&Ñ& q¨"¨b°!Ó4°YÕ?Õ?€GØ�lÕ# m°IÕ&>Õ>×CÑCÈÐCÓJ€Fð Ò%ð 	�wÕ×"Ñ"¨¯©¸f¿m¹mÐ"ÔLä×Ò˜f U�mÓ,ð ô
 �YŠY˜¨Ð0°aÔ8€FÜ—)’)œK¬¯ª¨h°{Õ.CÀVÓ(LÓMÓN×XÑXÐYZÐ\]Ó^€KØ˜oÕ.°¸Õ1HÕH×MÑMÐRSÐMÓT€JØ$ Q¨¨¨ VÕ,¨j¸Õ.?ˆK€Fô —i’i Ó)€OØ�|•_ v¨oÕ'>Õ>€NØ×Ñ˜rÓ" _×%<Ñ%<¸QÀÀ1ÀaÓ%HÈÕ%SÕS€Eð �e�^€FØ�^‰^˜J¨¨I°xÓ@€FàÒØ˜*Õ$ˆð �!„|Ø˜Ð+˜OÐ+Ð+Õ,ˆçØ�{Ð"Ð"à€Mùòq xs   Å8Q5c                   ó    a a€ ] tR tRt oRtR
V3R lV 3R lllt]P                  ! 4       R 4       t]	! R4      RV3R lR ll4       t
R	tVtV ;t# )Ú
BambaMixeri<  uK  
Compute âˆ†, A, B, C, and D the state space parameters and compute the `contextualized_states`.
A, D are input independent (see Mamba paper [1] Section 3.5.2 "Interpretation of A" for why A isn't selective)
âˆ†, B, C are input-dependent (this is a key difference between Mamba and the linear time invariant S4,
and is why Mamba is called **selective** state spaces)

The are a few differences between this and Mamba2Mixer:
- The variable use_precomputed_states is slightly different due to the hybrid cache structure
- Some extra variables that our layer doesn't need have been removed
c                ó,   <€ V ^8„  d   QhRS[ RS[RS[/# )r'   r(   r¶   Úinitialize_mixer_weights)r    rS   r   )r)   r*   s   "€r+   r,   ÚBambaMixer.__annotate__Q  s#   ø€ ÷ 08ñ 08™{ð 08±sð 08ÑVZñ 08r.   c           	     ó  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  V n        VP                  V n        VP                  V n
        \        VP                  V P                  ,          4      V n        W n        VP                  V n        VP"                  V n        \&        VP"                  ,          V n        VP*                  V n        VP.                  V n        VP2                  V n        VP6                  V n        VP:                  V n        V P                  ^V P0                  ,          V P                  ,          ,           V n        \>        P@                  ! V P<                  V P<                  VP                  V P                  V P<                  V P                  ^,
          R7      V n!        V P                  V P<                  ,           V P
                  ,           p\>        PD                  ! V P                  VV P                  R7      V n#        \>        PH                  ! \J        PL                  ! V P
                  4      4      V n'        \>        PH                  ! \J        PL                  ! V P
                  4      4      V n(        \S        V P                  V P,                  R7      V n*        \>        PH                  ! \J        PL                  ! V P
                  4      4      V n+        V'       d6   V PN                  PX                  PZ                  R8w  d   V P]                  4        \>        PD                  ! V P                  V P                  V P                  R7      V n/        VP`                  V,          V n1        R# )r'   )Úin_channelsÚout_channelsrº   Úkernel_sizer  r  r¹   ©rÛ   ÚmetaN)2r3   r4   Úmamba_proj_biasÚuse_biasÚmamba_n_headsrC  rP   Úmamba_d_stateÚssm_state_sizeÚmamba_d_convÚconv_kernel_sizerS   Úmamba_expandÚintermediate_sizer¶   Úmamba_conv_biasÚuse_conv_biasÚ
hidden_actr  r   ÚactÚrms_norm_epsÚlayer_norm_epsilonÚmamba_n_groupsÚn_groupsÚmamba_d_headrK   Úmamba_chunk_sizer÷   Útime_step_limitÚconv_dimr   ÚConv1dr  r½   Úin_projrÖ   rF   Úemptyr%  ÚA_logrÓ   Únormr'  r#   rg   Úinit_bamba_weightsÚout_projÚlayer_typesÚ
layer_type)r?   r(   r¶   rp  Úprojection_sizerA   s   &&&& €r+   r4   ÚBambaMixer.__init__Q  sf  ø€ Ü‰ÑÔØ×.Ñ.ˆŒØ×-Ñ-ˆŒØ!×-Ñ-ˆÔØ$×2Ñ2ˆÔØ &× 3Ñ 3ˆÔÜ!$ V×%8Ñ%8¸4×;KÑ;KÕ%KÓ!LˆÔØ"ŒØ#×3Ñ3ˆÔØ ×+Ñ+ˆŒÜ˜&×+Ñ+Õ,ˆŒØ"(×"5Ñ"5ˆÔØ×-Ñ-ˆŒØ×+Ñ+ˆŒØ ×1Ñ1ˆŒà%×5Ñ5ˆÔà×.Ñ.°°T·]±]Õ1BÀT×EXÑEXÕ1XÕXˆŒÜ—i’iØŸ™ØŸ™Ø×'Ñ'Ø×-Ñ-Ø—=‘=Ø×)Ñ)¨AÕ-ô
ˆŒð ×0Ñ0°4·=±=Õ@À4Ç>Á>ÕQˆÜ—y’yØ×ÑØØ—‘ô
ˆŒô —|’|¤E§K¢K°·±Ó$?Ó@ˆŒô —\’\¤%§+¢+¨d¯n©nÓ"=Ó>ˆŒ
Ü% d×&<Ñ&<À$×BYÑBYÔZˆŒ	Ü—’œeŸkšk¨$¯.©.Ó9Ó:ˆŒß#¨¯©×(;Ñ(;×(@Ñ(@ÀFÔ(JØ×#Ñ#Ô%ÜŸ	š	 $×"8Ñ"8¸$×:JÑ:JÐQU×Q^ÑQ^Ô_ˆŒà ×,Ñ,¨YÕ7ˆŽr.   c                óŠ  € \         P                  ! ^V P                  ^,           V P                  P                  \         P
                  R7      p\        P                  ! V P                  \         P                  ! V4      4       \        P                  ! V P                  4       \        P                  ! V P                  4       R# )r  rú   N)rF   rT   rC  r�  r#   rŸ   Úinitr  ÚlogÚones_r'  r%  )r?   r&  s   & r+   r’  ÚBambaMixer.init_bamba_weightsƒ  sj   € ä�LŠL˜˜DŸN™N¨QÕ.°t·z±z×7HÑ7HÔPU×P]ÑP]Ô^ˆÜ�
Š
�4—:‘:œuŸyšy¨›|Ô,Ü�
Š
�4—6‘6ÔÜ�
Š
�4—<‘<Ö r.   r  c                óp   <€ V ^8„  d   QhRS[ P                  RS[R,          RS[ P                  R,          /# )r'   r…   Úcache_paramsNr”   )rF   rG   r   )r)   r*   s   "€r+   r,   rq  ‹  s?   ø€ ÷ P%ñ P%á—|‘|ðP%ñ ˜d•lðP%ñ Ÿ™ tÕ+ñ	P%r.   c                ó  € VP                   w  rVpVP                  pVR J;'       d    VP                  V P                  4      p	\	        W4      pV P                  V4      p
\        P                  ! V P                  P                  4       4      ) pVRV P                  /,          pV P                  '       dô   Vfð   \        V
V P                  P                  P                  ^4      V P                  P                   V P"                  V3RV P$                  RV P&                  RV P(                  RV P*                  P                  RV P*                  P,                  RV P.                  P                  RV P.                  P                   R	V P0                  R
V P2                  RRRR/VB pVe   V# V
P5                  V P6                  V P8                  V P:                  .RR7      w  rïpV	'       d]   VP<                  V P                  ,          P>                  ^ ,          pVP<                  V P                  ,          P@                  ^ ,          pVPC                  ^^4      pV	'       d‡   V^8X  d€   VP<                  V P                  ,          PD                  '       gS   \G        VXV P                  P                  P                  ^4      V P                  P                   V P(                  R7      pM�Ve)   VPI                  VV P                  V PJ                  R7      p\M        VV P                  P                  P                  ^4      V P                  P                   3RV P(                  /VB pVe   VRRV) R 13,          p\	        VPC                  ^^4      V4      p\        P4                  ! VV P6                  V P2                  V PN                  ,          V P2                  V PN                  ,          .RR7      w  pppV	'       Ed]   V^8X  EdU   VPQ                  WPP:                  V P0                  4      pVPC                  ^^4      PS                  RRV P0                  4      pVR,          PS                  RV P0                  V PN                  4      pVPQ                  WPP2                  V PN                  4      pVPQ                  WPP2                  V PN                  4      pV P$                  R,          PS                  RV P0                  4      pV P"                  R,          PS                  RV P0                  4      p\U        XVVVVVV3RR RVRR/VB pVPQ                  V^R4      pEMVR Jp\W        VPQ                  WVV P:                  V P0                  4      VVVPQ                  WVV P2                  V PN                  4      VPQ                  WVV P2                  V PN                  4      3RV P&                  RV P$                  RR RTRV P"                  RRRV	'       d   XMR RV P                  /VB pV'       d$   Vw  ppVPY                  VV P                  R7       MTpVP[                  WVR4      pV P+                  VV4      pV P/                  VP]                  V4      4      pV# )Nr)  r'  r÷   r  r+  r,  r-  r.  r/  r0  r1  Fr*  ra   )r  )r~  rb   r<  r%  r;  Tr(  )r¶   rc   )rb   NNrS  )/re   rN   Úhas_previous_stater¶   r	  rŽ  rF   r@  r�  rH   r‹  rš   r4  r  rØ   Úsqueezerº   r%  r'  r÷   r  r‘  rÙ   r“  rK   rˆ  Úsplitr€  rŒ  rC  ÚlayersÚconv_statesÚrecurrent_statesri   Úrecord_pastr
  Úupdate_conv_stater~  r  r|  rÈ   rd   rL  rl  Úupdate_recurrent_stater‰   rU   )r?   r…   rž  r”   rV   rB  r  r  rN   Úuse_precomputed_statesÚprojected_statesr&  Úfused_kwargsÚfused_outputrä   Úhidden_states_B_Cr8  r  Úrecurrent_stater9  r:  r'  r%  Úscan_outputÚoutput_final_stateÚscan_resultrg  Úcontextualized_statess   &&&&,                       r+   rs   ÚBambaMixer.forwardŠ  s  € ð "/×!4Ñ!4Ñˆ
˜QØ×#Ñ#ˆØ!-°TÐ!9×!mÐ!m¸l×>]Ñ>]Ð^b×^lÑ^lÓ>mÐô 5°]ÓSˆØŸ<™<¨Ó6Ðä�YŠY�t—z‘z×'Ñ'Ó)Ó*Ð*ˆØ ¨T×-AÑ-AÐ BÕBˆØ�=�=ˆ=˜\Ò1Ü<Ø Ø—‘×"Ñ"×*Ñ*¨1Ó-Ø—‘× Ñ Ø—‘Øñð —&‘&ðð  Ÿ?™?ðð  Ÿ?™?ðð  $Ÿy™y×/Ñ/ðð !ŸI™I×6Ñ6ðð  $Ÿ}™}×3Ñ3ðð "Ÿ]™]×/Ñ/ðð Ÿ™ðð Ÿ™ðð "'ðð  %*ð!ð" ñ#ˆLð* Ò'Ø#Ð#à&6×&<Ñ&<Ø×#Ñ# T§]¡]°D·N±NÐCÈð '=ó '
Ñ#ˆ ÷ "Ø%×,Ñ,¨T¯^©^Õ<×HÑHÈÕKˆJØ*×1Ñ1°$·.±.ÕA×RÑRÐSTÕUˆOð .×7Ñ7¸¸1Ó=Ðß! g°¤l¸<×;NÑ;NÈtÏ~É~Õ;^×;j×;jÐ;jÜ 4Ø!ØØ—‘×"Ñ"×*Ñ*¨1Ó-Ø—‘× Ñ ØŸ?™?ô!Ñð Ò'Ø$0×$BÑ$BØ%Ø—N‘NØ%)×%:Ñ%:ð %Có %Ð!ô !1Ø!Ø—‘×"Ñ"×*Ñ*¨1Ó-Ø—‘× Ñ ñ!ð  Ÿ?™?ð	!ð
 ñ!Ðð Ò'Ø$5°a¸¸W¸H¹I°oÕ$FÐ!ô 9Ð9J×9TÑ9TÐUVÐXYÓ9ZÐ\jÓkÐÜ#ŸkškØØ×#Ñ# T§]¡]°T×5HÑ5HÕ%HÈ$Ï-É-ÐZ^×ZmÑZmÕJmÐnØô
Ñˆ�q˜!÷ "Ð! g°¥lØ)×.Ñ.¨z¿>¹>È4Ï=É=ÓYˆMØ—‘˜a Ó#×*Ñ*¨2¨r°4·=±=ÓAˆBØ�-Õ ×'Ñ'¨¨D¯M©M¸4×;NÑ;NÓOˆAØ—‘�z§=¡=°$×2EÑ2EÓFˆAØ—‘�z§=¡=°$×2EÑ2EÓFˆAØ—‘�w•×&Ñ& r¨4¯=©=Ó9ˆAØ—l‘l 7Õ+×2Ñ2°2°t·}±}ÓEˆGä7ØØØØØØØñð ðð  ðð !ðð ñˆKð &×*Ñ*¨:°q¸"Ó=ŠKð ".°TÐ!9ÐÜ+Ø×"Ñ" :¸¿¹ÈÏÉÓVØØØ—‘�z¨D¯M©M¸4×;NÑ;NÓOØ—‘�z¨D¯M©M¸4×;NÑ;NÓOñð  Ÿ?™?ðð —&‘&ðð ðð %7ðð Ÿ™ðð !ð÷ 3I™Èdðð ×-Ñ-ðð ñˆK÷" "Ø+6Ñ(�˜[Ø×3Ñ3°KÈ4Ï>É>Ð3ÕZà)�à%×-Ñ-¨jÀ2ÓFˆKà—i‘i ¨TÓ2ˆð !%§¡¨k¯n©n¸UÓ.CÓ DÐØ$Ð$r.   )r�  r'  r„  r  r÷   r  rŒ  r~  r%  rK   rP   rŽ  r€  r¶   r†  r•  rˆ  r‘  rC  r“  r|  r‹  ry  r‚  )T©NN)rv   rw   rx   ry   rÑ   r4   rF   r{   r’  r   rs   r|   r}   r~   r   s   @@r+   rn  rn  <  sV   ù‡ € ñ	÷08õ 08ðd ‡]‚]ƒ_ñ!ó ð!ñ ˜HÓ%÷P%ñ P%ó &÷P%ð P%r.   rn  c                   ó8   a a€ ] tR tRt oV 3R ltR tRtVtV ;t# )ÚBambaMLPi  c                ó  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        \
        P                  ! V P                  V P                  VP                  R 7      V n        \
        P                  ! V P                  V P                  VP                  R 7      V n	        \
        P                  ! V P                  V P                  VP                  R 7      V n
        \        VP                  ,          V n        R# )r¹   N)r3   r4   r(   rP   r€  r   r½   Úmlp_biasÚ	gate_projÚup_projÚ	down_projr   rƒ  Úact_fn©r?   r(   rA   s   &&€r+   r4   ÚBambaMLP.__init__  s¶   ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔÜŸš 4×#3Ñ#3°T×5KÑ5KÐRX×RaÑRaÔbˆŒÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPV×P_ÑP_Ô`ˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRX×RaÑRaÔbˆŒÜ˜V×.Ñ.Õ/ˆŽr.   c                ó�   € V P                  V P                  V P                  V4      4      V P                  V4      ,          4      pV# ru   )r»  r¼  r¹  rº  )r?   rm   r»  s   && r+   rs   ÚBambaMLP.forward)  s6   € Ø—N‘N 4§;¡;¨t¯~©~¸aÓ/@Ó#AÀDÇLÁLÐQRÃOÕ#SÓTˆ	ØÐr.   )r¼  r(   r»  r¹  rP   r€  rº  rë   r   s   @@r+   r¶  r¶    s   ù‡ € õ0÷ò r.   r¶  ÚRMSNormc                   óZ   a a€ ] tR tRt oRV3R lV 3R llltV3R lR ltR tRtVtV ;t	# )	ÚBambaRMSNormi.  c                ó$   <€ V ^8„  d   QhRS[ RR/# )r'   rÛ   rD   N)rH   )r)   r*   s   "€r+   r,   ÚBambaRMSNorm.__annotate__0  s   ø€ ÷ $ñ $©ð $¸$ñ $r.   c                óŽ   <€ \         SV `  4        \        P                  ! \        P
                  ! V4      4      V n        W n        R# )z+
BambaRMSNorm is equivalent to T5LayerNorm
NrÕ   rÚ   s   &&&€r+   r4   ÚBambaRMSNorm.__init__0  s/   ø€ ô 	‰ÑÔÜ—l’l¤5§:¢:¨kÓ#:Ó;ˆŒØ #Ör.   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r'   r…   rD   )rF   rG   )r)   r*   s   "€r+   r,   rÅ  8  s#   ø€ ÷ ;ñ ;¡U§\¡\ð ;±e·l±lñ ;r.   c                ó:  € VP                   pVP                  \        P                  4      pVP	                  ^4      P                  RRR7      pV\        P                  ! W0P                  ,           4      ,          pV P                  VP                  V4      ,          # )r'   TrÞ   rc   )	rN   rU   rF   rŸ   rá   râ   rã   rÙ   rØ   )r?   r…   rå   ræ   s   &&  r+   rs   ÚBambaRMSNorm.forward8  sw   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯ª°H×?TÑ?TÕ4TÓ(UÕUˆØ�{‰{˜]×-Ñ-¨kÓ:Õ:Ð:r.   c                ó^   € \        V P                  P                  4       R V P                   2# )z, eps=)rE   rØ   re   rÙ   )r?   s   &r+   Ú
extra_reprÚBambaRMSNorm.extra_repr?  s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr.   rè   ré   )
rv   rw   rx   ry   r4   rs   rÌ  r|   r}   r~   r   s   @@r+   rÃ  rÃ  .  s%   ù‡ € ÷$õ $÷;ð ;÷Jò Jr.   rÃ  c                   óX   a a€ ] tR tRt oRV3R lV 3R llltRV3R lR lltRtVtV ;t# )	ÚBambaDecoderLayeriC  c                ó,   <€ V ^8„  d   QhRS[ RS[RS[/# )r'   r(   r¶   r•  )r    rS   rh   )r)   r*   s   "€r+   r,   ÚBambaDecoderLayer.__annotate__D  s'   ø€ ÷ Dñ D™{ð D±sð DÉñ Dr.   c                ó„  <€ \         SV `  4        ^pV^8X  d   \        MRpV! V4      V n        \	        VP
                  VP                  R7      V n        \	        VP
                  VP                  R7      V n        W0n	        VR8X  d   \        WR7      V n        R# VR8X  d   \        W4      V n        R# \        RV: 24      h)r  Nrv  Úlinear_attention)r(   r¶   Úfull_attentionzInvalid layer_type: )r3   r4   r¶  Úfeed_forwardrÃ  rP   r…  Úinput_layernormÚpre_ff_layernormÚ
block_typern  Úmambar´   Ú	self_attnÚ
ValueError)r?   r(   r¶   r•  Únum_expertsÚffn_layer_classrA   s   &&&&  €r+   r4   ÚBambaDecoderLayer.__init__D  s¦   ø€ Ü‰ÑÔàˆØ&1°QÔ&6�(¸DˆÙ+¨FÓ3ˆÔÜ+¨F×,>Ñ,>ÀF×DWÑDWÔXˆÔÜ ,¨V×-?Ñ-?ÀV×EXÑEXÔ YˆÔà$ŒØÐ+Ô+Ü#¨6ÔGˆDŽJØÐ+Ô+Ü+¨FÓ>ˆDŽNäÐ3°J±>ÐBÓCÐCr.   c                ó�  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[ P                  S[ P                  3,          R,          RS[S[,          R	S[S[ P                  S[S[ P                  S[ P                  3,          R,          3,          /# )
r'   r…   r”   Nrn   rÆ   Ú	use_cacherÅ   rV   rD   )	rF   rG   Ú
LongTensorr   r   rE   r   r   ÚFloatTensor)r)   r*   s   "€r+   r,   rÑ  U  sÄ   ø€ ÷ (0ñ (0á—|‘|ð(0ñ Ÿ™ tÕ+ð(0ñ ×&Ñ&¨Õ-ð	(0ñ
  �ð(0ñ ˜$•;ð(0ñ #¡5§<¡<±·±Ð#=Õ>ÀÕEð(0ñ Ñ+Õ,ð(0ñ 
‰u× Ñ ¡%©×(9Ñ(9¹5×;LÑ;LÐ(LÕ"MÐPTÕ"TÐTÕ	Uñ(0r.   c                óR  € TpV P                  V4      pV P                  R 8X  d   V P                  ! R
RVRVRV/VB pRp	M1V P                  R8X  d!   V P                  ! R
RVRVRVRVRVR	V/VB w  rW�,           pTpV P	                  V4      pV P                  V4      pW�,           pVX	3# )rÓ  r…   rž  r”   NrÔ  rn   rÆ   rà  rÅ   r3  )rÖ  rØ  rÙ  rÚ  r×  rÕ  )
r?   r…   r”   rn   rÆ   rà  rÅ   rV   ÚresidualÚself_attn_weightss
   &&&&&&&,  r+   rs   ÚBambaDecoderLayer.forwardU  s  € ð !ˆà×,Ñ,¨]Ó;ˆà�?‰?Ð0Ô0Ø ŸJšJñ Ø+ðà,ðð  .ðð ñ	ˆMð !%ÑØ�_‰_Ð 0Ô0Ø/3¯~ª~ñ 0Ø+ð0à-ð0ð *ð0ð !0ð	0ð
 $ð0ð %8ð0ð ñ0Ñ,ˆMð !Õ0ˆà ˆØ×-Ñ-¨mÓ<ˆØ×)Ñ)¨-Ó8ˆØ Õ0ˆàÐ/Ð/Ð/r.   )rØ  rÕ  rÖ  rÙ  r×  rÚ  )rÓ  )NNNFNrë   r   s   @@r+   rÏ  rÏ  C  s    ù‡ € ÷Dõ D÷"(0÷ (0ò (0r.   rÏ  c                   ó–   a a€ ] tR tRt oRtRtR.tR.tRtRt	Rt
RtR]R]/t]P                   ! 4       V 3R l4       tV3R	 ltR
tVtV ;t# )ÚBambaPreTrainedModeli€  ÚmodelTrÏ  rÆ   r…   Ú
attentionsc           
     óŒ  <€ \         SV `  V4       \        V\        4      '       dž   \        P
                  ! VP                  4       \        P                  ! VP                  \        P                  ! \        P                  ! ^VP                  ^,           4      4      4       \        P
                  ! VP                  4       R# R# )r  N)r3   Ú_init_weightsrf   rn  r™  r›  r%  r  r�  rF   rš  rT   rC  r'  )r?   r�   rA   s   &&€r+   rì  Ú"BambaPreTrainedModel._init_weights�  st   ø€ ä‰Ñ˜fÔ%Ü�fœj×)Ò)Ü�JŠJ�v—~‘~Ô&Ü�JŠJ�v—|‘|¤U§Y¢Y¬u¯|ª|¸A¸v×?OÑ?OÐRSÕ?SÓ/TÓ%UÔVÜ�JŠJ�v—x‘xÖ ñ *r.   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r&   r   )r)   r*   s   "€r+   r,   Ú!BambaPreTrainedModel.__annotate__€  s   ø‡ ‚ áÑò r.   r3  )rv   rw   rx   ry   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_is_statefulÚ_can_compile_fullgraphrÏ  r´   Ú_can_record_outputsrF   r{   rì  Ú__annotate_func__r|   r}   r~   r   s   @@r+   rè  rè  €  so   ù‡ € ð  ÐØ&*Ð#Ø,Ð-ÐØ#4Ð"5ÐØÐØ€NØ€LØ!ÐàÐ*Ø�nðÐð
 ‡]‚]ƒ_ô!ó ð!÷# … r.   rè  c                   ór   a a€ ] tR tRt oV3R lV 3R llt]]]RV3R lR ll4       4       4       tRt	Vt
V ;t# )Ú
BambaModeli™  c                ó    <€ V ^8„  d   QhRS[ /# r&   r   )r)   r*   s   "€r+   r,   ÚBambaModel.__annotate__›  s   ø€ ÷ ñ ™{ñ r.   c           	     ój  <€ \         SV `  V4       VP                  V n        VP                  V n        \
        P                  ! VP                  VP                  V P                  4      V n        . p\        VP                  4       F0  pVP                  \        WVP                  V,          R 7      4       K2  	  \
        P                  ! V4      V n        VP                   V n        \#        VP                  VP$                  R7      V n        \)        VR7      V n        RV n        V P/                  4        R# ))r¶   r•  rv  ©r(   FN)r3   r4   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	EmbeddingrP   Úembed_tokensÚrangeÚnum_hidden_layersÚappendrÏ  Úlayers_block_typeÚ
ModuleListr£  rË   rÃ  r…  Úfinal_layernormr"   Ú
rotary_embÚgradient_checkpointingÚ	post_init)r?   r(   Údecoder_layersÚirA   s   &&  €r+   r4   ÚBambaModel.__init__›  sç   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸLšL¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔØˆÜ�v×/Ñ/Ö0ˆAØ×!Ñ!Ô"3°FÐTZ×TlÑTlÐmnÕToÔ"pÖqñ 1ä—m’m NÓ3ˆŒà$*×$?Ñ$?ˆÔ!Ü+¨F×,>Ñ,>ÀF×DWÑDWÔXˆÔÜ.°fÔ=ˆŒà&+ˆÔ#à�‰Ör.   c                óþ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[S[,          R	S[/# )
r'   Ú	input_idsNr”   rn   rÆ   Úinputs_embedsrà  rV   rD   )	rF   rá  rG   r   râ  r   r   r   r   )r)   r*   s   "€r+   r,   rý  ±  s“   ø€ ÷ 7
ñ 7
á×#Ñ# dÕ*ð7
ñ Ÿ™ tÕ+ð7
ñ ×&Ñ&¨Õ-ð	7
ñ
  �ð7
ñ ×(Ñ(¨4Õ/ð7
ñ ˜$•;ð7
ñ Ñ+Õ,ð7
ñ 
!ñ7
r.   c                óÐ  € VR J VR J,          '       d   \        R4      hVf   V P                  V4      pTpV'       d   Vf   \        V P                  R7      pVfC   \        P
                  ! VP                  ^,          VP                  R7      P                  ^ 4      p\        T;p	\        4      '       g/   RV P                  RVRVRVRV/p
R	\        R/ V
B R
\        R/ V
B /p	V P                  WƒR7      p\        V P                  4       F;  w  rÍV! V3RW�P                  P                   V,          ,          RVRVRVRV/VB w  rŽK=  	  V P#                  V4      p\%        VVR7      # )Nz:You must specify exactly one of input_ids or inputs_embedsrÿ  r>  r(   r  r”   rÆ   rn   rÔ  rÓ  )rn   rà  rÅ   )Úlast_hidden_staterÆ   r3  )rÛ  r  r   r(   rF   rT   re   r#   r§   rf   Údictr   r   r  Ú	enumerater£  r  r
  r   )r?   r  r”   rn   rÆ   r  rà  rV   r…   Úcausal_mask_mappingÚmask_kwargsrÅ   r  Údecoder_layerr£   s   &&&&&&&,       r+   rs   ÚBambaModel.forward®  s•  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ Ø ×-Ñ-¨iÓ8ˆMØ%ˆç˜Ò0Ü*°$·+±+Ô>ˆOàÒÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\×fÑfÐghÓiˆLä°Ð?Ð-Ä×FÒFð ˜$Ÿ+™+Ø Ø  .Ø! ?Ø ðˆKð !Ô"4Ñ"C°{Ñ"CØ"Ô$CÑ$RÀkÑ$Rð#Ðð #Ÿo™o¨m˜oÓWÐä )¨$¯+©+Ö 6ÑˆAÙ*7Øñ+à2·;±;×3PÑ3PÐQRÕ3SÕTð+ð *ð+ð !0ð	+ð
 $ð+ð %8ð+ð ñ+Ñ'ˆM™<ñ !7ð ×,Ñ,¨]Ó;ˆä&Ø+Ø+ô
ð 	
r.   )rË   r  r
  r  r£  r  r  r  )NNNNNN)rv   rw   rx   ry   r4   r   r   r   rs   r|   r}   r~   r   s   @@r+   rû  rû  ™  s=   ù‡ € ÷ó ð&  ØØ÷7
ñ 7
ó ó ó  ÷7
ð 7
r.   rû  c                   ó�   a a€ ] tR tRt oRR/tRR/tRR.R.3/tRR/tV 3R	 lt]	]
RV3R
 lR ll4       4       tV 3R ltRtVtV ;t# )ÚBambaForCausalLMië  zlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputr…   ÚlogitsÚkeep_full_weightc                ó  <€ \         SV `  V4       \        V4      V n        VP                  V n        \
        P                  ! VP                  VP                  R R7      V n        VP                  V n	        V P                  4        R# )Fr¹   N)r3   r4   rû  ré  r  r   r½   rP   r  Úz_loss_coefficientr  r½  s   &&€r+   r4   ÚBambaForCausalLM.__init__ò  sc   ø€ Ü‰Ñ˜Ô Ü Ó'ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒØ"(×";Ñ";ˆÔð 	�‰Ör.   c                ó:  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[S[ P                  ,          R
S[/	# )r'   r  Nr”   rn   rÆ   r  Úlabelsrà  Úlogits_to_keeprD   )rF   rá  rG   r   râ  r   rS   r   )r)   r*   s   "€r+   r,   ÚBambaForCausalLM.__annotate__þ  s«   ø€ ÷ =
ñ =
á×#Ñ# dÕ*ð=
ñ Ÿ™ tÕ+ð=
ñ ×&Ñ&¨Õ-ð	=
ñ
  �ð=
ñ ×(Ñ(¨4Õ/ð=
ñ × Ñ  4Õ'ð=
ñ ˜$•;ð=
ñ ™eŸl™lÕ*ð=
ñ 
 ñ=
r.   c	                ó|  € V P                   ! RRVRVRVRVRVRV/V	B p
V
P                  p\        V\        4      '       d   \	        V) R4      MTpV P                  VRVR3,          4      pRpVe    V P                  ! RR	VR
VRV P                  P                  /V	B pV P                  ^ 8”  dc   VP                  RR7      P                  VP                  R7      P                  ^4      P                  4       pWàP                  V,          ,           p\        VVV
P                   V
P"                  V
P$                  R7      # )aÊ  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

Example:

```python
>>> from transformers import AutoTokenizer, BambaForCausalLM

>>> model = BambaForCausalLM.from_pretrained("...")
>>> tokenizer = AutoTokenizer.from_pretrained("...")

>>> prompt = "Hey, are you conscious? Can you talk to me?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
```r  r”   rn   rÆ   r  rà  Nrb   r   r&  r  ra   rM   )Úlossr   rÆ   r…   rê  r3  rc   )ré  r  rf   rS   Úslicer  Úloss_functionr(   r  r#  Ú	logsumexprU   rN   rá   râ   r   rÆ   r…   rê  )r?   r  r”   rn   rÆ   r  r&  rà  r'  rV   Úoutputsr…   Úslice_indicesr   r*  Úz_losss   &&&&&&&&&,      r+   rs   ÚBambaForCausalLM.forwardü  sZ  € ðH ,0¯:ª:ñ ,
Øð,
à)ð,
ð &ð,
ð ,ð	,
ð
 (ð,
ð  ð,
ð ñ,
ˆð  ×1Ñ1ˆÜ8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒØ×%Ò%Ñp¨VÐp¸FÐpÈtÏ{É{×OeÑOeÐpÐioÑpˆDØ×&Ñ&¨Ô*Ø×)Ñ)¨bÐ)Ó1×4Ñ4¸4¿:¹:Ð4ÓF×JÑJÈ1ÓM×RÑRÓT�Ø×5Ñ5¸Õ>Õ>�ä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r.   c                ó\   <€ V P                   P                  VR &   \        SV `  ! V3/ VB pV# )r'  )r(   Únum_logits_to_keepr3   Úprepare_inputs_for_generation)r?   r  rV   Úmodel_inputsrA   s   &&, €r+   r4  Ú.BambaForCausalLM.prepare_inputs_for_generation=  s2   ø€ Ø#'§;¡;×#AÑ#AˆÐÑ Ü‘wÒ<¸YÑQÈ&ÑQˆØÐr.   )r  ré  r  r#  )NNNNNNNr   )rv   rw   rx   ry   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr4   r   r   rs   r4  r|   r}   r~   r   s   @@r+   r  r  ë  sl   ù‡ € à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÐ/Ð0€Jõð Ø÷=
ñ =
ó ó ð=
÷~õ r.   r  )rŽ   )r  r´  )NNFN)rû  r  rè  )NÚcollections.abcr   rF   Útorch.nn.functionalr   r�   r  Ú r   r™  Úactivationsr   Úcache_utilsr   r   Ú
generationr	   Úintegrationsr
   r   r   Úintegrations.accelerater   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   Úutils.deprecationr   Úutils.genericr   r   r   Úutils.output_capturingr   Úconfiguration_bambar    r—   r"   rƒ   r�   r¥   r²   r´   rÓ   rõ   rø   r  r	  r
  r  rH   r4  rL  rl  rn  r¶  rÃ  rÏ  rè  rû  r  Ú__all__r3  r.   r+   Ú<module>rO     sv  ðõ4 %ã ß Ð Ý å &Ý !ß .Ý )ß tÑ tÝ =ß PÝ 9ß Oß Kß FÝ &ß 5Ý 0ß [Ñ [Ý 5Ý ,ô8<˜2Ÿ9™9ô 8<òv(õ	U÷%ô4#ôL@)�R—Y‘Yô @)ôF;˜Ÿ™Ÿ™ô ;õ*Vò
ò(ò(	ñ (Ð(>ÀÓPö'ó Qð'ñ& (Ð(:¸OÓLö'ó Mð'ñ, (Ð(JÈKÓXð +/Ø%(©%°«,Ð$7Ø %ØØ*.ØØ*.Ø(,ØØØ!ö%ó Yðñ0 (Ð(@À+ÓNö:'ó Oð:'ñz (Ð(CÀ[ÓQð "Ø#'Ø*.ØØ%(©%°«,Ð$7Ø %ö^ó Rð^ñB àØØ)Ø%ØðóôV%�—‘ó V%óðV%ôrˆr�y‰yô ñ  ˜YÓ'ôJ�2—9‘9ó Jó (ðJô(:0Ð2ô :0ðz ô!˜?ó !ó ð!ð0 ôN
Ð%ó N
ó ðN
ðb ôTÐ+¨_ó Tó ðTòn E‚r.   