+
    œ°³j?D  ã                   ó  € R t ^ RIt^ RIHt ^RIHt ^RIHtHt ^RI	H
t
Ht ^RIHtHt ^RIHt ^RIHt ^R	IHtHtHt ^R
IHtHtHt ^RIHt ^RIHt ^RIH t H!t!H"t"H#t#H$t$H%t% ^RI&H't'H(t( ^RI)H*t* ]PV                  ! ],4      t- ! R R]$4      t.R%R lt/] ! R R] 4      4       t0 ! R R](4      t1 ! R R]'4      t2 ! R R]"4      t3 ! R R]#4      t4 ! R R]4      t5] ! R R ]4      4       t6] ! R! R"]64      4       t7 ! R# R$]!4      t8. R&Ot9R# )'zPyTorch Bamba model.N)Únn)Úinitialization)ÚCacheÚDynamicCache)Úcreate_causal_maskÚcreate_recurrent_attention_mask)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚPreTrainedModel)ÚUnpack)Úauto_docstringÚcan_return_tupleÚlogging)ÚTransformersKwargsÚmerge_with_config_defaultsÚno_inherit_decorator)Úcapture_outputs)ÚJambaAttentionDecoderLayer)ÚLlamaAttentionÚLlamaForCausalLMÚLlamaMLPÚLlamaRMSNormÚLlamaRotaryEmbeddingÚrotate_half)ÚMamba2MixerÚMambaRMSNormGated©ÚBambaConfigc                   ó6   a € ] tR t^4t o RV 3R lR lltRtV tR# )ÚBambaRotaryEmbeddingNc                óP   <€ V ^8„  d   QhRS[ RS[S[P                  S[3,          /# )é   ÚconfigÚreturn)r   ÚtupleÚtorchÚTensorÚfloat)ÚformatÚ__classdict__s   "€Úh/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/bamba/modular_bamba.pyÚ__annotate__Ú!BambaRotaryEmbedding.__annotate__5   s.   ø€ ÷ 5ñ 5±ð 5ÑW\Ñ]b×]iÑ]iÑkpÐ]pÕWqñ 5ó    c           	     óŠ  € V P                   R,          p\        V RR4      ;'       g    V P                  V P                  ,          pV P                   P	                  RR4      p\        WE,          4      pRpRV\        P                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚhead_dimNÚpartial_rotary_factorg      ð?©Údtype)
Úrope_parametersÚgetattrÚhidden_sizeÚnum_attention_headsÚgetÚintr%   Úaranger'   Úto)r"   ÚdeviceÚkwargsÚbaseÚdimr1   Úattention_factorÚinv_freqs   &&,     r*   Úcompute_default_rope_parametersÚ4BambaRotaryEmbedding.compute_default_rope_parameters5   s¦   € ð ×%Ñ% lÕ3ˆÜ�f˜j¨$Ó/×cÐc°6×3EÑ3EÈ×IcÑIcÕ3cˆØ &× 6Ñ 6× :Ñ :Ð;RÐTWÓ XÐÜ�#Õ-Ó.ˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r-   © ©N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__rB   Ú__static_attributes__Ú__classdictcell__)r)   s   @r*   r   r   4   s   ø‡ € ÷5÷ 5ð 5r-   r   c                ó¶  € VP                  V4      pVP                  V4      pVP                  R,          pV RRV13,          V RVR13,          rvVRRV13,          VRVR13,          r˜Wb,          \        V4      V,          ,           p
W‚,          \        V4      V,          ,           p\        P                  ! W§.RR7      p
\        P                  ! W¹.RR7      pW«3# )a{  Applies Rotary Position Embedding to the query and key tensors.

Removes the interleaving of cos and sin from GLM

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
.N©r?   éÿÿÿÿ)Ú	unsqueezeÚshaper   r%   Úcat)ÚqÚkÚcosÚsinÚunsqueeze_dimÚ
rotary_dimÚq_rotÚq_passÚk_rotÚk_passÚq_embedÚk_embeds   &&&&&       r*   Úapply_rotary_pos_embr^   K   sÐ   € ð( �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€Cð —‘˜2•€JØ�c˜;˜J˜;Ð&Õ'¨¨3°
±Ð+;Õ)<ˆ6Ø�c˜;˜J˜;Ð&Õ'¨¨3°
±Ð+;Õ)<ˆ6ð �{œ{¨5Ó1°CÕ7Õ8€GØ�{œ{¨5Ó1°CÕ7Õ8€Gô �iŠi˜Ð)¨rÔ2€GÜ�iŠi˜Ð)¨rÔ2€GØÐÐr-   c                   ó   € ] tR t^qtRtR# )ÚBambaAttentionrD   N©rF   rG   rH   rI   rJ   rD   r-   r*   r`   r`   q   s   † ãr-   r`   c                   ó   € ] tR t^vtRtR# )ÚBambaRMSNormGatedrD   Nra   rD   r-   r*   rc   rc   v   ó   † Ûr-   rc   c                   ót   a a€ ] tR t^zt oRtRV3R lV 3R lllt]P                  ! 4       R 4       tRt	Vt
V ;t# )Ú
BambaMixeruK  
Compute âˆ†, A, B, C, and D the state space parameters and compute the `contextualized_states`.
A, D are input independent (see Mamba paper [1] Section 3.5.2 "Interpretation of A" for why A isn't selective)
âˆ†, B, C are input-dependent (this is a key difference between Mamba and the linear time invariant S4,
and is why Mamba is called **selective** state spaces)

The are a few differences between this and Mamba2Mixer:
- The variable use_precomputed_states is slightly different due to the hybrid cache structure
- Some extra variables that our layer doesn't need have been removed
c                ó,   <€ V ^8„  d   QhRS[ RS[RS[/# )r!   r"   Ú	layer_idxÚinitialize_mixer_weights)r   r9   Úbool)r(   r)   s   "€r*   r+   ÚBambaMixer.__annotate__†   s#   ø€ ÷  ñ  ™{ð  ±sð  ÑVZñ  r-   c           	     ó  <€ VP                   V n        \        SV `  WV4       VP                  V n        VP                  V n        VP                  V n	        \        VP                  V P                  ,          4      V n        VP                  V n        VP                   V n        VP$                  V n        VP(                  V n        VP,                  V n        \1        V P                  V P"                  R 7      V n        \4        P6                  ! V P8                  V P8                  VP                  V P                  V P8                  V P                  ^,
          R7      V n        V P                  V P8                  ,           V P
                  ,           p\4        P<                  ! V P                  VV P                  R7      V n        \4        P<                  ! V P                  V P                  V P                  R7      V n         V =!V ="V =V =#V =$R# )©Úeps)Úin_channelsÚout_channelsÚbiasÚkernel_sizeÚgroupsÚpadding)rq   N)%Úmamba_proj_biasÚuse_biasÚsuperÚ__init__Úmamba_n_headsÚ	num_headsÚmamba_d_stateÚssm_state_sizeÚmamba_d_convÚconv_kernel_sizer9   Úmamba_expandr6   Úintermediate_sizeÚmamba_conv_biasÚuse_conv_biasÚrms_norm_epsÚlayer_norm_epsilonÚmamba_n_groupsÚn_groupsÚmamba_d_headr0   Úmamba_chunk_sizeÚ
chunk_sizerc   Únormr   ÚConv1dÚconv_dimÚconv1dÚLinearÚin_projÚout_projÚtime_step_floorÚtime_step_rankÚtime_step_minÚtime_step_max)Úselfr"   rh   ri   Úprojection_sizeÚ	__class__s   &&&& €r*   rx   ÚBambaMixer.__init__†   s‘  ø€ Ø×.Ñ.ˆŒÜ‰Ñ˜Ð,DÔEØ×-Ñ-ˆŒØ$×2Ñ2ˆÔØ &× 3Ñ 3ˆÔÜ!$ V×%8Ñ%8¸4×;KÑ;KÕ%KÓ!LˆÔØ#×3Ñ3ˆÔØ"(×"5Ñ"5ˆÔØ×-Ñ-ˆŒØ×+Ñ+ˆŒØ ×1Ñ1ˆŒÜ% d×&<Ñ&<À$×BYÑBYÔZˆŒ	Ü—i’iØŸ™ØŸ™Ø×'Ñ'Ø×-Ñ-Ø—=‘=Ø×)Ñ)¨AÕ-ô
ˆŒð ×0Ñ0°4·=±=Õ@À4Ç>Á>ÕQˆÜ—y’yØ×ÑØØ—‘ô
ˆŒô
 Ÿ	š	 $×"8Ñ"8¸$×:JÑ:JÐQU×Q^ÑQ^Ô_ˆŒØÐ ØÐØˆMØÐØÒr-   c                óŠ  € \         P                  ! ^V P                  ^,           V P                  P                  \         P
                  R7      p\        P                  ! V P                  \         P                  ! V4      4       \        P                  ! V P                  4       \        P                  ! V P                  4       R# )é   )r<   r3   N)r%   r:   rz   ÚA_logr<   Úfloat32ÚinitÚcopy_ÚlogÚones_ÚDÚdt_bias)r•   ÚAs   & r*   Úinit_bamba_weightsÚBambaMixer.init_bamba_weights¨   sj   € ä�LŠL˜˜DŸN™N¨QÕ.°t·z±z×7HÑ7HÔPU×P]ÑP]Ô^ˆÜ�
Š
�4—:‘:œuŸyšy¨›|Ô,Ü�
Š
�4—6‘6ÔÜ�
Š
�4—<‘<Ö r-   )r‰   r�   r~   r0   r�   r€   r„   r†   rŠ   rz   r�   r|   rv   r‚   )T)rF   rG   rH   rI   Ú__doc__rx   r%   Úno_gradr¤   rJ   rK   Ú__classcell__©r—   r)   s   @@r*   rf   rf   z   s4   ù‡ € ñ	÷ õ  ðD ‡]‚]ƒ_ñ!ó ÷!ð !r-   rf   c                   ó   € ] tR t^°tRtR# )ÚBambaMLPrD   Nra   rD   r-   r*   r«   r«   °   rd   r-   r«   c                   ó   € ] tR t^´tRtR# )ÚBambaRMSNormrD   Nra   rD   r-   r*   r­   r­   ´   rd   r-   r­   c                   óX   a a€ ] tR t^¸t oRV3R lV 3R llltRV3R lR lltRtVtV ;t# )ÚBambaDecoderLayerc                ó,   <€ V ^8„  d   QhRS[ RS[RS[/# )r!   r"   rh   Ú
layer_type)r   r9   Ústr)r(   r)   s   "€r*   r+   ÚBambaDecoderLayer.__annotate__¹   s'   ø€ ÷ Dñ D™{ð D±sð DÉñ Dr-   c                óò   <€ \         SV `  W4       V =^pV^8X  d   \        MRpV! V4      V n        W0n        VR8X  d   \        WR7      V n        R# VR8X  d   \        W4      V n        R# \        RV: 24      h)rš   NÚlinear_attention)r"   rh   Úfull_attentionzInvalid layer_type: )
rw   rx   Ú	self_attnr«   Úfeed_forwardÚ
block_typerf   Úmambar`   Ú
ValueError)r•   r"   rh   r±   Únum_expertsÚffn_layer_classr—   s   &&&&  €r*   rx   ÚBambaDecoderLayer.__init__¹   sw   ø€ Ü‰Ñ˜Ô+àˆNàˆØ&1°QÔ&6�(¸DˆÙ+¨FÓ3ˆÔà$ŒØÐ+Ô+Ü#¨6ÔGˆDŽJØÐ+Ô+Ü+¨FÓ>ˆDŽNäÐ3°J±>ÐBÓCÐCr-   c                ó�  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[ P                  S[ P                  3,          R,          RS[S[,          R	S[S[ P                  S[S[ P                  S[ P                  3,          R,          3,          /# )
r!   Úhidden_statesÚattention_maskNÚposition_idsÚpast_key_valuesÚ	use_cacheÚposition_embeddingsr=   r#   )	r%   r&   Ú
LongTensorr   rj   r$   r   r   ÚFloatTensor)r(   r)   s   "€r*   r+   r³   Ê   sÄ   ø€ ÷ (0ñ (0á—|‘|ð(0ñ Ÿ™ tÕ+ð(0ñ ×&Ñ&¨Õ-ð	(0ñ
  �ð(0ñ ˜$•;ð(0ñ #¡5§<¡<±·±Ð#=Õ>ÀÕEð(0ñ Ñ+Õ,ð(0ñ 
‰u× Ñ ¡%©×(9Ñ(9¹5×;LÑ;LÐ(LÕ"MÐPTÕ"TÐTÕ	Uñ(0r-   c                óR  € TpV P                  V4      pV P                  R 8X  d   V P                  ! R
RVRVRV/VB pRp	M1V P                  R8X  d!   V P                  ! R
RVRVRVRVRVR	V/VB w  rW�,           pTpV P	                  V4      pV P                  V4      pW�,           pVX	3# )rµ   rÀ   Úcache_paramsrÁ   Nr¶   rÂ   rÃ   rÄ   rÅ   rD   )Úinput_layernormr¹   rº   r·   Úpre_ff_layernormr¸   )
r•   rÀ   rÁ   rÂ   rÃ   rÄ   rÅ   r=   ÚresidualÚself_attn_weightss
   &&&&&&&,  r*   ÚforwardÚBambaDecoderLayer.forwardÊ   s  € ð !ˆà×,Ñ,¨]Ó;ˆà�?‰?Ð0Ô0Ø ŸJšJñ Ø+ðà,ðð  .ðð ñ	ˆMð !%ÑØ�_‰_Ð 0Ô0Ø/3¯~ª~ñ 0Ø+ð0à-ð0ð *ð0ð !0ð	0ð
 $ð0ð %8ð0ð ñ0Ñ,ˆMð !Õ0ˆà ˆØ×-Ñ-¨mÓ<ˆØ×)Ñ)¨-Ó8ˆØ Õ0ˆàÐ/Ð/Ð/r-   )r¹   r¸   rº   r·   )rµ   )NNNFN)	rF   rG   rH   rI   rx   rÎ   rJ   rK   r¨   r©   s   @@r*   r¯   r¯   ¸   s    ù‡ € ÷Dõ D÷"(0÷ (0ò (0r-   r¯   c                   ó–   a a€ ] tR t^õt oRtRtR.tR.tRtRt	Rt
RtR]R]/t]P                   ! 4       V 3R l4       tV3R ltR	tVtV ;t# )
ÚBambaPreTrainedModelÚmodelTr¯   rÃ   rÀ   Ú
attentionsc           
     óŒ  <€ \         SV `  V4       \        V\        4      '       dž   \        P
                  ! VP                  4       \        P                  ! VP                  \        P                  ! \        P                  ! ^VP                  ^,           4      4      4       \        P
                  ! VP                  4       R# R# )rš   N)rw   Ú_init_weightsÚ
isinstancerf   r�   r    r¢   rž   r›   r%   rŸ   r:   rz   r¡   )r•   Úmoduler—   s   &&€r*   rÕ   Ú"BambaPreTrainedModel._init_weights  st   ø€ ä‰Ñ˜fÔ%Ü�fœj×)Ò)Ü�JŠJ�v—~‘~Ô&Ü�JŠJ�v—|‘|¤U§Y¢Y¬u¯|ª|¸A¸v×?OÑ?OÐRSÕ?SÓ/TÓ%UÔVÜ�JŠJ�v—x‘xÖ ñ *r-   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # ©r!   r"   r   )r(   r)   s   "€r*   r+   Ú!BambaPreTrainedModel.__annotate__õ   s   ø‡ ‚ áÑò r-   rD   )rF   rG   rH   rI   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_is_statefulÚ_can_compile_fullgraphr¯   r`   Ú_can_record_outputsr%   r§   rÕ   Ú__annotate_func__rJ   rK   r¨   r©   s   @@r*   rÑ   rÑ   õ   so   ù‡ € ð  ÐØ&*Ð#Ø,Ð-ÐØ#4Ð"5ÐØÐØ€NØ€LØ!ÐàÐ*Ø�nðÐð
 ‡]‚]ƒ_ô!ó ð!÷# … r-   rÑ   c                   ór   a a€ ] tR tRt oV3R lV 3R llt]]]RV3R lR ll4       4       4       tRt	Vt
V ;t# )Ú
BambaModeli  c                ó    <€ V ^8„  d   QhRS[ /# rÚ   r   )r(   r)   s   "€r*   r+   ÚBambaModel.__annotate__  s   ø€ ÷ ñ ™{ñ r-   c           	     ój  <€ \         SV `  V4       VP                  V n        VP                  V n        \
        P                  ! VP                  VP                  V P                  4      V n        . p\        VP                  4       F0  pVP                  \        WVP                  V,          R 7      4       K2  	  \
        P                  ! V4      V n        VP                   V n        \#        VP                  VP$                  R7      V n        \)        VR7      V n        RV n        V P/                  4        R# ))rh   r±   rm   ©r"   FN)rw   rx   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingr6   Úembed_tokensÚrangeÚnum_hidden_layersÚappendr¯   Úlayers_block_typeÚ
ModuleListÚlayersÚ_attn_implementationr­   rƒ   Úfinal_layernormr   Ú
rotary_embÚgradient_checkpointingÚ	post_init)r•   r"   Údecoder_layersÚir—   s   &&  €r*   rx   ÚBambaModel.__init__  sç   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸLšL¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔØˆÜ�v×/Ñ/Ö0ˆAØ×!Ñ!Ô"3°FÐTZ×TlÑTlÐmnÕToÔ"pÖqñ 1ä—m’m NÓ3ˆŒà$*×$?Ñ$?ˆÔ!Ü+¨F×,>Ñ,>ÀF×DWÑDWÔXˆÔÜ.°fÔ=ˆŒà&+ˆÔ#à�‰Ör-   c                óþ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[S[,          R	S[/# )
r!   Ú	input_idsNrÁ   rÂ   rÃ   Úinputs_embedsrÄ   r=   r#   )	r%   rÆ   r&   r   rÇ   rj   r   r   r   )r(   r)   s   "€r*   r+   ré   &  s“   ø€ ÷ 7
ñ 7
á×#Ñ# dÕ*ð7
ñ Ÿ™ tÕ+ð7
ñ ×&Ñ&¨Õ-ð	7
ñ
  �ð7
ñ ×(Ñ(¨4Õ/ð7
ñ ˜$•;ð7
ñ Ñ+Õ,ð7
ñ 
!ñ7
r-   c                óÐ  € VR J VR J,          '       d   \        R4      hVf   V P                  V4      pTpV'       d   Vf   \        V P                  R7      pVfC   \        P
                  ! VP                  ^,          VP                  R7      P                  ^ 4      p\        T;p	\        4      '       g/   RV P                  RVRVRVRV/p
R	\        R/ V
B R
\        R/ V
B /p	V P                  WƒR7      p\        V P                  4       F;  w  rÍV! V3RW�P                  P                   V,          ,          RVRVRVRV/VB w  rŽK=  	  V P#                  V4      p\%        VVR7      # )Nz:You must specify exactly one of input_ids or inputs_embedsrë   )r<   r"   r  rÁ   rÃ   rÂ   r¶   rµ   )rÂ   rÄ   rÅ   )Úlast_hidden_staterÃ   rD   )r»   rð   r   r"   r%   r:   rP   r<   rO   rÖ   Údictr   r   rù   Ú	enumeraterö   rô   rø   r   )r•   r   rÁ   rÂ   rÃ   r  rÄ   r=   rÀ   Úcausal_mask_mappingÚmask_kwargsrÅ   rý   Údecoder_layerÚattn_weightss   &&&&&&&,       r*   rÎ   ÚBambaModel.forward#  s•  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ Ø ×-Ñ-¨iÓ8ˆMØ%ˆç˜Ò0Ü*°$·+±+Ô>ˆOàÒÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\×fÑfÐghÓiˆLä°Ð?Ð-Ä×FÒFð ˜$Ÿ+™+Ø Ø  .Ø! ?Ø ðˆKð !Ô"4Ñ"C°{Ñ"CØ"Ô$CÑ$RÀkÑ$Rð#Ðð #Ÿo™o¨m˜oÓWÐä )¨$¯+©+Ö 6ÑˆAÙ*7Øñ+à2·;±;×3PÑ3PÐQRÕ3SÕTð+ð *ð+ð !0ð	+ð
 $ð+ð %8ð+ð ñ+Ñ'ˆM™<ñ !7ð ×,Ñ,¨]Ó;ˆä&Ø+Ø+ô
ð 	
r-   )r÷   rð   rø   rú   rö   rí   rù   rî   )NNNNNN)rF   rG   rH   rI   rx   r   r   r   rÎ   rJ   rK   r¨   r©   s   @@r*   rç   rç     s=   ù‡ € ÷ó ð&  ØØ÷7
ñ 7
ó ó ó  ÷7
ð 7
r-   rç   c                   óh   a a€ ] tR tRt oV 3R lt]]RV3R lR ll4       4       tV 3R ltRt	Vt
V ;t# )ÚBambaForCausalLMi`  c                óh   <€ \         SV `  V4       VP                  V n        V P                  4        R # rE   )rw   rx   Úz_loss_coefficientrû   )r•   r"   r—   s   &&€r*   rx   ÚBambaForCausalLM.__init__a  s*   ø€ Ü‰Ñ˜Ô Ø"(×";Ñ";ˆÔð 	�‰Ör-   c                ó:  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[S[ P                  ,          R
S[/	# )r!   r   NrÁ   rÂ   rÃ   r  ÚlabelsrÄ   Úlogits_to_keepr#   )r%   rÆ   r&   r   rÇ   rj   r9   r	   )r(   r)   s   "€r*   r+   ÚBambaForCausalLM.__annotate__j  s«   ø€ ÷ =
ñ =
á×#Ñ# dÕ*ð=
ñ Ÿ™ tÕ+ð=
ñ ×&Ñ&¨Õ-ð	=
ñ
  �ð=
ñ ×(Ñ(¨4Õ/ð=
ñ × Ñ  4Õ'ð=
ñ ˜$•;ð=
ñ ™eŸl™lÕ*ð=
ñ 
 ñ=
r-   c	                ó|  € V P                   ! RRVRVRVRVRVRV/V	B p
V
P                  p\        V\        4      '       d   \	        V) R4      MTpV P                  VRVR3,          4      pRpVe    V P                  ! RR	VR
VRV P                  P                  /V	B pV P                  ^ 8”  dc   VP                  RR7      P                  VP                  R7      P                  ^4      P                  4       pWàP                  V,          ,           p\        VVV
P                   V
P"                  V
P$                  R7      # )aÊ  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

Example:

```python
>>> from transformers import AutoTokenizer, BambaForCausalLM

>>> model = BambaForCausalLM.from_pretrained("...")
>>> tokenizer = AutoTokenizer.from_pretrained("...")

>>> prompt = "Hey, are you conscious? Can you talk to me?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
```r   rÁ   rÂ   rÃ   r  rÄ   N:NNNÚlogitsr  rî   rM   r2   )Úlossr  rÃ   rÀ   rÓ   rD   rN   )rÒ   r  rÖ   r9   ÚsliceÚlm_headÚloss_functionr"   rî   r  Ú	logsumexpr;   r3   ÚpowÚmeanr	   rÃ   rÀ   rÓ   )r•   r   rÁ   rÂ   rÃ   r  r  rÄ   r  r=   ÚoutputsrÀ   Úslice_indicesr  r  Úz_losss   &&&&&&&&&,      r*   rÎ   ÚBambaForCausalLM.forwardh  sZ  € ðH ,0¯:ª:ñ ,
Øð,
à)ð,
ð &ð,
ð ,ð	,
ð
 (ð,
ð  ð,
ð ñ,
ˆð  ×1Ñ1ˆÜ8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒØ×%Ò%Ñp¨VÐp¸FÐpÈtÏ{É{×OeÑOeÐpÐioÑpˆDØ×&Ñ&¨Ô*Ø×)Ñ)¨bÐ)Ó1×4Ñ4¸4¿:¹:Ð4ÓF×JÑJÈ1ÓM×RÑRÓT�Ø×5Ñ5¸Õ>Õ>�ä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r-   c                ó\   <€ V P                   P                  VR &   \        SV `  ! V3/ VB pV# )r  )r"   Únum_logits_to_keeprw   Úprepare_inputs_for_generation)r•   r   r=   Úmodel_inputsr—   s   &&, €r*   r#  Ú.BambaForCausalLM.prepare_inputs_for_generation©  s2   ø€ Ø#'§;¡;×#AÑ#AˆÐÑ Ü‘wÒ<¸YÑQÈ&ÑQˆØÐr-   )r  )NNNNNNNé    )rF   rG   rH   rI   rx   r   r   rÎ   r#  rJ   rK   r¨   r©   s   @@r*   r  r  `  s6   ù‡ € õð Ø÷=
ñ =
ó ó ð=
÷~õ r-   r  )rš   )rç   r  rÑ   ):r¦   r%   r   Ú r   r�   Úcache_utilsr   r   Úmasking_utilsr   r   Úmodeling_outputsr   r	   Úmodeling_utilsr
   Úprocessing_utilsr   Úutilsr   r   r   Úutils.genericr   r   r   Úutils.output_capturingr   Újamba.modeling_jambar   Úllama.modeling_llamar   r   r   r   r   r   Úmamba2.modeling_mamba2r   r   Úconfiguration_bambar   Ú
get_loggerrF   Úloggerr   r^   r`   rc   rf   r«   r­   r¯   rÑ   rç   r  Ú__all__rD   r-   r*   Ú<module>r7     s  ðñ& ã Ý å &ß .ß Pß OÝ -Ý &ß >Ñ >ß aÑ aÝ 5Ý =÷÷ ÷õ -ð 
×	Ò	˜HÓ	%€ô5Ð/ô 5ô.#ðL ô	�^ó 	ó ð	ô	Ð)ô 	ô3!�ô 3!ôl	ˆxô 	ô	�<ô 	ô:0Ð2ô :0ðz ô!˜?ó !ó ð!ð0 ôN
Ð%ó N
ó ðN
ôbLÐ'ô Lò^ E‚r-   