+
    œ°³jR¬  ã                   óR  € ^ RI Ht ^ RIt^ RIHu Ht ^ RIHt ^RIHt	 ^RI
Ht ^RIHtHt ^RIHt ^RIHtHtHtHt ^R	IHt ^R
IHtHt ^RIHtHt ^RIHtH t  ^RI!H"t"H#t# ^RI$H%t% ^RI&H't'H(t(H)t) ^RI*H+t+ ^RI,H-t-H.t.H/t/H0t0 ^RI1H2t2H3t3 ^RI4H5t5 ]! R4       ! R R]Pl                  4      4       t7R t8]! R4      RJR l4       t9R R lt:RKR R llt;]! ]94       ! R R]Pl                  4      4       t<R  t=]! R!R"4      RLR# R$ ll4       t>]! R%R"4      RLR& R' ll4       t?]! R(R)4      RMR* R+ ll4       t@]! R,R)4      RNR- R. ll4       tA]! R/R)4      ROR0 R1 ll4       tB]! ]@]B]A]?]>.4       ! R2 R3]Pl                  4      4       tC ! R4 R5]Pl                  4      tD] ! R6 R7]Pl                  4      4       tE ! R8 R9]Pl                  4      tF ! R: R;]4      tG ! R< R=]4      tH ! R> R?]#4      tIR@]GRA]H/tJ]( ! RB RC]I4      4       tKRPRD RE lltL]( ! RF RG]I]4      4       tM ! RH RI]]I4      tN. RQOtOR# )Ré    )ÚCallableN)Únn)Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_experts_implementationÚuse_kernel_forward_from_hubÚ&use_kernel_func_from_hub_with_fallbackÚuse_kernelized_func)Úforce_accelerate_hooks)Úcreate_causal_maskÚcreate_recurrent_attention_mask)Ú GenericForSequenceClassificationÚGradientCheckpointingLayer)ÚMoeCausalLMOutputWithPastÚMoeModelOutputWithPast)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Úmerge_with_config_defaults)Úis_mambapy_availableÚis_torch_greater_or_equalÚis_torchdynamo_compilingÚis_torchdynamo_exporting)ÚOutputRecorderÚcapture_outputs©ÚJambaConfigÚRMSNormc                   óZ   a a€ ] tR t^;t oRV3R lV 3R llltV3R lR ltR tRtVtV ;t	# )ÚJambaRMSNormc                ó$   <€ V ^8„  d   QhRS[ RR/# )é   ÚepsÚreturnN)Úfloat)ÚformatÚ__classdict__s   "€Úi/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/jamba/modeling_jamba.pyÚ__annotate__ÚJambaRMSNorm.__annotate__=   s   ø€ ÷ $ñ $©ð $¸$ñ $ó    c                óŽ   <€ \         SV `  4        \        P                  ! \        P
                  ! V4      4      V n        W n        R# )z+
JambaRMSNorm is equivalent to T5LayerNorm
N)ÚsuperÚ__init__r   Ú	ParameterÚtorchÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizer)   Ú	__class__s   &&&€r.   r4   ÚJambaRMSNorm.__init__=   s/   ø€ ô 	‰ÑÔÜ—l’l¤5§:¢:¨kÓ#:Ó;ˆŒØ #Ör1   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r(   Úhidden_statesr*   ©r6   ÚTensor)r,   r-   s   "€r.   r/   r0   E   s#   ø€ ÷ ;ñ ;¡U§\¡\ð ;±e·l±lñ ;r1   c                ó:  € VP                   pVP                  \        P                  4      pVP	                  ^4      P                  RRR7      pV\        P                  ! W0P                  ,           4      ,          pV P                  VP                  V4      ,          # )r(   T)Úkeepdiméÿÿÿÿ)	ÚdtypeÚtor6   Úfloat32ÚpowÚmeanÚrsqrtr9   r8   )r:   r@   Úinput_dtypeÚvariances   &&  r.   ÚforwardÚJambaRMSNorm.forwardE   sw   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯ª°H×?TÑ?TÕ4TÓ(UÕUˆØ�{‰{˜]×-Ñ-¨kÓ:Õ:Ð:r1   c                ó^   € \        V P                  P                  4       R V P                   2# )z, eps=)Útupler8   Úshaper9   )r:   s   &r.   Ú
extra_reprÚJambaRMSNorm.extra_reprL   s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr1   )r9   r8   )ç�íµ ÷Æ°>)
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r4   rN   rS   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r<   r-   s   @@r.   r&   r&   ;   s%   ù‡ € ÷$õ $÷;ð ;÷Jò Jr1   r&   c                óÌ   € V RRV P                   R,          ^,          13,          pV RV P                   R,          ^,          R13,          p\        P                  ! V) V3RR7      # )z*Rotates half the hidden dims of the input..N©ÚdimrE   )rR   r6   Úcat)ÚxÚx1Úx2s   &  r.   Úrotate_halfre   P   sZ   € à	
ˆ3Ð"�!—'‘'˜"•+ Õ"Ð"Ð"Õ	#€BØ	
ˆ3�—‘˜•˜qÕ Ñ"Ð"Õ	#€BÜ�9Š9�r�c˜2�Y BÔ'Ð'r1   Úrotary_pos_embc                óÈ   € VP                  V4      pVP                  V4      pW,          \        V 4      V,          ,           pW,          \        V4      V,          ,           pWV3# )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)Ú	unsqueezere   )ÚqÚkÚcosÚsinÚunsqueeze_dimÚq_embedÚk_embeds   &&&&&  r.   Úapply_rotary_pos_embrp   W   sS   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�wœ; q›>¨CÕ/Õ0€GØ�wœ; q›>¨CÕ/Õ0€GØÐÐr1   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r(   r@   Ún_repr*   )r6   rB   Úint)r,   s   "r.   r/   r/   q   s.   € ÷ 	Uñ 	UœUŸ\™\ð 	U´#ð 	U¼%¿,¹,ñ 	Ur1   c                ó˜   € V P                   w  r#rEV^8X  d   V # V R,          P                  W#WV4      p V P                  W#V,          WE4      # )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
)ºNNNru   Nru   ru   )rR   ÚexpandÚreshape)r@   rr   ÚbatchÚnum_key_value_headsÚslenÚhead_dims   &&    r.   Ú	repeat_kvr|   q   sU   € ð
 2?×1DÑ1DÑ.€E Ø�„zØÐØ!Ð"2Õ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÕ(CÀTÓTÐTr1   c                ó   € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R\        R	\
        \        ,          /# )
r(   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNÚscalingÚdropoutÚkwargs)r   ÚModuler6   rB   r+   r   r   )r,   s   "r.   r/   r/   }   sw   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ð%ô ð%ô Ô'Õ(ñ%r1   c                 ó"  € \        W P                  4      p\        W0P                  4      p	\        P                  ! WP	                  ^^4      4      V,          p
Ve	   W¤,           p
\
        P                  P                  V
R\        P                  R7      P                  VP                  4      p
\
        P                  P                  W¦V P                  R7      p
\        P                  ! W©4      pVP	                  ^^4      P                  4       pWº3# )r(   ©r`   rF   )ÚpÚtrainingrE   )r|   Únum_key_value_groupsr6   ÚmatmulÚ	transposer   Ú
functionalÚsoftmaxrH   rG   rF   r„   rŠ   Ú
contiguous)r~   r   r€   r�   r‚   rƒ   r„   r…   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs   &&&&&&&,    r.   Úeager_attention_forwardr•   }   sÒ   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÕL€LØÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r1   c                   óX   a a€ ] tR t^–t oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )ÚJambaAttentionz=Multi-headed attention from 'Attention Is All You Need' paperc                ó&   <€ V ^8„  d   QhRS[ RS[/# ©r(   ÚconfigÚ	layer_idx©r#   rs   )r,   r-   s   "€r.   r/   ÚJambaAttention.__annotate__š   s   ø€ ÷ lñ l™{ð l±sñ lr1   c                óR  <€ \         SV `  4        Wn        W n        \	        VR VP
                  VP                  ,          4      V n        VP                  VP                  ,          V n	        V P                  R,          V n
        VP                  V n        RV n        \        P                  ! VP
                  VP                  V P                  ,          RR7      V n        \        P                  ! VP
                  VP                  V P                  ,          RR7      V n        \        P                  ! VP
                  VP                  V P                  ,          RR7      V n        \        P                  ! VP                  V P                  ,          VP
                  RR7      V n        R# )r{   TF©ÚbiasNg      à¿)r3   r4   rš   r›   Úgetattrr;   Únum_attention_headsr{   ry   r‹   rƒ   Úattention_dropoutÚ	is_causalr   ÚLinearÚq_projÚk_projÚv_projÚo_proj)r:   rš   r›   r<   s   &&&€r.   r4   ÚJambaAttention.__init__š   s(  ø€ Ü‰ÑÔØŒØ"ŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÕ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Õ$\ˆÔ!Ø—}‘} dÕ*ˆŒØ!'×!9Ñ!9ˆÔØˆŒÜ—i’i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Õ4^ÐejÔkˆŒÜ—i’i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Õ4^ÐejÔkˆŒÜ—i’i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Õ4^ÐejÔkˆŒÜ—i’i × :Ñ :¸T¿]¹]Õ JÈF×L^ÑL^ÐejÔkˆŽr1   c                óØ   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  S[ P                  R,          3,          /# )r(   r@   r‚   NÚpast_key_valuesr…   r*   )r6   rB   r   r   r   rQ   )r,   r-   s   "€r.   r/   r�   ¨   sk   ø€ ÷ ")ñ ")á—|‘|ð")ñ Ÿ™ tÕ+ð")ñ  �ð	")ñ
 Ñ+Õ,ð")ñ 
‰u�|‰|™UŸ\™\¨DÕ0Ð0Õ	1ñ")r1   c                óì  € VP                   R R p. VORNV P                  N5pV P                  V4      P                  V4      P	                  ^^4      pV P                  V4      P                  V4      P	                  ^^4      pV P                  V4      P                  V4      P	                  ^^4      p	Ve   VP                  W‰V P                  4      w  r‰\        P                  ! V P                  P                  \        4      p
T
! V VVV	V3RV P                  '       g   RMV P                  RV P                   /VB w  r¼VP"                  ! . VORN5!  P%                  4       pV P'                  V4      pW¼3# )Nr„   ç        rƒ   rE   )rR   r{   r¦   Úviewr�   r§   r¨   Úupdater›   r   Úget_interfacerš   Ú_attn_implementationr•   rŠ   r£   rƒ   rw   r�   r©   )r:   r@   r‚   r¬   r…   Úinput_shapeÚhidden_shapeÚquery_statesr‘   r’   Úattention_interfacer”   r“   s   &&&&,        r.   rN   ÚJambaAttention.forward¨   sp  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆàÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—k‘k +Ó.ˆØÐ(Ð(r1   )r£   rš   r{   r¤   r§   r›   r‹   r©   r¦   rƒ   r¨   ©NN©
rV   rW   rX   rY   Ú__doc__r4   rN   rZ   r[   r\   r]   s   @@r.   r—   r—   –   s#   ù‡ € áG÷ló l÷")÷ ")ò ")r1   r—   c                ób   € Ve+   V P                   pWR,          ,          P                  V4      p V # )ze
Tunes out the hidden states for padding tokens, see https://github.com/state-spaces/jamba/issues/66
)ru   ru   N)rF   rG   )r@   r‚   rF   s   && r.   Úapply_mask_to_padding_statesr¼   Í   s3   € ð
 Ò!Ø×#Ñ#ˆØ&¸
Õ)CÕC×GÑGÈÓNˆàÐr1   Úcausal_conv1d_updateÚcausal_conv1dc                óÀ   € V ^8„  d   QhR\         P                  R\         P                  R\        P                  R\        P                  R,          R\        R,          /# )r(   r@   Ú
conv_stater8   r    NÚ
activation©r6   rB   r   r5   Ústr)r,   s   "r.   r/   r/   Ú   sS   € ÷ 'ñ 'Ü—<‘<ð'ä—‘ð'ô �L‰Lð'ô �,‰,˜Õ
ð	'ô
 �d•
ñ'r1   c                 óº  € V P                   w  rVpVP                   R,          p\        P                  ! W.RR7      P                  VP                  4      p	VP                  V	RRV) R13,          4       \        P                  ! W’P                  ^4      V^ VR7      p
V
RRV) R13,          p
Ve   \        V,          ! V
4      p
V
P                  V P                  4      # )é   r_   ru   N)ÚpaddingÚgroupsrE   )
rR   r6   ra   rG   rF   Úcopy_ÚFÚconv1drh   r   )r@   rÀ   r8   r    rÁ   Ú_r;   Úseq_lenÚ	state_lenÚhidden_states_newÚouts   &&&&&      r.   r½   r½   Ù   sÅ   € ð ,×1Ñ1Ñ€A�GØ× Ñ  Õ$€IäŸ	š	 :Ð"=À2ÔF×IÑIÈ&Ï,É,ÓWÐØ×ÑÐ& q¨!¨i¨Z©[Ð'8Õ9Ô:Ü
�(Š(Ð$×&6Ñ&6°qÓ&9¸4ÈÐS^Ô
_€CØ
ˆa��W�H‘IˆoÕ
€CØÒÜ�ZÖ  Ó%ˆØ�6‰6�-×%Ñ%Ó&Ð&r1   Úcausal_conv1d_fnc          	      ó    € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R,          R\        R,          /# )r(   r@   r8   r    NrÁ   rÂ   )r,   s   "r.   r/   r/   î   sE   € ÷ 'ñ 'Ü—<‘<ð'ä�L‰Lð'ô �,‰,˜Õ
ð'ô �d•
ñ	'r1   c                 óV  € V P                   w  rVpVP                   R,          ^,
          p\        P                  ! V P                  VP                  4      VP                  ^4      VVVR7      RRRV13,          p	Ve   \        V,          ! V	4      p	V	P                  V P                  4      # )rÅ   )r8   r    rÆ   rÇ   ru   NrE   )rR   rÉ   rÊ   rG   rF   rh   r   )
r@   r8   r    rÁ   r…   rË   r;   rÌ   rÆ   rÏ   s
   &&&&,     r.   rÐ   rÐ   í   sž   € ð ,×1Ñ1Ñ€A�GØ�l‰l˜2Õ Õ"€Gä
�(Š(Ø×Ñ˜Ÿ™Ó&Ø×Ñ Ó"ØØØôð ˆˆHˆWˆH€nõ€Cð ÒÜ�ZÖ  Ó%ˆØ�6‰6�-×%Ñ%Ó&Ð&r1   Úmamba_inner_fnÚ	mamba_ssmc          "      óŽ  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R,          R\         P                  R\         P                  R\         P                  R\         P                  R,          R	\         P                  R
\         P                  R,          R\         P                  R,          R\         P                  R,          R\         P                  R,          R\        R\         P                  R,          R\         P                  R,          R\         P                  R,          R\        /# )r(   ÚxzÚconv1d_weightÚconv1d_biasNÚx_proj_weightÚdelta_proj_weightÚout_proj_weightÚout_proj_biasÚAÚBÚCÚDÚ
delta_biasÚdelta_softplusÚb_rms_weightÚc_rms_weightÚdt_rms_weightÚb_c_dt_rms_eps)r6   rB   Úboolr+   )r,   s   "r.   r/   r/     s  € ÷ ñ Ü�‰ðä—<‘<ðô —‘ Õ$ðô —<‘<ð	ô
 —|‘|ðô —\‘\ðô —<‘< $Õ&ðô ‡|�|ðô ‡|�|�dÕðô ‡|�|�dÕðô ‡|�|�dÕðô —‘˜tÕ#ðô ðô —,‘, Õ%ðô —,‘, Õ%ðô  —<‘< $Õ&ð!ô" ñ#r1   c                 ó   € R # ©N© )rÖ   r×   rØ   rÙ   rÚ   rÛ   rÜ   rÝ   rÞ   rß   rà   rá   râ   rã   rä   rå   ræ   r…   s   &&&&&&&&&&&&&&&&&,r.   rÓ   rÓ     s   € ñ* r1   Úselective_state_updatec                ón  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R,          R	\         P                  R,          R
\        R\         P                  R,          /
# )r(   Ústater@   ÚdtrÝ   rÞ   rß   rà   NÚdt_biasÚdt_softplusÚz©r6   rB   rç   )r,   s   "r.   r/   r/     sŸ   € ÷ *ñ *Ü�<‰<ð*ä—<‘<ð*ô 	�‰ð*ô ‡|�|ð	*ô
 ‡|�|ð*ô ‡|�|ð*ô ‡|�|�dÕð*ô �\‰\˜DÕ ð*ô ð*ô ‡|�|�dÕñ*r1   c
                 óª  € VP                   pVe"   W'P                  VP                   4      ,           pV'       d   \        P                  ! V4      p\        P
                  ! VP                  4       R,          VP                  4       ,          4      P                  V P                  R7      pVP                  4       R,          VP                  4       R,          ,          pWÑP                  4       R,          ,          pV P                  4       V,          V,           pV P                  VP                  V P                   4      4       \        P                  ! VP                  VP                   4      VP                  R4      4      P                  R4      pVe   VW,          ,           pV	e   V\        P                  ! V	4      ,          pVP                  V4      # )N©Údevice©.N)ru   Nru   rE   )rF   rG   rÉ   Úsoftplusr6   Úexpr+   rõ   rÈ   rŒ   rh   ÚsqueezeÚsilu)rí   r@   rî   rÝ   rÞ   rß   rà   rï   rð   rñ   r…   rL   ÚdAÚdBÚdBxÚ	ssm_staterÏ   s   &&&&&&&&&&,      r.   Úmamba_selective_state_updaterÿ     sE  € ð  ×%Ñ%€KàÒØ—*‘*˜RŸX™XÓ&Õ&ˆßÜ�ZŠZ˜‹^ˆô 
�Š�2—8‘8“:˜iÕ(¨1¯7©7«9Õ4Ó	5×	8Ñ	8ÀÇÁÐ	8Ó	M€Bð 
�‰‹�IÕ	 §¡£¨:Õ!6Õ	6€Bà
×"Ñ"Ó$ YÕ/Õ
/€Cð —‘“ Õ" SÕ(€IØ	‡K�K�	—‘˜UŸ[™[Ó)Ô*ô �,Š,�y—|‘| A§G¡GÓ,¨a¯k©k¸"«oÓ
>×
FÑ
FÀrÓ
J€Cð 	‚}Ø�MÕ%Õ%ˆà‚}Ø”A—F’F˜1“I�oˆà�6‰6�+ÓÐr1   Úselective_scan_fnc                ór  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R,          R\         P                  R,          R	\         P                  R,          R
\        R\        R\        R\        /# )r(   r@   rî   rÝ   rÞ   rß   rà   Nrñ   rá   râ   Úreturn_last_stateÚuse_mambapyÚuse_associative_scanrò   )r,   s   "r.   r/   r/   K  s½   € ÷ hñ hÜ—<‘<ðhä�‰ðhô ‡|�|ðhô ‡|�|ð	hô
 ‡|�|ðhô ‡|�|�dÕðhô ‡|�|�dÕðhô —‘˜tÕ#ðhô ðhô ðhô ðhô ñhr1   c                 ó0  € \        R 4      '       d   ^ RIHp MRp\        4       '       d   ^ RIHp MRpV P                  w  pppV P                  pVe)   WP                  VP                  4      R,          ,           pV'       d   \        P                  ! V4      pVP                  ^^4      pVP                  ^^4      p\        P                  ! VR,          VR,          ,          4      pVR,          VR,          P                  4       ,          pVV R,          P                  4       ,          pV
'       dp   Vel   V! VP                  ^^4      VP                  ^^4      4      pVVP                  R4      ,          P!                  ^4      P                  ^^4      pVR,          pEM¬V'       dÈ   VeÄ   \#        4       '       d´   \%        4       '       g¤   R pVP&                  P(                  R9   d   RMRpV! VVV3^VR7      w  pp\        P*                  ! VP                  ^^4      P                  V4      VP                  R4      4      pVP!                  R4      P                  ^^4      pVR,          pMÝ\        P,                  ! VVVP                  R,          VV P&                  R	7      p. p\/        V4       F~  pVRRV3,          V,          VRRV3,          ,           p\        P*                  ! VP                  V4      VRVR3,          P                  R4      4      pVP1                  VR,          4       K€  	  \        P2                  ! VRR
7      pVe   VWR,          ,          ,           pVe   V\        P4                  ! V4      ,          pV	'       d   VV3# V# )z2.9.0)Úassociative_scanN)Úpscanru   c                 ó@   € V w  r#Vw  rEW$,          WC,          V,           3# ré   rê   )ÚleftÚrightÚa_leftÚb_leftÚa_rightÚb_rights   &&    r.   Ú
combine_fnÚ(mamba_selective_scan.<locals>.combine_fnƒ  s'   € Ø!‰NˆFØ$ÑˆGØÕ# WÕ%5¸Õ%?Ð?Ð?r1   Ú	pointwiseÚgeneric)r`   Úcombine_mode©rF   rõ   r_   rö   )Nru   Nru   )ru   ru   ru   N)ru   Nru   ru   rE   )ru   rE   )ÚcudaÚxpu)ru   ru   rE   )ru   ru   r   )Nru   N)r   Ú(torch._higher_order_ops.associative_scanr  r   Úmambapy.pscanr  rR   rF   rG   rÉ   r÷   r�   r6   rø   r+   rh   rù   r   r   rõ   ÚtyperŒ   ÚzerosÚrangeÚappendÚstackrú   )r@   rî   rÝ   rÞ   rß   rà   rñ   rá   râ   r  r  r  r…   r  r  Ú
batch_sizeÚintermediate_sizerÌ   rL   Ú
discrete_AÚ
discrete_BÚdeltaB_uÚ
all_statesÚscan_outputrþ   r  r  rË   Úscan_outputsÚindexs   &&&&&&&&&&&&,                 r.   Úmamba_selective_scanr'  J  s  € ô" ! ×)Ò)ÞMàÐä×ÒÞ'àˆà-:×-@Ñ-@Ñ*€JÐ! 7Ø×%Ñ%€KàÒØ—-‘- §¡Ó)¨)Õ4Õ4ˆßÜ�ZŠZ˜‹^ˆð 	
�‰�A�qÓ€AØ	�‰�A�qÓ€Aô —’˜1Ð-Õ.°°MÕ1BÕBÓC€JØ�MÕ" Q }Õ%5×%;Ñ%;Ó%=Õ=€JØ˜M¨-Õ8×>Ñ>Ó@Õ@€Hç�uÒ(Ù˜:×/Ñ/°°1Ó5°x×7IÑ7IÈ!ÈQÓ7OÓPˆ
à! A§K¡K°£OÕ3×<Ñ<¸QÓ?×IÑIÈ!ÈQÓOˆØ˜uÕ%Š	÷ 	ØÒ(ä%×'Ò'Ô0H×0JÒ0Jò	@ð
 '1×&7Ñ&7×&<Ñ&<ÀÔ&O‘{ÐU^ˆÙ(ØØ˜Ð"ØØ%ô	
‰ˆˆ:ô —l’l :×#7Ñ#7¸¸1Ó#=×#@Ñ#@ÀÓ#MÈqÏ{É{Ð[]ËÓ_ˆØ!×)Ñ)¨"Ó-×7Ñ7¸¸1Ó=ˆØ˜xÕ(‰	ô —K’KØØØ�G‰G�B�KØØ ×'Ñ'ô
ˆ	ð ˆÜ˜7–^ˆEà" 1 a¨ ;Õ/°)Õ;¸hÀqÈ!ÈUÀ{Õ>SÕSˆIô  Ÿ,š, y§|¡|°KÓ'@À!ÀAÀuÈaÀKÅ.×BZÑBZÐ[]ÓB^Ó_ˆKØ×Ñ ¨GÕ 4Ö5ñ $ô —k’k ,°BÔ7ˆà‚}Ø! M°mÕ4DÕ$DÕDˆà‚}Ø!¤A§F¢F¨1£IÕ-ˆçØ˜IÐ%Ð%àÐr1   c                   ór   a a€ ] tR tRt oRtV3R lV 3R llt]! RR.4      R
V3R lR ll4       tR	tVt	V ;t
# )ÚJambaMambaMixeri¶  uo  
Compute âˆ†, A, B, C, and D the state space parameters and compute the `contextualized_states`.
A, D are input independent (see Mamba paper [1] Section 3.5.2 "Interpretation of A" for why A isn't selective)
âˆ†, B, C are input-dependent (this is a key difference between Mamba and the linear time invariant S4,
and is why Mamba is called **selective** state spaces)
c                ó    <€ V ^8„  d   QhRS[ /# ©r(   rš   r"   )r,   r-   s   "€r.   r/   ÚJambaMambaMixer.__annotate__Á  s   ø€ ÷ ,@ñ ,@™{ñ ,@r1   c           	     ó  <€ \         SV `  4        Wn        W n        VP                  V n        VP
                  V n        VP                  V n        VP                  VP                  ,          V n
        VP                  V n        VP                  V n        VP                  V n        \"        P$                  ! V P                  V P                  V P                  V P                  V P                  V P                  ^,
          R7      V n        VP(                  V n        \,        VP(                  ,          V n        \"        P0                  ! V P                  V P                  ^,          V P                   R7      V n        \"        P0                  ! V P                  V P                  V P                  ^,          ,           RR7      V n        \"        P0                  ! V P                  V P                  RR7      V n        \8        P:                  ! ^V P                  ^,           4      R,          pVP=                  V P                  R4      P?                  4       p\"        P@                  ! \8        PB                  ! V4      4      V n"        \"        P@                  ! \8        PF                  ! V P                  4      4      V n$        \"        P0                  ! V P                  V P                  V P                   R7      V n%        \M        V P                  VPN                  R7      V n(        \M        V P                  VPN                  R7      V n)        \M        V P                  VPN                  R7      V n*        VPV                  V n+        VPX                  V n,        R# )	rÅ   )Úin_channelsÚout_channelsr    Úkernel_sizerÇ   rÆ   rŸ   FTN©r)   ©Nru   rE   )-r3   r4   rš   r›   r;   Úmamba_d_stateÚssm_state_sizeÚmamba_d_convÚconv_kernel_sizeÚmamba_expandr  Úmamba_dt_rankÚtime_step_rankÚmamba_conv_biasÚuse_conv_biasÚmamba_proj_biasÚuse_biasr   ÚConv1drÊ   Ú
hidden_actrÁ   r   Úactr¥   Úin_projÚx_projÚdt_projr6   Úarangerv   r�   r5   ÚlogÚA_logr7   rà   Úout_projr&   Úrms_norm_epsÚdt_layernormÚb_layernormÚc_layernormr  r  )r:   rš   r›   rÝ   r<   s   &&& €r.   r4   ÚJambaMambaMixer.__init__Á  s}  ø€ Ü‰ÑÔØŒØ"ŒØ!×-Ñ-ˆÔØ$×2Ñ2ˆÔØ &× 3Ñ 3ˆÔØ!'×!4Ñ!4°v×7IÑ7IÕ!IˆÔØ$×2Ñ2ˆÔØ#×3Ñ3ˆÔØ×.Ñ.ˆŒÜ—i’iØ×.Ñ.Ø×/Ñ/Ø×#Ñ#Ø×-Ñ-Ø×)Ñ)Ø×)Ñ)¨AÕ-ô
ˆŒð !×+Ñ+ˆŒÜ˜&×+Ñ+Õ,ˆŒô —y’y ×!1Ñ!1°4×3IÑ3IÈAÕ3MÐTX×TaÑTaÔbˆŒä—i’i × 6Ñ 6¸×8KÑ8KÈd×NaÑNaÐdeÕNeÕ8eÐlqÔrˆŒä—y’y ×!4Ñ!4°d×6LÑ6LÐSWÔXˆŒô �LŠL˜˜D×/Ñ/°!Õ3Ó4°WÕ=ˆØ�H‰H�T×+Ñ+¨RÓ0×;Ñ;Ó=ˆä—\’\¤%§)¢)¨A£,Ó/ˆŒ
Ü—’œeŸjšj¨×)?Ñ)?Ó@ÓAˆŒÜŸ	š	 $×"8Ñ"8¸$×:JÑ:JÐQU×Q^ÑQ^Ô_ˆŒä(¨×)<Ñ)<À&×BUÑBUÔVˆÔÜ'¨×(;Ñ(;À×ATÑATÔUˆÔÜ'¨×(;Ñ(;À×ATÑATÔUˆÔà!×-Ñ-ˆÔØ$*×$?Ñ$?ˆÖ!r1   rÊ   rC  c                óp   <€ V ^8„  d   QhRS[ P                  RS[R,          RS[ P                  R,          /# )r(   r@   Úcache_paramsNr‚   )r6   rB   r   )r,   r-   s   "€r.   r/   r,  ð  s?   ø€ ÷ s%ñ s%á—|‘|ðs%ñ ˜d•lðs%ñ Ÿ™ tÕ+ñ	s%r1   c                ó¼	  € VP                   ^,          pVP                  pVRJ;'       d    VP                  V P                  4      p\	        W4      pV P                  V4      P                  ^^4      p\        P                  ! V P                  P                  4       4      ) p	VP                  ^^R7      w  r«V'       d]   VP                  V P                  ,          P                  ^ ,          pVP                  V P                  ,          P                  ^ ,          pV'       d‡   V^8X  d€   VP                  V P                  ,          P                  '       gS   \!        V
XV P"                  P$                  P'                  ^4      V P"                  P(                  V P*                  R7      p
M�Ve)   VP-                  V
V P                  V P.                  R7      p
\1        V
V P"                  P$                  P'                  ^4      V P"                  P(                  3RV P*                  /VB p
Ve   V
RRV) R13,          p
\	        V
P                  ^^4      V4      p
\        P2                  ! V P5                  V
4      V P6                  V P8                  V P8                  .RR7      w  rïpV P;                  V4      pV P=                  V4      pV P?                  V4      p\A        V PB                  R4      '       d>   V PE                  V4      V PD                  P(                  ,
          P                  ^^4      pM-V PD                  P$                  VP                  ^^4      ,          pV PD                  P(                  e%   V PD                  P(                  P                  4       MRpV'       dj   V^8X  dc   \G        XV
P                  ^^4      R,          VR,          V	VR,          VR,          V PH                  VR,          VRR	7
      PK                  R4      pM£VRJp\M        V
P                  ^^4      VV	VP                  ^^4      VP                  ^^4      V PH                  P                  4       VVRVV PN                  V PP                  R
7      pV'       d#   Vw  ppVPS                  VV P                  4       MTpV PU                  VP                  ^^4      PW                  V4      4      pV# )rÅ   Nr_   )rÁ   )r6  rÁ   ru   Ú_is_quantizedT)rñ   rï   rð   )rà   rñ   rá   râ   r  r  r  rE   ).r   )ru   r   ),rR   rF   Úhas_previous_stater›   r¼   rA  r�   r6   rø   rF  r+   ÚchunkÚlayersÚconv_statesÚrecurrent_statesÚrecord_pastr½   rÊ   r8   rù   r    rÁ   Úupdate_conv_stater6  rÐ   ÚsplitrB  r9  r4  rI  rJ  rK  Úhasattrrš   rC  rÿ   rà   rh   r'  r  r  Úupdate_recurrent_staterG  rG   )r:   r@   rN  r‚   r…   rÌ   rF   Úuse_precomputed_statesÚprojected_statesrÝ   Úhidden_states_B_CÚgaterÀ   Úrecurrent_stateÚ	time_steprÞ   rß   Útime_proj_biasr$  Úoutput_final_stateÚscan_resultÚfinal_stateÚcontextualized_statess   &&&&,                  r.   rN   ÚJambaMambaMixer.forwardï  s2  € ð  ×%Ñ% aÕ(ˆØ×#Ñ#ˆØ!-°TÐ!9×!mÐ!m¸l×>]Ñ>]Ð^b×^lÑ^lÓ>mÐô 5°]ÓSˆØŸ<™<¨Ó6×@Ñ@ÀÀAÓFÐô �YŠY�t—z‘z×'Ñ'Ó)Ó*Ð*ˆà"2×"8Ñ"8¸ÀÐ"8Ó"BÑÐç!Ø%×,Ñ,¨T¯^©^Õ<×HÑHÈÕKˆJØ*×1Ñ1°$·.±.ÕA×RÑRÐSTÕUˆO÷ " g°¤l¸<×;NÑ;NÈtÏ~É~Õ;^×;j×;jÐ;jÜ 4Ø!ØØ—‘×"Ñ"×*Ñ*¨1Ó-Ø—‘× Ñ ØŸ?™?ô!Ñð Ò'Ø$0×$BÑ$BØ%Ø—N‘NØ%)×%:Ñ%:ð %Có %Ð!ô !1Ø!Ø—‘×"Ñ"×*Ñ*¨1Ó-Ø—‘× Ñ ñ!ð  Ÿ?™?ð	!ð
 ñ!Ðð Ò'Ø$5°a¸¸W¸H¹I°oÕ$FÐ!ô 9Ð9J×9TÑ9TÐUVÐXYÓ9ZÐ\jÓkÐÜŸ+š+Ø�K‰KÐ)Ó*Ø× Ñ  $×"5Ñ"5°t×7JÑ7JÐKØô
‰ˆ	�að ×%Ñ% iÓ0ˆ	Ø×Ñ˜QÓˆØ×Ñ˜QÓˆô �4—;‘; ×0Ò0ØŸ™ iÓ0°4·<±<×3DÑ3DÕD×OÑOÐPQÐSTÓU‰IàŸ™×+Ñ+¨i×.AÑ.AÀ!ÀQÓ.GÕGˆIØ6:·l±l×6GÑ6GÒ6S˜Ÿ™×*Ñ*×0Ñ0Ô2ÐY]ˆ÷ " g°¤lÜ6ØØ!×+Ñ+¨A¨qÓ1°&Õ9Ø˜&Õ!ØØ�$•Ø�$•Ø—‘Ø�v•,Ø&Ø ô÷ ‰i˜‹mñ ð ".°TÐ!9ÐÜ.Ø!×+Ñ+¨A¨qÓ1ØØØ—‘˜A˜qÓ!Ø—‘˜A˜qÓ!Ø—&‘&—,‘,“.ØØ)Ø#Ø"4Ø ×,Ñ,Ø%)×%>Ñ%>ôˆK÷ "Ø+6Ñ(�˜[Ø×3Ñ3°KÀÇÁÕPà)�ð !%§¡¨k×.CÑ.CÀAÀqÓ.I×.LÑ.LÈUÓ.SÓ TÐØ$Ð$r1   )rF  rà   r@  rÁ   rJ  rK  rš   rÊ   r6  rI  rC  r;   rA  r  r›   rG  r4  r9  r  r=  r;  r  rB  r¸   )rV   rW   rX   rY   rº   r4   r   rN   rZ   r[   r\   r]   s   @@r.   r)  r)  ¶  sC   ù‡ € ñ÷,@ó ,@ñ\ ˜X yÐ1Ó2÷s%ñ s%ó 3÷s%ð s%r1   r)  c                   ó8   a a€ ] tR tRt oV 3R ltR tRtVtV ;t# )ÚJambaMLPif  c                óØ  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        \
        P                  ! V P                  V P                  R R7      V n        \
        P                  ! V P                  V P                  R R7      V n        \
        P                  ! V P                  V P                  R R7      V n	        \        VP                  ,          V n        R# ©FrŸ   N)r3   r4   rš   r;   r  r   r¥   Ú	gate_projÚup_projÚ	down_projr   r?  Úact_fn©r:   rš   r<   s   &&€r.   r4   ÚJambaMLP.__init__g  s¤   ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔÜŸš 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆŒÜ˜V×.Ñ.Õ/ˆŽr1   c                ó�   € V P                  V P                  V P                  V4      4      V P                  V4      ,          4      pV# ré   )rm  rn  rk  rl  )r:   rb   rm  s   && r.   rN   ÚJambaMLP.forwardq  s6   € Ø—N‘N 4§;¡;¨t¯~©~¸aÓ/@Ó#AÀDÇLÁLÐQRÃOÕ#SÓTˆ	ØÐr1   )rn  rš   rm  rk  r;   r  rl  ©	rV   rW   rX   rY   r4   rN   rZ   r[   r\   r]   s   @@r.   rh  rh  f  s   ù‡ € õ0÷ò r1   rh  c                   óT   a a€ ] tR tRt oRtV3R lV 3R lltV3R lR ltRtVtV ;t	# )ÚJambaExpertsiv  z2Collection of expert weights stored as 3D tensors.c                ó    <€ V ^8„  d   QhRS[ /# r+  r"   )r,   r-   s   "€r.   r/   ÚJambaExperts.__annotate__z  s   ø€ ÷ 0ñ 0™{ñ 0r1   c                ó  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  V n        \        P                  ! \        P                  ! V P                  ^V P                  ,          V P
                  4      4      V n        \        P                  ! \        P                  ! V P                  V P
                  V P                  4      4      V n        \        VP                  ,          V n        R# )r(   N)r3   r4   Únum_local_expertsÚnum_expertsr;   Ú
hidden_dimr  Úintermediate_dimr   r5   r6   ÚemptyÚgate_up_projrm  r   r?  rn  ro  s   &&€r.   r4   ÚJambaExperts.__init__z  s­   ø€ Ü‰ÑÔØ!×3Ñ3ˆÔØ ×,Ñ,ˆŒØ &× 8Ñ 8ˆÔÜŸLšL¬¯ª°T×5EÑ5EÀqÈ4×K`ÑK`ÕG`Ðbf×bqÑbqÓ)rÓsˆÔÜŸš¤e§k¢k°$×2BÑ2BÀDÇOÁOÐUY×UjÑUjÓ&kÓlˆŒÜ˜V×.Ñ.Õ/ˆŽr1   c                ó‚   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[ P                  /# )r(   r@   Útop_k_indexÚtop_k_weightsr*   rA   )r,   r-   s   "€r.   r/   rw  ƒ  sC   ø€ ÷ #ñ #á—|‘|ð#ñ —\‘\ð#ñ —|‘|ð	#ñ
 
�‰ñ#r1   c                óÆ  € \         P                  ! V4      p\         P                  ! 4       ;_uu_ 4        \         P                  P                  P                  W P                  R 7      pVP                  ^^^ 4      p\         P                  ! VP                  RR7      ^ 4      P                  4       pRRR4       X EF  pV^ ,          pWpP                  8X  d   K  \         P                  ! XV,          4      w  r‰W,          p
\        P                  P                  W P                  V,          4      P                  ^RR7      w  r¼V P                  V4      V,          p\        P                  P                  WÐP                   V,          4      pWÓW˜R3,          ,          pVP#                  ^ W�P%                  VP&                  4      4       EK  	  V#   + '       g   i     EL'; i))Únum_classesr_   NrE   )rE   éþÿÿÿ)r6   Ú
zeros_likeÚno_gradr   rŽ   Úone_hotrz  ÚpermuteÚgreaterÚsumÚnonzeroÚwhereÚlinearr~  rR  rn  rm  Ú
index_add_rG   rF   )r:   r@   r�  r‚  Úfinal_hidden_statesÚexpert_maskÚ
expert_hitÚ
expert_idxÚ	top_k_posÚ	token_idxÚcurrent_stater^  ÚupÚcurrent_hidden_statess   &&&&          r.   rN   ÚJambaExperts.forwardƒ  ss  € ô $×.Ò.¨}Ó=ÐÜ�]Š]�_�_ÜŸ(™(×-Ñ-×5Ñ5°k×O_ÑO_Ð5Ó`ˆKØ%×-Ñ-¨a°°AÓ6ˆKÜŸš {§¡¸8 Ó'DÀaÓH×PÑPÓRˆJ÷ ô
 %ˆJØ# A�ˆJØ×-Ñ-Ô-ÙÜ#(§;¢;¨{¸:Õ/FÓ#GÑ ˆIØ)Õ4ˆMÜ—}‘}×+Ñ+¨M×;LÑ;LÈZÕ;XÓY×_Ñ_Ð`aÐgiÐ_Ój‰HˆDØ$(§K¡K°Ó$5¸Õ$:Ð!Ü$&§M¡M×$8Ñ$8Ð9N×P^ÑP^Ð_iÕPjÓ$kÐ!Ø$9È)Ð`dÐJdÕ<eÕ$eÐ!Ø×*Ñ*¨1¨i×9QÑ9QÐRe×RkÑRkÓ9l×mñ %ð #Ð"÷# �_ˆ_ús   ´A=GÇG 	)rn  rm  r~  r{  r|  rz  r¹   r]   s   @@r.   ru  ru  v  s!   ù‡ € á<÷0ó 0÷#÷ #ð #r1   ru  c                   óZ   a a€ ] tR tRt oRtV3R lV 3R lltR tV3R lR ltRtVt	V ;t
# )	ÚJambaSparseMoeBlockiž  a¤  
This implementation is
strictly equivalent to standard MoE with full capacity (no
dropped tokens). It's faster since it formulates MoE operations
in terms of block-sparse operations to accommodate imbalanced
assignments of tokens to experts, whereas standard MoE either
(1) drop tokens at the cost of reduced performance or (2) set
capacity factor to number of experts and thus waste computation
and memory on padding.
c                ó    <€ V ^8„  d   QhRS[ /# r+  r"   )r,   r-   s   "€r.   r/   Ú JambaSparseMoeBlock.__annotate__ª  s   ø€ ÷ ,ñ ,™{ñ ,r1   c                ó0  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  V n        VP                  V n        \        P                  ! V P                  V P                  R R7      V n        \        V4      V n        R# rj  )r3   r4   r;   r{  r  Úffn_dimrz  Únum_experts_per_tokÚtop_kr   r¥   Úrouterru  Úexpertsro  s   &&€r.   r4   ÚJambaSparseMoeBlock.__init__ª  sm   ø€ Ü‰ÑÔØ ×,Ñ,ˆŒØ×/Ñ/ˆŒØ!×-Ñ-ˆÔØ×/Ñ/ˆŒ
ä—i’i §¡°×1AÑ1AÈÔNˆŒÜ# FÓ+ˆŽr1   c                óö   € \         P                  P                  P                  VR\         P                  R7      p\         P
                  ! W0P                  RR7      w  rEWTP                  VP                  4      3# )rÅ   rˆ   r_   rE   )	r6   r   rŽ   r�   r+   Útopkr¡  rG   rF   )r:   r@   Úrouter_logitsÚrouting_weightsr‚  r�  s   &&&   r.   Úroute_tokens_to_expertsÚ+JambaSparseMoeBlock.route_tokens_to_experts´  s\   € ÜŸ(™(×-Ñ-×5Ñ5°mÈÔSX×S^ÑS^Ð5Ó_ˆÜ%*§Z¢Z°ÇÁÐQSÔ%TÑ"ˆØ×,Ñ,¨]×-@Ñ-@ÓAÐAÐAr1   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r?   rA   )r,   r-   s   "€r.   r/   r�  ¹  s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r1   c                óØ   € VP                   w  r#pVP                  RV4      pV P                  V4      pV P                  W4      w  rgV P	                  WV4      pVP                  W#V4      pV# )rÅ   rE   )rR   r¯   r¢  r©  r£  rw   )r:   r@   r  Úsequence_lengthr{  r§  r�  r‚  s   &&      r.   rN   ÚJambaSparseMoeBlock.forward¹  sm   € Ø2?×2EÑ2EÑ/ˆ
 ZØ%×*Ñ*¨2¨zÓ:ˆØŸ™ MÓ2ˆØ%)×%AÑ%AÀ-Ó%_Ñ"ˆØŸ™ ]ÀÓOˆØ%×-Ñ-¨jÈ:ÓVˆØÐr1   )r£  rŸ  r{  rz  r¢  r¡  )rV   rW   rX   rY   rº   r4   r©  rN   rZ   r[   r\   r]   s   @@r.   r›  r›  ž  s)   ù‡ € ñ	÷,ó ,òB÷
÷ ð r1   r›  c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚJambaAttentionDecoderLayeriÃ  c                ó&   <€ V ^8„  d   QhRS[ RS[/# r™   rœ   )r,   r-   s   "€r.   r/   Ú'JambaAttentionDecoderLayer.__annotate__Ä  s   ø€ ÷ Zñ Z™{ð Z±sñ Zr1   c                ój  <€ \         SV `  4        VP                  '       d   VP                  V,          M^p\        W4      V n        V^8”  d   \
        M\        pV! V4      V n        \        VP                  VP                  R7      V n        \        VP                  VP                  R7      V n        R# )rÅ   r1  N)r3   r4   Úlayers_num_expertsr—   Ú	self_attnr›  rh  Úfeed_forwardr&   r;   rH  Úinput_layernormÚpre_ff_layernorm©r:   rš   r›   rz  Úffn_layer_classr<   s   &&&  €r.   r4   Ú#JambaAttentionDecoderLayer.__init__Ä  s‰   ø€ Ü‰ÑÔØ>D×>W×>WÐ>W�f×/Ñ/°	Ö:Ð]^ˆÜ'¨Ó:ˆŒà1<¸q´Õ-ÄhˆÙ+¨FÓ3ˆÔÜ+¨F×,>Ñ,>ÀF×DWÑDWÔXˆÔÜ ,¨V×-?Ñ-?ÀV×EXÑEXÔ YˆÖr1   c                óÜ   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[,          RS[ P                  /# )	r(   r@   r‚   NÚposition_idsr¬   Ú	use_cacher…   r*   )r6   rB   Ú
LongTensorr   rç   r   r   ÚFloatTensor)r,   r-   s   "€r.   r/   r²  Î  s}   ø€ ÷ ñ á—|‘|ðñ Ÿ™ tÕ+ðñ ×&Ñ&¨Õ-ð	ñ
  �ðñ ˜$•;ðñ Ñ+Õ,ðñ 
×	Ñ	ñr1   c                óÐ   € TpV P                  V4      pV P                  ! RR VRVRVRVRV/VB w  rWq,           pTpV P                  V4      pV P                  V4      pWq,           pV# )r@   r‚   r½  r¬   r¾  rê   )r·  rµ  r¸  r¶  )	r:   r@   r‚   r½  r¬   r¾  r…   ÚresidualrË   s	   &&&&&&,  r.   rN   Ú"JambaAttentionDecoderLayer.forwardÎ  sŸ   € ð !ˆØ×,Ñ,¨]Ó;ˆØŸ>š>ñ 
Ø'ð
à)ð
ð &ð
ð ,ð	
ð
  ð
ð ñ
Ñˆð !Õ0ˆØ ˆØ×-Ñ-¨mÓ<ˆØ×)Ñ)¨-Ó8ˆØ Õ0ˆØÐr1   )r¶  r·  r¸  rµ  )NNNFrs  r]   s   @@r.   r°  r°  Ã  s    ù‡ € ÷Zó Z÷÷ ò r1   r°  c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚJambaMambaDecoderLayerié  c                ó&   <€ V ^8„  d   QhRS[ RS[/# r™   rœ   )r,   r-   s   "€r.   r/   Ú#JambaMambaDecoderLayer.__annotate__ê  s   ø€ ÷ Zñ Z™{ð Z±sñ Zr1   c                ól  <€ \         SV `  4        VP                  '       d   VP                  V,          M^p\        WR7      V n        V^8”  d   \
        M\        pV! V4      V n        \        VP                  VP                  R7      V n        \        VP                  VP                  R7      V n        R# )rÅ   )rš   r›   r1  N)r3   r4   r´  r)  Úmambar›  rh  r¶  r&   r;   rH  r·  r¸  r¹  s   &&&  €r.   r4   ÚJambaMambaDecoderLayer.__init__ê  s‰   ø€ Ü‰ÑÔØ>D×>W×>WÐ>W�f×/Ñ/°	Ö:Ð]^ˆÜ$¨FÔHˆŒ
Ø1<¸q´Õ-ÄhˆÙ+¨FÓ3ˆÔÜ+¨F×,>Ñ,>ÀF×DWÑDWÔXˆÔÜ ,¨V×-?Ñ-?ÀV×EXÑEXÔ YˆÖr1   c                óÈ   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[S[,          RS[ P                  /# )r(   r@   r‚   Nr½  r¬   r…   r*   )r6   rB   r¿  r   r   r   rÀ  )r,   r-   s   "€r.   r/   rÇ  ó  so   ø€ ÷ ñ á—|‘|ðñ Ÿ™ tÕ+ðñ ×&Ñ&¨Õ-ð	ñ
  �ðñ Ñ+Õ,ðñ 
×	Ñ	ñr1   c                ó¼   € TpV P                  V4      pV P                  VVVR 7      pWa,           pTpV P                  V4      pV P                  V4      pWa,           pV# ))r@   rN  r‚   )r·  rÉ  r¸  r¶  )r:   r@   r‚   r½  r¬   r…   rÂ  s   &&&&&, r.   rN   ÚJambaMambaDecoderLayer.forwardó  sr   € ð !ˆØ×,Ñ,¨]Ó;ˆØŸ
™
Ø'Ø(Ø)ð #ó 
ˆð
 !Õ0ˆØ ˆØ×-Ñ-¨mÓ<ˆØ×)Ñ)¨-Ó8ˆØ Õ0ˆØÐr1   )r¶  r·  rÉ  r¸  )NNNrs  r]   s   @@r.   rÅ  rÅ  é  s    ù‡ € ÷Zó Z÷÷ ò r1   rÅ  c            
       óÄ   a a€ ] tR tRt oRtRtRR.tR.tRtRt	Rt
RtR]].R]R	]! ]P"                  R
R7      /t]P(                  ! 4       V 3R l4       tV3R ltRtVtV ;t# )ÚJambaPreTrainedModeli
  ÚmodelTr°  rÅ  r¬   r@   Ú
attentionsr§  r¢  )Ú
layer_namec                ó¾  <€ \         SV `  V4       \        V\        4      '       d±   \        P
                  ! ^VP                  ^,           4      R,          pVP                  VP                  R4      P                  4       p\        P                  ! VP                  \        P                  ! V4      4       \        P                  ! VP                  4       R# \        V\         4      '       dq   \        P"                  ! VP$                  RV P&                  P(                  R7       \        P"                  ! VP*                  RV P&                  P(                  R7       R# R# )rÅ   Nr®   )rJ   Ústdr2  rE   )r3   Ú_init_weightsÚ
isinstancer)  r6   rD  r4  rv   r  r�   ÚinitrÈ   rF  rE  Úones_rà   ru  Únormal_r~  rš   Úinitializer_rangerm  )r:   r~   rÝ   r<   s   && €r.   rÕ  Ú"JambaPreTrainedModel._init_weights  sØ   ø€ ä‰Ñ˜fÔ%Ü�fœo×.Ò.Ü—’˜Q × 5Ñ 5¸Õ 9Ó:¸7ÕCˆAØ—‘˜×1Ñ1°2Ó6×AÑAÓCˆAÜ�JŠJ�v—|‘|¤U§Y¢Y¨q£\Ô2Ü�JŠJ�v—x‘xÖ Ü˜¤×-Ò-Ü�LŠL˜×,Ñ,°3¸D¿K¹K×<YÑ<YÕZÜ�LŠL˜×)Ñ)°¸¿¹×9VÑ9V×Wñ .r1   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r+  r"   )r,   r-   s   "€r.   r/   Ú!JambaPreTrainedModel.__annotate__
  s   ø‡ ‚ ÙÑò r1   rê   )rV   rW   rX   rY   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_is_statefulÚ_can_compile_fullgraphr°  rÅ  r—   r    r   r¥   Ú_can_record_outputsr6   r‡  rÕ  Ú__annotate_func__rZ   r[   r\   r]   s   @@r.   rÏ  rÏ  
  sˆ   ù‡ € àÐØ&*Ð#Ø5Ð7OÐPÐØ#4Ð"5ÐØÐØ€NØ€LØ!ÐàÐ4Ð6LÐMØ�nØ™¨¯	©	¸hÔGðÐð ‡]‚]ƒ_ô	Xó ð	X÷# … r1   rÏ  Ú	attentionrÉ  c                   ór   a a€ ] tR tRt oV3R lV 3R llt]]]RV3R lR ll4       4       4       tRt	Vt
V ;t# )Ú
JambaModeli*  c                ó    <€ V ^8„  d   QhRS[ /# r+  r"   )r,   r-   s   "€r.   r/   ÚJambaModel.__annotate__,  s   ø€ ÷ ñ ™{ñ r1   c                ó8  <€ \         SV `  V4       VP                  V n        VP                  V n        \
        P                  ! VP                  VP                  V P                  4      V n        . p\        VP                  4       F9  p\        VP                  V,          ,          pVP                  V! WR 7      4       K;  	  \
        P                  ! V4      V n        \!        VP                  VP"                  R7      V n        RV n        V P)                  4        R# ))r›   r1  FN)r3   r4   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingr;   Úembed_tokensr  Únum_hidden_layersÚALL_DECODER_LAYER_TYPESÚlayers_block_typer  Ú
ModuleListrS  r&   rH  Úfinal_layernormÚgradient_checkpointingÚ	post_init)r:   rš   Údecoder_layersÚiÚlayer_classr<   s   &&   €r.   r4   ÚJambaModel.__init__,  sÔ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸLšL¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔØˆÜ�v×/Ñ/Ö0ˆAÜ1°&×2JÑ2JÈ1Õ2MÕNˆKØ×!Ñ!¡+¨fÔ"BÖCñ 1ô —m’m NÓ3ˆŒä+¨F×,>Ñ,>ÀF×DWÑDWÔXˆÔà&+ˆÔ#à�‰Ör1   c                óþ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[S[,          R	S[/# )
r(   Ú	input_idsNr‚   r½  r¬   Úinputs_embedsr¾  r…   r*   )	r6   r¿  rB   r   rÀ  rç   r   r   r   )r,   r-   s   "€r.   r/   rì  A  s“   ø€ ÷ 6
ñ 6
á×#Ñ# dÕ*ð6
ñ Ÿ™ tÕ+ð6
ñ ×&Ñ&¨Õ-ð	6
ñ
  �ð6
ñ ×(Ñ(¨4Õ/ð6
ñ ˜$•;ð6
ñ Ñ+Õ,ð6
ñ 
 ñ6
r1   c                óâ  € VR J VR J,          '       d   \        R4      hVf   V P                  V4      pV'       d   Vf   \        V P                  R7      pVfb   Ve   VP	                  4       M^ p\
        P                  ! VP                  ^,          VP                  R7      V,           pVP                  ^ 4      p\        T;p	\        4      '       g/   RV P                  RVRVRVRV/p
R	\        R/ V
B R
\        R/ V
B /p	Tp\        V P                  4       F7  w  rÍV! V3RW�P                  P                   V,          ,          RVRVRV/VB pK9  	  V P#                  V4      p\%        VVR7      # )Nz:You must specify exactly one of input_ids or inputs_embeds)rš   rô   rš   r   r‚   r¬   r½  Úfull_attentionÚlinear_attentionr¾  )Úlast_hidden_stater¬   rê   )Ú
ValueErrorrò  r   rš   Úget_seq_lengthr6   rD  rR   rõ   rh   rÖ  Údictr   r   Ú	enumeraterS  Úlayer_typesr÷  r   )r:   rÿ  r‚   r½  r¬   r   r¾  r…   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsr@   rû  Údecoder_layers   &&&&&&&,      r.   rN   ÚJambaModel.forward>  s•  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ Ø ×-Ñ-¨iÓ8ˆMç˜Ò0Ü*°$·+±+Ô>ˆOàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLä°Ð?Ð-Ä×FÒFð ˜$Ÿ+™+Ø Ø  .Ø! ?Ø ðˆKð !Ô"4Ñ"C°{Ñ"CØ"Ô$CÑ$RÀkÑ$Rð#Ðð &ˆÜ )¨$¯+©+Ö 6ÑˆAÙ)Øñà2·;±;×3JÑ3JÈ1Õ3MÕNðð *ðð !0ð	ð
 $ðð ñŠMñ !7ð ×,Ñ,¨]Ó;ˆä%Ø+Ø+ô
ð 	
r1   )rò  r÷  rø  rS  rï  rð  )NNNNNN)rV   rW   rX   rY   r4   r   r!   r   rN   rZ   r[   r\   r]   s   @@r.   rê  rê  *  s=   ù‡ € ÷ó ð$  ØØ÷6
ñ 6
ó ó ó  ÷6
ð 6
r1   rê  c          	      ó  € V ^8„  d   QhR\         P                  \        \         P                  ,          ,          R,          R\        R,          R\         P                  R,          R\         P                  \        ,          /# )r(   Úgate_logitsNrz  r‚   r*   )r6   rB   rQ   rs   )r,   s   "r.   r/   r/   z  sc   € ÷ B&ñ B&Ü—‘¤¤e§l¡lÕ 3Õ3°dÕ:ðB&ä�t•ðB&ô —L‘L 4Õ'ð	B&ô
 ‡\�\”CÕñB&r1   c           	     ó  € V e   \        V \        4      '       g   ^ # V ^ ,          P                  p\        P                  ! V\        P
                  VR7      p\        P                  ! V\        P
                  VR7      pRpVe1   VP                  R4      P                  V\        P
                  R7      pV  EFˆ  p	\        P                  P                  P                  V	P                  V4      RR7      p
\        P                  ! W¢RR7      w  r¼Vf~   V\        P                  ! VP                  R4      VR7      P                  4       ,           pWjP                  4       P                  ^ R7      ,           pWzP                  ^ ,          ,           pKÙ  V\        P                  ! V\        P
                  VR7      P!                  ^ VP                  R4      XP#                  V4      4      ,           pWjP                  4       VP%                  R4      ,          P                  ^ R7      ,           pWxP                  4       ,           pEK‹  	  WW,          pWg,          p\        P                  ! WÞP%                  ^ 4      ,          4      pWñ,          # )ax  
Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch.

See Switch Transformer (https://huggingface.co/papers/2101.03961) for more details. This function implements the loss
function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between
experts is too unbalanced.

Args:
    gate_logits:
        Logits from the `gate`, should be a tuple of model.config.num_hidden_layers tensors of
        shape [batch_size X sequence_length, num_experts].
    num_experts:
        Number of experts
    top_k:
        The number of experts to route per-token, can be also interpreted as the `top-k` routing
        parameter.
    attention_mask (`torch.Tensor`, *optional*):
        The attention_mask used in forward function
        shape [batch_size X sequence_length] if not None.

Returns:
    The auxiliary loss.
r  r®   )rõ   rF   r_   )Ú	minlengthrE   )rÖ  rQ   rõ   r6   r  rH   rw   rG   r   rŽ   r�   r¦  Úbincountr+   r‹  rR   Úscatter_add_Úrepeat_interleaverh   )r  rz  r¡  r‚   Úcompute_deviceÚtokens_per_expert_sumÚrouter_prob_sumÚ
total_rowsÚ	flat_maskÚ
layer_gater¨  rË   Úselected_expertsÚtokens_per_expertÚrouter_prob_per_expertÚoverall_losss   &&&&            r.   Úload_balancing_loss_funcr   z  sÿ  € ð: Ò¤*¨[¼%×"@Ò"@Ùð ! •^×*Ñ*€NÜ!ŸKšK¨¼5¿=¹=ÐQ_Ô`ÐÜ—k’k +´U·]±]È>ÔZ€OØ€JàÒ!à"×*Ñ*¨2Ó.×1Ñ1¸ÌuÏ}É}Ð1Ó]ˆ	ä!ˆ
ÜŸ(™(×-Ñ-×5Ñ5°j·m±mÀNÓ6SÐY[Ð5Ó\ˆÜ#Ÿjšj¨ÀRÔHÑˆØÒ!ð &¬¯ªÐ7G×7OÑ7OÐPRÓ7SÐ_jÔ(k×(qÑ(qÓ(sÕsð "ð .×0EÑ0EÓ0G×0KÑ0KÐPQÐ0KÓ0RÕRˆOØ#×&;Ñ&;¸AÕ&>Õ>ŠJð %:¼E¿KºKØ¤5§=¡=¸ô=ç‰l˜1Ð.×6Ñ6°rÓ:¸I×<WÑ<WÐX]Ó<^Ó_õ%`Ð!ð .×1FÑ1FÓ1HÈ9×K^ÑK^Ð_aÓKbÕ1b×0gÑ0gÐlmÐ0gÓ0nÕnˆOØ#§m¡m£oÕ5‹Jñ# "ð& .Õ:ÐØ,Õ9Ðä—9’9Ð.×1QÑ1QÐRSÓ1TÕTÓU€LØÕ%Ð%r1   c                   ó�   a a€ ] tR tRt oRR/tRR/tRR.R.3/tRR/tV3R	 lV 3R
 llt]	]
RV3R lR ll4       4       tRtVtV ;t# )ÚJambaForCausalLMi¿  zlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputr@   ÚlogitsÚkeep_full_weightc                ó    <€ V ^8„  d   QhRS[ /# r+  r"   )r,   r-   s   "€r.   r/   ÚJambaForCausalLM.__annotate__Æ  s   ø€ ÷ 
ñ 
™{ñ 
r1   c                óR  <€ \         SV `  V4       \        V4      V n        VP                  V n        \
        P                  ! VP                  VP                  R R7      V n        VP                  V n	        VP                  V n
        VP                  V n        V P                  4        R# rj  )r3   r4   rê  rÐ  rð  r   r¥   r;   r#  Úrouter_aux_loss_coefrz  r   rù  ro  s   &&€r.   r4   ÚJambaForCausalLM.__init__Æ  s   ø€ Ü‰Ñ˜Ô Ü Ó'ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒØ$*×$?Ñ$?ˆÔ!Ø!×-Ñ-ˆÔØ#)×#=Ñ#=ˆÔ ð 	�‰Ör1   c                ód  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[R,          R
S[S[ P                  ,          RS[S[,          RS[	/# )r(   rÿ  Nr‚   r½  r¬   r   Úlabelsr¾  Úoutput_router_logitsÚlogits_to_keepr…   r*   )
r6   r¿  rB   r   rÀ  rç   rs   r   r   r   )r,   r-   s   "€r.   r/   r(  Ô  sÖ   ø€ ÷ P
ñ P
á×#Ñ# dÕ*ðP
ñ Ÿ™ tÕ+ðP
ñ ×&Ñ&¨Õ-ð	P
ñ
  �ðP
ñ ×(Ñ(¨4Õ/ðP
ñ × Ñ  4Õ'ðP
ñ ˜$•;ðP
ñ # T�kðP
ñ ™eŸl™lÕ*ðP
ñ Ñ+Õ,ðP
ñ 
#ñP
r1   c
                ó¦  € Ve   TMV P                   P                  pV P                  ! RRVRVRVRVRVRVRV/V
B pVP                  p\	        V	\
        4      '       d   \        V	) R4      MT	pV P                  VR	VR	3,          4      pRpVe   V P                  ! WæV P                  3/ V
B pRpV'       dc   \        VP                  V P                  V P                  V4      pVe3   WðP                  VP                  VP                   4      ,          ,          p\#        VVVVP$                  VP&                  VP(                  VP                  R
7      # )aê  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

Example:

```python
>>> from transformers import AutoTokenizer, JambaForCausalLM

>>> model = JambaForCausalLM.from_pretrained("ai21labs/Jamba-v0.1")
>>> tokenizer = AutoTokenizer.from_pretrained("ai21labs/Jamba-v0.1")

>>> prompt = "Hey, are you conscious? Can you talk to me?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
```Nrÿ  r‚   r½  r¬   r   r¾  r.  ru   )ÚlossÚaux_lossr%  r¬   r@   rÑ  r§  rê   )rš   r.  rÐ  r  rÖ  rs   Úslicer#  Úloss_functionrð  r   r§  rz  r   r*  rG   rõ   r   r¬   r@   rÑ  )r:   rÿ  r‚   r½  r¬   r   r-  r¾  r.  r/  r…   Úoutputsr@   Úslice_indicesr%  r1  r2  s   &&&&&&&&&&,      r.   rN   ÚJambaForCausalLM.forwardÒ  sx  € ðN %9Ò$DÑ È$Ï+É+×JjÑJjð 	ð
 +/¯*ª*ñ 	+
Øð	+
à)ð	+
ð &ð	+
ð ,ð		+
ð
 (ð	+
ð  ð	+
ð "6ð	+
ð ñ	+
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒØ×%Ò% f°d·o±oÑPÈÑPˆDàˆßÜ/Ø×%Ñ%Ø× Ñ Ø×(Ñ(Øó	ˆHð Ò!Ø×1Ñ1°H·K±KÀÇÁÓ4LÕLÕL�ä(ØØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø!×/Ñ/ô
ð 	
r1   )r#  rÐ  rz  r   r*  rð  )	NNNNNNNNr   )rV   rW   rX   rY   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr4   r   r   rN   rZ   r[   r\   r]   s   @@r.   r"  r"  ¿  so   ù‡ € à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÐ/Ð0€J÷
ó 
ð Ø÷P
ñ P
ó ó ÷P
ð P
r1   r"  c                   ó   € ] tR tRtRtR# )ÚJambaForSequenceClassificationi'  rê   N)rV   rW   rX   rY   rZ   rê   r1   r.   r=  r=  '  s   † Ûr1   r=  )rÅ   )r®   r¸   )	NNNNTNNNrU   )NNFN)NNNFFFF)Nr(   N)r"  r=  rê  rÏ  )PÚcollections.abcr   r6   Útorch.nn.functionalr   rŽ   rÉ   Ú r   r×  Úactivationsr   Úcache_utilsr   r   Ú
generationr	   Úintegrationsr
   r   r   r   Úintegrations.accelerater   Úmasking_utilsr   r   Úmodeling_layersr   r   Úmodeling_outputsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.genericr   Úutils.import_utilsr   r   r   r   Úutils.output_capturingr    r!   Úconfiguration_jambar#   r†   r&   re   rp   r|   r•   r—   r¼   r½   rÐ   rÓ   rÿ   r'  r)  rh  ru  r›  r°  rÅ  rÏ  rô  rê  r   r"  r=  Ú__all__rê   r1   r.   Ú<module>rQ     s†  ðõ2 %ã ß Ð Ý å &Ý !ß .Ý )÷ó õ >ß Pß [ß Qß FÝ &ß IÑ IÝ 7÷ó ÷ FÝ ,ñ ˜YÓ'ôJ�2—9‘9ó Jó (ðJò((ñ Ð-Ó.óó /ðõ2	U÷%ñ2 Ð)Ó*ô3)�R—Y‘Yó 3)ó +ð3)òl	ñ (Ð(>ÀÓPö'ó Qð'ñ& (Ð(:¸OÓLö'ó Mð'ñ, (Ð(8¸+ÓFöó Gðñ. (Ð(@À+ÓNö*ó Oð*ñZ (Ð(;¸[ÓIöhó JðhñV ØÐ)Ð+GÐIYÐ[oÐpóôj%�b—i‘ió j%óðj%ôZˆr�y‰yô ð  ô$#�2—9‘9ó $#ó ð$#ôN"˜"Ÿ)™)ô "ôJ#Ð!;ô #ôLÐ7ô ôBX˜?ô Xð: 'Ð(BÀGÐMcÐdÐ ð ôL
Ð%ó L
ó ðL
÷^B&ðJ ôd
Ð+¨_ó d
ó ðd
ôN	Ð%EÐG[ô 	ò g‚r1   