+
    œ°³j}[  ã                   óŽ  € ^ RI Ht ^ RIt^ RIHt ^RIHt ^RIHt ^RI	H
t
Ht ^RIHtHt ^RIHtHt ^R	IHtHt ^R
IHtHt ^RIHt ^RIHtHtHt ^RIHt ^RI H!t!H"t" ^RI#H$t$ ^RI%H&t&H't'H(t( ^RI)H*t*H+t+H,t,H-t-H.t. ^RI/H0t0 ^RI1H2t2H3t3 ^RI4H5t5 ]Pl                  ! ]74      t8 ! R R]'4      t9 ! R R]&4      t: ! R R]$4      t; ! R R]04      t< ! R R]24      t= ! R R ]P|                  4      t? ! R! R"]4      t@ ! R# R$]4      tAR%]@R&]A/tB ! R' R(]4      tC] ! R) R*]C4      4       tD ! R+ R,]34      tE ! R- R.]]C4      tF. R/OtGR# )0é    )ÚCallableN)Únn)Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)Úcreate_causal_maskÚcreate_recurrent_attention_mask)Ú GenericForSequenceClassificationÚGradientCheckpointingLayer)ÚMoeCausalLMOutputWithPastÚMoeModelOutputWithPast)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚlogging)Úmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputs)ÚFalconMambaMixer)ÚLlamaAttentionÚLlamaRMSNormÚeager_attention_forward)Úapply_mask_to_padding_statesÚcausal_conv1d_fnÚcausal_conv1d_updateÚmamba_selective_scanÚmamba_selective_state_update)Ú
MistralMLP)ÚMixtralExpertsÚMixtralForCausalLM©ÚJambaConfigc                   ó   € ] tR t^4tRtR# )ÚJambaRMSNorm© N©Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__static_attributes__r(   ó    Úh/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/jamba/modular_jamba.pyr'   r'   4   ó   † Ûr/   r'   c                   óT   a a€ ] tR t^8t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚJambaAttentionc                ó&   <€ V ^8„  d   QhRS[ RS[/# ©é   ÚconfigÚ	layer_idx©r%   Úint)ÚformatÚ__classdict__s   "€r0   Ú__annotate__ÚJambaAttention.__annotate__9   s   ø€ ÷ lñ l™{ð l±sñ lr/   c                ó>  <€ \         SV `  W4       \        P                  ! VP                  VP
                  V P                  ,          R R7      V n        \        P                  ! VP                  VP                  V P                  ,          R R7      V n	        \        P                  ! VP                  VP                  V P                  ,          R R7      V n
        \        P                  ! VP
                  V P                  ,          VP                  R R7      V n        R# ©F©ÚbiasN)ÚsuperÚ__init__r   ÚLinearÚhidden_sizeÚnum_attention_headsÚhead_dimÚq_projÚnum_key_value_headsÚk_projÚv_projÚo_proj)Úselfr7   r8   Ú	__class__s   &&&€r0   rD   ÚJambaAttention.__init__9   sÁ   ø€ Ü‰Ñ˜Ô+Ü—i’i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Õ4^ÐejÔkˆŒÜ—i’i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Õ4^ÐejÔkˆŒÜ—i’i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Õ4^ÐejÔkˆŒÜ—i’i × :Ñ :¸T¿]¹]Õ JÈF×L^ÑL^ÐejÔkˆŽr/   c                óØ   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  S[ P                  R,          3,          /# )r6   Úhidden_statesÚattention_maskNÚpast_key_valuesÚkwargsÚreturn)ÚtorchÚTensorr   r   r   Útuple)r;   r<   s   "€r0   r=   r>   @   sk   ø€ ÷ ")ñ ")á—|‘|ð")ñ Ÿ™ tÕ+ð")ñ  �ð	")ñ
 Ñ+Õ,ð")ñ 
‰u�|‰|™UŸ\™\¨DÕ0Ð0Õ	1ñ")r/   c                óì  € VP                   R R p. VORNV P                  N5pV P                  V4      P                  V4      P	                  ^^4      pV P                  V4      P                  V4      P	                  ^^4      pV P                  V4      P                  V4      P	                  ^^4      p	Ve   VP                  W‰V P                  4      w  r‰\        P                  ! V P                  P                  \        4      p
T
! V VVV	V3RV P                  '       g   RMV P                  RV P                   /VB w  r¼VP"                  ! . VORN5!  P%                  4       pV P'                  V4      pW¼3# )NÚdropoutç        Úscalingéÿÿÿÿ)ÚshaperH   rI   ÚviewÚ	transposerK   rL   Úupdater8   r   Úget_interfacer7   Ú_attn_implementationr   ÚtrainingÚattention_dropoutr]   ÚreshapeÚ
contiguousrM   )rN   rR   rS   rT   rU   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚattention_interfaceÚattn_outputÚattn_weightss   &&&&,        r0   ÚforwardÚJambaAttention.forward@   sp  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆàÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—k‘k +Ó.ˆØÐ(Ð(r/   )rK   rM   rI   rL   ©NN©	r*   r+   r,   r-   rD   rq   r.   Ú__classdictcell__Ú__classcell__©rO   r<   s   @@r0   r3   r3   8   s    ù‡ € ÷ló l÷")÷ ")ò ")r/   r3   c                   óR   a € ] tR t^et o RtV 3R lR ltR tR	V 3R lR lltRtV t	R# )
ÚJambaMambaMixeruo  
Compute âˆ†, A, B, C, and D the state space parameters and compute the `contextualized_states`.
A, D are input independent (see Mamba paper [1] Section 3.5.2 "Interpretation of A" for why A isn't selective)
âˆ†, B, C are input-dependent (this is a key difference between Mamba and the linear time invariant S4,
and is why Mamba is called **selective** state spaces)
c                ó    <€ V ^8„  d   QhRS[ /# ©r6   r7   r$   )r;   r<   s   "€r0   r=   ÚJambaMambaMixer.__annotate__m   s   ø€ ÷ ,@ñ ,@™{ñ ,@r/   c           	     ó.  € \         P                  P                  V 4       Wn        W n        VP
                  V n        VP                  V n        VP                  V n	        VP                  VP
                  ,          V n        VP                  V n        VP                  V n        VP                   V n        \         P$                  ! V P                  V P                  V P                  V P                  V P                  V P                  ^,
          R7      V n        VP(                  V n        \,        VP(                  ,          V n        \         P0                  ! V P
                  V P                  ^,          V P"                  R7      V n        \         P0                  ! V P                  V P                  V P                  ^,          ,           RR7      V n        \         P0                  ! V P                  V P                  RR7      V n        \8        P:                  ! ^V P                  ^,           4      R,          pVP=                  V P                  R4      P?                  4       p\         P@                  ! \8        PB                  ! V4      4      V n"        \         P@                  ! \8        PF                  ! V P                  4      4      V n$        \         P0                  ! V P                  V P
                  V P"                  R7      V n%        \M        V P                  VPN                  R7      V n(        \M        V P                  VPN                  R7      V n)        \M        V P                  VPN                  R7      V n*        VPV                  V n+        VPX                  V n,        R# )	é   )Úin_channelsÚout_channelsrB   Úkernel_sizeÚgroupsÚpaddingrA   FTN©Úeps©NºNNNr^   )-r   ÚModulerD   r7   r8   rF   Úmamba_d_stateÚssm_state_sizeÚmamba_d_convÚconv_kernel_sizeÚmamba_expandÚintermediate_sizeÚmamba_dt_rankÚtime_step_rankÚmamba_conv_biasÚuse_conv_biasÚmamba_proj_biasÚuse_biasÚConv1dÚconv1dÚ
hidden_actÚ
activationr   ÚactrE   Úin_projÚx_projÚdt_projrW   ÚarangeÚexpandrh   Ú	ParameterÚlogÚA_logÚonesÚDÚout_projr'   Úrms_norm_epsÚdt_layernormÚb_layernormÚc_layernormÚuse_mambapyÚuse_associative_scan)rN   r7   r8   ÚAs   &&& r0   rD   ÚJambaMambaMixer.__init__m   sƒ  € Ü
�	‰	×Ñ˜4Ô ØŒØ"ŒØ!×-Ñ-ˆÔØ$×2Ñ2ˆÔØ &× 3Ñ 3ˆÔØ!'×!4Ñ!4°v×7IÑ7IÕ!IˆÔØ$×2Ñ2ˆÔØ#×3Ñ3ˆÔØ×.Ñ.ˆŒÜ—i’iØ×.Ñ.Ø×/Ñ/Ø×#Ñ#Ø×-Ñ-Ø×)Ñ)Ø×)Ñ)¨AÕ-ô
ˆŒð !×+Ñ+ˆŒÜ˜&×+Ñ+Õ,ˆŒô —y’y ×!1Ñ!1°4×3IÑ3IÈAÕ3MÐTX×TaÑTaÔbˆŒä—i’i × 6Ñ 6¸×8KÑ8KÈd×NaÑNaÐdeÕNeÕ8eÐlqÔrˆŒä—y’y ×!4Ñ!4°d×6LÑ6LÐSWÔXˆŒô �LŠL˜˜D×/Ñ/°!Õ3Ó4°WÕ=ˆØ�H‰H�T×+Ñ+¨RÓ0×;Ñ;Ó=ˆä—\’\¤%§)¢)¨A£,Ó/ˆŒ
Ü—’œeŸjšj¨×)?Ñ)?Ó@ÓAˆŒÜŸ	š	 $×"8Ñ"8¸$×:JÑ:JÐQU×Q^ÑQ^Ô_ˆŒä(¨×)<Ñ)<À&×BUÑBUÔVˆÔÜ'¨×(;Ñ(;À×ATÑATÔUˆÔÜ'¨×(;Ñ(;À×ATÑATÔUˆÔà!×-Ñ-ˆÔØ$*×$?Ñ$?ˆÖ!r/   c                ó   € \        R 4      h)zNot needed for jamba)ÚNotImplementedError)rN   s   &r0   Úinit_jamba_weightsÚ"JambaMambaMixer.init_jamba_weights›   s   € Ü!Ð"8Ó9Ð9r/   Nc                óp   <€ V ^8„  d   QhRS[ P                  RS[R,          RS[ P                  R,          /# )r6   rR   Úcache_paramsNrS   )rW   rX   r   )r;   r<   s   "€r0   r=   r|   ž   s?   ø€ ÷ s%ñ s%á—|‘|ðs%ñ ˜d•lðs%ñ Ÿ™ tÕ+ñ	s%r/   c                ó¼	  € VP                   ^,          pVP                  pVRJ;'       d    VP                  V P                  4      p\	        W4      pV P                  V4      P                  ^^4      p\        P                  ! V P                  P                  4       4      ) p	VP                  ^^R7      w  r«V'       d]   VP                  V P                  ,          P                  ^ ,          pVP                  V P                  ,          P                  ^ ,          pV'       d‡   V^8X  d€   VP                  V P                  ,          P                  '       gS   \!        V
XV P"                  P$                  P'                  ^4      V P"                  P(                  V P*                  R7      p
M�Ve)   VP-                  V
V P                  V P.                  R7      p
\1        V
V P"                  P$                  P'                  ^4      V P"                  P(                  3RV P*                  /VB p
Ve   V
RRV) R13,          p
\	        V
P                  ^^4      V4      p
\        P2                  ! V P5                  V
4      V P6                  V P8                  V P8                  .RR7      w  rïpV P;                  V4      pV P=                  V4      pV P?                  V4      p\A        V PB                  R4      '       d>   V PE                  V4      V PD                  P(                  ,
          P                  ^^4      pM-V PD                  P$                  VP                  ^^4      ,          pV PD                  P(                  e%   V PD                  P(                  P                  4       MRpV'       dj   V^8X  dc   \G        XV
P                  ^^4      R,          VR,          V	VR,          VR,          V PH                  VR,          VRR	7
      PK                  R4      pM£VRJp\M        V
P                  ^^4      VV	VP                  ^^4      VP                  ^^4      V PH                  P                  4       VVRVV PN                  V PP                  R
7      pV'       d#   Vw  ppVPS                  VV P                  4       MTpV PU                  VP                  ^^4      PW                  V4      4      pV# )r~   N©Údim)r˜   )rŒ   r˜   r‡   Ú_is_quantizedT)ÚzÚdt_biasÚdt_softplus)r£   r·   Ú
delta_biasÚdelta_softplusÚreturn_last_stater©   rª   r^   ).r   )r‡   r   ),r_   ÚdtypeÚhas_previous_stater8   r   rš   ra   rW   Úexpr¡   ÚfloatÚchunkÚlayersÚconv_statesÚrecurrent_statesÚrecord_pastr   r–   ÚweightÚsqueezerB   r˜   Úupdate_conv_staterŒ   r   Úsplitr›   r�   rŠ   r¦   r§   r¨   Úhasattrr7   rœ   r    r£   Ú	unsqueezer   r©   rª   Úupdate_recurrent_stater¤   Úto)rN   rR   r²   rS   rU   Úseq_lenr½   Úuse_precomputed_statesÚprojected_statesr«   Úhidden_states_B_CÚgateÚ
conv_stateÚrecurrent_stateÚ	time_stepÚBÚCÚtime_proj_biasÚscan_outputÚoutput_final_stateÚscan_resultÚfinal_stateÚcontextualized_statess   &&&&,                  r0   rq   ÚJambaMambaMixer.forwardž   s2  € ð  ×%Ñ% aÕ(ˆØ×#Ñ#ˆØ!-°TÐ!9×!mÐ!m¸l×>]Ñ>]Ð^b×^lÑ^lÓ>mÐô 5°]ÓSˆØŸ<™<¨Ó6×@Ñ@ÀÀAÓFÐô �YŠY�t—z‘z×'Ñ'Ó)Ó*Ð*ˆà"2×"8Ñ"8¸ÀÐ"8Ó"BÑÐç!Ø%×,Ñ,¨T¯^©^Õ<×HÑHÈÕKˆJØ*×1Ñ1°$·.±.ÕA×RÑRÐSTÕUˆO÷ " g°¤l¸<×;NÑ;NÈtÏ~É~Õ;^×;j×;jÐ;jÜ 4Ø!ØØ—‘×"Ñ"×*Ñ*¨1Ó-Ø—‘× Ñ ØŸ?™?ô!Ñð Ò'Ø$0×$BÑ$BØ%Ø—N‘NØ%)×%:Ñ%:ð %Có %Ð!ô !1Ø!Ø—‘×"Ñ"×*Ñ*¨1Ó-Ø—‘× Ñ ñ!ð  Ÿ?™?ð	!ð
 ñ!Ðð Ò'Ø$5°a¸¸W¸H¹I°oÕ$FÐ!ô 9Ð9J×9TÑ9TÐUVÐXYÓ9ZÐ\jÓkÐÜŸ+š+Ø�K‰KÐ)Ó*Ø× Ñ  $×"5Ñ"5°t×7JÑ7JÐKØô
‰ˆ	�að ×%Ñ% iÓ0ˆ	Ø×Ñ˜QÓˆØ×Ñ˜QÓˆô �4—;‘; ×0Ò0ØŸ™ iÓ0°4·<±<×3DÑ3DÕD×OÑOÐPQÐSTÓU‰IàŸ™×+Ñ+¨i×.AÑ.AÀ!ÀQÓ.GÕGˆIØ6:·l±l×6GÑ6GÒ6S˜Ÿ™×*Ñ*×0Ñ0Ô2ÐY]ˆ÷ " g°¤lÜ6ØØ!×+Ñ+¨A¨qÓ1°&Õ9Ø˜&Õ!ØØ�$•Ø�$•Ø—‘Ø�v•,Ø&Ø ô÷ ‰i˜‹mñ ð ".°TÐ!9ÐÜ.Ø!×+Ñ+¨A¨qÓ1ØØØ—‘˜A˜qÓ!Ø—‘˜A˜qÓ!Ø—&‘&—,‘,“.ØØ)Ø#Ø"4Ø ×,Ñ,Ø%)×%>Ñ%>ôˆK÷ "Ø+6Ñ(�˜[Ø×3Ñ3°KÀÇÁÕPà)�ð !%§¡¨k×.CÑ.CÀAÀqÓ.I×.LÑ.LÈUÓ.SÓ TÐØ$Ð$r/   )r¡   r£   r™   r˜   r§   r¨   r7   r–   rŒ   r¦   rœ   rF   rš   rŽ   r8   r¤   rŠ   r�   rª   r”   r’   r©   r›   rs   )
r*   r+   r,   r-   Ú__doc__rD   r¯   rq   r.   ru   )r<   s   @r0   ry   ry   e   s.   ø‡ € ñ÷,@ð ,@ò\:÷s%÷ s%ð s%r/   ry   c                   ó   € ] tR tRtRtR# )ÚJambaMLPi  r(   Nr)   r(   r/   r0   rá   rá     r1   r/   rá   c                   ó   € ] tR tRtRtR# )ÚJambaExpertsi  r(   Nr)   r(   r/   r0   rã   rã     r1   r/   rã   c                   óZ   a a€ ] tR tRt oRtV3R lV 3R lltR tV3R lR ltRtVt	V ;t
# )	ÚJambaSparseMoeBlocki  a¤  
This implementation is
strictly equivalent to standard MoE with full capacity (no
dropped tokens). It's faster since it formulates MoE operations
in terms of block-sparse operations to accommodate imbalanced
assignments of tokens to experts, whereas standard MoE either
(1) drop tokens at the cost of reduced performance or (2) set
capacity factor to number of experts and thus waste computation
and memory on padding.
c                ó    <€ V ^8„  d   QhRS[ /# r{   r$   )r;   r<   s   "€r0   r=   Ú JambaSparseMoeBlock.__annotate__(  s   ø€ ÷ ,ñ ,™{ñ ,r/   c                ó0  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  V n        VP                  V n        \        P                  ! V P                  V P                  R R7      V n        \        V4      V n        R# r@   )rC   rD   rF   Ú
hidden_dimrŽ   Úffn_dimÚnum_expertsÚnum_experts_per_tokÚtop_kr   rE   Úrouterrã   Úexperts©rN   r7   rO   s   &&€r0   rD   ÚJambaSparseMoeBlock.__init__(  sm   ø€ Ü‰ÑÔØ ×,Ñ,ˆŒØ×/Ñ/ˆŒØ!×-Ñ-ˆÔØ×/Ñ/ˆŒ
ä—i’i §¡°×1AÑ1AÈÔNˆŒÜ# FÓ+ˆŽr/   c                óö   € \         P                  P                  P                  VR\         P                  R7      p\         P
                  ! W0P                  RR7      w  rEWTP                  VP                  4      3# )r~   )rµ   r½   r´   r^   )	rW   r   Ú
functionalÚsoftmaxrÀ   Útopkrí   rÍ   r½   )rN   rR   Úrouter_logitsÚrouting_weightsÚtop_k_weightsÚtop_k_indexs   &&&   r0   Úroute_tokens_to_expertsÚ+JambaSparseMoeBlock.route_tokens_to_experts2  s\   € ÜŸ(™(×-Ñ-×5Ñ5°mÈÔSX×S^ÑS^Ð5Ó_ˆÜ%*§Z¢Z°ÇÁÐQSÔ%TÑ"ˆØ×,Ñ,¨]×-@Ñ-@ÓAÐAÐAr/   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r6   rR   rV   )rW   rX   )r;   r<   s   "€r0   r=   rç   7  s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r/   c                óØ   € VP                   w  r#pVP                  RV4      pV P                  V4      pV P                  W4      w  rgV P	                  WV4      pVP                  W#V4      pV# )r~   r^   )r_   r`   rî   rú   rï   rg   )rN   rR   Ú
batch_sizeÚsequence_lengthré   rö   rù   rø   s   &&      r0   rq   ÚJambaSparseMoeBlock.forward7  sm   € Ø2?×2EÑ2EÑ/ˆ
 ZØ%×*Ñ*¨2¨zÓ:ˆØŸ™ MÓ2ˆØ%)×%AÑ%AÀ-Ó%_Ñ"ˆØŸ™ ]ÀÓOˆØ%×-Ñ-¨jÈ:ÓVˆØÐr/   )rï   rê   ré   rë   rî   rí   )r*   r+   r,   r-   rß   rD   rú   rq   r.   ru   rv   rw   s   @@r0   rå   rå     s)   ù‡ € ñ	÷,ó ,òB÷
÷ ð r/   rå   c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚJambaAttentionDecoderLayeriA  c                ó&   <€ V ^8„  d   QhRS[ RS[/# r5   r9   )r;   r<   s   "€r0   r=   Ú'JambaAttentionDecoderLayer.__annotate__B  s   ø€ ÷ Zñ Z™{ð Z±sñ Zr/   c                ój  <€ \         SV `  4        VP                  '       d   VP                  V,          M^p\        W4      V n        V^8”  d   \
        M\        pV! V4      V n        \        VP                  VP                  R7      V n        \        VP                  VP                  R7      V n        R# )r~   r„   N)rC   rD   Úlayers_num_expertsr3   Ú	self_attnrå   rá   Úfeed_forwardr'   rF   r¥   Úinput_layernormÚpre_ff_layernorm©rN   r7   r8   rë   Úffn_layer_classrO   s   &&&  €r0   rD   Ú#JambaAttentionDecoderLayer.__init__B  s‰   ø€ Ü‰ÑÔØ>D×>W×>WÐ>W�f×/Ñ/°	Ö:Ð]^ˆÜ'¨Ó:ˆŒà1<¸q´Õ-ÄhˆÙ+¨FÓ3ˆÔÜ+¨F×,>Ñ,>ÀF×DWÑDWÔXˆÔÜ ,¨V×-?Ñ-?ÀV×EXÑEXÔ YˆÖr/   c                óÜ   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[,          RS[ P                  /# )	r6   rR   rS   NÚposition_idsrT   Ú	use_cacherU   rV   )rW   rX   Ú
LongTensorr   Úboolr   r   ÚFloatTensor)r;   r<   s   "€r0   r=   r  L  s}   ø€ ÷ ñ á—|‘|ðñ Ÿ™ tÕ+ðñ ×&Ñ&¨Õ-ð	ñ
  �ðñ ˜$•;ðñ Ñ+Õ,ðñ 
×	Ñ	ñr/   c                óÐ   € TpV P                  V4      pV P                  ! RR VRVRVRVRV/VB w  rWq,           pTpV P                  V4      pV P                  V4      pWq,           pV# )rR   rS   r  rT   r  r(   )r	  r  r
  r  )	rN   rR   rS   r  rT   r  rU   ÚresidualÚ_s	   &&&&&&,  r0   rq   Ú"JambaAttentionDecoderLayer.forwardL  sŸ   € ð !ˆØ×,Ñ,¨]Ó;ˆØŸ>š>ñ 
Ø'ð
à)ð
ð &ð
ð ,ð	
ð
  ð
ð ñ
Ñˆð !Õ0ˆØ ˆØ×-Ñ-¨mÓ<ˆØ×)Ñ)¨-Ó8ˆØ Õ0ˆØÐr/   )r  r	  r
  r  )NNNFrt   rw   s   @@r0   r  r  A  s    ù‡ € ÷Zó Z÷÷ ò r/   r  c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚJambaMambaDecoderLayerig  c                ó&   <€ V ^8„  d   QhRS[ RS[/# r5   r9   )r;   r<   s   "€r0   r=   Ú#JambaMambaDecoderLayer.__annotate__h  s   ø€ ÷ Zñ Z™{ð Z±sñ Zr/   c                ól  <€ \         SV `  4        VP                  '       d   VP                  V,          M^p\        WR7      V n        V^8”  d   \
        M\        pV! V4      V n        \        VP                  VP                  R7      V n        \        VP                  VP                  R7      V n        R# )r~   )r7   r8   r„   N)rC   rD   r  ry   Úmambarå   rá   r  r'   rF   r¥   r	  r
  r  s   &&&  €r0   rD   ÚJambaMambaDecoderLayer.__init__h  s‰   ø€ Ü‰ÑÔØ>D×>W×>WÐ>W�f×/Ñ/°	Ö:Ð]^ˆÜ$¨FÔHˆŒ
Ø1<¸q´Õ-ÄhˆÙ+¨FÓ3ˆÔÜ+¨F×,>Ñ,>ÀF×DWÑDWÔXˆÔÜ ,¨V×-?Ñ-?ÀV×EXÑEXÔ YˆÖr/   c                óÈ   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[S[,          RS[ P                  /# )r6   rR   rS   Nr  rT   rU   rV   )rW   rX   r  r   r   r   r  )r;   r<   s   "€r0   r=   r  q  so   ø€ ÷ ñ á—|‘|ðñ Ÿ™ tÕ+ðñ ×&Ñ&¨Õ-ð	ñ
  �ðñ Ñ+Õ,ðñ 
×	Ñ	ñr/   c                ó¼   € TpV P                  V4      pV P                  VVVR 7      pWa,           pTpV P                  V4      pV P                  V4      pWa,           pV# ))rR   r²   rS   )r	  r  r
  r  )rN   rR   rS   r  rT   rU   r  s   &&&&&, r0   rq   ÚJambaMambaDecoderLayer.forwardq  sr   € ð !ˆØ×,Ñ,¨]Ó;ˆØŸ
™
Ø'Ø(Ø)ð #ó 
ˆð
 !Õ0ˆØ ˆØ×-Ñ-¨mÓ<ˆØ×)Ñ)¨-Ó8ˆØ Õ0ˆØÐr/   )r  r	  r  r
  )NNNrt   rw   s   @@r0   r  r  g  s    ù‡ € ÷Zó Z÷÷ ò r/   r  Ú	attentionr  c            
       óÄ   a a€ ] tR tRt oRtRtRR.tR.tRtRt	Rt
RtR]].R]R	]! ]P"                  R
R7      /t]P(                  ! 4       V 3R l4       tV3R ltRtVtV ;t# )ÚJambaPreTrainedModeli‹  ÚmodelTr  r  rT   rR   Ú
attentionsrö   rî   )Ú
layer_namec                ó¾  <€ \         SV `  V4       \        V\        4      '       d±   \        P
                  ! ^VP                  ^,           4      R,          pVP                  VP                  R4      P                  4       p\        P                  ! VP                  \        P                  ! V4      4       \        P                  ! VP                  4       R# \        V\         4      '       dq   \        P"                  ! VP$                  RV P&                  P(                  R7       \        P"                  ! VP*                  RV P&                  P(                  R7       R# R# )r~   Nr\   )ÚmeanÚstdr†   r^   )rC   Ú_init_weightsÚ
isinstancery   rW   r�   rŠ   rž   rŽ   rh   ÚinitÚcopy_r¡   r    Úones_r£   rã   Únormal_Úgate_up_projr7   Úinitializer_rangeÚ	down_proj)rN   Úmoduler«   rO   s   && €r0   r+  Ú"JambaPreTrainedModel._init_weights›  sØ   ø€ ä‰Ñ˜fÔ%Ü�fœo×.Ò.Ü—’˜Q × 5Ñ 5¸Õ 9Ó:¸7ÕCˆAØ—‘˜×1Ñ1°2Ó6×AÑAÓCˆAÜ�JŠJ�v—|‘|¤U§Y¢Y¨q£\Ô2Ü�JŠJ�v—x‘xÖ Ü˜¤×-Ò-Ü�LŠL˜×,Ñ,°3¸D¿K¹K×<YÑ<YÕZÜ�LŠL˜×)Ñ)°¸¿¹×9VÑ9V×Wñ .r/   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r{   r$   )r;   r<   s   "€r0   r=   Ú!JambaPreTrainedModel.__annotate__‹  s   ø‡ ‚ ÙÑò r/   r(   )r*   r+   r,   r-   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_is_statefulÚ_can_compile_fullgraphr  r  r3   r   r   rE   Ú_can_record_outputsrW   Úno_gradr+  Ú__annotate_func__r.   ru   rv   rw   s   @@r0   r$  r$  ‹  sˆ   ù‡ € àÐØ&*Ð#Ø5Ð7OÐPÐØ#4Ð"5ÐØÐØ€NØ€LØ!ÐàÐ4Ð6LÐMØ�nØ™¨¯	©	¸hÔGðÐð ‡]‚]ƒ_ô	Xó ð	X÷# … r/   r$  c                   ór   a a€ ] tR tRt oV3R lV 3R llt]]]RV3R lR ll4       4       4       tRt	Vt
V ;t# )Ú
JambaModeli¨  c                ó    <€ V ^8„  d   QhRS[ /# r{   r$   )r;   r<   s   "€r0   r=   ÚJambaModel.__annotate__ª  s   ø€ ÷ ñ ™{ñ r/   c                ó8  <€ \         SV `  V4       VP                  V n        VP                  V n        \
        P                  ! VP                  VP                  V P                  4      V n        . p\        VP                  4       F9  p\        VP                  V,          ,          pVP                  V! WR 7      4       K;  	  \
        P                  ! V4      V n        \!        VP                  VP"                  R7      V n        RV n        V P)                  4        R# ))r8   r„   FN)rC   rD   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	EmbeddingrF   Úembed_tokensÚrangeÚnum_hidden_layersÚALL_DECODER_LAYER_TYPESÚlayers_block_typeÚappendÚ
ModuleListrÂ   r'   r¥   Úfinal_layernormÚgradient_checkpointingÚ	post_init)rN   r7   Údecoder_layersÚiÚlayer_classrO   s   &&   €r0   rD   ÚJambaModel.__init__ª  sÔ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸLšL¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔØˆÜ�v×/Ñ/Ö0ˆAÜ1°&×2JÑ2JÈ1Õ2MÕNˆKØ×!Ñ!¡+¨fÔ"BÖCñ 1ô —m’m NÓ3ˆŒä+¨F×,>Ñ,>ÀF×DWÑDWÔXˆÔà&+ˆÔ#à�‰Ör/   c                óþ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[S[,          R	S[/# )
r6   Ú	input_idsNrS   r  rT   Úinputs_embedsr  rU   rV   )	rW   r  rX   r   r  r  r   r   r   )r;   r<   s   "€r0   r=   rF  ¿  s“   ø€ ÷ 6
ñ 6
á×#Ñ# dÕ*ð6
ñ Ÿ™ tÕ+ð6
ñ ×&Ñ&¨Õ-ð	6
ñ
  �ð6
ñ ×(Ñ(¨4Õ/ð6
ñ ˜$•;ð6
ñ Ñ+Õ,ð6
ñ 
 ñ6
r/   c                óâ  € VR J VR J,          '       d   \        R4      hVf   V P                  V4      pV'       d   Vf   \        V P                  R7      pVfb   Ve   VP	                  4       M^ p\
        P                  ! VP                  ^,          VP                  R7      V,           pVP                  ^ 4      p\        T;p	\        4      '       g/   RV P                  RVRVRVRV/p
R	\        R/ V
B R
\        R/ V
B /p	Tp\        V P                  4       F7  w  rÍV! V3RW�P                  P                   V,          ,          RVRVRV/VB pK9  	  V P#                  V4      p\%        VVR7      # )Nz:You must specify exactly one of input_ids or inputs_embeds)r7   )Údevicer7   r\  rS   rT   r  Úfull_attentionÚlinear_attentionr  )Úlast_hidden_staterT   r(   )Ú
ValueErrorrL  r   r7   Úget_seq_lengthrW   r�   r_   r^  rË   r,  Údictr	   r
   Ú	enumeraterÂ   Úlayer_typesrS  r   )rN   r[  rS   r  rT   r\  r  rU   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsrR   rW  Údecoder_layers   &&&&&&&,      r0   rq   ÚJambaModel.forward¼  s•  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ Ø ×-Ñ-¨iÓ8ˆMç˜Ò0Ü*°$·+±+Ô>ˆOàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLä°Ð?Ð-Ä×FÒFð ˜$Ÿ+™+Ø Ø  .Ø! ?Ø ðˆKð !Ô"4Ñ"C°{Ñ"CØ"Ô$CÑ$RÀkÑ$Rð#Ðð &ˆÜ )¨$¯+©+Ö 6ÑˆAÙ)Øñà2·;±;×3JÑ3JÈ1Õ3MÕNðð *ðð !0ð	ð
 $ðð ñŠMñ !7ð ×,Ñ,¨]Ó;ˆä%Ø+Ø+ô
ð 	
r/   )rL  rS  rT  rÂ   rI  rJ  )NNNNNN)r*   r+   r,   r-   rD   r   r   r   rq   r.   ru   rv   rw   s   @@r0   rD  rD  ¨  s=   ù‡ € ÷ó ð$  ØØ÷6
ñ 6
ó ó ó  ÷6
ð 6
r/   rD  c                   óZ   a a€ ] tR tRt oV3R lV 3R lltRV3R lV 3R llltRtVtV ;t# )ÚJambaForCausalLMiø  c                ó    <€ V ^8„  d   QhRS[ /# r{   r$   )r;   r<   s   "€r0   r=   ÚJambaForCausalLM.__annotate__ù  s   ø€ ÷ .ñ .™{ñ .r/   c                óH   <€ \         SV `  V4       VP                  V n        R # )N)rC   rD   rë   rð   s   &&€r0   rD   ÚJambaForCausalLM.__init__ù  s   ø€ Ü‰Ñ˜Ô Ø!×-Ñ-ˆÖr/   c                ód  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[R,          R
S[S[ P                  ,          RS[S[,          RS[	/# )r6   r[  NrS   r  rT   r\  Úlabelsr  Úoutput_router_logitsÚlogits_to_keeprU   rV   )
rW   r  rX   r   r  r  r:   r   r   r   )r;   r<   s   "€r0   r=   ro  ý  sÉ   ø€ ÷ -
ñ -
á×#Ñ# dÕ*ð-
ñ Ÿ™ tÕ+ð-
ñ ×&Ñ&¨Õ-ð	-
ñ
  �ð-
ñ ×(Ñ(¨4Õ/ð-
ñ × Ñ  4Õ'ð-
ñ ˜$•;ð-
ñ # T�kð-
ñ ™eŸl™lÕ*ð-
ñ Ñ+Õ,ð-
ñ 
#ñ-
r/   c
           
     ó4   <€ \         SV `  ! VVVVVVVV	3/ V
B # )aê  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

Example:

```python
>>> from transformers import AutoTokenizer, JambaForCausalLM

>>> model = JambaForCausalLM.from_pretrained("ai21labs/Jamba-v0.1")
>>> tokenizer = AutoTokenizer.from_pretrained("ai21labs/Jamba-v0.1")

>>> prompt = "Hey, are you conscious? Can you talk to me?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
```)rC   rq   )rN   r[  rS   r  rT   r\  rs  r  rt  ru  rU   rO   s   &&&&&&&&&&,€r0   rq   ÚJambaForCausalLM.forwardý  s9   ø€ ôF ‰wŠØØØØØØØØñ

ð ñ

ð 
	
r/   )rë   )	NNNNNNNNr   rt   rw   s   @@r0   rm  rm  ø  s   ù‡ € ÷.ó .÷-
÷ -
õ -
r/   rm  c                   ó   € ] tR tRtRtR# )ÚJambaForSequenceClassificationi-  r(   Nr)   r(   r/   r0   ry  ry  -  r1   r/   ry  )rm  ry  rD  r$  )HÚcollections.abcr   rW   r   Ú r   r-  Úactivationsr   Úcache_utilsr   r   Úmasking_utilsr	   r
   Úmodeling_layersr   r   Úmodeling_outputsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.genericr   Úutils.output_capturingr   r   Ú"falcon_mamba.modeling_falcon_mambar   Úllama.modeling_llamar   r   r   Úmamba.modeling_mambar   r   r   r   r    Úmistral.modeling_mistralr!   Úmixtral.modeling_mixtralr"   r#   Úconfiguration_jambar%   Ú
get_loggerr*   Úloggerr'   r3   ry   rá   rã   rˆ   rå   r  r  rO  r$  rD  rm  ry  Ú__all__r(   r/   r0   Ú<module>r�     s9  ðõ& %ã Ý å &Ý !ß .ß Pß [ß Qß FÝ &ß @Ñ @Ý 7ß EÝ Aß XÑ X÷õ õ 2ß IÝ ,ð 
×	Ò	˜HÓ	%€ô	�<ô 	ô*)�^ô *)ôZl%Ð&ô l%ô^	ˆzô 	ô	�>ô 	ô"˜"Ÿ)™)ô "ôJ#Ð!;ô #ôLÐ7ô ðB 'Ð(BÀGÐMcÐdÐ ôX˜?ô Xð: ôL
Ð%ó L
ó ðL
ô^2
Ð)ô 2
ôj	Ð%EÐG[ô 	ò g‚r/   