+
    œ°³jõ  ã                   óê  € ^ RI t ^ RIHt ^ RIHt ^ RIt^ RIHu Ht	 ^ RIHt ^RI
Ht ^RIHt ^RIHtHt ^RIHt ^R	IHtHtHt ^R
IHt ^RIHtHt ^RIHt ^RIH t H!t!H"t" ^RI#H$t$H%t% ^RI&H't'H(t( ^RI)H*t* ^RI+H,t,H-t-H.t.H/t/ ^RI0H1t1 ^RI2H3t3H4t4 ^RI5H6t6 ^RI7H8t8 ]/Pr                  ! ]:4      t; ! R R]P                  Px                  4      t= ! R R]Px                  4      t> ! R R]Px                  4      t?R R lt@RUR R  lltAR! tB]! R"4      RVR# l4       tC ! R$ R%]Px                  4      tDR& R' ltER( tFR) tGR* tH]! R+R,4      RWR- R. ll4       tI]! R/R,4      RWR0 R1 ll4       tJ]! R2R34      RR]K! R44      3R5R6RR7RRR^R83R9 R: ll4       tL]! R;R34      RXR< R= ll4       tM]! R>R34      RRRR5R]K! R44      3R53R? R@ ll4       tN]! ]J]I]L]M]N.4       ! RA RB]Px                  4      4       tO ! RC RD]Px                  4      tP ! RE RF]Px                  4      tQ ! RG RH]4      tR ! RI RJ]4      tS]- ! RK RL](4      4       tT]- ! RM RN]T4      4       tU ! RO RP]T]4      tV]-! RQRR7       ! RS RT]T4      4       tW. RYOtXR# )Zé    N)ÚCallable)Úcycle)Únn)Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_kernel_forward_from_hubÚ&use_kernel_func_from_hub_with_fallbackÚuse_kernelized_func)Úforce_accelerate_hooks)Úcreate_causal_maskÚcreate_recurrent_attention_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚCausalLMOutputWithPastÚ SequenceClassifierOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputs©ÚZamba2Configc                   ó@   a a€ ] tR t^3t oRV 3R lltRR ltRtVtV ;t# )ÚZamba2RMSNormGatedc                óš   <€ \         SV `  4        \        P                  ! \        P
                  ! V4      4      V n        W0n        W n        R # ©N)	ÚsuperÚ__init__r   Ú	ParameterÚtorchÚonesÚweightÚvariance_epsilonÚ
group_size)ÚselfÚhidden_sizer/   ÚepsÚ	__class__s   &&&&€Úk/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/zamba2/modeling_zamba2.pyr)   ÚZamba2RMSNormGated.__init__4   s2   ø€ Ü‰ÑÔÜ—l’l¤5§:¢:¨kÓ#:Ó;ˆŒØ #ÔØ$Žó    c                ó”  € VP                   pVP                  \        P                  4      pVeD   V\        P
                  P                  VP                  \        P                  4      4      ,          pVP                  Ev rEWPP                  ,          pVP                  ! . VOVNV P                  N5!  pVP                  ^4      P                  RRR7      pV\        P                  ! W€P                  ,           4      ,          pVP                  ! . VOW`P                  ,          N5!  pV P                  VP                  V4      ,          # )NT©Úkeepdiméÿÿÿÿ)ÚdtypeÚtor+   Úfloat32r   Ú
functionalÚsiluÚshaper/   ÚviewÚpowÚmeanÚrsqrtr.   r-   )	r0   Úhidden_statesÚgateÚinput_dtypeÚprefix_dimsÚlast_dimÚgroup_countÚhidden_states_groupÚvariances	   &&&      r4   ÚforwardÚZamba2RMSNormGated.forward:   sÿ   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØÒØ)¬B¯M©M×,>Ñ,>¸t¿w¹wÄuÇ}Á}Ó?UÓ,VÕVˆMØ!.×!4Ñ!4ÑˆØ§/¡/Õ1ˆØ+×0Ò0Ð\°+Ð\¸{Ð\ÈDÏOÉOÓ\ÐØ&×*Ñ*¨1Ó-×2Ñ2°2¸tÐ2ÓDˆØ1´E·K²KÀ×K`ÑK`Õ@`Ó4aÕaÐØ+×0Ò0Ð]°+Ð]¸{Ï_É_Õ?\Ó]ˆØ�{‰{˜]×-Ñ-¨kÓ:Õ:Ð:r6   )r/   r.   r-   ©ç�íµ ÷Æ°>r'   ©	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r)   rM   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r3   Ú__classdict__s   @@r4   r%   r%   3   s   ù‡ € ÷%÷;ô ;r6   r%   c                   óZ   a a€ ] tR t^Ht oRV3R lV 3R llltV3R lR ltR tRtVtV ;t	# )ÚZamba2RMSNormc                ó$   <€ V ^8„  d   QhRS[ RR/# )é   r2   ÚreturnN)Úfloat)ÚformatrZ   s   "€r4   Ú__annotate__ÚZamba2RMSNorm.__annotate__I   s   ø€ ÷ $ñ $©ð $¸$ñ $r6   c                óŽ   <€ \         SV `  4        \        P                  ! \        P
                  ! V4      4      V n        W n        R# )z,
Zamba2RMSNorm is equivalent to T5LayerNorm
N)r(   r)   r   r*   r+   r,   r-   r.   )r0   r1   r2   r3   s   &&&€r4   r)   ÚZamba2RMSNorm.__init__I   s/   ø€ ô 	‰ÑÔÜ—l’l¤5§:¢:¨kÓ#:Ó;ˆŒØ #Ör6   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r^   rE   r_   )r+   ÚTensor)ra   rZ   s   "€r4   rb   rc   Q   s#   ø€ ÷ ;ñ ;¡U§\¡\ð ;±e·l±lñ ;r6   c                ó:  € VP                   pVP                  \        P                  4      pVP	                  ^4      P                  RRR7      pV\        P                  ! W0P                  ,           4      ,          pV P                  VP                  V4      ,          # )r^   Tr8   r:   )	r;   r<   r+   r=   rB   rC   rD   r.   r-   )r0   rE   rG   rL   s   &&  r4   rM   ÚZamba2RMSNorm.forwardQ   sw   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯ª°H×?TÑ?TÕ4TÓ(UÕUˆØ�{‰{˜]×-Ñ-¨kÓ:Õ:Ð:r6   c                ó^   € \        V P                  P                  4       R V P                   2# )z, eps=)Útupler-   r@   r.   )r0   s   &r4   Ú
extra_reprÚZamba2RMSNorm.extra_reprX   s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr6   )r.   r-   rO   )
rR   rS   rT   rU   r)   rM   rl   rV   rW   rX   rY   s   @@r4   r\   r\   H   s%   ù‡ € ÷$õ $÷;ð ;÷Jò Jr6   r\   c                   óÎ   a a€ ] tR t^\t o]! RRR7      R
V3R lV 3R lll4       t]]! RRR7      R
V3R lR ll4       4       t]P                  ! 4       ]
R 4       4       tR	tVtV ;t# )ÚZamba2RotaryEmbeddingÚdevicez5.18)Úversionc                ó    <€ V ^8„  d   QhRS[ /# ©r^   Úconfigr"   )ra   rZ   s   "€r4   rb   Ú"Zamba2RotaryEmbedding.__annotate__^   s   ø€ ÷ Oñ O™|ñ Or6   c                óä  <€ \         SV `  4        VP                  V n        VP                  V n        Wn        V P
                  P                  R ,          V n        V P                  pV P                  R8w  d   \        V P                  ,          pV! V P
                  V4      w  q@n
        \        P                  ! VRR7      V n        \        P                  ! VP                  4       RR7      V n        R# )Ú	rope_typeÚdefaultF)Ú
persistentN)r(   r)   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenrt   Úrope_parametersrw   Úcompute_default_rope_parametersr   Úattention_scalingr   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)r0   rt   rp   Úrope_init_fnr�   r3   s   &&&  €r4   r)   ÚZamba2RotaryEmbedding.__init__]   s¬   ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àŒàŸ™×4Ñ4°[ÕAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÔ&Ü.¨t¯~©~Õ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÔ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÔ!NˆÖr6   c                óP   <€ V ^8„  d   QhRS[ RS[S[P                  S[3,          /# )r^   rt   r_   )r#   rk   r+   rg   r`   )ra   rZ   s   "€r4   rb   ru   p   s.   ø€ ÷ 5ñ 5±ð 5ÑX]Ñ^c×^jÑ^jÑlqÐ^qÕXrñ 5r6   c           	     ó0  € V P                   R,          p\        V RR4      ;'       g    V P                  V P                  ,          pRpRV\        P
                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚhead_dimNg      ð?©r;   )r}   Úgetattrr1   Únum_attention_headsr+   Úaranger`   r<   )rt   rp   ÚkwargsÚbaseÚdimÚattention_factorr�   s   &&,    r4   r~   Ú5Zamba2RotaryEmbedding.compute_default_rope_parametersn   s~   € ð ×%Ñ% lÕ3ˆÜ�f˜j¨$Ó/×cÐc°6×3EÑ3EÈ×IcÑIcÕ3cˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r6   c                ó^  € V P                   R,          P                  VP                  ^ ,          R	^4      P                  \        P
                  VP                  R7      pVR
,          P                  4       p\        VP                  P                  \        4      '       d2   VP                  P                  R8w  d   VP                  P                  MRp\        VRR7      ;_uu_ 4        W4,          P                  ^^4      p\        P                  ! Wf3R	R7      pVP                  4       V P                  ,          pVP                  4       V P                  ,          p	R R R 4       XP                  VP                   R7      X	P                  VP                   R7      3#   + '       g   i     LH; i)N©r;   rp   ÚmpsÚcpuF)Údevice_typeÚenabled©r�   rŠ   )NºNNNNr:   )rš   Nrš   )r�   Úexpandr@   r<   r+   r`   rp   Ú
isinstanceÚtypeÚstrr   Ú	transposeÚcatÚcosr   Úsinr;   )
r0   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedr—   ÚfreqsÚembr¡   r¢   s
   &&&       r4   rM   ÚZamba2RotaryEmbedding.forward‚   s2  € ð �M‰M˜-Õ(×/Ñ/°×0BÑ0BÀ1Õ0EÀrÈ1ÓM×PÑPÔW\×WbÑWbÐkl×ksÑksÐPÓtð 	ð !-¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆä¨¸U×CÖCØ&Õ>×IÑIÈ!ÈQÓOˆEÜ—)’)˜U˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆC÷	 Dð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷ D×Cús   Ã(A4FÆF,	)r   rt   r�   r{   rƒ   r|   rw   r'   )rR   rS   rT   rU   r   r)   Ústaticmethodr~   r+   Úno_gradr   rM   rV   rW   rX   rY   s   @@r4   ro   ro   \   su   ù‡ € Ù�X vÔ.÷Oô Oó /ðOð  Ù�X vÔ.÷5ñ 5ó /ó ð5ð$ ‡]‚]ƒ_Øñ<ó ó ÷<ð <r6   ro   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r^   rE   Ún_repr_   ©r+   rg   Úint)ra   s   "r4   rb   rb   •   s.   € ÷ 	Uñ 	UœUŸ\™\ð 	U´#ð 	U¼%¿,¹,ñ 	Ur6   c                ó˜   € V P                   w  r#rEV^8X  d   V # V R,          P                  W#WV4      p V P                  W#V,          WE4      # )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
)rš   rš   Nrš   rš   )r@   r›   Úreshape)rE   r­   ÚbatchÚnum_key_value_headsÚslenr‰   s   &&    r4   Ú	repeat_kvrµ   •   sU   € ð
 2?×1DÑ1DÑ.€E Ø�„zØÐØ!Ð"2Õ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÕ(CÀTÓTÐTr6   ç        c                óÞ   € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R\        /# )	r^   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNÚscalingÚdropout)r   ÚModuler+   rg   r`   )ra   s   "r4   rb   rb   ¡   sg   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ð%ô ñ%r6   c                 ó"  € \        W P                  4      p\        W0P                  4      p	\        P                  ! WP	                  ^^4      4      V,          p
Ve	   W¤,           p
\
        P                  P                  V
R\        P                  R7      P                  VP                  4      p
\
        P                  P                  W¦V P                  R7      p
\        P                  ! W©4      pVP	                  ^^4      P                  4       pWº3# )r^   )r�   r;   )ÚpÚtrainingr:   )rµ   Únum_key_value_groupsr+   ÚmatmulrŸ   r   r>   Úsoftmaxr=   r<   r;   r¾   rÂ   Ú
contiguous)r¸   r¹   rº   r»   r¼   r½   r¾   rŽ   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs   &&&&&&&,    r4   Úeager_attention_forwardrË   ¡   sÒ   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÕL€LØÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r6   c                óÌ   € V RRV P                   R,          ^,          13,          pV RV P                   R,          ^,          R13,          p\        P                  ! V) V3RR7      # )z*Rotates half the hidden dims of the input..Nr™   r:   )r@   r+   r    )r£   Úx1Úx2s   &  r4   Úrotate_halfrÏ   º   sZ   € à	
ˆ3Ð"�!—'‘'˜"•+ Õ"Ð"Ð"Õ	#€BØ	
ˆ3�—‘˜•˜qÕ Ñ"Ð"Õ	#€BÜ�9Š9�r�c˜2�Y BÔ'Ð'r6   Úrotary_pos_embc                óÈ   € VP                  V4      pVP                  V4      pW,          \        V 4      V,          ,           pW,          \        V4      V,          ,           pWV3# )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)Ú	unsqueezerÏ   )ÚqÚkr¡   r¢   Úunsqueeze_dimÚq_embedÚk_embeds   &&&&&  r4   Úapply_rotary_pos_embrØ   Á   sS   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�wœ; q›>¨CÕ/Õ0€GØ�wœ; q›>¨CÕ/Õ0€GØÐÐr6   c                   ó\   a a€ ] tR t^Ût oRtRV3R lV 3R llltRV3R lR lltRtVtV ;t	# )ÚZamba2Attentiona*  
Multi-headed attention from 'Attention Is All You Need' paper.

Adapted from transformers.models.mistral.modeling_mistral.MistralAttention:
The input dimension here is attention_hidden_size = 2 * hidden_size, and head_dim = attention_hidden_size // num_heads.
The extra factor of 2 comes from the input being the concatenation of original_hidden_states with the output of the previous (mamba) layer
(see fig. 2 in https://huggingface.co/papers/2405.16712).
Additionally, replaced
attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) with
attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim/2)
Finally, this attention layer contributes to tied transformer blocks aimed to increasing compute without increasing model size. Because this
layer is tied, un-tied adapters (formally the same as LoRA but used in the base model) modules are added to the q, k, v projectors to increase
expressivity with a small memory overhead (see Fig. 2 of https://huggingface.co/papers/2411.15242).
c          	      ó\   <€ V ^8„  d   QhRS[ RS[R,          RS[R,          RS[R,          /# )r^   rt   Ú	layer_idxNÚnum_fwd_mem_blocksÚblock_id©r#   r¯   )ra   rZ   s   "€r4   rb   ÚZamba2Attention.__annotate__ë   sE   ø€ ÷ 6\ñ 6\áð6\ñ ˜•:ð6\ñ   $�Jð	6\ñ
 ˜•*ñ6\r6   c           
     ój	  <€ \         SV `  4        Wn        W n        VP                  V n        VP
                  V n        VP                  VP                  ,          V n	        VP                  V n
        V P                  ^,          R,          V n        RV n        VP                  V n        \        P                  ! VP                  VP                  V P                  ,          RR7      V n        \        P                  ! VP                  VP                  V P                  ,          RR7      V n        \        P                  ! VP                  VP                  V P                  ,          RR7      V n        \        P                  ! VP                  V P                  ,          VP&                  RR7      V n        W0n        VP,                  V n        W@n        VP2                  '       Edœ   \        P4                  ! . 4      V n        \        P4                  ! . 4      V n        \        P4                  ! . 4      V n        \=        V P*                  4       EF0  pWQP>                  ,          V8X  Ed…   \        P@                  ! \        P                  ! V P                  V P                  PB                  RR7      \        P                  ! V P                  PB                  V P                  RR7      4      p\        P@                  ! \        P                  ! V P                  V P                  PB                  RR7      \        P                  ! V P                  PB                  V P                  RR7      4      p\        P@                  ! \        P                  ! V P                  V P                  PB                  RR7      \        P                  ! V P                  PB                  V P                  RR7      4      pM?\        PD                  ! 4       p\        PD                  ! 4       p\        PD                  ! 4       pV P6                  PG                  V4       V P8                  PG                  V4       V P:                  PG                  V4       EK3  	  \I        V P.                  4       U	U
u/ uF  w  ršW©bK	  	  up
p	V n%        R# u up
p	i )r^   TF©ÚbiasNg      à¿)&r(   r)   rt   rÜ   Úattention_hidden_sizeÚattention_head_dimr‰   rŒ   r³   rÃ   rz   r½   Ú	is_causalÚattention_dropoutr   ÚLinearÚq_projÚk_projÚv_projr1   Úo_projrÝ   Úhybrid_layer_idsÚlayer_block_maprÞ   Úuse_shared_attention_adapterÚ
ModuleListÚlinear_q_adapter_listÚlinear_k_adapter_listÚlinear_v_adapter_listÚrangeÚnum_mem_blocksÚ
SequentialÚadapter_rankÚIdentityÚappendÚ	enumerateÚ	layer_dic)r0   rt   rÜ   rÝ   rÞ   ÚiÚlinear_q_adapterÚlinear_k_adapterÚlinear_v_adapterÚindexr»   r3   s   &&&&&      €r4   r)   ÚZamba2Attention.__init__ë   s>  ø€ ô 	‰ÑÔØŒØ"Œà%+×%AÑ%AˆÔ"Ø×1Ñ1ˆŒØ$*×$>Ñ$>À&×B\ÑB\Õ$\ˆÔ!Ø'-×'EÑ'EˆÔ$ØŸ™¨Õ)¨dÕ2ˆŒØˆŒØ!'×!9Ñ!9ˆÔä—i’i × <Ñ <¸f×>XÑ>XÐ[_×[hÑ[hÕ>hÐotÔuˆŒÜ—i’i × <Ñ <¸f×>XÑ>XÐ[_×[hÑ[hÕ>hÐotÔuˆŒÜ—i’i × <Ñ <¸f×>XÑ>XÐ[_×[hÑ[hÕ>hÐotÔuˆŒÜ—i’i × :Ñ :¸T¿]¹]Õ JÈF×L^ÑL^ÐejÔkˆŒØ"4ÔØ%×6Ñ6ˆÔØ Œà×.×.Ñ.Ü)+¯ª°rÓ):ˆDÔ&Ü)+¯ª°rÓ):ˆDÔ&Ü)+¯ª°rÓ):ˆDÔ&ä˜4×2Ñ2×3�Ø×,Ñ,Õ,°Õ8Ü')§}¢}ÜŸ	š	 $×"<Ñ"<¸d¿k¹k×>VÑ>VÐ]bÔcÜŸ	š	 $§+¡+×":Ñ":¸D×<VÑ<VÐ]bÔcó(Ð$ô (*§}¢}ÜŸ	š	 $×"<Ñ"<¸d¿k¹k×>VÑ>VÐ]bÔcÜŸ	š	 $§+¡+×":Ñ":¸D×<VÑ<VÐ]bÔcó(Ð$ô (*§}¢}ÜŸ	š	 $×"<Ñ"<¸d¿k¹k×>VÑ>VÐ]bÔcÜŸ	š	 $§+¡+×":Ñ":¸D×<VÑ<VÐ]bÔcó(Ñ$ô
 (*§{¢{£}Ð$Ü')§{¢{£}Ð$Ü')§{¢{£}Ð$Ø×*Ñ*×1Ñ1Ð2BÔCØ×*Ñ*×1Ñ1Ð2BÔCØ×*Ñ*×1Ñ1Ð2B×Cñ) 4ô, <EÀT×EYÑEYÔ;ZÔ[Ñ;Z©<¨5˜%š,Ñ;ZÒ[ˆŽùÓ[s   ÒR/c                óf  <€ V ^8„  d   QhRS[ P                  RS[RS[ P                  R,          RS[R,          RS[S[ P                  S[ P                  3,          R,          RS[S[,          RS[S[ P                  S[ P                  R,          S[S[ P                  ,          R,          3,          /# )	r^   rE   rÜ   r¼   NÚpast_key_valuesÚposition_embeddingsrŽ   r_   )r+   rg   r¯   r   rk   r   r   )ra   rZ   s   "€r4   rb   rà   #  s¦   ø€ ÷ 1)ñ 1)á—|‘|ð1)ñ ð1)ñ Ÿ™ tÕ+ð	1)ñ
  �ð1)ñ #¡5§<¡<±·±Ð#=Õ>ÀÕEð1)ñ Ñ+Õ,ð1)ñ 
‰u�|‰|™UŸ\™\¨DÕ0±%¹¿¹Õ2EÈÕ2LÐLÕ	Mñ1)r6   c                ó\  € VP                   R R p. VORNV P                  N5pV P                  V4      p	V P                  V4      p
V P	                  V4      pV P
                  P                  '       dq   V P                  V,          pW�P                  V,          ! V4      ,           p	W P                  V,          ! V4      ,           p
W°P                  V,          ! V4      ,           pV	P                  V4      P                  ^^4      p	V
P                  V4      P                  ^^4      p
VP                  V4      P                  ^^4      pV P
                  P                  '       d   Vw  rÞ\        WšWÞ4      w  ršVe   VP                  W«V4      w  r«\         P"                  ! V P
                  P$                  \&        4      pT! V V	V
VV3RV P(                  '       g   RMV P*                  RV P,                  /VB w  ppVP.                  ! . VORN5!  P1                  4       pV P3                  V4      pVV3# )Nr¾   r¶   r½   r:   )r@   r‰   ré   rê   rë   rt   rï   rû   rñ   rò   ró   rA   rŸ   Úuse_mem_roperØ   Úupdater   Úget_interfaceÚ_attn_implementationrË   rÂ   rç   r½   r±   rÆ   rì   )r0   rE   rÜ   r¼   r  r  rŽ   Úinput_shapeÚhidden_shapeÚquery_statesrÇ   rÈ   Úadapter_layer_idxr¡   r¢   Úattention_interfacerÊ   rÉ   s   &&&&&&,           r4   rM   ÚZamba2Attention.forward#  s  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1ˆØ—[‘[ Ó/ˆ
Ø—{‘{ =Ó1ˆØ�;‰;×3×3Ð3Ø $§¡¨yÕ 9ÐØ'×*DÑ*DÐEVÖ*WÐXeÓ*fÕfˆLØ#×&@Ñ&@ÐARÖ&SÐTaÓ&bÕbˆJØ'×*DÑ*DÐEVÖ*WÐXeÓ*fÕfˆLà#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆØ—_‘_ \Ó2×<Ñ<¸QÀÓBˆ
Ø#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆà�;‰;×#×#Ð#Ø*‰HˆCÜ';¸LÐVYÓ'_Ñ$ˆLàÒ&Ø'6×'=Ñ'=¸jÐXaÓ'bÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆ�\ð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r6   )rç   rä   rÞ   rt   r‰   ræ   rê   rî   rû   rÜ   rò   rñ   ró   rz   rÝ   rÃ   rì   ré   r½   rë   ©NNN)
rR   rS   rT   rU   Ú__doc__r)   rM   rV   rW   rX   rY   s   @@r4   rÚ   rÚ   Û   s&   ù‡ € ñ÷6\õ 6\÷p1)÷ 1)ò 1)r6   rÚ   c                óD   € V ^8„  d   QhR\         P                  R\        /# )r^   Úinput_tensorÚpad_sizer®   )ra   s   "r4   rb   rb   Z  s"   € ÷ Vñ V¤U§\¡\ð V¼Sñ Vr6   c                ó²   € \        V P                  4      ^8X  d   ^ ^ ^ ^ ^ V^ ^ 3M^ ^ ^ V^ ^ 3p\        P                  P                  P                  WR^ R7      # )zv
Padding x tensor with `pad_size` on the seq_len dim (dim=1)

Assumes that we only have tensors of either size 4 or 3
Úconstant)Úmoder»   )Úlenr@   r+   r   r>   Úpad)r  r  Ú	pad_shapes   && r4   Úpad_tensor_by_sizer  Z  sd   € ô 47°|×7IÑ7IÓ3JÈaÔ3O��A�q˜!˜Q ¨!¨QÑ/ÐVWÐYZÐ\]Ð_gÐijÐlmÐUn€Iä�8‰8×Ñ×"Ñ" <ÀÐSTÐ"ÓUÐUr6   c                óD  € \        W4      p \        V P                  4      ^8X  d6   V P                  V P                  ^ ,          RW P                  ^,          4      # V P                  V P                  ^ ,          RW P                  ^,          V P                  ^,          4      # )z°
Padding input_tensor with `pad_size` on the seq_len dim (dim=1) and
simultaneously splitting it into chunk sequences.

Assumes that we only have tensors of either size 4 or 3
r:   )r  r  r@   r±   )r  r  Ú
chunk_sizes   &&&r4   Úreshape_into_chunksr  e  s�   € ô & lÓ=€Lä
ˆ<×ÑÓ !Ô#à×#Ñ# L×$6Ñ$6°qÕ$9¸2¸z×K]ÑK]Ð^_ÕK`ÓaÐað ×#Ñ#Ø×Ñ˜qÕ! 2 z×3EÑ3EÀaÕ3HÈ,×J\ÑJ\Ð]^ÕJ_ó
ð 	
r6   c           	     ó2  € V P                  R4      pV R,          P                  ! . V P                  4       OVN5!  p \        P                  ! \        P                  ! WV P
                  \        P                  R7      RR7      pV P                  V( ^ 4      p \        P                  ! V RR7      p\        P                  ! \        P                  ! WV P
                  \        P                  R7      ^ R7      pVP                  V( \        P                  ) 4      pV# )zg
More stable segment sum calculation. Uses cumulative sums and masking instead of direct subtractions.
©rp   r;   )Údiagonalr™   r:   ©.Néþÿÿÿ)
Úsizer›   r+   Útrilr,   rp   ÚboolÚmasked_fillÚcumsumÚinf)r  r  ÚmaskÚtensor_segsums   &   r4   Úsegment_sumr,  y  sØ   € ð ×"Ñ" 2Ó&€Jð   	Õ*×1Ò1ÐS°<×3DÑ3DÓ3FÐSÈ
ÓS€Lä�:Š:”e—j’j À×@SÑ@SÔ[`×[eÑ[eÔfÐqsÔt€DØ×+Ñ+¨T¨E°1Ó5€Lä—L’L °2Ô6€Mô �:Š:”e—j’j À×@SÑ@SÔ[`×[eÑ[eÔfÐqrÔs€DØ!×-Ñ-¨t¨e´e·i±i°ZÓ@€MØÐr6   c                ób   € Ve+   V P                   pWR,          ,          P                  V4      p V # )ze
Tunes out the hidden states for padding tokens, see https://github.com/state-spaces/mamba/issues/66
©rš   rš   N)r;   r<   )rE   r¼   r;   s   && r4   Úapply_mask_to_padding_statesr/  �  s3   € ð
 Ò!Ø×#Ñ#ˆØ&¸
Õ)CÕC×GÑGÈÓNˆàÐr6   Úcausal_conv1d_updateÚcausal_conv1dc                óÀ   € V ^8„  d   QhR\         P                  R\         P                  R\        P                  R\        P                  R,          R\        R,          /# )r^   rE   Ú
conv_stater-   rã   NÚ
activation©r+   rg   r   r*   rž   )ra   s   "r4   rb   rb   š  sS   € ÷ 'ñ 'Ü—<‘<ð'ä—‘ð'ô �L‰Lð'ô �,‰,˜Õ
ð	'ô
 �d•
ñ'r6   c                 óº  € V P                   w  rVpVP                   R,          p\        P                  ! W.RR7      P                  VP                  4      p	VP                  V	RRV) R13,          4       \        P                  ! W’P                  ^4      V^ VR7      p
V
RRV) R13,          p
Ve   \        V,          ! V
4      p
V
P                  V P                  4      # )é   r™   rš   N)ÚpaddingÚgroupsr:   )
r@   r+   r    r<   r;   Úcopy_ÚFÚconv1drÒ   r   )rE   r3  r-   rã   r4  Ú_r1   Úseq_lenÚ	state_lenÚhidden_states_newÚouts   &&&&&      r4   r0  r0  ™  sÅ   € ð ,×1Ñ1Ñ€A�GØ× Ñ  Õ$€IäŸ	š	 :Ð"=À2ÔF×IÑIÈ&Ï,É,ÓWÐØ×ÑÐ& q¨!¨i¨Z©[Ð'8Õ9Ô:Ü
�(Š(Ð$×&6Ñ&6°qÓ&9¸4ÈÐS^Ô
_€CØ
ˆa��W�H‘IˆoÕ
€CØÒÜ�ZÖ  Ó%ˆØ�6‰6�-×%Ñ%Ó&Ð&r6   Úcausal_conv1d_fnc          	      ó    € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R,          R\        R,          /# )r^   rE   r-   rã   Nr4  r5  )ra   s   "r4   rb   rb   ®  sE   € ÷ 'ñ 'Ü—<‘<ð'ä�L‰Lð'ô �,‰,˜Õ
ð'ô �d•
ñ	'r6   c                 óV  € V P                   w  rVpVP                   R,          ^,
          p\        P                  ! V P                  VP                  4      VP                  ^4      VVVR7      RRRV13,          p	Ve   \        V,          ! V	4      p	V	P                  V P                  4      # )r7  )r-   rã   r8  r9  rš   Nr:   )r@   r;  r<  r<   r;   rÒ   r   )
rE   r-   rã   r4  rŽ   r=  r1   r>  r8  rA  s
   &&&&,     r4   rB  rB  ­  sž   € ð ,×1Ñ1Ñ€A�GØ�l‰l˜2Õ Õ"€Gä
�(Š(Ø×Ñ˜Ÿ™Ó&Ø×Ñ Ó"ØØØôð ˆˆHˆWˆH€nõ€Cð ÒÜ�ZÖ  Ó%ˆØ�6‰6�-×%Ñ%Ó&Ð&r6   Ú mamba_split_conv1d_scan_combinedÚ	mamba_ssmr)  Fr?   rP   Tc          $      ó.  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R,          R\         P                  R\         P                  R\         P                  R\        R	\         P                  R,          R
\        \        \        3,          R\
        R\        R\         P                  R,          R\        R\         P                  R,          R\         P                  R,          R\        R,          R\        R\
        /# )r^   ÚzxbcdtÚconv1d_weightÚconv1d_biasNÚdt_biasÚAÚDr  Úinitial_statesÚdt_limitÚreturn_final_statesr4  Úrmsnorm_weightÚrmsnorm_epsÚoutproj_weightÚoutproj_biasÚheaddimÚngroupsÚnorm_before_gate)r+   rg   r¯   rk   r`   r&  rž   )ra   s   "r4   rb   rb   Å  s
  € ÷ ñ Ü�L‰Lðä—<‘<ðô —‘ Õ$ðô �\‰\ð	ô
 ‡|�|ðô ‡|�|ðô ðô —L‘L 4Õ'ðô ”Eœ5�LÕ!ðô ðô ðô —L‘L 4Õ'ðô ðô —L‘L 4Õ'ðô —,‘, Õ%ðô  �4�Zð!ô" ð#ô$ ñ%r6   c                 ó   € R # r'   © )rH  rI  rJ  rK  rL  rM  r  rN  rO  rP  r4  rQ  rR  rS  rT  rU  rV  rW  rŽ   s   &&&&&&&&&&&&&&&&&&,r4   Ú!mamba2_split_conv1d_scan_combinedrZ  Ä  s   € ñ, r6   Úselective_state_updatec                ón  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R,          R	\         P                  R,          R
\        R\         P                  R,          /
# )r^   ÚstaterE   ÚdtrL  ÚBÚCrM  NrK  Údt_softplusÚz)r+   rg   r&  )ra   s   "r4   rb   rb   Þ  sŸ   € ÷ :'ñ :'Ü�<‰<ð:'ä—<‘<ð:'ô 	�‰ð:'ô ‡|�|ð	:'ô
 ‡|�|ð:'ô ‡|�|ð:'ô ‡|�|�dÕð:'ô �\‰\˜DÕ ð:'ô ð:'ô ‡|�|�dÕñ:'r6   c
                 ó†  € VP                   w  r¼pVP                   ^,          pVP                   R,          pVe"   W'P                  VP                  4      ,           pV'       d   \        P                  ! V4      pVR,          p\
        P                  ! VP                  4       VP                  4       ,          4      P                  V P                  R7      pVP                  W¾^V4      pVP                  W¾WÎ,          V4      P                  4       pVP                  W¼^V4      pW$,          pVVR,          ,          P                  V P                  R7      pV V,          V,           pV P                  VP                  V P                  4      4       VP                  W¾^V4      pVP                  W¾WÎ,          V4      P                  4       pVP                  W¼V4      pVP                  VP                  VP                  R7      pVP                  W¼,          Wß4      pVP                  W¼,          V^4      p\
        P                  ! VV4      pVP                  W¼V4      pVe)   VW,          ,           P                  VP                  4      pV	e   V\        P                  ! V	4      ,          pVP                  VP                  4      # )r7  ©rp   r   r:   r"  )r@   r<   r;   r;  Úsoftplusr+   Úexpr`   rp   r±   r›   rÆ   r:  rA   Úbmmr?   )r]  rE   r^  rL  r_  r`  rM  rK  ra  rb  rŽ   Ú
batch_sizeÚ	num_headsr‰   Ú
num_groupsÚ
state_sizeÚdAÚdBÚdBxÚ
ssm_statesÚssm_states_reshapedÚ
C_reshapedrA  s   &&&&&&&&&&,            r4   Úmamba2_selective_state_updaterr  Ý  s"  € ð '4×&9Ñ&9Ñ#€J˜8Ø—‘˜•€JØ—‘˜•€JàÒØ—*‘*˜RŸX™XÓ&Õ&ˆßÜ�ZŠZ˜‹^ˆØ	ˆI�€Bô 
�Š�2—8‘8“: §¡£	Õ)Ó	*×	-Ñ	-°U·\±\Ð	-Ó	B€Bð 	
�	‰	�*¨!¨ZÓ8€AØ	�‰�¨Õ)@À*ÓM×XÑXÓZ€AØ	�	‰	�*¨¨JÓ7€AØ	�€Bð �˜iÕ(Õ(×
,Ñ
,°E·L±LÐ
,Ó
A€Cð ˜•˜cÕ!€JØ	‡K�K�
—‘˜eŸk™kÓ*Ô+ð 	
�	‰	�*¨!¨ZÓ8€AØ	�‰�¨Õ)@À*ÓM×XÑXÓZ€AØ	�	‰	�*¨Ó4€Að —‘ a§h¡h°a·g±g�Ó>€JØ$Ÿ/™/¨*Õ*@À(ÓWÐØ—‘˜
Õ.°
¸AÓ>€JÜ
�)Š)Ð'¨Ó
4€CØ
�(‰(�:¨(Ó
3€Cð 	‚}Ø�]Õ&Õ&×*Ñ*¨3¯9©9Ó5ˆà‚}Ø”A—F’F˜1“I�oˆà�6‰6�-×%Ñ%Ó&Ð&r6   Úmamba_chunk_scan_combinedc                ó”  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R\        R\         P                  R,          R	\         P                  R,          R
\         P                  R,          R\        R\        \
        \
        3,          R\        /# )r^   rE   r^  rL  r_  r`  r  rM  NrK  rN  ra  rO  rP  )r+   rg   r¯   r&  rk   r`   )ra   s   "r4   rb   rb     sÆ   € ÷ ^ñ ^Ü—<‘<ð^ä�‰ð^ô ‡|�|ð^ô ‡|�|ð	^ô
 ‡|�|ð^ô ð^ô ‡|�|�dÕð^ô �\‰\˜DÕ ð^ô —L‘L 4Õ'ð^ô ð^ô ”Eœ5�LÕ!ð^ô ñ^r6   c                 óô  € V P                   w  rÞppVP                   ^,          pVe"   WP                  VP                  4      ,           pV	'       d   \        P                  ! V4      p\
        P                  ! W^ ,          V
^,          R7      pV P                  4       p VP                  4       P                  VV,          ^VR7      pVP                  4       P                  VV,          ^VR7      pW^V,          ,
          V,          pRpVe   VR,          \        V V4      ,          pWR,          P                  4       ,          p VP                  V P                  4      VP                  4       ,          pWW43 Uu. uF  p\        VVV4      NK  	  upw  rr4VP                  ^ ^^^4      p\
        P                  ! VRR7      p\
        P                  ! \        V4      4      pVR	,          VR
,          ,          P                  RR7      pVR,          VP                  ^ ^^^^4      R,          ,          P                  RR7      pVR,          V R,          ,          P                  ^R7      p\
        P                  ! VRRRRR13,          V,
          4      pVVP                  ^ RR^4      R,          ,          pVR,          V R,          ,          P                  ^R7      pVe/   VR,          P                  VP                  VP                   R7      M\
        P"                  ! VR,          4      p\
        P$                  ! VV.^R7      p\
        P                  ! \        \        P&                  ! VR,          R4      4      4      P)                  ^^4      pVR,          VR,          ,          P                  ^R7      pVRRR13,          VR,          p p\
        P                  ! V4      p!VR,          VR,          ,          p"V"P                  R4      V!P                  ^ ^^^4      R,          ,          p#VV#,           p$V$P+                  VRVV4      p$Ve
   V$V,           p$V^ 8”  d   V$RRV13,          p$V'       d   V$V 3# V$# u upi )r^   N)ÚminÚmax)r�   Úoutput_sizer™   rš   r”   r"  r:   )rš   rš   rš   Nrš   rš   )rš   rš   Nrš   rš   rš   r.  r#  ).Nrš   ©rš   N)rš   :Nr7  N)rš   rš   rš   r:   )r7  r   ).NN)rš   rš   N.)rš   r:   )r@   r<   r;   r;  re  r+   Úclampr`   Úrepeat_interleaver  r  Úpermuter(  rf  r,  Úsumrp   Ú
zeros_liker    r  rŸ   r±   )%rE   r^  rL  r_  r`  r  rM  rK  rN  ra  rO  rP  rŽ   rh  Úsequence_lengthri  r‰   rj  r  Ú
D_residualÚtensorÚA_cumsumÚLÚGÚMÚY_diagÚdecay_statesÚB_decayÚstatesÚprevious_statesÚdecay_chunkÚ
new_statesÚfinal_stateÚstate_decay_outÚC_times_statesÚY_offÚoutputs%   &&&&&&&&&&&&,                        r4   Úmamba2_chunk_scanr’    së  € ð  8E×7JÑ7JÑ4€J ¨HØ—‘˜•€JàÒØ—*‘*˜RŸX™XÓ&Õ&ˆßÜ�ZŠZ˜‹^ˆÜ	�Š�R a�[¨h°q­kÔ	:€Bà!×'Ñ'Ó)€MØ	�‰‹	×#Ñ# I°Õ$;ÀÐPYÐ#ÓZ€AØ	�‰‹	×#Ñ# I°Õ$;ÀÐPYÐ#ÓZ€Aà¨zÕ9Õ9¸ZÕG€HØ€JØ‚}Ø�y•\Ô$6°}ÀhÓ$OÕOˆ
ð " y¥M×$7Ñ$7Ó$9Õ9€MØ	�‰ˆ]× Ñ Ó! B§H¡H£JÕ.€Að `mÐqrÑ^vÓwÑ^vÐTZÔ1°&¸(ÀJÖOÑ^vÑwÑ€M�aà	�	‰	�!�Q˜˜1Ó€AÜ�|Š|˜A 2Ô&€Hô 	�	Š	”+˜a“.Ó!€Að 
ÐÕ	 !Ð$7Õ"8Õ	8×=Ñ=À"Ð=ÓE€Að 
ˆ9�˜Ÿ	™	 ! Q¨¨1¨aÓ0°Õ;Õ	;×@Ñ@ÀRÐ@ÓH€Að �	�l˜]¨:Õ6Õ6×;Ñ;ÀÐ;ÓB€Fô —9’9˜X a¨¨A¨r©s lÕ3°hÕ>Ó?€LØ�,×&Ñ& q¨"¨b°!Ó4°YÕ?Õ?€GØ�lÕ# m°IÕ&>Õ>×CÑCÈÐCÓJ€Fð Ò%ð 	�wÕ×"Ñ"¨¯©¸f¿m¹mÐ"ÔLä×Ò˜f U�mÓ,ð ô
 �YŠY˜¨Ð0°aÔ8€FÜ—)’)œK¬¯ª¨h°{Õ.CÀVÓ(LÓMÓN×XÑXÐYZÐ\]Ó^€KØ˜oÕ.°¸Õ1HÕH×MÑMÐRSÐMÓT€JØ$ Q¨¨¨ VÕ,¨j¸Õ.?ˆK€Fô —i’i Ó)€OØ�|•_ v¨oÕ'>Õ>€NØ×Ñ˜rÓ" _×%<Ñ%<¸QÀÀ1ÀaÓ%HÈÕ%SÕS€Eð �e�^€FØ�^‰^˜J¨¨I°xÓ@€FàÒØ˜*Õ$ˆð �!„|Ø˜Ð+˜OÐ+Ð+Õ,ˆçØ�{Ð"Ð"à€Mùòq xs   Å8Q5c                   ó    a a€ ] tR tRt oRtR
V3R lV 3R lllt]P                  ! 4       R 4       t]	! R4      RV3R lR ll4       t
R	tVtV ;t# )ÚZamba2MambaMixeri}  uo  
Compute âˆ†, A, B, C, and D the state space parameters and compute the `contextualized_states`.
A, D are input independent (see Mamba paper [1] Section 3.5.2 "Interpretation of A" for why A isn't selective)
âˆ†, B, C are input-dependent (this is a key difference between Mamba and the linear time invariant S4,
and is why Mamba is called **selective** state spaces)
c                ó:   <€ V ^8„  d   QhRS[ RS[R,          RS[/# )r^   rt   rÜ   NÚinitialize_mixer_weights)r#   r¯   r&  )ra   rZ   s   "€r4   rb   ÚZamba2MambaMixer.__annotate__Ž  s'   ø€ ÷ 48ñ 48™|ð 48¹¸d½
ð 48Ñeiñ 48r6   c           	     ó   <€ \         SV `  4        Wn        V P                  P                  V n        VP
                  V n        VP                  V n        VP                  V n	        \        VP                  V P
                  ,          4      V n        W n        VP                  V n        R V n        \         P"                  ! 4       V n        VP&                  V n        VP*                  V n        VP.                  V n        VP0                  \3        R4      3V n        V P                  ^V P(                  ,          V P                  ,          ,           V n        \         P8                  ! V P6                  V P6                  RVP                  V P6                  VP                  ^,
          R7      V n        V P                  V P6                  ,           V P                  ,           p\         P<                  ! V P
                  VVP>                  R7      V n         \         PB                  ! \D        PF                  ! V P                  4      4      V n$        \         PB                  ! \D        PF                  ! V P                  4      4      V n%        \M        V P                  V P                  V P(                  ,          RR7      V n'        \         PB                  ! \D        PF                  ! V P                  4      4      V n(        V'       d6   V PH                  PR                  PT                  R8w  d   V PW                  4        \         P<                  ! V P                  V P
                  VP>                  R7      V n,        VPZ                  V,          V n.        R# )	r?   r)  T)Úin_channelsÚout_channelsrã   Úkernel_sizer9  r8  râ   gñhãˆµøä>)r/   r2   ÚmetaN)/r(   r)   rt   Ún_mamba_headsri  r1   Úmamba_d_stateÚssm_state_sizeÚmamba_d_convÚconv_kernel_sizer¯   Úmamba_expandÚintermediate_sizerÜ   Úuse_conv_biasr4  r   ÚSiLUÚactÚmamba_ngroupsÚn_groupsÚmamba_headdimr‰   r  Útime_step_minr`   Útime_step_limitÚconv_dimÚConv1dr<  rè   Úadd_bias_linearÚin_projr*   r+   ÚemptyrK  ÚA_logr%   ÚnormrM  rp   r�   Úinit_zamba2_weightsÚout_projÚlayer_typesÚ
layer_type)r0   rt   rÜ   r–  Úprojection_sizer3   s   &&&& €r4   r)   ÚZamba2MambaMixer.__init__Ž  s_  ø€ Ü‰ÑÔØŒØŸ™×2Ñ2ˆŒØ!×-Ñ-ˆÔØ$×2Ñ2ˆÔØ &× 3Ñ 3ˆÔÜ!$ V×%8Ñ%8¸4×;KÑ;KÕ%KÓ!LˆÔØ"ŒØ#×1Ñ1ˆÔØ ˆŒÜ—7’7“9ˆŒØ×,Ñ,ˆŒØ×,Ñ,ˆŒØ ×+Ñ+ˆŒà &× 4Ñ 4´e¸E³lÐCˆÔà×.Ñ.°°T·]±]Õ1BÀT×EXÑEXÕ1XÕXˆŒä—i’iØŸ™ØŸ™ØØ×+Ñ+Ø—=‘=Ø×'Ñ'¨!Õ+ô
ˆŒð ×0Ñ0°4·=±=Õ@À4Ç>Á>ÕQˆÜ—y’yØ×ÑØØ×'Ñ'ô
ˆŒô —|’|¤E§K¢K°·±Ó$?Ó@ˆŒô —\’\¤%§+¢+¨d¯n©nÓ"=Ó>ˆŒ
Ü&Ø×"Ñ"¨t×/EÑ/EÈÏÉÕ/VÐ\`ô
ˆŒ	ô —’œeŸkšk¨$¯.©.Ó9Ó:ˆŒß#¨¯©×(;Ñ(;×(@Ñ(@ÀFÔ(JØ×$Ñ$Ô&ÜŸ	š	 $×"8Ñ"8¸$×:JÑ:JÐQW×QgÑQgÔhˆŒà ×,Ñ,¨YÕ7ˆŽr6   c                óŠ  € \         P                  ! ^V P                  ^,           V P                  P                  \         P
                  R7      p\        P                  ! V P                  \         P                  ! V4      4       \        P                  ! V P                  4       \        P                  ! V P                  4       R# )r7  r   N)r+   r�   ri  r±  rp   r=   Úinitr:  ÚlogÚones_rM  rK  )r0   rL  s   & r4   r³  Ú$Zamba2MambaMixer.init_zamba2_weightsÄ  sj   € ä�LŠL˜˜DŸN™N¨QÕ.°t·z±z×7HÑ7HÔPU×P]ÑP]Ô^ˆÜ�
Š
�4—:‘:œuŸyšy¨›|Ô,Ü�
Š
�4—6‘6ÔÜ�
Š
�4—<‘<Ö r6   r<  c                óp   <€ V ^8„  d   QhRS[ P                  RS[R,          RS[ P                  R,          /# )r^   rE   Úcache_paramsNr¼   )r+   rg   r   )ra   rZ   s   "€r4   rb   r—  Ì  s?   ø€ ÷ P%ñ P%á—|‘|ðP%ñ ˜d•lðP%ñ Ÿ™ tÕ+ñ	P%r6   c                ó  € VP                   w  rVpVP                  pVR J;'       d    VP                  V P                  4      p	\	        W4      pV P                  V4      p
\        P                  ! V P                  P                  4       4      ) pVRV P                  /,          pV P                  '       dô   Vfð   \        V
V P                  P                  P                  ^4      V P                  P                   V P"                  V3RV P$                  RV P&                  RV P(                  RV P*                  P                  RV P*                  P,                  RV P.                  P                  RV P.                  P                   R	V P0                  R
V P2                  RRRR/VB pVe   V# V
P5                  V P6                  V P8                  V P:                  .RR7      w  rïpV	'       d]   VP<                  V P                  ,          P>                  ^ ,          pVP<                  V P                  ,          P@                  ^ ,          pVPC                  ^^4      pV	'       d‡   V^8X  d€   VP<                  V P                  ,          PD                  '       gS   \G        VXV P                  P                  P                  ^4      V P                  P                   V P(                  R7      pM�Ve)   VPI                  VV P                  V PJ                  R7      p\M        VV P                  P                  P                  ^4      V P                  P                   3RV P(                  /VB pVe   VRRV) R 13,          p\	        VPC                  ^^4      V4      p\        P4                  ! VV P6                  V P2                  V PN                  ,          V P2                  V PN                  ,          .RR7      w  pppV	'       Ed]   V^8X  EdU   VPQ                  WPP:                  V P0                  4      pVPC                  ^^4      PS                  RRV P0                  4      pVR,          PS                  RV P0                  V PN                  4      pVPQ                  WPP2                  V PN                  4      pVPQ                  WPP2                  V PN                  4      pV P$                  R,          PS                  RV P0                  4      pV P"                  R,          PS                  RV P0                  4      p\U        XVVVVVV3RR RVRR/VB pVPQ                  V^R4      pEMVR Jp\W        VPQ                  WVV P:                  V P0                  4      VVVPQ                  WVV P2                  V PN                  4      VPQ                  WVV P2                  V PN                  4      3RV P&                  RV P$                  RR RTRV P"                  RRRV	'       d   XMR RV P                  /VB pV'       d$   Vw  ppVPY                  VV P                  R7       MTpVP[                  WVR4      pV P+                  VV4      pV P/                  VP]                  V4      4      pV# )NrO  rM  r  r4  rQ  rR  rS  rT  rU  rV  rW  FrP  r™   )r4  )r¡  rš   rb  rK  ra  TrN  ©rÜ   r:   )rš   NNry  )/r@   r;   Úhas_previous_staterÜ   r/  r¯  r+   rf  r±  r`   r«  rÂ   rZ  r<  r-   Úsqueezerã   rK  rM  r  r4  r²  r.   r´  r‰   r¨  Úsplitr£  r¬  ri  ÚlayersÚconv_statesÚrecurrent_statesrŸ   Úrecord_pastr0  Úupdate_conv_stater¡  rB  rŸ  rA   r›   rr  r’  Úupdate_recurrent_stater±   r<   )r0   rE   r¿  r¼   rŽ   rh  r>  r=  r;   Úuse_precomputed_statesÚprojected_statesrL  Úfused_kwargsÚfused_outputrF   Úhidden_states_B_Cr^  r3  Úrecurrent_stater_  r`  rM  rK  Úscan_outputÚoutput_final_stateÚscan_resultr�  Úcontextualized_statess   &&&&,                       r4   rM   ÚZamba2MambaMixer.forwardË  s  € ð "/×!4Ñ!4Ñˆ
˜QØ×#Ñ#ˆØ!-°TÐ!9×!mÐ!m¸l×>]Ñ>]Ð^b×^lÑ^lÓ>mÐô 5°]ÓSˆØŸ<™<¨Ó6Ðä�YŠY�t—z‘z×'Ñ'Ó)Ó*Ð*ˆØ ¨T×-AÑ-AÐ BÕBˆØ�=�=ˆ=˜\Ò1Ü<Ø Ø—‘×"Ñ"×*Ñ*¨1Ó-Ø—‘× Ñ Ø—‘Øñð —&‘&ðð  Ÿ?™?ðð  Ÿ?™?ðð  $Ÿy™y×/Ñ/ðð !ŸI™I×6Ñ6ðð  $Ÿ}™}×3Ñ3ðð "Ÿ]™]×/Ñ/ðð Ÿ™ðð Ÿ™ðð "'ðð  %*ð!ð" ñ#ˆLð* Ò'Ø#Ð#à&6×&<Ñ&<Ø×#Ñ# T§]¡]°D·N±NÐCÈð '=ó '
Ñ#ˆ ÷ "Ø%×,Ñ,¨T¯^©^Õ<×HÑHÈÕKˆJØ*×1Ñ1°$·.±.ÕA×RÑRÐSTÕUˆOð .×7Ñ7¸¸1Ó=Ðß! g°¤l¸<×;NÑ;NÈtÏ~É~Õ;^×;j×;jÐ;jÜ 4Ø!ØØ—‘×"Ñ"×*Ñ*¨1Ó-Ø—‘× Ñ ØŸ?™?ô!Ñð Ò'Ø$0×$BÑ$BØ%Ø—N‘NØ%)×%:Ñ%:ð %Có %Ð!ô !1Ø!Ø—‘×"Ñ"×*Ñ*¨1Ó-Ø—‘× Ñ ñ!ð  Ÿ?™?ð	!ð
 ñ!Ðð Ò'Ø$5°a¸¸W¸H¹I°oÕ$FÐ!ô 9Ð9J×9TÑ9TÐUVÐXYÓ9ZÐ\jÓkÐÜ#ŸkškØØ×#Ñ# T§]¡]°T×5HÑ5HÕ%HÈ$Ï-É-ÐZ^×ZmÑZmÕJmÐnØô
Ñˆ�q˜!÷ "Ð! g°¥lØ)×.Ñ.¨z¿>¹>È4Ï=É=ÓYˆMØ—‘˜a Ó#×*Ñ*¨2¨r°4·=±=ÓAˆBØ�-Õ ×'Ñ'¨¨D¯M©M¸4×;NÑ;NÓOˆAØ—‘�z§=¡=°$×2EÑ2EÓFˆAØ—‘�z§=¡=°$×2EÑ2EÓFˆAØ—‘�w•×&Ñ& r¨4¯=©=Ó9ˆAØ—l‘l 7Õ+×2Ñ2°2°t·}±}ÓEˆGä7ØØØØØØØñð ðð  ðð !ðð ñˆKð &×*Ñ*¨:°q¸"Ó=ŠKð ".°TÐ!9ÐÜ+Ø×"Ñ" :¸¿¹ÈÏÉÓVØØØ—‘�z¨D¯M©M¸4×;NÑ;NÓOØ—‘�z¨D¯M©M¸4×;NÑ;NÓOñð  Ÿ?™?ðð —&‘&ðð ðð %7ðð Ÿ™ðð !ð÷ 3I™Èdðð ×-Ñ-ðð ñˆK÷" "Ø+6Ñ(�˜[Ø×3Ñ3°KÈ4Ï>É>Ð3ÕZà)�à%×-Ñ-¨jÀ2ÓFˆKà—i‘i ¨TÓ2ˆð !%§¡¨k¯n©n¸UÓ.CÓ DÐØ$Ð$r6   )r±  rM  r¦  r4  r  rt   r<  r¬  r¡  rK  r‰   r1   r¯  r£  rÜ   r¶  r¨  r²  ri  r´  rŸ  r«  r¤  )NT©NN)rR   rS   rT   rU   r  r)   r+   r«   r³  r   rM   rV   rW   rX   rY   s   @@r4   r”  r”  }  sV   ù‡ € ñ÷48õ 48ðl ‡]‚]ƒ_ñ!ó ð!ñ ˜HÓ%÷P%ñ P%ó &÷P%ð P%r6   r”  c                   óL   a a€ ] tR tRt oRV3R lV 3R llltRR ltRtVtV ;t# )Ú	Zamba2MLPi_  c                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# )r^   rt   rÞ   Nrß   )ra   rZ   s   "€r4   rb   ÚZamba2MLP.__annotate__`  s&   ø€ ÷ Wñ W™|ð WÑPSÐVZÕPZñ Wr6   c           
     óP  <€ \         S	V `  4        Wn        VP                  V n        VP                  V n        W n        W0n        \        P                  ! V P                  ^V P                  ,          VP                  R7      V n
        \        P                  ! V P                  V P                  VP                  R7      V n        \        VP                  ,          V n        \        P                  ! . 4      V n        \#        V P
                  4       FÝ  pWAP$                  ,          V8X  d”   \        P&                  ! \        P                  ! V P                  P                  V P                  P(                  RR7      \        P                  ! V P                  P(                  ^V P                  ,          RR7      4      pM\        P*                  ! 4       pV P                   P-                  V4       Kß  	  VP.                  p\1        V4       UUu/ uF  w  rxW‡bK	  	  uppV n        R# u uppi )a9  
This MLP layer contributes to tied transformer blocks aimed to increasing compute without increasing model size. Because this layer
is tied, un-tied adapter modules (formally same as LoRA, but used in the base model) are added to the up and gate projectors to increase expressivity with a small memory overhead.
râ   FN)r(   r)   rt   r1   r£  rÝ   rÞ   r   rè   r®  Úgate_up_projÚ	down_projr   Ú
hidden_actÚact_fnrð   Úgate_up_proj_adapter_listrô   rõ   rö   r÷   rø   rù   rí   rú   rû   )
r0   rt   rÝ   rÞ   rü   Úgate_up_proj_adapterrî   r   r»   r3   s
   &&&&     €r4   r)   ÚZamba2MLP.__init__`  s‡  ø€ ô
 	‰ÑÔØŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔØ"4ÔØ ŒäŸIšI d×&6Ñ&6¸¸D×<RÑ<RÕ8RÐY_×YoÑYoÔpˆÔÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRX×RhÑRhÔiˆŒÜ˜V×.Ñ.Õ/ˆŒä)+¯ª°rÓ):ˆÔ&Ü�t×.Ñ.Ö/ˆAØ×(Ñ(Õ(¨HÔ4Ü')§}¢}Ü—I’I˜dŸk™k×5Ñ5°t·{±{×7OÑ7OÐV[Ô\Ü—I’I˜dŸk™k×6Ñ6¸¸D×<RÑ<RÕ8RÐY^Ô_ó(Ñ$ô
 (*§{¢{£}Ð$Ø×*Ñ*×1Ñ1Ð2FÖGñ 0ð !×1Ñ1ˆÜ;DÀ_Ô;UÔVÑ;U©<¨5˜%š,Ñ;UÒVˆŽùÓVs   È
H"c                ó,  € V P                  V4      pV P                  V,          pW0P                  V,          ! V4      ,           p\        P                  ! V^RR7      pV P                  V^ ,          4      V^,          ,          pV P                  V4      pV# )r^   r™   r:   )rÜ  rû   rà  r+   Úchunkrß  rÝ  )r0   Úhidden_staterÜ   Úgate_up_stater‘  s   &&&  r4   rM   ÚZamba2MLP.forward~  s{   € Ø×)Ñ)¨,Ó7ˆØ—N‘N 9Õ-ˆ	Ø%×(FÑ(FÀyÖ(QÐR^Ó(_Õ_ˆäŸš M°1¸"Ô=ˆØ—{‘{ =°Õ#3Ó4°}ÀQÕ7GÕGˆØ—‘ Ó-ˆØˆr6   )
rß  rÞ   rt   rÝ  rÜ  rà  r1   r£  rû   rÝ   rÖ  r'   rQ   rY   s   @@r4   rØ  rØ  _  s   ù‡ € ÷Wõ W÷<ô r6   rØ  c                   óX   a a€ ] tR tRt oRV3R lV 3R llltRV3R lR lltRtVtV ;t# )	ÚZamba2AttentionDecoderLayeri‰  c                óH   <€ V ^8„  d   QhRS[ RS[R,          RS[R,          /# )r^   rt   rÞ   NrÜ   rß   )ra   rZ   s   "€r4   rb   Ú(Zamba2AttentionDecoderLayer.__annotate__Š  s2   ø€ ÷ [ñ [™|ð [±s¸Tµzð [ÑUXÐ[_ÕU_ñ [r6   c                ó<  <€ \         SV `  4        W n        \        VP                  4      p\        VRWBR7      V n        \        WVR7      V n        \        VP                  VP                  R7      V n        \        VP                  VP                  R7      V n        R# )r7  )rÜ   rÝ   rÞ   )rÝ   rÞ   ©r2   Nr:   )r(   r)   rÞ   r  rí   rÚ   Ú	self_attnrØ  Úfeed_forwardr\   rä   Úrms_norm_epsÚinput_layernormr1   Úpre_ff_layernorm)r0   rt   rÞ   rÜ   Únum_gsr3   s   &&&& €r4   r)   Ú$Zamba2AttentionDecoderLayer.__init__Š  sz   ø€ Ü‰ÑÔØ ŒÜ�V×,Ñ,Ó-ˆÜ(¨¸2ÐRXÔlˆŒÜ% fÐRZÔ[ˆÔÜ,¨V×-IÑ-IÈv×ObÑObÔcˆÔÜ -¨f×.@Ñ.@Àf×FYÑFYÔ ZˆÖr6   c                óø   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[RS[ P                  R,          RS[R,          RS[ P                  R,          RS[S[,          R	S[S[ P                  ,          /# )
r^   rE   Úoriginal_hidden_statesrÜ   r¼   Nr  r  rŽ   r_   )	r+   rg   r¯   r   Ú
LongTensorr   r   rk   ÚFloatTensor)ra   rZ   s   "€r4   rb   rë  “  sŒ   ø€ ÷ )ñ )á—|‘|ð)ñ !&§¡ð)ñ ð	)ñ
 Ÿ™ tÕ+ð)ñ  �ð)ñ #×-Ñ-°Õ4ð)ñ Ñ+Õ,ð)ñ 
‰u× Ñ Õ	!ñ)r6   c                óÚ   € \         P                  ! W.RR7      pV P                  V4      pV P                  ! RRVRVRVRVRV/VB w  rV P	                  V4      pV P                  W4      pV# )	aš  
Args:
    hidden_states (`torch.FloatTensor`): output of previous Mamba layer of shape `(batch, seq_len, embed_dim)`
    original_hidden_states (`torch.FloatTensor`): word embedding output of shape `(batch, seq_len, embed_dim)`.
        This is concatenated with `hidden_states` (which is the output of the previous (mamba) layer). The
        concatenated tensor is then used as input of the pre-attention RMSNorm
        (see fig. 2 in https://huggingface.co/papers/2405.16712).
    attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
        `(batch, sequence_length)` where padding elements are indicated by 0.
    past_key_values (`Cache`, *optional*): cached past key and value projection states
    use_cache (`bool`, *optional*):
        If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
        (see `past_key_values`).
    position_embeddings (`tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*):
        Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,
        with `head_dim` being the embedding dimension of each attention head.
r™   rE   rÜ   r¼   r  r  r:   rY  )r+   Úconcatenaterñ  rî  rò  rï  )	r0   rE   rö  rÜ   r¼   r  r  rŽ   r=  s	   &&&&&&&, r4   rM   Ú#Zamba2AttentionDecoderLayer.forward“  s›   € ô6 ×)Ò)¨=Ð*QÐWYÔZˆØ×,Ñ,¨]Ó;ˆØŸ>š>ñ 
Ø'ð
àð
ð *ð
ð ,ð	
ð
 !4ð
ð ñ
Ñˆð ×-Ñ-¨mÓ<ˆØ×)Ñ)¨-ÓCˆàÐr6   )rÞ   rï  rñ  rò  rî  rÖ  r  rQ   rY   s   @@r4   ré  ré  ‰  s    ù‡ € ÷[õ [÷)÷ )ò )r6   ré  c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚZamba2MambaDecoderLayeri¿  c                ó&   <€ V ^8„  d   QhRS[ RS[/# )r^   rt   rÜ   rß   )ra   rZ   s   "€r4   rb   Ú$Zamba2MambaDecoderLayer.__annotate__À  s   ø€ ÷ #ñ #™|ð #¹ñ #r6   c                óž   <€ \         SV `  4        \        WR 7      V n        \	        VP
                  VP                  R7      V n        W n        R# ))rt   rÜ   rí  N)	r(   r)   r”  Úmambar\   r1   rð  rñ  rÜ   )r0   rt   rÜ   r3   s   &&&€r4   r)   Ú Zamba2MambaDecoderLayer.__init__À  s:   ø€ Ü‰ÑÔÜ%¨VÔIˆŒ
Ü,¨V×-?Ñ-?ÀV×EXÑEXÔYˆÔØ"Žr6   c                óh  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          R	S[ P
                  R,          R
S[ P                  R,          RS[S[,          RS[ P                  /# )r^   rE   rö  NrÜ   r¼   Úcausal_maskr  Ú	use_cacher¤   Útransformer_hidden_statesrŽ   r_   )r+   rg   r¯   r   r&  r÷  r   r   )ra   rZ   s   "€r4   rb   rÿ  Æ  sÂ   ø€ ÷ *ñ *á—|‘|ð*ñ !&§¡¨tÕ 3ð*ñ ˜•:ð	*ñ
 Ÿ™ tÕ+ð*ñ —\‘\ DÕ(ð*ñ  �ð*ñ ˜$•;ð*ñ ×&Ñ&¨Õ-ð*ñ $)§<¡<°$Õ#6ð*ñ Ñ+Õ,ð*ñ 
�‰ñ*r6   c
           	     óˆ   € TpV	e	   W,           MTpV P                  V4      pV P                  ! RRVRVRV/V
B pW±,           pV# )a  
Args:
    hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
    attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
        `(batch, sequence_length)` where padding elements are indicated by 0.
    past_key_values (`Cache`, *optional*): cached past key and value projection states
    use_cache (`bool`, *optional*):
        If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
        (see `past_key_values`).
rE   r¿  r¼   rY  )rñ  r  )r0   rE   rö  rÜ   r¼   r  r  r  r¤   r  rŽ   Úresiduals   &&&&&&&&&&, r4   rM   ÚZamba2MambaDecoderLayer.forwardÆ  ss   € ð0 !ˆð
 :SÒ9^ˆMÖ5Ðdqð 	ð ×,Ñ,¨]Ó;ˆàŸ
š
ñ 
Ø'ð
à(ð
ð *ð
ð ñ	
ˆð !Õ0ˆàÐr6   )rñ  rÜ   r  ©NNNNNFNNrQ   rY   s   @@r4   rý  rý  ¿  s   ù‡ € ÷#ó #÷*÷ *ò *r6   rý  c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚZamba2HybridLayerió  c                ó@   <€ V ^8„  d   QhRS[ RS[P                  RS[/# )r^   Úshared_transformerÚlinearr  )ré  r   rè   rý  )ra   rZ   s   "€r4   rb   ÚZamba2HybridLayer.__annotate__ô  s)   ø€ ÷ 5ñ 5Ù"=ð5ÙGIÇyÁyð5ÙYpñ5r6   c                óH   <€ \         SV `  4        W n        W0n        Wn        R # r'   )r(   r)   r  Úmamba_decoderr  )r0   r  r  r  r3   s   &&&&€r4   r)   ÚZamba2HybridLayer.__init__ô  s!   ø€ ô 	‰ÑÔØŒØ"ÔØ"4Ör6   c                óÊ  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          R	S[ P
                  R,          R
S[ P
                  R,          RS[S[,          RS[S[ P                  S[S[ P                  S[ P                  3,          R,          3,          /# )r^   rE   rö  NrÜ   r¼   r  r  r  r  r¤   rŽ   r_   )
r+   rg   r¯   r   r&  r÷  r   r   rk   rø  )ra   rZ   s   "€r4   rb   r  ü  sì   ø€ ÷ 4ñ 4á—|‘|ð4ñ !&§¡¨tÕ 3ð4ñ ˜•:ð	4ñ
 Ÿ™ tÕ+ð4ñ —\‘\ DÕ(ð4ñ  �ð4ñ ˜$•;ð4ñ #×-Ñ-°Õ4ð4ñ ×&Ñ&¨Õ-ð4ñ Ñ+Õ,ð4ñ 
‰u× Ñ ¡%©×(9Ñ(9¹5×;LÑ;LÐ(LÕ"MÐPTÕ"TÐTÕ	Uñ4r6   c
                ó    € V P                   ! V3RVRVRVRVRVRV	/V
B pV P                  V4      pV P                  ! V3RVRVRVRVRV/V
B pV# )	aø  
Args:
    hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
    original_hidden_states (`torch.FloatTensor`): word embedding output that will be concatenated with
    hidden activations to form the input of the shared transformer layer.
    layer_idx (`int`): layer number.
    attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
        `(batch, sequence_length)` where padding elements are indicated by 0.
    past_key_values (`Cache`, *optional*): cached past key and value projection states
    use_cache (`bool`, *optional*):
        If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
        (see `past_key_values`).
    position_embeddings (`tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*):
        Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,
        with `head_dim` being the embedding dimension of each attention head.
rö  rÜ   r¼   r  r  r¤   r  r  )r  r  r  )r0   rE   rö  rÜ   r¼   r  r  r  r  r¤   rŽ   r  s   &&&&&&&&&&, r4   rM   ÚZamba2HybridLayer.forwardü  sÃ   € ð< %)×$;Ò$;Øñ	%
à#9ð	%
ð  ð	%
ð 'ð		%
ð
 ,ð	%
ð !4ð	%
ð &ð	%
ð ñ	%
Ð!ð %)§K¡KÐ0IÓ$JÐ!à×*Ò*Øñ
à&?ð
ð *ð
ð ,ð	
ð
  ð
ð !4ð
ð ñ
ˆð Ðr6   )r  r  r  r
  rQ   rY   s   @@r4   r  r  ó  s   ù‡ € ÷5ó 5÷4÷ 4ò 4r6   r  c                   ó˜   a a€ ] tR tRt oRtRtRR.tR.tRtRt	Rt
RtR]R]/t]P                   ! 4       V 3R	 l4       tV3R
 ltRtVtV ;t# )ÚZamba2PreTrainedModeli3  ÚmodelTr  rý  r  rE   Ú
attentionsc                óØ  <€ \         SV `  V4       \        V\        4      '       EdÃ   \        P
                  ! \        P                  ! V P                  P                  4      \        P                  ! V P                  P                  4      \        P                  ! V P                  P                  4      ,
          ,          \        P                  ! V P                  P                  4      ,           4      P                  V P                  P                  R 7      pV\        P                  ! \        P                  ! V) 4      ) 4      ,           p\         P"                  ! VP$                  V4       \        P&                  ! ^VP(                  ^,           4      p\         P"                  ! VP*                  \        P                  ! V4      4       \         P,                  ! VP.                  4       R# R# ))rv  N)r(   Ú_init_weightsrœ   r”  r+   rf  Úrandrt   r�  Úmathr»  Útime_step_maxrª  rz  Útime_step_floorÚexpm1rº  r:  rK  r�   ri  r±  r¼  rM  )r0   r¸   r^  Úinv_dtrL  r3   s   &&   €r4   r  Ú#Zamba2PreTrainedModel._init_weightsC  s+  ø€ ä‰Ñ˜fÔ%Ü�fÔ.×/Ó/Ü—’Ü—
’
˜4Ÿ;™;×4Ñ4Ó5Ü—8’8˜DŸK™K×5Ñ5Ó6¼¿ºÀ$Ç+Á+×B[ÑB[Ó9\Õ\õ^ä—(’(˜4Ÿ;™;×4Ñ4Ó5õ6ó÷ ‰e˜Ÿ™×3Ñ3ˆeÓ4ð	 ð œ%Ÿ)š)¤U§[¢[°"°Ó%5Ð$5Ó6Õ6ˆFÜ�JŠJ�v—~‘~ vÔ.ä—’˜Q × 0Ñ 0°1Õ 4Ó5ˆAÜ�JŠJ�v—|‘|¤U§Y¢Y¨q£\Ô2Ü�JŠJ�v—x‘xÖ ñ 0r6   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # rs   r"   )ra   rZ   s   "€r4   rb   Ú"Zamba2PreTrainedModel.__annotate__3  s   ø‡ ‚ áÑò r6   rY  )rR   rS   rT   rU   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_flex_attnÚ_supports_sdpaÚ_is_statefulrý  rÚ   Ú_can_record_outputsr+   r«   r  Ú__annotate_func__rV   rW   rX   rY   s   @@r4   r  r  3  sr   ù‡ € ð  ÐØ&*Ð#Ø,Ð.GÐHÐØ#4Ð"5ÐØÐØÐØ€NØ€LàÐ0Ø�oðÐð
 ‡]‚]ƒ_ô!ó ð!÷# … r6   r  c                   ó|   a a€ ] tR tRt oRtV3R lV 3R llt]]]R	V3R lR ll4       4       4       t	R t
RtVtV ;t# )
ÚZamba2ModeliU  zX
Model consisting of *config.num_hidden_layers* layers.

Args:
    config: Zamba2Config
c                ó    <€ V ^8„  d   QhRS[ /# rs   r"   )ra   rZ   s   "€r4   rb   ÚZamba2Model.__annotate__^  s   ø€ ÷ ñ ™|ñ r6   c                óf  <€ \         SV `  V4       Wn        VP                  V n        VP
                  V n        \        P                  ! VP
                  VP                  V P                  4      V n	        VP                  V n
        V P                  4       V n        VP                  V n        \        VP                  VP                  R 7      V n        VP"                  '       d8   VP$                  '       d   \&        P)                  R4       \+        V4      V n        RV n        V P1                  4        R# )rí  ze`use_long_context` set to `True`: using rescaled `rope_theta` and extended `max_position_embeddings`.FN)r(   r)   rt   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingr1   Úembed_tokensÚlayers_block_typeÚ
get_layersrÅ  r	  r\   rð  Úfinal_layernormr  Úuse_long_contextÚloggerÚwarning_oncero   Ú
rotary_embÚgradient_checkpointingÚ	post_init©r0   rt   r3   s   &&€r4   r)   ÚZamba2Model.__init__^  sè   ø€ Ü‰Ñ˜Ô ØŒØ!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸLšL¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔØ!'×!9Ñ!9ˆÔØ—o‘oÓ'ˆŒà$*×$?Ñ$?ˆÔ!Ü,¨V×-?Ñ-?ÀV×EXÑEXÔYˆÔØ××ÐØ×&×&Ð&Ü×#Ñ#Ø{ôô 4°FÓ;ˆDŒOØ&+ˆÔ#ð 	�‰Ör6   c                ó  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[S[,          R	S[S[	,          /# )
r^   Ú	input_idsNr¼   r¤   r  Úinputs_embedsr  rŽ   r_   )
r+   r÷  rg   r   rø  r&  r   r   rk   r   )ra   rZ   s   "€r4   rb   r3  x  s£   ø€ ÷ E
ñ E
á×#Ñ# dÕ*ðE
ñ Ÿ™ tÕ+ðE
ñ ×&Ñ&¨Õ-ð	E
ñ
  �ðE
ñ ×(Ñ(¨4Õ/ðE
ñ ˜$•;ðE
ñ Ñ+Õ,ðE
ñ 
Ñ(Õ	(ñE
r6   c                óp  € VR J VR J,          '       d   \        R4      hVf   V P                  V4      pTp\        P                  ! V4      p	V'       d   Vf   \	        V P
                  R7      pVfb   Ve   VP                  4       M^ p
\        P                  ! VP                  ^,          VP                  R7      V
,           pVP                  ^ 4      p\        T;p\        4      '       g/   RV P
                  RVRVRVRV/pR	\        R/ VB R
\        R/ VB /pV P
                  P                  '       d   V P!                  WƒR7      pMR p\#        V P$                  4       F(  w  rïV! VV	VVR
,          VR	,          3RVRVRVRV/VB pK*  	  V P'                  V4      p\)        TV'       d   VR7      # R R7      # )NzaYou cannot specify both input_ids and inputs_embeds at the same time, and must specify either one)rt   rd  rt   rG  r¼   r  r¤   Úfull_attentionÚlinear_attention)r¤   r  r  )Úlast_hidden_stater  rY  )Ú
ValueErrorr9  r+   r‚   r	   rt   Úget_seq_lengthr�   r@   rp   rÒ   rœ   Údictr   r   r  r@  rú   rÅ  r<  r   )r0   rF  r¼   r¤   r  rG  r  rŽ   rE   rö  Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsr  rÜ   Úlayers   &&&&&&&,        r4   rM   ÚZamba2Model.forwardu  sè  € ð ˜Ð -°tÐ";×<Ô<ÜØsóð ð Ò Ø ×-Ñ-¨iÓ8ˆMà%ˆä!&§¢¨]Ó!;Ð÷ ˜Ò0Ü*°$·+±+Ô>ˆOàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLä°Ð?Ð-Ä×FÒFà˜$Ÿ+™+Ø Ø  .Ø! ?Ø ðˆKð !Ô"4Ñ"C°{Ñ"CØ"Ô$CÑ$RÀkÑ$Rð#Ðð �;‰;×#×#Ð#Ø"&§/¡/°- /Ó"[Ñà"&Ðä )¨$¯+©+Ö 6ÑˆIÙ!ØØ&ØØ#Ð$6Õ7Ø#Ð$4Õ5ñð !0ðð $ðð %8ðð *ðð ñŠMñ !7ð ×,Ñ,¨]Ó;ˆä&Ø+ß/8˜Oô
ð 	
à>Bô
ð 	
r6   c                ó`  € . p/ V n         ^ V n        . p^ p\        V P                  4       EFk  w  rE\	        V P
                  VR7      pVR8X  Ed6   RV R2p\        V\        4      '       d%   \        V4      V P
                  P                  8¼  dJ   \        V\        4      '       d   \        V4      p\        V4      pV P                   P                  Wx/4       MVP                  V4       W0P
                  P                  ,          p	V^,          p\        V P
                  V	R7      p
\        P                   ! V P
                  P"                  V P
                  P"                  RR7      pVP                  \%        W«V4      4       EKZ  VP                  V4       EKn  	  \        P&                  ! V4      # )r   rÁ  Úhybridzlayers.z.shared_transformer)rÞ   Frâ   )Ú_tied_weights_keysÚfirst_transformer_layer_idrú   r:  rý  rt   rœ   Úlistr  rõ   r   Únextr  rù   ré  r   rè   r1   r  rð   )r0   rÅ  Úunique_hybrid_blocksÚhybrid_layer_countÚlayer_idr¶  Úmamba_layerÚprefix_patternÚtarget_patternrÞ   Ú
attn_blockÚlinear_layers   &           r4   r;  ÚZamba2Model.get_layers¿  s^  € ØˆØ"$ˆÔØ*+ˆÔ'Ø!ÐØÐä$-¨d×.DÑ.D×$EÑ ˆHÜ1°$·+±+ÈÔRˆKØ˜XÕ%Ø#*¨8¨*Ð4GÐ!H�ô #Ð#7¼×>Ò>ÜÐ/Ó0°D·K±K×4NÑ4NÔNä!Ð"6¼×=Ò=Ü/4Ð5IÓ/JÐ,Ü%)Ð*>Ó%?�NØ×+Ñ+×2Ñ2°NÐ3SÕTð )×/Ñ/°Ô?ð
 .·±×0JÑ0JÕJ�Ø" aÕ'Ð"Ü8¸¿¹ÈxÔX�
Ü!Ÿyšy¨¯©×)@Ñ)@À$Ç+Á+×BYÑBYÐ`eÔf�Ø—‘Ô/°
È+ÓV×Wà—‘˜k×*ñ; %Fô< �}Š}˜VÓ$Ð$r6   )r	  rV  rt   r9  r<  rW  rA  rÅ  r:  r6  r@  r7  )NNNNNN)rR   rS   rT   rU   r  r)   r    r!   r   rM   r;  rV   rW   rX   rY   s   @@r4   r1  r1  U  sK   ù‡ € ñ÷ó ð.  ØØ÷E
ñ E
ó ó ó  ðE
÷N%%ò %%r6   r1  c                   ó|   a a€ ] tR tRt oRR/tV3R lV 3R llt]]R
V3R lR ll4       4       tV 3R lt	R	t
VtV ;t# )ÚZamba2ForCausalLMiè  zlm_head.weightzmodel.embed_tokens.weightc                ó    <€ V ^8„  d   QhRS[ /# rs   r"   )ra   rZ   s   "€r4   rb   ÚZamba2ForCausalLM.__annotate__ë  ó   ø€ ÷ ñ ™|ñ r6   c                óì   <€ \         SV `  V4       \        V4      V n        VP                  V n        \
        P                  ! VP                  VP                  R R7      V n        V P                  4        R# ©Frâ   N)
r(   r)   r1  r  r7  r   rè   r1   Úlm_headrB  rC  s   &&€r4   r)   ÚZamba2ForCausalLM.__init__ë  sU   ø€ Ü‰Ñ˜Ô Ü  Ó(ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒð 	�‰Ör6   c                ó`  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[S[ P                  ,          R
S[S[,          RS[	S[
,          /
# ©r^   rF  Nr¼   r¤   r  rG  Úlabelsr  Úlogits_to_keeprŽ   r_   )r+   r÷  rg   r   rø  r&  r¯   r   r   rk   r   )ra   rZ   s   "€r4   rb   rf  ö  sÍ   ø€ ÷ @
ñ @
á×#Ñ# dÕ*ð@
ñ Ÿ™ tÕ+ð@
ñ ×&Ñ&¨Õ-ð	@
ñ
  �ð@
ñ ×(Ñ(¨4Õ/ð@
ñ × Ñ  4Õ'ð@
ñ ˜$•;ð@
ñ ™eŸl™lÕ*ð@
ñ Ñ+Õ,ð@
ñ 
Ñ'Õ	'ñ@
r6   c	                ó|  € V P                   ! R
RVRVRVRVRVRV/V	B p
V
P                  p\        V\        4      '       d   \	        V) R4      MTpV P                  VRVR3,          4      pRpVe    V P                  ! VVV P                  3/ V	B p\        VVV
P                  V
P                  V
P                  R	7      # )aì  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

Example:

```python
>>> from transformers import AutoTokenizer, Zamba2ForCausalLM

>>> model = Zamba2ForCausalLM.from_pretrained("Zyphra/Zamba2-7B-v1")
>>> tokenizer = AutoTokenizer.from_pretrained("Zyphra/Zamba2-7B-v1")

>>> prompt = "Hey, are you conscious? Can you talk to me?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
```rF  r¼   r¤   r  rG  r  Nrš   ©ÚlossÚlogitsr  rE   r  rY  )r  rK  rœ   r¯   Úslicerj  Úloss_functionr7  r   r  rE   r  )r0   rF  r¼   r¤   r  rG  rn  r  ro  rŽ   ÚoutputsrE   Úslice_indicesrs  rr  s   &&&&&&&&&,     r4   rM   ÚZamba2ForCausalLM.forwardô  sü   € ðH ,0¯:ª:ñ ,
Øð,
à)ð,
ð &ð,
ð ,ð	,
ð
 (ð,
ð  ð,
ð ñ,
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒØ×%Ò%ØØØ—‘ñð ñ	ˆDô &ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r6   c                ó\   <€ V P                   P                  VR &   \        SV `  ! V3/ VB pV# )ro  )rt   Únum_logits_to_keepr(   Úprepare_inputs_for_generation)r0   rF  rŽ   Úmodel_inputsr3   s   &&, €r4   r{  Ú/Zamba2ForCausalLM.prepare_inputs_for_generation8  s2   ø€ Ø#'§;¡;×#AÑ#AˆÐÑ Ü‘wÒ<¸YÑQÈ&ÑQˆØÐr6   )rj  r  r7  ©NNNNNNNr   )rR   rS   rT   rU   rV  r)   r   r   rM   r{  rV   rW   rX   rY   s   @@r4   rd  rd  è  sJ   ù‡ € Ø*Ð,GÐHÐ÷ó ð Ø÷@
ñ @
ó ó ð@
÷Dõ r6   rd  aÌ  
    The Zamba2 Model with a sequence classification head on top (linear layer).

    [`Zamba2ForSequenceClassification`] uses the last token in order to do the classification, as other causal models
    (e.g. GPT-2) do.

    Since it does classification on the last token, it requires to know the position of the last token. If a
    `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
    no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
    padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
    each row of the batch).
    )Úcustom_introc                   óh   a a€ ] tR tRt oV3R lV 3R llt]]RV3R lR ll4       4       tRtVt	V ;t
# )ÚZamba2ForSequenceClassificationi>  c                ó    <€ V ^8„  d   QhRS[ /# rs   r"   )ra   rZ   s   "€r4   rb   Ú,Zamba2ForSequenceClassification.__annotate__M  rg  r6   c                óì   <€ \         SV `  V4       VP                  V n        \        V4      V n        \
        P                  ! VP                  V P                  R R7      V n        V P                  4        R# ri  )
r(   r)   Ú
num_labelsr1  r  r   rè   r1   ÚscorerB  rC  s   &&€r4   r)   Ú(Zamba2ForSequenceClassification.__init__M  sS   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒÜ  Ó(ˆŒ
Ü—Y’Y˜v×1Ñ1°4·?±?ÈÔOˆŒ
ð 	�‰Ör6   c                ó`  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[S[ P                  ,          R
S[S[,          RS[	S[
,          /
# rm  )r+   r÷  rg   r   rø  r&  r¯   r   r   rk   r   )ra   rZ   s   "€r4   rb   rƒ  X  sÍ   ø€ ÷ @
ñ @
á×#Ñ# dÕ*ð@
ñ Ÿ™ tÕ+ð@
ñ ×&Ñ&¨Õ-ð	@
ñ
  �ð@
ñ ×(Ñ(¨4Õ/ð@
ñ × Ñ  4Õ'ð@
ñ ˜$•;ð@
ñ ™eŸl™lÕ*ð@
ñ Ñ+Õ,ð@
ñ 
Ñ1Õ	1ñ@
r6   c	                óî  € V P                   ! V3RVRVRVRVRV/V	B p
V
^ ,          pV P                  V4      pVe   VP                  ^ ,          pMVP                  ^ ,          pV P                  P                  f   V^8w  d   \        R4      hV P                  P                  f   RpMÍVeœ   WP                  P                  8g  P                  VP                  \        P                  4      p\        P                  ! VP                  R,          VP                  \        P                  R7      pVV,          P                  R4      pM.Rp\        P                  V P                  P                   R	24       V\        P                  ! WÜP                  R
7      V3,          pRpVe%   V P                   ! RRVRVRVRV P                  /V	B p\#        VVV
P$                  V
P&                  V
P(                  R7      # )ae  
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
    `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
r¼   r¤   r  rG  r  Nz=Cannot handle batch sizes > 1 if no padding token is defined.r   zŠ will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`rd  rs  rn  Úpooled_logitsrt   rq  r:   rY  )r  r†  r@   rt   r5  rL  r<   rp   r+   Úint32r�   Úargmaxr>  r?  r3   rR   ru  r   r  rE   r  )r0   rF  r¼   r¤   r  rG  rn  r  ro  rŽ   Útransformer_outputsrE   rs  rh  Úlast_non_pad_tokenÚnon_pad_maskÚtoken_indicesrŠ  rr  s   &&&&&&&&&,         r4   rM   Ú'Zamba2ForSequenceClassification.forwardV  sú  € ð( 8<·z²zØñ8
à)ð8
ð &ð8
ð ,ð	8
ð
 (ð8
ð  ð8
ð ñ8
Ðð ,¨AÕ.ˆØ—‘˜MÓ*ˆàÒ Ø"Ÿ™¨Õ+‰Jà&×,Ñ,¨QÕ/ˆJà�;‰;×#Ñ#Ò+°
¸a´ÜÐ\Ó]Ð]Ø�;‰;×#Ñ#Ò+Ø!#ÑØÒ"Ø%¯©×)AÑ)AÑA×EÑEÀfÇmÁmÔUZ×U`ÑU`ÓaˆLÜ!ŸLšL¨¯©¸Õ)<ÀVÇ]Á]ÔZ_×ZeÑZeÔfˆMØ"/°,Õ">×!FÑ!FÀrÓ!JÑà!#ÐÜ×ÑØ—>‘>×*Ñ*Ð+ð ,Zð Zôð
 œuŸ|š|¨J¿}¹}ÔMÐOaÐaÕbˆàˆØÒØ×%Ò%ñ Ø$ðØ-3ðØCPðØY]×YdÑYdðØhnñˆDô 0ØØ Ø/×?Ñ?Ø-×;Ñ;Ø*×5Ñ5ô
ð 	
r6   )r  r…  r†  r~  )rR   rS   rT   rU   r)   r   r   rM   rV   rW   rX   rY   s   @@r4   r�  r�  >  s9   ù‡ € ÷ó ð Ø÷@
ñ @
ó ó ÷@
ð @
r6   r�  )r¶   )r7  rÖ  )NNFN)rd  r�  r1  r  )Yr  Úcollections.abcr   Ú	itertoolsr   r+   Útorch.nn.functionalr   r>   r;  Ú r   rº  Úactivationsr   Úcache_utilsr   r	   Ú
generationr
   Úintegrationsr   r   r   Úintegrations.accelerater   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.deprecationr   Úutils.genericr   r    Úutils.output_capturingr!   Úconfiguration_zamba2r#   Ú
get_loggerrR   r>  r¿   r%   r\   ro   rµ   rË   rÏ   rØ   rÚ   r  r  r,  r/  r0  rB  r`   rZ  rr  r’  r”  rØ  ré  rý  r  r  r1  rd  r�  Ú__all__rY  r6   r4   Ú<module>r¨     sË  ðó* Ý $Ý ã ß Ð Ý å &Ý !ß .Ý )ß tÑ tÝ =ß PÝ 9ß qÑ qß Kß FÝ &ß RÓ RÝ 0ß GÝ 5Ý .ð 
×	Ò	˜HÓ	%€ô;˜Ÿ™Ÿ™ô ;ô*J�B—I‘Iô Jô(6<˜BŸI™Iô 6<õr	U÷%ò2(ñ Ð-Ó.óó /ðô2y)�b—i‘iô y)õ~Vò
ò(ò(	ñ (Ð(>ÀÓPö'ó Qð'ñ& (Ð(:¸OÓLö'ó Mð'ñ, (Ð(JÈKÓXð +/Ø%(©%°«,Ð$7Ø %ØØ*.ØØ*.Ø(,ØØØ!ö%ó Yðñ0 (Ð(@À+ÓNö:'ó Oð:'ñz (Ð(CÀ[ÓQð "Ø#'Ø*.ØØ%(©%°«,Ð$7Ø %ö^ó Rð^ñB àØØ)Ø%ØðóôV%�r—y‘yó V%óðV%ôr'�—	‘	ô 'ôT3 "§)¡)ô 3ôl1Ð8ô 1ôh=Ð2ô =ð@ ô!˜Oó !ó ð!ðB ôN%Ð'ó N%ó ðN%ôdSÐ-¨ô Sñl ðôôL
Ð&;ó L
óðL
ò^ k‚r6   