+
    œ°³jã{  ã                   óò  € ^ RI Ht ^ RIHt ^ RIt^ RIHt ^RIHt ^RI	H
t
 ^RIHtHt ^RIHt ^R	IHt ^R
IHt ^RIHt ^RIHtHt ^RIHtHt ^RIHtHt ^RIH t  ^RI!H"t"H#t#H$t$ ^RI%H&t& ^RI'H(t(H)t) ^RI*H+t+ ^RI,H-t-  ! R R]P\                  4      t/R t0]! R4      R4R l4       t1R R lt2R5R R llt3 ! R R]P\                  4      t4 ! R  R!]P\                  4      t5 ! R" R#]P\                  4      t6 ! R$ R%]P\                  4      t7 ! R& R']P\                  4      t8 ! R( R)]P\                  4      t9 ! R* R+]4      t: ! R, R-]4      t;]# ! R. R/];4      4       t<R6R0 R1 llt= ! R2 R3];]4      t>. R7Ot?R# )8é    )ÚCallable)ÚAnyN)Únn)Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_kernel_forward_from_hub)Úcreate_causal_mask)ÚGradientCheckpointingLayer)ÚMoeCausalLMOutputWithPastÚMoeModelOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputs©Ú
DbrxConfigc                   óÎ   a a€ ] tR t^-t o]! RRR7      R
V3R lV 3R lll4       t]]! RRR7      R
V3R lR ll4       4       t]P                  ! 4       ]
R 4       4       tR	tVtV ;t# )ÚDbrxRotaryEmbeddingÚdevicez5.18)Úversionc                ó    <€ V ^8„  d   QhRS[ /# ©é   Úconfigr   )ÚformatÚ__classdict__s   "€Úg/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/dbrx/modeling_dbrx.pyÚ__annotate__Ú DbrxRotaryEmbedding.__annotate__/   s   ø€ ÷ Oñ O™zñ Oó    c                óä  <€ \         SV `  4        VP                  V n        VP                  V n        Wn        V P
                  P                  R ,          V n        V P                  pV P                  R8w  d   \        V P                  ,          pV! V P
                  V4      w  q@n
        \        P                  ! VRR7      V n        \        P                  ! VP                  4       RR7      V n        R# )Ú	rope_typeÚdefaultF)Ú
persistentN)ÚsuperÚ__init__Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr%   Úrope_parametersr-   Úcompute_default_rope_parametersr   Úattention_scalingr   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)Úselfr%   r    Úrope_init_fnr9   Ú	__class__s   &&&  €r(   r1   ÚDbrxRotaryEmbedding.__init__.   s¬   ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àŒàŸ™×4Ñ4°[ÕAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÔ&Ü.¨t¯~©~Õ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÔ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÔ!NˆÖr+   c                óP   <€ V ^8„  d   QhRS[ RS[S[P                  S[3,          /# )r$   r%   Úreturn)r   ÚtupleÚtorchÚTensorÚfloat)r&   r'   s   "€r(   r)   r*   A   s.   ø€ ÷ 5ñ 5±
ð 5ÑV[Ñ\a×\hÑ\hÑjoÐ\oÕVpñ 5r+   c           	     ó0  € V P                   R,          p\        V RR4      ;'       g    V P                  V P                  ,          pRpRV\        P
                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚhead_dimNç      ð?©Údtype)r5   ÚgetattrÚhidden_sizeÚnum_attention_headsrC   ÚarangerE   Úto)r%   r    ÚkwargsÚbaseÚdimÚattention_factorr9   s   &&,    r(   r6   Ú3DbrxRotaryEmbedding.compute_default_rope_parameters?   s~   € ð ×%Ñ% lÕ3ˆÜ�f˜j¨$Ó/×cÐc°6×3EÑ3EÈ×IcÑIcÕ3cˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r+   c                ó^  € V P                   R,          P                  VP                  ^ ,          R	^4      P                  \        P
                  VP                  R7      pVR
,          P                  4       p\        VP                  P                  \        4      '       d2   VP                  P                  R8w  d   VP                  P                  MRp\        VRR7      ;_uu_ 4        W4,          P                  ^^4      p\        P                  ! Wf3R	R7      pVP                  4       V P                  ,          pVP                  4       V P                  ,          p	R R R 4       XP                  VP                   R7      X	P                  VP                   R7      3#   + '       g   i     LH; i)N©rK   r    ÚmpsÚcpuF)Údevice_typeÚenabled©rS   rJ   )NºNNNNéÿÿÿÿ)r]   Nr]   )r9   ÚexpandÚshaperP   rC   rE   r    Ú
isinstanceÚtypeÚstrr   Ú	transposeÚcatÚcosr7   ÚsinrK   )
r<   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedrZ   ÚfreqsÚembrf   rg   s
   &&&       r(   ÚforwardÚDbrxRotaryEmbedding.forwardS   s2  € ð �M‰M˜-Õ(×/Ñ/°×0BÑ0BÀ1Õ0EÀrÈ1ÓM×PÑPÔW\×WbÑWbÐkl×ksÑksÐPÓtð 	ð !-¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆä¨¸U×CÖCØ&Õ>×IÑIÈ!ÈQÓOˆEÜ—)’)˜U˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆC÷	 Dð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷ D×Cús   Ã(A4FÆF,	)r7   r%   r9   r3   r;   r4   r-   ©N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r1   Ústaticmethodr6   rC   Úno_gradr   rn   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r>   r'   s   @@r(   r   r   -   su   ù‡ € Ù�X vÔ.÷Oô Oó /ðOð  Ù�X vÔ.÷5ñ 5ó /ó ð5ð$ ‡]‚]ƒ_Øñ<ó ó ÷<ð <r+   r   c                óÌ   € V RRV P                   R,          ^,          13,          pV RV P                   R,          ^,          R13,          p\        P                  ! V) V3RR7      # )z*Rotates half the hidden dims of the input..Nr\   r^   )r`   rC   re   )rh   Úx1Úx2s   &  r(   Úrotate_halfr~   f   sZ   € à	
ˆ3Ð"�!—'‘'˜"•+ Õ"Ð"Ð"Õ	#€BØ	
ˆ3�—‘˜•˜qÕ Ñ"Ð"Õ	#€BÜ�9Š9�r�c˜2�Y BÔ'Ð'r+   Úrotary_pos_embc                óÈ   € VP                  V4      pVP                  V4      pW,          \        V 4      V,          ,           pW,          \        V4      V,          ,           pWV3# )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)Ú	unsqueezer~   )ÚqÚkrf   rg   Úunsqueeze_dimÚq_embedÚk_embeds   &&&&&  r(   Úapply_rotary_pos_embr‡   m   sS   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�wœ; q›>¨CÕ/Õ0€GØ�wœ; q›>¨CÕ/Õ0€GØÐÐr+   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r$   Úhidden_statesÚn_reprA   )rC   rD   Úint)r&   s   "r(   r)   r)   ‡   s.   € ÷ 	Uñ 	UœUŸ\™\ð 	U´#ð 	U¼%¿,¹,ñ 	Ur+   c                ó˜   € V P                   w  r#rEV^8X  d   V # V R,          P                  W#WV4      p V P                  W#V,          WE4      # )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
)r]   r]   Nr]   r]   )r`   r_   Úreshape)r‰   rŠ   ÚbatchÚnum_key_value_headsÚslenrH   s   &&    r(   Ú	repeat_kvr‘   ‡   sU   € ð
 2?×1DÑ1DÑ.€E Ø�„zØÐØ!Ð"2Õ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÕ(CÀTÓTÐTr+   c                ó   € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R\        R	\
        \        ,          /# )
r$   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNÚscalingÚdropoutrQ   )r   ÚModulerC   rD   rE   r   r   )r&   s   "r(   r)   r)   “   sw   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ð%ô ð%ô Ô'Õ(ñ%r+   c                 ó"  € \        W P                  4      p\        W0P                  4      p	\        P                  ! WP	                  ^^4      4      V,          p
Ve	   W¤,           p
\
        P                  P                  V
R\        P                  R7      P                  VP                  4      p
\
        P                  P                  W¦V P                  R7      p
\        P                  ! W©4      pVP	                  ^^4      P                  4       pWº3# )r$   ©rS   rK   ©ÚpÚtrainingr^   )r‘   Únum_key_value_groupsrC   Úmatmulrd   r   Ú
functionalÚsoftmaxÚfloat32rP   rK   r™   rŸ   Ú
contiguous)r“   r”   r•   r–   r—   r˜   r™   rQ   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs   &&&&&&&,    r(   Úeager_attention_forwardrª   “   sÒ   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÕL€LØÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r+   c                   ó\   a a€ ] tR t^¬t oRtRV3R lV 3R llltRV3R lR lltRtVtV ;t	# )	ÚDbrxAttentionzYModular DBRX attention component that can be reused across different model architectures.c                ó.   <€ V ^8„  d   QhRS[ R,          /# )r$   Ú	layer_idxN)r‹   )r&   r'   s   "€r(   r)   ÚDbrxAttention.__annotate__¯   s   ø€ ÷ Rñ Rñ ˜•:ñRr+   c                ó"  <€ \         SV `  4        Wn        VP                  V n        VP
                  V n        V P                  V P                  ,          V n        VP                  V n	        W n
        VP                  pVP                  V n        VP                  V n        VP                  V n        V P                  V P                   ,          V n        V P                  R,          V n        VP&                  V n        RV n        \*        P,                  ! V P                  V P                  ^V P                   ,          V P                  ,          ,           RR7      V n        \*        P,                  ! V P                  V P                  RR7      V n        R# )g      à?TF©ÚbiasNg      à¿)r0   r1   r%   Úd_modelrM   Ún_headsÚ	num_headsrH   Úmax_seq_lenr2   r®   Úattn_configÚ
attn_pdropÚattention_dropoutÚclip_qkvÚ
kv_n_headsr�   r    r˜   rG   Ú	is_causalr   ÚLinearÚWqkvÚout_proj)r<   r%   r®   rQ   r·   r>   s   &&&, €r(   r1   ÚDbrxAttention.__init__¯   s&  ø€ ô 	‰ÑÔØŒØ!Ÿ>™>ˆÔØŸ™ˆŒØ×(Ñ(¨D¯N©NÕ:ˆŒØ'-×'9Ñ'9ˆÔ$Ø"Œà×(Ñ(ˆØ!,×!7Ñ!7ˆÔØ#×,Ñ,ˆŒØ#.×#9Ñ#9ˆÔ Ø$(§N¡N°d×6NÑ6NÕ$NˆÔ!Ø—}‘} dÕ*ˆŒØ%×0Ñ0ˆŒØˆŒä—I’IØ×Ñ˜d×.Ñ.°°T×5MÑ5MÕ1MÐPT×P]ÑP]Õ1]Õ]Ðdiô
ˆŒ	ô Ÿ	š	 $×"2Ñ"2°D×4DÑ4DÈ5ÔQˆŽr+   c                óÜ   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[S[ P                  S[ P                  3,          /# )r$   r‰   r—   NÚposition_embeddingsÚpast_key_valuesrA   )rC   rD   Ú
LongTensorr   rB   )r&   r'   s   "€r(   r)   r¯   Ë   sk   ø€ ÷ 3)ñ 3)á—|‘|ð3)ñ Ÿ™ tÕ+ð3)ñ #×-Ñ-°Õ4ð	3)ñ
  �ð3)ñ 
‰u�|‰|™UŸ\™\Ð)Õ	*ñ3)r+   c                ó   € VP                   R R p. VORNV P                  N5pV P                  V4      pV P                  e   V P                  ) MR p	VP	                  W�P                  R7      pVP                  V P                  V P                  V P                  ,          V P                  V P                  ,          .^R7      w  r«pV
P                  V4      P                  ^^4      p
VP                  V4      P                  ^^4      pVP                  V4      P                  ^^4      pVw  rÞ\        W«WÞ4      w  r«Ve   VP                  W¼V P                  4      w  r¼\        P                  ! V P                  P                   \"        4      pT! V V
VVV3RV P$                  '       g   RMV P&                  RV P(                  /VB w  ppVP*                  ! . VORN5!  P-                  4       pV P/                  V4      pVV3# )N)ÚminÚmaxr\   r™   ç        r˜   r^   )r`   rH   r¾   rº   ÚclampÚsplitrM   r�   Úviewrd   r‡   Úupdater®   r   Úget_interfacer%   Ú_attn_implementationrª   rŸ   r¹   r˜   r�   r¥   r¿   )r<   r‰   r—   rÂ   rÃ   rQ   Úinput_shapeÚhidden_shapeÚ
qkv_statesÚmin_valÚquery_statesr¦   r§   rf   rg   Úattention_interfacer©   r¨   s   &&&&&,            r(   rn   ÚDbrxAttention.forwardË   sú  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—Y‘Y˜}Ó-ˆ
Ø$(§M¡MÒ$=�4—=‘=‘.À4ˆØ×%Ñ%¨'·}±}Ð%ÓEˆ
à1;×1AÑ1Aà× Ñ Ø×(Ñ(¨4¯=©=Õ8Ø×(Ñ(¨4¯=©=Õ8ðð
 ð 2Bó 2
Ñ.ˆ ,ð $×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆØ—_‘_ \Ó2×<Ñ<¸QÀÓBˆ
Ø#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆ�\ð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—m‘m KÓ0ˆØ˜LÐ(Ð(r+   )r¾   r¹   rº   r%   rH   rM   r¼   r®   r2   rµ   r    r�   r¿   rG   r˜   rp   ©NNN)
rq   rr   rs   rt   Ú__doc__r1   rn   rw   rx   ry   rz   s   @@r(   r¬   r¬   ¬   s#   ù‡ € Ùc÷Rõ R÷83)÷ 3)ò 3)r+   r¬   c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚDbrxExpertGLUi  c                óä  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  V n        \
        P                  ! \        P                  ! V P                  V P                  ,          V P                  4      4      V n	        \
        P                  ! \        P                  ! V P                  V P                  ,          V P                  4      4      V n
        \
        P                  ! \        P                  ! V P                  V P                  ,          V P                  4      4      V n        VP                  P                  R R4      p\        V,          V n        R# )ÚnameÚsiluN)r0   r1   rM   Úffn_hidden_sizeÚmoe_num_expertsr   Ú	ParameterrC   ÚemptyÚw1Úv1Úw2Ú
ffn_act_fnÚgetr   Úactivation_fn)r<   r%   Úact_fn_namer>   s   && €r(   r1   ÚDbrxExpertGLU.__init__  sö   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØ%×5Ñ5ˆÔØ%×5Ñ5ˆÔä—,’,œuŸ{š{¨4×+?Ñ+?À$×BVÑBVÕ+VÐX\×XhÑXhÓiÓjˆŒÜ—,’,œuŸ{š{¨4×+?Ñ+?À$×BVÑBVÕ+VÐX\×XhÑXhÓiÓjˆŒÜ—,’,œuŸ{š{¨4×+?Ñ+?À$×BVÑBVÕ+VÐX\×XhÑXhÓiÓjˆŒà×'Ñ'×+Ñ+¨F°FÓ;ˆÜ# KÕ0ˆÖr+   c          
      óœ   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[ P                  RS[ P                  /# )r$   rh   Ú	expert_w1Ú	expert_v1Ú	expert_w2rA   ©rC   rD   )r&   r'   s   "€r(   r)   ÚDbrxExpertGLU.__annotate__  sK   ø€ ÷ ñ Ù—‘ðÙ*/¯,©,ðÙCHÇ<Á<ðÙ\a×\hÑ\hðá	�‰ñr+   c                óÆ   € VP                  VP                  4      pVP                  VP                  4      pV P                  V4      pWV,          pVP                  V4      pV# rp   )r¡   ÚTræ   )	r<   rh   rê   rë   rì   Ú	gate_projÚup_projÚintermediate_statesÚ	down_projs	   &&&&&    r(   rn   ÚDbrxExpertGLU.forward  sW   € ð —H‘H˜YŸ[™[Ó)ˆ	Ø—(‘(˜9Ÿ;™;Ó'ˆØ×&Ñ& yÓ1ˆ	Ø'Õ1ÐØ'×.Ñ.¨yÓ9ˆ	ØÐr+   )ræ   rÝ   rM   rÞ   râ   rá   rã   ©	rq   rr   rs   rt   r1   rn   rw   rx   ry   rz   s   @@r(   rÙ   rÙ     s   ù‡ € õ1÷÷ ð r+   rÙ   c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚDbrxExpertsi  c                óª   <€ \         SV `  4        \        V4      V n        VP                  V n        VP
                  V n        VP                  V n        R # rp   )r0   r1   rÙ   ÚmlprM   rÝ   rÞ   Únum_experts©r<   r%   r>   s   &&€r(   r1   ÚDbrxExperts.__init__  sD   ø€ Ü‰ÑÔÜ  Ó(ˆŒØ!×-Ñ-ˆÔØ%×5Ñ5ˆÔØ!×1Ñ1ˆÖr+   c                ó‚   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[ P                  /# )r$   r‰   Útop_k_indexÚtop_k_weightsrA   rí   )r&   r'   s   "€r(   r)   ÚDbrxExperts.__annotate__"  sC   ø€ ÷ ñ á—|‘|ðñ —\‘\ðñ —|‘|ð	ñ
 
�‰ñr+   c                ó&  € VP                   ^ ,          pVP                  RV P                  4      p\        P                  ! WP
                  VP                  R7      p\        P                  ! 4       ;_uu_ 4        \        P                  P                  P                  W P                  R7      pVP                  ^^^ 4      p\        P                  ! VP                  RR7      ^ 4      P                  4       pRRR4       RV P                   V P                  3pX EF.  p	V	^ ,          p	\        P                  ! 4       ;_uu_ 4        \        P"                  ! XV	,          4      w  r«RRR4       V P$                  P&                  P)                  V4      V	,          pV P$                  P*                  P)                  V4      V	,          pV P$                  P,                  P)                  V4      V	,          pV P%                  VX,          WÜV4      pVP)                  RV P                  4      W;X
R3,          ,          pVP/                  ^ W¿4       EK1  	  VP)                  VRV P                  4      pV#   + '       g   i     EL~; i  + '       g   i     EL%; i)r   rW   )Únum_classesr\   Nr^   )r^   éþÿÿÿ)r`   r�   rM   rC   Ú
zeros_likerK   r    rv   r   r¢   Úone_hotrû   ÚpermuteÚgreaterÚsumÚnonzerorÝ   Úwhererú   râ   rË   rá   rã   Ú
index_add_)r<   r‰   rÿ   r   Ú
batch_sizeÚnext_statesÚexpert_maskÚ
expert_hitÚsplit_expert_shapeÚ
expert_idxÚidxÚ	token_idxrâ   rá   rã   Ústatess   &&&&            r(   rn   ÚDbrxExperts.forward"  sç  € ð #×(Ñ(¨Õ+ˆ
Ø%×-Ñ-¨b°$×2BÑ2BÓCˆä×&Ò& }×<OÑ<OÐXe×XlÑXlÔmˆÜ�]Š]�_�_ÜŸ(™(×-Ñ-×5Ñ5°k×O_ÑO_Ð5Ó`ˆKØ%×-Ñ-¨a°°AÓ6ˆKÜŸš {§¡¸8 Ó'DÀaÓH×PÑPÓRˆJ÷ ð
 ! $×"6Ñ"6¸×8HÑ8HÐIÐÜ$ˆJØ# A�ˆJÜ—’—•Ü!&§¢¨[¸Õ-DÓ!E‘�÷ !à—‘—‘×!Ñ!Ð"4Ó5°jÕAˆBØ—‘—‘×!Ñ!Ð"4Ó5°jÕAˆBØ—‘—‘×!Ñ!Ð"4Ó5°jÕAˆBØ—X‘X˜m¨IÕ6¸ÀÓCˆFØ—[‘[  T×%5Ñ%5Ó6¸ÐRUÐW[ÐG[Õ9\Õ\ˆFØ×"Ñ" 1 i×8ñ %ð "×&Ñ& z°2°t×7GÑ7GÓHˆØÐ÷% �_ˆ_ú÷ !—�ús   Á9A=I+Å I?É+I<	É?J)rÝ   rM   rú   rû   rö   rz   s   @@r(   rø   rø     s   ù‡ € õ2÷÷ ð r+   rø   c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )Ú
DbrxRouteriA  c                óÌ   <€ \         SV `  4        VP                  V n        VP                  V n        \        P
                  ! V P                  VP                  R R7      V n        R# ©Fr±   N)r0   r1   rM   Úmoe_jitter_epsr   r½   rÞ   Úlayerrü   s   &&€r(   r1   ÚDbrxRouter.__init__B  sJ   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØ$×3Ñ3ˆÔÜ—Y’Y˜t×/Ñ/°×1GÑ1GÈeÔTˆŽ
r+   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r$   r‰   rA   rí   )r&   r'   s   "€r(   r)   ÚDbrxRouter.__annotate__H  s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r+   c                óL  € V P                   '       d^   V P                  eP   V\        P                  ! V4      P	                  RV P                  ,
          RV P                  ,           4      ,          pVP                  RVP                  R,          4      pV P                  V4      pV# )NrI   r^   )rŸ   r  rC   Ú
empty_likeÚuniform_rË   r`   r  )r<   r‰   Úrouter_logitss   && r(   rn   ÚDbrxRouter.forwardH  s†   € Ø�=�=ˆ=˜T×0Ñ0Ò<ØœU×-Ò-¨mÓ<×EÑEØ�d×)Ñ)Õ)¨3°×1DÑ1DÕ+Dóõ ˆMð &×*Ñ*¨2¨}×/BÑ/BÀ2Õ/FÓGˆØŸ
™
 =Ó1ˆØÐr+   )rM   r  r  rö   rz   s   @@r(   r  r  A  s   ù‡ € õU÷÷ ð r+   r  c                   óN   a a€ ] tR tRt oRtV 3R ltR tV3R lR ltRtVt	V ;t
# )ÚDbrxFFNiR  z0Modular DBRX MLP/FFN component with MoE support.c                óø   <€ \         SV `  4        \        VP                  4      V n        \        VP                  4      V n        VP                  P                  V n        VP                  P                  V n	        R # rp   )
r0   r1   r  Ú
ffn_configÚrouterrø   ÚexpertsÚmoe_normalize_expert_weightsÚ	moe_top_kÚtop_k)r<   r%   rQ   r>   s   &&,€r(   r1   ÚDbrxFFN.__init__U  sY   ø€ Ü‰ÑÔÜ  ×!2Ñ!2Ó3ˆŒÜ" 6×#4Ñ#4Ó5ˆŒà,2×,=Ñ,=×,ZÑ,ZˆÔ)Ø×&Ñ&×0Ñ0ˆŽ
r+   c                ó,  € \         P                  P                  P                  V^VP                  R7      p\         P
                  ! WP                  RR7      w  r#V P                  e+   V\         P                  ! W P                  RRR7      ,          pW#3# )é   rœ   r\   T)rž   rS   Úkeepdimr^   )	rC   r   r¢   r£   rK   Útopkr.  r,  Únorm)r<   r$  Úrouter_top_valueÚrouter_indicess   &&  r(   Úroute_tokens_to_expertsÚDbrxFFN.route_tokens_to_experts]  s~   € ÜŸ™×+Ñ+×3Ñ3°MÀqÐP]×PcÑPcÐ3ÓdˆÜ+0¯:ª:°mÇZÁZÐUWÔ+XÑ(ÐØ×,Ñ,Ò8Ø/´%·*²*Ø ×$EÑ$EÈ2ÐW[ô3õ  Ðð  Ð/Ð/r+   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r  rí   )r&   r'   s   "€r(   r)   ÚDbrxFFN.__annotate__f  s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r+   c                ór   € V P                  V4      pV P                  V4      w  r4V P                  WV4      pV# rp   )r*  r7  r+  )r<   r‰   r$  r   rÿ   Úoutputs   &&    r(   rn   ÚDbrxFFN.forwardf  s8   € ØŸ™ MÓ2ˆØ%)×%AÑ%AÀ-Ó%PÑ"ˆØ—‘˜m¸-ÓHˆØˆr+   )r+  r,  r*  r.  )rq   rr   rs   rt   r×   r1   r7  rn   rw   rx   ry   rz   s   @@r(   r'  r'  R  s!   ù‡ € Ù:õ1ò0÷÷ ð r+   r'  c                   óX   a a€ ] tR tRt oRV3R lV 3R llltRV3R lR lltRtVtV ;t# )	ÚDbrxNormAttentionNormim  c                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# )r$   r%   r®   N©r   r‹   )r&   r'   s   "€r(   r)   Ú"DbrxNormAttentionNorm.__annotate__n  s   ø€ ÷ 	?ñ 	?™zð 	?±c¸Dµjñ 	?r+   c                ó  <€ \         SV `  4        W n        VP                  V n        \        P
                  ! VP                  R R7      V n        \        VVR7      V n	        \        P
                  ! VP                  R R7      V n
        R# )Fr±   ©r%   r®   N)r0   r1   r®   Úresid_pdropr   Ú	LayerNormr³   Únorm_1r¬   ÚattnÚnorm_2©r<   r%   r®   r>   s   &&&€r(   r1   ÚDbrxNormAttentionNorm.__init__n  sa   ø€ Ü‰ÑÔØ"ŒØ!×-Ñ-ˆÔÜ—l’l 6§>¡>¸Ô>ˆŒÜ!ØØô
ˆŒ	ô —l’l 6§>¡>¸Ô>ˆŽr+   c                óÔ   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  R,          RS[R,          RS[RS[S[ P                  S[ P                  3,          /# )r$   r‰   rÂ   r—   NrÃ   rQ   rA   )rC   rD   rÄ   r   r   rB   )r&   r'   s   "€r(   r)   rB  y  sp   ø€ ÷ .ñ .á—|‘|ð.ñ #×-Ñ-ð.ñ Ÿ™ tÕ+ð	.ñ
  �ð.ñ ð.ñ 
‰u�|‰|™UŸ\™\Ð)Õ	*ñ.r+   c                ój  € TpV P                  V4      P                  VP                  4      pV P                  ! RR VRVRVRV/VB w  r\        P
                  P                  WP                  V P                  R7      pW,           pTpV P                  V4      P                  VP                  4      pWa3# ©r‰   r—   rÂ   rÃ   r�   © )
rG  rP   rK   rH  r   r¢   r™   rE  rŸ   rI  )r<   r‰   rÂ   r—   rÃ   rQ   Úresidual_statesÚ_s   &&&&&,  r(   rn   ÚDbrxNormAttentionNorm.forwardy  sÄ   € ð (ˆØŸ™ MÓ2×5Ñ5°m×6IÑ6IÓJˆàŸ9š9ñ 
Ø'ð
à)ð
ð !4ð
ð ,ð	
ð
 ñ
Ñˆô Ÿ™×-Ñ-¨m×?OÑ?OÐZ^×ZgÑZgÐ-ÓhˆØ%Õ7ˆà'ˆØŸ™ MÓ2×5Ñ5°m×6IÑ6IÓJˆàÐ-Ð-r+   )rH  r®   rG  rI  rE  rp   )NNrö   rz   s   @@r(   r?  r?  m  s   ù‡ € ÷	?õ 	?÷.÷ .ò .r+   r?  c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )Ú	DbrxBlocki•  c                ó&   <€ V ^8„  d   QhRS[ RS[/# )r$   r%   r®   rA  )r&   r'   s   "€r(   r)   ÚDbrxBlock.__annotate__–  s   ø€ ÷ 	*ñ 	*™zð 	*±cñ 	*r+   c                óº   <€ \         SV `  4        VP                  V n        VP                  V n        W n        \        VVR 7      V n        \        VR7      V n	        R# )rD  ©r%   N)
r0   r1   r³   rM   rE  r®   r?  Únorm_attn_normr'  ÚffnrJ  s   &&&€r(   r1   ÚDbrxBlock.__init__–  sN   ø€ Ü‰ÑÔØ!Ÿ>™>ˆÔØ!×-Ñ-ˆÔØ"ŒÜ3ØØô
ˆÔô  &Ô)ˆŽr+   c          
      óž   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[/# )r$   r‰   r—   NrÂ   rÃ   rQ   )rC   rD   rÄ   r   r   )r&   r'   s   "€r(   r)   rV  ¡  sY   ø€ ÷ ñ á—|‘|ðñ Ÿ™ tÕ+ðñ #×-Ñ-°Õ4ð	ñ
  �ðñ ñr+   c                óÚ   € V P                   ! RR VRVRVRV/VB w  raV P                  V4      p\        P                  P	                  WP
                  V P                  R7      pWa,           pV# rN  )rY  rZ  r   r¢   r™   rE  rŸ   )r<   r‰   r—   rÂ   rÃ   rQ   Úresid_statess   &&&&&, r(   rn   ÚDbrxBlock.forward¡  s‰   € ð '+×&9Ò&9ñ '
Ø'ð'
à)ð'
ð !4ð'
ð ,ð	'
ð
 ñ'
Ñ#ˆð Ÿ™ Ó/ˆÜŸ™×-Ñ-¨m×?OÑ?OÐZ^×ZgÑZgÐ-ÓhˆØ$Õ4ˆØÐr+   )rZ  rM   r®   rY  rE  rÖ   rö   rz   s   @@r(   rT  rT  •  s   ù‡ € ÷	*ó 	*÷÷ ò r+   rT  c                   ó¦   a a€ ] tR tRt oRtRtR.tR.tRtRt	Rt
RtRtR]R]/t]P"                  ! 4       V3R	 lV 3R
 ll4       tV3R ltRtVtV ;t# )ÚDbrxPreTrainedModeli·  ÚtransformerTrT  rÃ   Fr‰   Ú
attentionsc                ó4   <€ V ^8„  d   QhRS[ P                  /# )r$   r“   )r   rš   )r&   r'   s   "€r(   r)   Ú DbrxPreTrainedModel.__annotate__È  s   ø€ ÷ 7ñ 7¡B§I¡Iñ 7r+   c                óT  <€ \         SV `  V4       V P                  P                  p\	        V\
        4      '       dl   \        P                  ! VP                  R VR7       \        P                  ! VP                  R VR7       \        P                  ! VP                  R VR7       R# R# )rÈ   )ÚmeanÚstdN)r0   Ú_init_weightsr%   Úinitializer_rangera   rÙ   ÚinitÚnormal_rá   râ   rã   )r<   r“   rh  r>   s   && €r(   ri  Ú!DbrxPreTrainedModel._init_weightsÇ  sm   ø€ ä‰Ñ˜fÔ%Ø�k‰k×+Ñ+ˆÜ�fœm×,Ò,Ü�LŠL˜Ÿ™¨°#Õ6Ü�LŠL˜Ÿ™¨°#Õ6Ü�LŠL˜Ÿ™¨°#×6ñ -r+   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r#   r   )r&   r'   s   "€r(   r)   re  ·  s   ø‡ ‚ ÙÑò r+   rO  )rq   rr   rs   rt   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flex_attnÚ_supports_attention_backendÚ_supports_flash_attnÚ_supports_sdpaÚ_can_compile_fullgraphrT  r¬   Ú_can_record_outputsrC   rv   ri  Ú__annotate_func__rw   rx   ry   rz   s   @@r(   ra  ra  ·  sw   ù‡ € à%ÐØ&*Ð#Ø$˜ÐØ#4Ð"5ÐØÐØ"&ÐØÐØ€NØ"Ðà˜Ø�mðÐð
 ‡]‚]ƒ_÷7ò 7ó ð7÷# … r+   ra  c                   óš   a a€ ] tR tRt oRtV3R lV 3R lltV3R lR ltV3R lR lt]]	]
RV3R	 lR
 ll4       4       4       tRtVtV ;t# )Ú	DbrxModeliÑ  a•  Transformer decoder consisting of *config.num_hidden_layers*. Each layer is a [`DbrxBlock`] layer.

Args:
    config ([`DbrxConfig`]): Model configuration class with all parameters of the model.
        Initializing with a config file does not load the weights associated with the model, only the
        configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
c                ó    <€ V ^8„  d   QhRS[ /# r#   r   )r&   r'   s   "€r(   r)   ÚDbrxModel.__annotate__Û  s   ø€ ÷ ñ ™zñ r+   c           	     ó6  <€ \         SV `  V4       VP                  V n        VP                  V n        VP
                  V n        \        V4      V n        \        P                  ! VP                  VP                  V P                  4      V n        \        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n        \        P"                  ! VP                  R R7      V n        R V n        V P)                  4        R# u upi r  )r0   r1   Úpad_token_idÚpadding_idxÚ
vocab_sizeÚ	emb_pdropr   Ú
rotary_embr   Ú	Embeddingr³   ÚwteÚ
ModuleListÚrangeÚn_layersrT  ÚblocksrF  Únorm_fÚgradient_checkpointingÚ	post_initrJ  s   && €r(   r1   ÚDbrxModel.__init__Û  sË   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒØ×)Ñ)ˆŒÜ-¨fÓ5ˆŒÜ—<’< × 1Ñ 1°6·>±>À4×CSÑCSÓTˆŒÜ—m’mÔSXÐY_×YhÑYhÔSiÓ$jÑSiÀi¤Y¨vÖ%AÑSiÑ$jÓkˆŒÜ—l’l 6§>¡>¸Ô>ˆŒØ&+ˆÔ#ð 	�‰Öùò %ks   Â6Dc                ó4   <€ V ^8„  d   QhRS[ P                  /# ©r$   rA   ©r   r„  )r&   r'   s   "€r(   r)   r}  é  s   ø€ ÷ ñ ¡b§l¡lñ r+   c                ó   € V P                   # rp   ©r…  ©r<   s   &r(   Úget_input_embeddingsÚDbrxModel.get_input_embeddingsé  s   € Ø�x‰xˆr+   c                ó4   <€ V ^8„  d   QhRS[ P                  /# ©r$   r–   r�  )r&   r'   s   "€r(   r)   r}  ì  s   ø€ ÷ ñ ©"¯,©,ñ r+   c                ó   € Wn         R # rp   r’  ©r<   r–   s   &&r(   Úset_input_embeddingsÚDbrxModel.set_input_embeddingsì  s   € ØŽr+   c                óþ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[S[,          R	S[/# )
r$   Ú	input_idsNr—   ri   rÃ   Úinputs_embedsÚ	use_cacherQ   rA   )	rC   rÄ   rD   r   ÚFloatTensorÚboolr   r   r   )r&   r'   s   "€r(   r)   r}  ò  s“   ø€ ÷ 5
ñ 5
á×#Ñ# dÕ*ð5
ñ Ÿ™ tÕ+ð5
ñ ×&Ñ&¨Õ-ð	5
ñ
  �ð5
ñ ×(Ñ(¨4Õ/ð5
ñ ˜$•;ð5
ñ Ñ+Õ,ð5
ñ 
 ñ5
r+   c                ó†  € VR J VR J,          '       d   \        R4      hV'       d   Vf   \        V P                  R7      pVf   V P                  V4      pVfb   Ve   VP	                  4       M^ p\
        P                  ! VP                  ^,          VP                  R7      V,           pVP                  ^ 4      p\        V P                  VVVVR7      p	Tp
V P                  W£4      pV P                  R V P                  P                    F  pV! V
3RVRV	RVRVR	V/VB p
K  	  V P                  V
4      p
\        V
VR
7      # )Nz:You must specify exactly one of input_ids or inputs_embedsrX  )r    )r%   rž  r—   rÃ   ri   rÂ   r—   ri   rÃ   rŸ  )Úlast_hidden_staterÃ   )Ú
ValueErrorr	   r%   r…  Úget_seq_lengthrC   rO   r`   r    r�   r   rƒ  r‰  Únum_hidden_layersrŠ  r   )r<   r�  r—   ri   rÃ   rž  rŸ  rQ   Úpast_seen_tokensÚcausal_maskr‰   rÂ   Údecoder_layers   &&&&&&&,     r(   rn   ÚDbrxModel.forwardï  sc  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZç˜Ò0Ü*°$·+±+Ô>ˆOàÒ Ø ŸH™H YÓ/ˆMàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLä(Ø—;‘;Ø'Ø)Ø+Ø%ô
ˆð &ˆð #Ÿo™o¨mÓJÐà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÓIˆMÙ)Øñà$7ðð  +ðð *ð	ð
 !0ðð $ðð ñŠMñ Jð Ÿ™ MÓ2ˆä%Ø+Ø+ô
ð 	
r+   )r‰  r‚  r‹  rŠ  r€  rƒ  r�  r…  )NNNNNN)rq   rr   rs   rt   r×   r1   r”  rš  r   r   r   rn   rw   rx   ry   rz   s   @@r(   r{  r{  Ñ  sV   ù‡ € ñ÷ó ÷ð ÷ð ð  ØØ÷5
ñ 5
ó ó ó  ÷5
ð 5
r+   r{  c          	      ó  € V ^8„  d   QhR\         P                  \        \         P                  ,          ,          R,          R\        R,          R\         P                  R,          R\         P                  \        ,          /# )r$   Úgate_logitsNrû   r—   rA   )rC   rD   rB   r‹   )r&   s   "r(   r)   r)   *  sc   € ÷ B&ñ B&Ü—‘¤¤e§l¡lÕ 3Õ3°dÕ:ðB&ä�t•ðB&ô —L‘L 4Õ'ð	B&ô
 ‡\�\”CÕñB&r+   c           	     ó  € V e   \        V \        4      '       g   ^ # V ^ ,          P                  p\        P                  ! V\        P
                  VR7      p\        P                  ! V\        P
                  VR7      pRpVe1   VP                  R4      P                  V\        P
                  R7      pV  EFˆ  p	\        P                  P                  P                  V	P                  V4      RR7      p
\        P                  ! W¢RR7      w  r¼Vf~   V\        P                  ! VP                  R4      VR7      P                  4       ,           pWjP                  4       P                  ^ R7      ,           pWzP                  ^ ,          ,           pKÙ  V\        P                  ! V\        P
                  VR7      P!                  ^ VP                  R4      XP#                  V4      4      ,           pWjP                  4       VP%                  R4      ,          P                  ^ R7      ,           pWxP                  4       ,           pEK‹  	  WW,          pWg,          p\        P                  ! WÞP%                  ^ 4      ,          4      pWñ,          # )ax  
Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch.

See Switch Transformer (https://huggingface.co/papers/2101.03961) for more details. This function implements the loss
function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between
experts is too unbalanced.

Args:
    gate_logits:
        Logits from the `gate`, should be a tuple of model.config.num_hidden_layers tensors of
        shape [batch_size X sequence_length, num_experts].
    num_experts:
        Number of experts
    top_k:
        The number of experts to route per-token, can be also interpreted as the `top-k` routing
        parameter.
    attention_mask (`torch.Tensor`, *optional*):
        The attention_mask used in forward function
        shape [batch_size X sequence_length] if not None.

Returns:
    The auxiliary loss.
rW   rÈ   )r    rK   r\   )Ú	minlengthr^   )ra   rB   r    rC   Úzerosr¤   r�   rP   r   r¢   r£   r3  ÚbincountrE   r	  r`   Úscatter_add_Úrepeat_interleaver�   )r¬  rû   r.  r—   Úcompute_deviceÚtokens_per_expert_sumÚrouter_prob_sumÚ
total_rowsÚ	flat_maskÚ
layer_gateÚrouting_weightsrQ  Úselected_expertsÚtokens_per_expertÚrouter_prob_per_expertÚoverall_losss   &&&&            r(   Úload_balancing_loss_funcr¾  *  sÿ  € ð: Ò¤*¨[¼%×"@Ò"@Ùð ! •^×*Ñ*€NÜ!ŸKšK¨¼5¿=¹=ÐQ_Ô`ÐÜ—k’k +´U·]±]È>ÔZ€OØ€JàÒ!à"×*Ñ*¨2Ó.×1Ñ1¸ÌuÏ}É}Ð1Ó]ˆ	ä!ˆ
ÜŸ(™(×-Ñ-×5Ñ5°j·m±mÀNÓ6SÐY[Ð5Ó\ˆÜ#Ÿjšj¨ÀRÔHÑˆØÒ!ð &¬¯ªÐ7G×7OÑ7OÐPRÓ7SÐ_jÔ(k×(qÑ(qÓ(sÕsð "ð .×0EÑ0EÓ0G×0KÑ0KÐPQÐ0KÓ0RÕRˆOØ#×&;Ñ&;¸AÕ&>Õ>ŠJð %:¼E¿KºKØ¤5§=¡=¸ô=ç‰l˜1Ð.×6Ñ6°rÓ:¸I×<WÑ<WÐX]Ó<^Ó_õ%`Ð!ð .×1FÑ1FÓ1HÈ9×K^ÑK^Ð_aÓKbÕ1b×0gÑ0gÐlmÐ0gÓ0nÕnˆOØ#§m¡m£oÕ5‹Jñ# "ð& .Õ:ÐØ,Õ9Ðä—9’9Ð.×1QÑ1QÐRSÓ1TÕTÓU€LØÕ%Ð%r+   c                   óü   a a€ ] tR tRt oRR/tRR/tRR.R.3/tRR/tV3R	 lV 3R
 lltV3R lR lt	V3R lR lt
V3R lR ltV3R lR ltV3R lR ltV3R lR lt]]RV3R lR ll4       4       tRtVtV ;t# )ÚDbrxForCausalLMio  zlm_head.weightztransformer.wte.weightÚlm_headÚcolwise_gather_outputr‰   ÚlogitsÚkeep_full_weightc                ó    <€ V ^8„  d   QhRS[ /# r#   r   )r&   r'   s   "€r(   r)   ÚDbrxForCausalLM.__annotate__v  s   ø€ ÷ ñ ™zñ r+   c                óŽ  <€ \         SV `  V4       \        V4      V n        VP                  V n        \
        P                  ! VP                  VP                  R R7      V n        VP                  P                  V n        VP                  P                  V n        VP                  P                  V n        V P!                  4        R# r  )r0   r1   r{  rb  r�  r   r½   rM   rÁ  r)  Úmoe_loss_weightÚrouter_aux_loss_coefrÞ   rû   r-  Únum_experts_per_tokrŒ  rü   s   &&€r(   r1   ÚDbrxForCausalLM.__init__v  s�   ø€ Ü‰Ñ˜Ô Ü$ VÓ,ˆÔØ ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒØ$*×$5Ñ$5×$EÑ$EˆÔ!Ø!×,Ñ,×<Ñ<ˆÔØ#)×#4Ñ#4×#>Ñ#>ˆÔ Ø�‰Ör+   c                ó4   <€ V ^8„  d   QhRS[ P                  /# r�  r�  )r&   r'   s   "€r(   r)   rÆ  €  s   ø€ ÷ 7ñ 7¡b§l¡lñ 7r+   c                ó6   € V P                   P                  4       # rp   )rb  r”  r“  s   &r(   r”  Ú$DbrxForCausalLM.get_input_embeddings€  s   € Ø×Ñ×4Ñ4Ó6Ð6r+   c                ó4   <€ V ^8„  d   QhRS[ P                  /# r—  r�  )r&   r'   s   "€r(   r)   rÆ  ƒ  s   ø€ ÷ 5ñ 5©"¯,©,ñ 5r+   c                ó<   € V P                   P                  V4       R # rp   )rb  rš  r™  s   &&r(   rš  Ú$DbrxForCausalLM.set_input_embeddingsƒ  s   € Ø×Ñ×-Ñ-¨eÖ4r+   c                ó4   <€ V ^8„  d   QhRS[ P                  /# r�  ©r   r½   )r&   r'   s   "€r(   r)   rÆ  †  s   ø€ ÷ ñ ¡r§y¡yñ r+   c                ó   € V P                   # rp   ©rÁ  r“  s   &r(   Úget_output_embeddingsÚ%DbrxForCausalLM.get_output_embeddings†  s   € Ø�|‰|Ðr+   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r$   Únew_embeddingsrÓ  )r&   r'   s   "€r(   r)   rÆ  ‰  s   ø€ ÷ &ñ &±B·I±Iñ &r+   c                ó   € Wn         R # rp   rÕ  )r<   rÙ  s   &&r(   Úset_output_embeddingsÚ%DbrxForCausalLM.set_output_embeddings‰  s   € Ø%Žr+   c                ó    <€ V ^8„  d   QhRS[ /# )r$   Údecoder©r{  )r&   r'   s   "€r(   r)   rÆ  Œ  s   ø€ ÷ #ñ #¡9ñ #r+   c                ó   € Wn         R # rp   ©rb  )r<   rÞ  s   &&r(   Úset_decoderÚDbrxForCausalLM.set_decoderŒ  s   € Ø"Ör+   c                ó    <€ V ^8„  d   QhRS[ /# r�  rß  )r&   r'   s   "€r(   r)   rÆ  �  s   ø€ ÷  ñ  ™Yñ  r+   c                ó   € V P                   # rp   rá  r“  s   &r(   Úget_decoderÚDbrxForCausalLM.get_decoder�  s   € Ø×ÑÐr+   c                ód  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[R,          R
S[S[ P                  ,          RS[S[,          RS[	/# )r$   r�  Nr—   ri   rÃ   rž  ÚlabelsrŸ  Úoutput_router_logitsÚlogits_to_keeprQ   rA   )
rC   rÄ   rD   r   r   r¡  r‹   r   r   r   )r&   r'   s   "€r(   r)   rÆ  ”  sÖ   ø€ ÷ P
ñ P
á×#Ñ# dÕ*ðP
ñ Ÿ™ tÕ+ðP
ñ ×&Ñ&¨Õ-ð	P
ñ
  �ðP
ñ ×(Ñ(¨4Õ/ðP
ñ × Ñ  4Õ'ðP
ñ ˜$•;ðP
ñ # T�kðP
ñ ™eŸl™lÕ*ðP
ñ Ñ+Õ,ðP
ñ 
#ñP
r+   c
                ó¦  € Ve   TMV P                   P                  pV P                  ! RRVRVRVRVRVRVRV/V
B pVP                  p\	        V	\
        4      '       d   \        V	) R4      MT	pV P                  VR	VR	3,          4      pRpVe   V P                  ! WæV P                  3/ V
B pRpV'       dc   \        VP                  V P                  V P                  V4      pVe3   WðP                  VP                  VP                   4      ,          ,          p\#        VVVVP$                  VP&                  VP(                  VP                  R
7      # )a  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

Example:

```python
>> from transformers import AutoTokenizer, DbrxForCausalLM

>> model = DbrxForCausalLM.from_pretrained("transformers-community/dbrx-instruct")
>> tokenizer = AutoTokenizer.from_pretrained("transformers-community/dbrx-instruct")

>> prompt = "Hey, are you conscious? Can you talk to me?"
>> inputs = tokenizer(prompt, return_tensors="pt")

>> # Generate
>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
```
Nr�  r—   ri   rÃ   rž  rŸ  rê  r]   )ÚlossÚaux_lossrÃ  rÃ   r‰   rc  r$  rO  )r%   rê  rb  r£  ra   r‹   ÚslicerÁ  Úloss_functionr�  r¾  r$  rû   rÊ  rÉ  rP   r    r   rÃ   r‰   rc  )r<   r�  r—   ri   rÃ   rž  ré  rŸ  rê  rë  rQ   Úoutputsr‰   Úslice_indicesrÃ  rí  rî  s   &&&&&&&&&&,      r(   rn   ÚDbrxForCausalLM.forward’  sz  € ðN %9Ò$DÑ È$Ï+É+×JjÑJjð 	ð
 +/×*:Ò*:ñ 	+
Øð	+
à)ð	+
ð &ð	+
ð ,ð		+
ð
 (ð	+
ð  ð	+
ð "6ð	+
ð ñ	+
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒØ×%Ò% f°d·o±oÑPÈÑPˆDàˆßÜ/Ø×%Ñ%Ø× Ñ Ø×(Ñ(Øó	ˆHð Ò!Ø×1Ñ1°H·K±KÀÇÁÓ4LÕLÕL�ä(ØØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø!×/Ñ/ô
ð 	
r+   )rÁ  rû   rÊ  rÉ  rb  r�  )	NNNNNNNNr   )rq   rr   rs   rt   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr1   r”  rš  rÖ  rÛ  râ  ræ  r   r   rn   rw   rx   ry   rz   s   @@r(   rÀ  rÀ  o  s«   ù‡ € Ø*Ð,DÐEÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HàÐ/Ð0€J÷ó ÷7ð 7÷5ð 5÷ð ÷&ð &÷#ð #÷ ð  ð Ø÷P
ñ P
ó ó ÷P
ð P
r+   rÀ  )r1  )rÈ   )Nr$   N)rÀ  r{  ra  )@Úcollections.abcr   Útypingr   rC   r   Ú r   rk  Úactivationsr   Úcache_utilsr   r	   Ú
generationr
   Úintegrationsr   Úmasking_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   Úconfiguration_dbrxr   rš   r   r~   r‡   r‘   rª   r¬   rÙ   rø   r  r'  r?  rT  ra  r{  r¾  rÀ  Ú__all__rO  r+   r(   Ú<module>r     sA  ðõ* %Ý ã Ý å &Ý !ß .Ý )Ý 7Ý /Ý 9ß Qß Kß FÝ &ß IÑ IÝ 0ß GÝ 5Ý *ô6<˜"Ÿ)™)ô 6<òr(ñ Ð-Ó.óó /ðõ2	U÷%ô2R)�B—I‘Iô R)ôj�B—I‘Iô ô2$�"—)‘)ô $ôN�—‘ô ô"ˆb�i‰iô ô6%.˜BŸI™Iô %.ôPÐ*ô ôD7˜/ô 7ð4 ôU
Ð#ó U
ó ðU
÷pB&ôJu
Ð)¨?ô u
òp B‚r+   