+
    œ°³jâT  ã                   ó„  € R t ^ RIHt ^ RIHt ^ RIt^ RIHt ^RIHt	 ^RI
Ht ^RIHtHt ^RIHt ^R	IHt ^R
IHt ^RIHtHt ^RIHtHt ^RIHt ^RIHtHtH t  ^RI!H"t" ^RI#H$t$ ^RI%H&t&H't'H(t( ^RI)H*t* ^RI+H,t,  ! R R]&4      t- ! R R]P\                  4      t/ ! R R]P\                  4      t0 ! R R]P\                  4      t1 ! R R]P\                  4      t2 ! R R]P\                  4      t3 ! R  R!]P\                  4      t4 ! R" R#]4      t5 ! R$ R%]4      t6] ! R& R']64      4       t7 ! R( R)]6]4      t8. R*Ot9R# )+z"Modular components for DBRX model.)ÚCallable)ÚAnyN)Únn)Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úcreate_causal_mask)ÚGradientCheckpointingLayer)ÚMoeCausalLMOutputWithPastÚMoeModelOutputWithPast)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Úmerge_with_config_defaults)Úcapture_outputs)ÚLlamaRotaryEmbeddingÚapply_rotary_pos_embÚeager_attention_forward)Úload_balancing_loss_func©Ú
DbrxConfigc                   ó   € ] tR t^-tRtR# )ÚDbrxRotaryEmbedding© N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__static_attributes__r   ó    Úf/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/dbrx/modular_dbrx.pyr   r   -   s   † Ûr$   r   c                   ó\   a a€ ] tR t^1t oRtRV3R lV 3R llltRV3R lR lltRtVtV ;t	# )	ÚDbrxAttentionzYModular DBRX attention component that can be reused across different model architectures.c                ó.   <€ V ^8„  d   QhRS[ R,          /# )é   Ú	layer_idxN)Úint)ÚformatÚ__classdict__s   "€r%   Ú__annotate__ÚDbrxAttention.__annotate__4   s   ø€ ÷ Rñ Rñ ˜•:ñRr$   c                ó"  <€ \         SV `  4        Wn        VP                  V n        VP
                  V n        V P                  V P                  ,          V n        VP                  V n	        W n
        VP                  pVP                  V n        VP                  V n        VP                  V n        V P                  V P                   ,          V n        V P                  R,          V n        VP&                  V n        RV n        \*        P,                  ! V P                  V P                  ^V P                   ,          V P                  ,          ,           RR7      V n        \*        P,                  ! V P                  V P                  RR7      V n        R# )g      à?TF©ÚbiasNg      à¿)ÚsuperÚ__init__ÚconfigÚd_modelÚhidden_sizeÚn_headsÚ	num_headsÚhead_dimÚmax_seq_lenÚmax_position_embeddingsr*   Úattn_configÚ
attn_pdropÚattention_dropoutÚclip_qkvÚ
kv_n_headsÚnum_key_value_headsÚnum_key_value_groupsÚscalingÚ
rope_thetaÚ	is_causalr   ÚLinearÚWqkvÚout_proj)Úselfr5   r*   Úkwargsr=   Ú	__class__s   &&&, €r%   r4   ÚDbrxAttention.__init__4   s&  ø€ ô 	‰ÑÔØŒØ!Ÿ>™>ˆÔØŸ™ˆŒØ×(Ñ(¨D¯N©NÕ:ˆŒØ'-×'9Ñ'9ˆÔ$Ø"Œà×(Ñ(ˆØ!,×!7Ñ!7ˆÔØ#×,Ñ,ˆŒØ#.×#9Ñ#9ˆÔ Ø$(§N¡N°d×6NÑ6NÕ$NˆÔ!Ø—}‘} dÕ*ˆŒØ%×0Ñ0ˆŒØˆŒä—I’IØ×Ñ˜d×.Ñ.°°T×5MÑ5MÕ1MÐPT×P]ÑP]Õ1]Õ]Ðdiô
ˆŒ	ô Ÿ	š	 $×"2Ñ"2°D×4DÑ4DÈ5ÔQˆŽr$   c                óÜ   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[S[ P                  S[ P                  3,          /# )r)   Úhidden_statesÚattention_maskNÚposition_embeddingsÚpast_key_valuesÚreturn)ÚtorchÚTensorÚ
LongTensorr   Útuple)r,   r-   s   "€r%   r.   r/   P   sk   ø€ ÷ 3)ñ 3)á—|‘|ð3)ñ Ÿ™ tÕ+ð3)ñ #×-Ñ-°Õ4ð	3)ñ
  �ð3)ñ 
‰u�|‰|™UŸ\™\Ð)Õ	*ñ3)r$   c                ó   € VP                   R R p. VORNV P                  N5pV P                  V4      pV P                  e   V P                  ) MR p	VP	                  W�P                  R7      pVP                  V P                  V P                  V P                  ,          V P                  V P                  ,          .^R7      w  r«pV
P                  V4      P                  ^^4      p
VP                  V4      P                  ^^4      pVP                  V4      P                  ^^4      pVw  rÞ\        W«WÞ4      w  r«Ve   VP                  W¼V P                  4      w  r¼\        P                  ! V P                  P                   \"        4      pT! V V
VVV3RV P$                  '       g   RMV P&                  RV P(                  /VB w  ppVP*                  ! . VORN5!  P-                  4       pV P/                  V4      pVV3# )N)ÚminÚmax©ÚdimÚdropoutç        rD   éÿÿÿÿ)Úshaper:   rH   r@   ÚclampÚsplitr7   rB   ÚviewÚ	transposer   Úupdater*   r   Úget_interfacer5   Ú_attn_implementationr   Útrainingr?   rD   ÚreshapeÚ
contiguousrI   )rJ   rO   rP   rQ   rR   rK   Úinput_shapeÚhidden_shapeÚ
qkv_statesÚmin_valÚquery_statesÚ
key_statesÚvalue_statesÚcosÚsinÚattention_interfaceÚattn_outputÚattn_weightss   &&&&&,            r%   ÚforwardÚDbrxAttention.forwardP   sú  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—Y‘Y˜}Ó-ˆ
Ø$(§M¡MÒ$=�4—=‘=‘.À4ˆØ×%Ñ%¨'·}±}Ð%ÓEˆ
à1;×1AÑ1Aà× Ñ Ø×(Ñ(¨4¯=©=Õ8Ø×(Ñ(¨4¯=©=Õ8ðð
 ð 2Bó 2
Ñ.ˆ ,ð $×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆØ—_‘_ \Ó2×<Ñ<¸QÀÓBˆ
Ø#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆ�\ð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—m‘m KÓ0ˆØ˜LÐ(Ð(r$   )rH   r?   r@   r5   r:   r7   rF   r*   r<   r9   rC   rB   rI   rE   rD   ©N©NNN)
r   r    r!   r"   Ú__doc__r4   rw   r#   Ú__classdictcell__Ú__classcell__©rL   r-   s   @@r%   r'   r'   1   s#   ù‡ € Ùc÷Rõ R÷83)÷ 3)ò 3)r$   r'   c                   óD   a a€ ] tR t^†t oV 3R ltV3R lR ltRtVtV ;t# )ÚDbrxExpertGLUc                óä  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  V n        \
        P                  ! \        P                  ! V P                  V P                  ,          V P                  4      4      V n	        \
        P                  ! \        P                  ! V P                  V P                  ,          V P                  4      4      V n
        \
        P                  ! \        P                  ! V P                  V P                  ,          V P                  4      4      V n        VP                  P                  R R4      p\        V,          V n        R# )ÚnameÚsiluN)r3   r4   r7   Úffn_hidden_sizeÚmoe_num_expertsr   Ú	ParameterrT   ÚemptyÚw1Úv1Úw2Ú
ffn_act_fnÚgetr   Úactivation_fn)rJ   r5   Úact_fn_namerL   s   && €r%   r4   ÚDbrxExpertGLU.__init__‡   sö   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØ%×5Ñ5ˆÔØ%×5Ñ5ˆÔä—,’,œuŸ{š{¨4×+?Ñ+?À$×BVÑBVÕ+VÐX\×XhÑXhÓiÓjˆŒÜ—,’,œuŸ{š{¨4×+?Ñ+?À$×BVÑBVÕ+VÐX\×XhÑXhÓiÓjˆŒÜ—,’,œuŸ{š{¨4×+?Ñ+?À$×BVÑBVÕ+VÐX\×XhÑXhÓiÓjˆŒà×'Ñ'×+Ñ+¨F°FÓ;ˆÜ# KÕ0ˆÖr$   c          
      óœ   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[ P                  RS[ P                  /# )r)   ÚxÚ	expert_w1Ú	expert_v1Ú	expert_w2rS   ©rT   rU   )r,   r-   s   "€r%   r.   ÚDbrxExpertGLU.__annotate__”   sK   ø€ ÷ ñ Ù—‘ðÙ*/¯,©,ðÙCHÇ<Á<ðÙ\a×\hÑ\hðá	�‰ñr$   c                óÆ   € VP                  VP                  4      pVP                  VP                  4      pV P                  V4      pWV,          pVP                  V4      pV# ry   )ÚmatmulÚTr�   )	rJ   r‘   r’   r“   r”   Ú	gate_projÚup_projÚintermediate_statesÚ	down_projs	   &&&&&    r%   rw   ÚDbrxExpertGLU.forward”   sW   € ð —H‘H˜YŸ[™[Ó)ˆ	Ø—(‘(˜9Ÿ;™;Ó'ˆØ×&Ñ& yÓ1ˆ	Ø'Õ1ÐØ'×.Ñ.¨yÓ9ˆ	ØÐr$   )r�   r„   r7   r…   r‰   rˆ   rŠ   ©	r   r    r!   r"   r4   rw   r#   r|   r}   r~   s   @@r%   r€   r€   †   s   ù‡ € õ1÷÷ ð r$   r€   c                   óD   a a€ ] tR t^Ÿt oV 3R ltV3R lR ltRtVtV ;t# )ÚDbrxExpertsc                óª   <€ \         SV `  4        \        V4      V n        VP                  V n        VP
                  V n        VP                  V n        R # ry   )r3   r4   r€   Úmlpr7   r„   r…   Únum_experts©rJ   r5   rL   s   &&€r%   r4   ÚDbrxExperts.__init__    sD   ø€ Ü‰ÑÔÜ  Ó(ˆŒØ!×-Ñ-ˆÔØ%×5Ñ5ˆÔØ!×1Ñ1ˆÖr$   c                ó‚   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[ P                  /# )r)   rO   Útop_k_indexÚtop_k_weightsrS   r•   )r,   r-   s   "€r%   r.   ÚDbrxExperts.__annotate__§   sC   ø€ ÷ ñ á—|‘|ðñ —\‘\ðñ —|‘|ð	ñ
 
�‰ñr$   c                ó&  € VP                   ^ ,          pVP                  RV P                  4      p\        P                  ! WP
                  VP                  R7      p\        P                  ! 4       ;_uu_ 4        \        P                  P                  P                  W P                  R7      pVP                  ^^^ 4      p\        P                  ! VP                  RR7      ^ 4      P                  4       pRRR4       RV P                   V P                  3pX EF.  p	V	^ ,          p	\        P                  ! 4       ;_uu_ 4        \        P"                  ! XV	,          4      w  r«RRR4       V P$                  P&                  P)                  V4      V	,          pV P$                  P*                  P)                  V4      V	,          pV P$                  P,                  P)                  V4      V	,          pV P%                  VX,          WÜV4      pVP)                  RV P                  4      W;X
R3,          ,          pVP/                  ^ W¿4       EK1  	  VP)                  VRV P                  4      pV#   + '       g   i     EL~; i  + '       g   i     EL%; i)é    )ÚdtypeÚdevice)Únum_classesr[   Nr_   )r_   éþÿÿÿ)r`   ri   r7   rT   Ú
zeros_liker­   r®   Úno_gradr   Ú
functionalÚone_hotr¤   ÚpermuteÚgreaterÚsumÚnonzeror„   Úwherer£   r‰   rc   rˆ   rŠ   Ú
index_add_)rJ   rO   r¨   r©   Ú
batch_sizeÚnext_statesÚexpert_maskÚ
expert_hitÚsplit_expert_shapeÚ
expert_idxÚidxÚ	token_idxr‰   rˆ   rŠ   Ústatess   &&&&            r%   rw   ÚDbrxExperts.forward§   sç  € ð #×(Ñ(¨Õ+ˆ
Ø%×-Ñ-¨b°$×2BÑ2BÓCˆä×&Ò& }×<OÑ<OÐXe×XlÑXlÔmˆÜ�]Š]�_�_ÜŸ(™(×-Ñ-×5Ñ5°k×O_ÑO_Ð5Ó`ˆKØ%×-Ñ-¨a°°AÓ6ˆKÜŸš {§¡¸8 Ó'DÀaÓH×PÑPÓRˆJ÷ ð
 ! $×"6Ñ"6¸×8HÑ8HÐIÐÜ$ˆJØ# A�ˆJÜ—’—•Ü!&§¢¨[¸Õ-DÓ!E‘�÷ !à—‘—‘×!Ñ!Ð"4Ó5°jÕAˆBØ—‘—‘×!Ñ!Ð"4Ó5°jÕAˆBØ—‘—‘×!Ñ!Ð"4Ó5°jÕAˆBØ—X‘X˜m¨IÕ6¸ÀÓCˆFØ—[‘[  T×%5Ñ%5Ó6¸ÐRUÐW[ÐG[Õ9\Õ\ˆFØ×"Ñ" 1 i×8ñ %ð "×&Ñ& z°2°t×7GÑ7GÓHˆØÐ÷% �_ˆ_ú÷ !—�ús   Á9A=I+Å I?É+I<	É?J)r„   r7   r£   r¤   rŸ   r~   s   @@r%   r¡   r¡   Ÿ   s   ù‡ € õ2÷÷ ð r$   r¡   c                   óD   a a€ ] tR t^Æt oV 3R ltV3R lR ltRtVtV ;t# )Ú
DbrxRouterc                óÌ   <€ \         SV `  4        VP                  V n        VP                  V n        \        P
                  ! V P                  VP                  R R7      V n        R# ©Fr1   N)r3   r4   r7   Úmoe_jitter_epsr   rG   r…   Úlayerr¥   s   &&€r%   r4   ÚDbrxRouter.__init__Ç   sJ   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØ$×3Ñ3ˆÔÜ—Y’Y˜t×/Ñ/°×1GÑ1GÈeÔTˆŽ
r$   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r)   rO   rS   r•   )r,   r-   s   "€r%   r.   ÚDbrxRouter.__annotate__Í   s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r$   c                óL  € V P                   '       d^   V P                  eP   V\        P                  ! V4      P	                  RV P                  ,
          RV P                  ,           4      ,          pVP                  RVP                  R,          4      pV P                  V4      pV# )Ng      ð?r_   )rh   rÉ   rT   Ú
empty_likeÚuniform_rc   r`   rÊ   )rJ   rO   Úrouter_logitss   && r%   rw   ÚDbrxRouter.forwardÍ   s†   € Ø�=�=ˆ=˜T×0Ñ0Ò<ØœU×-Ò-¨mÓ<×EÑEØ�d×)Ñ)Õ)¨3°×1DÑ1DÕ+Dóõ ˆMð &×*Ñ*¨2¨}×/BÑ/BÀ2Õ/FÓGˆØŸ
™
 =Ó1ˆØÐr$   )r7   rÊ   rÉ   rŸ   r~   s   @@r%   rÆ   rÆ   Æ   s   ù‡ € õU÷÷ ð r$   rÆ   c                   óN   a a€ ] tR t^×t oRtV 3R ltR tV3R lR ltRtVt	V ;t
# )ÚDbrxFFNz0Modular DBRX MLP/FFN component with MoE support.c                óø   <€ \         SV `  4        \        VP                  4      V n        \        VP                  4      V n        VP                  P                  V n        VP                  P                  V n	        R # ry   )
r3   r4   rÆ   Ú
ffn_configÚrouterr¡   ÚexpertsÚmoe_normalize_expert_weightsÚ	moe_top_kÚtop_k)rJ   r5   rK   rL   s   &&,€r%   r4   ÚDbrxFFN.__init__Ú   sY   ø€ Ü‰ÑÔÜ  ×!2Ñ!2Ó3ˆŒÜ" 6×#4Ñ#4Ó5ˆŒà,2×,=Ñ,=×,ZÑ,ZˆÔ)Ø×&Ñ&×0Ñ0ˆŽ
r$   c                ó,  € \         P                  P                  P                  V^VP                  R7      p\         P
                  ! WP                  RR7      w  r#V P                  e+   V\         P                  ! W P                  RRR7      ,          pW#3# )é   )r\   r­   r[   T)Úpr\   Úkeepdimr_   )	rT   r   r³   Úsoftmaxr­   ÚtopkrÜ   rÚ   Únorm)rJ   rÒ   Úrouter_top_valueÚrouter_indicess   &&  r%   Úroute_tokens_to_expertsÚDbrxFFN.route_tokens_to_expertsâ   s~   € ÜŸ™×+Ñ+×3Ñ3°MÀqÐP]×PcÑPcÐ3ÓdˆÜ+0¯:ª:°mÇZÁZÐUWÔ+XÑ(ÐØ×,Ñ,Ò8Ø/´%·*²*Ø ×$EÑ$EÈ2ÐW[ô3õ  Ðð  Ð/Ð/r$   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rÍ   r•   )r,   r-   s   "€r%   r.   ÚDbrxFFN.__annotate__ë   s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r$   c                ór   € V P                  V4      pV P                  V4      w  r4V P                  WV4      pV# ry   )rØ   rç   rÙ   )rJ   rO   rÒ   r©   r¨   Úoutputs   &&    r%   rw   ÚDbrxFFN.forwardë   s8   € ØŸ™ MÓ2ˆØ%)×%AÑ%AÀ-Ó%PÑ"ˆØ—‘˜m¸-ÓHˆØˆr$   )rÙ   rÚ   rØ   rÜ   )r   r    r!   r"   r{   r4   rç   rw   r#   r|   r}   r~   s   @@r%   rÕ   rÕ   ×   s!   ù‡ € Ù:õ1ò0÷÷ ð r$   rÕ   c                   óX   a a€ ] tR t^òt oRV3R lV 3R llltRV3R lR lltRtVtV ;t# )ÚDbrxNormAttentionNormc                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# )r)   r5   r*   N©r   r+   )r,   r-   s   "€r%   r.   Ú"DbrxNormAttentionNorm.__annotate__ó   s   ø€ ÷ 	?ñ 	?™zð 	?±c¸Dµjñ 	?r$   c                ó  <€ \         SV `  4        W n        VP                  V n        \        P
                  ! VP                  R R7      V n        \        VVR7      V n	        \        P
                  ! VP                  R R7      V n
        R# )Fr1   ©r5   r*   N)r3   r4   r*   Úresid_pdropr   Ú	LayerNormr6   Únorm_1r'   ÚattnÚnorm_2©rJ   r5   r*   rL   s   &&&€r%   r4   ÚDbrxNormAttentionNorm.__init__ó   sa   ø€ Ü‰ÑÔØ"ŒØ!×-Ñ-ˆÔÜ—l’l 6§>¡>¸Ô>ˆŒÜ!ØØô
ˆŒ	ô —l’l 6§>¡>¸Ô>ˆŽr$   c                óÔ   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  R,          RS[R,          RS[RS[S[ P                  S[ P                  3,          /# )r)   rO   rQ   rP   NrR   rK   rS   )rT   rU   rV   r   r   rW   )r,   r-   s   "€r%   r.   rò   þ   sp   ø€ ÷ .ñ .á—|‘|ð.ñ #×-Ñ-ð.ñ Ÿ™ tÕ+ð	.ñ
  �ð.ñ ð.ñ 
‰u�|‰|™UŸ\™\Ð)Õ	*ñ.r$   c                ój  € TpV P                  V4      P                  VP                  4      pV P                  ! RR VRVRVRV/VB w  r\        P
                  P                  WP                  V P                  R7      pW,           pTpV P                  V4      P                  VP                  4      pWa3# ©rO   rP   rQ   rR   )rà   rh   r   )
r÷   Útor­   rø   r   r³   r]   rõ   rh   rù   )rJ   rO   rQ   rP   rR   rK   Úresidual_statesÚ_s   &&&&&,  r%   rw   ÚDbrxNormAttentionNorm.forwardþ   sÄ   € ð (ˆØŸ™ MÓ2×5Ñ5°m×6IÑ6IÓJˆàŸ9š9ñ 
Ø'ð
à)ð
ð !4ð
ð ,ð	
ð
 ñ
Ñˆô Ÿ™×-Ñ-¨m×?OÑ?OÐZ^×ZgÑZgÐ-ÓhˆØ%Õ7ˆà'ˆØŸ™ MÓ2×5Ñ5°m×6IÑ6IÓJˆàÐ-Ð-r$   )rø   r*   r÷   rù   rõ   ry   )NNrŸ   r~   s   @@r%   rï   rï   ò   s   ù‡ € ÷	?õ 	?÷.÷ .ò .r$   rï   c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )Ú	DbrxBlocki  c                ó&   <€ V ^8„  d   QhRS[ RS[/# )r)   r5   r*   rñ   )r,   r-   s   "€r%   r.   ÚDbrxBlock.__annotate__  s   ø€ ÷ 	*ñ 	*™zð 	*±cñ 	*r$   c                óº   <€ \         SV `  4        VP                  V n        VP                  V n        W n        \        VVR 7      V n        \        VR7      V n	        R# )rô   ©r5   N)
r3   r4   r6   r7   rõ   r*   rï   Únorm_attn_normrÕ   Úffnrú   s   &&&€r%   r4   ÚDbrxBlock.__init__  sN   ø€ Ü‰ÑÔØ!Ÿ>™>ˆÔØ!×-Ñ-ˆÔØ"ŒÜ3ØØô
ˆÔô  &Ô)ˆŽr$   c          
      óž   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[/# )r)   rO   rP   NrQ   rR   rK   )rT   rU   rV   r   r   )r,   r-   s   "€r%   r.   r  &  sY   ø€ ÷ ñ á—|‘|ðñ Ÿ™ tÕ+ðñ #×-Ñ-°Õ4ð	ñ
  �ðñ ñr$   c                óÚ   € V P                   ! RR VRVRVRV/VB w  raV P                  V4      p\        P                  P	                  WP
                  V P                  R7      pWa,           pV# rþ   )r	  r
  r   r³   r]   rõ   rh   )rJ   rO   rP   rQ   rR   rK   Úresid_statess   &&&&&, r%   rw   ÚDbrxBlock.forward&  s‰   € ð '+×&9Ò&9ñ '
Ø'ð'
à)ð'
ð !4ð'
ð ,ð	'
ð
 ñ'
Ñ#ˆð Ÿ™ Ó/ˆÜŸ™×-Ñ-¨m×?OÑ?OÐZ^×ZgÑZgÐ-ÓhˆØ$Õ4ˆØÐr$   )r
  r7   r*   r	  rõ   rz   rŸ   r~   s   @@r%   r  r    s   ù‡ € ÷	*ó 	*÷÷ ò r$   r  c                   ó¦   a a€ ] tR tRt oRtRtR.tR.tRtRt	Rt
RtRtR]R]/t]P"                  ! 4       V3R	 lV 3R
 ll4       tV3R ltRtVtV ;t# )ÚDbrxPreTrainedModeli<  ÚtransformerTr  rR   FrO   Ú
attentionsc                ó4   <€ V ^8„  d   QhRS[ P                  /# )r)   Úmodule)r   ÚModule)r,   r-   s   "€r%   r.   Ú DbrxPreTrainedModel.__annotate__M  s   ø€ ÷ 7ñ 7¡B§I¡Iñ 7r$   c                óT  <€ \         SV `  V4       V P                  P                  p\	        V\
        4      '       dl   \        P                  ! VP                  R VR7       \        P                  ! VP                  R VR7       \        P                  ! VP                  R VR7       R# R# )r^   )ÚmeanÚstdN)r3   Ú_init_weightsr5   Úinitializer_rangeÚ
isinstancer€   ÚinitÚnormal_rˆ   r‰   rŠ   )rJ   r  r  rL   s   && €r%   r  Ú!DbrxPreTrainedModel._init_weightsL  sm   ø€ ä‰Ñ˜fÔ%Ø�k‰k×+Ñ+ˆÜ�fœm×,Ò,Ü�LŠL˜Ÿ™¨°#Õ6Ü�LŠL˜Ÿ™¨°#Õ6Ü�LŠL˜Ÿ™¨°#×6ñ -r$   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # ©r)   r5   r   )r,   r-   s   "€r%   r.   r  <  s   ø‡ ‚ ÙÑò r$   r   )r   r    r!   r"   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flex_attnÚ_supports_attention_backendÚ_supports_flash_attnÚ_supports_sdpaÚ_can_compile_fullgraphr  r'   Ú_can_record_outputsrT   r²   r  Ú__annotate_func__r#   r|   r}   r~   s   @@r%   r  r  <  sw   ù‡ € à%ÐØ&*Ð#Ø$˜ÐØ#4Ð"5ÐØÐØ"&ÐØÐØ€NØ"Ðà˜Ø�mðÐð
 ‡]‚]ƒ_÷7ò 7ó ð7÷# … r$   r  c                   óš   a a€ ] tR tRt oRtV3R lV 3R lltV3R lR ltV3R lR lt]]	]
RV3R	 lR
 ll4       4       4       tRtVtV ;t# )Ú	DbrxModeliV  a•  Transformer decoder consisting of *config.num_hidden_layers*. Each layer is a [`DbrxBlock`] layer.

Args:
    config ([`DbrxConfig`]): Model configuration class with all parameters of the model.
        Initializing with a config file does not load the weights associated with the model, only the
        configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
c                ó    <€ V ^8„  d   QhRS[ /# r"  r   )r,   r-   s   "€r%   r.   ÚDbrxModel.__annotate__`  s   ø€ ÷ ñ ™zñ r$   c           	     ó6  <€ \         SV `  V4       VP                  V n        VP                  V n        VP
                  V n        \        V4      V n        \        P                  ! VP                  VP                  V P                  4      V n        \        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n        \        P"                  ! VP                  R R7      V n        R V n        V P)                  4        R# u upi rÈ   )r3   r4   Úpad_token_idÚpadding_idxÚ
vocab_sizeÚ	emb_pdropr   Ú
rotary_embr   Ú	Embeddingr6   ÚwteÚ
ModuleListÚrangeÚn_layersr  Úblocksrö   Únorm_fÚgradient_checkpointingÚ	post_initrú   s   && €r%   r4   ÚDbrxModel.__init__`  sË   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒØ×)Ñ)ˆŒÜ-¨fÓ5ˆŒÜ—<’< × 1Ñ 1°6·>±>À4×CSÑCSÓTˆŒÜ—m’mÔSXÐY_×YhÑYhÔSiÓ$jÑSiÀi¤Y¨vÖ%AÑSiÑ$jÓkˆŒÜ—l’l 6§>¡>¸Ô>ˆŒØ&+ˆÔ#ð 	�‰Öùò %ks   Â6Dc                ó4   <€ V ^8„  d   QhRS[ P                  /# ©r)   rS   ©r   r8  )r,   r-   s   "€r%   r.   r1  n  s   ø€ ÷ ñ ¡b§l¡lñ r$   c                ó   € V P                   # ry   ©r9  ©rJ   s   &r%   Úget_input_embeddingsÚDbrxModel.get_input_embeddingsn  s   € Ø�x‰xˆr$   c                ó4   <€ V ^8„  d   QhRS[ P                  /# ©r)   ÚvaluerD  )r,   r-   s   "€r%   r.   r1  q  s   ø€ ÷ ñ ©"¯,©,ñ r$   c                ó   € Wn         R # ry   rF  ©rJ   rL  s   &&r%   Úset_input_embeddingsÚDbrxModel.set_input_embeddingsq  s   € ØŽr$   c                óþ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[S[,          R	S[/# )
r)   Ú	input_idsNrP   Úposition_idsrR   Úinputs_embedsÚ	use_cacherK   rS   )	rT   rV   rU   r   ÚFloatTensorÚboolr   r   r   )r,   r-   s   "€r%   r.   r1  w  s“   ø€ ÷ 5
ñ 5
á×#Ñ# dÕ*ð5
ñ Ÿ™ tÕ+ð5
ñ ×&Ñ&¨Õ-ð	5
ñ
  �ð5
ñ ×(Ñ(¨4Õ/ð5
ñ ˜$•;ð5
ñ Ñ+Õ,ð5
ñ 
 ñ5
r$   c                ó†  € VR J VR J,          '       d   \        R4      hV'       d   Vf   \        V P                  R7      pVf   V P                  V4      pVfb   Ve   VP	                  4       M^ p\
        P                  ! VP                  ^,          VP                  R7      V,           pVP                  ^ 4      p\        V P                  VVVVR7      p	Tp
V P                  W£4      pV P                  R V P                  P                    F  pV! V
3RVRV	RVRVR	V/VB p
K  	  V P                  V
4      p
\        V
VR
7      # )Nz:You must specify exactly one of input_ids or inputs_embedsr  )r®   )r5   rT  rP   rR   rS  rQ   rP   rS  rR   rU  )Úlast_hidden_staterR   )Ú
ValueErrorr   r5   r9  Úget_seq_lengthrT   Úaranger`   r®   Ú	unsqueezer
   r7  r=  Únum_hidden_layersr>  r   )rJ   rR  rP   rS  rR   rT  rU  rK   Úpast_seen_tokensÚcausal_maskrO   rQ   Údecoder_layers   &&&&&&&,     r%   rw   ÚDbrxModel.forwardt  sc  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZç˜Ò0Ü*°$·+±+Ô>ˆOàÒ Ø ŸH™H YÓ/ˆMàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLä(Ø—;‘;Ø'Ø)Ø+Ø%ô
ˆð &ˆð #Ÿo™o¨mÓJÐà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÓIˆMÙ)Øñà$7ðð  +ðð *ð	ð
 !0ðð $ðð ñŠMñ Jð Ÿ™ MÓ2ˆä%Ø+Ø+ô
ð 	
r$   )r=  r6  r?  r>  r4  r7  r5  r9  )NNNNNN)r   r    r!   r"   r{   r4   rH  rO  r   r   r   rw   r#   r|   r}   r~   s   @@r%   r/  r/  V  sV   ù‡ € ñ÷ó ÷ð ÷ð ð  ØØ÷5
ñ 5
ó ó ó  ÷5
ð 5
r$   r/  c                   óü   a a€ ] tR tRt oRR/tRR/tRR.R.3/tRR/tV3R	 lV 3R
 lltV3R lR lt	V3R lR lt
V3R lR ltV3R lR ltV3R lR ltV3R lR lt]]RV3R lR ll4       4       tRtVtV ;t# )ÚDbrxForCausalLMi¯  zlm_head.weightztransformer.wte.weightÚlm_headÚcolwise_gather_outputrO   ÚlogitsÚkeep_full_weightc                ó    <€ V ^8„  d   QhRS[ /# r"  r   )r,   r-   s   "€r%   r.   ÚDbrxForCausalLM.__annotate__¶  s   ø€ ÷ ñ ™zñ r$   c                óŽ  <€ \         SV `  V4       \        V4      V n        VP                  V n        \
        P                  ! VP                  VP                  R R7      V n        VP                  P                  V n        VP                  P                  V n        VP                  P                  V n        V P!                  4        R# rÈ   )r3   r4   r/  r  r5  r   rG   r7   re  r×   Úmoe_loss_weightÚrouter_aux_loss_coefr…   r¤   rÛ   Únum_experts_per_tokr@  r¥   s   &&€r%   r4   ÚDbrxForCausalLM.__init__¶  s�   ø€ Ü‰Ñ˜Ô Ü$ VÓ,ˆÔØ ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒØ$*×$5Ñ$5×$EÑ$EˆÔ!Ø!×,Ñ,×<Ñ<ˆÔØ#)×#4Ñ#4×#>Ñ#>ˆÔ Ø�‰Ör$   c                ó4   <€ V ^8„  d   QhRS[ P                  /# rC  rD  )r,   r-   s   "€r%   r.   rj  À  s   ø€ ÷ 7ñ 7¡b§l¡lñ 7r$   c                ó6   € V P                   P                  4       # ry   )r  rH  rG  s   &r%   rH  Ú$DbrxForCausalLM.get_input_embeddingsÀ  s   € Ø×Ñ×4Ñ4Ó6Ð6r$   c                ó4   <€ V ^8„  d   QhRS[ P                  /# rK  rD  )r,   r-   s   "€r%   r.   rj  Ã  s   ø€ ÷ 5ñ 5©"¯,©,ñ 5r$   c                ó<   € V P                   P                  V4       R # ry   )r  rO  rN  s   &&r%   rO  Ú$DbrxForCausalLM.set_input_embeddingsÃ  s   € Ø×Ñ×-Ñ-¨eÖ4r$   c                ó4   <€ V ^8„  d   QhRS[ P                  /# rC  ©r   rG   )r,   r-   s   "€r%   r.   rj  Æ  s   ø€ ÷ ñ ¡r§y¡yñ r$   c                ó   € V P                   # ry   ©re  rG  s   &r%   Úget_output_embeddingsÚ%DbrxForCausalLM.get_output_embeddingsÆ  s   € Ø�|‰|Ðr$   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r)   Únew_embeddingsrw  )r,   r-   s   "€r%   r.   rj  É  s   ø€ ÷ &ñ &±B·I±Iñ &r$   c                ó   € Wn         R # ry   ry  )rJ   r}  s   &&r%   Úset_output_embeddingsÚ%DbrxForCausalLM.set_output_embeddingsÉ  s   € Ø%Žr$   c                ó    <€ V ^8„  d   QhRS[ /# )r)   Údecoder©r/  )r,   r-   s   "€r%   r.   rj  Ì  s   ø€ ÷ #ñ #¡9ñ #r$   c                ó   € Wn         R # ry   ©r  )rJ   r‚  s   &&r%   Úset_decoderÚDbrxForCausalLM.set_decoderÌ  s   € Ø"Ör$   c                ó    <€ V ^8„  d   QhRS[ /# rC  rƒ  )r,   r-   s   "€r%   r.   rj  Ï  s   ø€ ÷  ñ  ™Yñ  r$   c                ó   € V P                   # ry   r…  rG  s   &r%   Úget_decoderÚDbrxForCausalLM.get_decoderÏ  s   € Ø×ÑÐr$   c                ód  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[R,          R
S[S[ P                  ,          RS[S[,          RS[	/# )r)   rR  NrP   rS  rR   rT  ÚlabelsrU  Úoutput_router_logitsÚlogits_to_keeprK   rS   )
rT   rV   rU   r   rV  rW  r+   r   r   r   )r,   r-   s   "€r%   r.   rj  Ô  sÖ   ø€ ÷ P
ñ P
á×#Ñ# dÕ*ðP
ñ Ÿ™ tÕ+ðP
ñ ×&Ñ&¨Õ-ð	P
ñ
  �ðP
ñ ×(Ñ(¨4Õ/ðP
ñ × Ñ  4Õ'ðP
ñ ˜$•;ðP
ñ # T�kðP
ñ ™eŸl™lÕ*ðP
ñ Ñ+Õ,ðP
ñ 
#ñP
r$   c
                ó¦  € Ve   TMV P                   P                  pV P                  ! RRVRVRVRVRVRVRV/V
B pVP                  p\	        V	\
        4      '       d   \        V	) R4      MT	pV P                  VR	VR	3,          4      pRpVe   V P                  ! WæV P                  3/ V
B pRpV'       dc   \        VP                  V P                  V P                  V4      pVe3   WðP                  VP                  VP                   4      ,          ,          p\#        VVVVP$                  VP&                  VP(                  VP                  R
7      # )a  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

Example:

```python
>> from transformers import AutoTokenizer, DbrxForCausalLM

>> model = DbrxForCausalLM.from_pretrained("transformers-community/dbrx-instruct")
>> tokenizer = AutoTokenizer.from_pretrained("transformers-community/dbrx-instruct")

>> prompt = "Hey, are you conscious? Can you talk to me?"
>> inputs = tokenizer(prompt, return_tensors="pt")

>> # Generate
>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
```
NrR  rP   rS  rR   rT  rU  rŽ  :NNN)ÚlossÚaux_lossrg  rR   rO   r  rÒ   r   )r5   rŽ  r  rY  r  r+   Úslicere  Úloss_functionr5  r   rÒ   r¤   rn  rm  rÿ   r®   r   rR   rO   r  )rJ   rR  rP   rS  rR   rT  r�  rU  rŽ  r�  rK   ÚoutputsrO   Úslice_indicesrg  r‘  r’  s   &&&&&&&&&&,      r%   rw   ÚDbrxForCausalLM.forwardÒ  sz  € ðN %9Ò$DÑ È$Ï+É+×JjÑJjð 	ð
 +/×*:Ò*:ñ 	+
Øð	+
à)ð	+
ð &ð	+
ð ,ð		+
ð
 (ð	+
ð  ð	+
ð "6ð	+
ð ñ	+
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒØ×%Ò% f°d·o±oÑPÈÑPˆDàˆßÜ/Ø×%Ñ%Ø× Ñ Ø×(Ñ(Øó	ˆHð Ò!Ø×1Ñ1°H·K±KÀÇÁÓ4LÕLÕL�ä(ØØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø!×/Ñ/ô
ð 	
r$   )re  r¤   rn  rm  r  r5  )	NNNNNNNNr¬   )r   r    r!   r"   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr4   rH  rO  rz  r  r†  rŠ  r   r   rw   r#   r|   r}   r~   s   @@r%   rd  rd  ¯  s«   ù‡ € Ø*Ð,DÐEÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HàÐ/Ð0€J÷ó ÷7ð 7÷5ð 5÷ð ÷&ð &÷#ð #÷ ð  ð Ø÷P
ñ P
ó ó ÷P
ð P
r$   rd  )rd  r/  r  ):r{   Úcollections.abcr   Útypingr   rT   r   Ú r   r  Úactivationsr   Úcache_utilsr   r   Ú
generationr	   Úmasking_utilsr
   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.genericr   Úutils.output_capturingr   Úllama.modeling_llamar   r   r   Úmixtral.modeling_mixtralr   Úconfiguration_dbrxr   r   r  r'   r€   r¡   rÆ   rÕ   rï   r  r  r/  rd  Ú__all__r   r$   r%   Ú<module>r®     s  ðñ )å $Ý ã Ý å &Ý !ß .Ý )Ý /õ÷ Rß FÝ &ß IÑ IÝ 7Ý 5÷ñ õ
 @Ý *ô	Ð.ô 	ôR)�B—I‘Iô R)ôj�B—I‘Iô ô2$�"—)‘)ô $ôN�—‘ô ô"ˆb�i‰iô ô6%.˜BŸI™Iô %.ôPÐ*ô ôD7˜/ô 7ð4 ôU
Ð#ó U
ó ðU
ôpu
Ð)¨?ô u
òp B‚r$   