+
    œ°³jjj  ã                   óþ  € R t ^ RIt^ RIHt ^ RIHt ^ RIt^ RIHu H	t
 ^ RIHt ^ RIHt ^RIHt ^RIHt ^RIHt ^R	IHt ^R
IHt ^RIHt ^RIHtHt ^RIHt ^RIH t H!t! ^RI"H#t# ^RI$H%t%H&t&H't'H(t( ^RI)H*t* ^RI+H,t,H-t-H.t.H/t/H0t0H1t1H2t2H3t3 ^RI4H5t5H6t6 ](Pn                  ! ]84      t9]'! 4       '       d   ^ RI:H;t; ]&! RR7      ] ! R R]4      4       4       t< ! R R]/4      t= ! R R]04      t>R2R R llt?] ! 4       t@]?]@R&    ! R  R!]P‚                  4      tB ! R" R#]-4      tC ! R$ R%]P‚                  4      tD ! R& R']4      tE ! R( R)].4      tF ! R* R+]64      tGR3R, R- lltH ! R. R/]54      tI ! R0 R1],4      tJ. R4OtKR# )5zPyTorch Doge model.N)ÚCallable)ÚUnion)Ústrict)Únn)Úinitialization)ÚACT2FN)ÚCache)ÚPreTrainedConfig)Úcompile_friendly_flex_attention)ÚGradientCheckpointingLayer)ÚMoeCausalLMOutputWithPastÚMoeModelOutputWithPast)ÚRopeParameters)ÚAttentionInterfaceÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚis_torch_flex_attn_availableÚlogging)ÚOutputRecorder)ÚLlamaForSequenceClassificationÚLlamaMLPÚLlamaPreTrainedModelÚLlamaRMSNormÚLlamaRotaryEmbeddingÚapply_rotary_pos_embÚeager_attention_forwardÚ	repeat_kv)ÚMixtralForCausalLMÚMixtralModel)Ú	BlockMaskzSmallDoge/Doge-320M)Ú
checkpointc                   ó  a a€ ] tR t^:t oRtRtR.tRRRRRRRR	R
R	RRRRRR	RRRRRR/tRR.R.3RRR.R.3RR.R.3/tRt	Rt
Rt^ tRtRtRtR tR!tR"tRtR#t^tR#tR"tRtR"tR#tRtR"tR$t^@tR"tR"t R%t!R#t"R#t#R#t$V 3R& lt%V3R' lt&R(t'Vt(V ;t)# ))Ú
DogeConfiga´  
keep_window_size (`int`, *optional*, defaults to 2048):
    The window size of tokens that are not dynamically masked, and dynamic masking is only performed when the sequence length exceeds this value.
is_moe (`bool`, *optional*, defaults to `False`):
    Whether to use the Cross Domain Mixture of Experts, if `True`, the MoE will inherit the MLP to initialize.

```python
>>> from transformers import DogeConfig, DogeModel

>>> # Initializing a Doge-320M style configuration
>>> configuration = DogeConfig()

>>> # Initializing a model from the Doge-320M style configuration
>>> model = DogeModel(configuration)

>>> # Accessing the model configuration
>>> configuration = model.config
```ÚdogeÚpast_key_valueszlayers.*.self_attn.q_projÚcolwisezlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.dt_projÚrowwisezlayers.*.self_attn.o_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projzlayers.*.mlp.router_gateÚcolwise_gather_outputzlayers.*.mlp.down_embedÚrowwise_split_inputzlayers.*.mlp.up_embedÚembed_tokensÚ	input_idsÚinputs_embedsÚlayersÚhidden_statesÚattention_maskÚnormi €  i   é   ç        Úsilug{®Gáz”?g�íµ ÷Æ°>TFNi @  gü©ñÒMbP?c                óf   <€ V P                   f   V P                  V n         \        SV `  ! R/ VB  R # )N© )Únum_key_value_headsÚnum_attention_headsÚsuperÚ__post_init__)ÚselfÚkwargsÚ	__class__s   &,€Úf/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/doge/modular_doge.pyr:   ÚDogeConfig.__post_init__ƒ   s-   ø€ à×#Ñ#Ò+Ø'+×'?Ñ'?ˆDÔ$ä‰ÒÑ' Ô'ó    c                ó  <€ V ^8„  d   Qh/ S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[S[ ,          ;R&   S[;R&   S[;R&   S[;R&   S[;R	&   S[;R
&   S[ ;R&   S[S[,          R,          ;R&   S[ ;R&   S[ R,          ;R&   S[;R&   S[R,          ;R&   S[;R&   S[ R,          ;R&   S[ ;R&   S[;R&   S[ ;R&   S[ ;R&   S[;R&   S[;R&   S[;R&   S[ R,          ;R&   S[ R,          ;R&   S[ S[S[ ,          ,          R,          ;R&   # )é   Ú
vocab_sizeÚhidden_sizeÚintermediate_sizeÚnum_hidden_layersÚhidden_dropoutÚ
hidden_actÚinitializer_rangeÚrms_norm_epsÚ	use_cacheÚtie_word_embeddingsÚmax_position_embeddingsNÚrope_parametersr8   r7   Úattention_biasÚattention_dropoutÚmlp_biasÚsliding_windowÚkeep_window_sizeÚis_moeÚnum_expertsÚnum_experts_per_tokÚnorm_topk_probÚoutput_router_logitsÚrouter_aux_loss_coefÚpad_token_idÚbos_token_idÚeos_token_id)ÚintÚfloatÚstrÚboolr   ÚdictÚlist)ÚformatÚ__classdict__s   "€r>   Ú__annotate__ÚDogeConfig.__annotate__:   sØ  ø‡ ‚ ñX ÑñY ñZ Ññ[ ñ\ Ñ!ñ] ñ^ Ññ_ ñ` ™C•KÑ%ña ñb Ññc ñd Ñ#ñe ñf Ññg ñh Ññi ñj Ñ%ñk ñl !Ñ'ñm ñn $¡dÕ*¨TÕ1Ñ8ño ñp Ñ ñq ñr ˜t�Ñ*ñs ñt Ñ ñu ñv ˜t•|Ñ)ñw ñx Ññy ñz ˜$•JÑ%ñ{ ñ| Ñ ñ} ñ~ Ññ ñ@ ÑñA ñB Ñ!ñC ñD Ñ ñE ñF Ñ&ñG ñH  Ñ'ñI ñJ ˜•*Ñ#ñK ñL ˜•*Ñ#ñM ñN ™™S�	•/ DÕ(Ñ/òO r@   )r7   )*Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferenceÚbase_model_tp_planÚbase_model_pp_planrC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   rM   rN   r8   r7   rO   rP   rQ   rR   rS   rT   rU   rV   rW   rX   rY   rZ   r[   r\   r:   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r=   rd   s   @@r>   r$   r$   :   sA  ù‡ € ñð& €JØ#4Ð"5Ðð 	$ YØ# YØ# YØ$ iØ# YØ  )Ø 	Ø  )Ø"Ð$;Ø!Ð#8ØÐ!6ðÐð 	˜+˜¨Ð(9Ð:Ø�OÐ%5Ð6¸Ð8IÐJØ�/Ð" _Ð$5Ð6ðÐð €JØ€KØ!ÐØÐØ"%€NØ€JØ#ÐØ€LØ€IØ %ÐØ#'ÐØ48€OØ ÐØ&*ÐØ €NØ&)ÐØ€HØ!%€NØ ÐØ€FØ€KØ!ÐØ €NØ!&ÐØ"'ÐØ#€LØ#€LØ+/€Lõ(÷S … r@   r$   c                   ó   € ] tR t^‹tRtR# )ÚDogeRMSNormr6   N©rg   rh   ri   rj   rq   r6   r@   r>   rv   rv   ‹   ó   † Ûr@   rv   c                   ó   € ] tR t^�tRtR# )ÚDogeRotaryEmbeddingr6   Nrw   r6   r@   r>   rz   rz   �   rx   r@   rz   c                ó\  € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        \        P                  R3,          R\
        R,          R	\
        R,          R
\        \        P                  \        P                  3,          /# )rB   ÚmoduleÚqueryÚkeyÚvaluer0   r!   ÚscalingNÚsoftcapÚreturn)r   ÚModuleÚtorchÚTensorr   r^   Útuple)rc   s   "r>   re   re   “   s�   € ÷ +*ñ +*Ü�I‰Ið+*ä�<‰<ð+*ô 
�‰ð+*ô �<‰<ð	+*ô
 œ%Ÿ,™,¨Ð3Õ4ð+*ô �T�\ð+*ô �T�\ð+*ô Œ5�<‰<œŸ™Ð%Õ&ñ+*r@   c                 ó@  aa€ R pR o\        V\        4      '       d   TpMVoSe!   SRRRR VP                  R,          13,          oVV3R lp	\        VVVV	VRVRR7      w  r«VP	                  VP
                  4      pV
P                  ^^4      P                  4       p
W«3# )NºNNNc                 óª   <€ Se%   S\         P                  ! V S,          4      ,          p Se&   V SV,          V,          V,          V,          ,           p V # ©N)r„   Útanh)ÚscoreÚ	batch_idxÚhead_idxÚq_idxÚkv_idxÚcausal_maskr�   s   &&&&&€€r>   Ú	score_modÚ)flex_attention_forward.<locals>.score_mod§   sI   ø€ ØÒØœeŸjšj¨°­Ó9Õ9ˆEØÒ"Ø˜K¨	Õ2°8Õ<¸UÕCÀFÕKÕKˆEØˆr@   T)r’   Ú
block_maskÚ
enable_gqaÚscaleÚ
return_lseéþÿÿÿ)Ú
isinstancer!   Úshaper
   ÚtoÚdtypeÚ	transposeÚ
contiguous)r|   r}   r~   r   r0   r€   r�   r<   r”   r’   Úattn_outputÚattention_weightsr‘   s   &&&&&&f,    @r>   Úflex_attention_forwardr¡   “   sµ   ù€ ð €JØ€KÜ�.¤)×,Ò,Ø#‰
à$ˆàÒØ! ! Q¨¨?¨S¯Y©Y°r­]¨?Ð":Õ;ˆöô &EØØØØØØØð ô&Ñ"€Kð *×,Ñ,¨U¯[©[Ó9ÐØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ)Ð)r@   Údoge_flex_attentionc                   ón   a a€ ] tR t^Åt oRV3R lV 3R llltR	V3R lR lltR
V3R lR lltRtVtV ;t	# )ÚDogeAttentionc                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# ©rB   ÚconfigÚ	layer_idxN©r$   r]   )rc   rd   s   "€r>   re   ÚDogeAttention.__annotate__Æ   s#   ø€ ÷ Jñ J™zð J±c¸Dµjñ Jr@   c                óZ  <€ \         SV `  4        Wn        W n        \	        VR VP
                  VP                  ,          4      V n        VP                  VP                  ,          V n	        V P                  R,          V n
        VP                  V n        VP                  V n        \        P                  ! VP
                  VP                  V P                  ,          VP                  R7      V n        \        P                  ! VP
                  VP                  V P                  ,          VP                  R7      V n        \        P                  ! VP
                  VP                  V P                  ,          VP                  R7      V n        \        P&                  ! \(        P*                  ! VP                  4      4      V n        \        P                  ! VP                  V P                  ,          VP                  VP                  R7      V n        \        P                  ! VP                  V P                  ,          VP
                  VP                  R7      V n        \3        V P                  VP4                  R7      V n        \3        V P                  VP4                  R7      V n        R# )Úhead_dim©Úbias©ÚepsNg      à¿)r9   Ú__init__r§   r¨   ÚgetattrrD   r8   r¬   r7   Únum_key_value_groupsr€   rP   rS   r   ÚLinearrO   Úq_projÚk_projÚv_projÚ	Parameterr„   ÚzerosÚAÚdt_projÚo_projrv   rJ   Úq_normÚk_norm©r;   r§   r¨   r=   s   &&&€r>   r±   ÚDogeAttention.__init__Æ   sÖ  ø€ Ü‰ÑÔØŒØ"ŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÕ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Õ$\ˆÔ!Ø—}‘} dÕ*ˆŒØ!'×!9Ñ!9ˆÔØ &× 7Ñ 7ˆÔä—i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —’œeŸkšk¨&×*DÑ*DÓEÓFˆŒÜ—y’yØ×&Ñ&¨¯©Õ6¸×8RÑ8RÐY_×YnÑYnô
ˆŒô —i’iØ×&Ñ&¨¯©Õ6¸×8JÑ8JÐQW×QfÑQfô
ˆŒô " $§-¡-°V×5HÑ5HÔIˆŒÜ! $§-¡-°V×5HÑ5HÔIˆŽr@   c                ó<  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          RS[ P                  R,          RS[R,          RS[S[ P                  S[ P                  R,          S[S[ P                  ,          R,          3,          /# )rB   r/   Úposition_embeddingsr0   Nr&   r‚   )r„   r…   r†   r   )rc   rd   s   "€r>   re   rª   ä   s‡   ø€ ÷ 3)ñ 3)á—|‘|ð3)ñ #¡5§<¡<±·±Ð#=Õ>ð3)ñ Ÿ™ tÕ+ð	3)ñ
  �ð3)ñ 
‰u�|‰|™UŸ\™\¨DÕ0±%¹¿¹Õ2EÈÕ2LÐLÕ	Mñ3)r@   c           	     óö  € VP                   R R p. VORNV P                  N5pV P                  V P                  V4      P	                  V4      4      P                  ^^4      pV P                  V P                  V4      P	                  V4      4      P                  ^^4      p	V P                  V4      P	                  V4      P                  ^^4      p
Vw  r¼\        W‰W¼4      w  r‰Ve   VP                  WšV P                  4      w  ršV P                  V
P                  ^^4      P                  V
P                   ^ ,          V
P                   R,          R4      4      p\        P                  ! V P                   \"        P$                  ! V4      ,          4      P                  RR4      pV P'                  VVV P(                  VR7      p\+        WàP,                  4      p\.        P1                  V P2                  P4                  \6        4      pT! V VV	V
3RTRV P8                  '       g   RMV P:                  RV P<                  /VB w  ppVP                  ! . VORN5!  P?                  4       pV PA                  V4      pVV3# )N)r/   Ú	dt_statesrS   r0   r0   Údropoutr3   r€   éÿÿÿÿr˜   )!rš   r¬   r½   rµ   Úviewr�   r¾   r¶   r·   r   Úupdater¨   r»   Úreshaper„   Úexprº   ÚFÚsoftplusÚprepare_dynamic_maskrS   r   r³   ÚALL_ATTENTION_FUNCTIONSÚget_interfacer§   Ú_attn_implementationr   ÚtrainingrP   r€   rž   r¼   )r;   r/   rÂ   r0   r&   r<   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚcosÚsinrÄ   Ú	attn_maskÚattention_interfacerŸ   Úattn_weightss   &&&&&,            r>   ÚforwardÚDogeAttention.forwardä   sU  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ 4§;¡;¨}Ó#=×#BÑ#BÀ<Ó#PÓQ×[Ñ[Ð\]Ð_`ÓaˆØ—[‘[ §¡¨]Ó!;×!@Ñ!@ÀÓ!NÓO×YÑYÐZ[Ð]^Ó_ˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJð —L‘LØ×"Ñ" 1 aÓ(×0Ñ0°×1CÑ1CÀAÕ1FÈ×HZÑHZÐ[]ÕH^Ð`bÓcó
ˆ	ô —I’I˜dŸf™f¤q§z¢z°)Ó'<Õ<Ó=×GÑGÈÈBÓOˆ	Ø×-Ñ-Ø'ØØ!×2Ñ2Ø)ð	 .ó 
ˆ	ô ˜i×)BÑ)BÓCˆ	ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØñ		%
ð
 %ð	%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆ�\ð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r@   c          	      ó|   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[RS[ P                  R,          /# )rB   r/   rÄ   rS   r0   N)r„   r…   r]   )rc   rd   s   "€r>   re   rª     sD   ø€ ÷ #ñ #á—|‘|ð#ñ —<‘<ð#ñ ð	#ñ
 Ÿ™ tÕ+ñ#r@   c           
     ó:  € \         P                  ! VP                  4      P                  pVP                  pVR
,          P	                  RRVP
                  ^,          R4      pVe¯   \        V\        4      '       g™   VP                  \         P                  8X  dG   VP                  p\         P                  ! V\         P                  ! RRVP                  VR7      V4      pVP                  VRRRRVP
                  R,          13,          ^ 8g  V4      pVP
                  R,          V8”  do   \         P                  ! WvVP                  R7      p\         P                  ! WsRRRR7      P                  p	VP!                  RV	R	4      pVP                  VR8H  V4      pV# )aø  
The core idea of DMA is to calculate the dynamic attention mask to mask the tokens that should be masked, so as to form sparse attention.

Combine `dt_states` with `attention_mask` to generate the final `attn_mask`.

Args:
    hidden_states (`torch.Tensor`): The input hidden_states, used to determine the minimum value of the current input precision.
    dt_states (`torch.Tensor`): dt_states of shape `(batch_size, num_heads, key_sequence_length)`.
    keep_window_size (`int`): The window size of tokens that are not dynamically masked, and dynamic masking is only performed when the sequence length exceeds this value.
    attention_mask (`torch.Tensor`, *optional*): attention mask of shape `(batch_size, 1, query_sequence_length, key_sequence_length)`.
rˆ   Nr3   )Údevicerœ   ©rœ   rà   TF)ÚdimÚlargestÚsortedg      ð?)rˆ   rˆ   Nrˆ   rÆ   r6   )r„   Úfinforœ   ÚminÚexpandrš   r™   r!   r`   ÚwhereÚfullrà   Úmasked_fillÚ
zeros_likeÚtopkÚindicesÚscatter)
r;   r/   rÄ   rS   r0   Ú	min_dtyperœ   rÙ   Úactive_maskÚtopk_indicess
   &&&&&     r>   rÍ   Ú"DogeAttention.prepare_dynamic_mask  s_  € ô$ —K’K × 3Ñ 3Ó4×8Ñ8ˆ	Ø×#Ñ#ˆØ˜mÕ,×3Ñ3Ø��M×'Ñ'¨Õ*¨Bó
ˆ	ð Ò%¬j¸Ì×.SÒ.SØ×#Ñ#¤u§z¡zÔ1Ø%×+Ñ+�Ü!&§¢Ø"¤E§J¢J¨r°3¸~×?TÑ?TÐ\aÔ$bÐdmó"�ð "×-Ñ-¨n¸QÀÀ1ÐF[È	ÏÉÐXZÕH[ÐF[Ð=[Õ.\Ð`aÑ.aÐclÓmˆIØ�?‰?˜2ÕÐ!1Ô1Ü×*Ò*¨9È)×JZÑJZÔ[ˆKÜ Ÿ:š: iÀrÐSWÐ`eÔf×nÑnˆLØ%×-Ñ-¨b°,ÀÓDˆKØ!×-Ñ-¨k¸SÑ.@À)ÓLˆIØÐr@   )rº   rP   r§   r»   r¬   r¾   r¶   rS   r¨   r³   r¼   r½   rµ   r€   r·   rŠ   ©NN)r2   N)
rg   rh   ri   rj   r±   rÜ   rÍ   rq   rr   rs   rt   s   @@r>   r¤   r¤   Å   s+   ù‡ € ÷Jõ J÷<3)ò 3)÷j#÷ #ò #r@   r¤   c                   ó   € ] tR tRtRtR# )ÚDogeMLPi?  r6   Nrw   r6   r@   r>   rõ   rõ   ?  rx   r@   rõ   c                   óP   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltRtVtV ;t# )Ú	DogeCDMoEiC  c                ó    <€ V ^8„  d   QhRS[ /# )rB   r§   )r$   )rc   rd   s   "€r>   re   ÚDogeCDMoE.__annotate__D  s   ø€ ÷ Iñ I™zñ Ir@   c                ó  <€ \         SV `  4        VP                  V n        VP                  V n        \        VP
                  ,          V n        VP                  V n        \        P                  ! \        P                  ! V P                  4      4      V n        VP                  V n        VP                  V n        \        P                   ! V P                  V P                  VP"                  R 7      V n        \        P                   ! V P                  V P                  VP"                  R 7      V n        \        P                   ! V P                  V P                  VP"                  R 7      V n        \        P                   ! V P                  V P                  ^,          RR 7      V n        \        P,                  ! V P                  V P                  4      V n        \        P,                  ! V P                  V P                  4      V n        R# )r­   FN)r9   r±   rD   rE   r   rH   Úact_fnrU   ÚmathÚfloorÚsqrtÚnum_keysrV   Útop_krW   r   r´   rQ   Ú	gate_projÚup_projÚ	down_projÚrouter_gateÚ	EmbeddingÚ
down_embedÚup_embed©r;   r§   r=   s   &&€r>   r±   ÚDogeCDMoE.__init__D  s_  ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔÜ˜V×.Ñ.Õ/ˆŒà!×-Ñ-ˆÔÜŸ
š
¤4§9¢9¨T×-=Ñ-=Ó#>Ó?ˆŒØ×/Ñ/ˆŒ
Ø$×3Ñ3ˆÔô Ÿš 4×#3Ñ#3°T×5KÑ5KÐRX×RaÑRaÔbˆŒÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPV×P_ÑP_Ô`ˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRX×RaÑRaÔbˆŒô Ÿ9š9 T×%5Ñ%5°t·}±}ÀqÕ7HÈuÔUˆÔô Ÿ,š, t×'7Ñ'7¸×9IÑ9IÓJˆŒÜŸš T×%5Ñ%5°t×7GÑ7GÓHˆŽr@   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )rB   r/   r‚   )r„   r…   )rc   rd   s   "€r>   re   rù   [  s'   ø€ ÷ ,ñ ,á—|‘|ð,ñ 
�‰ñ	,r@   c                ó  € VP                   w  r4pV P                  V4      P                  ^W4,          R4      pVP                  V P                  RR7      w  w  rxw  ršVP                  R4      VP                  R4      ,           pV	P                  R4      V P                  ,          V
P                  R4      ,           pVP                  ! . VP                   RR ORN5!  pVP                  ! . VP                   RR ORN5!  pVP                  V P                  RR7      w  rÞVP                  RV4      p\        P                  ! VRR7      pV P                  '       d   VVP                  RRR7      ,          pV P                  V4      pV P                  V4      p\        P                  ! VVP                  W4,          R^4      4      P                  W4,          R4      pV P!                  V4      V,          p\        P                  ! VP                  W4,          ^R4      V4      P                  W4R4      pV P#                  V P!                  V P%                  V4      4      V P'                  V4      ,          4      pVV,           pW3# )rB   ©râ   NT)râ   ÚkeepdimrÆ   r˜   )rš   r  rÇ   rì   rÿ   Ú	unsqueezer   ÚgatherrË   ÚsoftmaxrW   Úsumr  r  r„   Úmatmulrû   r  r  r  )r;   r/   r<   ÚbszÚseq_lenÚ_Úrouter_logitsÚscores_xÚscores_yÚ	indices_xÚ	indices_yÚ
all_scoresÚall_indicesÚscoresÚposition_indicesrí   Úrouting_weightsr  r  Úexperts_weightsÚexperts_statess   &&,                  r>   rÜ   ÚDogeCDMoE.forward[  s.  € ð
 (×-Ñ-‰ˆ�að ×(Ñ(¨Ó7×<Ñ<¸QÀÅÈrÓRˆð 8E×7IÑ7IÈ$Ï-É-Ð]_Ð7IÓ7`Ñ4ÑˆÑ4˜yØ×'Ñ'¨Ó+¨h×.@Ñ.@ÀÓ.DÕDˆ
Ø×)Ñ)¨"Ó-°·±Õ=À	×@SÑ@SÐTVÓ@WÕWˆØ—_’_Ð@ j×&6Ñ&6°s¸Ð&;Ð@¸RÓ@ˆ
Ø!×&Ò&ÐC¨×(9Ñ(9¸#¸2Ð(>ÐCÀÓCˆØ#-§?¡?°4·:±:À2 ?Ó#FÑ ˆØ×$Ñ$ RÐ)9Ó:ˆÜŸ)š) F°Ô3ˆØ××ÐØ˜×2Ñ2°rÀ4Ð2ÓHÕHˆOð —_‘_ WÓ-ˆ
Ø—=‘= Ó)ˆÜŸ,š, z°=×3EÑ3EÀcÅmÐUWÐYZÓ3[Ó\×aÑaÐbeÕboÐqsÓtˆØŸ+™+ oÓ6¸ÕHˆÜŸš o×&:Ñ&:¸3½=È!ÈRÓ&PÐRZÓ[×`Ñ`ÐadÐoqÓrˆØŸ™ t§{¡{°4·>±>À-Ó3PÓ'QÐTX×T`ÑT`ÐanÓToÕ'oÓpˆØ%¨Õ6ˆØÐ+Ð+r@   )rû   r  r  r  rD   rE   rW   rU   rÿ   r  r   r  r  ©	rg   rh   ri   rj   r±   rÜ   rq   rr   rs   rt   s   @@r>   r÷   r÷   C  s    ù‡ € ÷Ió I÷.,÷ ,ð ,r@   r÷   c                   óX   a a€ ] tR tRt oRV3R lV 3R llltRV3R lR lltRtVtV ;t# )	ÚDogeDecoderLayeri|  c                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# r¦   r©   )rc   rd   s   "€r>   re   ÚDogeDecoderLayer.__annotate__}  s#   ø€ ÷ 
Tñ 
T™zð 
T±c¸Dµjñ 
Tr@   c                ó>  <€ \         SV `  4        VP                  V n        \        VP                  VP
                  R 7      V n        \        WR7      V n        \        P                  ! \        P                  ! VP                  4      4      V n        \        VP                  VP
                  R 7      V n        VP                  '       g   \!        V4      M
\#        V4      V n        \        P                  ! \        P                  ! VP                  4      4      V n        R# )r¯   )r§   r¨   N)r9   r±   rG   rv   rD   rJ   Úinput_layernormr¤   Ú	self_attnr   r¸   r„   ÚonesÚinput_residualÚpost_attention_layernormrT   rõ   r÷   ÚmlpÚpost_attention_residualr¿   s   &&&€r>   r±   ÚDogeDecoderLayer.__init__}  s¸   ø€ Ü‰ÑÔØ$×3Ñ3ˆÔä*¨6×+=Ñ+=À6×CVÑCVÔWˆÔÜ&¨fÔJˆŒÜ Ÿlšl¬5¯:ª:°f×6HÑ6HÓ+IÓJˆÔä(3°F×4FÑ4FÈF×L_ÑL_Ô(`ˆÔ%Ø*0¯-¯-¨-”7˜6”?¼YÀvÓ=NˆŒÜ')§|¢|´E·J²J¸v×?QÑ?QÓ4RÓ'SˆÖ$r@   c                ó�  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[,          R	S[S[ P                  S[S[ P                  S[ P                  3,          R,          3,          /# )
rB   r/   rÂ   Nr0   Úposition_idsr&   rK   r<   r‚   )	r„   r…   r†   Ú
LongTensorr   r`   r   r   ÚFloatTensor)rc   rd   s   "€r>   re   r'  ‰  sÄ   ø€ ÷  ñ  á—|‘|ð ñ #¡5§<¡<±·±Ð#=Õ>ÀÕEð ñ Ÿ™ tÕ+ð	 ñ
 ×&Ñ&¨Õ-ð ñ  �ð ñ ˜$•;ð ñ Ñ+Õ,ð ñ 
‰u× Ñ ¡%©×(9Ñ(9¹5×;LÑ;LÐ(LÕ"MÐPTÕ"TÐTÕ	Uñ r@   c                óÌ  € TpV P                  V4      pV P                  ! RR VRVRVRVRVRV/VB w  r\        P                  ! WP                  V P
                  R7      pV P                  V,          V,           pTpV P                  V4      pV P                  V4      p\        P                  ! WP                  V P
                  R7      pV P                  V,          V,           pV# )r/   rÂ   r0   r2  r&   rK   )ÚprÑ   r6   )
r)  r*  rË   rÅ   rG   rÑ   r,  r-  r.  r/  )
r;   r/   rÂ   r0   r2  r&   rK   r<   ÚresidualÚself_attn_weightss
   &&&&&&&,  r>   rÜ   ÚDogeDecoderLayer.forward‰  sý   € ð !ˆØ×,Ñ,¨]Ó;ˆØ+/¯>ª>ñ ,
Ø'ð,
à 3ð,
ð *ð,
ð &ð	,
ð
 ,ð,
ð  ð,
ð ñ,
Ñ(ˆô Ÿ	š	 -×3FÑ3FÐQU×Q^ÑQ^Ô_ˆØ×+Ñ+¨hÕ6¸ÕFˆð !ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆÜŸ	š	 -×3FÑ3FÐQU×Q^ÑQ^Ô_ˆØ×4Ñ4°xÕ?À-ÕOˆàÐr@   )rG   r)  r,  r.  r-  r/  r*  rŠ   )NNNNFr#  rt   s   @@r>   r%  r%  |  s    ù‡ € ÷
Tõ 
T÷ ÷  ò  r@   r%  c                   óv   a € ] tR tRt o RtRtR]! ]^R7      R]R]	/t
]P                  ! 4       R 4       tRtV tR	# )
ÚDogePreTrainedModeli¬  Fr  )Úindexr/   Ú
attentionsc                óÆ  € \         P                  ! W4       \        V\        4      '       d7   \	        VR4      '       d#   \
        P                  ! VP                  4       R# R# \        V\        4      '       di   \	        VR4      '       d!   \
        P                  ! VP                  4       \	        VR4      '       d#   \
        P                  ! VP                  4       R# R# R# )zInitialize the weightsrº   r,  r/  N)r   Ú_init_weightsr™   r¤   ÚhasattrÚinitÚzeros_rº   r%  Úones_r,  r/  )r;   r|   s   &&r>   r?  Ú!DogePreTrainedModel._init_weightsµ  s™   € ô 	×%Ò% dÔ3Ü�fœm×,Ò,Ü�v˜s×#Ò#Ü—’˜FŸH™HÖ%ñ $ä˜Ô 0×1Ò1Ü�vÐ/×0Ò0Ü—
’
˜6×0Ñ0Ô1Ü�vÐ8×9Ò9Ü—
’
˜6×9Ñ9Ö:ñ :ñ 2r@   r6   N)rg   rh   ri   rj   Ú_supports_flash_attnÚ_can_compile_fullgraphr   r÷   r%  r¤   Ú_can_record_outputsr„   Úno_gradr?  rq   rr   )rd   s   @r>   r;  r;  ¬  sJ   ø‡ € Ø ÐØ"Ðà™¨	¸Ô;ØÐ)Ø�mðÐð ‡]‚]ƒ_ñ
;ó ö
;r@   r;  c                   ó   € ] tR tRtRtR# )Ú	DogeModeliÃ  r6   Nrw   r6   r@   r>   rJ  rJ  Ã  rx   r@   rJ  c                ó*  € V ^8„  d   QhR\         P                  \        \         P                  ,          ,          R,          R\        R,          R\        R,          R\        R\         P                  R,          R\         P                  \        ,          /# )rB   Úgate_logitsNrU   rÿ   r   r0   r‚   )r„   r…   r†   r]   )rc   s   "r>   re   re   Ç  s}   € ÷ g&ñ g&Ü—‘¤¤e§l¡lÕ 3Õ3°dÕ:ðg&ä�t•ðg&ô �D�jðg&ô ð	g&ô
 —L‘L 4Õ'ðg&ô ‡\�\”CÕñg&r@   c                óN  € V e   \        V \        4      '       g   ^ # V ^ ,          P                  pV ^ ,          P                  p. p. pV  EF*  p	V	P	                  V4      p	V	P                  VRR7      w  w  r«w  rÍV
P                  R4      VP                  R4      ,           pVP                  R4      V,          VP                  R4      ,           pVP                  ! . VP                  RR ORN5!  pVP                  ! . VP                  RR ORN5!  pVP                  VRR7      w  ppVP                  RV4      p\        P                  ! VRR7      pVP                  V4       VP                  V4       EK-  	  \        P                  ! V^ R7      p\        P                  ! V^ R7      pVf‡   VP                  R4      p\        P                  ! WVR7      p\        P                   ! WuVR7      pVP#                  ^ VV4      VP                  ^ ,          ,          p\        P$                  ! V^ R7      pEMOVP                  w  pp\'        V 4      pVR,          P)                  VVVV34      P+                  R4      P	                  V4      pVP                  R4      VP-                  4       ,          p\        P                  ! WVR7      p\        P                   ! WuVR7      pVP#                  ^ VV4      \        P.                  ! V4      ,          pVR,          P)                  VVVV34      P+                  RV4      P	                  V4      p\        P.                  ! VV,          ^ R7      \        P.                  ! V^ R7      ,          p\        P.                  ! VV,          4      pVV,          # )a¤  
Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch.

See Switch Transformer (https://huggingface.co/papers/2101.03961) for more details. This function implements the loss
function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between
experts is too unbalanced.

Args:
    gate_logits:
        Logits from the `router_gate`, should be a tuple of model.config.num_hidden_layers tensors of
        shape [2, batch_size * sequence_length, num_keys].
    num_experts:
        Number of experts
    num_keys:
        Number of keys
    top_k:
        The number of experts to route per-token, can be also interpreted as the `top-k` routing
        parameter.
    attention_mask (`torch.Tensor`, *optional*):
        The attention_mask used in forward function
        shape [batch_size X sequence_length] if not None.

Returns:
    The auxiliary loss.
Nr  rá   rÆ   r˜   )Nrˆ   rˆ   N)r™   r†   rœ   rà   r›   rì   r  rÇ   rš   r  rË   r  Úappendr„   Úcatr¹   Ú	ones_likeÚscatter_add_ÚmeanÚlenrç   rÉ   r`   r  )rL  rU   rÿ   r   r0   Úcompute_dtypeÚcompute_deviceÚall_expert_indicesÚall_routing_weightsÚlayer_gate_logitsr  r  r  r  r  r  r  r  Úexpert_indicesr  Útokens_per_expertÚpadÚrouter_prob_per_expertÚ
batch_sizeÚsequence_lengthrF   Úexpert_attention_maskÚ router_per_expert_attention_maskÚoverall_losss   &&&&&                        r>   Úload_balancing_loss_funcrb  Ç  sY  € ð@ Ò¤*¨[¼%×"@Ò"@Ùà •N×(Ñ(€MØ  •^×*Ñ*€NØÐØÐä(ÐØ-×0Ñ0°Ó@Ðà7H×7MÑ7MÈhÐ\^Ð7MÓ7_Ñ4ÑˆÑ4˜yà×'Ñ'¨Ó+¨h×.@Ñ.@ÀÓ.DÕDˆ
Ø×)Ñ)¨"Ó-°Õ8¸9×;NÑ;NÈrÓ;RÕRˆØ—_’_Ð@ j×&6Ñ&6°s¸Ð&;Ð@¸RÓ@ˆ
Ø!×&Ò&ÐC¨×(9Ñ(9¸#¸2Ð(>ÐCÀÓCˆà(Ÿo™o¨e¸˜oÓ<ÑˆÐØ$×+Ñ+¨BÐ0@ÓAˆäŸ)š) J°BÔ7ˆà×!Ñ! .Ô1Ø×"Ñ" ?×3ñ! )ô" ŸšÐ#5¸1Ô=ÐÜŸ)š)Ð$7¸QÔ?ÐàÒà/×4Ñ4°RÓ8ÐÜ!ŸKšK¨ÐQ_Ô`ÐÜ�oŠoÐ0ÈnÔ]ˆØ-×:Ñ:¸1Ð>PÐRUÓVÐYk×YqÑYqÐrsÕYtÕtÐô "'§¢Ð,?ÀQÔ!GÒà&4×&:Ñ&:Ñ#ˆ
�OÜ Ó,Ðð Ð+Õ,ß‰VÐ&¨
°OÀUÐKÓLß‰W�R‹[ß‰R�Óð	 	ð 0×4Ñ4°RÓ8Ð9N×9SÑ9SÓ9UÕVÐô "ŸKšK¨ÐQ_Ô`ÐÜ�oŠoÐ0ÈnÔ]ˆØ-×:Ñ:¸1Ð>PÐRUÓVÔY^×YbÒYbØ!óZ
õ 
Ðð Ð+Õ,ß‰VÐ&¨
°OÀ[ÐQÓRß‰W�R˜Ó%ß‰R�Óð	 	)ô "'§¢Ð+>ÐAaÕ+aÐghÔ!iÔlq×luÒluØ,°!ôm
õ "
Ðô —9’9Ð.Ð1GÕGÓH€LØ˜+Õ%Ð%r@   c                   óH   a a€ ] tR tRt oV 3R ltRV3R lR lltRtVtV ;t# )ÚDogeForCausalLMi1  c                óh   <€ \         SV `  V4       \        V4      V n        VP                  V n        R # rŠ   )r9   r±   rJ  ÚmodelrU   r  s   &&€r>   r±   ÚDogeForCausalLM.__init__2  s*   ø€ Ü‰Ñ˜Ô Ü˜vÓ&ˆŒ
Ø!×-Ñ-ˆÖr@   c                ód  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[S[ P                  ,          R
S[R,          RS[S[,          RS[	/# )rB   r,   Nr0   r2  r&   r-   ÚlabelsrK   Úlogits_to_keeprX   r<   r‚   )
r„   r3  r…   r   r4  r`   r]   r   r   r   )rc   rd   s   "€r>   re   ÚDogeForCausalLM.__annotate__7  sÖ   ø€ ÷ O
ñ O
á×#Ñ# dÕ*ðO
ñ Ÿ™ tÕ+ðO
ñ ×&Ñ&¨Õ-ð	O
ñ
  �ðO
ñ ×(Ñ(¨4Õ/ðO
ñ × Ñ  4Õ'ðO
ñ ˜$•;ðO
ñ ™eŸl™lÕ*ðO
ñ # T�kðO
ñ Ñ+Õ,ðO
ñ 
#ñO
r@   c
                ó  € V	e   T	MV P                   P                  p	V P                  ! R
RVRVRVRVRVRV/V
B pVP                  p\	        V\
        4      '       d   \        V) R4      MTpV P                  VRVR3,          4      pRpVe   V P                  ! WæV P                  3/ V
B pRpV	'       d–   \        VP                  V P                  \        P                  ! \        P                  ! V P                  4      4      V P                   V4      pVe3   WðP"                  VP%                  VP&                  4      ,          ,          p\)        VVVVP*                  VP,                  VP.                  VP                  R	7      # )aè  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

Example:

```python
>>> from transformers import AutoTokenizer, DogeForCausalLM

>>> model = DogeForCausalLM.from_pretrained("SmallDoge/Doge-320M")
>>> tokenizer = AutoTokenizer.from_pretrained("SmallDoge/Doge-320M")

>>> prompt = "Hey, are you conscious? Can you talk to me?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
```Nr,   r0   r2  r&   r-   rK   rˆ   )ÚlossÚaux_lossÚlogitsr&   r/   r=  r  r6   )r§   rX   rf  Úlast_hidden_stater™   r]   ÚsliceÚlm_headÚloss_functionrC   rb  r  rU   rü   rý   rþ   rV   rY   r›   rà   r   r&   r/   r=  )r;   r,   r0   r2  r&   r-   ri  rK   rj  rX   r<   Úoutputsr/   Úslice_indicesro  rm  rn  s   &&&&&&&&&&,      r>   rÜ   ÚDogeForCausalLM.forward7  s‰  € ðH %9Ò$DÑ È$Ï+É+×JjÑJjð 	ð
 +/¯*ª*ñ +
Øð+
à)ð+
ð &ð+
ð ,ð	+
ð
 (ð+
ð  ð+
ð ñ+
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒØ×%Ò% f°d·o±oÑPÈÑPˆDàˆßÜ/Ø×%Ñ%Ø× Ñ Ü—
’
œ4Ÿ9š9 T×%5Ñ%5Ó6Ó7Ø×(Ñ(ØóˆHð Ò!Ø×1Ñ1°H·K±KÀÇÁÓ4LÕLÕL�ä(ØØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø!×/Ñ/ô
ð 	
r@   )rf  rU   )	NNNNNNNé    Nr#  rt   s   @@r>   rd  rd  1  s   ù‡ € õ.÷
O
÷ O
ò O
r@   rd  c                   ó   € ] tR tRtRtR# )ÚDogeForSequenceClassificationi‰  r6   Nrw   r6   r@   r>   ry  ry  ‰  rx   r@   ry  ró   )NNrB   N)r$   rd  rJ  r;  ry  )Lrk   rü   Úcollections.abcr   Útypingr   r„   Útorch.nn.functionalr   Ú
functionalrË   Úhuggingface_hub.dataclassesr   Ú r   rA  Úactivationsr   Úcache_utilsr   Úconfiguration_utilsr	   Úintegrations.flex_attentionr
   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.output_capturingr   Úllama.modeling_llamar   r   r   r   r   r   r   r   Úmixtral.modeling_mixtralr   r    Ú
get_loggerrg   ÚloggerÚ!torch.nn.attention.flex_attentionr!   r$   rv   rz   r¡   rÎ   rƒ   r¤   rõ   r÷   r%  r;  rJ  rb  rd  ry  Ú__all__r6   r@   r>   Ú<module>r‘     sb  ðñ  ã Ý $Ý ã ß Ð Ý .Ý å &Ý !Ý  Ý 3Ý JÝ 9ß QÝ 1ß AÝ &ß ^Ó ^Ý 4÷	÷ 	ó 	÷ Hð 
×	Ò	˜HÓ	%€á×!Ò!Ý;ñ Ð0Ô1ØôL(Ð!ó L(ó ó 2ðL(ô^	�,ô 	ô	Ð.ô 	÷+*ñ\ -Ó.Ð Ø1GÐ Ð-Ñ .ôw�B—I‘Iô wôt	ˆhô 	ô6,�—	‘	ô 6,ôr-Ð1ô -ô`;Ð.ô ;ô.	�ô 	÷g&ôTU
Ð(ô U
ôp	Ð$Bô 	ò‚r@   