+
    œ°³j$F  ã                   óÚ  € R t ^ RIHt ^ RIt^ RIHu Ht ^ RIH	t	 ^ RIHt ^RI
Ht ^RIHt ^RIHt ^RIHt ^R	IHtHt ^R
IHtHt ^RIHt ^RIHtHt ^RIHt ^RI H!t!H"t"H#t# ^RI$H%t% ^RI&H't'H(t(H)t)H*t*H+t+H,t,H-t-H.t.H/t/H0t0H1t1 ]Pd                  ! ]34      t4]! RR7      ]	 ! R R]%4      4       4       t5 ! R R]!4      t6 ! R R]4      t7 ! R R]Pp                  4      t9 ! R R]/4      t: ! R R]+4      t; ! R R ]04      t< ! R! R"])4      t= ! R# R$]-4      t> ! R% R&]'4      t? ! R' R(](4      t@ ! R) R*].4      tA ! R+ R,],4      tB ! R- R.]*4      tC ! R/ R0]]A4      tD ! R1 R2]]A4      tE. R3OtFR# )4zPyTorch A.X-K2 model.)ÚCallableN)Ústrict)Únn)Úinitialization)ÚCache)ÚPreTrainedConfig)ÚFlashAttentionKwargs)Ú GenericForSequenceClassificationÚGenericForTokenClassification)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)Úauto_docstringÚlogging)ÚCLIPMLP)ÚDeepseekV3RMSNormÚapply_rotary_pos_emb_interleaveÚeager_attention_forward)ÚDeepseekV32Config)ÚDeepseekV32AttentionÚDeepseekV32DecoderLayerÚDeepseekV32ExpertsÚDeepseekV32ForCausalLMÚDeepseekV32IndexerÚDeepseekV32ModelÚDeepseekV32MoEÚDeepseekV32PreTrainedModelÚDeepseekV32RotaryEmbeddingÚDeepseekV32TopkRouterÚyarn_apply_mscalez
skt/A.X-K2)Ú
checkpointc                   óü   a € ] tR t^8t o RtRtRtRtRtRt	^0t
^ t^ t^t^€tRt^€tRt^ t^@t^@t^tR	tR
tRtRtRt^€t^t^tRtRt] ! 4       t!] ! 4       t"RRRRRRRRRRRRRRRRRRRRRRRRRR/t#R t$R  t%V 3R! lt&R"t'V t(R# )#Ú
AXK2Configa}  
n_group (`int`, *optional*):
    Number of expert groups for grouped routing, used by the larger A.X-K2 releases. `None` (the
    A.X-K2-Light default) routes over all experts without group restriction.
topk_group (`int`, *optional*):
    Number of expert groups the top-k selection is restricted to when `n_group` is set.
mlp_layer_types (`list`, *optional*):
    MLP type pattern for each layer (`"dense"` or `"sparse"`). Derived from the (legacy) kwargs
    `first_k_dense_replace` / `moe_layer_freq` when not provided.
index_topk (`int`, *optional*, defaults to 2048):
    Number of top tokens selected by the indexer for sparse attention.
index_head_dim (`int`, *optional*, defaults to 128):
    Head dimension for the indexer projections (DSA).
index_n_heads (`int`, *optional*, defaults to 16):
    Number of heads for the indexer projections (DSA).
gated_norm_rank (`int`, *optional*, defaults to 16):
    Bottleneck rank for the low-rank input-dependent gate used by `AXK2GatedRMSNorm`. The gate wraps
    `input_layernorm` on every layer and `post_attention_layernorm` on MoE layers.

```python
>>> from transformers import AXK2Config, AXK2Model

>>> # Initializing an A.X-K2 style configuration
>>> configuration = AXK2Config()

>>> # Initializing a model from the configuration
>>> model = AXK2Model(configuration)

>>> # Accessing the model configuration
>>> configuration = model.config
```Úaxk2i € i   i   i   g      @i€  i   g�íµ ÷Æ°>ik Nzlayers.*.self_attn.q_gate_projÚcolwisez%layers.*.self_attn.kv_a_proj_with_mqaÚmla_kv_a_projzlayers.*.self_attn.kv_b_projzlayers.*.self_attn.o_projÚrowwisez!layers.*.mlp.experts.gate_up_projÚpacked_colwisezlayers.*.mlp.experts.down_projzlayers.*.mlp.expertsÚmoe_tp_expertsz%layers.*.mlp.shared_experts.gate_projz#layers.*.mlp.shared_experts.up_projz%layers.*.mlp.shared_experts.down_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projc                óÔ  € V P                   V P                  ,           V n        V P                  V n        V P                  fd   VP                  R^4      pVP                  R^4      p\        V P                  4       Uu. uF  pWB8¼  d   WC,          ^ 8X  d   RMRNK  	  upV n        V P                  f   R.V P                  ,          V n        \        P                  ! V 3/ VB  R # u upi )NÚfirst_k_dense_replaceÚmoe_layer_freqÚsparseÚdenseÚdeepseek_sparse_attention)Úqk_nope_head_dimÚqk_rope_head_dimÚqk_head_dimÚhead_dimÚmlp_layer_typesÚpopÚrangeÚnum_hidden_layersÚlayer_typesr   Ú__post_init__)ÚselfÚkwargsr*   r+   Úis   &,   Úf/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/axk2/modular_axk2.pyr8   ÚAXK2Config.__post_init__�   sÚ   € Ø×0Ñ0°4×3HÑ3HÕHˆÔð ×-Ñ-ˆŒð ×ÑÒ'Ø$*§J¡JÐ/FÈÓ$JÐ!Ø#ŸZ™ZÐ(8¸!Ó<ˆNô ˜t×5Ñ5Ô6ó$á6�Að Ô6¸1Õ;MÐQRÔ;R‘ÐX_Ò_Ù6ñ$ˆDÔ ð ×ÑÒ#Ø ;Ð<¸t×?UÑ?UÕUˆDÔä×&Ò& tÑ6¨vÔ6ùò$s   Á= C%c                ó^  € \         P                  ! V 4       V P                  e   V P                  ^ 8:  d   \        RV P                   R24      hV P                  R J V P
                  R J 8w  d'   \        RV P                   RV P
                   R24      hV P                  eŒ   V P                  V P                  ,          ^ 8w  d'   \        RV P                   RV P                   R24      hV P
                  V P                  8”  d'   \        RV P
                   R	V P                   R24      hR # R # )
NzkA.X-K2 requires a positive `q_lora_rank` (the indexer and output gate read the query LoRA bottleneck), got Ú.zc`n_group` and `topk_group` must be set together (both `None` for non-grouped routing), got n_group=z, topk_group=z`n_routed_experts` (z") must be divisible by `n_group` (z).z`topk_group` (z) cannot exceed `n_group` ()r   Úvalidate_architectureÚq_lora_rankÚ
ValueErrorÚn_groupÚ
topk_groupÚn_routed_experts)r9   s   &r<   r@   Ú AXK2Config.validate_architecture¢   s0  € Ü×.Ò.¨tÔ4Ø×ÑÒ# t×'7Ñ'7¸1Ô'<Üð$Ø$(×$4Ñ$4Ð#5°Qð8óð ð �L‰L˜DÐ  d§o¡o¸Ð&=Ô>ÜðØŸ<™<˜.¨°d·o±oÐ5FÀaðIóð ð �<‰<Ò#Ø×$Ñ$ t§|¡|Õ3°qÔ8Ü Ø*¨4×+@Ñ+@Ð*AÐAcÐdh×dpÑdpÐcqÐqsÐtóð ð �‰ §¡Ô-Ü  >°$·/±/Ð1BÐB]Ð^b×^jÑ^jÐ]kÐkmÐ!nÓoÐoñ .ñ $ó    c                óª  <€ V ^8„  d   Qh/ S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R	&   S[;R
&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[;R&   S[ R,          ;R&   S[ S[S[ ,          ,          R,          ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ R,          ;R&   S[ R,          ;R&   # )é   Ú
vocab_sizeÚhidden_sizeÚintermediate_sizeÚmoe_intermediate_sizer6   Únum_attention_headsÚnum_key_value_headsÚn_shared_expertsrE   Úrouted_scaling_factorÚkv_lora_rankrA   r0   Ú
v_head_dimr/   Únum_experts_per_tokÚmax_position_embeddingsÚrms_norm_epsNÚbos_token_idÚeos_token_idÚ
index_topkÚindex_head_dimÚindex_n_headsÚgated_norm_rankrC   rD   )ÚintÚfloatÚlist)ÚformatÚ__classdict__s   "€r<   Ú__annotate__ÚAXK2Config.__annotate__8   s¤  ø‡ ‚ ñJ ÑñK ñL ÑñM ñN Ñ!ñO ñP Ñ$ñQ ñR ÑñS ñT Ñ!ñU ñV Ñ!ñW ñX ÑñY ñZ Ññ[ ñ\ !Ñ&ñ] ñ^ Ññ_ ñ` Ñña ñb Ññc ñd Ññe ñf Ññg ñh Ñ ñi ñj !Ñ)ñk ñl Ññm ñn ˜•*Ñ%ño ñp ™™S�	•/ DÕ(Ñ1ñq ñr Ññs ñt Ññu ñv Ññw ñx Ññy ñ~ �4�ZÑñ ñ@ �d•
Ñ!òA rG   )r2   r7   r3   r1   ))Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Ú
model_typerJ   rK   rL   rM   r6   rN   rO   rP   rE   rQ   rR   rA   r0   rS   r/   rT   rU   rV   rW   rX   rY   rZ   r[   r\   rC   rD   ÚAttributeErrorr*   Úmlp_biasÚbase_model_tp_planr8   r@   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__©ra   s   @r<   r"   r"   8   s  ø‡ € ñð@ €Jà€JØ€KØ!ÐØ!$ÐØÐØ!ÐØ!ÐØÐØÐØ#&ÐØ€LØ€KØÐØ€JØÐØ ÐØ#)ÐØ€LØ%€LØ+1€LØ€JØ€NØ€MØ€Oð €GØ!€Já*Ó,ÐÙÓ€Hð 	)¨)Ø/°Ø&¨	Ø# YØ+Ð-=Ø(¨)ØÐ 0Ø/°Ø-¨yØ/°Ø  )Ø 	Ø  )ðÐò 7ò*p÷U ƒ rG   r"   c                   ó   € ] tR t^·tRtR# )ÚAXK2RMSNorm© N©rd   re   rf   rg   rn   rs   rG   r<   rr   rr   ·   ó   † ÛrG   rr   c                   ó>   a a€ ] tR t^»t oV3R lV 3R lltRtVtV ;t# )ÚAXK2GateMLPc                ó    <€ V ^8„  d   QhRS[ /# ©rI   Úconfig©r"   )r`   ra   s   "€r<   rb   ÚAXK2GateMLP.__annotate__¼   s   ø€ ÷ Uñ U™zñ UrG   c                ó"  <€ \         SV `  V4       \        P                  ! 4       V n        \        P
                  ! VP                  VP                  R R7      V n        \        P
                  ! VP                  VP                  R R7      V n	        R# )F©ÚbiasN)
ÚsuperÚ__init__r   ÚSiLUÚactivation_fnÚLinearrK   r\   Úfc1Úfc2©r9   rz   Ú	__class__s   &&€r<   r�   ÚAXK2GateMLP.__init__¼   s^   ø€ Ü‰Ñ˜Ô ÜŸWšW›YˆÔÜ—9’9˜V×/Ñ/°×1GÑ1GÈeÔTˆŒÜ—9’9˜V×3Ñ3°V×5GÑ5GÈeÔTˆŽrG   )rƒ   r…   r†   ©rd   re   rf   rg   r�   rn   ro   Ú__classcell__©rˆ   ra   s   @@r<   rw   rw   »   s   ù‡ € ÷U÷ Uó UrG   rw   c                   óT   a a€ ] tR t^Ãt oRtV3R lV 3R lltV3R lR ltRtVtV ;t	# )ÚAXK2GatedRMSNormz‹RMSNorm followed by a low-rank input-dependent sigmoid gate (Megatron `GatedNormWrapper`):

y = RMSNorm(x)
return y * sigmoid(gate_mlp(y))
c                ó    <€ V ^8„  d   QhRS[ /# ry   r{   )r`   ra   s   "€r<   rb   ÚAXK2GatedRMSNorm.__annotate__Ê   s   ø€ ÷ 'ñ '™zñ 'rG   c                ó�   <€ \         SV `  4        \        VP                  VP                  R 7      V n        \        V4      V n        R# )©ÚepsN)r€   r�   rr   rK   rV   Únormrw   Úmlpr‡   s   &&€r<   r�   ÚAXK2GatedRMSNorm.__init__Ê   s4   ø€ Ü‰ÑÔÜ × 2Ñ 2¸×8KÑ8KÔLˆŒ	Ü˜vÓ&ˆŽrG   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )rI   ÚxÚreturn)ÚtorchÚTensor)r`   ra   s   "€r<   rb   r�   Ï   s'   ø€ ÷ Dñ D™Ÿ™ð D©%¯,©,ñ DrG   c                óÊ   € V P                  V4      pV\        P                  ! V P                  V4      P	                  4       4      ,          P                  VP                  4      # )N)r”   rš   Úsigmoidr•   r^   ÚtoÚdtype)r9   r˜   Úys   && r<   ÚforwardÚAXK2GatedRMSNorm.forwardÏ   sC   € Ø�I‰I�a‹LˆØ”E—M’M $§(¡(¨1£+×"3Ñ"3Ó"5Ó6Õ6×:Ñ:¸1¿7¹7ÓCÐCrG   )r•   r”   )
rd   re   rf   rg   rh   r�   r¡   rn   ro   r‹   rŒ   s   @@r<   rŽ   rŽ   Ã   s&   ù‡ € ñ÷'ó '÷
D÷ Dð DrG   rŽ   c                   ó   € ] tR t^ÔtRtR# )ÚAXK2RotaryEmbeddingrs   Nrt   rs   rG   r<   r¤   r¤   Ô   ru   rG   r¤   c                   ó   € ] tR t^ØtRtR# )ÚAXK2Indexerrs   Nrt   rs   rG   r<   r¦   r¦   Ø   ru   rG   r¦   c                   ó,   a € ] tR t^Üt o R tR tRtV tR# )ÚAXK2TopkRouterc                óp  € VP                  RV P                  V P                  V P                  ,          4      P                  ^RR7      ^ ,          P	                  RR7      p\
        P                  ! W P                  RRR7      ^,          p\
        P                  ! V4      pVP                  ^V^4       VP                  R4      P                  RV P                  V P                  V P                  ,          4      P                  RV P                  4      pVP                  VP                  4       ( R4      pV# )z$Apply DeepSeek style grouped scoring©ÚdimF©Úkr«   Úsortedç        éÿÿÿÿ)ÚviewÚ	num_groupÚnum_expertsÚtopkÚsumrš   rD   Ú
zeros_likeÚscatter_Ú	unsqueezeÚexpandÚreshapeÚmasked_fillÚbool)r9   Úscores_for_choiceÚgroup_scoresÚ	group_idxÚ
group_maskÚ
score_masks   &&    r<   Úapply_group_scoringÚ"AXK2TopkRouter.apply_group_scoringÝ   sø   € ð ×"Ñ" 2 t§~¡~°t×7GÑ7GÈ4Ï>É>Õ7YÓZß‰T�!˜ˆT‹_˜Qõ ç‰S�RˆS‹[ð 	ô
 —J’J˜|¯©ÀBÈuÔUÐVWÕXˆ	Ü×%Ò% lÓ3ˆ
Ø×Ñ˜A˜y¨!Ô,à× Ñ  Ó$ß‰V�B˜Ÿ™¨×(8Ñ(8¸D¿N¹NÕ(JÓKß‰W�R˜×)Ñ)Ó*ð 	ð .×9Ñ9¸:¿?¹?Ó;LÐ:LÈcÓRÐØ Ð rG   c                ó~  € VP                  RV P                  4      p\        P                  ! VP	                  \
        P                  4      V P                  P	                  \
        P                  4      4      pVP                  4       pW0P                  ,           pV P                  e   V P                  V4      p\
        P                  ! W@P                  RRR7      ^,          pVP                  ^V4      pV P                  '       d#   VP!                  RRR7      R,           pWg,          pW`P"                  ,          pW&V3# )é   Fr¬   T)r«   Úkeepdimg#B’¡œÇ;r°   )r±   Ú
hidden_dimÚFÚlinearÚtyperš   Úfloat32Úweightr�   Úe_score_correction_biasr²   rÂ   r´   Útop_kÚgatherÚnorm_topk_probrµ   rQ   )r9   Úhidden_statesÚrouter_logitsÚscoresr½   Útopk_indicesÚtopk_weightsÚdenominators   &&      r<   r¡   ÚAXK2TopkRouter.forwardð   sý   € Ø%×*Ñ*¨2¨t¯©Ó?ˆÜŸš ×!3Ñ!3´E·M±MÓ!BÀDÇKÁK×DTÑDTÔUZ×UbÑUbÓDcÓdˆØ×&Ñ&Ó(ˆØ"×%AÑ%AÕAÐð �>‰>Ò%Ø $× 8Ñ 8Ð9JÓ KÐä—z’zÐ"3·z±zÀrÐRWÔXÐYZÕ[ˆØ—}‘} Q¨Ó5ˆØ××ÐØ&×*Ñ*¨r¸4Ð*Ó@À5ÕHˆKØÕ'ˆLØ#×&@Ñ&@Õ@ˆØ¨LÐ8Ð8rG   rs   N)rd   re   rf   rg   rÂ   r¡   rn   ro   rp   s   @r<   r¨   r¨   Ü   s   ø‡ € ò!÷&9ð 9rG   r¨   c                   ó   € ] tR tRtRtR# )ÚAXK2Expertsi  rs   Nrt   rs   rG   r<   rÙ   rÙ     ru   rG   rÙ   c                   ó   € ] tR tRtRtR# )ÚAXK2MoEi  rs   Nrt   rs   rG   r<   rÛ   rÛ     ru   rG   rÛ   c                   óH   a € ] tR tRt o V 3R lR ltRV 3R lR lltRtV tR# )	ÚAXK2Attentioni  c                ó&   <€ V ^8„  d   QhRS[ RS[/# ©rI   rz   Ú	layer_idx©r"   r]   )r`   ra   s   "€r<   rb   ÚAXK2Attention.__annotate__  s   ø€ ÷ ,6ñ ,6™zð ,6±cñ ,6rG   c                óì  € \         P                  P                  V 4       Wn        W n        VP
                  V n        VP                  V n        VP                  V n        VP                  V n	        VP                  V n
        VP                  V n        VP                  V n        VP                  V n        VP                  VP                  ,           V n        VP                  VP                  ,          V n        R V n        \         P$                  ! V P                  V P                  VP&                  R7      V n        \+        V P                  4      V n        \         P$                  ! ^V P                  ,          V P                  V P                  V P                  ,           ,          RR7      V n        \         P$                  ! V P                  V P                  V P                  ,           VP&                  R7      V n        \+        V P                  4      V n        \         P$                  ! V P                  V P                  V P                  V P                  ,           ,          RR7      V n        \         P$                  ! V P                  V P                  ,          V P                  VP&                  R7      V n        \9        VP:                  V P                  R,          4      V n        \?        W4      V n         R# )Tr~   FNg      à¿)!r   ÚModuler�   rz   rà   Úattention_dropoutrK   rN   Ú	num_headsrA   r0   rR   rS   r/   r1   rO   Únum_key_value_groupsÚ	is_causalr„   Úattention_biasÚq_a_projrr   Úq_a_layernormÚq_gate_projÚkv_a_proj_with_mqaÚkv_a_layernormÚ	kv_b_projÚo_projr   Úrope_parametersÚscalingr¦   Úindexer)r9   rz   rà   s   &&&r<   r�   ÚAXK2Attention.__init__  s!  € Ü
�	‰	×Ñ˜4Ô ØŒØ"ŒØ!'×!9Ñ!9ˆÔØ!×-Ñ-ˆÔØ×3Ñ3ˆŒà!×-Ñ-ˆÔØ &× 7Ñ 7ˆÔØ"×/Ñ/ˆÔØ ×+Ñ+ˆŒØ &× 7Ñ 7ˆÔØ!×2Ñ2°V×5LÑ5LÕLˆÔØ$*×$>Ñ$>À&×B\ÑB\Õ$\ˆÔ!àˆŒäŸ	š	 $×"2Ñ"2°D×4DÑ4DÈ6×K`ÑK`ÔaˆŒÜ(¨×)9Ñ)9Ó:ˆÔäŸ9š9Ø�× Ñ Õ  $§.¡.°D×4DÑ4DÀtÇÁÕ4VÕ"WÐ^cô
ˆÔô #%§)¢)Ø×ÑØ×Ñ × 5Ñ 5Õ5Ø×&Ñ&ô#
ˆÔô
 *¨$×*;Ñ*;Ó<ˆÔÜŸšØ×ÑØ�N‰N˜d×3Ñ3°d·o±oÕEÕFØô
ˆŒô —i’iØ�N‰N˜TŸ_™_Õ,Ø×ÑØ×&Ñ&ô
ˆŒô )¨×)?Ñ)?À×AQÑAQÐVZÕA[Ó\ˆŒÜ" 6Ó5ˆŽrG   Nc                ó6  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          RS[ P                  RS[R,          RS[ P                  R,          RS[S[,          RS[S[ P                  S[ P                  R,          3,          /# )	rI   rÑ   Úposition_embeddingsÚattention_maskÚpast_key_valuesNÚposition_idsr:   r™   )rš   r›   Útupler   r   r   )r`   ra   s   "€r<   rb   râ   :  sž   ø€ ÷ T)ñ T)á—|‘|ðT)ñ #¡5§<¡<±·±Ð#=Õ>ðT)ñ Ÿ™ð	T)ñ
  �ðT)ñ —l‘l TÕ)ðT)ñ Ñ-Õ.ðT)ñ 
‰u�|‰|™UŸ\™\¨DÕ0Ð0Õ	1ñT)rG   c           	     óÞ  € VP                   R R	 w  rxWxR	V P                  V P                  ,           3p	V P                  V4      p
V P	                  V
4      pV P                  \        P                  ! Wº.R	R7      4      P                  V	4      p\        P                  ! WÀP                  V P                  .R	R7      w  rÞVP                  ^^4      pVP                  WxR	4      p\        P                  ! WÐP                  V P                  .R	R7      w  ppV P                  V4      p\        P                  ! VV P                  V P                  .R	R7      w  ppV P!                  V4      P                  V^W€P                  4      pVP                  V^W€P                  4      pVw  pp\#        VVVV4      w  pp\        P                  ! VV3R	R7      pV P%                  VV4      w  ppVe!   VP'                  VVV P(                  4      w  ppV P+                  VVVVR
,          VVR7      pR pV P,                  P.                  R9   dÈ   VP1                  WxVP                   ^,          3\        P2                  R7      P5                  R	VP7                  4       R4      P9                  ^4      pVP:                  \        P2                  8X  d   VV( ,          pM=VP=                  V\        P>                  ! VP:                  4      P@                  4      pMTp\B        PD                  ! V P,                  P.                  \F        4      pT! V VVVV3RV PH                  '       g   RMV PJ                  RV PL                  RV/VB w  ppVP                  WxR	4      PO                  4       pV\        PP                  ! VPS                  4       4      ,          PU                  VP:                  4      pV PW                  V4      pVV3# )Nrª   )rø   )rŸ   FÚdropoutr¯   rò   Úindicesr°   )ºNNNé    rþ   rþ   )ÚeagerÚsdpa),Úshaper1   rS   rê   rë   rì   rš   Úcatr±   ÚsplitÚ	transposerº   r/   r0   rí   rR   rî   r   Ú	expand_kvÚupdaterà   ró   rz   Ú_attn_implementationÚnew_onesr¼   ÚscatterÚlongr¸   rŸ   r»   ÚfinfoÚminr   Úget_interfacer   Útrainingrå   rò   Ú
contiguousr�   r^   rž   rð   )r9   rÑ   rö   r÷   rø   rù   r:   Ú
batch_sizeÚ
seq_lengthÚquery_gate_shapeÚq_compressedÚq_residÚquery_gate_statesÚquery_statesÚgate_statesÚq_passÚq_rotÚcompressed_kvÚkv_passÚk_rotÚk_passÚcosÚsinÚ
key_statesÚvalue_statesrÔ   Úsparse_indicesÚ
index_maskÚattention_interfaceÚattn_outputÚattn_weightss   &&&&&&,                        r<   r¡   ÚAXK2Attention.forward:  s˜  € ð "/×!4Ñ!4°S°bÐ!9Ñˆ
Ø&°B¸×8HÑ8HÈ4Ï?É?Õ8ZÐ[Ðð —}‘} ]Ó3ˆØ×$Ñ$ \Ó2ˆà ×,Ñ,¬U¯YªY¸Ð7NÐTVÔ-WÓX×]Ñ]Ð^nÓoÐÜ$)§K¢KÐ0A×DTÑDTÐVZ×VeÑVeÐCfÐlnÔ$oÑ!ˆØ#×-Ñ-¨a°Ó3ˆØ!×)Ñ)¨*À"ÓEˆäŸš L×3HÑ3HÈ$×J_ÑJ_Ð2`ÐfhÔi‰ˆ�à×/Ñ/°Ó>ˆÜŸš ]°T×5FÑ5FÈ×H]ÑH]Ð4^ÐdfÔg‰ˆ�à×$Ñ$ WÓ-×2Ñ2°:¸qÀ*×N_ÑN_Ó`ˆØ—
‘
˜: q¨*×6KÑ6KÓLˆà&‰ˆˆSÜ6°u¸eÀSÈ#ÓN‰ˆˆuä—y’y &¨% °bÔ9ˆà#'§>¡>°&¸%Ó#@Ñ ˆ
�Lð Ò&Ø'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜ð —|‘|ØØØØ˜:Õ&ØØ+ð $ó 
ˆð ˆØ�;‰;×+Ñ+Ð/@Ô@à×%Ñ% z¸z×?OÑ?OÐPQÕ?RÐ&SÔ[`×[eÑ[eÐ%Ófß‘˜˜\×.Ñ.Ó0°%Ó8ß‘˜1“ð ð ×#Ñ#¤u§z¡zÔ1Ø!/°:°+Õ!=‘à!/×!;Ñ!;¸JÌÏÊÐTa×TgÑTgÓHh×HlÑHlÓ!m‘à)ˆNä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ
%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð
%
ð —L‘Lð
%
ð #ð
%
ð ñ
%
Ñ!ˆ�\ð "×)Ñ)¨*À"ÓE×PÑPÓRˆà"¤U§]¢]°;×3DÑ3DÓ3FÓ%GÕG×KÑKÈK×L]ÑL]Ó^ˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(rG   )rå   rz   rK   ró   rè   rî   rí   rï   rR   rà   ræ   rç   rð   rë   rê   rì   rA   r1   r/   r0   rò   rS   )NN)rd   re   rf   rg   r�   r¡   rn   ro   rp   s   @r<   rÝ   rÝ     s"   ø‡ € ÷,6ð ,6÷\T)÷ T)ð T)rG   rÝ   c                   ó>   a a€ ] tR tRt oV3R lV 3R lltRtVtV ;t# )ÚAXK2DecoderLayeri‘  c                ó&   <€ V ^8„  d   QhRS[ RS[/# rß   rá   )r`   ra   s   "€r<   rb   ÚAXK2DecoderLayer.__annotate__’  s   ø€ ÷ 
ñ 
™zð 
±cñ 
rG   c                óæ   <€ \         SV `  W4       \        V4      V n        VP                  V,          R 8X  d   \        V4      V n        R# \        VP                  VP                  R7      V n        R# )r,   r’   N)	r€   r�   rŽ   Úinput_layernormr3   rr   rK   rV   Úpost_attention_layernorm)r9   rz   rà   rˆ   s   &&&€r<   r�   ÚAXK2DecoderLayer.__init__’  sd   ø€ Ü‰Ñ˜Ô+Ü/°Ó7ˆÔð ×%Ñ% iÕ0°HÔ<ô ˜VÓ$ð 	Ö%ô ˜V×/Ñ/°V×5HÑ5HÔIð 	Ö%rG   )r.  r/  rŠ   rŒ   s   @@r<   r*  r*  ‘  s   ù‡ € ÷
÷ 
ó 
rG   r*  c                   óT   a € ] tR tRt o R.t]P                  ! 4       R 4       tRtV t	R# )ÚAXK2PreTrainedModeliœ  Úinv_freqc                ó  € \         P                  ! W4       \        V\        4      '       dZ   \        P
                  ! VP                  R V P                  P                  R7       \        P                  ! VP                  4       R# \        V\        4      '       dq   \        P
                  ! VP                  R V P                  P                  R7       \        P
                  ! VP                  R V P                  P                  R7       R# R# )r¯   )ÚmeanÚstdN)r   Ú_init_weightsÚ
isinstancer¨   ÚinitÚnormal_rÌ   rz   Úinitializer_rangeÚzeros_rÍ   rÙ   Úgate_up_projÚ	down_proj)r9   Úmodules   &&r<   r7  Ú!AXK2PreTrainedModel._init_weightsŸ  sŸ   € ä×%Ò% dÔ3Ü�fœn×-Ò-Ü�LŠL˜Ÿ™¨S°d·k±k×6SÑ6SÕTÜ�KŠK˜×6Ñ6Ö7Ü˜¤×,Ò,Ü�LŠL˜×,Ñ,°3¸D¿K¹K×<YÑ<YÕZÜ�LŠL˜×)Ñ)°¸¿¹×9VÑ9V×Wñ -rG   rs   N)
rd   re   rf   rg   Ú"_keys_to_ignore_on_load_unexpectedrš   Úno_gradr7  rn   ro   rp   s   @r<   r2  r2  œ  s'   ø‡ € Ø*4¨Ð&à
‡]‚]ƒ_ñXó öXrG   r2  c                   ó   € ] tR tRtRtR# )Ú	AXK2Modeliª  rs   Nrt   rs   rG   r<   rD  rD  ª  ru   rG   rD  c                   ó   € ] tR tRtRtR# )ÚAXK2ForCausalLMi®  rs   Nrt   rs   rG   r<   rF  rF  ®  ru   rG   rF  c                   ó   € ] tR tRtRtR# )ÚAXK2ForSequenceClassificationi²  rs   Nrt   rs   rG   r<   rH  rH  ²  ru   rG   rH  c                   ó   € ] tR tRtRtR# )ÚAXK2ForTokenClassificationi¶  rs   Nrt   rs   rG   r<   rJ  rJ  ¶  ru   rG   rJ  )r"   r2  rD  rF  rH  rJ  )Grh   Úcollections.abcr   rš   Útorch.nn.functionalr   Ú
functionalrÈ   Úhuggingface_hub.dataclassesr   Ú r   r9  Úcache_utilsr   Úconfiguration_utilsr   Úmodeling_flash_attention_utilsr   Úmodeling_layersr	   r
   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   Úclip.modeling_clipr   Ú deepseek_v3.modeling_deepseek_v3r   r   r   Ú'deepseek_v32.configuration_deepseek_v32r   Ú"deepseek_v32.modeling_deepseek_v32r   r   r   r   r   r   r   r   r   r   r   Ú
get_loggerrd   Úloggerr"   rr   rw   rä   rŽ   r¤   r¦   r¨   rÙ   rÛ   rÝ   r*  r2  rD  rF  rH  rJ  Ú__all__rs   rG   r<   Ú<module>r^     sx  ðñ å $ã ß Ð Ý .Ý å &Ý  Ý 3Ý Bß ^ß FÝ &ß ,Ý (÷ñ õ
 H÷÷ ÷ ñ ð 
×	Ò	˜HÓ	%€ñ ˜<Ô(ØôzpÐ"ó zpó ó )ðzpôz	Ð#ô 	ôU�'ô UôD�r—y‘yô Dô"	Ð4ô 	ô	Ð$ô 	ô$9Ð*ô $9ôN	Ð$ô 	ô	ˆnô 	ôC)Ð(ô C)ôL
Ð.ô 
ôXÐ4ô Xô	Ð ô 	ô	Ð,ô 	ô	Ð$DÐFYô 	ô	Ð!>Ð@Sô 	ò‚rG   