+
    œ°³jî¡  ã                   ó  € ^ RI t ^ RIHt ^ RIt^ RIHu Ht ^ RIHt ^RIH	t
 ^RIHt ^RIHtHt ^RIHt ^RIHtHt ^R	IHt ^R
IHt ^RIHtHtHt ^RIHtHt ^RI H!t!H"t" ^RI#H$t$H%t% ^RI&H't' ^RI(H)t)H*t* ^RI+H,t, ^RI-H.t.H/t/H0t0 ^RI1H2t2 ^RI3H4t4 ]! R4       ! R R]Pj                  4      4       t6 ! R R]Pj                  4      t7 ! R R]Pj                  4      t8 ! R R]Pj                  4      t9R t:]! R4      R@R  l4       t; ! R! R"]Pj                  4      t< ! R# R$]Pj                  4      t=] ! R% R&]Pj                  4      4       t> ! R' R(]Pj                  4      t? ! R) R*]Pj                  4      t@R+ R, ltARAR- R. lltBRBR/ ltCR0 tDRCR1 ltE ! R2 R3]Pj                  4      tF ! R4 R5]4      tG]* ! R6 R7]%4      4       tH]* ! R8 R9]H4      4       tI]* ! R: R;]H]4      4       tJ ! R< R=]]H4      tK ! R> R?]]H4      tL. RDOtMR# )Eé    N)ÚCallable)Únn)Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_experts_implementationÚuse_kernel_forward_from_hub)Úcreate_causal_mask)ÚFlashAttentionKwargs)Ú GenericForSequenceClassificationÚGenericForTokenClassificationÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstring)Údeprecate_kwarg)Úcan_return_tupleÚmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputs©Ú
AXK2ConfigÚRMSNormc                   óZ   a a€ ] tR t^3t oRV3R lV 3R llltV3R lR ltR tRtVtV ;t	# )ÚAXK2RMSNormc                ó$   <€ V ^8„  d   QhRS[ RR/# )é   ÚepsÚreturnN)Úfloat)ÚformatÚ__classdict__s   "€Úg/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/axk2/modeling_axk2.pyÚ__annotate__ÚAXK2RMSNorm.__annotate__5   s   ø€ ÷ $ñ $©ð $¸$ñ $ó    c                óŽ   <€ \         SV `  4        \        P                  ! \        P
                  ! V4      4      V n        W n        R# )z*
AXK2RMSNorm is equivalent to T5LayerNorm
N)ÚsuperÚ__init__r   Ú	ParameterÚtorchÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizer&   Ú	__class__s   &&&€r+   r1   ÚAXK2RMSNorm.__init__5   s/   ø€ ô 	‰ÑÔÜ—l’l¤5§:¢:¨kÓ#:Ó;ˆŒØ #Ör.   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r%   Úhidden_statesr'   ©r3   ÚTensor)r)   r*   s   "€r+   r,   r-   =   s#   ø€ ÷ ;ñ ;¡U§\¡\ð ;±e·l±lñ ;r.   c                ó:  € VP                   pVP                  \        P                  4      pVP	                  ^4      P                  RRR7      pV\        P                  ! W0P                  ,           4      ,          pV P                  VP                  V4      ,          # )r%   T)Úkeepdiméÿÿÿÿ)	ÚdtypeÚtor3   Úfloat32ÚpowÚmeanÚrsqrtr6   r5   )r7   r=   Úinput_dtypeÚvariances   &&  r+   ÚforwardÚAXK2RMSNorm.forward=   sw   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯ª°H×?TÑ?TÕ4TÓ(UÕUˆØ�{‰{˜]×-Ñ-¨kÓ:Õ:Ð:r.   c                ó^   € \        V P                  P                  4       R V P                   2# )z, eps=)Útupler5   Úshaper6   )r7   s   &r+   Ú
extra_reprÚAXK2RMSNorm.extra_reprD   s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr.   )r6   r5   )ç�íµ ÷Æ°>)
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r1   rK   rP   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r9   r*   s   @@r+   r#   r#   3   s%   ù‡ € ÷$õ $÷;ð ;÷Jò Jr.   r#   c                   óP   a a€ ] tR t^Ht oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚAXK2GateMLPc                ó    <€ V ^8„  d   QhRS[ /# ©r%   Úconfigr   )r)   r*   s   "€r+   r,   ÚAXK2GateMLP.__annotate__I   s   ø€ ÷ Uñ U™zñ Ur.   c                ó,  <€ \         SV `  4        Wn        \        P                  ! 4       V n        \        P                  ! VP                  VP                  R R7      V n	        \        P                  ! VP                  VP                  R R7      V n
        R# ©F©ÚbiasN)r0   r1   r_   r   ÚSiLUÚactivation_fnÚLinearr8   Úgated_norm_rankÚfc1Úfc2©r7   r_   r9   s   &&€r+   r1   ÚAXK2GateMLP.__init__I   sa   ø€ Ü‰ÑÔØŒÜŸWšW›YˆÔÜ—9’9˜V×/Ñ/°×1GÑ1GÈeÔTˆŒÜ—9’9˜V×3Ñ3°V×5GÑ5GÈeÔTˆŽr.   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r<   r>   )r)   r*   s   "€r+   r,   r`   P   s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r.   c                ól   € V P                  V4      pV P                  V4      pV P                  V4      pV# ©N)ri   rf   rj   )r7   r=   s   &&r+   rK   ÚAXK2GateMLP.forwardP   s4   € ØŸ™ Ó/ˆØ×*Ñ*¨=Ó9ˆØŸ™ Ó/ˆØÐr.   )rf   r_   ri   rj   ©	rS   rT   rU   rV   r1   rK   rW   rX   rY   rZ   s   @@r+   r\   r\   H   s    ù‡ € ÷Uó U÷÷ ð r.   r\   c                   óT   a a€ ] tR t^Wt oRtV3R lV 3R lltV3R lR ltRtVtV ;t	# )ÚAXK2GatedRMSNormz‹RMSNorm followed by a low-rank input-dependent sigmoid gate (Megatron `GatedNormWrapper`):

y = RMSNorm(x)
return y * sigmoid(gate_mlp(y))
c                ó    <€ V ^8„  d   QhRS[ /# r^   r   )r)   r*   s   "€r+   r,   ÚAXK2GatedRMSNorm.__annotate__^   s   ø€ ÷ 'ñ '™zñ 'r.   c                ó�   <€ \         SV `  4        \        VP                  VP                  R 7      V n        \        V4      V n        R# )©r&   N)r0   r1   r#   r8   Úrms_norm_epsÚnormr\   Úmlprk   s   &&€r+   r1   ÚAXK2GatedRMSNorm.__init__^   s4   ø€ Ü‰ÑÔÜ × 2Ñ 2¸×8KÑ8KÔLˆŒ	Ü˜vÓ&ˆŽr.   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r%   Úxr'   r>   )r)   r*   s   "€r+   r,   ru   c   s'   ø€ ÷ Dñ D™Ÿ™ð D©%¯,©,ñ Dr.   c                óÊ   € V P                  V4      pV\        P                  ! V P                  V4      P	                  4       4      ,          P                  VP                  4      # ro   )ry   r3   Úsigmoidrz   r(   rD   rC   )r7   r}   Úys   && r+   rK   ÚAXK2GatedRMSNorm.forwardc   sC   € Ø�I‰I�a‹LˆØ”E—M’M $§(¡(¨1£+×"3Ñ"3Ó"5Ó6Õ6×:Ñ:¸1¿7¹7ÓCÐCr.   )rz   ry   ©
rS   rT   rU   rV   Ú__doc__r1   rK   rW   rX   rY   rZ   s   @@r+   rs   rs   W   s&   ù‡ € ñ÷'ó '÷
D÷ Dð Dr.   rs   c                   óÎ   a a€ ] tR t^ht o]! RRR7      R
V3R lV 3R lll4       t]]! RRR7      R
V3R lR ll4       4       t]P                  ! 4       ]
R 4       4       tR	tVtV ;t# )ÚAXK2RotaryEmbeddingÚdevicez5.18)Úversionc                ó    <€ V ^8„  d   QhRS[ /# r^   r   )r)   r*   s   "€r+   r,   Ú AXK2RotaryEmbedding.__annotate__j   s   ø€ ÷ Oñ O™zñ Or.   c                óä  <€ \         SV `  4        VP                  V n        VP                  V n        Wn        V P
                  P                  R ,          V n        V P                  pV P                  R8w  d   \        V P                  ,          pV! V P
                  V4      w  q@n
        \        P                  ! VRR7      V n        \        P                  ! VP                  4       RR7      V n        R# )Ú	rope_typeÚdefaultF)Ú
persistentN)r0   r1   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr_   Úrope_parametersr‹   Úcompute_default_rope_parametersr   Úattention_scalingr   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)r7   r_   r†   Úrope_init_fnr•   r9   s   &&&  €r+   r1   ÚAXK2RotaryEmbedding.__init__i   s¬   ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àŒàŸ™×4Ñ4°[ÕAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÔ&Ü.¨t¯~©~Õ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÔ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÔ!NˆÖr.   c                óP   <€ V ^8„  d   QhRS[ RS[S[P                  S[3,          /# )r%   r_   r'   )r    rN   r3   r?   r(   )r)   r*   s   "€r+   r,   r‰   |   s.   ø€ ÷ 5ñ 5±
ð 5ÑV[Ñ\a×\hÑ\hÑjoÐ\oÕVpñ 5r.   c           	     ó0  € V P                   R,          p\        V RR4      ;'       g    V P                  V P                  ,          pRpRV\        P
                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚhead_dimNç      ð?©rC   )r‘   Úgetattrr8   Únum_attention_headsr3   Úaranger(   rD   )r_   r†   ÚkwargsÚbaseÚdimÚattention_factorr•   s   &&,    r+   r’   Ú3AXK2RotaryEmbedding.compute_default_rope_parametersz   s~   € ð ×%Ñ% lÕ3ˆÜ�f˜j¨$Ó/×cÐc°6×3EÑ3EÈ×IcÑIcÕ3cˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r.   c                ó^  € V P                   R,          P                  VP                  ^ ,          R	^4      P                  \        P
                  VP                  R7      pVR
,          P                  4       p\        VP                  P                  \        4      '       d2   VP                  P                  R8w  d   VP                  P                  MRp\        VRR7      ;_uu_ 4        W4,          P                  ^^4      p\        P                  ! Wf3R	R7      pVP                  4       V P                  ,          pVP                  4       V P                  ,          p	R R R 4       XP                  VP                   R7      X	P                  VP                   R7      3#   + '       g   i     LH; i)N)rC   r†   ÚmpsÚcpuF)Údevice_typeÚenabled©r¥   rŸ   )NºNNNNrB   )r®   Nr®   )r•   ÚexpandrO   rD   r3   r(   r†   Ú
isinstanceÚtypeÚstrr   Ú	transposeÚcatÚcosr“   ÚsinrC   )
r7   r}   Úposition_idsÚinv_freq_expandedÚposition_ids_expandedr«   ÚfreqsÚembrµ   r¶   s
   &&&       r+   rK   ÚAXK2RotaryEmbedding.forwardŽ   s2  € ð �M‰M˜-Õ(×/Ñ/°×0BÑ0BÀ1Õ0EÀrÈ1ÓM×PÑPÔW\×WbÑWbÐkl×ksÑksÐPÓtð 	ð !-¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆä¨¸U×CÖCØ&Õ>×IÑIÈ!ÈQÓOˆEÜ—)’)˜U˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆC÷	 Dð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷ D×Cús   Ã(A4FÆF,	)r“   r_   r•   r�   r—   r�   r‹   ro   )rS   rT   rU   rV   r   r1   Ústaticmethodr’   r3   Úno_gradr   rK   rW   rX   rY   rZ   s   @@r+   r…   r…   h   su   ù‡ € Ù�X vÔ.÷Oô Oó /ðOð  Ù�X vÔ.÷5ñ 5ó /ó ð5ð$ ‡]‚]ƒ_Øñ<ó ó ÷<ð <r.   r…   c                óÌ   € V RRV P                   R,          ^,          13,          pV RV P                   R,          ^,          R13,          p\        P                  ! V) V3RR7      # )z*Rotates half the hidden dims of the input..Nr­   rB   )rO   r3   r´   )r}   Úx1Úx2s   &  r+   Úrotate_halfrÂ   ¡   sZ   € à	
ˆ3Ð"�!—'‘'˜"•+ Õ"Ð"Ð"Õ	#€BØ	
ˆ3�—‘˜•˜qÕ Ñ"Ð"Õ	#€BÜ�9Š9�r�c˜2�Y BÔ'Ð'r.   Úrotary_pos_embc                óÈ   € VP                  V4      pVP                  V4      pW,          \        V 4      V,          ,           pW,          \        V4      V,          ,           pWV3# )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)Ú	unsqueezerÂ   )ÚqÚkrµ   r¶   Úunsqueeze_dimÚq_embedÚk_embeds   &&&&&  r+   Úapply_rotary_pos_embrË   ¨   sS   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�wœ; q›>¨CÕ/Õ0€GØ�wœ; q›>¨CÕ/Õ0€GØÐÐr.   c                   ó€   a a€ ] tR t^Ât oRtV3R lV 3R llt]P                  ! 4       RV3R lR ll4       tRt	Vt
V ;t# )ÚAXK2Indexera5  
DeepSeek Sparse Attention (DSA) indexer for selecting top-k tokens.

The Indexer has its own lightweight projections (wq_b, wk) separate from the main MLA attention,
and returns the additive top-k sparse mask directly (`0` at the selected tokens, `-inf` elsewhere);
the raw top-k indices are only ever scattered into that mask, so they are not surfaced.

**Cache strategy**: the indexer key cache lives on the per-layer `DynamicIndexedLayer` (or the
`StaticIndexedLayer` for static caches) inside the shared cache, accessed via
`past_key_values.update_indexer()`.
c                ó$   <€ V ^8„  d   QhRRRS[ /# )r%   r_   r    Ú	layer_idx)Úint)r)   r*   s   "€r+   r,   ÚAXK2Indexer.__annotate__Ï   s   ø€ ÷ 1ñ 1˜|ð 1¹ñ 1r.   c                óÔ  <€ \         SV `  4        Wn        W n        VP                  V n        VP
                  V n        VP                  V n        VP                  V n	        VP                  V n
        VP                  V n        \        P                  ! V P                  V P                  V P                  ,          R R7      V n        \        P                  ! V P                  V P                  R R7      V n        \        P                   ! V P                  RR7      V n        \        P                  ! V P                  V P                  R R7      V n        V P                  R,          V n        R# )Frc   rR   rw   Nç      à¿)r0   r1   r_   rÏ   r8   Úindex_n_headsÚn_headsÚindex_head_dimr�   Úqk_rope_head_dimÚ
index_topkÚq_lora_rankr   rg   Úwq_bÚwkÚ	LayerNormÚk_normÚweights_projÚsoftmax_scale©r7   r_   rÏ   r9   s   &&&€r+   r1   ÚAXK2Indexer.__init__Ï   sõ   ø€ Ü‰ÑÔØŒØ"Œà &× 2Ñ 2ˆÔØ"×0Ñ0ˆŒØ#×2Ñ2ˆŒØ%+×%<Ñ%<ˆÔØ%×0Ñ0ˆŒØ &× 2Ñ 2ˆÔä—I’I˜d×.Ñ.°·±¸t¿}¹}Õ0LÐSXÔYˆŒ	Ü—)’)˜D×,Ñ,¨d¯m©mÀ%ÔHˆŒÜ—l’l 4§=¡=°dÔ;ˆŒÜŸIšI d×&6Ñ&6¸¿¹È5ÔQˆÔØ!Ÿ]™]¨DÕ0ˆÖr.   c                óô   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[S[ P                  S[ P                  3,          RS[ P                  RS[ P                  RS[R,          RS[ P                  /# )	r%   r=   Úq_residÚposition_embeddingsÚattention_maskr·   Úpast_key_valuesNr'   )r3   r?   rN   r   )r)   r*   s   "€r+   r,   rÑ   â   s„   ø€ ÷ >Gñ >Gá—|‘|ð>Gñ —‘ð>Gñ #¡5§<¡<±·±Ð#=Õ>ð	>Gñ
 Ÿ™ð>Gñ —l‘lð>Gñ  �ð>Gñ 
�‰ñ>Gr.   c                óR  € VP                   w  rxp	Vw  r«V P                  V4      pVP                  WxV P                  V P                  4      p\
        P                  ! WÀP                  V P                  V P                  ,
          .RR7      w  rÞV P                  V P                  V4      4      P                  ^4      p\
        P                  ! WðP                  V P                  V P                  ,
          .RR7      w  pp\        VVW«^R7      w  pp\
        P                  ! WÞ.RR7      p\
        P                  ! VV.RR7      P                  ^4      pVe   VP                  WðP                  4      p\
        P                   ! VP#                  4       VP%                  RR4      P#                  4       P                  ^4      4      V P&                  ,          p\(        P*                  ! V4      pV P-                  VP/                  V P,                  P0                  P2                  4      4      P#                  4       V P                  R,          ,          p\
        P                   ! VP                  R4      V4      P                  R4      pVP2                  \
        P4                  8X  d   VP7                  V( \#        R4      4      pM	VV,           p\9        V P:                  VP                   R,          4      pVP=                  VRR7      P>                  P/                  \
        P@                  4      # )uW  
Selects the top-k tokens per query for DeepSeek Sparse Attention (DSA).

This is the bf16 equivalent of the reference Indexer which uses `rotate_activation` (Hadamard transform)
and `fp8_index` (FP8 quantized scoring kernel). Since the Hadamard transform is orthogonal (dot products
are preserved: HqÂ·Hk = qÂ·k), and FP8 quantization is a precision optimization, we skip both and compute
scores directly in bf16/fp32.

The scoring logic computes:
    index_score[b,s,t] = Î£_h (weight[b,s,h] Â· softmax_scale Â· q[b,s,h,:] Â· k[b,t,:])

Args:
    hidden_states: Input hidden states `[B, S, hidden_size]`.
    q_resid: Query residual from `q_a_layernorm(q_a_proj(x))`, shape `[B, S, q_lora_rank]`.
    position_embeddings: `(cos, sin)` from RotaryEmbedding.
    attention_mask: Causal mask, broadcastable to `[B, S, T]`.
    past_key_values: Cache object containing the indexer key cache for this layer.

Returns:
    `torch.Tensor`: the `int32` top-k token indices of shape `[B, S, topk]`. The eager / SDPA paths
        turn these into an additive sparse mask; the `flash-mla` kernel consumes them directly.
r­   )rÈ   z-infrB   éþÿÿÿrÓ   )!rO   rÚ   ÚviewrÕ   r�   r3   Úsplitr×   rÝ   rÛ   rÅ   rË   r´   ÚsqueezeÚupdate_indexerrÏ   Úmatmulr(   r³   rß   ÚFÚrelurÞ   rD   r5   rC   ÚboolÚmasked_fillÚminrØ   ÚtopkÚindicesÚint32)r7   r=   rã   rä   rå   r·   ræ   Ú
batch_sizeÚseq_lenÚ_rµ   r¶   rÆ   Úq_rotÚq_passrÇ   Úk_rotÚk_passÚscoresÚweightsÚindex_scoresró   s   &&&&&&&               r+   rK   ÚAXK2Indexer.forwardá   s^  € ð@ "/×!4Ñ!4Ñˆ
˜QØ&‰ˆØ�I‰I�gÓˆØ�F‰F�:¨¯©°d·m±mÓDˆÜŸš A×(=Ñ(=¸t¿}¹}Èt×OdÑOdÕ?dÐ'eÐkmÔn‰ˆà�K‰K˜Ÿ™ Ó.Ó/×9Ñ9¸!Ó<ˆÜŸš A×(=Ñ(=¸t¿}¹}Èt×OdÑOdÕ?dÐ'eÐkmÔn‰ˆˆvô ,¨E°5¸#ÐRSÔT‰ˆˆuÜ�IŠI�u�o¨2Ô.ˆÜ�IŠI�u˜f�o¨2Ô.×6Ñ6°qÓ9ˆàÒ&Ø×.Ñ.¨q·.±.ÓAˆAä—’˜aŸg™g›i¨¯©°R¸Ó)<×)BÑ)BÓ)D×)NÑ)NÈqÓ)QÓRÐUY×UgÑUgÕgˆÜ—’˜“ˆð ×#Ñ# M×$4Ñ$4°T×5FÑ5F×5MÑ5M×5SÑ5SÓ$TÓU×[Ñ[Ó]Ðae×amÑamÐosÕasÕtˆÜ—|’| G×$5Ñ$5°bÓ$9¸6ÓB×JÑJÈ2ÓNˆð ×Ñ¤5§:¡:Ô-Ø'×3Ñ3°^°OÄUÈ6Ã]ÓS‰Là'¨.Õ8ˆLä�4—?‘? L×$6Ñ$6°rÕ$:Ó;ˆØ× Ñ  ¨2Ð Ó.×6Ñ6×9Ñ9¼%¿+¹+ÓFÐFr.   )r_   r�   r8   rØ   rÝ   rÏ   rÕ   rÙ   r×   rß   rÞ   rÛ   rÚ   ro   )rS   rT   rU   rV   rƒ   r1   r3   r¾   rK   rW   rX   rY   rZ   s   @@r+   rÍ   rÍ   Â   s<   ù‡ € ñ
÷1ó 1ð$ ‡]‚]ƒ_÷>Gñ >Gó ÷>Gð >Gr.   rÍ   c                   óJ   a a€ ] tR tRt oV3R lV 3R lltR tR tRtVtV ;t	# )ÚAXK2TopkRouteri#  c                ó    <€ V ^8„  d   QhRS[ /# r^   r   )r)   r*   s   "€r+   r,   ÚAXK2TopkRouter.__annotate__$  s   ø€ ÷ 
gñ 
g™zñ 
gr.   c                ó,  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  V n        \        P                  ! \        P                  ! V P
                  V P                  4      4      V n        VP                  V n        VP                  V n        VP                   V n        VP"                  V n        \        P$                  ! \        P                  ! V P
                  \        P&                  R 7      4      V n        R# )rŸ   N)r0   r1   Únum_experts_per_tokÚtop_kÚnum_local_expertsÚnum_expertsr8   Ú
hidden_dimr   r2   r3   Úzerosr5   Úrouted_scaling_factorÚn_groupÚ	num_groupÚ
topk_groupÚnorm_topk_probr”   rE   Úe_score_correction_biasrk   s   &&€r+   r1   ÚAXK2TopkRouter.__init__$  s¸   ø€ Ü‰ÑÔØ×/Ñ/ˆŒ
Ø!×3Ñ3ˆÔØ ×,Ñ,ˆŒÜ—l’l¤5§;¢;¨t×/?Ñ/?ÀÇÁÓ#QÓRˆŒØ%+×%AÑ%AˆÔ"ØŸ™ˆŒØ ×+Ñ+ˆŒØ$×3Ñ3ˆÔÜ')§y¢y´·²¸d×>NÑ>NÔW\×WdÑWdÔ1eÓ'fˆÖ$r.   c                ó~  € VP                  RV P                  4      p\        P                  ! VP	                  \
        P                  4      V P                  P	                  \
        P                  4      4      pVP                  4       pW0P                  ,           pV P                  e   V P                  V4      p\
        P                  ! W@P                  RRR7      ^,          pVP                  ^V4      pV P                  '       d#   VP!                  RRR7      R,           pWg,          pW`P"                  ,          pW&V3# )é   F©rÇ   r¥   ÚsortedT)r¥   rA   g#B’¡œÇ;rB   )ré   r
  rî   Úlinearr±   r3   rE   r5   r   r  r  Úapply_group_scoringró   r  Úgatherr  Úsumr  )r7   r=   Úrouter_logitsrý   Úscores_for_choiceÚtopk_indicesÚtopk_weightsÚdenominators   &&      r+   rK   ÚAXK2TopkRouter.forward0  sý   € Ø%×*Ñ*¨2¨t¯©Ó?ˆÜŸš ×!3Ñ!3´E·M±MÓ!BÀDÇKÁK×DTÑDTÔUZ×UbÑUbÓDcÓdˆØ×&Ñ&Ó(ˆØ"×%AÑ%AÕAÐð �>‰>Ò%Ø $× 8Ñ 8Ð9JÓ KÐä—z’zÐ"3·z±zÀrÐRWÔXÐYZÕ[ˆØ—}‘} Q¨Ó5ˆØ××ÐØ&×*Ñ*¨r¸4Ð*Ó@À5ÕHˆKØÕ'ˆLØ#×&@Ñ&@Õ@ˆØ¨LÐ8Ð8r.   c                óp  € VP                  RV P                  V P                  V P                  ,          4      P                  ^RR7      ^ ,          P	                  RR7      p\
        P                  ! W P                  RRR7      ^,          p\
        P                  ! V4      pVP                  ^V^4       VP                  R4      P                  RV P                  V P                  V P                  ,          4      P                  RV P                  4      pVP                  VP                  4       ( R4      pV# )z$Apply DeepSeek style grouped scoringr­   Fr  ç        rB   )ré   r  r	  ró   r  r3   r  Ú
zeros_likeÚscatter_rÅ   r¯   Úreshaperñ   rð   )r7   r  Úgroup_scoresÚ	group_idxÚ
group_maskÚ
score_masks   &&    r+   r  Ú"AXK2TopkRouter.apply_group_scoringB  sø   € ð ×"Ñ" 2 t§~¡~°t×7GÑ7GÈ4Ï>É>Õ7YÓZß‰T�!˜ˆT‹_˜Qõ ç‰S�RˆS‹[ð 	ô
 —J’J˜|¯©ÀBÈuÔUÐVWÕXˆ	Ü×%Ò% lÓ3ˆ
Ø×Ñ˜A˜y¨!Ô,à× Ñ  Ó$ß‰V�B˜Ÿ™¨×(8Ñ(8¸D¿N¹NÕ(JÓKß‰W�R˜×)Ñ)Ó*ð 	ð .×9Ñ9¸:¿?¹?Ó;LÐ:LÈcÓRÐØ Ð r.   )	r  r
  r  r	  r  r  r  r  r5   )
rS   rT   rU   rV   r1   rK   r  rW   rX   rY   rZ   s   @@r+   r  r  #  s    ù‡ € ÷
gó 
gò9÷$!ò !r.   r  c                   óH   a a€ ] tR tRt oRtV 3R ltV3R lR ltRtVtV ;t	# )ÚAXK2ExpertsiV  z2Collection of expert weights stored as 3D tensors.c                ó  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  V n        \        P                  ! \        P                  ! V P                  ^V P                  ,          V P
                  4      4      V n        \        P                  ! \        P                  ! V P                  V P
                  V P                  4      4      V n        \        VP                  ,          V n        R# )r%   N)r0   r1   r  r	  r8   r
  Úmoe_intermediate_sizeÚintermediate_dimr   r2   r3   ÚemptyÚgate_up_projÚ	down_projr   Ú
hidden_actÚact_fnrk   s   &&€r+   r1   ÚAXK2Experts.__init__Z  s­   ø€ Ü‰ÑÔØ!×3Ñ3ˆÔØ ×,Ñ,ˆŒØ &× <Ñ <ˆÔÜŸLšL¬¯ª°T×5EÑ5EÀqÈ4×K`ÑK`ÕG`Ðbf×bqÑbqÓ)rÓsˆÔÜŸš¤e§k¢k°$×2BÑ2BÀDÇOÁOÐUY×UjÑUjÓ&kÓlˆŒÜ˜V×.Ñ.Õ/ˆŽr.   c                ó‚   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[ P                  /# )r%   r=   Útop_k_indexÚtop_k_weightsr'   r>   )r)   r*   s   "€r+   r,   ÚAXK2Experts.__annotate__c  sC   ø€ ÷ #ñ #á—|‘|ð#ñ —\‘\ð#ñ —|‘|ð	#ñ
 
�‰ñ#r.   c                óÆ  € \         P                  ! V4      p\         P                  ! 4       ;_uu_ 4        \         P                  P                  P                  W P                  R 7      pVP                  ^^^ 4      p\         P                  ! VP                  RR7      ^ 4      P                  4       pRRR4       X EF  pV^ ,          pWpP                  8X  d   K  \         P                  ! XV,          4      w  r‰W,          p
\        P                  P                  W P                  V,          4      P                  ^RR7      w  r¼V P                  V4      V,          p\        P                  P                  WÐP                   V,          4      pWÓW˜R3,          ,          pVP#                  ^ W�P%                  VP&                  4      4       EK  	  V#   + '       g   i     EL'; i))Únum_classesr­   NrB   )rB   rè   )r3   r#  r¾   r   Ú
functionalÚone_hotr	  ÚpermuteÚgreaterr  ÚnonzeroÚwherer  r1  Úchunkr4  r2  Ú
index_add_rD   rC   )r7   r=   r7  r8  Úfinal_hidden_statesÚexpert_maskÚ
expert_hitÚ
expert_idxÚ	top_k_posÚ	token_idxÚcurrent_stateÚgateÚupÚcurrent_hidden_statess   &&&&          r+   rK   ÚAXK2Experts.forwardc  ss  € ô $×.Ò.¨}Ó=ÐÜ�]Š]�_�_ÜŸ(™(×-Ñ-×5Ñ5°k×O_ÑO_Ð5Ó`ˆKØ%×-Ñ-¨a°°AÓ6ˆKÜŸš {§¡¸8 Ó'DÀaÓH×PÑPÓRˆJ÷ ô
 %ˆJØ# A�ˆJØ×-Ñ-Ô-ÙÜ#(§;¢;¨{¸:Õ/FÓ#GÑ ˆIØ)Õ4ˆMÜ—}‘}×+Ñ+¨M×;LÑ;LÈZÕ;XÓY×_Ñ_Ð`aÐgiÐ_Ój‰HˆDØ$(§K¡K°Ó$5¸Õ$:Ð!Ü$&§M¡M×$8Ñ$8Ð9N×P^ÑP^Ð_iÕPjÓ$kÐ!Ø$9È)Ð`dÐJdÕ<eÕ$eÐ!Ø×*Ñ*¨1¨i×9QÑ9QÐRe×RkÑRkÓ9l×mñ %ð #Ð"÷# �_ˆ_ús   ´A=GÇG 	)r4  r2  r1  r
  r/  r	  r‚   rZ   s   @@r+   r,  r,  V  s   ù‡ € á<õ0÷#÷ #ð #r.   r,  c                   ó<   a a€ ] tR tRt oRV 3R lltR tRtVtV ;t# )ÚAXK2MLPi~  c                óä  <€ \         SV `  4        Wn        VP                  V n        Vf   VP                  MTV n        \
        P                  ! V P                  V P                  RR7      V n        \
        P                  ! V P                  V P                  RR7      V n        \
        P                  ! V P                  V P                  RR7      V n	        \        VP                  ,          V n        R # )NFrc   )r0   r1   r_   r8   Úintermediate_sizer   rg   Ú	gate_projÚup_projr2  r   r3  r4  )r7   r_   rR  r9   s   &&&€r+   r1   ÚAXK2MLP.__init__  s¬   ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ=NÒ=V ×!9Ò!9Ð\mˆÔÜŸš 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆŒÜ˜V×.Ñ.Õ/ˆŽr.   c                ó�   € V P                  V P                  V P                  V4      4      V P                  V4      ,          4      pV# ro   )r2  r4  rS  rT  )r7   r}   r2  s   && r+   rK   ÚAXK2MLP.forward‰  s6   € Ø—N‘N 4§;¡;¨t¯~©~¸aÓ/@Ó#AÀDÇLÁLÐQRÃOÕ#SÓTˆ	ØÐr.   )r4  r_   r2  rS  r8   rR  rT  ro   rq   rZ   s   @@r+   rP  rP  ~  s   ù‡ € ÷0÷ò r.   rP  c                   óT   a a€ ] tR tRt oRtV3R lV 3R lltV3R lR ltRtVtV ;t	# )ÚAXK2MoEiŽ  z2
A mixed expert module containing shared experts.
c                ó    <€ V ^8„  d   QhRS[ /# r^   r   )r)   r*   s   "€r+   r,   ÚAXK2MoE.__annotate__“  s   ø€ ÷ 
ñ 
™zñ 
r.   c                óÈ   <€ \         SV `  4        Wn        \        V4      V n        \        V4      V n        \        WP                  VP                  ,          R 7      V n
        R# ))r_   rR  N)r0   r1   r_   r,  Úexpertsr  rK  rP  r.  Ún_shared_expertsÚshared_expertsrk   s   &&€r+   r1   ÚAXK2MoE.__init__“  sL   ø€ Ü‰ÑÔØŒÜ" 6Ó*ˆŒÜ" 6Ó*ˆŒ	Ü%Ø×-IÑ-IÈF×LcÑLcÕ-cô
ˆÖr.   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r<   r>   )r)   r*   s   "€r+   r,   r[  œ  s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r.   c                óþ   € TpVP                   pV P                  V4      w  rEpVP                  RVP                   R,          4      pV P                  WV4      P                  ! V!  pWP	                  V4      ,           pV# )r  rB   )rO   rK  ré   r]  r_  )r7   r=   Ú	residualsÚ
orig_shaperø   r  r  s   &&     r+   rK   ÚAXK2MoE.forwardœ  sw   € Ø!ˆ	Ø"×(Ñ(ˆ
Ø(,¯	©	°-Ó(@Ñ%ˆ˜Ø%×*Ñ*¨2¨}×/BÑ/BÀ2Õ/FÓGˆØŸ™ ]À,ÓO×TÒTÐV`ÑaˆØ%×(;Ñ(;¸IÓ(FÕFˆØÐr.   )r_   r]  rK  r_  r‚   rZ   s   @@r+   rY  rY  Ž  s#   ù‡ € ñ÷
ó 
÷÷ ð r.   rY  c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r%   r=   Ún_repr'   )r3   r?   rÐ   )r)   s   "r+   r,   r,   ¦  s.   € ÷ 	Uñ 	UœUŸ\™\ð 	U´#ð 	U¼%¿,¹,ñ 	Ur.   c                ó˜   € V P                   w  r#rEV^8X  d   V # V R,          P                  W#WV4      p V P                  W#V,          WE4      # )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
)r®   r®   Nr®   r®   )rO   r¯   r%  )r=   rg  ÚbatchÚnum_key_value_headsÚslenr�   s   &&    r+   Ú	repeat_kvrl  ¦  sU   € ð
 2?×1DÑ1DÑ.€E Ø�„zØÐØ!Ð"2Õ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÕ(CÀTÓTÐTr.   c                ó   € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R\        R	\
        \        ,          /# )
r%   ÚmoduleÚqueryÚkeyÚvaluerå   NÚscalingÚdropoutr£   )r   ÚModuler3   r?   r(   r   r   )r)   s   "r+   r,   r,   ²  sw   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ð%ô ð%ô Ô'Õ(ñ%r.   c                 ó"  € \        W P                  4      p\        W0P                  4      p	\        P                  ! WP	                  ^^4      4      V,          p
Ve	   W¤,           p
\
        P                  P                  V
R\        P                  R7      P                  VP                  4      p
\
        P                  P                  W¦V P                  R7      p
\        P                  ! W©4      pVP	                  ^^4      P                  4       pWº3# )r%   )r¥   rC   )ÚpÚtrainingrB   )rl  Únum_key_value_groupsr3   rí   r³   r   r<  ÚsoftmaxrE   rD   rC   rs  rw  Ú
contiguous)rn  ro  rp  rq  rå   rr  rs  r£   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs   &&&&&&&,    r+   Úeager_attention_forwardr  ²  sÒ   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÕL€LØÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r.   c                 ój   € V ^8:  d   R# RV,          \         P                  ! V 4      ,          R,           # )r  rž   gš™™™™™¹?)ÚmathÚlog)ÚscaleÚmscales   &&r+   Úyarn_get_mscaler…  Ë  s(   € Ø�„zÙØ��<œ$Ÿ(š( 5›/Õ)¨CÕ/Ð/r.   c                 ó®   € V P                  R R4      R8w  d>   V P                  R^ 4      pV R,          pV'       d   \        W24      pW,          V,          pV# )r‹   rŒ   Úmscale_all_dimÚfactor)Úgetr…  )r‘   rr  r‡  Úscaling_factorr„  s   &&   r+   Úyarn_apply_mscaler‹  Ñ  sS   € Ø×Ñ˜;¨	Ó2°iÔ?Ø(×,Ñ,Ð-=¸qÓAˆØ(¨Õ2ˆßÜ$ ^ÓDˆFØÕ&¨Õ/ˆGØ€Nr.   c                ó  € VRRVP                   R,          ^,          13,          P                  V4      pVRRVP                   R,          ^,          13,          P                  V4      pV R,          V R,          rvVR,          VR,          r˜\        P                  ! Wb,          Ws,          ,
          Wr,          Wc,          ,           .RR7      p
\        P                  ! W‚,          W“,          ,
          W’,          Wƒ,          ,           .RR7      pW«3# )aŸ  
Applies interleaved Rotary Position Embedding to the query and key tensors.

DeepSeek lays the rotary dimensions out in interleaved pairs `(x0, x1), (x2, x3), ...`, each rotated by a
single frequency. We compute that rotation directly on the even/odd slices instead of de-interleaving with a
`view`/`transpose`/`reshape`; the output is bit-identical to the de-interleaved `rotate_half` formulation while
avoiding the extra contiguous copy.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    position_ids (`torch.Tensor`):
        The position indices of the tokens corresponding to the query and key tensors. For example, this can be
        used to pass offsetted position ids when working with a KV-cache.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
.Nr­   rB   ).:r   Nr%   ).:r  Nr%   )rO   rÅ   r3   r´   )rÆ   rÇ   rµ   r¶   r·   rÈ   Úq1Úq2Úk1Úk2rÉ   rÊ   s   &&&&&&      r+   Úapply_rotary_pos_emb_interleaver‘  Û  sÓ   € ð8 ˆcÐ'�S—Y‘Y˜r•] aÕ'Ð'Ð'Õ
(×
2Ñ
2°=Ó
A€CØ
ˆcÐ'�S—Y‘Y˜r•] aÕ'Ð'Ð'Õ
(×
2Ñ
2°=Ó
A€Càˆy�\˜1˜Y�<ˆØˆy�\˜1˜Y�<ˆä�iŠi˜� B¥HÕ,¨b­h¸½Õ.AÐBÈÔK€GÜ�iŠi˜� B¥HÕ,¨b­h¸½Õ.AÐBÈÔK€GØÐÐr.   c                   ój   a a€ ] tR tRt oRtV3R lV 3R lltV3R lR ltR
V3R lR lltR	tVt	V ;t
# )ÚAXK2Attentioni  zª
DeepSeek-V3 MLA, with a DSA indexer whose top-k sparse mask is folded into the attention mask.
Qlora rank formulation is dropped as it is never used in released models.
c                ó&   <€ V ^8„  d   QhRS[ RS[/# ©r%   r_   rÏ   ©r    rÐ   )r)   r*   s   "€r+   r,   ÚAXK2Attention.__annotate__  s   ø€ ÷ ,6ñ ,6™zð ,6±cñ ,6r.   c                óÌ  <€ \         SV `  4        Wn        W n        VP                  V n        VP
                  V n        VP                  V n        VP                  V n        VP                  V n	        VP                  V n
        VP                  V n        VP                  V n        VP                  VP                  ,           V n        VP                  VP                  ,          V n        R V n        \"        P$                  ! V P
                  V P                  VP&                  R7      V n        \+        V P                  4      V n        \"        P$                  ! ^V P                  ,          V P                  V P                  V P                  ,           ,          RR7      V n        \"        P$                  ! V P
                  V P                  V P                  ,           VP&                  R7      V n        \+        V P                  4      V n        \"        P$                  ! V P                  V P                  V P                  V P                  ,           ,          RR7      V n        \"        P$                  ! V P                  V P                  ,          V P
                  VP&                  R7      V n        \9        VP:                  V P                  R,          4      V n        \?        W4      V n         R# )Trc   FNrÓ   )!r0   r1   r_   rÏ   Úattention_dropoutr8   r¡   Ú	num_headsrÙ   r×   Úkv_lora_rankÚ
v_head_dimÚqk_nope_head_dimÚqk_head_dimrj  rx  Ú	is_causalr   rg   Úattention_biasÚq_a_projr#   Úq_a_layernormÚq_gate_projÚkv_a_proj_with_mqaÚkv_a_layernormÚ	kv_b_projÚo_projr‹  r‘   rr  rÍ   Úindexerrà   s   &&&€r+   r1   ÚAXK2Attention.__init__  s  ø€ Ü‰ÑÔØŒØ"ŒØ!'×!9Ñ!9ˆÔØ!×-Ñ-ˆÔØ×3Ñ3ˆŒà!×-Ñ-ˆÔØ &× 7Ñ 7ˆÔØ"×/Ñ/ˆÔØ ×+Ñ+ˆŒØ &× 7Ñ 7ˆÔØ!×2Ñ2°V×5LÑ5LÕLˆÔØ$*×$>Ñ$>À&×B\ÑB\Õ$\ˆÔ!àˆŒäŸ	š	 $×"2Ñ"2°D×4DÑ4DÈ6×K`ÑK`ÔaˆŒÜ(¨×)9Ñ)9Ó:ˆÔäŸ9š9Ø�× Ñ Õ  $§.¡.°D×4DÑ4DÀtÇÁÕ4VÕ"WÐ^cô
ˆÔô #%§)¢)Ø×ÑØ×Ñ × 5Ñ 5Õ5Ø×&Ñ&ô#
ˆÔô
 *¨$×*;Ñ*;Ó<ˆÔÜŸšØ×ÑØ�N‰N˜d×3Ñ3°d·o±oÕEÕFØô
ˆŒô —i’iØ�N‰N˜TŸ_™_Õ,Ø×ÑØ×&Ñ&ô
ˆŒô )¨×)?Ñ)?À×AQÑAQÐVZÕA[Ó\ˆŒÜ" 6Ó5ˆŽr.   c                ó’   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[S[ P                  S[ P                  3,          /# )r%   Úkv_noperû   r'   )r3   r?   rN   )r)   r*   s   "€r+   r,   r—  6  s?   ø€ ÷ (ñ (¡§¡ð (±e·l±lð (ÁuÉUÏ\É\Ñ[`×[gÑ[gÐMgÕGhñ (r.   c                ó‚  € VP                   w  r4rTW5RV P                  V P                  ,           3pV P                  V4      P	                  V4      P                  ^^4      p\        P                  ! WP                  V P                  .RR7      w  rxVP                  RVP                   ^,          RR4      pVP                  ! . VP                   RR OV P                  V P                  ,           N5!  p	V	RRV P                  13,          P                  V4       V	RV P                  R13,          P                  V4       W˜3# )as  Expands the compressed latents into key and value states. Args:
    - kv_nope: key + value without positional encoding, shape [batch_size, 1, seqlen, self.kv_lora_rank]
    - k_rot: shared key with positional encoding, shape [batch_size, 1, seqlen, self.qk_rope_head_dim]
Returns the key and value states, two tensors of shape [batch, num_heads, seq, (k or v)_head_dim].
r­   N.rB   )rO   r�  rœ  r¦  ré   r³   r3   rê   r¯   Ú	new_emptyr×   Úcopy_)
r7   r«  rû   rö   rø   Ú
seq_lengthÚ	key_shapeÚk_noper|  r{  s
   &&&       r+   Ú	expand_kvÚAXK2Attention.expand_kv6  s  € ð (/§}¡}Ñ$ˆ
�zØ¨R°×1FÑ1FÈÏÉÕ1XÐYˆ	à—.‘. Ó)×.Ñ.¨yÓ9×CÑCÀAÀqÓIˆÜ$Ÿ{š{¨7×5JÑ5JÈDÏOÉOÐ4\ÐbdÔeÑˆØ—‘˜R §¡¨a¥°"°bÓ9ˆð ×&Ò&Ðj¨¯©°c°rÐ(:Ðj¸D×<QÑ<QÐTX×TiÑTiÕ<iÓjˆ
Ø�3Ð/˜$×/Ñ/Ð/Ð/Õ0×6Ñ6°vÔ>Ø�3˜×-Ñ-Ñ/Ð/Õ0×6Ñ6°uÔ=ØÐ'Ð'r.   c                ó6  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          RS[ P                  RS[R,          RS[ P                  R,          RS[S[,          RS[S[ P                  S[ P                  R,          3,          /# )	r%   r=   rä   rå   ræ   Nr·   r£   r'   )r3   r?   rN   r   r   r   )r)   r*   s   "€r+   r,   r—  I  sž   ø€ ÷ T)ñ T)á—|‘|ðT)ñ #¡5§<¡<±·±Ð#=Õ>ðT)ñ Ÿ™ð	T)ñ
  �ðT)ñ —l‘l TÕ)ðT)ñ Ñ-Õ.ðT)ñ 
‰u�|‰|™UŸ\™\¨DÕ0Ð0Õ	1ñT)r.   c           	     óÞ  € VP                   R R	 w  rxWxR	V P                  V P                  ,           3p	V P                  V4      p
V P	                  V
4      pV P                  \        P                  ! Wº.R	R7      4      P                  V	4      p\        P                  ! WÀP                  V P                  .R	R7      w  rÞVP                  ^^4      pVP                  WxR	4      p\        P                  ! WÐP                  V P                  .R	R7      w  ppV P                  V4      p\        P                  ! VV P                  V P                  .R	R7      w  ppV P!                  V4      P                  V^W€P                  4      pVP                  V^W€P                  4      pVw  pp\#        VVVV4      w  pp\        P                  ! VV3R	R7      pV P%                  VV4      w  ppVe!   VP'                  VVV P(                  4      w  ppV P+                  VVVVR
,          VVR7      pR pV P,                  P.                  R9   dÈ   VP1                  WxVP                   ^,          3\        P2                  R7      P5                  R	VP7                  4       R4      P9                  ^4      pVP:                  \        P2                  8X  d   VV( ,          pM=VP=                  V\        P>                  ! VP:                  4      P@                  4      pMTp\B        PD                  ! V P,                  P.                  \F        4      pT! V VVVV3RV PH                  '       g   RMV PJ                  RV PL                  RV/VB w  ppVP                  WxR	4      PO                  4       pV\        PP                  ! VPS                  4       4      ,          PU                  VP:                  4      pV PW                  V4      pVV3# )Nr­   )ræ   rŸ   Frs  r"  rr  rô   rB   )r®   r   r®   r®   )ÚeagerÚsdpa),rO   rž  rœ  r¡  r¢  r£  r3   r´   ré   rê   r³   r%  r�  r×   r¤  r›  r¥  r‘  r²  ÚupdaterÏ   r¨  r_   Ú_attn_implementationÚnew_onesrð   ÚscatterÚlongrÅ   rC   rñ   Úfinforò   r   Úget_interfacer  rw  r™  rr  rz  r   r(   rD   r§  )r7   r=   rä   rå   ræ   r·   r£   rö   r¯  Úquery_gate_shapeÚq_compressedrã   Úquery_gate_statesÚquery_statesÚgate_statesrú   rù   Úcompressed_kvÚkv_passrû   rü   rµ   r¶   r{  r|  r  Úsparse_indicesÚ
index_maskÚattention_interfacer~  r}  s   &&&&&&,                        r+   rK   ÚAXK2Attention.forwardI  s˜  € ð "/×!4Ñ!4°S°bÐ!9Ñˆ
Ø&°B¸×8HÑ8HÈ4Ï?É?Õ8ZÐ[Ðð —}‘} ]Ó3ˆØ×$Ñ$ \Ó2ˆà ×,Ñ,¬U¯YªY¸Ð7NÐTVÔ-WÓX×]Ñ]Ð^nÓoÐÜ$)§K¢KÐ0A×DTÑDTÐVZ×VeÑVeÐCfÐlnÔ$oÑ!ˆØ#×-Ñ-¨a°Ó3ˆØ!×)Ñ)¨*À"ÓEˆäŸš L×3HÑ3HÈ$×J_ÑJ_Ð2`ÐfhÔi‰ˆ�à×/Ñ/°Ó>ˆÜŸš ]°T×5FÑ5FÈ×H]ÑH]Ð4^ÐdfÔg‰ˆ�à×$Ñ$ WÓ-×2Ñ2°:¸qÀ*×N_ÑN_Ó`ˆØ—
‘
˜: q¨*×6KÑ6KÓLˆà&‰ˆˆSÜ6°u¸eÀSÈ#ÓN‰ˆˆuä—y’y &¨% °bÔ9ˆà#'§>¡>°&¸%Ó#@Ñ ˆ
�Lð Ò&Ø'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜ð —|‘|ØØØØ˜:Õ&ØØ+ð $ó 
ˆð ˆØ�;‰;×+Ñ+Ð/@Ô@à×%Ñ% z¸z×?OÑ?OÐPQÕ?RÐ&SÔ[`×[eÑ[eÐ%Ófß‘˜˜\×.Ñ.Ó0°%Ó8ß‘˜1“ð ð ×#Ñ#¤u§z¡zÔ1Ø!/°:°+Õ!=‘à!/×!;Ñ!;¸JÌÏÊÐTa×TgÑTgÓHh×HlÑHlÓ!m‘à)ˆNä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ
%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð
%
ð —L‘Lð
%
ð #ð
%
ð ñ
%
Ñ!ˆ�\ð "×)Ñ)¨*À"ÓE×PÑPÓRˆà"¤U§]¢]°;×3DÑ3DÓ3FÓ%GÕG×KÑKÈK×L]ÑL]Ó^ˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r.   )r™  r_   r8   r¨  rŸ  r¥  r¤  r¦  r›  rÏ   rš  rx  r§  r¢  r¡  r£  rÙ   rž  r�  r×   rr  rœ  )NN)rS   rT   rU   rV   rƒ   r1   r²  rK   rW   rX   rY   rZ   s   @@r+   r“  r“    s1   ù‡ € ñ÷
,6ó ,6÷\(ð (÷&T)÷ T)ò T)r.   r“  c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚAXK2DecoderLayeri   c                ó&   <€ V ^8„  d   QhRS[ RS[/# r•  r–  )r)   r*   s   "€r+   r,   ÚAXK2DecoderLayer.__annotate__¡  s   ø€ ÷ 
ñ 
™zð 
±cñ 
r.   c                óŒ  <€ \         SV `  4        VP                  V n        \        W4      V n        VP
                  V,          R 8X  d   \        V4      M
\        V4      V n        \        V4      V n
        VP
                  V,          R 8X  d   \        V4      V n        R# \        VP                  VP                  R7      V n        R# )Úsparserw   N)r0   r1   r8   r“  Ú	self_attnÚmlp_layer_typesrY  rP  rz   rs   Úinput_layernormr#   rx   Úpost_attention_layernormrà   s   &&&€r+   r1   ÚAXK2DecoderLayer.__init__¡  s¢   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔÜ& vÓ9ˆŒà&,×&<Ñ&<¸YÕ&GÈ8Ô&S”7˜6”?ÔY`ÐagÓYhˆŒÜ/°Ó7ˆÔð ×%Ñ% iÕ0°HÔ<ô ˜VÓ$ð 	Ö%ô ˜V×/Ñ/°V×5HÑ5HÔIð 	Ö%r.   c                ó.  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[ P                  S[ P                  3,          R,          RS[S[,          R	S[ P                  /# )
r%   r=   rå   Nr·   ræ   Ú	use_cacherä   r£   r'   )r3   r?   Ú
LongTensorr   rð   rN   r   r   )r)   r*   s   "€r+   r,   rÍ  ®  sœ   ø€ ÷ ñ á—|‘|ðñ Ÿ™ tÕ+ðñ ×&Ñ&¨Õ-ð	ñ
  �ðñ ˜$•;ðñ #¡5§<¡<±·±Ð#=Õ>ÀÕEðñ Ñ+Õ,ðñ 
�‰ñr.   c                óÔ   € TpV P                  V4      pV P                  ! RR VRVRVRVRVRV/VB w  rW�,           pTpV P                  V4      pV P                  V4      pW�,           pV# )r=   rå   r·   ræ   rÖ  rä   © )rÒ  rÐ  rÓ  rz   )
r7   r=   rå   r·   ræ   rÖ  rä   r£   Úresidualrø   s
   &&&&&&&,  r+   rK   ÚAXK2DecoderLayer.forward®  s©   € ð !ˆØ×,Ñ,¨]Ó;ˆàŸ>š>ñ 
Ø'ð
à)ð
ð &ð
ð ,ð	
ð
  ð
ð !4ð
ð ñ
Ñˆð !Õ0ˆð !ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ Õ0ˆØÐr.   )r8   rÒ  rz   rÓ  rÐ  )NNNFNrq   rZ   s   @@r+   rË  rË     s   ù‡ € ÷
ó 
÷÷ ò r.   rË  c                   ó¬   a a€ ] tR tRt oRtRtR.tR.tRtRt	Rt
RtRtR]R]/tR	.tR
.tR.t]P(                  ! 4       V 3R l4       tV3R ltRtVtV ;t# )ÚAXK2PreTrainedModeliÎ  ÚmodelTrË  ræ   Fr=   Ú
attentionsr  r•   zindexer.weights_projc                ó  <€ \         SV `  V4       \        V\        4      '       dZ   \        P
                  ! VP                  R V P                  P                  R7       \        P                  ! VP                  4       R# \        V\        4      '       dq   \        P
                  ! VP                  R V P                  P                  R7       \        P
                  ! VP                  R V P                  P                  R7       R# R# )r"  )rG   ÚstdN)r0   Ú_init_weightsr°   r  ÚinitÚnormal_r5   r_   Úinitializer_rangeÚzeros_r  r,  r1  r2  )r7   rn  r9   s   &&€r+   râ  Ú!AXK2PreTrainedModel._init_weightsã  sŸ   ø€ ä‰Ñ˜fÔ%Ü�fœn×-Ò-Ü�LŠL˜Ÿ™¨S°d·k±k×6SÑ6SÕTÜ�KŠK˜×6Ñ6Ö7Ü˜¤×,Ò,Ü�LŠL˜×,Ñ,°3¸D¿K¹K×<YÑ<YÕZÜ�LŠL˜×)Ñ)°¸¿¹×9VÑ9V×Wñ -r.   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r^   r   )r)   r*   s   "€r+   r,   Ú AXK2PreTrainedModel.__annotate__Î  s   ø‡ ‚ áÑò r.   rÙ  )rS   rT   rU   rV   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendrË  r“  Ú_can_record_outputsÚ_keep_in_fp32_modules_strictÚ"_keys_to_ignore_on_load_unexpectedÚ_keep_in_fp32_modulesr3   r¾   râ  Ú__annotate_func__rW   rX   rY   rZ   s   @@r+   rÝ  rÝ  Î  s’   ù‡ € ð  ÐØ&*Ð#Ø+Ð,ÐØ#4Ð"5ÐØ ÐØ€NØÐà!ÐØ"&ÐàÐ)Ø�mðÐð %>Ð#>Ð Ø*4¨Ð&Ø3Ð4Ðà
‡]‚]ƒ_ôXó ðX÷- … r.   rÝ  c                   ór   a a€ ] tR tRt oV3R lV 3R llt]]]RV3R lR ll4       4       4       tRt	Vt
V ;t# )Ú	AXK2Modeliî  c                ó    <€ V ^8„  d   QhRS[ /# r^   r   )r)   r*   s   "€r+   r,   ÚAXK2Model.__annotate__ð  s   ø€ ÷ ñ ™zñ r.   c           	     ó  <€ \         SV `  V4       VP                  V n        VP                  V n        \
        P                  ! VP                  VP                  V P                  4      V n        \
        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n        \        VP                  VP                  R 7      V n        \#        VR7      V n        RV n        V P)                  4        R# u upi )rw   ©r_   FN)r0   r1   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingr8   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersrË  Úlayersr#   rx   ry   r…   Ú
rotary_embÚgradient_checkpointingÚ	post_initrà   s   && €r+   r1   ÚAXK2Model.__init__ð  sÊ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸLšL¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m’mÜBGÈ×H`ÑH`ÔBaÓbÑBa°YÔ˜fÖ0ÑBaÑbó
ˆŒô   × 2Ñ 2¸×8KÑ8KÔLˆŒ	Ü-°VÔ<ˆŒØ&+ˆÔ#ð 	�‰Öùò cs   ÂDc                óþ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[S[,          R	S[/# )
r%   Ú	input_idsNrå   r·   ræ   Úinputs_embedsrÖ  r£   r'   )	r3   r×  r?   r   ÚFloatTensorrð   r   r   r   )r)   r*   s   "€r+   r,   rû    s“   ø€ ÷ 6
ñ 6
á×#Ñ# dÕ*ð6
ñ Ÿ™ tÕ+ð6
ñ ×&Ñ&¨Õ-ð	6
ñ
  �ð6
ñ ×(Ñ(¨4Õ/ð6
ñ ˜$•;ð6
ñ Ñ+Õ,ð6
ñ 
!ñ6
r.   c                ó&  € VR J VR J,          '       d   \        R4      hVf   V P                  V4      pV'       d   Vf   \        V P                  R7      pVfb   Ve   VP	                  4       M^ p\
        P                  ! VP                  ^,          VP                  R7      V,           pVP                  ^ 4      p\        T;p	\        4      '       g&   RV P                  RVRVRVRVR	R
/p
R\        R/ V
B /p	TpV P                  W³R7      p\        V P                  R V P                  P                    4       F9  w  rÞV! V3RW�P                  P"                  V,          ,          RVRVRVRV/VB pK;  	  V P%                  V4      p\'        VVR7      # )Nz:You must specify exactly one of input_ids or inputs_embedsrý  )r†   r_   r  rå   ræ   r·   Úallow_is_causal_skipFÚdeepseek_sparse_attention)r·   rä   rÖ  )Úlast_hidden_stateræ   rÙ  )Ú
ValueErrorr  r   r_   Úget_seq_lengthr3   r¢   rO   r†   rÅ   r°   Údictr   r  Ú	enumerater  r  Úlayer_typesry   r   )r7   r  rå   r·   ræ   r  rÖ  r£   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsr=   rä   ÚiÚdecoder_layers   &&&&&&&,       r+   rK   ÚAXK2Model.forward   sµ  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ Ø*.×*;Ñ*;¸IÓ*FˆMç˜Ò0Ü*°$·+±+Ô>ˆOàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLô °Ð?Ð-Ä×FÒFà˜$Ÿ+™+Ø Ø  .Ø! ?Ø Ø&¨ðˆKð $?Ô@RÑ@aÐU`Ñ@aÐ"bÐà%ˆØ"Ÿo™o¨m˜oÓWÐä )¨$¯+©+Ð6U¸¿¹×8UÑ8UÐ*VÖ WÑˆAÙ)Øñà2·;±;×3JÑ3JÈ1Õ3MÕNðð %8ðð *ð	ð
 !0ðð $ðð ñŠMñ !Xð Ÿ	™	 -Ó0ˆÜ&Ø+Ø+ô
ð 	
r.   )r  r  r  ry   rÿ  r  r   )NNNNNN)rS   rT   rU   rV   r1   r   r   r   rK   rW   rX   rY   rZ   s   @@r+   rù  rù  î  s=   ù‡ € ÷ó ð   ØØ÷6
ñ 6
ó ó ó  ÷6
ð 6
r.   rù  c                   ó„   a a€ ] tR tRt oRR/tRR/tRR.R.3/tRR/tV 3R	 lt]	]
RV3R
 lR ll4       4       tRtVtV ;t# )ÚAXK2ForCausalLMi<  zlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputr=   ÚlogitsÚkeep_full_weightc                óì   <€ \         SV `  V4       \        V4      V n        VP                  V n        \
        P                  ! VP                  VP                  R R7      V n        V P                  4        R# rb   )
r0   r1   rù  rÞ  r   r   rg   r8   r   r	  rk   s   &&€r+   r1   ÚAXK2ForCausalLM.__init__C  sU   ø€ Ü‰Ñ˜Ô Ü˜vÓ&ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒð 	�‰Ör.   c                óP  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[S[ P                  ,          R
S[S[,          RS[	/
# )r%   r  Nrå   r·   ræ   r  ÚlabelsrÖ  Úlogits_to_keepr£   r'   )
r3   r×  r?   r   r  rð   rÐ   r   r   r   )r)   r*   s   "€r+   r,   ÚAXK2ForCausalLM.__annotate__N  s»   ø€ ÷ 6
ñ 6
á×#Ñ# dÕ*ð6
ñ Ÿ™ tÕ+ð6
ñ ×&Ñ&¨Õ-ð	6
ñ
  �ð6
ñ ×(Ñ(¨4Õ/ð6
ñ × Ñ  4Õ'ð6
ñ ˜$•;ð6
ñ ™eŸl™lÕ*ð6
ñ Ñ+Õ,ð6
ñ 
 ñ6
r.   c	                ó–  € V P                   ! RRVRVRVRVRVRV/V	B p
V
P                  p\        V\        4      '       d   \	        V) R4      MTpV P                  VRVR3,          4      pRpVe-   V P                  ! RR	VR
VRV P                  P                  /V	B p\        VVV
P                  V
P                  V
P                  R7      # )ai  
Example:

```python
>>> from transformers import AutoTokenizer, AXK2ForCausalLM

>>> model = AXK2ForCausalLM.from_pretrained("meta-axk2/AXK2-2-7b-hf")
>>> tokenizer = AutoTokenizer.from_pretrained("meta-axk2/AXK2-2-7b-hf")

>>> prompt = "Hey, are you conscious? Can you talk to me?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
```r  rå   r·   ræ   r  rÖ  Nr®   r"  r'  r   )Úlossr"  ræ   r=   rß  rÙ  )rÞ  r  r°   rÐ   Úslicer   Úloss_functionr_   r   r   ræ   r=   rß  )r7   r  rå   r·   ræ   r  r'  rÖ  r(  r£   Úoutputsr=   Úslice_indicesr"  r+  s   &&&&&&&&&,     r+   rK   ÚAXK2ForCausalLM.forwardL  sÿ   € ð> ,0¯:ª:ñ ,
Øð,
à)ð,
ð &ð,
ð ,ð	,
ð
 (ð,
ð  ð,
ð ñ,
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒØ×%Ò%Ñp¨VÐp¸FÐpÈtÏ{É{×OeÑOeÐpÐioÑpˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r.   )r   rÞ  r   )NNNNNNNr   )rS   rT   rU   rV   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr1   r   r   rK   rW   rX   rY   rZ   s   @@r+   r  r  <  sf   ù‡ € à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÐ/Ð0€Jõð Ø÷6
ñ 6
ó ó ÷6
ð 6
r.   r  c                   ó   € ] tR tRtRtR# )ÚAXK2ForSequenceClassificationi‡  rÙ  N©rS   rT   rU   rV   rW   rÙ  r.   r+   r6  r6  ‡  ó   † Ûr.   r6  c                   ó   € ] tR tRtRtR# )ÚAXK2ForTokenClassificationi‹  rÙ  Nr7  rÙ  r.   r+   r:  r:  ‹  r8  r.   r:  )r  )r"  )r  r  )Nr  )rÝ  rù  r  r6  r:  )Nr�  Úcollections.abcr   r3   Útorch.nn.functionalr   r<  rî   Ú r   rã  Úactivationsr   Úcache_utilsr   r   Ú
generationr	   Úintegrationsr
   r   Úmasking_utilsr   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   r   r   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   Úutils.deprecationr   Úutils.genericr   r   r   Úutils.output_capturingr   Úconfiguration_axk2r    rt  r#   r\   rs   r…   rÂ   rË   rÍ   r  r,  rP  rY  rl  r  r…  r‹  r‘  r“  rË  rÝ  rù  r  r6  r:  Ú__all__rÙ  r.   r+   Ú<module>rO     sû  ðó* Ý $ã ß Ð Ý å &Ý !ß .Ý )ß SÝ /Ý B÷ñ ÷
 Pß Kß FÝ &ß 7Ý 0ß YÑ YÝ 5Ý *ñ ˜YÓ'ôJ�"—)‘)ó Jó (ðJô(�"—)‘)ô ôD�r—y‘yô Dô"6<˜"Ÿ)™)ô 6<òr(ñ Ð-Ó.óó /ðô2^G�"—)‘)ô ^GôB0!�R—Y‘Yô 0!ðf ô$#�"—)‘)ó $#ó ð$#ôNˆb�i‰iô ô ˆb�i‰iô õ0	U÷%ô20òô$ôN[)�B—I‘Iô [)ô|+Ð1ô +ð\ ôX˜/ó Xó ðXð> ôJ
Ð#ó J
ó ðJ
ðZ ôG
Ð)¨?ó G
ó ðG
ôT	Ð$DÐFYô 	ô	Ð!>Ð@Sô 	ò‚r.   