+
    œ°³j&D  ã                   óf  € R t ^ RIHt ^ RIt^ RIHt ^RIHt ^RIH	t	H
t
 ^RIHt ^RIHtHt ^RIHt ^R	IHtHt ^R
IHtHt ^RIHt ^RIHtHtHtHt ^RIH t  ^RI!H"t"H#t# ^RI$H%t% ^RI&H't'H(t(H)t)H*t*H+t+ ^RI,H-t-H.t. ^RI/H0t0 ]Pb                  ! ]24      t3 ! R R])4      t4 ! R R]%4      t5 ! R R].4      t6 ! R R]Pn                  4      t8 ! R R]-4      t9 ! R R]Pn                  4      t: ! R R ]'4      t; ! R! R"]4      t< ! R# R$]4      t=] ! R% R&]=4      4       t> ! R' R(](]=]4      t?. R)Ot@R# )*zPyTorch AFMoE model.)ÚCallableN)Únn)Úinitialization)ÚCacheÚDynamicCache)ÚGenerationMixin)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚGradientCheckpointingLayer)ÚMoeCausalLMOutputWithPastÚMoeModelOutputWithPast)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Úmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputs)ÚGptOssRMSNorm)ÚLlamaAttentionÚLlamaForCausalLMÚLlamaRotaryEmbeddingÚapply_rotary_pos_embÚeager_attention_forward)ÚQwen2MoeExpertsÚQwen2MoeMLP©ÚAfmoeConfigc                   ó   € ] tR t^/tRtR# )ÚAfmoeRotaryEmbedding© N©Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__static_attributes__r#   ó    Úh/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/afmoe/modular_afmoe.pyr"   r"   /   ó   † Ûr*   r"   c                   ó   € ] tR t^3tRtR# )ÚAfmoeRMSNormr#   Nr$   r#   r*   r+   r.   r.   3   r,   r*   r.   c                   ó   € ] tR t^7tRtR# )ÚAfmoeMLPr#   Nr$   r#   r*   r+   r0   r0   7   r,   r*   r0   c                   óH   a a€ ] tR t^;t oRtV 3R ltV3R lR ltRtVtV ;t	# )ÚAfmoeTokenChoiceRouterz�
Token-choice top-K router for MoE routing.

This router assigns each token to the top-K experts based on sigmoid scores, matching the released checkpoints.
c                óú   <€ \         SV `  4        Wn        VP                  V n        VP
                  V n        VP                  V n        \        P                  ! VP                  VP
                  R R7      V n
        R# ©F©ÚbiasN)ÚsuperÚ__init__ÚconfigÚnum_experts_per_tokÚtop_kÚnum_expertsÚroute_scaler   ÚLinearÚhidden_sizeÚgate©Úselfr9   Ú	__class__s   &&€r+   r8   ÚAfmoeTokenChoiceRouter.__init__B   s\   ø€ Ü‰ÑÔØŒØ×/Ñ/ˆŒ
Ø!×-Ñ-ˆÔØ!×-Ñ-ˆÔÜ—I’I˜f×0Ñ0°&×2DÑ2DÈ5ÔQˆŽ	r*   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )é   Úhidden_statesÚexpert_bias)ÚtorchÚTensor)ÚformatÚ__classdict__s   "€r+   Ú__annotate__Ú#AfmoeTokenChoiceRouter.__annotate__J   s#   ø€ ÷ ;ñ ;¡U§\¡\ð ;ÁÇÁñ ;r*   c                ó¸  € VP                   w   r4VP                  RV4      pV P                  V4      P                  \        P
                  4      p\        P                  ! V4      p\        P                  ! Wb,           V P                  ^R7      w  r7VP                  ^VR7      pVP                  RRR7      R,           p	W‰,          pW€P                  ,          pWXV3# )é   )ÚkÚdim)rR   ÚindexT)rR   Úkeepdimg#B’¡œÇ;éÿÿÿÿ)ÚshapeÚviewr@   ÚtorI   Úfloat32ÚsigmoidÚtopkr;   ÚgatherÚsumr=   )
rB   rG   rH   Ú_Ú
hidden_dimÚrouter_logitsÚscoresÚselected_expertsÚ
top_scoresÚdenominators
   &&&       r+   ÚforwardÚAfmoeTokenChoiceRouter.forwardJ   s¸   € Ø(×.Ñ.Ñˆˆ1Ø%×*Ñ*¨2¨zÓ:ˆàŸ	™	 -Ó0×3Ñ3´E·M±MÓBˆÜ—’˜}Ó-ˆä#Ÿjšj¨Õ)=ÀÇÁÐQRÔSÑˆØ—]‘] qÐ0@�]ÓAˆ
Ø —n‘n¨°T�nÓ:¸UÕBˆØÕ-ˆ
Ø×"2Ñ"2Õ2ˆ
ØÐ*:Ð:Ð:r*   )r9   r@   r<   r=   r;   ©
r%   r&   r'   r(   Ú__doc__r8   re   r)   Ú__classdictcell__Ú__classcell__©rC   rL   s   @@r+   r2   r2   ;   s   ù‡ € ñõR÷;÷ ;ð ;r*   r2   c                   ó   € ] tR t^YtRtR# )ÚAfmoeExpertsr#   Nr$   r#   r*   r+   rm   rm   Y   r,   r*   rm   c                   ó<   a a€ ] tR t^]t oRtV 3R ltR tRtVtV ;t	# )ÚAfmoeSparseMoeBlockz¾
Mixture of Experts (MoE) module for AFMoE.

This module implements a sparse MoE layer with both shared experts (always active) and
routed experts (activated based on token-choice routing).
c                ó<  <€ \         SV `  4        Wn        \        V4      V n        \        WP                  VP                  ,          4      V n        \        V4      V n
        \        P                  ! \        P                  ! VP                  4      R R7      V n        R# )F)Úrequires_gradN)r7   r8   r9   r2   Úrouterr0   Úmoe_intermediate_sizeÚnum_shared_expertsÚshared_expertsrm   Úexpertsr   Ú	ParameterrI   Úzerosr<   rH   rA   s   &&€r+   r8   ÚAfmoeSparseMoeBlock.__init__e   sl   ø€ Ü‰ÑÔØŒÜ,¨VÓ4ˆŒÜ& v×/KÑ/KÈf×NgÑNgÕ/gÓhˆÔÜ# FÓ+ˆŒÜŸ<š<¬¯ª°F×4FÑ4FÓ(GÐW\Ô]ˆÖr*   c                ó  € VP                   w  r#pVP                  RV4      pV P                  WP                  4      w  rgpV P	                  V4      P                  W#V4      p	V P                  WXV4      P                  W#V4      p
Wš,           # )rP   rU   )rV   rW   rr   rH   ru   rv   )rB   rG   Ú
batch_sizeÚseq_lenr_   Úhidden_states_flatr^   rc   rb   Úshared_outputÚrouted_outputs   &&         r+   re   ÚAfmoeSparseMoeBlock.forwardm   sŽ   € Ø*7×*=Ñ*=Ñ'ˆ
˜ZØ*×/Ñ/°°JÓ?Ðð +/¯+©+°m×EUÑEUÓ*VÑ'ˆÐ'ð ×+Ñ+Ð,>Ó?×DÑDÀZÐZdÓeˆØŸ™Ð%7È:ÓV×[Ñ[Ø ó
ˆð Õ,Ð,r*   )r9   rH   rv   rr   ru   rg   rk   s   @@r+   ro   ro   ]   s   ù‡ € ñõ^÷-ò -r*   ro   c                   óX   a a€ ] tR t^|t oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )ÚAfmoeAttentiona6  
Multi-headed attention module with optional sliding window and gating.

This attention mechanism supports both full attention and sliding window attention,
and includes Q/K normalization and gating of the output. It inherits from [`LlamaAttention`] to minimize the amount
of custom logic we need to maintain.
c                ó&   <€ V ^8„  d   QhRS[ RS[/# ©rF   r9   Ú	layer_idx©r    Úint)rK   rL   s   "€r+   rM   ÚAfmoeAttention.__annotate__…   s   ø€ ÷ 	oñ 	o™{ð 	o±sñ 	or*   c                óÄ  <€ \         SV `  W4       VP                  V,          R 8H  V n        V P                  '       d   VP                  MRV n        \        V P                  VP                  R7      V n        \        V P                  VP                  R7      V n	        \        P                  ! VP                  VP                  V P                  ,          RR7      V n        R# )Úsliding_attentionN©ÚepsFr5   )r7   r8   Úlayer_typesÚis_local_attentionÚsliding_windowr.   Úhead_dimÚrms_norm_epsÚq_normÚk_normr   r>   r?   Únum_attention_headsÚ	gate_proj©rB   r9   r…   rC   s   &&&€r+   r8   ÚAfmoeAttention.__init__…   s¤   ø€ Ü‰Ñ˜Ô+ð #)×"4Ñ"4°YÕ"?ÐCVÑ"VˆÔØ7;×7N×7NÐ7N˜f×3Ò3ÐTXˆÔä" 4§=¡=°f×6IÑ6IÔJˆŒÜ" 4§=¡=°f×6IÑ6IÔJˆŒÜŸš 6×#5Ñ#5°v×7QÑ7QÐTX×TaÑTaÕ7aÐhmÔnˆŽr*   c                ó  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  S[ P                  3,          /# )rF   rG   Úposition_embeddingsÚattention_maskNÚpast_key_valuesÚkwargsÚreturn)rI   rJ   Útupler   r   r   )rK   rL   s   "€r+   rM   rˆ   �   s‚   ø€ ÷ .)ñ .)á—|‘|ð.)ñ #¡5§<¡<±·±Ð#=Õ>ð.)ñ Ÿ™ tÕ+ð	.)ñ
  �ð.)ñ Ñ+Õ,ð.)ñ 
‰u�|‰|™UŸ\™\Ð)Õ	*ñ.)r*   c                óö  € VP                   R R p. VORNV P                  N5pV P                  V4      P                  V4      pV P	                  V4      P                  V4      p	V P                  V4      P                  V4      p
V P                  V4      pV P                  V4      P                  ^^4      pV P                  V	4      P                  ^^4      p	V
P                  ^^4      p
V P                  '       d   Vw  rÍ\        W‰WÍ4      w  r‰Ve   VP                  WšV P                  4      w  rš\        P                  ! V P                   P"                  \$        4      pT! V VV	V
3RTRV P&                  '       g   RMV P(                  RV P*                  RV P,                  /VB w  ppVP                  ! . VORN5!  P/                  4       pV\0        P2                  ! V4      ,          pV P5                  V4      pVV3# )Nrš   Údropoutç        Úscalingr�   rU   )rV   r�   Úq_projrW   Úk_projÚv_projr•   r’   Ú	transposer“   rŽ   r   Úupdater…   r   Úget_interfacer9   Ú_attn_implementationr   ÚtrainingÚattention_dropoutr¢   r�   Ú
contiguousrI   rZ   Úo_proj)rB   rG   r™   rš   r›   rœ   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚgate_statesÚcosÚsinÚattention_interfaceÚoutputÚattn_weightsÚattn_outputs   &&&&&,            r+   re   ÚAfmoeAttention.forward�   së  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓDˆØ—[‘[ Ó/×4Ñ4°\ÓBˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓDˆØ—n‘n ]Ó3ˆà—{‘{ <Ó0×:Ñ:¸1¸aÓ@ˆØ—[‘[ Ó,×6Ñ6°q¸!Ó<ˆ
Ø#×-Ñ-¨a°Ó3ˆà×"×"Ð"Ø*‰HˆCÜ';¸LÐVYÓ'_Ñ$ˆLàÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ  3ØØØØñ	
 
ð
 *ð
 
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð
 
ð —L‘Lð
 
ð  ×.Ñ.ð
 
ð ñ
 
Ñˆ�ð —’Ð.˜kÐ.¨2Ó.×9Ñ9Ó;ˆØœ%Ÿ-š-¨Ó4Õ4ˆØ—k‘k &Ó)ˆØ˜LÐ(Ð(r*   )r•   rŽ   r“   r’   r�   )Nrg   rk   s   @@r+   r‚   r‚   |   s%   ù‡ € ñ÷	oó 	o÷.)÷ .)ò .)r*   r‚   c                   óX   a a€ ] tR t^Át oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )ÚAfmoeDecoderLayerz¼
AFMoE decoder layer with dual normalization.

This layer applies self-attention followed by either a dense MLP or MoE block,
with dual normalization (pre and post) around each component.
c                ó&   <€ V ^8„  d   QhRS[ RS[/# r„   r†   )rK   rL   s   "€r+   rM   ÚAfmoeDecoderLayer.__annotate__É   s   ø€ ÷ (ñ (™{ð (±sñ (r*   c                ó2  <€ \         SV `  4        VP                  V n        W n        \	        WR 7      V n        \        VP                  VP                  R7      V n        \        VP                  VP                  R7      V n	        \        VP                  VP                  R7      V n
        \        VP                  VP                  R7      V n        W!P                  8¬  V n        V P                  '       d   \        V4      V n        R# \!        V4      V n        R# ))r9   r…   r‹   N)r7   r8   r?   r…   r‚   Ú	self_attnr.   r‘   Úinput_layernormÚpost_attention_layernormÚpre_mlp_layernormÚpost_mlp_layernormÚnum_dense_layersÚmoe_enabledro   Úmlpr0   r–   s   &&&€r+   r8   ÚAfmoeDecoderLayer.__init__É   sÔ   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØ"Œä'¨vÔKˆŒô  ,¨F×,>Ñ,>ÀF×DWÑDWÔXˆÔÜ(4°V×5GÑ5GÈV×M`ÑM`Ô(aˆÔ%ô ".¨f×.@Ñ.@Àf×FYÑFYÔ!ZˆÔÜ".¨v×/AÑ/AÀv×GZÑGZÔ"[ˆÔð %×(?Ñ(?Ñ?ˆÔØ××ÐÜ*¨6Ó2ˆDŽHä Ó'ˆDŽHr*   c                ó.  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[ P                  S[ P                  3,          R,          RS[S[,          R	S[ P                  /# )
rF   rG   rš   NÚposition_idsr›   Ú	use_cacher™   rœ   r�   )	rI   rJ   Ú
LongTensorr   Úboolrž   r   r   ÚFloatTensor)rK   rL   s   "€r+   rM   r¾   ß   sž   ø€ ÷ !ñ !á—|‘|ð!ñ Ÿ™ tÕ+ð!ñ ×&Ñ&¨Õ-ð	!ñ
  �ð!ñ ˜$•;ð!ñ #¡5§<¡<±·±Ð#=Õ>ÀÕEð!ñ Ñ+Õ,ð!ñ 
×	Ñ	ñ!r*   c                ó  € TpV P                  V4      pV P                  ! RR VRVRVRVRVRV/VB w  rV P                  V4      pW�,           pTpV P                  V4      pV P	                  V4      pV P                  V4      pW�,           pV# )rG   rš   rÊ   r›   rË   r™   r#   )rÁ   rÀ   rÂ   rÃ   rÇ   rÄ   )
rB   rG   rš   rÊ   r›   rË   r™   rœ   Úresidualr^   s
   &&&&&&&,  r+   re   ÚAfmoeDecoderLayer.forwardß   sË   € ð !ˆð ×,Ñ,¨]Ó;ˆØŸ>š>ñ 
Ø'ð
à)ð
ð &ð
ð ,ð	
ð
  ð
ð !4ð
ð ñ
Ñˆð ×5Ñ5°mÓDˆØ Õ0ˆð !ˆØ×.Ñ.¨}Ó=ˆØŸ™ Ó/ˆØ×/Ñ/°Ó>ˆà Õ0ˆØÐr*   )	r?   rÁ   r…   rÇ   rÆ   rÂ   rÄ   rÃ   rÀ   )NNNNNrg   rk   s   @@r+   r¼   r¼   Á   s#   ù‡ € ñ÷(ó (÷,!÷ !ò !r*   r¼   c                   ó’   a a€ ] tR tRt oRtRtR.tR.tR]! ]	^ R7      R]
R	]/t. ROtR
tR
tR
tR
tR
tR
tV 3R ltV3R ltRtVtV ;t# )ÚAfmoePreTrainedModeli  zz
An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
models.
Úmodelr¼   r›   r`   )rS   rG   Ú
attentionsTc                ó  <€ \         SV `  V4       V P                  P                  p\	        V\
        4      '       dI   \        P                  ! VP                  RVR7       \        P                  ! VP                  RVR7       R# \	        V\        4      '       d-   \        P                  ! VP                  P                  4       R# \	        V\        4      '       d#   \        P                  ! VP                  4       R# R# )zInitialize the weightsr¡   )ÚmeanÚstdN)r7   Ú_init_weightsr9   Úinitializer_rangeÚ
isinstancerm   ÚinitÚnormal_Úgate_up_projÚ	down_projr2   Úzeros_r@   Úweightro   rH   )rB   ÚmodulerØ   rC   s   && €r+   rÙ   Ú"AfmoePreTrainedModel._init_weights#  s¡   ø€ ä‰Ñ˜fÔ%Ø�k‰k×+Ñ+ˆÜ�fœl×+Ò+Ü�LŠL˜×,Ñ,°3¸CÕ@Ü�LŠL˜×)Ñ)°¸×=Ü˜Ô 6×7Ò7Ü�KŠK˜Ÿ™×*Ñ*Ö+Ü˜Ô 3×4Ò4Ü�KŠK˜×*Ñ*Ö+ñ 5r*   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # ©rF   r9   r   )rK   rL   s   "€r+   rM   Ú!AfmoePreTrainedModel.__annotate__  s   ø‡ ‚ ñ Ñò r*   r#   )rÁ   rÂ   rÃ   rÄ   r’   r“   ÚnormrH   )r%   r&   r'   r(   rh   Úbase_model_prefixÚ_no_split_modulesÚ_skip_keys_device_placementr   r2   r¼   r‚   Ú_can_record_outputsÚ_keep_in_fp32_modulesÚ_supports_sdpaÚ_supports_flash_attnÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendÚsupports_gradient_checkpointingrÙ   Ú__annotate_func__r)   ri   rj   rk   s   @@r+   rÓ   rÓ     s~   ù‡ € ñð  ÐØ,Ð-ÐØ#4Ð"5Ðà™Ð(>ÀaÔHØÐ*Ø�nðÐò
	Ðð €NØÐØÐØ!ÐØ"&ÐØ&*Ð#õ
,÷A … r*   rÓ   c                   óv   a a€ ] tR tRt oRtV3R lV 3R llt]]]RV3R lR ll4       4       4       t	Rt
VtV ;t# )	Ú
AfmoeModeli0  z‹
Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`AfmoeDecoderLayer`]

Args:
    config: AfmoeConfig
c                ó    <€ V ^8„  d   QhRS[ /# rå   r   )rK   rL   s   "€r+   rM   ÚAfmoeModel.__annotate__9  s   ø€ ÷ ñ ™{ñ r*   c           	     ó  <€ \         SV `  V4       VP                  V n        VP                  V n        \
        P                  ! VP                  VP                  V P                  4      V n        \
        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n        \        VP                  VP                  R 7      V n        \#        VR7      V n        RV n        V P)                  4        R# u upi )r‹   ©r9   FN)r7   r8   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingr?   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersr¼   Úlayersr.   r‘   rç   r"   Ú
rotary_embÚgradient_checkpointingÚ	post_initr–   s   && €r+   r8   ÚAfmoeModel.__init__9  sÈ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸLšL¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m’mÜCHÈ×IaÑIaÔCbÓcÑCb°iÔ˜vÖ1ÑCbÑcó
ˆŒô ! ×!3Ñ!3¸×9LÑ9LÔMˆŒ	Ü.°fÔ=ˆŒØ&+ˆÔ#à�‰Öùò ds   ÂDc                ó  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[,          R	S[S[	,          /# )
rF   Ú	input_idsNrš   Úinputs_embedsrÊ   r›   rË   rœ   r�   )
rI   rÌ   rJ   rÎ   r   rÍ   r   r   rž   r   )rK   rL   s   "€r+   rM   r÷   K  s™   ø€ ÷ <
ñ <
á×#Ñ# dÕ*ð<
ñ Ÿ™ tÕ+ð<
ñ ×(Ñ(¨4Õ/ð	<
ñ
 ×&Ñ&¨Õ-ð<
ñ  �ð<
ñ ˜$•;ð<
ñ Ñ+Õ,ð<
ñ 
Ñ'Õ	'ñ<
r*   c                ó   € VR J VR J,          '       d   \        R4      hV'       d   Vf   \        V P                  R7      pVf   V P                  V4      pVfb   Ve   VP	                  4       M^ p\
        P                  ! VP                  ^,          VP                  R7      V,           pVP                  ^ 4      p\        T;p	\        4      '       g-   RV P                  RVRVRV/p
R\        R/ V
B R	\        R/ V
B /p	TpV P                  P                  '       d$   W°P                  P                  R
,          ,          pV P!                  W´4      p\#        V P$                  4       F9  w  rÞV! V3RW�P                  P&                  V,          ,          RVRVRVRV/VB pK;  	  V P)                  V4      p\+        TV'       d   VR7      # R R7      # )Nz:You must specify exactly one of input_ids or inputs_embedsrù   )Údevicer9   r	  rš   r›   Úfull_attentionrŠ   g      à?rÊ   rË   r™   )Úlast_hidden_stater›   r#   )Ú
ValueErrorr   r9   rþ   Úget_seq_lengthrI   ÚarangerV   r  Ú	unsqueezerÛ   Údictr   r	   Úmup_enabledr?   r  Ú	enumerater  r�   rç   r   )rB   r  rš   r	  rÊ   r›   rË   rœ   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsrG   r™   ÚiÚdecoder_layers   &&&&&&&,       r+   re   ÚAfmoeModel.forwardH  sß  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZç˜Ò0Ü*°$·+±+Ô>ˆOàÒ Ø ×-Ñ-¨iÓ8ˆMàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLô °Ð?Ð-Ä×FÒFà˜$Ÿ+™+Ø Ø  .Ø! ?ð	ˆKð !Ô"4Ñ"C°{Ñ"CØ#Ô%FÑ%UÈÑ%Uð#Ðð
 &ˆð �;‰;×"×"Ð"Ø)¯[©[×-DÑ-DÀcÕ-IÕJˆMà"Ÿo™o¨mÓJÐä )¨$¯+©+Ö 6ÑˆAÙ)Øñà2·;±;×3JÑ3JÈ1Õ3MÕNðð *ðð !0ð	ð
 $ðð %8ðð ñŠMñ !7ð Ÿ	™	 -Ó0ˆÜ%Ø+ß/8˜Oô
ð 	
à>Bô
ð 	
r*   )rþ   r  r  rç   rû   r  rü   )NNNNNN)r%   r&   r'   r(   rh   r8   r   r   r   re   r)   ri   rj   rk   s   @@r+   rõ   rõ   0  sB   ù‡ € ñ÷ó ð ØØ÷<
ñ <
ó ó  ó ÷<
ð <
r*   rõ   c                   óx   a € ] tR tRt o RR/tRR/tRR.R.3/tRR/tR	 t]	]
RV 3R lR ll4       4       tRtV tR
# )ÚAfmoeForCausalLMiŠ  zlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputrG   ÚlogitsÚkeep_full_weightc                óö   € \         P                  W4       \        V4      V n        VP                  V n        \
        P                  ! VP                  VP                  R R7      V n        V P                  4        R# r4   )
rÓ   r8   rõ   rÔ   rü   r   r>   r?   r  r  )rB   r9   s   &&r+   r8   ÚAfmoeForCausalLM.__init__‘  sS   € Ü×%Ñ% dÔ3Ü Ó'ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒØ�‰Ör*   Nc                ód  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[R,          R
S[S[ P                  ,          RS[S[,          RS[	/# )rF   r  Nrš   rÊ   r›   r	  ÚlabelsrË   Úoutput_router_logitsÚlogits_to_keeprœ   r�   )
rI   rÌ   rJ   r   rÎ   rÍ   r‡   r   r   r   )rK   rL   s   "€r+   rM   ÚAfmoeForCausalLM.__annotate__š  sÉ   ø€ ÷ +
ñ +
á×#Ñ# dÕ*ð+
ñ Ÿ™ tÕ+ð+
ñ ×&Ñ&¨Õ-ð	+
ñ
  �ð+
ñ ×(Ñ(¨4Õ/ð+
ñ × Ñ  4Õ'ð+
ñ ˜$•;ð+
ñ # T�kð+
ñ ™eŸl™lÕ*ð+
ñ Ñ+Õ,ð+
ñ 
#ñ+
r*   c
                óÌ  € Ve   TMV P                   P                  pV P                  ! R
RVRVRVRVRVRVRV/V
B pVP                  p\	        V	\
        4      '       d   \        V	) R 4      MT	pV P                  VRVR3,          4      pR pVe   V P                  ! WæV P                  3/ V
B p\        VVVP                  VP                  VP                  VP                  R	7      # )Nr  rš   rÊ   r›   r	  rË   r%  :NNN)Úlossr  r›   rG   rÕ   r`   r#   )r9   r%  rÔ   r  rÛ   r‡   Úslicer  Úloss_functionrü   r   r›   rG   rÕ   r`   )rB   r  rš   rÊ   r›   r	  r$  rË   r%  r&  rœ   ÚoutputsrG   Úslice_indicesr  r)  s   &&&&&&&&&&,     r+   re   ÚAfmoeForCausalLM.forward˜  s  € ð  %9Ò$DÑ È$Ï+É+×JjÑJjð 	ð +/¯*ª*ñ 	+
Øð	+
à)ð	+
ð &ð	+
ð ,ð		+
ð
 (ð	+
ð  ð	+
ð "6ð	+
ð ñ	+
ˆð  ×1Ñ1ˆÜ8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒØ×%Ò% f°d·o±oÑPÈÑPˆDä(ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø!×/Ñ/ô
ð 	
r*   )r  rÔ   rü   )	NNNNNNNNé    )r%   r&   r'   r(   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr8   r   r   re   r)   ri   )rL   s   @r+   r  r  Š  sa   ø‡ € Ø*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HàÐ/Ð0€Jòð Ø÷+
ñ +
ó ó ö+
r*   r  )r  rõ   rÓ   )Arh   Úcollections.abcr   rI   r   Ú r   rÜ   Úcache_utilsr   r   Ú
generationr   Úmasking_utilsr   r	   Úmodeling_layersr
   Úmodeling_outputsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.genericr   Úutils.output_capturingr   r   Úgpt_oss.modeling_gpt_ossr   Úllama.modeling_llamar   r   r   r   r   Úqwen2_moe.modeling_qwen2_moer   r   Úconfiguration_afmoer    Ú
get_loggerr%   Úloggerr"   r.   r0   ÚModuler2   rm   ro   r‚   r¼   rÓ   rõ   r  Ú__all__r#   r*   r+   Ú<module>rH     s  ðñ å $ã Ý å &ß .Ý )ß RÝ 9ß Qß FÝ &ß RÓ RÝ 7ß EÝ 4÷õ ÷ HÝ ,ð 
×	Ò	˜HÓ	%€ô	Ð/ô 	ô	�=ô 	ô	ˆ{ô 	ô;˜RŸY™Yô ;ô<	�?ô 	ô-˜"Ÿ)™)ô -ô>B)�^ô B)ôJ?Ð2ô ?ôD*,˜?ô *,ðZ ôV
Ð%ó V
ó ðV
ôr;
Ð'Ð)=¸ô ;
ò|‚r*   