+
    œ°³j”q  ã                   ó&  € ^ RI Ht ^ RIt^ RIHt ^RIHt ^RIHt ^RI	H
t
Ht ^RIHt ^RIHtHtHt ^R	IHtHt ^R
IHt ^RIHtHt ^RIHtHt ^RIHtHt ^RI H!t! ^RI"H#t#H$t$H%t% ^RI&H't' ^RI(H)t)H*t* ^RI+H,t,H-t- ^RI.H/t/  ! R R]P`                  4      t1]! R4       ! R R]P`                  4      4       t2 ! R R]P`                  4      t3 ! R R]P`                  4      t4] ! R R]P`                  4      4       t5 ! R R ]P`                  4      t6R! t7]! R"4      R2R# l4       t8R$ R% lt9R3R& R' llt:]! ]84       ! R( R)]P`                  4      4       t; ! R* R+]4      t< ! R, R-]4      t=]$ ! R. R/]=4      4       t>]$ ! R0 R1]=]4      4       t?. R4Ot@R# )5é    )ÚCallableN)Únn)Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_experts_implementationÚuse_kernel_forward_from_hubÚuse_kernelized_func)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚGradientCheckpointingLayer)ÚMoeCausalLMOutputWithPastÚMoeModelOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputs©ÚAfmoeConfigc                   óÎ   a a€ ] tR t^,t o]! RRR7      R
V3R lV 3R lll4       t]]! RRR7      R
V3R lR ll4       4       t]P                  ! 4       ]
R 4       4       tR	tVtV ;t# )ÚAfmoeRotaryEmbeddingÚdevicez5.18)Úversionc                ó    <€ V ^8„  d   QhRS[ /# ©é   Úconfigr   )ÚformatÚ__classdict__s   "€Úi/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/afmoe/modeling_afmoe.pyÚ__annotate__Ú!AfmoeRotaryEmbedding.__annotate__.   s   ø€ ÷ Oñ O™{ñ Oó    c                óä  <€ \         SV `  4        VP                  V n        VP                  V n        Wn        V P
                  P                  R ,          V n        V P                  pV P                  R8w  d   \        V P                  ,          pV! V P
                  V4      w  q@n
        \        P                  ! VRR7      V n        \        P                  ! VP                  4       RR7      V n        R# )Ú	rope_typeÚdefaultF)Ú
persistentN)ÚsuperÚ__init__Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr(   Úrope_parametersr0   Úcompute_default_rope_parametersr   Úattention_scalingr   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)Úselfr(   r#   Úrope_init_fnr<   Ú	__class__s   &&&  €r+   r4   ÚAfmoeRotaryEmbedding.__init__-   s¬   ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àŒàŸ™×4Ñ4°[ÕAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÔ&Ü.¨t¯~©~Õ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÔ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÔ!NˆÖr.   c                óP   <€ V ^8„  d   QhRS[ RS[S[P                  S[3,          /# )r'   r(   Úreturn)r    ÚtupleÚtorchÚTensorÚfloat)r)   r*   s   "€r+   r,   r-   @   s.   ø€ ÷ 5ñ 5±ð 5ÑW\Ñ]b×]iÑ]iÑkpÐ]pÕWqñ 5r.   c           	     ó0  € V P                   R,          p\        V RR4      ;'       g    V P                  V P                  ,          pRpRV\        P
                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚhead_dimNg      ð?©Údtype)r8   ÚgetattrÚhidden_sizeÚnum_attention_headsrF   ÚarangerH   Úto)r(   r#   ÚkwargsÚbaseÚdimÚattention_factorr<   s   &&,    r+   r9   Ú4AfmoeRotaryEmbedding.compute_default_rope_parameters>   s~   € ð ×%Ñ% lÕ3ˆÜ�f˜j¨$Ó/×cÐc°6×3EÑ3EÈ×IcÑIcÕ3cˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r.   c                ó^  € V P                   R,          P                  VP                  ^ ,          R	^4      P                  \        P
                  VP                  R7      pVR
,          P                  4       p\        VP                  P                  \        4      '       d2   VP                  P                  R8w  d   VP                  P                  MRp\        VRR7      ;_uu_ 4        W4,          P                  ^^4      p\        P                  ! Wf3R	R7      pVP                  4       V P                  ,          pVP                  4       V P                  ,          p	R R R 4       XP                  VP                   R7      X	P                  VP                   R7      3#   + '       g   i     LH; i)N)rM   r#   ÚmpsÚcpuF)Údevice_typeÚenabled©rU   rL   )NºNNNNéÿÿÿÿ)r^   Nr^   )r<   ÚexpandÚshaperR   rF   rH   r#   Ú
isinstanceÚtypeÚstrr   Ú	transposeÚcatÚcosr:   ÚsinrM   )
r?   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedr[   ÚfreqsÚembrg   rh   s
   &&&       r+   ÚforwardÚAfmoeRotaryEmbedding.forwardR   s2  € ð �M‰M˜-Õ(×/Ñ/°×0BÑ0BÀ1Õ0EÀrÈ1ÓM×PÑPÔW\×WbÑWbÐkl×ksÑksÐPÓtð 	ð !-¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆä¨¸U×CÖCØ&Õ>×IÑIÈ!ÈQÓOˆEÜ—)’)˜U˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆC÷	 Dð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷ D×Cús   Ã(A4FÆF,	)r:   r(   r<   r6   r>   r7   r0   ©N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r4   Ústaticmethodr9   rF   Úno_gradr   ro   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©rA   r*   s   @@r+   r"   r"   ,   su   ù‡ € Ù�X vÔ.÷Oô Oó /ðOð  Ù�X vÔ.÷5ñ 5ó /ó ð5ð$ ‡]‚]ƒ_Øñ<ó ó ÷<ð <r.   r"   ÚRMSNormc                   óZ   a a€ ] tR t^et oRV3R lV 3R llltV3R lR ltR tRtVtV ;t	# )ÚAfmoeRMSNormc                ó$   <€ V ^8„  d   QhRS[ RR/# )r'   ÚepsrD   N)rH   )r)   r*   s   "€r+   r,   ÚAfmoeRMSNorm.__annotate__g   s   ø€ ÷ $ñ $©ð $¸$ñ $r.   c                óŽ   <€ \         SV `  4        \        P                  ! \        P
                  ! V4      4      V n        W n        R# )z+
AfmoeRMSNorm is equivalent to T5LayerNorm
N)r3   r4   r   Ú	ParameterrF   ÚonesÚweightÚvariance_epsilon)r?   rO   r€   rA   s   &&&€r+   r4   ÚAfmoeRMSNorm.__init__g   s/   ø€ ô 	‰ÑÔÜ—l’l¤5§:¢:¨kÓ#:Ó;ˆŒØ #Ör.   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r'   rD   ©rF   rG   )r)   r*   s   "€r+   r,   r�   o   s   ø€ ÷ =ñ =©¯©ñ =r.   c                ó:  € VP                   pVP                  \        P                  4      pVP	                  ^4      P                  RRR7      pV\        P                  ! W0P                  ,           4      ,          pV P                  V,          P                  V4      # )r'   T)Úkeepdimr_   )	rM   rR   rF   Úfloat32ÚpowÚmeanÚrsqrtr†   r…   )r?   Úhidden_statesÚinput_dtypeÚvariances   &&  r+   ro   ÚAfmoeRMSNorm.forwardo   sw   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯ª°H×?TÑ?TÕ4TÓ(UÕUˆØ—‘˜mÕ+×/Ñ/°Ó<Ð<r.   c                ó^   € \        V P                  P                  4       R V P                   2# )z, eps=)rE   r…   ra   r†   )r?   s   &r+   Ú
extra_reprÚAfmoeRMSNorm.extra_reprv   s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr.   )r†   r…   )g�íµ ÷Æ°>)
rr   rs   rt   ru   r4   ro   r•   rx   ry   rz   r{   s   @@r+   r~   r~   e   s%   ù‡ € ÷$õ $÷=ð =÷Jò Jr.   r~   c                   ó<   a a€ ] tR t^zt oRV 3R lltR tRtVtV ;t# )ÚAfmoeMLPc                óä  <€ \         SV `  4        Wn        VP                  V n        Vf   VP                  MTV n        \
        P                  ! V P                  V P                  RR7      V n        \
        P                  ! V P                  V P                  RR7      V n        \
        P                  ! V P                  V P                  RR7      V n	        \        VP                  ,          V n        R # )NF©Úbias)r3   r4   r(   rO   Úintermediate_sizer   ÚLinearÚ	gate_projÚup_projÚ	down_projr   Ú
hidden_actÚact_fn)r?   r(   rœ   rA   s   &&&€r+   r4   ÚAfmoeMLP.__init__{   s¬   ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ=NÒ=V ×!9Ò!9Ð\mˆÔÜŸš 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆŒÜ˜V×.Ñ.Õ/ˆŽr.   c                ó�   € V P                  V P                  V P                  V4      4      V P                  V4      ,          4      pV# rq   )r    r¢   rž   rŸ   )r?   ri   r    s   && r+   ro   ÚAfmoeMLP.forward…   s6   € Ø—N‘N 4§;¡;¨t¯~©~¸aÓ/@Ó#AÀDÇLÁLÐQRÃOÕ#SÓTˆ	ØÐr.   )r¢   r(   r    rž   rO   rœ   rŸ   rq   )	rr   rs   rt   ru   r4   ro   rx   ry   rz   r{   s   @@r+   r˜   r˜   z   s   ù‡ € ÷0÷ò r.   r˜   c                   óH   a a€ ] tR t^Št oRtV 3R ltV3R lR ltRtVtV ;t	# )ÚAfmoeTokenChoiceRouterz�
Token-choice top-K router for MoE routing.

This router assigns each token to the top-K experts based on sigmoid scores, matching the released checkpoints.
c                óú   <€ \         SV `  4        Wn        VP                  V n        VP
                  V n        VP                  V n        \        P                  ! VP                  VP
                  R R7      V n
        R# ©Frš   N)r3   r4   r(   Únum_experts_per_tokÚtop_kÚnum_expertsÚroute_scaler   r�   rO   Úgate©r?   r(   rA   s   &&€r+   r4   ÚAfmoeTokenChoiceRouter.__init__‘   s\   ø€ Ü‰ÑÔØŒØ×/Ñ/ˆŒ
Ø!×-Ñ-ˆÔØ!×-Ñ-ˆÔÜ—I’I˜f×0Ñ0°&×2DÑ2DÈ5ÔQˆŽ	r.   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r'   r�   Úexpert_biasr‰   )r)   r*   s   "€r+   r,   Ú#AfmoeTokenChoiceRouter.__annotate__™   s#   ø€ ÷ ;ñ ;¡U§\¡\ð ;ÁÇÁñ ;r.   c                ó¸  € VP                   w   r4VP                  RV4      pV P                  V4      P                  \        P
                  4      p\        P                  ! V4      p\        P                  ! Wb,           V P                  ^R7      w  r7VP                  ^VR7      pVP                  RRR7      R,           p	W‰,          pW€P                  ,          pWXV3# )é   )ÚkrU   )rU   ÚindexT)rU   r‹   g#B’¡œÇ;r_   )ra   Úviewr®   rR   rF   rŒ   ÚsigmoidÚtopkr«   ÚgatherÚsumr­   )
r?   r�   r²   Ú_Ú
hidden_dimÚrouter_logitsÚscoresÚselected_expertsÚ
top_scoresÚdenominators
   &&&       r+   ro   ÚAfmoeTokenChoiceRouter.forward™   s¸   € Ø(×.Ñ.Ñˆˆ1Ø%×*Ñ*¨2¨zÓ:ˆàŸ	™	 -Ó0×3Ñ3´E·M±MÓBˆÜ—’˜}Ó-ˆä#Ÿjšj¨Õ)=ÀÇÁÐQRÔSÑˆØ—]‘] qÐ0@�]ÓAˆ
Ø —n‘n¨°T�nÓ:¸UÕBˆØÕ-ˆ
Ø×"2Ñ"2Õ2ˆ
ØÐ*:Ð:Ð:r.   )r(   r®   r¬   r­   r«   ©
rr   rs   rt   ru   Ú__doc__r4   ro   rx   ry   rz   r{   s   @@r+   r§   r§   Š   s   ù‡ € ñõR÷;÷ ;ð ;r.   r§   c                   óH   a a€ ] tR t^¨t oRtV 3R ltV3R lR ltRtVtV ;t	# )ÚAfmoeExpertsz2Collection of expert weights stored as 3D tensors.c                ó  <€ \         SV `  4        VP                  V n        VP                  V n        VP
                  V n        \        P                  ! \        P                  ! V P                  ^V P                  ,          V P                  4      4      V n        \        P                  ! \        P                  ! V P                  V P                  V P                  4      4      V n        \        VP                  ,          V n        R# )r'   N)r3   r4   r¬   rO   r¾   Úmoe_intermediate_sizeÚintermediate_dimr   rƒ   rF   ÚemptyÚgate_up_projr    r   r¡   r¢   r¯   s   &&€r+   r4   ÚAfmoeExperts.__init__¬   s­   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØ ×,Ñ,ˆŒØ &× <Ñ <ˆÔÜŸLšL¬¯ª°T×5EÑ5EÀqÈ4×K`ÑK`ÕG`Ðbf×bqÑbqÓ)rÓsˆÔÜŸš¤e§k¢k°$×2BÑ2BÀDÇOÁOÐUY×UjÑUjÓ&kÓlˆŒÜ˜V×.Ñ.Õ/ˆŽr.   c                ó‚   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[ P                  /# )r'   r�   Útop_k_indexÚtop_k_weightsrD   r‰   )r)   r*   s   "€r+   r,   ÚAfmoeExperts.__annotate__µ   sC   ø€ ÷ #ñ #á—|‘|ð#ñ —\‘\ð#ñ —|‘|ð	#ñ
 
�‰ñ#r.   c                óÆ  € \         P                  ! V4      p\         P                  ! 4       ;_uu_ 4        \         P                  P                  P                  W P                  R 7      pVP                  ^^^ 4      p\         P                  ! VP                  RR7      ^ 4      P                  4       pRRR4       X EF  pV^ ,          pWpP                  8X  d   K  \         P                  ! XV,          4      w  r‰W,          p
\        P                  P                  W P                  V,          4      P                  ^RR7      w  r¼V P                  V4      V,          p\        P                  P                  WÐP                   V,          4      pWÓW˜R3,          ,          pVP#                  ^ W�P%                  VP&                  4      4       EK  	  V#   + '       g   i     EL'; i))Únum_classesr]   Nr_   )r_   éþÿÿÿ)rF   Ú
zeros_likerw   r   Ú
functionalÚone_hotr¬   ÚpermuteÚgreaterr¼   ÚnonzeroÚwhereÚlinearrÍ   Úchunkr¢   r    Ú
index_add_rR   rM   )r?   r�   rÐ   rÑ   Úfinal_hidden_statesÚexpert_maskÚ
expert_hitÚ
expert_idxÚ	top_k_posÚ	token_idxÚcurrent_stater®   ÚupÚcurrent_hidden_statess   &&&&          r+   ro   ÚAfmoeExperts.forwardµ   ss  € ô $×.Ò.¨}Ó=ÐÜ�]Š]�_�_ÜŸ(™(×-Ñ-×5Ñ5°k×O_ÑO_Ð5Ó`ˆKØ%×-Ñ-¨a°°AÓ6ˆKÜŸš {§¡¸8 Ó'DÀaÓH×PÑPÓRˆJ÷ ô
 %ˆJØ# A�ˆJØ×-Ñ-Ô-ÙÜ#(§;¢;¨{¸:Õ/FÓ#GÑ ˆIØ)Õ4ˆMÜ—}‘}×+Ñ+¨M×;LÑ;LÈZÕ;XÓY×_Ñ_Ð`aÐgiÐ_Ój‰HˆDØ$(§K¡K°Ó$5¸Õ$:Ð!Ü$&§M¡M×$8Ñ$8Ð9N×P^ÑP^Ð_iÕPjÓ$kÐ!Ø$9È)Ð`dÐJdÕ<eÕ$eÐ!Ø×*Ñ*¨1¨i×9QÑ9QÐRe×RkÑRkÓ9l×mñ %ð #Ð"÷# �_ˆ_ús   ´A=GÇG 	)r¢   r    rÍ   r¾   rË   r¬   rÅ   r{   s   @@r+   rÈ   rÈ   ¨   s   ù‡ € á<õ0÷#÷ #ð #r.   rÈ   c                   ó<   a a€ ] tR t^Ðt oRtV 3R ltR tRtVtV ;t	# )ÚAfmoeSparseMoeBlockz¾
Mixture of Experts (MoE) module for AFMoE.

This module implements a sparse MoE layer with both shared experts (always active) and
routed experts (activated based on token-choice routing).
c                ó<  <€ \         SV `  4        Wn        \        V4      V n        \        WP                  VP                  ,          4      V n        \        V4      V n
        \        P                  ! \        P                  ! VP                  4      R R7      V n        R# )F)Úrequires_gradN)r3   r4   r(   r§   Úrouterr˜   rÊ   Únum_shared_expertsÚshared_expertsrÈ   Úexpertsr   rƒ   rF   Úzerosr¬   r²   r¯   s   &&€r+   r4   ÚAfmoeSparseMoeBlock.__init__Ø   sl   ø€ Ü‰ÑÔØŒÜ,¨VÓ4ˆŒÜ& v×/KÑ/KÈf×NgÑNgÕ/gÓhˆÔÜ# FÓ+ˆŒÜŸ<š<¬¯ª°F×4FÑ4FÓ(GÐW\Ô]ˆÖr.   c                ó  € VP                   w  r#pVP                  RV4      pV P                  WP                  4      w  rgpV P	                  V4      P                  W#V4      p	V P                  WXV4      P                  W#V4      p
Wš,           # )rµ   r_   )ra   r¸   rî   r²   rð   rñ   )r?   r�   Ú
batch_sizeÚseq_lenr¾   Úhidden_states_flatr½   rÂ   rÁ   Úshared_outputÚrouted_outputs   &&         r+   ro   ÚAfmoeSparseMoeBlock.forwardà   sŽ   € Ø*7×*=Ñ*=Ñ'ˆ
˜ZØ*×/Ñ/°°JÓ?Ðð +/¯+©+°m×EUÑEUÓ*VÑ'ˆÐ'ð ×+Ñ+Ð,>Ó?×DÑDÀZÐZdÓeˆØŸ™Ð%7È:ÓV×[Ñ[Ø ó
ˆð Õ,Ð,r.   )r(   r²   rñ   rî   rð   rÅ   r{   s   @@r+   rë   rë   Ð   s   ù‡ € ñõ^÷-ò -r.   rë   c                óÌ   € V RRV P                   R,          ^,          13,          pV RV P                   R,          ^,          R13,          p\        P                  ! V) V3RR7      # )z*Rotates half the hidden dims of the input..Nr]   r_   )ra   rF   rf   )ri   Úx1Úx2s   &  r+   Úrotate_halfrþ   ï   sZ   € à	
ˆ3Ð"�!—'‘'˜"•+ Õ"Ð"Ð"Õ	#€BØ	
ˆ3�—‘˜•˜qÕ Ñ"Ð"Õ	#€BÜ�9Š9�r�c˜2�Y BÔ'Ð'r.   Úrotary_pos_embc                óÈ   € VP                  V4      pVP                  V4      pW,          \        V 4      V,          ,           pW,          \        V4      V,          ,           pWV3# )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)Ú	unsqueezerþ   )Úqr¶   rg   rh   Úunsqueeze_dimÚq_embedÚk_embeds   &&&&&  r+   Úapply_rotary_pos_embr  ö   sS   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�wœ; q›>¨CÕ/Õ0€GØ�wœ; q›>¨CÕ/Õ0€GØÐÐr.   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r'   r�   Ún_reprD   )rF   rG   Úint)r)   s   "r+   r,   r,     s.   € ÷ 	Uñ 	UœUŸ\™\ð 	U´#ð 	U¼%¿,¹,ñ 	Ur.   c                ó˜   € V P                   w  r#rEV^8X  d   V # V R,          P                  W#WV4      p V P                  W#V,          WE4      # )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
)r^   r^   Nr^   r^   )ra   r`   Úreshape)r�   r  ÚbatchÚnum_key_value_headsÚslenrK   s   &&    r+   Ú	repeat_kvr    sU   € ð
 2?×1DÑ1DÑ.€E Ø�„zØÐØ!Ð"2Õ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÕ(CÀTÓTÐTr.   c                ó   € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R\        R	\
        \        ,          /# )
r'   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNÚscalingÚdropoutrS   )r   ÚModulerF   rG   rH   r   r   )r)   s   "r+   r,   r,     sw   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ð%ô ð%ô Ô'Õ(ñ%r.   c                 ó"  € \        W P                  4      p\        W0P                  4      p	\        P                  ! WP	                  ^^4      4      V,          p
Ve	   W¤,           p
\
        P                  P                  V
R\        P                  R7      P                  VP                  4      p
\
        P                  P                  W¦V P                  R7      p
\        P                  ! W©4      pVP	                  ^^4      P                  4       pWº3# )r'   )rU   rM   )ÚpÚtrainingr_   )r  Únum_key_value_groupsrF   Úmatmulre   r   r×   ÚsoftmaxrŒ   rR   rM   r  r  Ú
contiguous)r  r  r  r  r  r  r  rS   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs   &&&&&&&,    r+   Úeager_attention_forwardr$    sÒ   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÕL€LØÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r.   c                   óX   a a€ ] tR tRt oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )	ÚAfmoeAttentioni5  a6  
Multi-headed attention module with optional sliding window and gating.

This attention mechanism supports both full attention and sliding window attention,
and includes Q/K normalization and gating of the output. It inherits from [`LlamaAttention`] to minimize the amount
of custom logic we need to maintain.
c                ó&   <€ V ^8„  d   QhRS[ RS[/# ©r'   r(   Ú	layer_idx©r    r	  )r)   r*   s   "€r+   r,   ÚAfmoeAttention.__annotate__?  s   ø€ ÷ oñ o™{ð o±sñ or.   c                ó@  <€ \         SV `  4        Wn        W n        \	        VR VP
                  VP                  ,          4      V n        VP                  VP                  ,          V n	        V P                  R,          V n
        VP                  V n        RV n        \        P                  ! VP
                  VP                  V P                  ,          VP                  R7      V n        \        P                  ! VP
                  VP                  V P                  ,          VP                  R7      V n        \        P                  ! VP
                  VP                  V P                  ,          VP                  R7      V n        \        P                  ! VP                  V P                  ,          VP
                  VP                  R7      V n        VP(                  V,          R8H  V n        V P*                  '       d   VP,                  MRV n        \/        V P                  VP0                  R7      V n        \/        V P                  VP0                  R7      V n        \        P                  ! VP
                  VP                  V P                  ,          RR7      V n        R# )rK   Trš   Úsliding_attentionN©r€   Fg      à¿)r3   r4   r(   r)  rN   rO   rP   rK   r  r  r  Úattention_dropoutÚ	is_causalr   r�   Úattention_biasÚq_projÚk_projÚv_projÚo_projÚlayer_typesÚis_local_attentionÚsliding_windowr~   Úrms_norm_epsÚq_normÚk_normrž   ©r?   r(   r)  rA   s   &&&€r+   r4   ÚAfmoeAttention.__init__?  sÝ  ø€ Ü‰ÑÔØŒØ"ŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÕ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Õ$\ˆÔ!Ø—}‘} dÕ*ˆŒØ!'×!9Ñ!9ˆÔØˆŒä—i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×&Ñ&¨¯©Õ6¸×8JÑ8JÐQW×QfÑQfô
ˆŒð
 #)×"4Ñ"4°YÕ"?ÐCVÑ"VˆÔØ7;×7N×7NÐ7N˜f×3Ò3ÐTXˆÔä" 4§=¡=°f×6IÑ6IÔJˆŒÜ" 4§=¡=°f×6IÑ6IÔJˆŒÜŸš 6×#5Ñ#5°v×7QÑ7QÐTX×TaÑTaÕ7aÐhmÔnˆŽr.   c                ó  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  S[ P                  3,          /# )r'   r�   Úposition_embeddingsr  NÚpast_key_valuesrS   rD   )rF   rG   rE   r   r   r   )r)   r*   s   "€r+   r,   r+  ^  s‚   ø€ ÷ .)ñ .)á—|‘|ð.)ñ #¡5§<¡<±·±Ð#=Õ>ð.)ñ Ÿ™ tÕ+ð	.)ñ
  �ð.)ñ Ñ+Õ,ð.)ñ 
‰u�|‰|™UŸ\™\Ð)Õ	*ñ.)r.   c                óö  € VP                   R R p. VORNV P                  N5pV P                  V4      P                  V4      pV P	                  V4      P                  V4      p	V P                  V4      P                  V4      p
V P                  V4      pV P                  V4      P                  ^^4      pV P                  V	4      P                  ^^4      p	V
P                  ^^4      p
V P                  '       d   Vw  rÍ\        W‰WÍ4      w  r‰Ve   VP                  WšV P                  4      w  rš\        P                  ! V P                   P"                  \$        4      pT! V VV	V
3RTRV P&                  '       g   RMV P(                  RV P*                  RV P,                  /VB w  ppVP                  ! . VORN5!  P/                  4       pV\0        P2                  ! V4      ,          pV P5                  V4      pVV3# )Nr  r  ç        r  r8  r_   )ra   rK   r2  r¸   r3  r4  rž   r:  re   r;  r7  r  Úupdater)  r   Úget_interfacer(   Ú_attn_implementationr$  r  r/  r  r8  r  rF   r¹   r5  )r?   r�   r?  r  r@  rS   Úinput_shapeÚhidden_shapeÚquery_statesr   r!  Úgate_statesrg   rh   Úattention_interfaceÚoutputr"  r#  s   &&&&&,            r+   ro   ÚAfmoeAttention.forward^  së  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓDˆØ—[‘[ Ó/×4Ñ4°\ÓBˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓDˆØ—n‘n ]Ó3ˆà—{‘{ <Ó0×:Ñ:¸1¸aÓ@ˆØ—[‘[ Ó,×6Ñ6°q¸!Ó<ˆ
Ø#×-Ñ-¨a°Ó3ˆà×"×"Ð"Ø*‰HˆCÜ';¸LÐVYÓ'_Ñ$ˆLàÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ  3ØØØØñ	
 
ð
 *ð
 
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð
 
ð —L‘Lð
 
ð  ×.Ñ.ð
 
ð ñ
 
Ñˆ�ð —’Ð.˜kÐ.¨2Ó.×9Ñ9Ó;ˆØœ%Ÿ-š-¨Ó4Õ4ˆØ—k‘k &Ó)ˆØ˜LÐ(Ð(r.   )r/  r(   rž   rK   r0  r7  r;  r3  r)  r  r5  r:  r2  r  r8  r4  rq   rÅ   r{   s   @@r+   r&  r&  5  s%   ù‡ € ñ÷oó o÷>.)÷ .)ò .)r.   r&  c                   óX   a a€ ] tR tRt oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )	ÚAfmoeDecoderLayeri�  z¼
AFMoE decoder layer with dual normalization.

This layer applies self-attention followed by either a dense MLP or MoE block,
with dual normalization (pre and post) around each component.
c                ó&   <€ V ^8„  d   QhRS[ RS[/# r(  r*  )r)   r*   s   "€r+   r,   ÚAfmoeDecoderLayer.__annotate__—  s   ø€ ÷ (ñ (™{ð (±sñ (r.   c                ó2  <€ \         SV `  4        VP                  V n        W n        \	        WR 7      V n        \        VP                  VP                  R7      V n        \        VP                  VP                  R7      V n	        \        VP                  VP                  R7      V n
        \        VP                  VP                  R7      V n        W!P                  8¬  V n        V P                  '       d   \        V4      V n        R# \!        V4      V n        R# ))r(   r)  r.  N)r3   r4   rO   r)  r&  Ú	self_attnr~   r9  Úinput_layernormÚpost_attention_layernormÚpre_mlp_layernormÚpost_mlp_layernormÚnum_dense_layersÚmoe_enabledrë   Úmlpr˜   r<  s   &&&€r+   r4   ÚAfmoeDecoderLayer.__init__—  sÔ   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØ"Œä'¨vÔKˆŒô  ,¨F×,>Ñ,>ÀF×DWÑDWÔXˆÔÜ(4°V×5GÑ5GÈV×M`ÑM`Ô(aˆÔ%ô ".¨f×.@Ñ.@Àf×FYÑFYÔ!ZˆÔÜ".¨v×/AÑ/AÀv×GZÑGZÔ"[ˆÔð %×(?Ñ(?Ñ?ˆÔØ××ÐÜ*¨6Ó2ˆDŽHä Ó'ˆDŽHr.   c                ó.  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[ P                  S[ P                  3,          R,          RS[S[,          R	S[ P                  /# )
r'   r�   r  Nrj   r@  Ú	use_cacher?  rS   rD   )	rF   rG   Ú
LongTensorr   ÚboolrE   r   r   ÚFloatTensor)r)   r*   s   "€r+   r,   rP  ­  sž   ø€ ÷ !ñ !á—|‘|ð!ñ Ÿ™ tÕ+ð!ñ ×&Ñ&¨Õ-ð	!ñ
  �ð!ñ ˜$•;ð!ñ #¡5§<¡<±·±Ð#=Õ>ÀÕEð!ñ Ñ+Õ,ð!ñ 
×	Ñ	ñ!r.   c                ó  € TpV P                  V4      pV P                  ! RR VRVRVRVRVRV/VB w  rV P                  V4      pW�,           pTpV P                  V4      pV P	                  V4      pV P                  V4      pW�,           pV# )r�   r  rj   r@  r\  r?  © )rS  rR  rT  rU  rY  rV  )
r?   r�   r  rj   r@  r\  r?  rS   Úresidualr½   s
   &&&&&&&,  r+   ro   ÚAfmoeDecoderLayer.forward­  sË   € ð !ˆð ×,Ñ,¨]Ó;ˆØŸ>š>ñ 
Ø'ð
à)ð
ð &ð
ð ,ð	
ð
  ð
ð !4ð
ð ñ
Ñˆð ×5Ñ5°mÓDˆØ Õ0ˆð !ˆØ×.Ñ.¨}Ó=ˆØŸ™ Ó/ˆØ×/Ñ/°Ó>ˆà Õ0ˆØÐr.   )	rO   rS  r)  rY  rX  rT  rV  rU  rR  )NNNNNrÅ   r{   s   @@r+   rN  rN  �  s#   ù‡ € ñ÷(ó (÷,!÷ !ò !r.   rN  c                   ó’   a a€ ] tR tRt oRtRtR.tR.tR]! ]	^ R7      R]
R	]/t. ROtR
tR
tR
tR
tR
tR
tV 3R ltV3R ltRtVtV ;t# )ÚAfmoePreTrainedModeliÑ  zz
An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
models.
ÚmodelrN  r@  r¿   )r·   r�   Ú
attentionsTc                ó  <€ \         SV `  V4       V P                  P                  p\	        V\
        4      '       dI   \        P                  ! VP                  RVR7       \        P                  ! VP                  RVR7       R# \	        V\        4      '       d-   \        P                  ! VP                  P                  4       R# \	        V\        4      '       d#   \        P                  ! VP                  4       R# R# )zInitialize the weightsrB  )rŽ   ÚstdN)r3   Ú_init_weightsr(   Úinitializer_rangerb   rÈ   ÚinitÚnormal_rÍ   r    r§   Úzeros_r®   r…   rë   r²   )r?   r  ri  rA   s   && €r+   rj  Ú"AfmoePreTrainedModel._init_weightsñ  s¡   ø€ ä‰Ñ˜fÔ%Ø�k‰k×+Ñ+ˆÜ�fœl×+Ò+Ü�LŠL˜×,Ñ,°3¸CÕ@Ü�LŠL˜×)Ñ)°¸×=Ü˜Ô 6×7Ò7Ü�KŠK˜Ÿ™×*Ñ*Ö+Ü˜Ô 3×4Ò4Ü�KŠK˜×*Ñ*Ö+ñ 5r.   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r&   r   )r)   r*   s   "€r+   r,   Ú!AfmoePreTrainedModel.__annotate__Ñ  s   ø‡ ‚ ñ Ñò r.   ra  )rS  rT  rU  rV  r:  r;  Únormr²   )rr   rs   rt   ru   rÆ   Úbase_model_prefixÚ_no_split_modulesÚ_skip_keys_device_placementr   r§   rN  r&  Ú_can_record_outputsÚ_keep_in_fp32_modulesÚ_supports_sdpaÚ_supports_flash_attnÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendÚsupports_gradient_checkpointingrj  Ú__annotate_func__rx   ry   rz   r{   s   @@r+   re  re  Ñ  s~   ù‡ € ñð  ÐØ,Ð-ÐØ#4Ð"5Ðà™Ð(>ÀaÔHØÐ*Ø�nðÐò
	Ðð €NØÐØÐØ!ÐØ"&ÐØ&*Ð#õ
,÷A … r.   re  c                   óv   a a€ ] tR tRt oRtV3R lV 3R llt]]]RV3R lR ll4       4       4       t	Rt
VtV ;t# )	Ú
AfmoeModeliþ  z‹
Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`AfmoeDecoderLayer`]

Args:
    config: AfmoeConfig
c                ó    <€ V ^8„  d   QhRS[ /# r&   r   )r)   r*   s   "€r+   r,   ÚAfmoeModel.__annotate__  s   ø€ ÷ ñ ™{ñ r.   c           	     ó  <€ \         SV `  V4       VP                  V n        VP                  V n        \
        P                  ! VP                  VP                  V P                  4      V n        \
        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n        \        VP                  VP                  R 7      V n        \#        VR7      V n        RV n        V P)                  4        R# u upi )r.  ©r(   FN)r3   r4   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	EmbeddingrO   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersrN  Úlayersr~   r9  rr  r"   Ú
rotary_embÚgradient_checkpointingÚ	post_initr<  s   && €r+   r4   ÚAfmoeModel.__init__  sÈ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸLšL¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m’mÜCHÈ×IaÑIaÔCbÓcÑCb°iÔ˜vÖ1ÑCbÑcó
ˆŒô ! ×!3Ñ!3¸×9LÑ9LÔMˆŒ	Ü.°fÔ=ˆŒØ&+ˆÔ#à�‰Öùò ds   ÂDc                ó  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[,          R	S[S[	,          /# )
r'   Ú	input_idsNr  Úinputs_embedsrj   r@  r\  rS   rD   )
rF   r]  rG   r_  r   r^  r   r   rE   r   )r)   r*   s   "€r+   r,   r‚    s™   ø€ ÷ <
ñ <
á×#Ñ# dÕ*ð<
ñ Ÿ™ tÕ+ð<
ñ ×(Ñ(¨4Õ/ð	<
ñ
 ×&Ñ&¨Õ-ð<
ñ  �ð<
ñ ˜$•;ð<
ñ Ñ+Õ,ð<
ñ 
Ñ'Õ	'ñ<
r.   c                ó   € VR J VR J,          '       d   \        R4      hV'       d   Vf   \        V P                  R7      pVf   V P                  V4      pVfb   Ve   VP	                  4       M^ p\
        P                  ! VP                  ^,          VP                  R7      V,           pVP                  ^ 4      p\        T;p	\        4      '       g-   RV P                  RVRVRV/p
R\        R/ V
B R	\        R/ V
B /p	TpV P                  P                  '       d$   W°P                  P                  R
,          ,          pV P!                  W´4      p\#        V P$                  4       F9  w  rÞV! V3RW�P                  P&                  V,          ,          RVRVRVRV/VB pK;  	  V P)                  V4      p\+        TV'       d   VR7      # R R7      # )Nz:You must specify exactly one of input_ids or inputs_embedsr„  )r#   r(   r”  r  r@  Úfull_attentionr-  g      à?rj   r\  r?  )Úlast_hidden_stater@  ra  )Ú
ValueErrorr   r(   r‰  Úget_seq_lengthrF   rQ   ra   r#   r  rb   Údictr   r   Úmup_enabledrO   rŽ  Ú	enumerater�  r6  rr  r   )r?   r“  r  r”  rj   r@  r\  rS   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsr�   r?  ÚiÚdecoder_layers   &&&&&&&,       r+   ro   ÚAfmoeModel.forward  sß  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZç˜Ò0Ü*°$·+±+Ô>ˆOàÒ Ø ×-Ñ-¨iÓ8ˆMàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLô °Ð?Ð-Ä×FÒFà˜$Ÿ+™+Ø Ø  .Ø! ?ð	ˆKð !Ô"4Ñ"C°{Ñ"CØ#Ô%FÑ%UÈÑ%Uð#Ðð
 &ˆð �;‰;×"×"Ð"Ø)¯[©[×-DÑ-DÀcÕ-IÕJˆMà"Ÿo™o¨mÓJÐä )¨$¯+©+Ö 6ÑˆAÙ)Øñà2·;±;×3JÑ3JÈ1Õ3MÕNðð *ðð !0ð	ð
 $ðð %8ðð ñŠMñ !7ð Ÿ	™	 -Ó0ˆÜ%Ø+ß/8˜Oô
ð 	
à>Bô
ð 	
r.   )r‰  r�  r�  rr  r†  rŽ  r‡  )NNNNNN)rr   rs   rt   ru   rÆ   r4   r   r   r   ro   rx   ry   rz   r{   s   @@r+   r€  r€  þ  sB   ù‡ € ñ÷ó ð ØØ÷<
ñ <
ó ó  ó ÷<
ð <
r.   r€  c                   ó„   a a€ ] tR tRt oRR/tRR/tRR.R.3/tRR/tV 3R	 lt]	]
RV3R
 lR ll4       4       tRtVtV ;t# )ÚAfmoeForCausalLMiX  zlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputr�   ÚlogitsÚkeep_full_weightc                óì   <€ \         SV `  V4       \        V4      V n        VP                  V n        \
        P                  ! VP                  VP                  R R7      V n        V P                  4        R# r©   )
r3   r4   r€  rf  r‡  r   r�   rO   r¥  r�  r¯   s   &&€r+   r4   ÚAfmoeForCausalLM.__init__`  sS   ø€ Ü‰Ñ˜Ô Ü Ó'ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒØ�‰Ör.   c                ód  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[R,          R
S[S[ P                  ,          RS[S[,          RS[	/# )r'   r“  Nr  rj   r@  r”  Úlabelsr\  Úoutput_router_logitsÚlogits_to_keeprS   rD   )
rF   r]  rG   r   r_  r^  r	  r   r   r   )r)   r*   s   "€r+   r,   ÚAfmoeForCausalLM.__annotate__i  sÉ   ø€ ÷ <
ñ <
á×#Ñ# dÕ*ð<
ñ Ÿ™ tÕ+ð<
ñ ×&Ñ&¨Õ-ð	<
ñ
  �ð<
ñ ×(Ñ(¨4Õ/ð<
ñ × Ñ  4Õ'ð<
ñ ˜$•;ð<
ñ # T�kð<
ñ ™eŸl™lÕ*ð<
ñ Ñ+Õ,ð<
ñ 
#ñ<
r.   c
                óÌ  € Ve   TMV P                   P                  pV P                  ! RRVRVRVRVRVRVRV/V
B pVP                  p\	        V	\
        4      '       d   \        V	) R4      MT	pV P                  VR	VR	3,          4      pRpVe   V P                  ! WæV P                  3/ V
B p\        VVVP                  VP                  VP                  VP                  R
7      # )ao  
Example:

```python
>>> from transformers import AutoTokenizer, AfmoeForCausalLM

>>> model = AfmoeForCausalLM.from_pretrained("meta-afmoe/Afmoe-2-7b-hf")
>>> tokenizer = AutoTokenizer.from_pretrained("meta-afmoe/Afmoe-2-7b-hf")

>>> prompt = "Hey, are you conscious? Can you talk to me?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
```Nr“  r  rj   r@  r”  r\  r­  r^   )Úlossr§  r@  r�   rg  r¿   ra  )r(   r­  rf  r—  rb   r	  Úslicer¥  Úloss_functionr‡  r   r@  r�   rg  r¿   )r?   r“  r  rj   r@  r”  r¬  r\  r­  r®  rS   Úoutputsr�   Úslice_indicesr§  r±  s   &&&&&&&&&&,     r+   ro   ÚAfmoeForCausalLM.forwardg  s  € ðB %9Ò$DÑ È$Ï+É+×JjÑJjð 	ð +/¯*ª*ñ 	+
Øð	+
à)ð	+
ð &ð	+
ð ,ð		+
ð
 (ð	+
ð  ð	+
ð "6ð	+
ð ñ	+
ˆð  ×1Ñ1ˆÜ8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒØ×%Ò% f°d·o±oÑPÈÑPˆDä(ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø!×/Ñ/ô
ð 	
r.   )r¥  rf  r‡  )	NNNNNNNNr   )rr   rs   rt   ru   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr4   r   r   ro   rx   ry   rz   r{   s   @@r+   r¤  r¤  X  sf   ù‡ € à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HàÐ/Ð0€Jõð Ø÷<
ñ <
ó ó ÷<
ð <
r.   r¤  )rµ   )rB  )r¤  r€  re  )AÚcollections.abcr   rF   r   Ú r   rl  Úactivationsr   Úcache_utilsr   r   Ú
generationr	   Úintegrationsr
   r   r   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   r   Úconfiguration_afmoer    r  r"   r~   r˜   r§   rÈ   rë   rþ   r  r  r$  r&  rN  re  r€  r¤  Ú__all__ra  r.   r+   Ú<module>rÍ     s†  ðõ* %ã Ý å &Ý !ß .Ý )ß hÑ hß RÝ 9ß Qß Kß FÝ &ß IÑ IÝ 0ß Gß EÝ ,ô6<˜2Ÿ9™9ô 6<ñr ˜YÓ'ôJ�2—9‘9ó Jó (ðJô(ˆr�y‰yô ô ;˜RŸY™Yô ;ð< ô$#�2—9‘9ó $#ó ð$#ôN-˜"Ÿ)™)ô -ò>(ñ Ð-Ó.óó /ðõ2	U÷%ñ2 Ð)Ó*ôV)�R—Y‘Yó V)ó +ðV)ôr?Ð2ô ?ôD*,˜?ô *,ðZ ôV
Ð%ó V
ó ðV
ðr ôL
Ð+¨_ó L
ó ðL
ò^ E‚r.   