+
    œ°³jÄ”  ã                   ó:  € ^ RI Ht ^ RIt^ RIHt ^RIHt ^RIHt ^RI	H
t
Ht ^RIHt ^RIHtHtHt ^R	IHtHt ^R
IHtHt ^RIHtHt ^RIHtHt ^RIHtH t  ^RI!H"t" ^RI#H$t$H%t%H&t& ^RI'H(t( ^RI)H*t*H+t+ ^RI,H-t-H.t. ^RI/H0t0  ! R R]Pb                  4      t2R t3]! R4      R4R l4       t4R R lt5R5R R llt6]! ]44       ! R R]Pb                  4      4       t7 ! R R ]Pp                  Pr                  4      t:] ! R! R"]Pb                  4      4       t;R6R# lt< ! R$ R%]Pz                  4      t> ! R& R']Pb                  4      t? ! R( R)]4      t@]% ! R* R+] 4      4       tA]% ! R, R-]A4      4       tBR7R. R/ lltC]% ! R0 R1]A]4      4       tD ! R2 R3]]A4      tE. R8OtFR# )9é    )ÚCallableN)Únn)Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_experts_implementationÚuse_kernel_forward_from_hubÚuse_kernelized_func)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)Ú GenericForSequenceClassificationÚGradientCheckpointingLayer)ÚMoeCausalLMOutputWithPastÚMoeModelOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputs©ÚPhimoeConfigc                   óÒ   a a€ ] tR t^-t o]! RRR7      R
V3R lV 3R lll4       t]]! RRR7      R
V3R lR ll4       4       t]P                  ! 4       ]
R
R l4       4       tR	tVtV ;t# )ÚPhimoeRotaryEmbeddingÚdevicez5.18)Úversionc                ó    <€ V ^8„  d   QhRS[ /# ©é   Úconfigr    )ÚformatÚ__classdict__s   "€Úk/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/phimoe/modeling_phimoe.pyÚ__annotate__Ú"PhimoeRotaryEmbedding.__annotate__/   s   ø€ ÷ Oñ O™|ñ Oó    c                ó
  <€ \         SV `  4        VP                  V n        VP                  V n        Wn        V P
                  P                  R ,          V n        V P                  V n	        V P                  R8w  d   \        V P                  ,          V n	        V P                  V P
                  V4      w  q0n        \        P                  ! VRR7      V n        \        P                  ! VP                  4       RR7      V n        R# )Ú	rope_typeÚdefaultF)Ú
persistentN)ÚsuperÚ__init__Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr)   Úrope_parametersr1   Úcompute_default_rope_parametersÚrope_init_fnr   Úattention_scalingr   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)Úselfr)   r$   r>   Ú	__class__s   &&& €r,   r5   ÚPhimoeRotaryEmbedding.__init__.   s¸   ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àŒØŸ™×4Ñ4°[ÕAˆŒØ&*×&JÑ&JˆÔØ�>‰>˜YÔ&Ü 3°D·N±NÕ CˆDÔØ+/×+<Ñ+<¸T¿[¹[È&Ó+QÑ(ˆÔ(äŸ	š	 (°uÔ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÔ!NˆÖr/   c                óP   <€ V ^8„  d   QhRS[ RS[S[P                  S[3,          /# )r(   r)   Úreturn)r!   ÚtupleÚtorchÚTensorÚfloat)r*   r+   s   "€r,   r-   r.   @   s.   ø€ ÷ 5ñ 5±ð 5ÑX]Ñ^c×^jÑ^jÑlqÐ^qÕXrñ 5r/   c           	     ó0  € V P                   R,          p\        V RR4      ;'       g    V P                  V P                  ,          pRpRV\        P
                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚhead_dimNç      ð?)Údtype)r9   ÚgetattrÚhidden_sizeÚnum_attention_headsrG   ÚarangerI   Úto)r)   r$   ÚkwargsÚbaseÚdimÚattention_factorr>   s   &&,    r,   r:   Ú5PhimoeRotaryEmbedding.compute_default_rope_parameters>   s~   € ð ×%Ñ% lÕ3ˆÜ�f˜j¨$Ó/×cÐc°6×3EÑ3EÈ×IcÑIcÕ3cˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r/   c                ó°  € R p\         P                  ! V4      ^,           pV P                  P                  R,          R8w  dd   V'       d\   W@P                  P                  R,          8”  d   V P                  P                  R,          MV P                  P                  R,          pV P	                  V P                  4      w  rVVf   TMTpVR,          P                  VP                  ^ ,          R^4      P                  \         P                  VP                  R7      pVR,          P                  4       p\        VP                  P                  \        4      '       d2   VP                  P                  R8w  d   VP                  P                  MRp	\        V	R	R
7      ;_uu_ 4        Wx,          P                  ^^4      p
\         P                  ! Wª3RR7      pVP!                  4       V,          pVP#                  4       V,          pR R R 4       XP                  VP$                  4      XP                  VP$                  4      3#   + '       g   i     LF; i)Nr1   r2   Ú original_max_position_embeddingsÚlong_mscaleÚshort_mscale©rN   r$   ÚmpsÚcpuF)Údevice_typeÚenabled©rV   )NºNNNNéÿÿÿÿ)rc   Nrc   )rG   Úmaxr)   r9   r;   ÚexpandÚshaperS   rI   r$   Ú
isinstanceÚtypeÚstrr   Ú	transposeÚcatÚcosÚsinrN   )rA   ÚxÚposition_idsÚmscaleÚseq_lenr>   r<   Úinv_freq_expandedÚposition_ids_expandedr`   ÚfreqsÚembrm   rn   s   &&&           r,   ÚforwardÚPhimoeRotaryEmbedding.forwardR   sµ  € ð ˆÜ—)’)˜LÓ)¨AÕ-ˆØ�;‰;×&Ñ& {Õ3°yÔ@ÇWð Ÿ[™[×8Ñ8Ð9[Õ\Ô\ð —‘×+Ñ+¨MÖ:à—[‘[×0Ñ0°Õ@ð ð
 '+×&7Ñ&7¸¿¹Ó&DÑ#ˆØ&,¢nÑ"¸&ˆà�]Õ#×*Ñ*¨<×+=Ñ+=¸aÕ+@À"ÀaÓH×KÑKÔRW×R]ÑR]Ðfg×fnÑfnÐKÓoð 	ð !-¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆÜ¨¸U×CÖCØ&Õ>×IÑIÈ!ÈQÓOˆEÜ—)’)˜U˜N°Ô3ˆCØ—'‘'“)˜fÕ$ˆCØ—'‘'“)˜fÕ$ˆC÷	 Dð
 �v‰v�a—g‘g‹ §¡ q§w¡w£Ð/Ð/÷ D×Cús   Æ'A IÉI	)r<   r)   r>   r7   r@   r8   r;   r1   ©N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r5   Ústaticmethodr:   rG   Úno_gradr   rw   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©rB   r+   s   @@r,   r#   r#   -   su   ù‡ € Ù�X vÔ.÷Oô Oó /ðOð Ù�X vÔ.÷5ñ 5ó /ó ð5ð$ ‡]‚]ƒ_Øó0ó ó ÷0ð 0r/   r#   c                óÌ   € V RRV P                   R,          ^,          13,          pV RV P                   R,          ^,          R13,          p\        P                  ! V) V3RR7      # )z*Rotates half the hidden dims of the input..Nrb   rd   )rg   rG   rl   )ro   Úx1Úx2s   &  r,   Úrotate_halfr‡   m   sZ   € à	
ˆ3Ð"�!—'‘'˜"•+ Õ"Ð"Ð"Õ	#€BØ	
ˆ3�—‘˜•˜qÕ Ñ"Ð"Õ	#€BÜ�9Š9�r�c˜2�Y BÔ'Ð'r/   Úrotary_pos_embc                óÈ   € VP                  V4      pVP                  V4      pW,          \        V 4      V,          ,           pW,          \        V4      V,          ,           pWV3# )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)Ú	unsqueezer‡   )ÚqÚkrm   rn   Úunsqueeze_dimÚq_embedÚk_embeds   &&&&&  r,   Úapply_rotary_pos_embr�   t   sS   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�wœ; q›>¨CÕ/Õ0€GØ�wœ; q›>¨CÕ/Õ0€GØÐÐr/   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r(   Úhidden_statesÚn_reprE   )rG   rH   Úint)r*   s   "r,   r-   r-   Ž   s.   € ÷ 	Uñ 	UœUŸ\™\ð 	U´#ð 	U¼%¿,¹,ñ 	Ur/   c                ó˜   € V P                   w  r#rEV^8X  d   V # V R,          P                  W#WV4      p V P                  W#V,          WE4      # )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
)rc   rc   Nrc   rc   )rg   rf   Úreshape)r’   r“   ÚbatchÚnum_key_value_headsÚslenrL   s   &&    r,   Ú	repeat_kvrš   Ž   sU   € ð
 2?×1DÑ1DÑ.€E Ø�„zØÐØ!Ð"2Õ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÕ(CÀTÓTÐTr/   c                ó   € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R\        R	\
        \        ,          /# )
r(   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNÚscalingÚdropoutrT   )r   ÚModulerG   rH   rI   r   r   )r*   s   "r,   r-   r-   š   sw   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ð%ô ð%ô Ô'Õ(ñ%r/   c                 ó"  € \        W P                  4      p\        W0P                  4      p	\        P                  ! WP	                  ^^4      4      V,          p
Ve	   W¤,           p
\
        P                  P                  V
R\        P                  R7      P                  VP                  4      p
\
        P                  P                  W¦V P                  R7      p
\        P                  ! W©4      pVP	                  ^^4      P                  4       pWº3# )r(   )rV   rN   )ÚpÚtrainingrd   )rš   Únum_key_value_groupsrG   Úmatmulrk   r   Ú
functionalÚsoftmaxÚfloat32rS   rN   r¢   r¦   Ú
contiguous)rœ   r�   rž   rŸ   r    r¡   r¢   rT   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs   &&&&&&&,    r,   Úeager_attention_forwardr±   š   sÒ   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÕL€LØÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r/   c                   óX   a a€ ] tR t^³t oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )ÚPhimoeAttentionz=Multi-headed attention from 'Attention Is All You Need' paperc                ó&   <€ V ^8„  d   QhRS[ RS[/# ©r(   r)   Ú	layer_idx©r!   r”   )r*   r+   s   "€r,   r-   ÚPhimoeAttention.__annotate__·   s   ø€ ÷ 
ñ 
™|ð 
¹ñ 
r/   c                ó¢  <€ \         SV `  4        Wn        W n        \	        VR VP
                  VP                  ,          4      V n        VP                  VP                  ,          V n	        V P                  R,          V n
        VP                  V n        RV n        \        P                  ! VP
                  VP                  V P                  ,          VP                  R7      V n        \        P                  ! VP
                  VP                  V P                  ,          VP                  R7      V n        \        P                  ! VP
                  VP                  V P                  ,          VP                  R7      V n        \        P                  ! VP                  V P                  ,          VP
                  VP                  R7      V n        R# )rL   T©ÚbiasNg      à¿)r4   r5   r)   r¶   rO   rP   rQ   rL   r˜   r§   r¡   Úattention_dropoutÚ	is_causalr   ÚLinearÚattention_biasÚq_projÚk_projÚv_projÚo_proj©rA   r)   r¶   rB   s   &&&€r,   r5   ÚPhimoeAttention.__init__·   sI  ø€ Ü‰ÑÔØŒØ"ŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÕ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Õ$\ˆÔ!Ø—}‘} dÕ*ˆŒØ!'×!9Ñ!9ˆÔØˆŒä—i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×&Ñ&¨¯©Õ6¸×8JÑ8JÐQW×QfÑQfô
ˆŽr/   c                ó  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          R,          RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  S[ P                  3,          /# )r(   r’   Úposition_embeddingsNr    Úpast_key_valuesrT   rE   )rG   rH   rF   r   r   r   )r*   r+   s   "€r,   r-   r¸   Î   s‡   ø€ ÷ &)ñ &)á—|‘|ð&)ñ #¡5§<¡<±·±Ð#=Õ>ÀÕEð&)ñ Ÿ™ tÕ+ð	&)ñ
  �ð&)ñ Ñ+Õ,ð&)ñ 
‰u�|‰|™UŸ\™\Ð)Õ	*ñ&)r/   c                ó  € VP                   R R p. VORNV P                  N5pV P                  V4      P                  V4      P	                  ^^4      pV P                  V4      P                  V4      P	                  ^^4      p	V P                  V4      P                  V4      P	                  ^^4      p
Vw  r¼\        W‰W¼4      w  r‰Ve   VP                  WšV P                  4      w  rš\        P                  ! V P                  P                  \        4      pT! V VV	V
V3RV P                  '       g   RMV P                   RV P"                  /VB w  rïVP$                  ! . VORN5!  P'                  4       pV P)                  V4      pWï3# )Nr¢   ç        r¡   rd   )rg   rL   rÀ   Úviewrk   rÁ   rÂ   r�   Úupdater¶   r   Úget_interfacer)   Ú_attn_implementationr±   r¦   r¼   r¡   r–   r¬   rÃ   )rA   r’   rÇ   r    rÈ   rT   Úinput_shapeÚhidden_shapeÚquery_statesr­   r®   rm   rn   Úattention_interfacer°   r¯   s   &&&&&,          r,   rw   ÚPhimoeAttention.forwardÎ   s‡  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—k‘k +Ó.ˆØÐ(Ð(r/   )r¼   r)   rL   r½   rÁ   r¶   r§   rÃ   rÀ   r¡   rÂ   )NNN©
rz   r{   r|   r}   Ú__doc__r5   rw   r€   r�   r‚   rƒ   s   @@r,   r³   r³   ³   s!   ù‡ € áG÷
ó 
÷.&)÷ &)ò &)r/   r³   c                   óX   a € ] tR t^÷t o ]V 3R lR l4       t]V 3R lR l4       tRtV tR# )ÚPhimoeMultiplierc          
      óœ   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[ P                  RS[ P                  /# )r(   ÚscoresÚ
multiplierÚselected_expertsÚmasked_gatesÚmask_for_one©rG   rH   )r*   r+   s   "€r,   r-   ÚPhimoeMultiplier.__annotate__ù   sQ   ø€ ÷ )ñ )á—‘ð)ñ —L‘Lð)ñ  Ÿ,™,ð	)ñ
 —l‘lð)ñ —l‘lñ)r/   c                ó6   € V P                  W#V4       W%,          # )aÄ  
Forward pass for the custom autograd function.

Args:
    ctx: Context object to save information for backward computation.
    scores (torch.Tensor): Input scores tensor.
    multiplier (torch.Tensor): Multiplier tensor.
    selected_experts (torch.Tensor): Tensor of selected experts.
    masked_gates (torch.Tensor): Masked gates tensor.
    mask_for_one (torch.Tensor): Mask for one tensor.

Returns:
    torch.Tensor: Result of the forward pass.
)Úsave_for_backward)ÚctxrÙ   rÚ   rÛ   rÜ   rÝ   s   &&&&&&r,   rw   ÚPhimoeMultiplier.forwardø   s   € ð. 	×Ñ˜j¸LÔIØÕ(Ð(r/   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r(   Úgrad_at_outputrÞ   )r*   r+   s   "€r,   r-   rß     s   ø€ ÷ 
ñ 
áŸ™ñ
r/   c                ó”   € V P                   w  r#pW,          pWAP                  R4      ,          pVP                  RVVR7       VRRRR3# )a
  
Backward pass for the custom autograd function.

Args:
    ctx: Context object with saved tensors from the forward pass.
    grad_at_output (torch.Tensor): Gradient at the output.

Returns:
    tuple[torch.Tensor, None, None, None, None]: Gradients for the inputs.
)rV   ÚindexÚsrcNrd   )Úsaved_tensorsÚmulÚscatter_add_)râ   rå   rÚ   rÛ   rÜ   Úgrad_at_scores_expandeds   &&    r,   ÚbackwardÚPhimoeMultiplier.backward  sg   € ð 69×5FÑ5FÑ2ˆ
 là'Õ4ˆà".×1CÑ1CÀBÓ1GÕ"GÐØ×,Ñ,ØØ"Øð 	-ô 	
ð $ØØØØð
ð 	
r/   © N)	rz   r{   r|   r}   r~   rw   rí   r€   r�   )r+   s   @r,   r×   r×   ÷   s+   ø‡ € Ø÷)ó ð)ð2 ÷
ó ö
r/   r×   c                   óT   a a€ ] tR tRt oRtV3R lV 3R lltV3R lR ltRtVtV ;t	# )ÚPhimoeExpertsi5  z2Collection of expert weights stored as 3D tensors.c                ó    <€ V ^8„  d   QhRS[ /# r'   r    )r*   r+   s   "€r,   r-   ÚPhimoeExperts.__annotate__9  s   ø€ ÷ 0ñ 0™|ñ 0r/   c                ó  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  V n        \        P                  ! \        P                  ! V P                  ^V P                  ,          V P
                  4      4      V n        \        P                  ! \        P                  ! V P                  V P
                  V P                  4      4      V n        \        VP                  ,          V n        R# )r(   N)r4   r5   Únum_local_expertsÚnum_expertsrP   Ú
hidden_dimÚintermediate_sizeÚintermediate_dimr   Ú	ParameterrG   ÚemptyÚgate_up_projÚ	down_projr   Ú
hidden_actÚact_fn©rA   r)   rB   s   &&€r,   r5   ÚPhimoeExperts.__init__9  s­   ø€ Ü‰ÑÔØ!×3Ñ3ˆÔØ ×,Ñ,ˆŒØ &× 8Ñ 8ˆÔÜŸLšL¬¯ª°T×5EÑ5EÀqÈ4×K`ÑK`ÕG`Ðbf×bqÑbqÓ)rÓsˆÔÜŸš¤e§k¢k°$×2BÑ2BÀDÇOÁOÐUY×UjÑUjÓ&kÓlˆŒÜ˜V×.Ñ.Õ/ˆŽr/   c                ó‚   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[ P                  /# )r(   r’   Útop_k_indexÚtop_k_weightsrE   rÞ   )r*   r+   s   "€r,   r-   ró   B  sC   ø€ ÷ #ñ #á—|‘|ð#ñ —\‘\ð#ñ —|‘|ð	#ñ
 
�‰ñ#r/   c                óÆ  € \         P                  ! V4      p\         P                  ! 4       ;_uu_ 4        \         P                  P                  P                  W P                  R 7      pVP                  ^^^ 4      p\         P                  ! VP                  RR7      ^ 4      P                  4       pRRR4       X EF  pV^ ,          pWpP                  8X  d   K  \         P                  ! XV,          4      w  r‰W,          p
\        P                  P                  W P                  V,          4      P                  ^RR7      w  r¼V P                  V4      V,          p\        P                  P                  WÐP                   V,          4      pWÓW˜R3,          ,          pVP#                  ^ W�P%                  VP&                  4      4       EK  	  V#   + '       g   i     EL'; i))Únum_classesrb   Nrd   )rd   éþÿÿÿ)rG   Ú
zeros_liker   r   r©   Úone_hotrö   ÚpermuteÚgreaterÚsumÚnonzeroÚwhereÚlinearrü   Úchunkrÿ   rý   Ú
index_add_rS   rN   )rA   r’   r  r  Úfinal_hidden_statesÚexpert_maskÚ
expert_hitÚ
expert_idxÚ	top_k_posÚ	token_idxÚcurrent_stateÚgateÚupÚcurrent_hidden_statess   &&&&          r,   rw   ÚPhimoeExperts.forwardB  ss  € ô $×.Ò.¨}Ó=ÐÜ�]Š]�_�_ÜŸ(™(×-Ñ-×5Ñ5°k×O_ÑO_Ð5Ó`ˆKØ%×-Ñ-¨a°°AÓ6ˆKÜŸš {§¡¸8 Ó'DÀaÓH×PÑPÓRˆJ÷ ô
 %ˆJØ# A�ˆJØ×-Ñ-Ô-ÙÜ#(§;¢;¨{¸:Õ/FÓ#GÑ ˆIØ)Õ4ˆMÜ—}‘}×+Ñ+¨M×;LÑ;LÈZÕ;XÓY×_Ñ_Ð`aÐgiÐ_Ój‰HˆDØ$(§K¡K°Ó$5¸Õ$:Ð!Ü$&§M¡M×$8Ñ$8Ð9N×P^ÑP^Ð_iÕPjÓ$kÐ!Ø$9È)Ð`dÐJdÕ<eÕ$eÐ!Ø×*Ñ*¨1¨i×9QÑ9QÐRe×RkÑRkÓ9l×mñ %ð #Ð"÷# �_ˆ_ús   ´A=GÇG 	)rÿ   rý   rü   r÷   rù   rö   rÔ   rƒ   s   @@r,   rñ   rñ   5  s!   ù‡ € á<÷0ó 0÷#÷ #ð #r/   rñ   c                ó4  € \         P                  ! 4       ;_uu_ 4        V P                  RRR7      w  rEV P                  4       P	                  VR7      pW@,
          V,          ^V,          8„  pRRR4       V P                  X\        R4      4      pV'       dq   V\         P                  ! V\         P                  R7      P                  4       P                  4       ,
          P                  RR7      ^,          P                  R4      pMXp\         P                  ! VRR7      pVP                  RVR7      p	V'       dˆ   VP                  RRR7      w  r¥\         P                  ! W…8H  \         P                  ! V
4      R	8„  4      p\         P                   ! R
VRR7      P#                  V4      p\$        P'                  V V	VVV4      pMT	p\         P(                  ! V RV\        R4      4      p\         P                  ! 4       ;_uu_ 4        VP                  RRR7      w  rEV P                  4       P	                  VR7      pW@,
          V,          ^V,          8„  pRRR4       VP                  V\        R4      4      pV'       dq   V\         P                  ! V\         P                  R7      P                  4       P                  4       ,
          P                  RR7      ^,          P                  R4      pMXp\         P                  ! VRR7      pVP                  RVR7      pV'       d–   VP                  RRR7      w  r¥\         P                  ! Wõ8H  \         P                  ! V
4      P+                  4       R	8„  4      p\         P                   ! R
VRR7      P#                  V4      p\$        P'                  V VVVV4      pMTp\         P,                  ! VV3RR7      p\         P,                  ! W�3RR7      pVV3#   + '       g   i     ELŽ; i  + '       g   i     EL»; i)u,  
Sparse mixer function to select top-k experts and compute multipliers.
Based on the paper: https://huggingface.co/papers/2409.12136
We first replace the TopK(Â·) function as random sampling of discrete variables
in model training. Then, following Liu et al. (2023a) and Liu et al. (2023b), we apply Heun's
third order method to approximate the expert routing gradient and construct a modified
back-propagation to give a mathematically sound gradient estimation for expert routing.

Args:
    scores (torch.Tensor): Input scores tensor.
    jitter_eps (float): Jitter epsilon for numerical stability.
    training (bool): Flag indicating if the model is in training mode.
    top_k (int): Number of top experts to select.

Returns:
    tuple[torch.Tensor, torch.Tensor]: Multiplier and selected experts tensors.
T)rV   Úkeepdim)ÚminNz-inf)Úmemory_formatrb   )rV   rç   g      è?gioð…ÉTÕ?gKÈ=›Uå?)Úalphard   )rG   r   re   ÚabsÚclampÚmasked_fillrI   Ú
empty_likeÚlegacy_contiguous_formatÚexponential_ÚlogrŠ   rª   ÚgatherÚ
logical_orÚ	rand_likeÚaddÚtype_asr×   ÚapplyÚscatterÚuniform_Úconcat)rÙ   Ú
jitter_epsr¦   Útop_kÚmask_logits_thresholdÚmax_indÚfactorrÜ   rÛ   Úmultiplier_oÚ
max_scoresrÝ   rÚ   Úmasked_scoresÚmasked_gates_top2Úselected_experts_top2Úmultiplier_top2_oÚmask_for_one_top2Úmultiplier_top2s   &&&&               r,   Úsparsemixerr?  ]  s‹  € ô$ 
�Š��à)/¯©¸ÀD¨Ó)IÑ&ÐØ—‘“×#Ñ#Ð(=Ð#Ó>ˆØ"7Õ"@ÀFÕ!JÈqÐS]Í~Ñ ^Ð÷	 
ð ×%Ñ%Ð&;¼UÀ6»]ÓK€Lßð Ü×"Ò" <¼u×?]Ñ?]Ô^×kÑkÓm×qÑqÓsõt÷ ‰S�RˆS‹[˜õ	÷
 ‰Y�r‹]ñ 	ð #Ðô —=’= °2Ô6€LØ×&Ñ&¨2Ð5EÐ&ÓF€Lçà*×.Ñ.°2¸tÐ.ÓDÑˆ
Ü×'Ò'ØÑ'Ü�OŠO˜JÓ'¨$Ñ.ó
ˆô
 —y’y ¨¸VÔD×LÑLÈ\ÓZˆä%×+Ñ+ØØØØØó
‰
ð "ˆ
ô —M’MØØ
ØÜˆf‹ó	€Mô 
�Š��à)6×):Ñ):¸rÈ4Ð):Ó)PÑ&ÐØ—‘“×#Ñ#Ð(=Ð#Ó>ˆØ"7Õ"@ÀFÕ!JÈqÐS]Í~Ñ ^Ð÷	 
ð &×1Ñ1Ð2GÌÈvËÓWÐßð "Ü×"Ò"Ð#4ÄE×DbÑDbÔcß‘“ß‘“õ÷
 ‰S�RˆS‹[˜õ÷ ‰Y�r‹]ñ 	ð !(ÐäŸšÐ&7¸RÔ@ÐØ)×0Ñ0°RÐ?TÐ0ÓUÐçà/×3Ñ3¸ÀDÐ3ÓIÑˆ
Ü!×,Ò,Ø!Ñ,Ü�OŠO˜JÓ'×0Ñ0Ó2°TÑ9ó
Ðô
 "ŸIšI fÐ.?ÀvÔN×VÑVÐWhÓiÐä*×0Ñ0ØØØ!ØØó
‰ð ,ˆä—’˜z¨?Ð;ÀÔD€JÜ—|’|Ð%5Ð$MÐSUÔVÐð 	Øðð ÷G 
�ˆú÷f 
�ˆús   žAO2ÈAPÏ2P	ÐP	c                   óV   a a€ ] tR tRt oV3R lV 3R lltV3R lV 3R lltRtVtV ;t# )ÚPhimoeTopKRouteriØ  c                ó    <€ V ^8„  d   QhRS[ /# r'   r    )r*   r+   s   "€r,   r-   ÚPhimoeTopKRouter.__annotate__Ù  s   ø€ ÷ 4ñ 4™|ñ 4r/   c                óÜ   <€ \         SV `  VP                  VP                  R R7       VP                  V n        VP
                  V n        VP                  V n        VP                  V n        R# )Frº   N)	r4   r5   rP   rõ   Úrouter_jitter_noiseÚinput_jitter_noiseÚnum_experts_per_tokr3  rö   r   s   &&€r,   r5   ÚPhimoeTopKRouter.__init__Ù  sZ   ø€ Ü‰Ñ˜×+Ñ+¨V×-EÑ-EÈEÐÔRØ#)×#=Ñ#=ˆÔ Ø"(×";Ñ";ˆÔØ×/Ñ/ˆŒ
Ø!×3Ñ3ˆÖr/   c                óx   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          /# ©r(   r’   rE   )rG   rH   rF   )r*   r+   s   "€r,   r-   rC  à  s5   ø€ ÷ 	@ñ 	@¡U§\¡\ð 	@±e¹E¿L¹LÉ%Ï,É,Ð<VÕ6Wñ 	@r/   c                ój  <€ V P                   '       da   V P                  ^ 8”  dP   V\        P                  ! V4      P	                  RV P                  ,
          RV P                  ,           4      ,          p\
        SV `  V4      p\        W P                  V P                   V P                  R7      w  r4W#V3# )r   rM   )r2  r¦   r3  )
r¦   rF  rG   r%  r0  r4   rw   r?  rE  r3  )rA   r’   Úrouter_logitsÚrouting_weightsrÛ   rB   s   &&   €r,   rw   ÚPhimoeTopKRouter.forwardà  s™   ø€ Ø�=�=ˆ=˜T×4Ñ4°qÔ8ØœU×-Ò-¨mÓ<×EÑEØ�d×-Ñ-Õ-¨s°T×5LÑ5LÕ/Lóõ ˆMô ™™¨Ó6ˆÜ,7Ø×&>Ñ&>ÈÏÉÐ^b×^hÑ^hô-
Ñ)ˆð Ð/?Ð?Ð?r/   )rF  rö   rE  r3  ©	rz   r{   r|   r}   r5   rw   r€   r�   r‚   rƒ   s   @@r,   rA  rA  Ø  s!   ù‡ € ÷4ó 4÷	@÷ 	@ó 	@r/   rA  c                   óH   a a€ ] tR tRt oRtV 3R ltV3R lR ltRtVtV ;t	# )ÚPhimoeSparseMoeBlockiì  a¤  
This implementation is
strictly equivalent to standard MoE with full capacity (no
dropped tokens). It's faster since it formulates MoE operations
in terms of block-sparse operations to accommodate imbalanced
assignments of tokens to experts, whereas standard MoE either
(1) drop tokens at the cost of reduced performance or (2) set
capacity factor to number of experts and thus waste computation
and memory on padding.
c                ó  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  V n        VP                  V n	        \        V4      V n        \        V4      V n        VP                  V n        R # ry   )r4   r5   rP   r÷   rø   Úffn_dimrõ   rö   rG  r3  rA  Úrouterrñ   ÚexpertsrF  r   s   &&€r,   r5   ÚPhimoeSparseMoeBlock.__init__ø  si   ø€ Ü‰ÑÔØ ×,Ñ,ˆŒØ×/Ñ/ˆŒØ!×3Ñ3ˆÔØ×/Ñ/ˆŒ
Ü& vÓ.ˆŒÜ$ VÓ,ˆŒØ"(×";Ñ";ˆÖr/   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rJ  rÞ   )r*   r+   s   "€r,   r-   Ú!PhimoeSparseMoeBlock.__annotate__  s'   ø€ ÷ Tñ T¡U§\¡\ð T±e·l±lñ Tr/   c                ó¶  € VP                   w  r#pV P                  '       da   V P                  ^ 8”  dP   V\        P                  ! V4      P                  RV P                  ,
          RV P                  ,           4      ,          pVP                   w  r#pVP                  RV4      pV P                  V4      w  rVpV P                  WV4      pVP                  W#V4      # )r   rM   rd   )	rg   r¦   rF  rG   r%  r0  r–   rT  rU  )	rA   r’   Ú
batch_sizeÚsequence_lengthr÷   Ú_rM  rÛ   r  s	   &&       r,   rw   ÚPhimoeSparseMoeBlock.forward  sÂ   € Ø2?×2EÑ2EÑ/ˆ
 ZØ�=�=ˆ=˜T×4Ñ4°qÔ8ØœU×-Ò-¨mÓ<×EÑEØ�d×-Ñ-Õ-¨s°T×5LÑ5LÕ/Lóõ ˆMð 3@×2EÑ2EÑ/ˆ
 ZØ%×-Ñ-¨b°*Ó=ˆØ/3¯{©{¸=Ó/IÑ,ˆÐ,Ø"Ÿl™l¨=ÈOÓ\ÐØ"×*Ñ*¨:È
ÓSÐSr/   )rU  rS  r÷   rF  rö   rT  r3  rÔ   rƒ   s   @@r,   rQ  rQ  ì  s!   ù‡ € ñ	õ<÷T÷ Tð Tr/   rQ  c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚPhimoeDecoderLayeri  c                ó&   <€ V ^8„  d   QhRS[ RS[/# rµ   r·   )r*   r+   s   "€r,   r-   ÚPhimoeDecoderLayer.__annotate__  s   ø€ ÷ 
ñ 
™|ð 
¹ñ 
r/   c                óN  <€ \         SV `  4        VP                  V n        \        W4      V n        \        V4      V n        \        P                  ! VP                  VP                  R R7      V n
        \        P                  ! VP                  VP                  R R7      V n        R# )T©ÚepsÚelementwise_affineN)r4   r5   rP   r³   Ú	self_attnrQ  Úmlpr   Ú	LayerNormÚrms_norm_epsÚinput_layernormÚpost_attention_layernormrÄ   s   &&&€r,   r5   ÚPhimoeDecoderLayer.__init__  s|   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔä(¨Ó;ˆŒä'¨Ó/ˆŒô  "Ÿ|š|¨F×,>Ñ,>ÀF×DWÑDWÐlpÔqˆÔÜ(*¯ªØ×Ñ F×$7Ñ$7ÈDô)
ˆÖ%r/   c                ó  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[S[,          RS[ P                  /# )	r(   r’   rÇ   Nr    rp   rÈ   rT   rE   )rG   rH   rF   Ú
LongTensorr   r   r   )r*   r+   s   "€r,   r-   ra    sŽ   ø€ ÷ ñ á—|‘|ðñ #¡5§<¡<±·±Ð#=Õ>ÀÕEðñ Ÿ™ tÕ+ð	ñ
 ×&Ñ&¨Õ-ðñ  �ðñ Ñ+Õ,ðñ 
�‰ñr/   c                óÐ   € TpV P                  V4      pV P                  ! RR VRVRVRVRV/VB w  rWq,           pTpV P                  V4      pV P                  V4      pWq,           pV# )r’   rÇ   r    rp   rÈ   rï   )rj  rf  rk  rg  )	rA   r’   rÇ   r    rp   rÈ   rT   Úresidualr\  s	   &&&&&&,  r,   rw   ÚPhimoeDecoderLayer.forward  s�   € ð !ˆØ×,Ñ,¨]Ó;ˆØŸ>š>ñ 
Ø'ð
à 3ð
ð *ð
ð &ð	
ð
 ,ð
ð ñ
Ñˆð !Õ0ˆØ ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ Õ0ˆØÐr/   )rP   rj  rg  rk  rf  )NNNNrO  rƒ   s   @@r,   r_  r_    s   ù‡ € ÷
ó 
÷÷ ò r/   r_  c                   ó®   a a€ ] tR tRt oRtRtR.tR.tRtRt	Rt
RtRtR]! ]^ R7      R]R	]/t]P&                  ! 4       V 3R
 l4       tV3R ltRtVtV ;t# )ÚPhimoePreTrainedModeli:  ÚmodelTr_  rÈ   rL  )rç   r’   Ú
attentionsc                ó„  <€ \         SV `  V4       V P                  P                  p\	        V\
        4      '       dI   \        P                  ! VP                  R VR7       \        P                  ! VP                  R VR7       R# \	        V\        4      '       d&   \        P                  ! VP                  R VR7       R# R# )rÊ   )ÚmeanÚstdN)r4   Ú_init_weightsr)   Úinitializer_rangerh   rñ   ÚinitÚnormal_rü   rý   rA  Úweight)rA   rœ   rx  rB   s   && €r,   ry  Ú#PhimoePreTrainedModel._init_weightsM  s   ø€ ä‰Ñ˜fÔ%Ø�k‰k×+Ñ+ˆÜ�fœm×,Ò,Ü�LŠL˜×,Ñ,°3¸CÕ@Ü�LŠL˜×)Ñ)°¸×=Ü˜Ô 0×1Ò1Ü�LŠL˜Ÿ™¨S°c×:ñ 2r/   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r'   r    )r*   r+   s   "€r,   r-   Ú"PhimoePreTrainedModel.__annotate__:  s   ø‡ ‚ áÑò r/   rï   )rz   r{   r|   r}   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendr   rA  r_  r³   Ú_can_record_outputsrG   r   ry  Ú__annotate_func__r€   r�   r‚   rƒ   s   @@r,   rs  rs  :  sƒ   ù‡ € ð  ÐØ&*Ð#Ø-Ð.ÐØ#4Ð"5ÐØÐØ€NØÐà!ÐØ"&Ðà™Ð(8ÀÔBØÐ+Ø�oðÐð ‡]‚]ƒ_ô;ó ð;÷) … r/   rs  c                   ór   a a€ ] tR tRt oV3R lV 3R llt]]]RV3R lR ll4       4       4       tRt	Vt
V ;t# )ÚPhimoeModeliX  c                ó    <€ V ^8„  d   QhRS[ /# r'   r    )r*   r+   s   "€r,   r-   ÚPhimoeModel.__annotate__Z  s   ø€ ÷ ñ ™|ñ r/   c           	     ó,  <€ \         SV `  V4       VP                  V n        VP                  V n        \
        P                  ! VP                  VP                  V P                  4      V n        \
        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n        \
        P                  ! VP                  VP                  R R7      V n        \#        VR7      V n        RV n        V P)                  4        R# u upi )Trc  ©r)   FN)r4   r5   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	EmbeddingrP   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersr_  Úlayersrh  ri  Únormr#   Ú
rotary_embÚgradient_checkpointingÚ	post_initrÄ   s   && €r,   r5   ÚPhimoeModel.__init__Z  sÑ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸLšL¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m’mÜDIÈ&×JbÑJbÔDcÓdÑDc°yÔ Ö2ÑDcÑdó
ˆŒô —L’L ×!3Ñ!3¸×9LÑ9LÐaeÔfˆŒ	Ü/°vÔ>ˆŒØ&+ˆÔ#ð 	�‰Öùò es   ÂDc                óþ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[S[,          R	S[/# )
r(   Ú	input_idsNr    rp   rÈ   Úinputs_embedsÚ	use_cacherT   rE   )	rG   rn  rH   r   ÚFloatTensorÚboolr   r   r   )r*   r+   s   "€r,   r-   r�  m  s“   ø€ ÷ 4
ñ 4
á×#Ñ# dÕ*ð4
ñ Ÿ™ tÕ+ð4
ñ ×&Ñ&¨Õ-ð	4
ñ
  �ð4
ñ ×(Ñ(¨4Õ/ð4
ñ ˜$•;ð4
ñ Ñ+Õ,ð4
ñ 
 ñ4
r/   c                óÊ  € VR J VR J,          '       d   \        R4      hV'       d   Vf   \        V P                  R7      pVf   V P                  V4      pVfb   Ve   VP	                  4       M^ p\
        P                  ! VP                  ^,          VP                  R7      V,           pVP                  ^ 4      pV P                  P                  f   \        M\        p	V	! V P                  VVVVR7      p
TpV P                  W³R7      pV P                  R V P                  P                    F  pV! V3RV
RVRVR	VR
V/VB pK  	  V P!                  V4      p\#        VVR7      # )Nz:You must specify exactly one of input_ids or inputs_embedsr‘  )r$   )r)   r¢  r    rÈ   rp   )rp   r    rp   rÈ   r£  rÇ   )Úlast_hidden_staterÈ   )Ú
ValueErrorr   r)   r–  Úget_seq_lengthrG   rR   rg   r$   rŠ   Úsliding_windowr   r   rœ  rš  r™  r›  r   )rA   r¡  r    rp   rÈ   r¢  r£  rT   Úpast_seen_tokensÚmask_functionÚcausal_maskr’   rÇ   Údecoder_layers   &&&&&&&,      r,   rw   ÚPhimoeModel.forwardj  s}  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZç˜Ò0Ü*°$·+±+Ô>ˆOàÒ Ø ×-Ñ-¨iÓ8ˆMàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLà.2¯k©k×.HÑ.HÒ.PÕ*ÔVwˆÙ#Ø—;‘;Ø'Ø)Ø+Ø%ô
ˆð &ˆØ"Ÿo™o¨m˜oÓWÐà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÓIˆMÙ)Øñà*ðð *ðð !0ð	ð
 $ðð %8ðð ñŠMñ Jð Ÿ	™	 -Ó0ˆä%Ø+Ø+ô
ð 	
r/   )r–  r�  rš  r›  r“  rœ  r”  )NNNNNN)rz   r{   r|   r}   r5   r   r   r   rw   r€   r�   r‚   rƒ   s   @@r,   r�  r�  X  s=   ù‡ € ÷ó ð   ØØ÷4
ñ 4
ó ó ó  ÷4
ð 4
r/   r�  c          	      ó  € V ^8„  d   QhR\         P                  \        \         P                  ,          ,          R,          R\        R,          R\         P                  R,          R\         P                  \        ,          /# )r(   Úgate_logitsNrö   r    rE   )rG   rH   rF   r”   )r*   s   "r,   r-   r-   ¤  sc   € ÷ B&ñ B&Ü—‘¤¤e§l¡lÕ 3Õ3°dÕ:ðB&ä�t•ðB&ô —L‘L 4Õ'ð	B&ô
 ‡\�\”CÕñB&r/   c           	     ó  € V e   \        V \        4      '       g   ^ # V ^ ,          P                  p\        P                  ! V\        P
                  VR7      p\        P                  ! V\        P
                  VR7      pRpVe1   VP                  R4      P                  V\        P
                  R7      pV  EFˆ  p	\        P                  P                  P                  V	P                  V4      RR7      p
\        P                  ! W¢RR7      w  r¼Vf~   V\        P                  ! VP                  R4      VR7      P                  4       ,           pWjP                  4       P                  ^ R7      ,           pWzP                  ^ ,          ,           pKÙ  V\        P                  ! V\        P
                  VR7      P!                  ^ VP                  R4      XP#                  V4      4      ,           pWjP                  4       VP%                  R4      ,          P                  ^ R7      ,           pWxP                  4       ,           pEK‹  	  WW,          pWg,          p\        P                  ! WÞP%                  ^ 4      ,          4      pWñ,          # )ax  
Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch.

See Switch Transformer (https://huggingface.co/papers/2101.03961) for more details. This function implements the loss
function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between
experts is too unbalanced.

Args:
    gate_logits:
        Logits from the `gate`, should be a tuple of model.config.num_hidden_layers tensors of
        shape [batch_size X sequence_length, num_experts].
    num_experts:
        Number of experts
    top_k:
        The number of experts to route per-token, can be also interpreted as the `top-k` routing
        parameter.
    attention_mask (`torch.Tensor`, *optional*):
        The attention_mask used in forward function
        shape [batch_size X sequence_length] if not None.

Returns:
    The auxiliary loss.
r]   rÊ   )r$   rN   rb   )Ú	minlengthrd   )rh   rF   r$   rG   Úzerosr«   r–   rS   r   r©   rª   ÚtopkÚbincountrI   r  rg   rë   Úrepeat_interleaverŠ   )r±  rö   r3  r    Úcompute_deviceÚtokens_per_expert_sumÚrouter_prob_sumÚ
total_rowsÚ	flat_maskÚ
layer_gaterM  r\  rÛ   Útokens_per_expertÚrouter_prob_per_expertÚoverall_losss   &&&&            r,   Úload_balancing_loss_funcrÁ  ¤  sÿ  € ð: Ò¤*¨[¼%×"@Ò"@Ùð ! •^×*Ñ*€NÜ!ŸKšK¨¼5¿=¹=ÐQ_Ô`ÐÜ—k’k +´U·]±]È>ÔZ€OØ€JàÒ!à"×*Ñ*¨2Ó.×1Ñ1¸ÌuÏ}É}Ð1Ó]ˆ	ä!ˆ
ÜŸ(™(×-Ñ-×5Ñ5°j·m±mÀNÓ6SÐY[Ð5Ó\ˆÜ#Ÿjšj¨ÀRÔHÑˆØÒ!ð &¬¯ªÐ7G×7OÑ7OÐPRÓ7SÐ_jÔ(k×(qÑ(qÓ(sÕsð "ð .×0EÑ0EÓ0G×0KÑ0KÐPQÐ0KÓ0RÕRˆOØ#×&;Ñ&;¸AÕ&>Õ>ŠJð %:¼E¿KºKØ¤5§=¡=¸ô=ç‰l˜1Ð.×6Ñ6°rÓ:¸I×<WÑ<WÐX]Ó<^Ó_õ%`Ð!ð .×1FÑ1FÓ1HÈ9×K^ÑK^Ð_aÓKbÕ1b×0gÑ0gÐlmÐ0gÓ0nÕnˆOØ#§m¡m£oÕ5‹Jñ# "ð& .Õ:ÐØ,Õ9Ðä—9’9Ð.×1QÑ1QÐRSÓ1TÕTÓU€LØÕ%Ð%r/   c                   ó”   a a€ ] tR tRt oRR/tRR/tRR.R.3/tRR/tV 3R	 lt]	]
RV3R
 lR ll4       4       tRV 3R lltRtVtV ;t# )ÚPhimoeForCausalLMié  zlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputr’   ÚlogitsÚkeep_full_weightc                óz  <€ \         SV `  V4       \        V4      V n        VP                  V n        \
        P                  ! VP                  VP                  V P                  P                  R 7      V n
        VP                  V n        VP                  V n        VP                  V n        V P                  4        R# )rº   N)r4   r5   r�  rt  r”  r   r¾   rP   r)   Úlm_head_biasrÄ  Úrouter_aux_loss_coefrõ   rö   rG  rž  r   s   &&€r,   r5   ÚPhimoeForCausalLM.__init__ð  s‰   ø€ Ü‰Ñ˜Ô Ü  Ó(ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈTÏ[É[×MeÑMeÔfˆŒØ$*×$?Ñ$?ˆÔ!Ø!×3Ñ3ˆÔØ#)×#=Ñ#=ˆÔ ð 	�‰Ör/   c                ód  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[R,          R
S[S[ P                  ,          RS[S[,          RS[	/# )r(   r¡  Nr    rp   rÈ   r¢  Úlabelsr£  Úoutput_router_logitsÚlogits_to_keeprT   rE   )
rG   rn  rH   r   r¤  r¥  r”   r   r   r   )r*   r+   s   "€r,   r-   ÚPhimoeForCausalLM.__annotate__þ  sÖ   ø€ ÷ P
ñ P
á×#Ñ# dÕ*ðP
ñ Ÿ™ tÕ+ðP
ñ ×&Ñ&¨Õ-ð	P
ñ
  �ðP
ñ ×(Ñ(¨4Õ/ðP
ñ × Ñ  4Õ'ðP
ñ ˜$•;ðP
ñ # T�kðP
ñ ™eŸl™lÕ*ðP
ñ Ñ+Õ,ðP
ñ 
#ñP
r/   c
                ó¦  € Ve   TMV P                   P                  pV P                  ! RRVRVRVRVRVRVRV/V
B pVP                  p\	        V	\
        4      '       d   \        V	) R4      MT	pV P                  VR	VR	3,          4      pRpVe   V P                  ! WæV P                  3/ V
B pRpV'       dc   \        VP                  V P                  V P                  V4      pVe3   WðP                  VP                  VP                   4      ,          ,          p\#        VVVVP$                  VP&                  VP(                  VP                  R
7      # )aú  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

Example:

```python
>>> from transformers import AutoTokenizer, PhimoeForCausalLM

>>> model = PhimoeForCausalLM.from_pretrained("mistralai/Phimoe-8x7B-v0.1")
>>> tokenizer = AutoTokenizer.from_pretrained("mistralai/Phimoe-8x7B-v0.1")

>>> prompt = "Hey, are you conscious? Can you talk to me?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
```Nr¡  r    rp   rÈ   r¢  r£  rÎ  rc   )ÚlossÚaux_lossrÆ  rÈ   r’   ru  rL  rï   )r)   rÎ  rt  r§  rh   r”   ÚslicerÄ  Úloss_functionr”  rÁ  rL  rö   rG  rÊ  rS   r$   r   rÈ   r’   ru  )rA   r¡  r    rp   rÈ   r¢  rÍ  r£  rÎ  rÏ  rT   Úoutputsr’   Úslice_indicesrÆ  rÒ  rÓ  s   &&&&&&&&&&,      r,   rw   ÚPhimoeForCausalLM.forwardü  sx  € ðN %9Ò$DÑ È$Ï+É+×JjÑJjð 	ð
 +/¯*ª*ñ 	+
Øð	+
à)ð	+
ð &ð	+
ð ,ð		+
ð
 (ð	+
ð  ð	+
ð "6ð	+
ð ñ	+
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒØ×%Ò% f°d·o±oÑPÈÑPˆDàˆßÜ/Ø×%Ñ%Ø× Ñ Ø×(Ñ(Øó	ˆHð Ò!Ø×1Ñ1°H·K±KÀÇÁÓ4LÕLÕL�ä(ØØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø!×/Ñ/ô
ð 	
r/   c                óJ  <€ V'       d|   \        V P                  R 4      '       d`   VP                  ^,          V P                  P                  ^,           8¼  d-   VP	                  4       p	W�P                  P                  8:  d   Rp\
        SV `  ! R	RVRVRVRVRVRVRV/VB p
V
# )
rZ   Nr¡  rÈ   r    r¢  rp   r£  rÏ  rï   )Úhasattrr)   rg   rZ   r©  r4   Úprepare_inputs_for_generation)rA   r¡  rÈ   r    r¢  rp   r£  rÏ  rT   Úpast_lengthÚmodel_inputsrB   s   &&&&&&&&,  €r,   rÛ  Ú/PhimoeForCausalLM.prepare_inputs_for_generationQ  sÁ   ø€ ÷" Ü˜Ÿ™Ð%G×HÒHØ—‘ Õ" d§k¡k×&RÑ&RÐUVÕ&VÔVà)×8Ñ8Ó:ˆKØŸk™k×JÑJÔJØ"&�ä‘wÒ<ñ 	
Øð	
à+ð	
ð *ð	
ð (ð		
ð
 &ð	
ð  ð	
ð *ð	
ð ñ	
ˆð Ðr/   )rÄ  rt  rö   rG  rÊ  r”  )	NNNNNNNNr   )NNNNTr   )rz   r{   r|   r}   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr5   r   r   rw   rÛ  r€   r�   r‚   rƒ   s   @@r,   rÃ  rÃ  é  so   ù‡ € à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÐ/Ð0€Jõ
ð Ø÷P
ñ P
ó ó ðP
÷f#÷ #r/   rÃ  c                   ó   € ] tR tRtRtR# )ÚPhimoeForSequenceClassificationiw  rï   N)rz   r{   r|   r}   r€   rï   r/   r,   rä  rä  w  s   † Ó`cr/   rä  )é   )rÊ   )r(   )Nr(   N)rs  r�  rÃ  rä  )GÚcollections.abcr   rG   r   Ú r   r{  Úactivationsr   Úcache_utilsr   r   Ú
generationr	   Úintegrationsr
   r   r   Úmasking_utilsr   r   Úmodeling_layersr   r   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   r   Úconfiguration_phimoer!   r£   r#   r‡   r�   rš   r±   r³   ÚautogradÚFunctionr×   rñ   r?  r¾   rA  rQ  r_  rs  r�  rÁ  rÃ  rä  Ú__all__rï   r/   r,   Ú<module>rú     s–  ðõ, %ã Ý å &Ý !ß .Ý )ß hÑ hß Rß [ß Qß Kß FÝ &ß IÑ IÝ 0ß Gß EÝ .ô=0˜BŸI™Iô =0ò@(ñ Ð-Ó.óó /ðõ2	U÷%ñ2 Ð)Ó*ô@)�b—i‘ió @)ó +ð@)ôF;
�u—~‘~×.Ñ.ô ;
ð| ô$#�B—I‘Ió $#ó ð$#ôNxôv@�r—y‘yô @ô(!T˜2Ÿ9™9ô !TôH'Ð3ô 'ðT ô;˜Oó ;ó ð;ð: ôH
Ð'ó H
ó ðH
÷VB&ðJ ôJÐ-¨ó Jó ðJôZ dÐ&FÐH]Ô cò k‚r/   