+
    œ°³jî‡  ã                   óZ  € ^ RI Ht ^ RIt^ RIHt ^ RIHt ^RIHt	 ^RI
Ht ^RIHtHt ^RIHt ^R	IHt ^R
IHt ^RIHtHt ^RIHtHt ^RIHtHt ^RIHtH t  ^RI!H"t" ^RI#H$t$H%t%H&t&H't' ^RI(H)t) ^RI*H+t+H,t, ^RI-H.t.H/t/ ^RI0H1t1 ]'Pd                  ! ]34      t4]! R4       ! R R]Pj                  4      4       t6 ! R R]Pj                  4      t7 ! R R]Pj                  4      t8 ! R R]Pj                  4      t9 ! R R]Pj                  4      t: ! R  R!]Pj                  4      t;R" t<]! R#4      R7R$ l4       t=R% R& lt>R8R' R( llt? ! R) R*]Pj                  4      t@ ! R+ R,]4      tA]% ! R- R.] 4      4       tB]% ! R/ R0]B4      4       tCR9R1 R2 lltD ! R3 R4]B]4      tE ! R5 R6]]B4      tF. R:OtGR# );é    )ÚCallableN)Únn)Ú
functional)Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_kernel_forward_from_hub)Úcreate_causal_mask)Ú GenericForSequenceClassificationÚGradientCheckpointingLayer)ÚMoeCausalLMOutputWithPastÚMoeModelOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputs©ÚJetMoeConfigÚRMSNormc                   óZ   a a€ ] tR t^0t oRV3R lV 3R llltV3R lR ltR tRtVtV ;t	# )ÚJetMoeRMSNormc                ó$   <€ V ^8„  d   QhRS[ RR/# )é   ÚepsÚreturnN)Úfloat)ÚformatÚ__classdict__s   "€Úk/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/jetmoe/modeling_jetmoe.pyÚ__annotate__ÚJetMoeRMSNorm.__annotate__2   s   ø€ ÷ $ñ $©ð $¸$ñ $ó    c                óŽ   <€ \         SV `  4        \        P                  ! \        P
                  ! V4      4      V n        W n        R# )z,
JetMoeRMSNorm is equivalent to T5LayerNorm
N)ÚsuperÚ__init__r   Ú	ParameterÚtorchÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizer&   Ú	__class__s   &&&€r+   r1   ÚJetMoeRMSNorm.__init__2   s/   ø€ ô 	‰ÑÔÜ—l’l¤5§:¢:¨kÓ#:Ó;ˆŒØ #Ör.   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r%   Úhidden_statesr'   )r3   ÚTensor)r)   r*   s   "€r+   r,   r-   :   s#   ø€ ÷ ;ñ ;¡U§\¡\ð ;±e·l±lñ ;r.   c                ó:  € VP                   pVP                  \        P                  4      pVP	                  ^4      P                  RRR7      pV\        P                  ! W0P                  ,           4      ,          pV P                  VP                  V4      ,          # )r%   T)Úkeepdiméÿÿÿÿ)	ÚdtypeÚtor3   Úfloat32ÚpowÚmeanÚrsqrtr6   r5   )r7   r<   Úinput_dtypeÚvariances   &&  r+   ÚforwardÚJetMoeRMSNorm.forward:   sw   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯ª°H×?TÑ?TÕ4TÓ(UÕUˆØ�{‰{˜]×-Ñ-¨kÓ:Õ:Ð:r.   c                ó^   € \        V P                  P                  4       R V P                   2# )z, eps=)Útupler5   Úshaper6   )r7   s   &r+   Ú
extra_reprÚJetMoeRMSNorm.extra_reprA   s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr.   )r6   r5   )g�íµ ÷Æ°>)
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r1   rI   rN   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r9   r*   s   @@r+   r#   r#   0   s%   ù‡ € ÷$õ $÷;ð ;÷Jò Jr.   r#   c                   óÎ   a a€ ] tR t^Et o]! RRR7      R
V3R lV 3R lll4       t]]! RRR7      R
V3R lR ll4       4       t]P                  ! 4       ]
R 4       4       tR	tVtV ;t# )ÚJetMoeRotaryEmbeddingÚdevicez5.18)Úversionc                ó    <€ V ^8„  d   QhRS[ /# ©r%   Úconfigr   )r)   r*   s   "€r+   r,   Ú"JetMoeRotaryEmbedding.__annotate__G   s   ø€ ÷ Oñ O™|ñ Or.   c                óä  <€ \         SV `  4        VP                  V n        VP                  V n        Wn        V P
                  P                  R ,          V n        V P                  pV P                  R8w  d   \        V P                  ,          pV! V P
                  V4      w  q@n
        \        P                  ! VRR7      V n        \        P                  ! VP                  4       RR7      V n        R# )Ú	rope_typeÚdefaultF)Ú
persistentN)r0   r1   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr^   Úrope_parametersra   Úcompute_default_rope_parametersr   Úattention_scalingr   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)r7   r^   rZ   Úrope_init_fnrk   r9   s   &&&  €r+   r1   ÚJetMoeRotaryEmbedding.__init__F   s¬   ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àŒàŸ™×4Ñ4°[ÕAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÔ&Ü.¨t¯~©~Õ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÔ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÔ!NˆÖr.   c                óP   <€ V ^8„  d   QhRS[ RS[S[P                  S[3,          /# )r%   r^   r'   )r    rL   r3   r=   r(   )r)   r*   s   "€r+   r,   r_   Y   s.   ø€ ÷ 5ñ 5±ð 5ÑX]Ñ^c×^jÑ^jÑlqÐ^qÕXrñ 5r.   c           	     ó0  € V P                   R,          p\        V RR4      ;'       g    V P                  V P                  ,          pRpRV\        P
                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚhead_dimNg      ð?©rA   )rg   Úgetattrr8   Únum_attention_headsr3   Úaranger(   rB   )r^   rZ   ÚkwargsÚbaseÚdimÚattention_factorrk   s   &&,    r+   rh   Ú5JetMoeRotaryEmbedding.compute_default_rope_parametersW   s~   € ð ×%Ñ% lÕ3ˆÜ�f˜j¨$Ó/×cÐc°6×3EÑ3EÈ×IcÑIcÕ3cˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r.   c                ó^  € V P                   R,          P                  VP                  ^ ,          R	^4      P                  \        P
                  VP                  R7      pVR
,          P                  4       p\        VP                  P                  \        4      '       d2   VP                  P                  R8w  d   VP                  P                  MRp\        VRR7      ;_uu_ 4        W4,          P                  ^^4      p\        P                  ! Wf3R	R7      pVP                  4       V P                  ,          pVP                  4       V P                  ,          p	R R R 4       XP                  VP                   R7      X	P                  VP                   R7      3#   + '       g   i     LH; i)N©rA   rZ   ÚmpsÚcpuF)Údevice_typeÚenabled©rz   rt   )NºNNNNr@   )r„   Nr„   )rk   ÚexpandrM   rB   r3   r(   rZ   Ú
isinstanceÚtypeÚstrr   Ú	transposeÚcatÚcosri   ÚsinrA   )
r7   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedr�   ÚfreqsÚembr‹   rŒ   s
   &&&       r+   rI   ÚJetMoeRotaryEmbedding.forwardk   s2  € ð �M‰M˜-Õ(×/Ñ/°×0BÑ0BÀ1Õ0EÀrÈ1ÓM×PÑPÔW\×WbÑWbÐkl×ksÑksÐPÓtð 	ð !-¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆä¨¸U×CÖCØ&Õ>×IÑIÈ!ÈQÓOˆEÜ—)’)˜U˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆC÷	 Dð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷ D×Cús   Ã(A4FÆF,	)ri   r^   rk   re   rm   rf   ra   ©N)rP   rQ   rR   rS   r   r1   Ústaticmethodrh   r3   Úno_gradr   rI   rT   rU   rV   rW   s   @@r+   rY   rY   E   su   ù‡ € Ù�X vÔ.÷Oô Oó /ðOð  Ù�X vÔ.÷5ñ 5ó /ó ð5ð$ ‡]‚]ƒ_Øñ<ó ó ÷<ð <r.   rY   c                   óD   a a€ ] tR t^~t oV3R lV 3R lltR tRtVtV ;t# )ÚJetMoeParallelExpertsc                ó0   <€ V ^8„  d   QhRS[ RS[ RS[ RR/# )r%   Únum_expertsÚ
input_sizeÚoutput_sizer'   N©Úint)r)   r*   s   "€r+   r,   Ú"JetMoeParallelExperts.__annotate__   s)   ø€ ÷ 'ñ '¡Cð '±Sð 'Ásð 'Ètñ 'r.   c                ó¨   <€ \         SV `  4        \        P                  ! \        P
                  ! WV4      4      V n        Wn        W n        W0n	        R# )aS  
Initialize the JetMoeParallelExperts module.
The experts weights are stored in [num_experts, output_size, input_size] format. Such that it's compatible with
many MoE libraries, such as [Megablock](https://github.com/databricks/megablocks) and
[ScatterMoE](https://github.com/shawntan/scattermoe), as well as the
[MoE kernel](https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/layers/fused_moe/fused_moe.py)
used in vllm.

Args:
    num_experts (int):
        Number of experts.
    input_size (int):
        Size of the input.
    output_size (int):
        Size of the output.
N)
r0   r1   r   r2   r3   Úemptyr5   rš   r›   rœ   )r7   rš   r›   rœ   r9   s   &&&&€r+   r1   ÚJetMoeParallelExperts.__init__   s<   ø€ ô" 	‰ÑÔÜ—l’l¤5§;¢;¨{ÈÓ#TÓUˆŒØ&ÔØ$ŒØ&Ör.   c                ó  € VP                  V^ R7      p. p\        V P                  4       F@  pVP                  \        P
                  ! W5,          V P                  V,          4      4       KB  	  \        P                  ! V^ R7      pV# )z¼
Forward pass of the JetMoeParallelExperts module.

Args:
    inputs (Tensor):
        Input tensor.
    expert_size:
        Expert size information.

Returns:
    Tensor: Output tensor.
rƒ   )	ÚsplitÚrangerš   ÚappendÚFÚlinearr5   r3   rŠ   )r7   ÚinputsÚexpert_sizeÚ
input_listÚoutput_listÚiÚresultss   &&&    r+   rI   ÚJetMoeParallelExperts.forward–   sh   € ð —\‘\ +°1�\Ó5ˆ
ØˆÜ�t×'Ñ'Ö(ˆAØ×ÑœqŸxšx¨
­°t·{±{À1µ~ÓFÖGñ )ä—)’)˜K¨QÔ/ˆØˆr.   )r›   rš   rœ   r5   ©	rP   rQ   rR   rS   r1   rI   rT   rU   rV   rW   s   @@r+   r˜   r˜   ~   s   ù‡ € ÷'ó '÷.ò r.   r˜   c                   óD   a a€ ] tR t^«t oV3R lV 3R lltR tRtVtV ;t# )ÚJetMoeTopKGatingc                ó,   <€ V ^8„  d   QhRS[ RS[ RS[ /# )r%   r›   rš   Útop_kr�   )r)   r*   s   "€r+   r,   ÚJetMoeTopKGating.__annotate__¬   s'   ø€ ÷ Dñ D¡3ð D±Sð DÁñ Dr.   c                ó‚   <€ \         SV `  4        W n        Wn        W0n        \
        P                  ! WRR7      V n        R# )zÒ
Initialize the top-k gating mechanism.

Args:
    input_size (`int`):
        Size of the input.
    num_experts (`int`):
        Number of experts.
    top_k (`int`):
        Number of top experts to select.
F©ÚbiasN)r0   r1   rš   r›   r´   r   ÚLinearÚlayer)r7   r›   rš   r´   r9   s   &&&&€r+   r1   ÚJetMoeTopKGating.__init__¬   s2   ø€ ô 	‰ÑÔà&ÔØ$ŒØŒ
ä—Y’Y˜z¸UÔCˆŽ
r.   c                óš  € V P                  V4      P                  4       pVP                  V P                  ^R7      w  r4\        P
                  ! V^R7      P                  V4      p\        P                  ! VP                  ^ 4      V P                  .VP                  VP                  R7      pVP                  ^V^4      pVP                  4       P                  ^ 4      pVP                  4       pVP!                  4       p	V	P#                  ^ 4      w  r«VP%                  V P                  RR7      pVP!                  4       pW[,          pW¼WØV3# )é   rƒ   r~   Útrunc)Úrounding_mode)rº   r(   Útopkr´   r3   ÚsoftmaxÚtype_asÚzerosÚsizerš   rA   rZ   ÚscatterÚlongÚsumÚtolistÚflattenÚsortÚdiv)r7   r<   ÚlogitsÚtop_k_logitsÚtop_k_indicesÚtop_k_gatesrÃ   Úgatesrª   Útop_k_expertsÚ_Úindex_sorted_expertsÚbatch_indexÚbatch_gatess   &&            r+   rI   ÚJetMoeTopKGating.forwardÀ   s"  € à—‘˜MÓ*×0Ñ0Ó2ˆØ&,§k¡k°$·*±*À! kÓ&DÑ#ˆÜ—m’m L°aÔ8×@Ñ@ÀÓOˆô —’Ø×Ñ˜aÓ  $×"2Ñ"2Ð3¸;×;LÑ;LÐU`×UgÑUgô
ˆð —‘˜a °Ó2ˆØ—j‘j“l×&Ñ& qÓ)ˆð "×(Ñ(Ó*ˆð &×-Ñ-Ó/ˆØ"/×"4Ñ"4°QÓ"7ÑˆØ*×.Ñ.¨t¯z©zÈÐ.ÓQˆð "×)Ñ)Ó+ˆØ!Õ7ˆà#°+ÈFÐRÐRr.   )r›   rº   rš   r´   r°   rW   s   @@r+   r²   r²   «   s   ù‡ € ÷Dó D÷(Sò Sr.   r²   c                   óH   a a€ ] tR t^Üt oRtV3R lV 3R lltR tRtVtV ;t	# )Ú	JetMoeMoEz¦
A Sparsely gated mixture of experts layer with 1-layer Feed-Forward networks as experts.

Args:
    config:
        Configuration object with model hyperparameters.
c                ó    <€ V ^8„  d   QhRS[ /# r]   r   )r)   r*   s   "€r+   r,   ÚJetMoeMoE.__annotate__å   s   ø€ ÷ 
ñ 
™|ñ 
r.   c                óT  <€ \         SV `  4        VP                  V n        VP                  V n        \
        VP                  ,          V n        \        P                  P                  \        P                  ! V P                  4      4      V n        \        VP                  V P                  V P                  ^,          4      V n        \        VP                  V P                  V P                  4      V n        \#        V P                  VP                  VP$                  R7      V n        R# )r%   ©r›   rš   r´   N)r0   r1   r8   r›   Úintermediate_sizer   Úactivation_functionÚ
activationr3   r   r2   r¡   r¸   r˜   Únum_local_expertsÚinput_linearÚoutput_linearr²   Únum_experts_per_tokÚrouter©r7   r^   r9   s   &&€r+   r1   ÚJetMoeMoE.__init__å   sÏ   ø€ Ü‰ÑÔà ×,Ñ,ˆŒØ!×3Ñ3ˆÔÜ  ×!;Ñ!;Õ<ˆŒÜ—H‘H×&Ñ&¤u§{¢{°4·?±?Ó'CÓDˆŒ	Ü1°&×2JÑ2JÈDÏOÉOÐ]a×]mÑ]mÐpqÕ]qÓrˆÔÜ2°6×3KÑ3KÈT×M]ÑM]Ð_c×_nÑ_nÓoˆÔä&Ø—‘Ø×0Ñ0Ø×,Ñ,ô
ˆŽr.   c                ó\  € VP                  4       w  r#pVP                  RV4      pV P                  V4      w  rVrxp	W,          p
V P                  W¨4      pVP	                  ^RR7      pV P                  V^ ,          4      V^,          ,          pV P                  W¸4      pW×R,          ,          p\        P                  ! W#,          V P                  3VP                  VP                  R7      pVP                  ^ Wm4      pVP                  W#V P                  4      pWðP                  ,           pV# )z¶
Forward pass of the mixture of experts layer.

Args:
    layer_input (Tensor):
        Input tensor.

Returns:
    Tensor:
        Output tensor.
    Tensor:
        Router logits.
rƒ   r~   r@   ©r„   N)rÄ   Úreshaperä   rá   Úchunkrß   râ   r3   rÃ   r›   rA   rZ   Ú	index_addÚviewr¸   )r7   Úlayer_inputÚbszÚlengthÚemb_sizerÒ   rÔ   rÕ   rª   Úrouter_logitsÚexpert_inputsr<   Úchunked_hidden_statesÚexpert_outputsrÃ   Úlayer_outputs   &&              r+   rI   ÚJetMoeMoE.forwardõ   s  € ð !,× 0Ñ 0Ó 2Ñˆ�XØ!×)Ñ)¨"¨hÓ7ˆØBFÇ+Á+ÈkÓBZÑ?ˆ˜°-à#Õ0ˆØ×)Ñ)¨-ÓEˆØ -× 3Ñ 3°A¸2Ð 3Ó >ÐØŸ™Ð(=¸aÕ(@ÓAÐDYÐZ[ÕD\Õ\ˆØ×+Ñ+¨MÓGˆà'°gÕ*>Õ>ˆä—’˜S�\¨4¯?©?Ð;À>×CWÑCWÐ`n×`uÑ`uÔvˆØ—‘ q¨+ÓFˆØ#×(Ñ(¨°d·o±oÓFˆØ#§i¡iÕ/ˆØÐr.   )rß   r¸   r8   rá   r›   râ   rä   ©
rP   rQ   rR   rS   Ú__doc__r1   rI   rT   rU   rV   rW   s   @@r+   rØ   rØ   Ü   s   ù‡ € ñ÷
ó 
÷ ò r.   rØ   c                   óT   a a€ ] tR tRt oRtV3R lV 3R lltR tR tR tRt	Vt
V ;t# )	Ú	JetMoeMoAi  z±
A Sparsely gated mixture of attention layer with pairs of query- and output-projections as experts.

Args:
    config:
        Configuration object with model hyperparameters.
c                ó    <€ V ^8„  d   QhRS[ /# r]   r   )r)   r*   s   "€r+   r,   ÚJetMoeMoA.__annotate__  s   ø€ ÷ 
ñ 
™|ñ 
r.   c                ót  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  VP                  ,          V n        VP                  V n	        \        P                  P                  \        P                  ! V P
                  4      4      V n        \        V P                  V P
                  V P                  4      V n        \        V P                  V P                  V P
                  4      V n        \%        V P
                  V P                  V P                  R 7      V n        R# )rÜ   N)r0   r1   rà   rš   r8   r›   Úkv_channelsÚnum_key_value_headsrã   r´   r3   r   r2   r¡   r¸   r˜   rá   râ   r²   rä   rå   s   &&€r+   r1   ÚJetMoeMoA.__init__  sÛ   ø€ Ü‰ÑÔà!×3Ñ3ˆÔØ ×,Ñ,ˆŒØ!×-Ñ-°×0JÑ0JÕJˆÔØ×/Ñ/ˆŒ
Ü—H‘H×&Ñ&¤u§{¢{°4·?±?Ó'CÓDˆŒ	ä1°$×2BÑ2BÀDÇOÁOÐUY×UeÑUeÓfˆÔÜ2°4×3CÑ3CÀT×EUÑEUÐW[×WfÑWfÓgˆÔä&Ø—‘Ø×(Ñ(Ø—*‘*ô
ˆŽr.   c                ó¶  € VP                  4       w  r#pVP                  RV4      pV P                  V4      w  rVrxp	WVWx3p
W,          pV P                  W¸4      p\        P
                  ! W#,          V P                  ,          V P                  3VP                  VP                  R7      pVP                  ^ W\4      pVP                  W#V P                  R4      pWéV
3# )zq
Map inputs to attention experts according to routing decision and compute query projection inside each experts.
r~   r@   )rÄ   ré   rä   rá   r3   rÃ   r´   r8   rA   rZ   rë   rì   )r7   rí   rî   rï   rð   rÓ   rÔ   rÕ   rª   rñ   Ú	topo_inforò   rô   rÃ   rõ   s   &&             r+   ÚmapÚJetMoeMoA.map1  sÖ   € ð !,× 0Ñ 0Ó 2Ñˆ�XØ!×)Ñ)¨"¨hÓ7ˆØUY×U`ÑU`ÐalÓUmÑRÐ¨;À]Ø)¸ÐQˆ	ð $Õ0ˆØ×*Ñ*¨=ÓFˆô —’Ø�\˜DŸJ™JÕ&¨×(8Ñ(8Ð9À×AUÑAUÐ^l×^sÑ^sô
ˆð —‘ qÐ*>ÓOˆØ#×(Ñ(¨°d·j±jÀ"ÓEˆØ¨IÐ5Ð5r.   c                ó¨  € VP                  4       w  r4rVVP                  RV4      pVw  rxršW,          pV P                  Wº4      pWÉR,          ,          p\        P                  ! W4,          V P
                  3VP                  VP                  R7      pVP                  ^ WŒ4      pVP                  W4V P
                  4      pWàP                  ,           pV# )ze
Compute output projection inside each attention experts and merge the outputs of different experts.
r~   r@   rè   )rÄ   ré   râ   r3   rÃ   r›   rA   rZ   rë   rì   r¸   )r7   rí   r  rî   rï   Úkr8   rÓ   rÔ   rÕ   rª   rò   rô   rÃ   rõ   s   &&&            r+   ÚreduceÚJetMoeMoA.reduceH  sÁ   € ð '2×&6Ñ&6Ó&8Ñ#ˆ�QØ!×)Ñ)¨"¨kÓ:ˆØFOÑCÐ¨;ð $Õ9ˆØ×+Ñ+¨MÓGˆð (°gÕ*>Õ>ˆô —’˜S�\¨4¯?©?Ð;À>×CWÑCWÐ`n×`uÑ`uÔvˆØ—‘ q¨+ÓFˆØ#×(Ñ(¨°d·o±oÓFˆØ#§i¡iÕ/ˆØÐr.   c                ó   € \        R 4      h)z-This module doesn't support call and forward.)ÚNotImplementedError)r7   rí   s   &&r+   rI   ÚJetMoeMoA.forward^  s   € Ü!Ð"QÓRÐRr.   )r¸   r8   rá   r›   rš   râ   rä   r´   )rP   rQ   rR   rS   rø   r1   r  r  rI   rT   rU   rV   rW   s   @@r+   rú   rú     s*   ù‡ € ñ÷
ó 
ò$6ò.÷,Sò Sr.   rú   c                óÌ   € V RRV P                   R,          ^,          13,          pV RV P                   R,          ^,          R13,          p\        P                  ! V) V3RR7      # )z*Rotates half the hidden dims of the input..Nrƒ   r@   )rM   r3   rŠ   )r�   Úx1Úx2s   &  r+   Úrotate_halfr  b  sZ   € à	
ˆ3Ð"�!—'‘'˜"•+ Õ"Ð"Ð"Õ	#€BØ	
ˆ3�—‘˜•˜qÕ Ñ"Ð"Õ	#€BÜ�9Š9�r�c˜2�Y BÔ'Ð'r.   Úrotary_pos_embc                óÈ   € VP                  V4      pVP                  V4      pW,          \        V 4      V,          ,           pW,          \        V4      V,          ,           pWV3# )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)Ú	unsqueezer  )Úqr  r‹   rŒ   Úunsqueeze_dimÚq_embedÚk_embeds   &&&&&  r+   Úapply_rotary_pos_embr  i  sS   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�wœ; q›>¨CÕ/Õ0€GØ�wœ; q›>¨CÕ/Õ0€GØÐÐr.   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r%   r<   Ún_repr'   )r3   r=   rž   )r)   s   "r+   r,   r,   ƒ  s.   € ÷ 	Uñ 	UœUŸ\™\ð 	U´#ð 	U¼%¿,¹,ñ 	Ur.   c                ó˜   € V P                   w  r#rEV^8X  d   V # V R,          P                  W#WV4      p V P                  W#V,          WE4      # )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
)r„   r„   Nr„   r„   )rM   r…   ré   )r<   r  Úbatchrÿ   Úslenrs   s   &&    r+   Ú	repeat_kvr  ƒ  sU   € ð
 2?×1DÑ1DÑ.€E Ø�„zØÐØ!Ð"2Õ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÕ(CÀTÓTÐTr.   c                ó   € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R\        R	\
        \        ,          /# )
r%   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNÚscalingÚdropoutrx   )r   ÚModuler3   r=   r(   r   r   )r)   s   "r+   r,   r,   �  sw   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ð%ô ð%ô Ô'Õ(ñ%r.   c                 ó"  € \        W P                  4      p\        W0P                  4      p	\        P                  ! WP	                  ^^4      4      V,          p
Ve	   W¤,           p
\
        P                  P                  V
R\        P                  R7      P                  VP                  4      p
\
        P                  P                  W¦V P                  R7      p
\        P                  ! W©4      pVP	                  ^^4      P                  4       pWº3# )r%   )rz   rA   )ÚpÚtrainingr@   )r  Únum_key_value_groupsr3   Úmatmulr‰   r   r   rÁ   rC   rB   rA   r%  r)  Ú
contiguous)r  r   r!  r"  r#  r$  r%  rx   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs   &&&&&&&,    r+   Úeager_attention_forwardr1  �  sÒ   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÕL€LØÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r.   c                   ó\   a a€ ] tR tRt oRtRV3R lV 3R llltR	V3R lR lltRtVtV ;t	# )
ÚJetMoeAttentioni¨  z@
Multi-headed attention from 'Attention Is All You Need' paper.
c                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# ©r%   r^   Ú	layer_idxN©r    rž   )r)   r*   s   "€r+   r,   ÚJetMoeAttention.__annotate__­  s#   ø€ ÷ dñ d™|ð d¹¸d½
ñ dr.   c                ó|  <€ \         SV `  4        Wn        W n        RV n        Vf.   \
        P                  RV P                  P                   R24       ^V n	        VP                  V n        VP                  V n        VP                  VP                  ,          V n        VP                  V n        VP                   V n        VP                  V n        V P$                  R,          V n        \)        V4      V n        \,        P.                  P1                  VP2                  V P                  ^,          RR7      V n        R# )z¬
Initialize the JetMoeAttention module.

Args:
    config:
        Configuration object with model hyperparameters.
    layer_idx:
        Index of the layer in the model.
TNzInstantiating z¹ without passing a `layer_idx` is not recommended and will lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` when creating this class.Fr·   g      à¿)r0   r1   r^   r6  Ú	is_causalÚloggerÚwarning_oncer9   rP   r*  rã   r´   Úattention_dropoutrþ   rÿ   Úkv_projection_sizerv   Ú	num_headsrs   r$  rú   Úexpertsr3   r   r¹   r8   Úkv_proj©r7   r^   r6  r9   s   &&&€r+   r1   ÚJetMoeAttention.__init__­  s   ø€ ô 	‰ÑÔØŒØ"ŒØˆŒØÒÜ×ÑØ  §¡×!8Ñ!8Ð 9ð :,ð ,ôð %&ˆÔ!Ø×/Ñ/ˆŒ
Ø!'×!9Ñ!9ˆÔØ"(×"4Ñ"4°v×7QÑ7QÕ"QˆÔØ#)×#=Ñ#=ˆÔ Ø×3Ñ3ˆŒØ×*Ñ*ˆŒØ—}‘} dÕ*ˆŒÜ  Ó(ˆŒä—x‘x—‘ v×'9Ñ'9¸4×;RÑ;RÐUVÕ;VÐ]b�ÓcˆŽr.   c                ó   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[S[ P                  S[ P                  R,          S[S[ P                  ,          R,          3,          /# )r%   r<   r#  NÚposition_embeddingsÚpast_key_valuesr'   )r3   r=   Ú
LongTensorr   rL   )r)   r*   s   "€r+   r,   r8  Î  s€   ø€ ÷ /8ñ /8á—|‘|ð/8ñ Ÿ™ tÕ+ð/8ñ #×-Ñ-°Õ4ð	/8ñ
  �ð/8ñ 
‰u�|‰|™UŸ\™\¨DÕ0±%¹¿¹Õ2EÈÕ2LÐLÕ	Mñ/8r.   c                óô  € VP                   R R p. VORNV P                  N5pV P                  P                  V4      w  r‰p
V P	                  V4      P                  ^RR7      w  r¼VP                  V4      P                  ^^4      pVP                  V4      P                  ^^4      pVP                  V4      P                  ^^4      pVw  rÞ\        W‹WÞ4      w  r‹Ve   VP                  W¼V P                  4      w  r¼\        P                  ! V P                  P                  \        4      pVP!                  ^V P"                  ^^4      pVP!                  ^V P"                  ^^4      pT! V VVVV3RV P$                  '       g   RMV P&                  RV P(                  /VB w  ppVP                  ! . VOV P"                  NRN5!  pV P                  P+                  VV
4      pVP                  ! . VORN5!  pVVV	3# )Nrƒ   r%  ç        r$  r@   )rM   rs   r@  r  rA  rê   rì   r‰   r  Úupdater6  r   Úget_interfacer^   Ú_attn_implementationr1  Úrepeatr´   r)  r=  r$  r  )r7   r<   r#  rE  rF  rx   Úinput_shapeÚhidden_shapeÚquery_statesrñ   r  r-  r.  r‹   rŒ   Úattention_interfacer0  r/  s   &&&&&,            r+   rI   ÚJetMoeAttention.forwardÎ  sû  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà15·±×1AÑ1AÀ-Ó1PÑ.ˆ YØ#'§<¡<°Ó#>×#DÑ#DÀQÈBÐ#DÓ#OÑ ˆ
à#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆØ—_‘_ \Ó2×<Ñ<¸QÀÓBˆ
Ø#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðð  ×&Ñ& q¨$¯*©*°a¸Ó;ˆ
Ø#×*Ñ*¨1¨d¯j©j¸!¸QÓ?ˆá$7ØØØØØñ	%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆ�\ð "×&Ò&ÐD¨ÐD°T·Z±ZÐDÀÓDˆØ—l‘l×)Ñ)¨+°yÓAˆØ!×&Ò&Ð8¨Ð8°RÓ8ˆØ˜L¨-Ð7Ð7r.   )r=  r^   r@  rs   r:  rA  r>  r6  r?  r*  rÿ   r$  r´   r”   )NNNr÷   rW   s   @@r+   r3  r3  ¨  s&   ù‡ € ñ÷dõ d÷B/8÷ /8ò /8r.   r3  c                   óX   a a€ ] tR tRt oRV3R lV 3R llltRV3R lR lltRtVtV ;t# )	ÚJetMoeDecoderLayeri   c                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# r5  r7  )r)   r*   s   "€r+   r,   ÚJetMoeDecoderLayer.__annotate__  s#   ø€ ÷ Añ A™|ð A¹¸d½
ñ Ar.   c                óî   <€ \         SV `  4        VP                  V n        \        V4      V n        \        VP                  4      V n        \        VP                  4      V n        \        W4      V n	        R # r”   )
r0   r1   r8   rØ   Úmlpr#   Úinput_layernormÚpost_attention_layernormr3  Úself_attentionrB  s   &&&€r+   r1   ÚJetMoeDecoderLayer.__init__  s[   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔÜ˜VÓ$ˆŒÜ,¨V×-?Ñ-?Ó@ˆÔÜ(5°f×6HÑ6HÓ(IˆÔ%Ü-¨fÓ@ˆÖr.   c                ó.  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[ P                  S[ P                  3,          R,          RS[S[,          R	S[ P                  /# )
r%   r<   r#  NrŽ   rF  Ú	use_cacherE  rx   r'   )r3   r=   rG  r   ÚboolrL   r   r   )r)   r*   s   "€r+   r,   rV  	  sœ   ø€ ÷ ñ á—|‘|ðñ Ÿ™ tÕ+ðñ ×&Ñ&¨Õ-ð	ñ
  �ðñ ˜$•;ðñ #¡5§<¡<±·±Ð#=Õ>ÀÕEðñ Ñ+Õ,ðñ 
�‰ñr.   c                óØ   € TpV P                  V4      pV P                  ! RR VRVRVRVRVRV/VB w  p p	W�,           pTpV P                  V4      pV P                  V4      pW�,           pV# )r<   r#  rŽ   rF  r^  rE  © )rY  r[  rZ  rX  )
r7   r<   r#  rŽ   rF  r^  rE  rx   ÚresidualrÒ   s
   &&&&&&&,  r+   rI   ÚJetMoeDecoderLayer.forward	  s¯   € ð !ˆØ×,Ñ,¨]Ó;ˆà"×1Ò1ñ 
Ø'ð
à)ð
ð &ð
ð ,ð	
ð
  ð
ð !4ð
ð ñ
Ñˆ�q˜!ð !Õ0ˆð !ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ Õ0ˆØÐr.   )r8   rY  rX  rZ  r[  r”   )NNNFNr°   rW   s   @@r+   rT  rT     s    ù‡ € ÷Aõ A÷÷ ò r.   rT  c            
       óÒ   a a€ ] tR tRt oRtRtR.tR.tRtRt	Rt
RtRtR]! ]^R7      ]! ]^R7      .R	]R
]! ]^R7      /t]P&                  ! 4       V 3R l4       tV3R ltRtVtV ;t# )ÚJetMoePreTrainedModeli)  ÚmodelFrT  rF  Trñ   )Úindexr<   Ú
attentionsc                óJ  <€ \         SV `  V4       \        V\        4      '       d:   \        P
                  ! VP                  RV P                  P                  R7       R# \        V\        \        ,          4      '       d#   \        P                  ! VP                  4       R# R# )zInitialize the weights.rI  )rE   ÚstdN)r0   Ú_init_weightsr†   r˜   ÚinitÚnormal_r5   r^   Úinitializer_rangerú   rØ   Úzeros_r¸   )r7   r  r9   s   &&€r+   rk  Ú#JetMoePreTrainedModel._init_weights;  se   ø€ ô 	‰Ñ˜fÔ%Ü�fÔ3×4Ò4Ü�LŠL˜Ÿ™¨S°d·k±k×6SÑ6S×TÜ˜¤	¬IÕ 5×6Ò6Ü�KŠK˜Ÿ™Ö$ñ 7r.   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r]   r   )r)   r*   s   "€r+   r,   Ú"JetMoePreTrainedModel.__annotate__)  s   ø‡ ‚ áÑò r.   ra  )rP   rQ   rR   rS   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendr   r3  r²   rT  Ú_can_record_outputsr3   r–   rk  Ú__annotate_func__rT   rU   rV   rW   s   @@r+   re  re  )  s—   ù‡ € ð  ÐØ&+Ð#Ø-Ð.ÐØ#4Ð"5ÐØÐØ€NØÐØ"ÐØ"&Ðà™.¨ÀÔBÁNÐScÐklÔDmÐnØÐ+Ø‘n _¸AÔ>ðÐð ‡]‚]ƒ_ô%ó ð%÷' … r.   re  c                   ór   a a€ ] tR tRt oV3R lV 3R llt]]]RV3R lR ll4       4       4       tRt	Vt
V ;t# )ÚJetMoeModeliE  c                ó    <€ V ^8„  d   QhRS[ /# r]   r   )r)   r*   s   "€r+   r,   ÚJetMoeModel.__annotate__G  s   ø€ ÷ ñ ™|ñ r.   c           	     ó6  <€ \         SV `  V4       VP                  V n        VP                  V n        \
        P                  ! VP                  VP                  V P                  4      V n        \
        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n        \        VP                  VP                  R 7      V n        \#        VR7      V n        RV n        VP(                  V n        V P+                  4        R# u upi ))r&   ©r^   FN)r0   r1   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingr8   Úembed_tokensÚ
ModuleListr¥   Únum_hidden_layersrT  Úlayersr#   Úrms_norm_epsÚnormrY   Ú
rotary_embÚgradient_checkpointingrL  Ú	post_initrB  s   && €r+   r1   ÚJetMoeModel.__init__G  sØ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸLšL¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m’mÜDIÈ&×JbÑJbÔDcÓdÑDc°yÔ Ö2ÑDcÑdó
ˆŒô " &×"4Ñ"4¸&×:MÑ:MÔNˆŒ	Ü/°vÔ>ˆŒØ&+ˆÔ#Ø$*×$?Ñ$?ˆÔ!ð 	�‰Öùò es   ÂDc                óþ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[S[,          R	S[/# )
r%   Ú	input_idsNr#  rŽ   rF  Úinputs_embedsr^  rx   r'   )	r3   rG  r=   r   ÚFloatTensorr_  r   r   r   )r)   r*   s   "€r+   r,   r�  [  s“   ø€ ÷ 5
ñ 5
á×#Ñ# dÕ*ð5
ñ Ÿ™ tÕ+ð5
ñ ×&Ñ&¨Õ-ð	5
ñ
  �ð5
ñ ×(Ñ(¨4Õ/ð5
ñ ˜$•;ð5
ñ Ñ+Õ,ð5
ñ 
 ñ5
r.   c                ó†  € VR J VR J,          '       d   \        R4      hV'       d   Vf   \        V P                  R7      pVf   V P                  V4      pVfb   Ve   VP	                  4       M^ p\
        P                  ! VP                  ^,          VP                  R7      V,           pVP                  ^ 4      p\        V P                  VVVVR7      p	Tp
V P                  W£4      pV P                  R V P                  P                    F  pV! V
3RVRV	RVRVR	V/VB p
K  	  V P                  V
4      p
\        V
VR
7      # )Nz:You must specify exactly one of input_ids or inputs_embedsrƒ  )rZ   )r^   r”  r#  rF  rŽ   rE  r#  rF  r^  rŽ   )Úlast_hidden_staterF  )Ú
ValueErrorr	   r^   rˆ  Úget_seq_lengthr3   rw   rM   rZ   r  r   rŽ  r‹  rŠ  r�  r   )r7   r“  r#  rŽ   rF  r”  r^  rx   Úpast_seen_tokensÚcausal_maskr<   rE  Údecoder_layers   &&&&&&&,     r+   rI   ÚJetMoeModel.forwardX  se  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZç˜Ò0Ü*°$·+±+Ô>ˆOàÒ Ø ×-Ñ-¨iÓ8ˆMàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLä(Ø—;‘;Ø'Ø)Ø+Ø%ô
ˆð &ˆð #Ÿo™o¨mÓJÐà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÓIˆMÙ)Øñà$7ðð  +ðð !0ð	ð
 $ðð *ðð ñŠMñ Jð Ÿ	™	 -Ó0ˆä%Ø+Ø+ô
ð 	
r.   )rL  rˆ  r�  r‹  r�  r…  rŽ  r†  )NNNNNN)rP   rQ   rR   rS   r1   r   r   r   rI   rT   rU   rV   rW   s   @@r+   r  r  E  s=   ù‡ € ÷ó ð"  ØØ÷5
ñ 5
ó ó ó  ÷5
ð 5
r.   r  c          	      ó  € V ^8„  d   QhR\         P                  \        \         P                  ,          ,          R,          R\        R,          R\         P                  R,          R\         P                  \        ,          /# )r%   Úgate_logitsNrš   r#  r'   )r3   r=   rL   rž   )r)   s   "r+   r,   r,   “  sc   € ÷ B&ñ B&Ü—‘¤¤e§l¡lÕ 3Õ3°dÕ:ðB&ä�t•ðB&ô —L‘L 4Õ'ð	B&ô
 ‡\�\”CÕñB&r.   c           	     ó  € V e   \        V \        4      '       g   ^ # V ^ ,          P                  p\        P                  ! V\        P
                  VR7      p\        P                  ! V\        P
                  VR7      pRpVe1   VP                  R4      P                  V\        P
                  R7      pV  EFˆ  p	\        P                  P                  P                  V	P                  V4      RR7      p
\        P                  ! W¢RR7      w  r¼Vf~   V\        P                  ! VP                  R4      VR7      P                  4       ,           pWjP                  4       P                  ^ R7      ,           pWzP                  ^ ,          ,           pKÙ  V\        P                  ! V\        P
                  VR7      P!                  ^ VP                  R4      XP#                  V4      4      ,           pWjP                  4       VP%                  R4      ,          P                  ^ R7      ,           pWxP                  4       ,           pEK‹  	  WW,          pWg,          p\        P                  ! WÞP%                  ^ 4      ,          4      pWñ,          # )ax  
Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch.

See Switch Transformer (https://huggingface.co/papers/2101.03961) for more details. This function implements the loss
function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between
experts is too unbalanced.

Args:
    gate_logits:
        Logits from the `gate`, should be a tuple of model.config.num_hidden_layers tensors of
        shape [batch_size X sequence_length, num_experts].
    num_experts:
        Number of experts
    top_k:
        The number of experts to route per-token, can be also interpreted as the `top-k` routing
        parameter.
    attention_mask (`torch.Tensor`, *optional*):
        The attention_mask used in forward function
        shape [batch_size X sequence_length] if not None.

Returns:
    The auxiliary loss.
r~   rI  )rZ   rA   rƒ   )Ú	minlengthr@   )r†   rL   rZ   r3   rÃ   rC   ré   rB   r   r   rÁ   rÀ   Úbincountr(   rÇ   rM   Úscatter_add_Úrepeat_interleaver  )rŸ  rš   r´   r#  Úcompute_deviceÚtokens_per_expert_sumÚrouter_prob_sumÚ
total_rowsÚ	flat_maskÚ
layer_gateÚrouting_weightsrÒ   Úselected_expertsÚtokens_per_expertÚrouter_prob_per_expertÚoverall_losss   &&&&            r+   Úload_balancing_loss_funcr°  “  sÿ  € ð: Ò¤*¨[¼%×"@Ò"@Ùð ! •^×*Ñ*€NÜ!ŸKšK¨¼5¿=¹=ÐQ_Ô`ÐÜ—k’k +´U·]±]È>ÔZ€OØ€JàÒ!à"×*Ñ*¨2Ó.×1Ñ1¸ÌuÏ}É}Ð1Ó]ˆ	ä!ˆ
ÜŸ(™(×-Ñ-×5Ñ5°j·m±mÀNÓ6SÐY[Ð5Ó\ˆÜ#Ÿjšj¨ÀRÔHÑˆØÒ!ð &¬¯ªÐ7G×7OÑ7OÐPRÓ7SÐ_jÔ(k×(qÑ(qÓ(sÕsð "ð .×0EÑ0EÓ0G×0KÑ0KÐPQÐ0KÓ0RÕRˆOØ#×&;Ñ&;¸AÕ&>Õ>ŠJð %:¼E¿KºKØ¤5§=¡=¸ô=ç‰l˜1Ð.×6Ñ6°rÓ:¸I×<WÑ<WÐX]Ó<^Ó_õ%`Ð!ð .×1FÑ1FÓ1HÈ9×K^ÑK^Ð_aÓKbÕ1b×0gÑ0gÐlmÐ0gÓ0nÕnˆOØ#§m¡m£oÕ5‹Jñ# "ð& .Õ:ÐØ,Õ9Ðä—9’9Ð.×1QÑ1QÐRSÓ1TÕTÓU€LØÕ%Ð%r.   c                   ód   a a€ ] tR tRt oRR/tV 3R lt]]RV3R lR ll4       4       tRt	Vt
V ;t# )	ÚJetMoeForCausalLMiØ  zlm_head.weightzmodel.embed_tokens.weightc                ót  <€ \         SV `  V4       \        V4      V n        VP                  V n        VP
                  V n        \        P                  ! VP                  VP                  R R7      V n	        VP                  V n
        VP                  V n        VP                  V n        V P                  4        R# )Fr·   N)r0   r1   r  rf  r†  Úaux_loss_coefr   r¹   r8   Úlm_headÚtie_word_embeddingsrà   rš   rã   r�  rå   s   &&€r+   r1   ÚJetMoeForCausalLM.__init__Û  s�   ø€ Ü‰Ñ˜Ô Ü  Ó(ˆŒ
Ø ×+Ñ+ˆŒØ#×1Ñ1ˆÔÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒØ#)×#=Ñ#=ˆÔ Ø!×3Ñ3ˆÔØ#)×#=Ñ#=ˆÔ ð 	�‰Ör.   c                óN  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[S[ P                  ,          R
S[R,          RS[/
# )r%   r“  Nr#  rŽ   rF  r”  Úlabelsr^  Úlogits_to_keepÚoutput_router_logitsr'   )r3   rG  r=   r   r•  r_  rž   r   )r)   r*   s   "€r+   r,   ÚJetMoeForCausalLM.__annotate__ê  s¹   ø€ ÷ 9
ñ 9
á×#Ñ# dÕ*ð9
ñ Ÿ™ tÕ+ð9
ñ ×&Ñ&¨Õ-ð	9
ñ
  �ð9
ñ ×(Ñ(¨4Õ/ð9
ñ × Ñ  4Õ'ð9
ñ ˜$•;ð9
ñ ™eŸl™lÕ*ð9
ñ # T�kð9
ñ 
#ñ9
r.   c
                ó†  € V P                   ! RR VRVRVRVRVRVRV	/V
B pVP                  p\        V\        4      '       d   \	        V) R4      MTpV P                  VRVR3,          4      pRpVe+   V P                  ! VV3R	V P                  P                  /V
B pRpV	'       dc   \        VP                  V P                  V P                  V4      pVe3   WðP                  VP                  VP                  4      ,          ,          p\!        VVVVP"                  VP$                  VP&                  VP                  R
7      # )r“  r#  rŽ   rF  r”  r^  r»  Nr„   r†  )ÚlossÚaux_lossrÌ   rF  r<   rh  rñ   ra  )rf  r—  r†   rž   Úslicerµ  Úloss_functionr^   r†  r°  rñ   rš   rã   r´  rB   rZ   r   rF  r<   rh  )r7   r“  r#  rŽ   rF  r”  r¹  r^  rº  r»  rx   Úoutputsr<   Úslice_indicesrÌ   r¾  r¿  s   &&&&&&&&&&,      r+   rI   ÚJetMoeForCausalLM.forwardè  sv  € ð +/¯*ª*ñ 	+
Øð	+
à)ð	+
ð &ð	+
ð ,ð		+
ð
 (ð	+
ð  ð	+
ð "6ð	+
ð ñ	+
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒØ×%Ò%ØØñð  Ÿ;™;×1Ñ1ðð ñ	ˆDð ˆßÜ/Ø×%Ñ%Ø× Ñ Ø×(Ñ(Øó	ˆHð Ò!Ø×*Ñ*¨X¯[©[¸¿¹Ó-EÕEÕE�ä(ØØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø!×/Ñ/ô
ð 	
r.   )r´  rµ  rf  rš   rã   r¶  r†  )	NNNNNNNr   F)rP   rQ   rR   rS   Ú_tied_weights_keysr1   r   r   rI   rT   rU   rV   rW   s   @@r+   r²  r²  Ø  s<   ù‡ € Ø*Ð,GÐHÐõð Ø÷9
ñ 9
ó ó ÷9
ð 9
r.   r²  c                   ó   € ] tR tRtRtR# )ÚJetMoeForSequenceClassificationi&  ra  N)rP   rQ   rR   rS   rT   ra  r.   r+   rÇ  rÇ  &  s   † Ó`cr.   rÇ  )r½   )rI  )Nr%   N)r²  r  re  rÇ  )HÚcollections.abcr   r3   r   Útorch.nnr   r§   Ú r   rl  Úactivationsr   Úcache_utilsr   r	   Ú
generationr
   Úintegrationsr   Úmasking_utilsr   Úmodeling_layersr   r   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   r   Úconfiguration_jetmoer    Ú
get_loggerrP   r;  r&  r#   rY   r˜   r²   rØ   rú   r  r  r  r1  r3  rT  re  r  r°  r²  rÇ  Ú__all__ra  r.   r+   Ú<module>rÜ     s�  ðõ* %ã Ý Ý $å &Ý !ß .Ý )Ý 7Ý /ß [ß Qß Kß FÝ &ß RÓ RÝ 0ß Gß EÝ .ð 
×	Ò	˜HÓ	%€ñ ˜YÓ'ôJ�B—I‘Ió Jó (ðJô(6<˜BŸI™Iô 6<ôr*˜BŸI™Iô *ôZ.S�r—y‘yô .Sôb7�—	‘	ô 7ôtIS�—	‘	ô ISòX(ñ Ð-Ó.óó /ðõ2	U÷%ô2U8�b—i‘iô U8ôp&Ð3ô &ðR ô%˜Oó %ó ð%ð6 ôJ
Ð'ó J
ó ðJ
÷ZB&ôJK
Ð-¨ô K
ô\ dÐ&FÐH]Ô cò k‚r.   