+
    œ°³j]  ã                   óÎ  € R t ^ RIHt ^ RIt^ RIHt ^ RIHt ^RIH	t
 ^RIHt ^RIHtHt ^RIHt ^R	IHt ^R
IHt ^RIHtHt ^RIHtHt ^RIHt ^RIHtH t H!t!H"t" ^RI#H$t$ ^RI%H&t&H't' ^RI(H)t) ^RI*H+t+H,t,H-t-H.t.H/t/H0t0H1t1 ^RI2H3t3 ]"Ph                  ! ]54      t6 ! R R]-4      t7 ! R R].4      t8 ! R R]Pr                  4      t: ! R R]Pr                  4      t; ! R R]Pr                  4      t< ! R R]Pr                  4      t= ! R  R!]Pr                  4      t> ! R" R#])4      t?]  ! R$ R%],4      4       t@]  ! R& R']+4      4       tA ! R( R)]@]4      tB ! R* R+]]@4      tC. R,OtDR# )-zPyTorch JetMoe model.)ÚCallableN)Únn)Ú
functional)Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úcreate_causal_mask)Ú GenericForSequenceClassification)ÚMoeCausalLMOutputWithPastÚMoeModelOutputWithPast)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Úmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputs)ÚLlamaDecoderLayer)ÚMixtralModelÚMixtralPreTrainedModelÚMixtralRMSNormÚMixtralRotaryEmbeddingÚapply_rotary_pos_embÚeager_attention_forwardÚload_balancing_loss_func©ÚJetMoeConfigc                   ó   € ] tR t^4tRtR# )ÚJetMoeRMSNorm© N©Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__static_attributes__r$   ó    Új/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/jetmoe/modular_jetmoe.pyr#   r#   4   ó   † Ûr+   r#   c                   ó   € ] tR t^8tRtR# )ÚJetMoeRotaryEmbeddingr$   Nr%   r$   r+   r,   r/   r/   8   r-   r+   r/   c                   óD   a a€ ] tR t^<t oV3R lV 3R lltR tRtVtV ;t# )ÚJetMoeParallelExpertsc                ó0   <€ V ^8„  d   QhRS[ RS[ RS[ RR/# )é   Únum_expertsÚ
input_sizeÚoutput_sizeÚreturnN©Úint)ÚformatÚ__classdict__s   "€r,   Ú__annotate__Ú"JetMoeParallelExperts.__annotate__=   s)   ø€ ÷ 'ñ '¡Cð '±Sð 'Ásð 'Ètñ 'r+   c                ó¨   <€ \         SV `  4        \        P                  ! \        P
                  ! WV4      4      V n        Wn        W n        W0n	        R# )aS  
Initialize the JetMoeParallelExperts module.
The experts weights are stored in [num_experts, output_size, input_size] format. Such that it's compatible with
many MoE libraries, such as [Megablock](https://github.com/databricks/megablocks) and
[ScatterMoE](https://github.com/shawntan/scattermoe), as well as the
[MoE kernel](https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/layers/fused_moe/fused_moe.py)
used in vllm.

Args:
    num_experts (int):
        Number of experts.
    input_size (int):
        Size of the input.
    output_size (int):
        Size of the output.
N)
ÚsuperÚ__init__r   Ú	ParameterÚtorchÚemptyÚweightr4   r5   r6   )Úselfr4   r5   r6   Ú	__class__s   &&&&€r,   r@   ÚJetMoeParallelExperts.__init__=   s<   ø€ ô" 	‰ÑÔÜ—l’l¤5§;¢;¨{ÈÓ#TÓUˆŒØ&ÔØ$ŒØ&Ör+   c                ó  € VP                  V^ R7      p. p\        V P                  4       F@  pVP                  \        P
                  ! W5,          V P                  V,          4      4       KB  	  \        P                  ! V^ R7      pV# )z¼
Forward pass of the JetMoeParallelExperts module.

Args:
    inputs (Tensor):
        Input tensor.
    expert_size:
        Expert size information.

Returns:
    Tensor: Output tensor.
©Údim)	ÚsplitÚranger4   ÚappendÚFÚlinearrD   rB   Úcat)rE   ÚinputsÚexpert_sizeÚ
input_listÚoutput_listÚiÚresultss   &&&    r,   ÚforwardÚJetMoeParallelExperts.forwardT   sh   € ð —\‘\ +°1�\Ó5ˆ
ØˆÜ�t×'Ñ'Ö(ˆAØ×ÑœqŸxšx¨
­°t·{±{À1µ~ÓFÖGñ )ä—)’)˜K¨QÔ/ˆØˆr+   )r5   r4   r6   rD   ©	r&   r'   r(   r)   r@   rW   r*   Ú__classdictcell__Ú__classcell__©rF   r;   s   @@r,   r1   r1   <   s   ù‡ € ÷'ó '÷.ò r+   r1   c                   óD   a a€ ] tR t^it oV3R lV 3R lltR tRtVtV ;t# )ÚJetMoeTopKGatingc                ó,   <€ V ^8„  d   QhRS[ RS[ RS[ /# )r3   r5   r4   Útop_kr8   )r:   r;   s   "€r,   r<   ÚJetMoeTopKGating.__annotate__j   s'   ø€ ÷ Dñ D¡3ð D±Sð DÁñ Dr+   c                ó‚   <€ \         SV `  4        W n        Wn        W0n        \
        P                  ! WRR7      V n        R# )zÒ
Initialize the top-k gating mechanism.

Args:
    input_size (`int`):
        Size of the input.
    num_experts (`int`):
        Number of experts.
    top_k (`int`):
        Number of top experts to select.
F©ÚbiasN)r?   r@   r4   r5   r`   r   ÚLinearÚlayer)rE   r5   r4   r`   rF   s   &&&&€r,   r@   ÚJetMoeTopKGating.__init__j   s2   ø€ ô 	‰ÑÔà&ÔØ$ŒØŒ
ä—Y’Y˜z¸UÔCˆŽ
r+   c                óš  € V P                  V4      P                  4       pVP                  V P                  ^R7      w  r4\        P
                  ! V^R7      P                  V4      p\        P                  ! VP                  ^ 4      V P                  .VP                  VP                  R7      pVP                  ^V^4      pVP                  4       P                  ^ 4      pVP                  4       pVP!                  4       p	V	P#                  ^ 4      w  r«VP%                  V P                  RR7      pVP!                  4       pW[,          pW¼WØV3# )é   rI   ©ÚdtypeÚdeviceÚtrunc)Úrounding_mode)rf   ÚfloatÚtopkr`   rB   ÚsoftmaxÚtype_asÚzerosÚsizer4   rk   rl   ÚscatterÚlongÚsumÚtolistÚflattenÚsortÚdiv)rE   Úhidden_statesÚlogitsÚtop_k_logitsÚtop_k_indicesÚtop_k_gatesrs   ÚgatesrR   Útop_k_expertsÚ_Úindex_sorted_expertsÚbatch_indexÚbatch_gatess   &&            r,   rW   ÚJetMoeTopKGating.forward~   s"  € à—‘˜MÓ*×0Ñ0Ó2ˆØ&,§k¡k°$·*±*À! kÓ&DÑ#ˆÜ—m’m L°aÔ8×@Ñ@ÀÓOˆô —’Ø×Ñ˜aÓ  $×"2Ñ"2Ð3¸;×;LÑ;LÐU`×UgÑUgô
ˆð —‘˜a °Ó2ˆØ—j‘j“l×&Ñ& qÓ)ˆð "×(Ñ(Ó*ˆð &×-Ñ-Ó/ˆØ"/×"4Ñ"4°QÓ"7ÑˆØ*×.Ñ.¨t¯z©zÈÐ.ÓQˆð "×)Ñ)Ó+ˆØ!Õ7ˆà#°+ÈFÐRÐRr+   )r5   rf   r4   r`   rY   r\   s   @@r,   r^   r^   i   s   ù‡ € ÷Dó D÷(Sò Sr+   r^   c                   óH   a a€ ] tR t^št oRtV3R lV 3R lltR tRtVtV ;t	# )Ú	JetMoeMoEz¦
A Sparsely gated mixture of experts layer with 1-layer Feed-Forward networks as experts.

Args:
    config:
        Configuration object with model hyperparameters.
c                ó    <€ V ^8„  d   QhRS[ /# ©r3   Úconfigr    )r:   r;   s   "€r,   r<   ÚJetMoeMoE.__annotate__£   s   ø€ ÷ 
ñ 
™|ñ 
r+   c                óT  <€ \         SV `  4        VP                  V n        VP                  V n        \
        VP                  ,          V n        \        P                  P                  \        P                  ! V P                  4      4      V n        \        VP                  V P                  V P                  ^,          4      V n        \        VP                  V P                  V P                  4      V n        \#        V P                  VP                  VP$                  R7      V n        R# )r3   ©r5   r4   r`   N)r?   r@   Úhidden_sizer5   Úintermediate_sizer   Úactivation_functionÚ
activationrB   r   rA   rC   rd   r1   Únum_local_expertsÚinput_linearÚoutput_linearr^   Únum_experts_per_tokÚrouter©rE   rŒ   rF   s   &&€r,   r@   ÚJetMoeMoE.__init__£   sÏ   ø€ Ü‰ÑÔà ×,Ñ,ˆŒØ!×3Ñ3ˆÔÜ  ×!;Ñ!;Õ<ˆŒÜ—H‘H×&Ñ&¤u§{¢{°4·?±?Ó'CÓDˆŒ	Ü1°&×2JÑ2JÈDÏOÉOÐ]a×]mÑ]mÐpqÕ]qÓrˆÔÜ2°6×3KÑ3KÈT×M]ÑM]Ð_c×_nÑ_nÓoˆÔä&Ø—‘Ø×0Ñ0Ø×,Ñ,ô
ˆŽr+   c                ó\  € VP                  4       w  r#pVP                  RV4      pV P                  V4      w  rVrxp	W,          p
V P                  W¨4      pVP	                  ^RR7      pV P                  V^ ,          4      V^,          ,          pV P                  W¸4      pW×R,          ,          p\        P                  ! W#,          V P                  3VP                  VP                  R7      pVP                  ^ Wm4      pVP                  W#V P                  4      pWðP                  ,           pV# )z¶
Forward pass of the mixture of experts layer.

Args:
    layer_input (Tensor):
        Input tensor.

Returns:
    Tensor:
        Output tensor.
    Tensor:
        Router logits.
rI   rj   éÿÿÿÿ©ºNNNN)rt   Úreshaper˜   r•   Úchunkr“   r–   rB   rs   r5   rk   rl   Ú	index_addÚviewrd   )rE   Úlayer_inputÚbszÚlengthÚemb_sizerƒ   r…   r†   rR   Úrouter_logitsÚexpert_inputsr|   Úchunked_hidden_statesÚexpert_outputsrs   Úlayer_outputs   &&              r,   rW   ÚJetMoeMoE.forward³   s  € ð !,× 0Ñ 0Ó 2Ñˆ�XØ!×)Ñ)¨"¨hÓ7ˆØBFÇ+Á+ÈkÓBZÑ?ˆ˜°-à#Õ0ˆØ×)Ñ)¨-ÓEˆØ -× 3Ñ 3°A¸2Ð 3Ó >ÐØŸ™Ð(=¸aÕ(@ÓAÐDYÐZ[ÕD\Õ\ˆØ×+Ñ+¨MÓGˆà'°gÕ*>Õ>ˆä—’˜S�\¨4¯?©?Ð;À>×CWÑCWÐ`n×`uÑ`uÔvˆØ—‘ q¨+ÓFˆØ#×(Ñ(¨°d·o±oÓFˆØ#§i¡iÕ/ˆØÐr+   )r“   rd   r�   r•   r5   r–   r˜   ©
r&   r'   r(   r)   Ú__doc__r@   rW   r*   rZ   r[   r\   s   @@r,   r‰   r‰   š   s   ù‡ € ñ÷
ó 
÷ ò r+   r‰   c                   óT   a a€ ] tR t^Ôt oRtV3R lV 3R lltR tR tR tRt	Vt
V ;t# )Ú	JetMoeMoAz±
A Sparsely gated mixture of attention layer with pairs of query- and output-projections as experts.

Args:
    config:
        Configuration object with model hyperparameters.
c                ó    <€ V ^8„  d   QhRS[ /# r‹   r    )r:   r;   s   "€r,   r<   ÚJetMoeMoA.__annotate__Ý   s   ø€ ÷ 
ñ 
™|ñ 
r+   c                ót  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  VP                  ,          V n        VP                  V n	        \        P                  P                  \        P                  ! V P
                  4      4      V n        \        V P                  V P
                  V P                  4      V n        \        V P                  V P                  V P
                  4      V n        \%        V P
                  V P                  V P                  R 7      V n        R# )r�   N)r?   r@   r”   r4   r�   r5   Úkv_channelsÚnum_key_value_headsr—   r`   rB   r   rA   rC   rd   r1   r•   r–   r^   r˜   r™   s   &&€r,   r@   ÚJetMoeMoA.__init__Ý   sÛ   ø€ Ü‰ÑÔà!×3Ñ3ˆÔØ ×,Ñ,ˆŒØ!×-Ñ-°×0JÑ0JÕJˆÔØ×/Ñ/ˆŒ
Ü—H‘H×&Ñ&¤u§{¢{°4·?±?Ó'CÓDˆŒ	ä1°$×2BÑ2BÀDÇOÁOÐUY×UeÑUeÓfˆÔÜ2°4×3CÑ3CÀT×EUÑEUÐW[×WfÑWfÓgˆÔä&Ø—‘Ø×(Ñ(Ø—*‘*ô
ˆŽr+   c                ó¶  € VP                  4       w  r#pVP                  RV4      pV P                  V4      w  rVrxp	WVWx3p
W,          pV P                  W¸4      p\        P
                  ! W#,          V P                  ,          V P                  3VP                  VP                  R7      pVP                  ^ W\4      pVP                  W#V P                  R4      pWéV
3# )zq
Map inputs to attention experts according to routing decision and compute query projection inside each experts.
rj   rœ   )rt   rŸ   r˜   r•   rB   rs   r`   r�   rk   rl   r¡   r¢   )rE   r£   r¤   r¥   r¦   r„   r…   r†   rR   r§   Ú	topo_infor¨   rª   rs   r«   s   &&             r,   ÚmapÚJetMoeMoA.mapï   sÖ   € ð !,× 0Ñ 0Ó 2Ñˆ�XØ!×)Ñ)¨"¨hÓ7ˆØUY×U`ÑU`ÐalÓUmÑRÐ¨;À]Ø)¸ÐQˆ	ð $Õ0ˆØ×*Ñ*¨=ÓFˆô —’Ø�\˜DŸJ™JÕ&¨×(8Ñ(8Ð9À×AUÑAUÐ^l×^sÑ^sô
ˆð —‘ qÐ*>ÓOˆØ#×(Ñ(¨°d·j±jÀ"ÓEˆØ¨IÐ5Ð5r+   c                ó¨  € VP                  4       w  r4rVVP                  RV4      pVw  rxršW,          pV P                  Wº4      pWÉR,          ,          p\        P                  ! W4,          V P
                  3VP                  VP                  R7      pVP                  ^ WŒ4      pVP                  W4V P
                  4      pWàP                  ,           pV# )ze
Compute output projection inside each attention experts and merge the outputs of different experts.
rj   rœ   r�   )rt   rŸ   r–   rB   rs   r5   rk   rl   r¡   r¢   rd   )rE   r£   r¸   r¤   r¥   Úkr�   r„   r…   r†   rR   r¨   rª   rs   r«   s   &&&            r,   ÚreduceÚJetMoeMoA.reduce  sÁ   € ð '2×&6Ñ&6Ó&8Ñ#ˆ�QØ!×)Ñ)¨"¨kÓ:ˆØFOÑCÐ¨;ð $Õ9ˆØ×+Ñ+¨MÓGˆð (°gÕ*>Õ>ˆô —’˜S�\¨4¯?©?Ð;À>×CWÑCWÐ`n×`uÑ`uÔvˆØ—‘ q¨+ÓFˆØ#×(Ñ(¨°d·o±oÓFˆØ#§i¡iÕ/ˆØÐr+   c                ó   € \        R 4      h)z-This module doesn't support call and forward.)ÚNotImplementedError)rE   r£   s   &&r,   rW   ÚJetMoeMoA.forward  s   € Ü!Ð"QÓRÐRr+   )rd   r�   r•   r5   r4   r–   r˜   r`   )r&   r'   r(   r)   r®   r@   r¹   r½   rW   r*   rZ   r[   r\   s   @@r,   r°   r°   Ô   s*   ù‡ € ñ÷
ó 
ò$6ò.÷,Sò Sr+   r°   c                   ó\   a a€ ] tR tRt oRtRV3R lV 3R llltR	V3R lR lltRtVtV ;t	# )
ÚJetMoeAttentioni   z@
Multi-headed attention from 'Attention Is All You Need' paper.
c                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# ©r3   rŒ   Ú	layer_idxN©r!   r9   )r:   r;   s   "€r,   r<   ÚJetMoeAttention.__annotate__%  s#   ø€ ÷ dñ d™|ð d¹¸d½
ñ dr+   c                ó|  <€ \         SV `  4        Wn        W n        RV n        Vf.   \
        P                  RV P                  P                   R24       ^V n	        VP                  V n        VP                  V n        VP                  VP                  ,          V n        VP                  V n        VP                   V n        VP                  V n        V P$                  R,          V n        \)        V4      V n        \,        P.                  P1                  VP2                  V P                  ^,          RR7      V n        R# )z¬
Initialize the JetMoeAttention module.

Args:
    config:
        Configuration object with model hyperparameters.
    layer_idx:
        Index of the layer in the model.
TNzInstantiating z¹ without passing a `layer_idx` is not recommended and will lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` when creating this class.Frc   g      à¿)r?   r@   rŒ   rÆ   Ú	is_causalÚloggerÚwarning_oncerF   r&   Únum_key_value_groupsr—   r`   Úattention_dropoutr´   rµ   Úkv_projection_sizeÚnum_attention_headsÚ	num_headsÚhead_dimÚscalingr°   ÚexpertsrB   r   re   r�   Úkv_proj©rE   rŒ   rÆ   rF   s   &&&€r,   r@   ÚJetMoeAttention.__init__%  s   ø€ ô 	‰ÑÔØŒØ"ŒØˆŒØÒÜ×ÑØ  §¡×!8Ñ!8Ð 9ð :,ð ,ôð %&ˆÔ!Ø×/Ñ/ˆŒ
Ø!'×!9Ñ!9ˆÔØ"(×"4Ñ"4°v×7QÑ7QÕ"QˆÔØ#)×#=Ñ#=ˆÔ Ø×3Ñ3ˆŒØ×*Ñ*ˆŒØ—}‘} dÕ*ˆŒÜ  Ó(ˆŒä—x‘x—‘ v×'9Ñ'9¸4×;RÑ;RÐUVÕ;VÐ]b�ÓcˆŽr+   c                ó   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[S[ P                  S[ P                  R,          S[S[ P                  ,          R,          3,          /# )r3   r|   Úattention_maskNÚposition_embeddingsÚpast_key_valuesr7   )rB   ÚTensorÚ
LongTensorr   Útuple)r:   r;   s   "€r,   r<   rÈ   F  s€   ø€ ÷ /8ñ /8á—|‘|ð/8ñ Ÿ™ tÕ+ð/8ñ #×-Ñ-°Õ4ð	/8ñ
  �ð/8ñ 
‰u�|‰|™UŸ\™\¨DÕ0±%¹¿¹Õ2EÈÕ2LÐLÕ	Mñ/8r+   c                óô  € VP                   R R p. VORNV P                  N5pV P                  P                  V4      w  r‰p
V P	                  V4      P                  ^RR7      w  r¼VP                  V4      P                  ^^4      pVP                  V4      P                  ^^4      pVP                  V4      P                  ^^4      pVw  rÞ\        W‹WÞ4      w  r‹Ve   VP                  W¼V P                  4      w  r¼\        P                  ! V P                  P                  \        4      pVP!                  ^V P"                  ^^4      pVP!                  ^V P"                  ^^4      pT! V VVVV3RV P$                  '       g   RMV P&                  RV P(                  /VB w  ppVP                  ! . VOV P"                  NRN5!  pV P                  P+                  VV
4      pVP                  ! . VORN5!  pVVV	3# )NrI   Údropoutç        rÓ   rœ   )ÚshaperÒ   rÔ   r¹   rÕ   r    r¢   Ú	transposer   ÚupdaterÆ   r   Úget_interfacerŒ   Ú_attn_implementationr   Úrepeatr`   ÚtrainingrÎ   rÓ   r½   )rE   r|   rÙ   rÚ   rÛ   ÚkwargsÚinput_shapeÚhidden_shapeÚquery_statesr§   r¸   Ú
key_statesÚvalue_statesÚcosÚsinÚattention_interfaceÚattn_outputÚattn_weightss   &&&&&,            r,   rW   ÚJetMoeAttention.forwardF  sû  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà15·±×1AÑ1AÀ-Ó1PÑ.ˆ YØ#'§<¡<°Ó#>×#DÑ#DÀQÈBÐ#DÓ#OÑ ˆ
à#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆØ—_‘_ \Ó2×<Ñ<¸QÀÓBˆ
Ø#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðð  ×&Ñ& q¨$¯*©*°a¸Ó;ˆ
Ø#×*Ñ*¨1¨d¯j©j¸!¸QÓ?ˆá$7ØØØØØñ	%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆ�\ð "×&Ò&ÐD¨ÐD°T·Z±ZÐDÀÓDˆØ—l‘l×)Ñ)¨+°yÓAˆØ!×&Ò&Ð8¨Ð8°RÓ8ˆØ˜L¨-Ð7Ð7r+   )rÎ   rŒ   rÔ   rÒ   rÊ   rÕ   rÏ   rÆ   rÑ   rÍ   rµ   rÓ   r`   ©N)NNNr­   r\   s   @@r,   rÃ   rÃ      s&   ù‡ € ñ÷dõ d÷B/8÷ /8ò /8r+   rÃ   c                   óX   a a€ ] tR tRt oRV3R lV 3R llltRV3R lR lltRtVtV ;t# )	ÚJetMoeDecoderLayerix  c                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# rÅ   rÇ   )r:   r;   s   "€r,   r<   ÚJetMoeDecoderLayer.__annotate__y  s   ø€ ÷ ñ ™|ð ¹¸d½
ñ r+   c                óÒ   <€ \         SV `  W4       \        VP                  4      V n        \        W4      V n        \        VP                  4      V n        \        V4      V n	        V =
R # rõ   )r?   r@   r#   r�   Úinput_layernormrÃ   Úself_attentionÚpost_attention_layernormr‰   ÚmlpÚ	self_attnrÖ   s   &&&€r,   r@   ÚJetMoeDecoderLayer.__init__y  sT   ø€ Ü‰Ñ˜Ô+Ü,¨V×-?Ñ-?Ó@ˆÔÜ-¨fÓ@ˆÔÜ(5°f×6HÑ6HÓ(IˆÔ%Ü˜VÓ$ˆŒØŠNr+   c                ó.  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[ P                  S[ P                  3,          R,          RS[S[,          R	S[ P                  /# )
r3   r|   rÙ   NÚposition_idsrÛ   Ú	use_cacherÚ   ré   r7   )rB   rÜ   rÝ   r   ÚboolrÞ   r   r   )r:   r;   s   "€r,   r<   rù   �  sœ   ø€ ÷ ñ á—|‘|ðñ Ÿ™ tÕ+ðñ ×&Ñ&¨Õ-ð	ñ
  �ðñ ˜$•;ðñ #¡5§<¡<±·±Ð#=Õ>ÀÕEðñ Ñ+Õ,ðñ 
�‰ñr+   c                óØ   € TpV P                  V4      pV P                  ! RR VRVRVRVRVRV/VB w  p p	W�,           pTpV P                  V4      pV P                  V4      pW�,           pV# )r|   rÙ   r  rÛ   r  rÚ   r$   )rû   rü   rý   rþ   )
rE   r|   rÙ   r  rÛ   r  rÚ   ré   Úresidualrƒ   s
   &&&&&&&,  r,   rW   ÚJetMoeDecoderLayer.forward�  s¯   € ð !ˆØ×,Ñ,¨]Ó;ˆà"×1Ò1ñ 
Ø'ð
à)ð
ð &ð
ð ,ð	
ð
  ð
ð !4ð
ð ñ
Ñˆ�q˜!ð !Õ0ˆð !ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ Õ0ˆØÐr+   )rû   rþ   rý   rü   rõ   )NNNFNrY   r\   s   @@r,   r÷   r÷   x  s   ù‡ € ÷õ ÷÷ ò r+   r÷   c            
       ó¾   a € ] tR tRt o R]! ]^R7      ]! ]^R7      .R]R]! ]^R7      /tRt	Rt
R.tR	.tR
tR
tRt]P"                  ! 4       R 4       tV 3R ltRtV tR# )ÚJetMoePreTrainedModeli¡  r§   )Úindexr|   Ú
attentionsÚmodelFr÷   rÛ   Tc                óV  € \         P                  ! W4       \        V\        4      '       d:   \        P
                  ! VP                  RV P                  P                  R7       R# \        V\        \        ,          4      '       d#   \        P                  ! VP                  4       R# R# )zInitialize the weights.rá   )ÚmeanÚstdN)r   Ú_init_weightsÚ
isinstancer1   ÚinitÚnormal_rD   rŒ   Úinitializer_ranger°   r‰   Úzeros_rd   )rE   Úmodules   &&r,   r  Ú#JetMoePreTrainedModel._init_weights±  se   € ô 	×%Ò% dÔ3Ü�fÔ3×4Ò4Ü�LŠL˜Ÿ™¨S°d·k±k×6SÑ6S×TÜ˜¤	¬IÕ 5×6Ò6Ü�KŠK˜Ÿ™Ö$ñ 7r+   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r‹   r    )r:   r;   s   "€r,   r<   Ú"JetMoePreTrainedModel.__annotate__¡  s   ø‡ ‚ ñ Ñò r+   r$   N)r&   r'   r(   r)   r   rÃ   r^   r÷   Ú_can_record_outputsÚbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_can_compile_fullgraphrB   Úno_gradr  Ú__annotate_func__r*   rZ   )r;   s   @r,   r	  r	  ¡  s‹   ø‡ € ð 	™.¨ÀÔBÁNÐScÐklÔDmÐnØÐ+Ø‘n _¸AÔ>ðÐð  ÐØ&+Ð#Ø-Ð.ÐØ#4Ð"5ÐØÐØ€NØ"Ðà
‡]‚]ƒ_ñ%ó ð%÷# ƒ r+   r	  c                   ór   a a€ ] tR tRt oV3R lV 3R llt]]]RV3R lR ll4       4       4       tRt	Vt
V ;t# )ÚJetMoeModeli»  c                ó    <€ V ^8„  d   QhRS[ /# r‹   r    )r:   r;   s   "€r,   r<   ÚJetMoeModel.__annotate__½  s   ø€ ÷ 
Oñ 
O™|ñ 
Or+   c           	     óæ  <€ \         SV `  V4       VP                  V n        VP                  V n        \
        P                  ! VP                  VP                  V P                  4      V n        \
        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n        VP                  V n        \        VP                  VP                   R 7      V n        R# u upi ))ÚepsN)r?   r@   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingr�   Úembed_tokensÚ
ModuleListrL   Únum_hidden_layersr÷   Úlayersræ   r#   Úrms_norm_epsÚnormrÖ   s   && €r,   r@   ÚJetMoeModel.__init__½  s¸   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸLšL¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m’mÜDIÈ&×JbÑJbÔDcÓdÑDc°yÔ Ö2ÑDcÑdó
ˆŒð %+×$?Ñ$?ˆÔ!Ü! &×"4Ñ"4¸&×:MÑ:MÔNˆŽ	ùò es   ÂC.c                óþ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[S[,          R	S[/# )
r3   Ú	input_idsNrÙ   r  rÛ   Úinputs_embedsr  ré   r7   )	rB   rÝ   rÜ   r   ÚFloatTensorr  r   r   r   )r:   r;   s   "€r,   r<   r'  Ì  s“   ø€ ÷ 5
ñ 5
á×#Ñ# dÕ*ð5
ñ Ÿ™ tÕ+ð5
ñ ×&Ñ&¨Õ-ð	5
ñ
  �ð5
ñ ×(Ñ(¨4Õ/ð5
ñ ˜$•;ð5
ñ Ñ+Õ,ð5
ñ 
 ñ5
r+   c                ó†  € VR J VR J,          '       d   \        R4      hV'       d   Vf   \        V P                  R7      pVf   V P                  V4      pVfb   Ve   VP	                  4       M^ p\
        P                  ! VP                  ^,          VP                  R7      V,           pVP                  ^ 4      p\        V P                  VVVVR7      p	Tp
V P                  W£4      pV P                  R V P                  P                    F  pV! V
3RVRV	RVRVR	V/VB p
K  	  V P                  V
4      p
\        V
VR
7      # )Nz:You must specify exactly one of input_ids or inputs_embeds)rŒ   )rl   )rŒ   r7  rÙ   rÛ   r  rÚ   rÙ   rÛ   r  r  )Úlast_hidden_staterÛ   )Ú
ValueErrorr   rŒ   r.  Úget_seq_lengthrB   Úarangerâ   rl   Ú	unsqueezer
   Ú
rotary_embr1  r0  r3  r   )rE   r6  rÙ   r  rÛ   r7  r  ré   Úpast_seen_tokensÚcausal_maskr|   rÚ   Údecoder_layers   &&&&&&&,     r,   rW   ÚJetMoeModel.forwardÉ  se  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZç˜Ò0Ü*°$·+±+Ô>ˆOàÒ Ø ×-Ñ-¨iÓ8ˆMàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLä(Ø—;‘;Ø'Ø)Ø+Ø%ô
ˆð &ˆð #Ÿo™o¨mÓJÐà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÓIˆMÙ)Øñà$7ðð  +ðð !0ð	ð
 $ðð *ðð ñŠMñ Jð Ÿ	™	 -Ó0ˆä%Ø+Ø+ô
ð 	
r+   )ræ   r.  r1  r3  r+  r,  )NNNNNN)r&   r'   r(   r)   r@   r   r   r   rW   r*   rZ   r[   r\   s   @@r,   r%  r%  »  s?   ù‡ € ÷
Oó 
Oð  ØØ÷5
ñ 5
ó ó ó  ÷5
ð 5
r+   r%  c                   ód   a a€ ] tR tRt oRR/tV 3R lt]]RV3R lR ll4       4       tRt	Vt
V ;t# )	ÚJetMoeForCausalLMi  zlm_head.weightzmodel.embed_tokens.weightc                ót  <€ \         SV `  V4       \        V4      V n        VP                  V n        VP
                  V n        \        P                  ! VP                  VP                  R R7      V n	        VP                  V n
        VP                  V n        VP                  V n        V P                  4        R# )Frc   N)r?   r@   r%  r  r,  Úaux_loss_coefr   re   r�   Úlm_headÚtie_word_embeddingsr”   r4   r—   Ú	post_initr™   s   &&€r,   r@   ÚJetMoeForCausalLM.__init__  s�   ø€ Ü‰Ñ˜Ô Ü  Ó(ˆŒ
Ø ×+Ñ+ˆŒØ#×1Ñ1ˆÔÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒØ#)×#=Ñ#=ˆÔ Ø!×3Ñ3ˆÔØ#)×#=Ñ#=ˆÔ ð 	�‰Ör+   c                óN  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[S[ P                  ,          R
S[R,          RS[/
# )r3   r6  NrÙ   r  rÛ   r7  Úlabelsr  Úlogits_to_keepÚoutput_router_logitsr7   )rB   rÝ   rÜ   r   r8  r  r9   r   )r:   r;   s   "€r,   r<   ÚJetMoeForCausalLM.__annotate__  s¹   ø€ ÷ 9
ñ 9
á×#Ñ# dÕ*ð9
ñ Ÿ™ tÕ+ð9
ñ ×&Ñ&¨Õ-ð	9
ñ
  �ð9
ñ ×(Ñ(¨4Õ/ð9
ñ × Ñ  4Õ'ð9
ñ ˜$•;ð9
ñ ™eŸl™lÕ*ð9
ñ # T�kð9
ñ 
#ñ9
r+   c
                ó†  € V P                   ! RR VRVRVRVRVRVRV	/V
B pVP                  p\        V\        4      '       d   \	        V) R4      MTpV P                  VRVR3,          4      pRpVe+   V P                  ! VV3R	V P                  P                  /V
B pRpV	'       dc   \        VP                  V P                  V P                  V4      pVe3   WðP                  VP                  VP                  4      ,          ,          p\!        VVVVP"                  VP$                  VP&                  VP                  R
7      # )r6  rÙ   r  rÛ   r7  r  rO  Nrž   r,  )ÚlossÚaux_lossr}   rÛ   r|   r  r§   r$   )r  r:  r  r9   ÚslicerH  Úloss_functionrŒ   r,  r   r§   r4   r—   rG  Útorl   r   rÛ   r|   r  )rE   r6  rÙ   r  rÛ   r7  rM  r  rN  rO  ré   Úoutputsr|   Úslice_indicesr}   rR  rS  s   &&&&&&&&&&,      r,   rW   ÚJetMoeForCausalLM.forward  sv  € ð +/¯*ª*ñ 	+
Øð	+
à)ð	+
ð &ð	+
ð ,ð		+
ð
 (ð	+
ð  ð	+
ð "6ð	+
ð ñ	+
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒØ×%Ò%ØØñð  Ÿ;™;×1Ñ1ðð ñ	ˆDð ˆßÜ/Ø×%Ñ%Ø× Ñ Ø×(Ñ(Øó	ˆHð Ò!Ø×*Ñ*¨X¯[©[¸¿¹Ó-EÕEÕE�ä(ØØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø!×/Ñ/ô
ð 	
r+   )rG  rH  r  r4   r—   rI  r,  )	NNNNNNNé    F)r&   r'   r(   r)   Ú_tied_weights_keysr@   r   r   rW   r*   rZ   r[   r\   s   @@r,   rE  rE    s<   ù‡ € Ø*Ð,GÐHÐõð Ø÷9
ñ 9
ó ó ÷9
ð 9
r+   rE  c                   ó   € ] tR tRtRtR# )ÚJetMoeForSequenceClassificationiR  r$   Nr%   r$   r+   r,   r]  r]  R  s   † Ó`cr+   r]  )rE  r%  r	  r]  )Er®   Úcollections.abcr   rB   r   Útorch.nnr   rN   Ú r   r  Úactivationsr   Úcache_utilsr   r   Ú
generationr	   Úmasking_utilsr
   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.genericr   Úutils.output_capturingr   r   Úllama.modeling_llamar   Úmixtral.modeling_mixtralr   r   r   r   r   r   r   Úconfiguration_jetmoer!   Ú
get_loggerr&   rË   r#   r/   ÚModuler1   r^   r‰   r°   rÃ   r÷   r	  r%  rE  r]  Ú__all__r$   r+   r,   Ú<module>rr     sS  ðñ å $ã Ý Ý $å &Ý !ß .Ý )Ý /õ÷ Rß FÝ &ß RÓ RÝ 7ß EÝ 4÷÷ ñ õ /ð 
×	Ò	˜HÓ	%€ô	�Nô 	ô	Ð2ô 	ô*˜BŸI™Iô *ôZ.S�r—y‘yô .Sôb7�—	‘	ô 7ôtIS�—	‘	ô ISôXU8�b—i‘iô U8ôp&Ð*ô &ðR ô%Ð2ó %ó ð%ð2 ôE
�,ó E
ó ðE
ôPK
Ð-¨ô K
ô\ dÐ&FÐH]Ô cò k‚r+   