+
    œ°³jæF  ã                   óœ  € R t ^ RIt^ RIHu Ht ^ RIHt ^RIHt ^RI	H
t
 ^RIHtHt ^RIHt ^RIHtHt ^RIHt ^R	IHtHt ^R
IHt ^RIHt ^RIHtHt ^RIH t  ^RI!H"t"H#t#H$t$H%t%H&t&H't'H(t(H)t)H*t* ^RI+H,t, ]PZ                  ! ].4      t/R,R R llt0] ! R R]Pb                  4      4       t2 ! R R]Pb                  4      t3 ! R R]Pb                  4      t4 ! R R])4      t5 ! R R]*4      t6 ! R R]"4      t7 ! R R]4      t8 ! R  R!](4      t9 ! R" R#]'4      t: ! R$ R%]#4      t; ! R& R']%4      t< ! R( R)]&4      t= ! R* R+]$4      t>. R-Ot?R# ).zPyTorch Mixtral model.N)Únn)Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)Úuse_experts_implementation)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚGradientCheckpointingLayer)ÚMoeCausalLMOutputWithPastÚMoeModelOutputWithPast)ÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚlogging)ÚOutputRecorder)	ÚMistralAttentionÚMistralForCausalLMÚMistralForQuestionAnsweringÚ MistralForSequenceClassificationÚMistralForTokenClassificationÚMistralModelÚMistralPreTrainedModelÚMistralRMSNormÚMistralRotaryEmbedding©ÚMixtralConfigc          	      ó  € V ^8„  d   QhR\         P                  \        \         P                  ,          ,          R,          R\        R,          R\         P                  R,          R\         P                  \        ,          /# )é   Úgate_logitsNÚnum_expertsÚattention_maskÚreturn)ÚtorchÚTensorÚtupleÚint)Úformats   "Úl/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/mixtral/modular_mixtral.pyÚ__annotate__r)   5   sc   € ÷ B&ñ B&Ü—‘¤¤e§l¡lÕ 3Õ3°dÕ:ðB&ä�t•ðB&ô —L‘L 4Õ'ð	B&ô
 ‡\�\”CÕñB&ó    c           	     ó  € V e   \        V \        4      '       g   ^ # V ^ ,          P                  p\        P                  ! V\        P
                  VR7      p\        P                  ! V\        P
                  VR7      pRpVe1   VP                  R4      P                  V\        P
                  R7      pV  EFˆ  p	\        P                  P                  P                  V	P                  V4      RR7      p
\        P                  ! W¢RR7      w  r¼Vf~   V\        P                  ! VP                  R4      VR7      P                  4       ,           pWjP                  4       P                  ^ R7      ,           pWzP                  ^ ,          ,           pKÙ  V\        P                  ! V\        P
                  VR7      P!                  ^ VP                  R4      XP#                  V4      4      ,           pWjP                  4       VP%                  R4      ,          P                  ^ R7      ,           pWxP                  4       ,           pEK‹  	  WW,          pWg,          p\        P                  ! WÞP%                  ^ 4      ,          4      pWñ,          # )ax  
Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch.

See Switch Transformer (https://huggingface.co/papers/2101.03961) for more details. This function implements the loss
function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between
experts is too unbalanced.

Args:
    gate_logits:
        Logits from the `gate`, should be a tuple of model.config.num_hidden_layers tensors of
        shape [batch_size X sequence_length, num_experts].
    num_experts:
        Number of experts
    top_k:
        The number of experts to route per-token, can be also interpreted as the `top-k` routing
        parameter.
    attention_mask (`torch.Tensor`, *optional*):
        The attention_mask used in forward function
        shape [batch_size X sequence_length] if not None.

Returns:
    The auxiliary loss.
)ÚdtypeÚdeviceç        )r-   r,   ©Údim)Ú	minlengthéÿÿÿÿ)Ú
isinstancer%   r-   r#   ÚzerosÚfloat32ÚreshapeÚtor   Ú
functionalÚsoftmaxÚtopkÚbincountÚfloatÚsumÚshapeÚscatter_add_Úrepeat_interleaveÚ	unsqueeze)r   r    Útop_kr!   Úcompute_deviceÚtokens_per_expert_sumÚrouter_prob_sumÚ
total_rowsÚ	flat_maskÚ
layer_gateÚrouting_weightsÚ_Úselected_expertsÚtokens_per_expertÚrouter_prob_per_expertÚoverall_losss   &&&&            r(   Úload_balancing_loss_funcrO   5   sÿ  € ð: Ò¤*¨[¼%×"@Ò"@Ùð ! •^×*Ñ*€NÜ!ŸKšK¨¼5¿=¹=ÐQ_Ô`ÐÜ—k’k +´U·]±]È>ÔZ€OØ€JàÒ!à"×*Ñ*¨2Ó.×1Ñ1¸ÌuÏ}É}Ð1Ó]ˆ	ä!ˆ
ÜŸ(™(×-Ñ-×5Ñ5°j·m±mÀNÓ6SÐY[Ð5Ó\ˆÜ#Ÿjšj¨ÀRÔHÑˆØÒ!ð &¬¯ªÐ7G×7OÑ7OÐPRÓ7SÐ_jÔ(k×(qÑ(qÓ(sÕsð "ð .×0EÑ0EÓ0G×0KÑ0KÐPQÐ0KÓ0RÕRˆOØ#×&;Ñ&;¸AÕ&>Õ>ŠJð %:¼E¿KºKØ¤5§=¡=¸ô=ç‰l˜1Ð.×6Ñ6°rÓ:¸I×<WÑ<WÐX]Ó<^Ó_õ%`Ð!ð .×1FÑ1FÓ1HÈ9×K^ÑK^Ð_aÓKbÕ1b×0gÑ0gÐlmÐ0gÓ0nÕnˆOØ#§m¡m£oÕ5‹Jñ# "ð& .Õ:ÐØ,Õ9Ðä—9’9Ð.×1QÑ1QÐRSÓ1TÕTÓU€LØÕ%Ð%r*   c                   óT   a a€ ] tR t^zt oRtV3R lV 3R lltV3R lR ltRtVtV ;t	# )ÚMixtralExpertsz2Collection of expert weights stored as 3D tensors.c                ó    <€ V ^8„  d   QhRS[ /# )r   Úconfigr   )r'   Ú__classdict__s   "€r(   r)   ÚMixtralExperts.__annotate__~   s   ø€ ÷ 0ñ 0™}ñ 0r*   c                ó  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  V n        \        P                  ! \        P                  ! V P                  ^V P                  ,          V P
                  4      4      V n        \        P                  ! \        P                  ! V P                  V P
                  V P                  4      4      V n        \        VP                  ,          V n        R# )r   N)ÚsuperÚ__init__Únum_local_expertsr    Úhidden_sizeÚ
hidden_dimÚintermediate_sizeÚintermediate_dimr   Ú	Parameterr#   ÚemptyÚgate_up_projÚ	down_projr   Ú
hidden_actÚact_fn©ÚselfrS   Ú	__class__s   &&€r(   rX   ÚMixtralExperts.__init__~   s­   ø€ Ü‰ÑÔØ!×3Ñ3ˆÔØ ×,Ñ,ˆŒØ &× 8Ñ 8ˆÔÜŸLšL¬¯ª°T×5EÑ5EÀqÈ4×K`ÑK`ÕG`Ðbf×bqÑbqÓ)rÓsˆÔÜŸš¤e§k¢k°$×2BÑ2BÀDÇOÁOÐUY×UjÑUjÓ&kÓlˆŒÜ˜V×.Ñ.Õ/ˆŽr*   c                ó‚   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[ P                  /# )r   Úhidden_statesÚtop_k_indexÚtop_k_weightsr"   ©r#   r$   )r'   rT   s   "€r(   r)   rU   ‡   sC   ø€ ÷ #ñ #á—|‘|ð#ñ —\‘\ð#ñ —|‘|ð	#ñ
 
�‰ñ#r*   c                óÆ  € \         P                  ! V4      p\         P                  ! 4       ;_uu_ 4        \         P                  P                  P                  W P                  R 7      pVP                  ^^^ 4      p\         P                  ! VP                  RR7      ^ 4      P                  4       pRRR4       X EF  pV^ ,          pWpP                  8X  d   K  \         P                  ! XV,          4      w  r‰W,          p
\        P                  P                  W P                  V,          4      P                  ^RR7      w  r¼V P                  V4      V,          p\        P                  P                  WÐP                   V,          4      pWÓW˜R3,          ,          pVP#                  ^ W�P%                  VP&                  4      4       EK  	  V#   + '       g   i     EL'; i))Únum_classesr/   Nr2   )r2   éþÿÿÿ)r#   Ú
zeros_likeÚno_gradr   r8   Úone_hotr    ÚpermuteÚgreaterr=   ÚnonzeroÚwhereÚlinearr`   Úchunkrc   ra   Ú
index_add_r7   r,   )re   ri   rj   rk   Úfinal_hidden_statesÚexpert_maskÚ
expert_hitÚ
expert_idxÚ	top_k_posÚ	token_idxÚcurrent_stateÚgateÚupÚcurrent_hidden_statess   &&&&          r(   ÚforwardÚMixtralExperts.forward‡   ss  € ô $×.Ò.¨}Ó=ÐÜ�]Š]�_�_ÜŸ(™(×-Ñ-×5Ñ5°k×O_ÑO_Ð5Ó`ˆKØ%×-Ñ-¨a°°AÓ6ˆKÜŸš {§¡¸8 Ó'DÀaÓH×PÑPÓRˆJ÷ ô
 %ˆJØ# A�ˆJØ×-Ñ-Ô-ÙÜ#(§;¢;¨{¸:Õ/FÓ#GÑ ˆIØ)Õ4ˆMÜ—}‘}×+Ñ+¨M×;LÑ;LÈZÕ;XÓY×_Ñ_Ð`aÐgiÐ_Ój‰HˆDØ$(§K¡K°Ó$5¸Õ$:Ð!Ü$&§M¡M×$8Ñ$8Ð9N×P^ÑP^Ð_iÕPjÓ$kÐ!Ø$9È)Ð`dÐJdÕ<eÕ$eÐ!Ø×*Ñ*¨1¨i×9QÑ9QÐRe×RkÑRkÓ9l×mñ %ð #Ð"÷# �_ˆ_ús   ´A=GÇG 	)rc   ra   r`   r[   r]   r    )
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__rX   r„   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©rf   rT   s   @@r(   rQ   rQ   z   s!   ù‡ € á<÷0ó 0÷#÷ #ð #r*   rQ   c                   ó8   a a€ ] tR t^¢t oV 3R ltR tRtVtV ;t# )ÚMixtralTopKRouterc                ó  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  V n        \        P                  ! \        P                  ! V P
                  V P                  4      4      V n        R # ©N)rW   rX   Únum_experts_per_tokrB   rY   r    rZ   r[   r   r^   r#   r_   Úweightrd   s   &&€r(   rX   ÚMixtralTopKRouter.__init__£   s[   ø€ Ü‰ÑÔØ×/Ñ/ˆŒ
Ø!×3Ñ3ˆÔØ ×,Ñ,ˆŒÜ—l’l¤5§;¢;¨t×/?Ñ/?ÀÇÁÓ#QÓRˆŽr*   c                ór  € VP                  RV P                  4      p\        P                  ! WP                  4      p\
        P                  P                  P                  VP                  4       RR7      p\
        P                  ! W0P                  RR7      w  rEWDP                  RRR7      ,          pTpW&V3# )é   r/   T)r0   Úkeepdimr2   )r6   r[   ÚFrw   r”   r#   r   r8   r9   r<   r:   rB   r=   )re   ri   Úrouter_logitsÚrouter_probsÚrouter_top_valueÚrouter_indicesÚrouter_scoress   &&     r(   r„   ÚMixtralTopKRouter.forwardª   s”   € Ø%×-Ñ-¨b°$·/±/ÓBˆÜŸš ·±Ó<ˆÜ—x‘x×*Ñ*×2Ñ2°=×3FÑ3FÓ3HÈbÐ2ÓQˆÜ+0¯:ª:°lÇJÁJÐTVÔ+WÑ(ÐØ×0Ñ0°RÀÐ0ÓFÕFÐØ(ˆØ¨^Ð;Ð;r*   )r[   r    rB   r”   ©	r†   r‡   rˆ   r‰   rX   r„   r‹   rŒ   r�   rŽ   s   @@r(   r�   r�   ¢   s   ù‡ € õS÷<ò <r*   r�   c                   óD   a a€ ] tR t^´t oV 3R ltV3R lR ltRtVtV ;t# )ÚMixtralSparseMoeBlockc                ó¨   <€ \         SV `  4        VP                  V n        VP                  V n        \        V4      V n        \        V4      V n	        R # r’   )
rW   rX   r“   rB   Úrouter_jitter_noiseÚjitter_noiser�   r�   rQ   Úexpertsrd   s   &&€r(   rX   ÚMixtralSparseMoeBlock.__init__µ   sA   ø€ Ü‰ÑÔØ×/Ñ/ˆŒ
Ø"×6Ñ6ˆÔÜ% fÓ-ˆŒ	Ü% fÓ-ˆŽr*   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r   ri   r"   rl   )r'   rT   s   "€r(   r)   Ú"MixtralSparseMoeBlock.__annotate__¼   s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r*   c                ó¾  € VP                   w  r#pV P                  '       da   V P                  ^ 8”  dP   V\        P                  ! V4      P                  RV P                  ,
          RV P                  ,           4      ,          pVP                  RVP                   R,          4      pV P                  V4      w  rVpV P                  WV4      pVP                  W#V4      pV# )é    g      ð?r2   )
r>   Útrainingr¥   r#   Ú
empty_likeÚuniform_Úviewr�   r¦   r6   )re   ri   Ú
batch_sizeÚsequence_lengthr[   rJ   rk   rj   s   &&      r(   r„   ÚMixtralSparseMoeBlock.forward¼   sº   € Ø2?×2EÑ2EÑ/ˆ
 ZØ�=�=ˆ=˜T×.Ñ.°Ô2ØœU×-Ò-¨mÓ<×EÑEÀcÈD×L]ÑL]ÕF]Ð_bÐei×evÑevÕ_vÓwÕwˆMØ%×*Ñ*¨2¨}×/BÑ/BÀ2Õ/FÓGˆØ(,¯	©	°-Ó(@Ñ%ˆ˜+ØŸ™ ]ÀÓOˆØ%×-Ñ-¨jÈ:ÓVˆØÐr*   )r¦   r�   r¥   rB   r    rŽ   s   @@r(   r¢   r¢   ´   s   ù‡ € õ.÷÷ ð r*   r¢   c                   ó   € ] tR t^ÇtRtR# )ÚMixtralRMSNorm© N©r†   r‡   rˆ   r‰   r‹   rµ   r*   r(   r´   r´   Ç   ó   † Ûr*   r´   c                   ó   € ] tR t^ËtRtR# )ÚMixtralRotaryEmbeddingrµ   Nr¶   rµ   r*   r(   r¹   r¹   Ë   r·   r*   r¹   c                   ó   € ] tR t^ÏtRtR# )ÚMixtralAttentionrµ   Nr¶   rµ   r*   r(   r»   r»   Ï   r·   r*   r»   c                   óT   a a€ ] tR t^Ót oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚMixtralDecoderLayerc                ó&   <€ V ^8„  d   QhRS[ RS[/# )r   rS   Ú	layer_idx)r   r&   )r'   rT   s   "€r(   r)   Ú MixtralDecoderLayer.__annotate__Ô   s   ø€ ÷ dñ d™}ð d¹ñ dr*   c                ó  <€ \         SV `  4        VP                  V n        \        W4      V n        \        V4      V n        \        VP                  VP                  R 7      V n	        \        VP                  VP                  R 7      V n
        R# ))ÚepsN)rW   rX   rZ   r»   Ú	self_attnr¢   Úmlpr´   Úrms_norm_epsÚinput_layernormÚpost_attention_layernorm)re   rS   r¿   rf   s   &&&€r(   rX   ÚMixtralDecoderLayer.__init__Ô   sj   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔä)¨&Ó<ˆŒä(¨Ó0ˆŒÜ-¨f×.@Ñ.@Àf×FYÑFYÔZˆÔÜ(6°v×7IÑ7IÈv×ObÑObÔ(cˆÖ%r*   c                ó  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[S[,          RS[ P                  /# )	r   ri   Úposition_embeddingsNr!   Úposition_idsÚpast_key_valuesÚkwargsr"   )r#   r$   r%   Ú
LongTensorr   r   r   )r'   rT   s   "€r(   r)   rÀ   Þ   sŽ   ø€ ÷ ñ á—|‘|ðñ #¡5§<¡<±·±Ð#=Õ>ÀÕEðñ Ÿ™ tÕ+ð	ñ
 ×&Ñ&¨Õ-ðñ  �ðñ Ñ+Õ,ðñ 
�‰ñr*   c                óÐ   € TpV P                  V4      pV P                  ! RR VRVRVRVRV/VB w  rWq,           pTpV P                  V4      pV P                  V4      pWq,           pV# )ri   rÊ   r!   rË   rÌ   rµ   )rÆ   rÃ   rÇ   rÄ   )	re   ri   rÊ   r!   rË   rÌ   rÍ   ÚresidualrJ   s	   &&&&&&,  r(   r„   ÚMixtralDecoderLayer.forwardÞ   s�   € ð !ˆØ×,Ñ,¨]Ó;ˆØŸ>š>ñ 
Ø'ð
à 3ð
ð *ð
ð &ð	
ð
 ,ð
ð ñ
Ñˆð !Õ0ˆØ ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ Õ0ˆØÐr*   )rZ   rÆ   rÄ   rÇ   rÃ   )NNNNr    rŽ   s   @@r(   r½   r½   Ó   s    ù‡ € ÷dó d÷÷ ò r*   r½   c                   ón   a € ] tR t^ùt o R]! ]^ R7      R]R]/t]	P                  ! 4       R 4       tRtV tR# )ÚMixtralPreTrainedModelrš   )Úindexri   Ú
attentionsc                ó�  € \         P                  ! W4       V P                  P                  p\	        V\
        4      '       dI   \        P                  ! VP                  R VR7       \        P                  ! VP                  R VR7       R# \	        V\        4      '       d&   \        P                  ! VP                  R VR7       R# R# )r.   )ÚmeanÚstdN)r   Ú_init_weightsrS   Úinitializer_ranger3   rQ   ÚinitÚnormal_r`   ra   r�   r”   )re   ÚmodulerØ   s   && r(   rÙ   Ú$MixtralPreTrainedModel._init_weights   s   € ä×%Ò% dÔ3Ø�k‰k×+Ñ+ˆÜ�fœn×-Ò-Ü�LŠL˜×,Ñ,°3¸CÕ@Ü�LŠL˜×)Ñ)°¸×=Ü˜Ô 1×2Ò2Ü�LŠL˜Ÿ™¨S°c×:ñ 3r*   rµ   N)r†   r‡   rˆ   r‰   r   r�   r½   r»   Ú_can_record_outputsr#   rq   rÙ   r‹   rŒ   ©rT   s   @r(   rÓ   rÓ   ù   s@   ø‡ € à™Ð(9ÀÔCØÐ,ØÐ&ðÐð ‡]‚]ƒ_ñ;ó ö;r*   rÓ   c                   ó6   a € ] tR tRt o RV 3R lR lltRtV tR# )ÚMixtralModeli  Nc                óþ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[S[,          R	S[/# )
r   Ú	input_idsNr!   rË   rÌ   Úinputs_embedsÚ	use_cacherÍ   r"   )	r#   rÎ   r$   r   ÚFloatTensorÚboolr   r   r   )r'   rT   s   "€r(   r)   ÚMixtralModel.__annotate__  s“   ø€ ÷ 4
ñ 4
á×#Ñ# dÕ*ð4
ñ Ÿ™ tÕ+ð4
ñ ×&Ñ&¨Õ-ð	4
ñ
  �ð4
ñ ×(Ñ(¨4Õ/ð4
ñ ˜$•;ð4
ñ Ñ+Õ,ð4
ñ 
 ñ4
r*   c                óÊ  € VR J VR J,          '       d   \        R4      hV'       d   Vf   \        V P                  R7      pVf   V P                  V4      pVfb   Ve   VP	                  4       M^ p\
        P                  ! VP                  ^,          VP                  R7      V,           pVP                  ^ 4      pV P                  P                  f   \        M\        p	V	! V P                  VVVVR7      p
TpV P                  W³R7      pV P                  R V P                  P                    F  pV! V3RV
RVRVR	VR
V/VB pK  	  V P!                  V4      p\#        VVR7      # )Nz:You must specify exactly one of input_ids or inputs_embeds)rS   )r-   )rS   rå   r!   rÌ   rË   )rË   r!   rË   rÌ   ræ   rÊ   )Úlast_hidden_staterÌ   )Ú
ValueErrorr   rS   Úembed_tokensÚget_seq_lengthr#   Úaranger>   r-   rA   Úsliding_windowr   r	   Ú
rotary_embÚlayersÚnum_hidden_layersÚnormr   )re   rä   r!   rË   rÌ   rå   ræ   rÍ   Úpast_seen_tokensÚmask_functionÚcausal_maskri   rÊ   Údecoder_layers   &&&&&&&,      r(   r„   ÚMixtralModel.forward  s}  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZç˜Ò0Ü*°$·+±+Ô>ˆOàÒ Ø ×-Ñ-¨iÓ8ˆMàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLà.2¯k©k×.HÑ.HÒ.PÕ*ÔVwˆÙ#Ø—;‘;Ø'Ø)Ø+Ø%ô
ˆð &ˆØ"Ÿo™o¨m˜oÓWÐà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÓIˆMÙ)Øñà*ðð *ðð !0ð	ð
 $ðð %8ðð ñŠMñ Jð Ÿ	™	 -Ó0ˆä%Ø+Ø+ô
ð 	
r*   rµ   )NNNNNN)r†   r‡   rˆ   r‰   r„   r‹   rŒ   rà   s   @r(   râ   râ     s   ø‡ € ÷4
÷ 4
ð 4
r*   râ   c                   óX   a a€ ] tR tRt oRR/tRR/tV 3R ltR
V3R lR lltR	tVt	V ;t
# )ÚMixtralForCausalLMiC  zlm_head.weightzmodel.embed_tokens.weightÚlm_headÚkeep_full_weightc                ó¬   <€ \         SV `  V4       \        V4      V n        VP                  V n        VP
                  V n        VP                  V n        R # r’   )rW   rX   râ   ÚmodelÚrouter_aux_loss_coefrY   r    r“   rd   s   &&€r(   rX   ÚMixtralForCausalLM.__init__G  sF   ø€ Ü‰Ñ˜Ô Ü! &Ó)ˆŒ
Ø$*×$?Ñ$?ˆÔ!Ø!×3Ñ3ˆÔØ#)×#=Ñ#=ˆÖ r*   c                ód  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[R,          R
S[S[ P                  ,          RS[S[,          RS[	/# )r   rä   Nr!   rË   rÌ   rå   Úlabelsræ   Úoutput_router_logitsÚlogits_to_keeprÍ   r"   )
r#   rÎ   r$   r   rç   rè   r&   r   r   r   )r'   rT   s   "€r(   r)   ÚMixtralForCausalLM.__annotate__N  sÖ   ø€ ÷ P
ñ P
á×#Ñ# dÕ*ðP
ñ Ÿ™ tÕ+ðP
ñ ×&Ñ&¨Õ-ð	P
ñ
  �ðP
ñ ×(Ñ(¨4Õ/ðP
ñ × Ñ  4Õ'ðP
ñ ˜$•;ðP
ñ # T�kðP
ñ ™eŸl™lÕ*ðP
ñ Ñ+Õ,ðP
ñ 
#ñP
r*   c
                ó¦  € Ve   TMV P                   P                  pV P                  ! RRVRVRVRVRVRVRV/V
B pVP                  p\	        V	\
        4      '       d   \        V	) R4      MT	pV P                  VR	VR	3,          4      pRpVe   V P                  ! WæV P                  3/ V
B pRpV'       dc   \        VP                  V P                  V P                  V4      pVe3   WðP                  VP                  VP                   4      ,          ,          p\#        VVVVP$                  VP&                  VP(                  VP                  R
7      # )aþ  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

Example:

```python
>>> from transformers import AutoTokenizer, MixtralForCausalLM

>>> model = MixtralForCausalLM.from_pretrained("mistralai/Mixtral-8x7B-v0.1")
>>> tokenizer = AutoTokenizer.from_pretrained("mistralai/Mixtral-8x7B-v0.1")

>>> prompt = "Hey, are you conscious? Can you talk to me?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
```Nrä   r!   rË   rÌ   rå   ræ   r  :NNN)ÚlossÚaux_lossÚlogitsrÌ   ri   rÕ   rš   rµ   )rS   r  rÿ   rë   r3   r&   Úslicerü   Úloss_functionÚ
vocab_sizerO   rš   r    r“   r   r7   r-   r   rÌ   ri   rÕ   )re   rä   r!   rË   rÌ   rå   r  ræ   r  r  rÍ   Úoutputsri   Úslice_indicesr
  r  r	  s   &&&&&&&&&&,      r(   r„   ÚMixtralForCausalLM.forwardN  sx  € ðJ %9Ò$DÑ È$Ï+É+×JjÑJjð 	ð
 +/¯*ª*ñ 	+
Øð	+
à)ð	+
ð &ð	+
ð ,ð		+
ð
 (ð	+
ð  ð	+
ð "6ð	+
ð ñ	+
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒØ×%Ò% f°d·o±oÑPÈÑPˆDàˆßÜ/Ø×%Ñ%Ø× Ñ Ø×(Ñ(Øó	ˆHð Ò!Ø×1Ñ1°H·K±KÀÇÁÓ4LÕLÕL�ä(ØØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø!×/Ñ/ô
ð 	
r*   )rÿ   r    r“   r   )	NNNNNNNNr«   )r†   r‡   rˆ   r‰   Ú_tied_weights_keysÚ
_fsdp_planrX   r„   r‹   rŒ   r�   rŽ   s   @@r(   rû   rû   C  s3   ù‡ € Ø*Ð,GÐHÐØÐ/Ð0€Jõ>÷P
÷ P
ò P
r*   rû   c                   ó   € ] tR tRtRtR# )Ú MixtralForSequenceClassificationi¡  rµ   Nr¶   rµ   r*   r(   r  r  ¡  r·   r*   r  c                   ó   € ] tR tRtRtR# )ÚMixtralForTokenClassificationi¥  rµ   Nr¶   rµ   r*   r(   r  r  ¥  r·   r*   r  c                   ó   € ] tR tRtRtR# )ÚMixtralForQuestionAnsweringi©  rµ   Nr¶   rµ   r*   r(   r  r  ©  r·   r*   r  )Nr   N)rû   r  râ   rÓ   r  r  )@rŠ   r#   Útorch.nn.functionalr   r8   r™   Ú r   rÛ   Úactivationsr   Úcache_utilsr   r   Úintegrationsr   Úmasking_utilsr   r	   Úmodeling_layersr
   Úmodeling_outputsr   r   Úmodeling_utilsr   Úprocessing_utilsr   Úutilsr   r   Úutils.output_capturingr   Úmistral.modeling_mistralr   r   r   r   r   r   r   r   r   Úconfiguration_mixtralr   Ú
get_loggerr†   ÚloggerrO   ÚModulerQ   r�   r¢   r´   r¹   r»   r½   rÓ   râ   rû   r  r  r  Ú__all__rµ   r*   r(   Ú<module>r+     s0  ðñ& ã ß Ð Ý å &Ý !ß .Ý 6ß RÝ 9ß QÝ -Ý &ß 0Ý 4÷
÷ 
õ 
õ 1ð 
×	Ò	˜HÓ	%€÷B&ðJ ô$#�R—Y‘Yó $#ó ð$#ôN<˜Ÿ	™	ô <ô$˜BŸI™Iô ô&	�^ô 	ô	Ð3ô 	ô	Ð'ô 	ô#Ð4ô #ôL;Ð3ô ;ô$5
�<ô 5
ôp[
Ð+ô [
ô|	Ð'Gô 	ô	Ð$Aô 	ô	Ð"=ô 	ò‚r*   