+
    œ°³jÙu  ã                   óˆ  € ^ RI Ht ^ RIt^ RIHu Ht ^ RIHt ^RIHt	 ^RI
Ht ^RIHtHt ^RIHt ^RIHtHtHt ^R	IHtHt ^R
IHt ^RIHtHtHtHt ^RIH t H!t! ^RI"H#t#H$t$ ^RI%H&t&H't' ^RI(H)t) ^RI*H+t+H,t,H-t- ^RI.H/t/ ^RI0H1t1H2t2 ^RI3H4t4H5t5 ^RI6H7t7 ] ! R R]Pp                  4      4       t9 ! R R]Pp                  4      t: ! R R]Pp                  4      t;]! R4       ! R R]Pp                  4      4       t< ! R R]Pp                  4      t=R  t>]! R!4      R9R" l4       t?R# R$ lt@R:R% R& lltA]! ]?4       ! R' R(]Pp                  4      4       tB ! R) R*]4      tC], ! R+ R,]'4      4       tD], ! R- R.]D4      4       tER;R/ R0 lltF], ! R1 R2]D]4      4       tG ! R3 R4]]D4      tH ! R5 R6]]D4      tI ! R7 R8]]D4      tJ. R<OtKR# )=é    )ÚCallableN)Únn)Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_experts_implementationÚuse_kernel_forward_from_hubÚuse_kernelized_func)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚFlashAttentionKwargs)ÚGenericForQuestionAnsweringÚ GenericForSequenceClassificationÚGenericForTokenClassificationÚGradientCheckpointingLayer)ÚMoeCausalLMOutputWithPastÚMoeModelOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputs©ÚMixtralConfigc                   óT   a a€ ] tR t^8t oRtV3R lV 3R lltV3R lR ltRtVtV ;t	# )ÚMixtralExpertsz2Collection of expert weights stored as 3D tensors.c                ó    <€ V ^8„  d   QhRS[ /# ©é   Úconfigr#   )ÚformatÚ__classdict__s   "€Úm/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/mixtral/modeling_mixtral.pyÚ__annotate__ÚMixtralExperts.__annotate__<   s   ø€ ÷ 0ñ 0™}ñ 0ó    c                ó  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  V n        \        P                  ! \        P                  ! V P                  ^V P                  ,          V P
                  4      4      V n        \        P                  ! \        P                  ! V P                  V P
                  V P                  4      4      V n        \        VP                  ,          V n        R# )r)   N)ÚsuperÚ__init__Únum_local_expertsÚnum_expertsÚhidden_sizeÚ
hidden_dimÚintermediate_sizeÚintermediate_dimr   Ú	ParameterÚtorchÚemptyÚgate_up_projÚ	down_projr   Ú
hidden_actÚact_fn©Úselfr*   Ú	__class__s   &&€r-   r3   ÚMixtralExperts.__init__<   s­   ø€ Ü‰ÑÔØ!×3Ñ3ˆÔØ ×,Ñ,ˆŒØ &× 8Ñ 8ˆÔÜŸLšL¬¯ª°T×5EÑ5EÀqÈ4×K`ÑK`ÕG`Ðbf×bqÑbqÓ)rÓsˆÔÜŸš¤e§k¢k°$×2BÑ2BÀDÇOÁOÐUY×UjÑUjÓ&kÓlˆŒÜ˜V×.Ñ.Õ/ˆŽr0   c                ó‚   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[ P                  /# )r)   Úhidden_statesÚtop_k_indexÚtop_k_weightsÚreturn©r;   ÚTensor)r+   r,   s   "€r-   r.   r/   E   sC   ø€ ÷ #ñ #á—|‘|ð#ñ —\‘\ð#ñ —|‘|ð	#ñ
 
�‰ñ#r0   c                óÆ  € \         P                  ! V4      p\         P                  ! 4       ;_uu_ 4        \         P                  P                  P                  W P                  R 7      pVP                  ^^^ 4      p\         P                  ! VP                  RR7      ^ 4      P                  4       pRRR4       X EF  pV^ ,          pWpP                  8X  d   K  \         P                  ! XV,          4      w  r‰W,          p
\        P                  P                  W P                  V,          4      P                  ^RR7      w  r¼V P                  V4      V,          p\        P                  P                  WÐP                   V,          4      pWÓW˜R3,          ,          pVP#                  ^ W�P%                  VP&                  4      4       EK  	  V#   + '       g   i     EL'; i))Únum_classes©ÚdimNéÿÿÿÿ)rP   éþÿÿÿ)r;   Ú
zeros_likeÚno_gradr   Ú
functionalÚone_hotr5   ÚpermuteÚgreaterÚsumÚnonzeroÚwhereÚlinearr=   Úchunkr@   r>   Ú
index_add_ÚtoÚdtype)rB   rF   rG   rH   Úfinal_hidden_statesÚexpert_maskÚ
expert_hitÚ
expert_idxÚ	top_k_posÚ	token_idxÚcurrent_stateÚgateÚupÚcurrent_hidden_statess   &&&&          r-   ÚforwardÚMixtralExperts.forwardE   ss  € ô $×.Ò.¨}Ó=ÐÜ�]Š]�_�_ÜŸ(™(×-Ñ-×5Ñ5°k×O_ÑO_Ð5Ó`ˆKØ%×-Ñ-¨a°°AÓ6ˆKÜŸš {§¡¸8 Ó'DÀaÓH×PÑPÓRˆJ÷ ô
 %ˆJØ# A�ˆJØ×-Ñ-Ô-ÙÜ#(§;¢;¨{¸:Õ/FÓ#GÑ ˆIØ)Õ4ˆMÜ—}‘}×+Ñ+¨M×;LÑ;LÈZÕ;XÓY×_Ñ_Ð`aÐgiÐ_Ój‰HˆDØ$(§K¡K°Ó$5¸Õ$:Ð!Ü$&§M¡M×$8Ñ$8Ð9N×P^ÑP^Ð_iÕPjÓ$kÐ!Ø$9È)Ð`dÐJdÕ<eÕ$eÐ!Ø×*Ñ*¨1¨i×9QÑ9QÐRe×RkÑRkÓ9l×mñ %ð #Ð"÷# �_ˆ_ús   ´A=GÇG 	)r@   r>   r=   r7   r9   r5   ©
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r3   rj   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©rC   r,   s   @@r-   r&   r&   8   s!   ù‡ € á<÷0ó 0÷#÷ #ð #r0   r&   c                   ó8   a a€ ] tR t^`t oV 3R ltR tRtVtV ;t# )ÚMixtralTopKRouterc                ó  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  V n        \        P                  ! \        P                  ! V P
                  V P                  4      4      V n        R # ©N)r2   r3   Únum_experts_per_tokÚtop_kr4   r5   r6   r7   r   r:   r;   r<   ÚweightrA   s   &&€r-   r3   ÚMixtralTopKRouter.__init__a   s[   ø€ Ü‰ÑÔØ×/Ñ/ˆŒ
Ø!×3Ñ3ˆÔØ ×,Ñ,ˆŒÜ—l’l¤5§;¢;¨t×/?Ñ/?ÀÇÁÓ#QÓRˆŽr0   c                ór  € VP                  RV P                  4      p\        P                  ! WP                  4      p\
        P                  P                  P                  VP                  4       RR7      p\
        P                  ! W0P                  RR7      w  rEWDP                  RRR7      ,          pTpW&V3# )é   rN   T)rO   ÚkeepdimrP   )Úreshaper7   ÚFr[   r|   r;   r   rT   ÚsoftmaxÚfloatÚtopkr{   rX   )rB   rF   Úrouter_logitsÚrouter_probsÚrouter_top_valueÚrouter_indicesÚrouter_scoress   &&     r-   rj   ÚMixtralTopKRouter.forwardh   s”   € Ø%×-Ñ-¨b°$·/±/ÓBˆÜŸš ·±Ó<ˆÜ—x‘x×*Ñ*×2Ñ2°=×3FÑ3FÓ3HÈbÐ2ÓQˆÜ+0¯:ª:°lÇJÁJÐTVÔ+WÑ(ÐØ×0Ñ0°RÀÐ0ÓFÕFÐØ(ˆØ¨^Ð;Ð;r0   )r7   r5   r{   r|   ©	rm   rn   ro   rp   r3   rj   rr   rs   rt   ru   s   @@r-   rw   rw   `   s   ù‡ € õS÷<ò <r0   rw   c                   óD   a a€ ] tR t^rt oV 3R ltV3R lR ltRtVtV ;t# )ÚMixtralSparseMoeBlockc                ó¨   <€ \         SV `  4        VP                  V n        VP                  V n        \        V4      V n        \        V4      V n	        R # ry   )
r2   r3   rz   r{   Úrouter_jitter_noiseÚjitter_noiserw   rg   r&   ÚexpertsrA   s   &&€r-   r3   ÚMixtralSparseMoeBlock.__init__s   sA   ø€ Ü‰ÑÔØ×/Ñ/ˆŒ
Ø"×6Ñ6ˆÔÜ% fÓ-ˆŒ	Ü% fÓ-ˆŽr0   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r)   rF   rI   rJ   )r+   r,   s   "€r-   r.   Ú"MixtralSparseMoeBlock.__annotate__z   s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r0   c                ó¾  € VP                   w  r#pV P                  '       da   V P                  ^ 8”  dP   V\        P                  ! V4      P                  RV P                  ,
          RV P                  ,           4      ,          pVP                  RVP                   R,          4      pV P                  V4      w  rVpV P                  WV4      pVP                  W#V4      pV# )r   ç      ð?rP   )
ÚshapeÚtrainingr‘   r;   Ú
empty_likeÚuniform_Úviewrg   r’   r�   )rB   rF   Ú
batch_sizeÚsequence_lengthr7   Ú_rH   rG   s   &&      r-   rj   ÚMixtralSparseMoeBlock.forwardz   sº   € Ø2?×2EÑ2EÑ/ˆ
 ZØ�=�=ˆ=˜T×.Ñ.°Ô2ØœU×-Ò-¨mÓ<×EÑEÀcÈD×L]ÑL]ÕF]Ð_bÐei×evÑevÕ_vÓwÕwˆMØ%×*Ñ*¨2¨}×/BÑ/BÀ2Õ/FÓGˆØ(,¯	©	°-Ó(@Ñ%ˆ˜+ØŸ™ ]ÀÓOˆØ%×-Ñ-¨jÈ:ÓVˆØÐr0   )r’   rg   r‘   r{   rŒ   ru   s   @@r-   rŽ   rŽ   r   s   ù‡ € õ.÷÷ ð r0   rŽ   ÚRMSNormc                   óZ   a a€ ] tR t^…t oRV3R lV 3R llltV3R lR ltR tRtVtV ;t	# )ÚMixtralRMSNormc                ó$   <€ V ^8„  d   QhRS[ RR/# )r)   ÚepsrI   N)r„   )r+   r,   s   "€r-   r.   ÚMixtralRMSNorm.__annotate__‡   s   ø€ ÷ $ñ $©ð $¸$ñ $r0   c                óŽ   <€ \         SV `  4        \        P                  ! \        P
                  ! V4      4      V n        W n        R# )z-
MixtralRMSNorm is equivalent to T5LayerNorm
N)r2   r3   r   r:   r;   Úonesr|   Úvariance_epsilon)rB   r6   r¦   rC   s   &&&€r-   r3   ÚMixtralRMSNorm.__init__‡   s/   ø€ ô 	‰ÑÔÜ—l’l¤5§:¢:¨kÓ#:Ó;ˆŒØ #Ör0   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r•   rJ   )r+   r,   s   "€r-   r.   r§   �   s#   ø€ ÷ ;ñ ;¡U§\¡\ð ;±e·l±lñ ;r0   c                ó:  € VP                   pVP                  \        P                  4      pVP	                  ^4      P                  RRR7      pV\        P                  ! W0P                  ,           4      ,          pV P                  VP                  V4      ,          # )r)   T)r€   rP   )	r_   r^   r;   Úfloat32ÚpowÚmeanÚrsqrtrª   r|   )rB   rF   Úinput_dtypeÚvariances   &&  r-   rj   ÚMixtralRMSNorm.forward�   sw   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯ª°H×?TÑ?TÕ4TÓ(UÕUˆØ�{‰{˜]×-Ñ-¨kÓ:Õ:Ð:r0   c                ó^   € \        V P                  P                  4       R V P                   2# )z, eps=)Útupler|   r™   rª   )rB   s   &r-   Ú
extra_reprÚMixtralRMSNorm.extra_repr–   s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr0   )rª   r|   )g�íµ ÷Æ°>)
rm   rn   ro   rp   r3   rj   r·   rr   rs   rt   ru   s   @@r-   r¤   r¤   …   s%   ù‡ € ÷$õ $÷;ð ;÷Jò Jr0   r¤   c                   óÎ   a a€ ] tR t^št o]! RRR7      R
V3R lV 3R lll4       t]]! RRR7      R
V3R lR ll4       4       t]P                  ! 4       ]
R 4       4       tR	tVtV ;t# )ÚMixtralRotaryEmbeddingÚdevicez5.18)Úversionc                ó    <€ V ^8„  d   QhRS[ /# r(   r#   )r+   r,   s   "€r-   r.   Ú#MixtralRotaryEmbedding.__annotate__œ   s   ø€ ÷ Oñ O™}ñ Or0   c                óä  <€ \         SV `  4        VP                  V n        VP                  V n        Wn        V P
                  P                  R ,          V n        V P                  pV P                  R8w  d   \        V P                  ,          pV! V P
                  V4      w  q@n
        \        P                  ! VRR7      V n        \        P                  ! VP                  4       RR7      V n        R# )Ú	rope_typeÚdefaultF)Ú
persistentN)r2   r3   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr*   Úrope_parametersrÀ   Úcompute_default_rope_parametersr   Úattention_scalingr   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)rB   r*   r»   Úrope_init_fnrÊ   rC   s   &&&  €r-   r3   ÚMixtralRotaryEmbedding.__init__›   s¬   ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àŒàŸ™×4Ñ4°[ÕAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÔ&Ü.¨t¯~©~Õ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÔ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÔ!NˆÖr0   c                óP   <€ V ^8„  d   QhRS[ RS[S[P                  S[3,          /# )r)   r*   rI   )r$   r¶   r;   rK   r„   )r+   r,   s   "€r-   r.   r¾   ®   s.   ø€ ÷ 5ñ 5±ð 5ÑY^Ñ_d×_kÑ_kÑmrÐ_rÕYsñ 5r0   c           	     ó0  € V P                   R,          p\        V RR4      ;'       g    V P                  V P                  ,          pRpRV\        P
                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚhead_dimNr˜   ©r_   )rÆ   Úgetattrr6   Únum_attention_headsr;   Úaranger„   r^   )r*   r»   ÚkwargsÚbaserO   Úattention_factorrÊ   s   &&,    r-   rÇ   Ú6MixtralRotaryEmbedding.compute_default_rope_parameters¬   s~   € ð ×%Ñ% lÕ3ˆÜ�f˜j¨$Ó/×cÐc°6×3EÑ3EÈ×IcÑIcÕ3cˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r0   c                ó^  € V P                   R,          P                  VP                  ^ ,          R	^4      P                  \        P
                  VP                  R7      pVR
,          P                  4       p\        VP                  P                  \        4      '       d2   VP                  P                  R8w  d   VP                  P                  MRp\        VRR7      ;_uu_ 4        W4,          P                  ^^4      p\        P                  ! Wf3R	R7      pVP                  4       V P                  ,          pVP                  4       V P                  ,          p	R R R 4       XP                  VP                   R7      X	P                  VP                   R7      3#   + '       g   i     LH; i)N©r_   r»   ÚmpsÚcpuF)Údevice_typeÚenabledrN   rÓ   )NºNNNNrP   )rá   Nrá   )rÊ   Úexpandr™   r^   r;   r„   r»   Ú
isinstanceÚtypeÚstrr   Ú	transposeÚcatÚcosrÈ   Úsinr_   )
rB   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedrß   ÚfreqsÚembrè   ré   s
   &&&       r-   rj   ÚMixtralRotaryEmbedding.forwardÀ   s2  € ð �M‰M˜-Õ(×/Ñ/°×0BÑ0BÀ1Õ0EÀrÈ1ÓM×PÑPÔW\×WbÑWbÐkl×ksÑksÐPÓtð 	ð !-¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆä¨¸U×CÖCØ&Õ>×IÑIÈ!ÈQÓOˆEÜ—)’)˜U˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆC÷	 Dð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷ D×Cús   Ã(A4FÆF,	)rÈ   r*   rÊ   rÄ   rÌ   rÅ   rÀ   ry   )rm   rn   ro   rp   r   r3   ÚstaticmethodrÇ   r;   rS   r   rj   rr   rs   rt   ru   s   @@r-   rº   rº   š   su   ù‡ € Ù�X vÔ.÷Oô Oó /ðOð  Ù�X vÔ.÷5ñ 5ó /ó ð5ð$ ‡]‚]ƒ_Øñ<ó ó ÷<ð <r0   rº   c                óÌ   € V RRV P                   R,          ^,          13,          pV RV P                   R,          ^,          R13,          p\        P                  ! V) V3RR7      # )z*Rotates half the hidden dims of the input..NrN   rP   )r™   r;   rç   )rê   Úx1Úx2s   &  r-   Úrotate_halfrõ   Ó   sZ   € à	
ˆ3Ð"�!—'‘'˜"•+ Õ"Ð"Ð"Õ	#€BØ	
ˆ3�—‘˜•˜qÕ Ñ"Ð"Õ	#€BÜ�9Š9�r�c˜2�Y BÔ'Ð'r0   Úrotary_pos_embc                óÈ   € VP                  V4      pVP                  V4      pW,          \        V 4      V,          ,           pW,          \        V4      V,          ,           pWV3# )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)Ú	unsqueezerõ   )ÚqÚkrè   ré   Úunsqueeze_dimÚq_embedÚk_embeds   &&&&&  r-   Úapply_rotary_pos_embrþ   Ú   sS   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�wœ; q›>¨CÕ/Õ0€GØ�wœ; q›>¨CÕ/Õ0€GØÐÐr0   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r)   rF   Ún_reprI   )r;   rK   Úint)r+   s   "r-   r.   r.   ô   s.   € ÷ 	Uñ 	UœUŸ\™\ð 	U´#ð 	U¼%¿,¹,ñ 	Ur0   c                ó˜   € V P                   w  r#rEV^8X  d   V # V R,          P                  W#WV4      p V P                  W#V,          WE4      # )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
)rá   rá   Nrá   rá   )r™   râ   r�   )rF   r   ÚbatchÚnum_key_value_headsÚslenrÒ   s   &&    r-   Ú	repeat_kvr  ô   sU   € ð
 2?×1DÑ1DÑ.€E Ø�„zØÐØ!Ð"2Õ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÕ(CÀTÓTÐTr0   c                ó   € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R\        R	\
        \        ,          /# )
r)   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNÚscalingÚdropoutr×   )r   ÚModuler;   rK   r„   r   r   )r+   s   "r-   r.   r.      sw   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ð%ô ð%ô Ô'Õ(ñ%r0   c                 ó"  € \        W P                  4      p\        W0P                  4      p	\        P                  ! WP	                  ^^4      4      V,          p
Ve	   W¤,           p
\
        P                  P                  V
R\        P                  R7      P                  VP                  4      p
\
        P                  P                  W¦V P                  R7      p
\        P                  ! W©4      pVP	                  ^^4      P                  4       pWº3# )r)   )rO   r_   )Úprš   rP   )r  Únum_key_value_groupsr;   Úmatmulræ   r   rT   rƒ   r®   r^   r_   r  rš   Ú
contiguous)r  r	  r
  r  r  r  r  r×   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs   &&&&&&&,    r-   Úeager_attention_forwardr     sÒ   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÕL€LØÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r0   c                   óX   a a€ ] tR tRt oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )	ÚMixtralAttentioni  z=Multi-headed attention from 'Attention Is All You Need' paperc                ó&   <€ V ^8„  d   QhRS[ RS[/# ©r)   r*   Ú	layer_idx©r$   r  )r+   r,   s   "€r-   r.   ÚMixtralAttention.__annotate__  s   ø€ ÷ lñ l™}ð l¹ñ lr0   c                óf  <€ \         SV `  4        Wn        W n        \	        VR R4      ;'       g    VP
                  VP                  ,          V n        VP                  VP                  ,          V n	        V P                  R,          V n
        VP                  V n        RV n        \        P                  ! VP
                  VP                  V P                  ,          RR7      V n        \        P                  ! VP
                  VP                  V P                  ,          RR7      V n        \        P                  ! VP
                  VP                  V P                  ,          RR7      V n        \        P                  ! VP                  V P                  ,          VP
                  RR7      V n        R# )rÒ   NTF©Úbiasg      à¿)r2   r3   r*   r  rÔ   r6   rÕ   rÒ   r  r  r  Úattention_dropoutÚ	is_causalr   ÚLinearÚq_projÚk_projÚv_projÚo_proj©rB   r*   r  rC   s   &&&€r-   r3   ÚMixtralAttention.__init__  s1  ø€ Ü‰ÑÔØŒØ"ŒÜ ¨
°DÓ9×mÐm¸V×=OÑ=OÐSY×SmÑSmÕ=mˆŒØ$*×$>Ñ$>À&×B\ÑB\Õ$\ˆÔ!Ø—}‘} dÕ*ˆŒØ!'×!9Ñ!9ˆÔØˆŒÜ—i’i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Õ4^ÐejÔkˆŒÜ—i’i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Õ4^ÐejÔkˆŒÜ—i’i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Õ4^ÐejÔkˆŒÜ—i’i × :Ñ :¸T¿]¹]Õ JÈF×L^ÑL^ÐejÔkˆŽr0   c                ó  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  S[ P                  R,          3,          /# )r)   rF   Úposition_embeddingsr  NÚpast_key_valuesr×   rI   )r;   rK   r¶   r   r   r   )r+   r,   s   "€r-   r.   r   +  s‡   ø€ ÷ ')ñ ')á—|‘|ð')ñ #¡5§<¡<±·±Ð#=Õ>ð')ñ Ÿ™ tÕ+ð	')ñ
  �ð')ñ Ñ-Õ.ð')ñ 
‰u�|‰|™UŸ\™\¨DÕ0Ð0Õ	1ñ')r0   c                ó>  € VP                   R R p. VORNV P                  N5pV P                  V4      P                  V4      P	                  ^^4      pV P                  V4      P                  V4      P	                  ^^4      p	V P                  V4      P                  V4      P	                  ^^4      p
Vw  r¼\        W‰W¼4      w  r‰Ve   VP                  WšV P                  4      w  rš\        P                  ! V P                  P                  \        4      pT! V VV	V
V3RV P                  '       g   RMV P                   RV P"                  R\%        V P                  RR 4      /VB w  rïVP&                  ! . VORN5!  P)                  4       pV P+                  V4      pWï3# )Nr  ç        r  Úsliding_windowrP   )r™   rÒ   r'  r�   ræ   r(  r)  rþ   Úupdater  r   Úget_interfacer*   Ú_attn_implementationr  rš   r$  r  rÔ   r�   r  r*  )rB   rF   r.  r  r/  r×   Úinput_shapeÚhidden_shapeÚquery_statesr  r  rè   ré   Úattention_interfacer  r  s   &&&&&,          r-   rj   ÚMixtralAttention.forward+  sŸ  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ
%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð
%
ð —L‘Lð
%
ô # 4§;¡;Ð0@À$ÓGð
%
ð ñ
%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—k‘k +Ó.ˆØÐ(Ð(r0   )r$  r*   rÒ   r%  r(  r  r  r*  r'  r  r)  ry   rl   ru   s   @@r-   r  r    s#   ù‡ € áG÷ló l÷')÷ ')ò ')r0   r  c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚMixtralDecoderLayeriU  c                ó&   <€ V ^8„  d   QhRS[ RS[/# r  r  )r+   r,   s   "€r-   r.   Ú MixtralDecoderLayer.__annotate__V  s   ø€ ÷ dñ d™}ð d¹ñ dr0   c                ó  <€ \         SV `  4        VP                  V n        \        W4      V n        \        V4      V n        \        VP                  VP                  R 7      V n	        \        VP                  VP                  R 7      V n
        R# )©r¦   N)r2   r3   r6   r  Ú	self_attnrŽ   Úmlpr¤   Úrms_norm_epsÚinput_layernormÚpost_attention_layernormr+  s   &&&€r-   r3   ÚMixtralDecoderLayer.__init__V  sj   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔä)¨&Ó<ˆŒä(¨Ó0ˆŒÜ-¨f×.@Ñ.@Àf×FYÑFYÔZˆÔÜ(6°v×7IÑ7IÈv×ObÑObÔ(cˆÖ%r0   c                ó  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[S[,          RS[ P                  /# )	r)   rF   r.  Nr  rë   r/  r×   rI   )r;   rK   r¶   Ú
LongTensorr   r   r   )r+   r,   s   "€r-   r.   r>  `  sŽ   ø€ ÷ ñ á—|‘|ðñ #¡5§<¡<±·±Ð#=Õ>ÀÕEðñ Ÿ™ tÕ+ð	ñ
 ×&Ñ&¨Õ-ðñ  �ðñ Ñ+Õ,ðñ 
�‰ñr0   c                óÐ   € TpV P                  V4      pV P                  ! RR VRVRVRVRV/VB w  rWq,           pTpV P                  V4      pV P                  V4      pWq,           pV# )rF   r.  r  rë   r/  © )rD  rA  rE  rB  )	rB   rF   r.  r  rë   r/  r×   Úresidualr    s	   &&&&&&,  r-   rj   ÚMixtralDecoderLayer.forward`  s�   € ð !ˆØ×,Ñ,¨]Ó;ˆØŸ>š>ñ 
Ø'ð
à 3ð
ð *ð
ð &ð	
ð
 ,ð
ð ñ
Ñˆð !Õ0ˆØ ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ Õ0ˆØÐr0   )r6   rD  rB  rE  rA  )NNNNrŒ   ru   s   @@r-   r<  r<  U  s    ù‡ € ÷dó d÷÷ ò r0   r<  c                   ó®   a a€ ] tR tRt oRtRtR.tR.tRtRt	Rt
RtRtR]! ]^ R7      R]R	]/t]P&                  ! 4       V 3R
 l4       tV3R ltRtVtV ;t# )ÚMixtralPreTrainedModeli{  ÚmodelTr<  r/  r†   )ÚindexrF   Ú
attentionsc                ó„  <€ \         SV `  V4       V P                  P                  p\	        V\
        4      '       dI   \        P                  ! VP                  R VR7       \        P                  ! VP                  R VR7       R# \	        V\        4      '       d&   \        P                  ! VP                  R VR7       R# R# )r1  )r°   ÚstdN)r2   Ú_init_weightsr*   Úinitializer_rangerã   r&   ÚinitÚnormal_r=   r>   rw   r|   )rB   r  rS  rC   s   && €r-   rT  Ú$MixtralPreTrainedModel._init_weightsŽ  s   ø€ ä‰Ñ˜fÔ%Ø�k‰k×+Ñ+ˆÜ�fœn×-Ò-Ü�LŠL˜×,Ñ,°3¸CÕ@Ü�LŠL˜×)Ñ)°¸×=Ü˜Ô 1×2Ò2Ü�LŠL˜Ÿ™¨S°c×:ñ 3r0   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r(   r#   )r+   r,   s   "€r-   r.   Ú#MixtralPreTrainedModel.__annotate__{  s   ø‡ ‚ áÑò r0   rJ  )rm   rn   ro   rp   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendr!   rw   r<  r  Ú_can_record_outputsr;   rS   rT  Ú__annotate_func__rr   rs   rt   ru   s   @@r-   rN  rN  {  s„   ù‡ € ð  ÐØ&*Ð#Ø.Ð/ÐØ#4Ð"5ÐØÐØ€NØÐà!ÐØ"&Ðà™Ð(9ÀÔCØÐ,ØÐ&ðÐð ‡]‚]ƒ_ô;ó ð;÷) … r0   rN  c                   ór   a a€ ] tR tRt oV3R lV 3R llt]]]RV3R lR ll4       4       4       tRt	Vt
V ;t# )ÚMixtralModeli™  c                ó    <€ V ^8„  d   QhRS[ /# r(   r#   )r+   r,   s   "€r-   r.   ÚMixtralModel.__annotate__›  s   ø€ ÷ ñ ™}ñ r0   c           	     ó  <€ \         SV `  V4       VP                  V n        VP                  V n        \
        P                  ! VP                  VP                  V P                  4      V n        \
        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n        \        VP                  VP                  R 7      V n        \#        VR7      V n        RV n        V P)                  4        R# u upi )r@  ©r*   FN)r2   r3   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingr6   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersr<  Úlayersr¤   rC  Únormrº   Ú
rotary_embÚgradient_checkpointingÚ	post_initr+  s   && €r-   r3   ÚMixtralModel.__init__›  sÊ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸLšL¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m’mÜEJÈ6×KcÑKcÔEdÓeÑEd¸	Ô  Ö3ÑEdÑeó
ˆŒô # 6×#5Ñ#5¸6×;NÑ;NÔOˆŒ	Ü0¸Ô?ˆŒØ&+ˆÔ#ð 	�‰Öùò fs   ÂDc                óþ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[S[,          R	S[/# )
r)   Ú	input_idsNr  rë   r/  Úinputs_embedsÚ	use_cacher×   rI   )	r;   rH  rK   r   ÚFloatTensorÚboolr   r   r   )r+   r,   s   "€r-   r.   ri  ®  s“   ø€ ÷ 4
ñ 4
á×#Ñ# dÕ*ð4
ñ Ÿ™ tÕ+ð4
ñ ×&Ñ&¨Õ-ð	4
ñ
  �ð4
ñ ×(Ñ(¨4Õ/ð4
ñ ˜$•;ð4
ñ Ñ+Õ,ð4
ñ 
 ñ4
r0   c                óÊ  € VR J VR J,          '       d   \        R4      hV'       d   Vf   \        V P                  R7      pVf   V P                  V4      pVfb   Ve   VP	                  4       M^ p\
        P                  ! VP                  ^,          VP                  R7      V,           pVP                  ^ 4      pV P                  P                  f   \        M\        p	V	! V P                  VVVVR7      p
TpV P                  W³R7      pV P                  R V P                  P                    F  pV! V3RV
RVRVR	VR
V/VB pK  	  V P!                  V4      p\#        VVR7      # )Nz:You must specify exactly one of input_ids or inputs_embedsrk  )r»   )r*   r|  r  r/  rë   )rë   r  rë   r/  r}  r.  )Úlast_hidden_stater/  )Ú
ValueErrorr   r*   rp  Úget_seq_lengthr;   rÖ   r™   r»   rø   r2  r   r   rv  rt  rs  ru  r   )rB   r{  r  rë   r/  r|  r}  r×   Úpast_seen_tokensÚmask_functionÚcausal_maskrF   r.  Údecoder_layers   &&&&&&&,      r-   rj   ÚMixtralModel.forward«  s}  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZç˜Ò0Ü*°$·+±+Ô>ˆOàÒ Ø ×-Ñ-¨iÓ8ˆMàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLà.2¯k©k×.HÑ.HÒ.PÕ*ÔVwˆÙ#Ø—;‘;Ø'Ø)Ø+Ø%ô
ˆð &ˆØ"Ÿo™o¨m˜oÓWÐà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÓIˆMÙ)Øñà*ðð *ðð !0ð	ð
 $ðð %8ðð ñŠMñ Jð Ÿ	™	 -Ó0ˆä%Ø+Ø+ô
ð 	
r0   )rp  rw  rt  ru  rm  rv  rn  )NNNNNN)rm   rn   ro   rp   r3   r    r"   r   rj   rr   rs   rt   ru   s   @@r-   rg  rg  ™  s=   ù‡ € ÷ó ð   ØØ÷4
ñ 4
ó ó ó  ÷4
ð 4
r0   rg  c          	      ó  € V ^8„  d   QhR\         P                  \        \         P                  ,          ,          R,          R\        R,          R\         P                  R,          R\         P                  \        ,          /# )r)   Úgate_logitsNr5   r  rI   )r;   rK   r¶   r  )r+   s   "r-   r.   r.   å  sc   € ÷ B&ñ B&Ü—‘¤¤e§l¡lÕ 3Õ3°dÕ:ðB&ä�t•ðB&ô —L‘L 4Õ'ð	B&ô
 ‡\�\”CÕñB&r0   c           	     ó  € V e   \        V \        4      '       g   ^ # V ^ ,          P                  p\        P                  ! V\        P
                  VR7      p\        P                  ! V\        P
                  VR7      pRpVe1   VP                  R4      P                  V\        P
                  R7      pV  EFˆ  p	\        P                  P                  P                  V	P                  V4      RR7      p
\        P                  ! W¢RR7      w  r¼Vf~   V\        P                  ! VP                  R4      VR7      P                  4       ,           pWjP                  4       P                  ^ R7      ,           pWzP                  ^ ,          ,           pKÙ  V\        P                  ! V\        P
                  VR7      P!                  ^ VP                  R4      XP#                  V4      4      ,           pWjP                  4       VP%                  R4      ,          P                  ^ R7      ,           pWxP                  4       ,           pEK‹  	  WW,          pWg,          p\        P                  ! WÞP%                  ^ 4      ,          4      pWñ,          # )ax  
Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch.

See Switch Transformer (https://huggingface.co/papers/2101.03961) for more details. This function implements the loss
function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between
experts is too unbalanced.

Args:
    gate_logits:
        Logits from the `gate`, should be a tuple of model.config.num_hidden_layers tensors of
        shape [batch_size X sequence_length, num_experts].
    num_experts:
        Number of experts
    top_k:
        The number of experts to route per-token, can be also interpreted as the `top-k` routing
        parameter.
    attention_mask (`torch.Tensor`, *optional*):
        The attention_mask used in forward function
        shape [batch_size X sequence_length] if not None.

Returns:
    The auxiliary loss.
rÜ   r1  )r»   r_   rN   )Ú	minlengthrP   )rã   r¶   r»   r;   Úzerosr®   r�   r^   r   rT   rƒ   r…   Úbincountr„   rX   r™   Úscatter_add_Úrepeat_interleaverø   )rŠ  r5   r{   r  Úcompute_deviceÚtokens_per_expert_sumÚrouter_prob_sumÚ
total_rowsÚ	flat_maskÚ
layer_gateÚrouting_weightsr    Úselected_expertsÚtokens_per_expertÚrouter_prob_per_expertÚoverall_losss   &&&&            r-   Úload_balancing_loss_funcrœ  å  sÿ  € ð: Ò¤*¨[¼%×"@Ò"@Ùð ! •^×*Ñ*€NÜ!ŸKšK¨¼5¿=¹=ÐQ_Ô`ÐÜ—k’k +´U·]±]È>ÔZ€OØ€JàÒ!à"×*Ñ*¨2Ó.×1Ñ1¸ÌuÏ}É}Ð1Ó]ˆ	ä!ˆ
ÜŸ(™(×-Ñ-×5Ñ5°j·m±mÀNÓ6SÐY[Ð5Ó\ˆÜ#Ÿjšj¨ÀRÔHÑˆØÒ!ð &¬¯ªÐ7G×7OÑ7OÐPRÓ7SÐ_jÔ(k×(qÑ(qÓ(sÕsð "ð .×0EÑ0EÓ0G×0KÑ0KÐPQÐ0KÓ0RÕRˆOØ#×&;Ñ&;¸AÕ&>Õ>ŠJð %:¼E¿KºKØ¤5§=¡=¸ô=ç‰l˜1Ð.×6Ñ6°rÓ:¸I×<WÑ<WÐX]Ó<^Ó_õ%`Ð!ð .×1FÑ1FÓ1HÈ9×K^ÑK^Ð_aÓKbÕ1b×0gÑ0gÐlmÐ0gÓ0nÕnˆOØ#§m¡m£oÕ5‹Jñ# "ð& .Õ:ÐØ,Õ9Ðä—9’9Ð.×1QÑ1QÐRSÓ1TÕTÓU€LØÕ%Ð%r0   c                   ó„   a a€ ] tR tRt oRR/tRR/tRR.R.3/tRR/tV 3R	 lt]	]
RV3R
 lR ll4       4       tRtVtV ;t# )ÚMixtralForCausalLMi*  zlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputrF   ÚlogitsÚkeep_full_weightc                óR  <€ \         SV `  V4       \        V4      V n        VP                  V n        \
        P                  ! VP                  VP                  R R7      V n        VP                  V n	        VP                  V n        VP                  V n        V P                  4        R# )Fr"  N)r2   r3   rg  rO  rn  r   r&  r6   rŸ  Úrouter_aux_loss_coefr4   r5   rz   rx  rA   s   &&€r-   r3   ÚMixtralForCausalLM.__init__1  s   ø€ Ü‰Ñ˜Ô Ü! &Ó)ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒØ$*×$?Ñ$?ˆÔ!Ø!×3Ñ3ˆÔØ#)×#=Ñ#=ˆÔ ð 	�‰Ör0   c                ód  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[R,          R
S[S[ P                  ,          RS[S[,          RS[	/# )r)   r{  Nr  rë   r/  r|  Úlabelsr}  Úoutput_router_logitsÚlogits_to_keepr×   rI   )
r;   rH  rK   r   r~  r  r  r   r   r   )r+   r,   s   "€r-   r.   ÚMixtralForCausalLM.__annotate__?  sÖ   ø€ ÷ P
ñ P
á×#Ñ# dÕ*ðP
ñ Ÿ™ tÕ+ðP
ñ ×&Ñ&¨Õ-ð	P
ñ
  �ðP
ñ ×(Ñ(¨4Õ/ðP
ñ × Ñ  4Õ'ðP
ñ ˜$•;ðP
ñ # T�kðP
ñ ™eŸl™lÕ*ðP
ñ Ñ+Õ,ðP
ñ 
#ñP
r0   c
                ó¦  € Ve   TMV P                   P                  pV P                  ! RRVRVRVRVRVRVRV/V
B pVP                  p\	        V	\
        4      '       d   \        V	) R4      MT	pV P                  VR	VR	3,          4      pRpVe   V P                  ! WæV P                  3/ V
B pRpV'       dc   \        VP                  V P                  V P                  V4      pVe3   WðP                  VP                  VP                   4      ,          ,          p\#        VVVVP$                  VP&                  VP(                  VP                  R
7      # )aþ  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

Example:

```python
>>> from transformers import AutoTokenizer, MixtralForCausalLM

>>> model = MixtralForCausalLM.from_pretrained("mistralai/Mixtral-8x7B-v0.1")
>>> tokenizer = AutoTokenizer.from_pretrained("mistralai/Mixtral-8x7B-v0.1")

>>> prompt = "Hey, are you conscious? Can you talk to me?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
```Nr{  r  rë   r/  r|  r}  r¨  rá   )ÚlossÚaux_lossr¡  r/  rF   rQ  r†   rJ  )r*   r¨  rO  r�  rã   r  ÚslicerŸ  Úloss_functionrn  rœ  r†   r5   rz   r¤  r^   r»   r   r/  rF   rQ  )rB   r{  r  rë   r/  r|  r§  r}  r¨  r©  r×   ÚoutputsrF   Úslice_indicesr¡  r¬  r­  s   &&&&&&&&&&,      r-   rj   ÚMixtralForCausalLM.forward=  sx  € ðN %9Ò$DÑ È$Ï+É+×JjÑJjð 	ð
 +/¯*ª*ñ 	+
Øð	+
à)ð	+
ð &ð	+
ð ,ð		+
ð
 (ð	+
ð  ð	+
ð "6ð	+
ð ñ	+
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒØ×%Ò% f°d·o±oÑPÈÑPˆDàˆßÜ/Ø×%Ñ%Ø× Ñ Ø×(Ñ(Øó	ˆHð Ò!Ø×1Ñ1°H·K±KÀÇÁÓ4LÕLÕL�ä(ØØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø!×/Ñ/ô
ð 	
r0   )rŸ  rO  r5   rz   r¤  rn  )	NNNNNNNNr   )rm   rn   ro   rp   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr3   r   r   rj   rr   rs   rt   ru   s   @@r-   rž  rž  *  sj   ù‡ € à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÐ/Ð0€Jõ
ð Ø÷P
ñ P
ó ó ÷P
ð P
r0   rž  c                   ó   € ] tR tRtRtR# )Ú MixtralForSequenceClassificationi’  rJ  N©rm   rn   ro   rp   rr   rJ  r0   r-   r¸  r¸  ’  ó   † Ûr0   r¸  c                   ó   € ] tR tRtRtR# )ÚMixtralForTokenClassificationi–  rJ  Nr¹  rJ  r0   r-   r¼  r¼  –  rº  r0   r¼  c                   ó   € ] tR tRtRtR# )ÚMixtralForQuestionAnsweringiš  rJ  Nr¹  rJ  r0   r-   r¾  r¾  š  rº  r0   r¾  )r   )r1  )Nr)   N)rž  r¾  rg  rN  r¸  r¼  )LÚcollections.abcr   r;   Útorch.nn.functionalr   rT   r‚   Ú r   rV  Úactivationsr   Úcache_utilsr   r   Ú
generationr	   Úintegrationsr
   r   r   Úmasking_utilsr   r   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   r   r   r   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.deprecationr   Úutils.genericr   r    Úutils.output_capturingr!   r"   Úconfiguration_mixtralr$   r  r&   rw   rŽ   r¤   rº   rõ   rþ   r  r  r  r<  rN  rg  rœ  rž  r¸  r¼  r¾  Ú__all__rJ  r0   r-   Ú<module>rÓ     sÊ  ðõ4 %ã ß Ð Ý å &Ý !ß .Ý )ß hÑ hß RÝ B÷ó ÷ Rß Kß FÝ &ß IÑ IÝ 0ß Gß EÝ 0ð ô$#�R—Y‘Yó $#ó ð$#ôN<˜Ÿ	™	ô <ô$˜BŸI™Iô ñ& ˜YÓ'ôJ�R—Y‘Yó Jó (ðJô(6<˜RŸY™Yô 6<òr(ñ Ð-Ó.óó /ðõ2	U÷%ñ2 Ð)Ó*ô8)�r—y‘yó 8)ó +ð8)ôv#Ð4ô #ðL ô;˜_ó ;ó ð;ð: ôH
Ð)ó H
ó ðH
÷VB&ðJ ôd
Ð/°ó d
ó ðd
ôN	Ð'GÐI_ô 	ô	Ð$AÐCYô 	ô	Ð"=Ð?Uô 	ò‚r0   