+
    œ°³j°Q  ã                   óô  € ^ RI Ht ^ RIt^ RIHt ^RIHt ^RIHtHt ^RI	H
t
 ^RIHtHt ^RIHtHt ^R	IHt ^R
IHtHtHtHt ^RIHtHt ^RIHtHt ^RIHtH t  ^RI!H"t" ^RI#H$t$H%t%H&t& ^RI'H(t( ^RI)H*t*H+t+ ^RI,H-t- ^RI.H/t/  ! R R]P`                  4      t1R t2]! R4      R2R l4       t3R R lt4R3R R llt5]! ]34       ! R R]P`                  4      4       t6]! R4       ! R  R!]P`                  4      4       t7 ! R" R#]4      t8]% ! R$ R%] 4      4       t9 ! R& R']P`                  4      t:]% ! R( R)]94      4       t;]% ! R* R+]9]
4      4       t< ! R, R-]]94      t= ! R. R/]]94      t> ! R0 R1]]94      t?. R4Ot@R# )5é    )ÚCallableN)Únn)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_kernel_forward_from_hubÚuse_kernelized_func)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚFlashAttentionKwargs)ÚGenericForQuestionAnsweringÚ GenericForSequenceClassificationÚGenericForTokenClassificationÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputs©ÚMistralConfigc                   ó8   a a€ ] tR t^#t oV 3R ltR tRtVtV ;t# )Ú
MistralMLPc                óØ  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        \
        P                  ! V P                  V P                  R R7      V n        \
        P                  ! V P                  V P                  R R7      V n        \
        P                  ! V P                  V P                  R R7      V n	        \        VP                  ,          V n        R# ©F©ÚbiasN)ÚsuperÚ__init__ÚconfigÚhidden_sizeÚintermediate_sizer   ÚLinearÚ	gate_projÚup_projÚ	down_projr   Ú
hidden_actÚact_fn©Úselfr*   Ú	__class__s   &&€Úm/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/mistral/modeling_mistral.pyr)   ÚMistralMLP.__init__$   s¤   ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔÜŸš 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆŒÜ˜V×.Ñ.Õ/ˆŽó    c                ó�   € V P                  V P                  V P                  V4      4      V P                  V4      ,          4      pV# ©N)r0   r2   r.   r/   )r4   Úxr0   s   && r6   ÚforwardÚMistralMLP.forward.   s6   € Ø—N‘N 4§;¡;¨t¯~©~¸aÓ/@Ó#AÀDÇLÁLÐQRÃOÕ#SÓTˆ	ØÐr8   )r2   r*   r0   r.   r+   r,   r/   ©	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r)   r<   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r5   Ú__classdict__s   @@r6   r#   r#   #   s   ù‡ € õ0÷ò r8   r#   c                óÌ   € V RRV P                   R,          ^,          13,          pV RV P                   R,          ^,          R13,          p\        P                  ! V) V3RR7      # )z*Rotates half the hidden dims of the input..N©Údiméÿÿÿÿ)ÚshapeÚtorchÚcat)r;   Úx1Úx2s   &  r6   Úrotate_halfrQ   3   sZ   € à	
ˆ3Ð"�!—'‘'˜"•+ Õ"Ð"Ð"Õ	#€BØ	
ˆ3�—‘˜•˜qÕ Ñ"Ð"Õ	#€BÜ�9Š9�r�c˜2�Y BÔ'Ð'r8   Úrotary_pos_embc                óÈ   € VP                  V4      pVP                  V4      pW,          \        V 4      V,          ,           pW,          \        V4      V,          ,           pWV3# )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)Ú	unsqueezerQ   )ÚqÚkÚcosÚsinÚunsqueeze_dimÚq_embedÚk_embeds   &&&&&  r6   Úapply_rotary_pos_embr\   :   sS   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�wœ; q›>¨CÕ/Õ0€GØ�wœ; q›>¨CÕ/Õ0€GØÐÐr8   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )é   Úhidden_statesÚn_repÚreturn)rM   ÚTensorÚint)Úformats   "r6   Ú__annotate__re   T   s.   € ÷ 	Uñ 	UœUŸ\™\ð 	U´#ð 	U¼%¿,¹,ñ 	Ur8   c                ó˜   € V P                   w  r#rEV^8X  d   V # V R,          P                  W#WV4      p V P                  W#V,          WE4      # )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
)ºNNNrg   Nrg   rg   )rL   ÚexpandÚreshape)r_   r`   ÚbatchÚnum_key_value_headsÚslenÚhead_dims   &&    r6   Ú	repeat_kvrn   T   sU   € ð
 2?×1DÑ1DÑ.€E Ø�„zØÐØ!Ð"2Õ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÕ(CÀTÓTÐTr8   c                ó   € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R\        R	\
        \        ,          /# )
r^   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNÚscalingÚdropoutÚkwargs)r   ÚModulerM   rb   Úfloatr   r   )rd   s   "r6   re   re   `   sw   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ð%ô ð%ô Ô'Õ(ñ%r8   c                 ó"  € \        W P                  4      p\        W0P                  4      p	\        P                  ! WP	                  ^^4      4      V,          p
Ve	   W¤,           p
\
        P                  P                  V
R\        P                  R7      P                  VP                  4      p
\
        P                  P                  W¦V P                  R7      p
\        P                  ! W©4      pVP	                  ^^4      P                  4       pWº3# )r^   )rJ   Údtype)ÚpÚtrainingrK   )rn   Únum_key_value_groupsrM   ÚmatmulÚ	transposer   Ú
functionalÚsoftmaxÚfloat32Útor{   rv   r}   Ú
contiguous)rp   rq   rr   rs   rt   ru   rv   rw   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs   &&&&&&&,    r6   Úeager_attention_forwardrŠ   `   sÒ   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÕL€LØÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r8   c                   óX   a a€ ] tR t^yt oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )ÚMistralAttentionz=Multi-headed attention from 'Attention Is All You Need' paperc                ó&   <€ V ^8„  d   QhRS[ RS[/# ©r^   r*   Ú	layer_idx©r!   rc   )rd   rG   s   "€r6   re   ÚMistralAttention.__annotate__}   s   ø€ ÷ lñ l™}ð l¹ñ lr8   c                óf  <€ \         SV `  4        Wn        W n        \	        VR R4      ;'       g    VP
                  VP                  ,          V n        VP                  VP                  ,          V n	        V P                  R,          V n
        VP                  V n        RV n        \        P                  ! VP
                  VP                  V P                  ,          RR7      V n        \        P                  ! VP
                  VP                  V P                  ,          RR7      V n        \        P                  ! VP
                  VP                  V P                  ,          RR7      V n        \        P                  ! VP                  V P                  ,          VP
                  RR7      V n        R# )rm   NTFr&   g      à¿)r(   r)   r*   r�   Úgetattrr+   Únum_attention_headsrm   rk   r~   ru   Úattention_dropoutÚ	is_causalr   r-   Úq_projÚk_projÚv_projÚo_proj©r4   r*   r�   r5   s   &&&€r6   r)   ÚMistralAttention.__init__}   s1  ø€ Ü‰ÑÔØŒØ"ŒÜ ¨
°DÓ9×mÐm¸V×=OÑ=OÐSY×SmÑSmÕ=mˆŒØ$*×$>Ñ$>À&×B\ÑB\Õ$\ˆÔ!Ø—}‘} dÕ*ˆŒØ!'×!9Ñ!9ˆÔØˆŒÜ—i’i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Õ4^ÐejÔkˆŒÜ—i’i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Õ4^ÐejÔkˆŒÜ—i’i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Õ4^ÐejÔkˆŒÜ—i’i × :Ñ :¸T¿]¹]Õ JÈF×L^ÑL^ÐejÔkˆŽr8   c                ó  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  S[ P                  R,          3,          /# )r^   r_   Úposition_embeddingsrt   NÚpast_key_valuesrw   ra   )rM   rb   Útupler   r   r   )rd   rG   s   "€r6   re   r‘   ‹   s‡   ø€ ÷ ')ñ ')á—|‘|ð')ñ #¡5§<¡<±·±Ð#=Õ>ð')ñ Ÿ™ tÕ+ð	')ñ
  �ð')ñ Ñ-Õ.ð')ñ 
‰u�|‰|™UŸ\™\¨DÕ0Ð0Õ	1ñ')r8   c                ó>  € VP                   R R p. VORNV P                  N5pV P                  V4      P                  V4      P	                  ^^4      pV P                  V4      P                  V4      P	                  ^^4      p	V P                  V4      P                  V4      P	                  ^^4      p
Vw  r¼\        W‰W¼4      w  r‰Ve   VP                  WšV P                  4      w  rš\        P                  ! V P                  P                  \        4      pT! V VV	V
V3RV P                  '       g   RMV P                   RV P"                  R\%        V P                  RR 4      /VB w  rïVP&                  ! . VORN5!  P)                  4       pV P+                  V4      pWï3# )Nrv   ç        ru   Úsliding_windowrK   )rL   rm   r—   Úviewr€   r˜   r™   r\   Úupdater�   r   Úget_interfacer*   Ú_attn_implementationrŠ   r}   r•   ru   r“   ri   r…   rš   )r4   r_   rž   rt   rŸ   rw   Úinput_shapeÚhidden_shapeÚquery_statesr†   r‡   rW   rX   Úattention_interfacer‰   rˆ   s   &&&&&,          r6   r<   ÚMistralAttention.forward‹   sŸ  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ
%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð
%
ð —L‘Lð
%
ô # 4§;¡;Ð0@À$ÓGð
%
ð ñ
%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—k‘k +Ó.ˆØÐ(Ð(r8   )r•   r*   rm   r–   r˜   r�   r~   rš   r—   ru   r™   r:   )
r?   r@   rA   rB   Ú__doc__r)   r<   rC   rD   rE   rF   s   @@r6   rŒ   rŒ   y   s#   ù‡ € áG÷ló l÷')÷ ')ò ')r8   rŒ   ÚRMSNormc                   óZ   a a€ ] tR t^µt oRV3R lV 3R llltV3R lR ltR tRtVtV ;t	# )ÚMistralRMSNormc                ó$   <€ V ^8„  d   QhRS[ RR/# )r^   Úepsra   N)ry   )rd   rG   s   "€r6   re   ÚMistralRMSNorm.__annotate__·   s   ø€ ÷ $ñ $©ð $¸$ñ $r8   c                óŽ   <€ \         SV `  4        \        P                  ! \        P
                  ! V4      4      V n        W n        R# )z-
MistralRMSNorm is equivalent to T5LayerNorm
N)r(   r)   r   Ú	ParameterrM   ÚonesÚweightÚvariance_epsilon)r4   r+   r²   r5   s   &&&€r6   r)   ÚMistralRMSNorm.__init__·   s/   ø€ ô 	‰ÑÔÜ—l’l¤5§:¢:¨kÓ#:Ó;ˆŒØ #Ör8   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r^   r_   ra   )rM   rb   )rd   rG   s   "€r6   re   r³   ¿   s#   ø€ ÷ ;ñ ;¡U§\¡\ð ;±e·l±lñ ;r8   c                ó:  € VP                   pVP                  \        P                  4      pVP	                  ^4      P                  RRR7      pV\        P                  ! W0P                  ,           4      ,          pV P                  VP                  V4      ,          # )r^   T)ÚkeepdimrK   )	r{   r„   rM   rƒ   ÚpowÚmeanÚrsqrtr¸   r·   )r4   r_   Úinput_dtypeÚvariances   &&  r6   r<   ÚMistralRMSNorm.forward¿   sw   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯ª°H×?TÑ?TÕ4TÓ(UÕUˆØ�{‰{˜]×-Ñ-¨kÓ:Õ:Ð:r8   c                ó^   € \        V P                  P                  4       R V P                   2# )z, eps=)r    r·   rL   r¸   )r4   s   &r6   Ú
extra_reprÚMistralRMSNorm.extra_reprÆ   s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr8   )r¸   r·   )g�íµ ÷Æ°>)
r?   r@   rA   rB   r)   r<   rÄ   rC   rD   rE   rF   s   @@r6   r°   r°   µ   s%   ù‡ € ÷$õ $÷;ð ;÷Jò Jr8   r°   c                   óT   a a€ ] tR t^Êt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚMistralDecoderLayerc                ó&   <€ V ^8„  d   QhRS[ RS[/# rŽ   r�   )rd   rG   s   "€r6   re   Ú MistralDecoderLayer.__annotate__Ë   s   ø€ ÷ dñ d™}ð d¹ñ dr8   c                ó   <€ \         SV `  4        VP                  V n        \        WR 7      V n        \        V4      V n        \        VP                  VP                  R7      V n	        \        VP                  VP                  R7      V n
        R# ))r*   r�   ©r²   N)r(   r)   r+   rŒ   Ú	self_attnr#   Úmlpr°   Úrms_norm_epsÚinput_layernormÚpost_attention_layernormr›   s   &&&€r6   r)   ÚMistralDecoderLayer.__init__Ë   sj   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔÜ)°ÔMˆŒÜ˜fÓ%ˆŒÜ-¨f×.@Ñ.@Àf×FYÑFYÔZˆÔÜ(6°v×7IÑ7IÈv×ObÑObÔ(cˆÖ%r8   c                ó.  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[ P                  S[ P                  3,          R,          RS[S[,          R	S[ P                  /# )
r^   r_   rt   NÚposition_idsrŸ   Ú	use_cacherž   rw   ra   )rM   rb   Ú
LongTensorr   Úboolr    r   r   )rd   rG   s   "€r6   re   rÉ   Ó   sœ   ø€ ÷ ñ á—|‘|ðñ Ÿ™ tÕ+ðñ ×&Ñ&¨Õ-ð	ñ
  �ðñ ˜$•;ðñ #¡5§<¡<±·±Ð#=Õ>ÀÕEðñ Ñ+Õ,ðñ 
�‰ñr8   c                óÔ   € TpV P                  V4      pV P                  ! RR VRVRVRVRVRV/VB w  rW�,           pTpV P                  V4      pV P                  V4      pW�,           pV# )r_   rt   rÓ   rŸ   rÔ   rž   © )rÏ   rÌ   rÐ   rÍ   )
r4   r_   rt   rÓ   rŸ   rÔ   rž   rw   ÚresidualÚ_s
   &&&&&&&,  r6   r<   ÚMistralDecoderLayer.forwardÓ   s©   € ð !ˆØ×,Ñ,¨]Ó;ˆàŸ>š>ñ 
Ø'ð
à)ð
ð &ð
ð ,ð	
ð
  ð
ð !4ð
ð ñ
Ñˆð !Õ0ˆð !ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ Õ0ˆØÐr8   )r+   rÏ   rÍ   rÐ   rÌ   )NNNFNr>   rF   s   @@r6   rÇ   rÇ   Ê   s    ù‡ € ÷dó d÷÷ ò r8   rÇ   c                   ó`   a € ] tR t^ót o RtRtR.tR.tRtRt	Rt
RtRtR]R]/tV 3R ltRtV tR	# )
ÚMistralPreTrainedModelÚmodelTrÇ   rŸ   r_   Ú
attentionsc                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # ©r^   r*   r    )rd   rG   s   "€r6   re   Ú#MistralPreTrainedModel.__annotate__ó   s   ø‡ ‚ áÑò r8   rØ   N)r?   r@   rA   rB   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendrÇ   rŒ   Ú_can_record_outputsÚ__annotate_func__rC   rD   )rG   s   @r6   rÝ   rÝ   ó   s]   ø‡ € ð  ÐØ&*Ð#Ø.Ð/ÐØ#4Ð"5ÐØÐØ€NØÐà!ÐØ"&ÐàÐ,ØÐ&ðÐ÷ ƒ r8   rÝ   c                   óÎ   a a€ ] tR tRt o]! RRR7      RV3R lV 3R lll4       t]]! RRR7      RV3R lR ll4       4       t]P                  ! 4       ]
R	 4       4       tR
tVtV ;t# )ÚMistralRotaryEmbeddingi  Údevicez5.18)Úversionc                ó    <€ V ^8„  d   QhRS[ /# rá   r    )rd   rG   s   "€r6   re   Ú#MistralRotaryEmbedding.__annotate__  s   ø€ ÷ Oñ O™}ñ Or8   c                óä  <€ \         SV `  4        VP                  V n        VP                  V n        Wn        V P
                  P                  R ,          V n        V P                  pV P                  R8w  d   \        V P                  ,          pV! V P
                  V4      w  q@n
        \        P                  ! VRR7      V n        \        P                  ! VP                  4       RR7      V n        R# )Ú	rope_typeÚdefaultF)Ú
persistentN)r(   r)   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr*   Úrope_parametersrõ   Úcompute_default_rope_parametersr   Úattention_scalingr   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)r4   r*   rð   Úrope_init_fnrÿ   r5   s   &&&  €r6   r)   ÚMistralRotaryEmbedding.__init__  s¬   ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àŒàŸ™×4Ñ4°[ÕAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÔ&Ü.¨t¯~©~Õ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÔ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÔ!NˆÖr8   c                óP   <€ V ^8„  d   QhRS[ RS[S[P                  S[3,          /# )r^   r*   ra   )r!   r    rM   rb   ry   )rd   rG   s   "€r6   re   ró     s.   ø€ ÷ 5ñ 5±ð 5ÑY^Ñ_d×_kÑ_kÑmrÐ_rÕYsñ 5r8   c           	     ó0  € V P                   R,          p\        V RR4      ;'       g    V P                  V P                  ,          pRpRV\        P
                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetarm   Ng      ð?©r{   )rû   r“   r+   r”   rM   Úarangery   r„   )r*   rð   rw   ÚbaserJ   Úattention_factorrÿ   s   &&,    r6   rü   Ú6MistralRotaryEmbedding.compute_default_rope_parameters  s~   € ð ×%Ñ% lÕ3ˆÜ�f˜j¨$Ó/×cÐc°6×3EÑ3EÈ×IcÑIcÕ3cˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r8   c                ó^  € V P                   R,          P                  VP                  ^ ,          R	^4      P                  \        P
                  VP                  R7      pVR
,          P                  4       p\        VP                  P                  \        4      '       d2   VP                  P                  R8w  d   VP                  P                  MRp\        VRR7      ;_uu_ 4        W4,          P                  ^^4      p\        P                  ! Wf3R	R7      pVP                  4       V P                  ,          pVP                  4       V P                  ,          p	R R R 4       XP                  VP                   R7      X	P                  VP                   R7      3#   + '       g   i     LH; i)N)r{   rð   ÚmpsÚcpuF)Údevice_typeÚenabledrI   r  )Nrg   NrK   )rg   Nrg   )rÿ   rh   rL   r„   rM   ry   rð   Ú
isinstanceÚtypeÚstrr   r€   rN   rW   rý   rX   r{   )
r4   r;   rÓ   Úinv_freq_expandedÚposition_ids_expandedr  ÚfreqsÚembrW   rX   s
   &&&       r6   r<   ÚMistralRotaryEmbedding.forward,  s2  € ð �M‰M˜-Õ(×/Ñ/°×0BÑ0BÀ1Õ0EÀrÈ1ÓM×PÑPÔW\×WbÑWbÐkl×ksÑksÐPÓtð 	ð !-¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆä¨¸U×CÖCØ&Õ>×IÑIÈ!ÈQÓOˆEÜ—)’)˜U˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆC÷	 Dð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷ D×Cús   Ã(A4FÆF,	)rý   r*   rÿ   rù   r  rú   rõ   r:   )r?   r@   rA   rB   r   r)   Ústaticmethodrü   rM   Úno_gradr   r<   rC   rD   rE   rF   s   @@r6   rï   rï     su   ù‡ € Ù�X vÔ.÷Oô Oó /ðOð  Ù�X vÔ.÷5ñ 5ó /ó ð5ð$ ‡]‚]ƒ_Øñ<ó ó ÷<ð <r8   rï   c                   ór   a a€ ] tR tRt oV3R lV 3R llt]]]RV3R lR ll4       4       4       tRt	Vt
V ;t# )ÚMistralModeli?  c                ó    <€ V ^8„  d   QhRS[ /# rá   r    )rd   rG   s   "€r6   re   ÚMistralModel.__annotate__A  s   ø€ ÷ ñ ™}ñ r8   c           	     ó  <€ \         SV `  V4       VP                  V n        VP                  V n        \
        P                  ! VP                  VP                  V P                  4      V n        \
        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n        \        VP                  VP                  R 7      V n        \#        VR7      V n        RV n        V P)                  4        R# u upi )rË   ©r*   FN)r(   r)   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingr+   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersrÇ   Úlayersr°   rÎ   Únormrï   Ú
rotary_embÚgradient_checkpointingÚ	post_initr›   s   && €r6   r)   ÚMistralModel.__init__A  sÊ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸLšL¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m’mÜEJÈ6×KcÑKcÔEdÓeÑEd¸	Ô  Ö3ÑEdÑeó
ˆŒô # 6×#5Ñ#5¸6×;NÑ;NÔOˆŒ	Ü0¸Ô?ˆŒØ&+ˆÔ#ð 	�‰Öùò fs   ÂDc                óþ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[S[,          R	S[/# )
r^   Ú	input_idsNrt   rÓ   rŸ   Úinputs_embedsrÔ   rw   ra   )	rM   rÕ   rb   r   ÚFloatTensorrÖ   r   r   r   )rd   rG   s   "€r6   re   r  T  s“   ø€ ÷ 2
ñ 2
á×#Ñ# dÕ*ð2
ñ Ÿ™ tÕ+ð2
ñ ×&Ñ&¨Õ-ð	2
ñ
  �ð2
ñ ×(Ñ(¨4Õ/ð2
ñ ˜$•;ð2
ñ Ñ+Õ,ð2
ñ 
!ñ2
r8   c                óè  € VR J VR J,          '       d   \        R4      hVf   V P                  V4      pV'       d   Vf   \        V P                  R7      pVfb   Ve   VP	                  4       M^ p\
        P                  ! VP                  ^,          VP                  R7      V,           pVP                  ^ 4      pV P                  P                  f   \        M\        p	V	! V P                  VVVVR7      p
TpV P                  W³R7      pV P                  R V P                  P                    F  pV! V3RV
RVRVR	VR
V/VB pK  	  V P!                  V4      p\#        TV'       d   VR7      # R R7      # )Nz:You must specify exactly one of input_ids or inputs_embedsr   )rð   )r*   r1  rt   rŸ   rÓ   )rÓ   rt   rÓ   rŸ   rÔ   rž   )Úlast_hidden_staterŸ   )Ú
ValueErrorr%  r   r*   Úget_seq_lengthrM   r  rL   rð   rT   r£   r   r   r+  r)  r(  r*  r   )r4   r0  rt   rÓ   rŸ   r1  rÔ   rw   Úpast_seen_tokensÚmask_functionÚcausal_maskr_   rž   Údecoder_layers   &&&&&&&,      r6   r<   ÚMistralModel.forwardQ  sŒ  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ Ø ×-Ñ-¨iÓ8ˆMç˜Ò0Ü*°$·+±+Ô>ˆOàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLà.2¯k©k×.HÑ.HÒ.PÕ*ÔVwˆÙ#Ø—;‘;Ø'Ø)Ø+Ø%ô
ˆð &ˆØ"Ÿo™o¨m˜oÓWÐà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÓIˆMÙ)Øñà*ðð *ðð !0ð	ð
 $ðð %8ðð ñŠMñ Jð Ÿ	™	 -Ó0ˆÜ&Ø+ß/8˜Oô
ð 	
à>Bô
ð 	
r8   )r%  r,  r)  r*  r"  r+  r#  )NNNNNN)r?   r@   rA   rB   r)   r   r   r   r<   rC   rD   rE   rF   s   @@r6   r  r  ?  s=   ù‡ € ÷ó ð   ØØ÷2
ñ 2
ó ó ó  ÷2
ð 2
r8   r  c                   ó„   a a€ ] tR tRt oRR/tRR/tRR.R.3/tRR/tV 3R	 lt]	]
RV3R
 lR ll4       4       tRtVtV ;t# )ÚMistralForCausalLMi‰  zlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputr_   ÚlogitsÚkeep_full_weightc                óì   <€ \         SV `  V4       \        V4      V n        VP                  V n        \
        P                  ! VP                  VP                  R R7      V n        V P                  4        R# r%   )
r(   r)   r  rÞ   r#  r   r-   r+   r>  r-  r3   s   &&€r6   r)   ÚMistralForCausalLM.__init__�  sU   ø€ Ü‰Ñ˜Ô Ü! &Ó)ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒð 	�‰Ör8   c                óP  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[S[ P                  ,          R
S[S[,          RS[	/
# )r^   r0  Nrt   rÓ   rŸ   r1  ÚlabelsrÔ   Úlogits_to_keeprw   ra   )
rM   rÕ   rb   r   r2  rÖ   rc   r   r   r   )rd   rG   s   "€r6   re   ÚMistralForCausalLM.__annotate__›  s»   ø€ ÷ 6
ñ 6
á×#Ñ# dÕ*ð6
ñ Ÿ™ tÕ+ð6
ñ ×&Ñ&¨Õ-ð	6
ñ
  �ð6
ñ ×(Ñ(¨4Õ/ð6
ñ × Ñ  4Õ'ð6
ñ ˜$•;ð6
ñ ™eŸl™lÕ*ð6
ñ Ñ+Õ,ð6
ñ 
 ñ6
r8   c	                ó–  € V P                   ! RRVRVRVRVRVRV/V	B p
V
P                  p\        V\        4      '       d   \	        V) R4      MTpV P                  VRVR3,          4      pRpVe-   V P                  ! RR	VR
VRV P                  P                  /V	B p\        VVV
P                  V
P                  V
P                  R7      # )a{  
Example:

```python
>>> from transformers import AutoTokenizer, MistralForCausalLM

>>> model = MistralForCausalLM.from_pretrained("meta-mistral/Mistral-2-7b-hf")
>>> tokenizer = AutoTokenizer.from_pretrained("meta-mistral/Mistral-2-7b-hf")

>>> prompt = "Hey, are you conscious? Can you talk to me?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
```r0  rt   rÓ   rŸ   r1  rÔ   Nrg   r@  rE  r#  )Úlossr@  rŸ   r_   rß   rØ   )rÞ   r4  r  rc   Úslicer>  Úloss_functionr*   r#  r   rŸ   r_   rß   )r4   r0  rt   rÓ   rŸ   r1  rE  rÔ   rF  rw   Úoutputsr_   Úslice_indicesr@  rI  s   &&&&&&&&&,     r6   r<   ÚMistralForCausalLM.forward™  sÿ   € ð> ,0¯:ª:ñ ,
Øð,
à)ð,
ð &ð,
ð ,ð	,
ð
 (ð,
ð  ð,
ð ñ,
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒØ×%Ò%Ñp¨VÐp¸FÐpÈtÏ{É{×OeÑOeÐpÐioÑpˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r8   )r>  rÞ   r#  )NNNNNNNr   )r?   r@   rA   rB   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr)   r   r   r<   rC   rD   rE   rF   s   @@r6   r=  r=  ‰  sf   ù‡ € à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÐ/Ð0€Jõð Ø÷6
ñ 6
ó ó ÷6
ð 6
r8   r=  c                   ó   € ] tR tRtRtR# )ÚMistralForTokenClassificationiÔ  rØ   N©r?   r@   rA   rB   rC   rØ   r8   r6   rT  rT  Ô  ó   † Ûr8   rT  c                   ó   € ] tR tRtRtR# )Ú MistralForSequenceClassificationiØ  rØ   NrU  rØ   r8   r6   rX  rX  Ø  rV  r8   rX  c                   ó   € ] tR tRtRtR# )ÚMistralForQuestionAnsweringiÜ  rØ   NrU  rØ   r8   r6   rZ  rZ  Ü  s   † ÓX[r8   rZ  )é   )r¢   )r=  rZ  r  rÝ   rX  rT  )AÚcollections.abcr   rM   r   Úactivationsr   Úcache_utilsr   r   Ú
generationr   Úintegrationsr	   r
   Úmasking_utilsr   r   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   r   r   r   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   Úconfiguration_mistralr!   rx   r#   rQ   r\   rn   rŠ   rŒ   r°   rÇ   rÝ   rï   r  r=  rT  rX  rZ  Ú__all__rØ   r8   r6   Ú<module>rn     s†  ðõ %ã Ý å !ß .Ý )ß Lß RÝ B÷ó ÷ Pß Kß FÝ &ß IÑ IÝ 0ß GÝ 5Ý 0ô�—‘ô ò (ñ Ð-Ó.óó /ðõ2	U÷%ñ2 Ð)Ó*ô8)�r—y‘yó 8)ó +ð8)ñv ˜YÓ'ôJ�R—Y‘Yó Jó (ðJô(&Ð4ô &ðR ô˜_ó ó ðô$6<˜RŸY™Yô 6<ðr ôF
Ð)ó F
ó ðF
ðR ôG
Ð/°ó G
ó ðG
ôT	Ð$AÐCYô 	ô	Ð'GÐI_ô 	ô \Ð"=Ð?UÔ [ò‚r8   