+
    œ°³j}!  ã                   óN  € ^ RI Ht ^ RIt^ RIHu Ht ^ RIHt ^RIHt	 ^RI
Ht ^RIHt ^RIHtHt ^RIHtHt ^R	IHt ^R
IHt ^RIHt ^RIHtHtHtHtHt ^RI H!t!H"t"H#t#H$t$H%t%H&t& ^RI'H(t( ^RI)H*t* ^RI+H,t, ]PZ                  ! ].4      t/ ! R R]#4      t0 ! R R]$4      t1 ! R R]*4      t2 ! R R]4      t3 ! R R]4      t4 ! R R]4      t5 ! R R]4      t6 ! R R ]4      t7 ! R! R"]4      t8 ! R# R$]"4      t9 ! R% R&]!4      t: ! R' R(]]84      t; ! R) R*]]84      t<. R+Ot=R# ),é    )ÚCallableN)Únn)Úinitialization)ÚCache)ÚFlashAttentionKwargs)Ú GenericForSequenceClassificationÚGenericForTokenClassification)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)Úlogging)ÚDeepseekV2TopkRouter)ÚDeepseekV3AttentionÚDeepseekV3DecoderLayerÚDeepseekV3ExpertsÚDeepseekV3MoEÚapply_rotary_pos_emb_interleave)ÚLlamaForCausalLMÚ
LlamaModelÚLlamaRMSNormÚLlamaRotaryEmbeddingÚapply_rotary_pos_embÚeager_attention_forward)Úget_llama_4_attn_scale)ÚQwen2MoeMLP©ÚMistral4Configc                   ó   € ] tR t^3tRtR# )ÚMistral4RMSNorm© N©Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__static_attributes__r    ó    Ún/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/mistral4/modular_mistral4.pyr   r   3   ó   † Ûr'   r   c                   ó   € ] tR t^7tRtR# )ÚMistral4RotaryEmbeddingr    Nr!   r    r'   r(   r+   r+   7   r)   r'   r+   c                   ó   € ] tR t^;tRtR# )ÚMistral4MLPr    Nr!   r    r'   r(   r-   r-   ;   r)   r'   r-   c                   ó8   a a€ ] tR t^?t oV 3R ltR tRtVtV ;t# )ÚMistral4TopkRouterc                ón   <€ \         SV `  V4       V =VP                  V n        VP
                  V n        R # ©N)ÚsuperÚ__init__Útopk_methodÚnum_local_expertsÚnum_expertsÚnorm_topk_prob)ÚselfÚconfigÚ	__class__s   &&€r(   r3   ÚMistral4TopkRouter.__init__@   s2   ø€ Ü‰Ñ˜Ô ØÐØ!×3Ñ3ˆÔØ$×3Ñ3ˆÖr'   c                ó  € VP                  RV P                  4      p\        P                  ! WP                  4      pVP                  R4      pVP                  RV P                  V P                  V P                  ,          4      P                  ^RR7      ^ ,          P                  RR7      p\        P                  ! W@P                  RRR7      ^,          p\        P                  ! V4      pVP                  ^V^4       VP                  R4      P                  RV P                  V P                  V P                  ,          4      P!                  RV P                  4      pVP#                  VP%                  4       ( R4      p\        P                  ! W€P&                  RRR7      ^,          p	VP)                  ^V	4      p
V P*                  '       d#   V
P                  RRR7      R,           pW«,          p
W P,                  ,          p
W*V	3# )	é   ©ÚdimF)Úkr?   Úsortedç        T)r?   Úkeepdimg#B’¡œÇ;éÿÿÿÿ)ÚviewÚ
hidden_dimÚFÚlinearÚweightÚsoftmaxÚ	num_groupr6   ÚtopkÚsumÚtorchÚ
topk_groupÚ
zeros_likeÚscatter_Ú	unsqueezeÚexpandÚreshapeÚmasked_fillÚboolÚtop_kÚgatherr7   Úrouted_scaling_factor)r8   Úhidden_statesÚrouter_logitsÚscoresÚgroup_scoresÚ	group_idxÚ
group_maskÚ
score_maskÚscores_for_choiceÚtopk_indicesÚtopk_weightsÚdenominators   &&          r(   ÚforwardÚMistral4TopkRouter.forwardF   s£  € Ø%×*Ñ*¨2¨t¯©Ó?ˆÜŸš ·±Ó<ˆØ×&Ñ& rÓ*ˆà�K‰K˜˜DŸN™N¨D×,<Ñ,<ÀÇÁÕ,NÓO×TÑTÐUVÐ\^ÐTÓ_Ð`aÕb×fÑfÐkmÐfÓnð 	ô —J’J˜|¯©ÀBÈuÔUÐVWÕXˆ	Ü×%Ò% lÓ3ˆ
Ø×Ñ˜A˜y¨!Ô,à× Ñ  Ó$ß‰V�B˜Ÿ™¨×(8Ñ(8¸D¿N¹NÕ(JÓKß‰W�R˜×)Ñ)Ó*ð 	ð
 #×.Ñ.°
·±Ó0AÐ/AÀ3ÓGÐÜ—z’zÐ"3·z±zÀrÐRWÔXÐYZÕ[ˆØ—}‘} Q¨Ó5ˆØ××ÐØ&×*Ñ*¨r¸4Ð*Ó@À5ÕHˆKØÕ'ˆLØ#×&@Ñ&@Õ@ˆØ¨LÐ8Ð8r'   )r7   r6   )	r"   r#   r$   r%   r3   re   r&   Ú__classdictcell__Ú__classcell__©r:   Ú__classdict__s   @@r(   r/   r/   ?   s   ù‡ € õ4÷9ò 9r'   r/   c                   ó   € ] tR t^_tRtR# )ÚMistral4Expertsr    Nr!   r    r'   r(   rl   rl   _   r)   r'   rl   c                   ó   € ] tR t^ctRtR# )ÚMistral4MoEr    Nr!   r    r'   r(   rn   rn   c   r)   r'   rn   c                   ó6   a € ] tR t^gt o RV 3R lR lltRtV tR# )ÚMistral4AttentionNc                ól  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          RS[ P                  R,          RS[ P                  RS[R,          RS[S[,          RS[S[ P                  S[ P                  R,          S[S[ P                  ,          R,          3,          /# )	é   rZ   Úposition_embeddingsÚattention_maskNÚposition_idsÚpast_key_valuesÚkwargsÚreturn)rN   ÚTensorÚtupler   r   r   )Úformatrj   s   "€r(   Ú__annotate__ÚMistral4Attention.__annotate__h   s¥   ø€ ÷ >)ñ >)á—|‘|ð>)ñ #¡5§<¡<±·±Ð#=Õ>ð>)ñ Ÿ™ tÕ+ð	>)ñ
 —l‘lð>)ñ  �ð>)ñ Ñ-Õ.ð>)ñ 
‰u�|‰|™UŸ\™\¨DÕ0±%¹¿¹Õ2EÈÕ2LÐLÕ	Mñ>)r'   c                óÞ  € VP                   R R w  rxWxRV P                  3p	V P                  f   V P                  V4      p
M/V P	                  V P                  V P                  V4      4      4      p
V
P                  V	4      P                  ^^4      p
\        P                  ! W P                  V P                  .RR7      w  r¼V P                  V4      p\        P                  ! WÐP                  V P                  .RR7      w  rïV P                  V4      P                  V^W€P                  4      pVP                  V^W€P                  4      pVw  ppV P                   P"                  '       d   \%        WÏVV4      w  rÏM\'        WÏVV4      w  rÏVe    VP)                  VWðP*                  4      w  pp\        P,                  ! W¼3RR7      pV\/        VV P                   P0                  P3                  R4      V P                   P0                  P3                  R4      4      P5                  VP6                  4      ,          pV P9                  VV4      w  pp\:        P<                  ! V P                   P>                  \@        4      pT! V VVVV3RV PB                  '       g   RMV PD                  RV PF                  /VB w  ppVPI                  WxR4      PK                  4       pV PM                  V4      pVV3# )Nr>   Úllama_4_scaling_betaÚ original_max_position_embeddingsÚdropoutrB   ÚscalingrD   )'ÚshapeÚqk_head_dimÚq_lora_rankÚq_projÚq_b_projÚq_a_layernormÚq_a_projrE   Ú	transposerN   ÚsplitÚqk_nope_head_dimÚqk_rope_head_dimÚkv_a_proj_with_mqaÚkv_lora_rankÚkv_a_layernormr9   Úrope_interleaver   r   ÚupdateÚ	layer_idxÚcatr   Úrope_parametersÚgetÚtoÚdtypeÚ	expand_kvr
   Úget_interfaceÚ_attn_implementationr   ÚtrainingÚattention_dropoutr‚   rT   Ú
contiguousÚo_proj)r8   rZ   rs   rt   ru   rv   rw   Ú
batch_sizeÚ
seq_lengthÚquery_shapeÚq_statesÚq_passÚq_rotÚcompressed_kvÚkv_passÚk_rotÚk_passÚcosÚsinÚquery_statesÚ
key_statesÚvalue_statesÚattention_interfaceÚattn_outputÚattn_weightss   &&&&&&,                  r(   re   ÚMistral4Attention.forwardh   sŸ  € ð "/×!4Ñ!4°S°bÐ!9Ñˆ
Ø!¨r°4×3CÑ3CÐDˆà×ÑÒ#Ø—{‘{ =Ó1‰Hà—}‘} T×%7Ñ%7¸¿¹ÀmÓ8TÓ%UÓVˆHØ—=‘= Ó-×7Ñ7¸¸1Ó=ˆÜŸš H×/DÑ/DÀd×F[ÑF[Ð.\ÐbdÔe‰ˆà×/Ñ/°Ó>ˆÜŸš ]×5FÑ5FÈ×H]ÑH]Ð4^ÐdfÔg‰ˆà×$Ñ$ WÓ-×2Ñ2°:¸qÀ*×N_ÑN_Ó`ˆØ—
‘
˜: q¨*×6KÑ6KÓLˆà&‰ˆˆSØ�;‰;×&×&Ð&Ü:¸5ÈÈcÓR‰LˆE�5ä/°¸cÀ3ÓG‰LˆEð Ò&Ø+×2Ñ2°6¸5Ç.Á.ÓQ‰MˆF�Eä—y’y & °bÔ9ˆà#Ô&<ØØ�K‰K×'Ñ'×+Ñ+Ð,BÓCØ�K‰K×'Ñ'×+Ñ+Ð,NÓOó'
÷ ‰"ˆ\×ÑÓ
 õ	!ˆð $(§>¡>°&¸%Ó#@Ñ ˆ
�Lä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆ�\ð "×)Ñ)¨*À"ÓE×PÑPÓRˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r'   r    r1   )r"   r#   r$   r%   re   r&   rg   ©rj   s   @r(   rp   rp   g   s   ø‡ € ÷>)÷ >)ð >)r'   rp   c                   ó2   a € ] tR t^©t o V 3R lR ltRtV tR# )ÚMistral4DecoderLayerc                ó&   <€ V ^8„  d   QhRS[ RS[/# )rr   r9   r“   )r   Úint)r{   rj   s   "€r(   r|   Ú!Mistral4DecoderLayer.__annotate__ª   s   ø€ ÷ eñ e™~ð e¹#ñ er'   c                ó‚  € \         P                  P                  V 4       VP                  V n        \	        WR 7      V n        W!P                  8¼  d   \        V4      V n        M\        V4      V n        \        VP                  VP                  R7      V n        \        VP                  VP                  R7      V n        R# ))r9   r“   )ÚepsN)r   ÚModuler3   Úhidden_sizerp   Ú	self_attnÚfirst_k_dense_replacern   Úmlpr-   r   Úrms_norm_epsÚinput_layernormÚpost_attention_layernorm)r8   r9   r“   s   &&&r(   r3   ÚMistral4DecoderLayer.__init__ª   s‰   € Ü
�	‰	×Ñ˜4Ô Ø!×-Ñ-ˆÔä*°&ÔNˆŒà×4Ñ4Ô4Ü" 6Ó*ˆD�Hä" 6Ó*ˆDŒHä.¨v×/AÑ/AÀv×GZÑGZÔ[ˆÔÜ(7¸×8JÑ8JÐPV×PcÑPcÔ(dˆÖ%r'   )r¼   rÁ   r¿   rÂ   r½   N)r"   r#   r$   r%   r3   r&   rg   r³   s   @r(   rµ   rµ   ©   s   ø‡ € ÷eö er'   rµ   c                   ó¢   a a€ ] tR t^¹t oRtRtR.tR.tRtRt	Rt
RtRtR]R]/t. t. t]P&                  ! 4       V 3R l4       tV3R ltR	tVtV ;t# )
ÚMistral4PreTrainedModelÚmodelTrµ   rv   rZ   Ú
attentionsc                óÐ  <€ \         SV `  V4       \        V\        4      '       d:   \        P
                  ! VP                  R V P                  P                  R7       R# \        V\        4      '       dq   \        P
                  ! VP                  R V P                  P                  R7       \        P
                  ! VP                  R V P                  P                  R7       R# R# )rB   )ÚmeanÚstdN)r2   Ú_init_weightsÚ
isinstancer/   ÚinitÚnormal_rI   r9   Úinitializer_rangerl   Úgate_up_projÚ	down_proj)r8   Úmoduler:   s   &&€r(   rË   Ú%Mistral4PreTrainedModel._init_weightsÌ   sŽ   ø€ ä‰Ñ˜fÔ%Ü�fÔ0×1Ò1Ü�LŠL˜Ÿ™¨S°d·k±k×6SÑ6S×TÜ˜¤×0Ò0Ü�LŠL˜×,Ñ,°3¸D¿K¹K×<YÑ<YÕZÜ�LŠL˜×)Ñ)°¸¿¹×9VÑ9V×Wñ 1r'   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # )rr   r9   r   )r{   rj   s   "€r(   r|   Ú$Mistral4PreTrainedModel.__annotate__¹   s   ø‡ ‚ ÙÑò r'   r    )r"   r#   r$   r%   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendrµ   rp   Ú_can_record_outputsÚ_keep_in_fp32_modules_strictÚ"_keys_to_ignore_on_load_unexpectedrN   Úno_gradrË   Ú__annotate_func__r&   rg   rh   ri   s   @@r(   rÅ   rÅ   ¹   sƒ   ù‡ € àÐØ&*Ð#Ø/Ð0ÐØ#4Ð"5ÐØÐØ€NØÐà!ÐØ"&ÐàÐ-ØÐ'ðÐð $&Ð Ø)+Ð&à
‡]‚]ƒ_ôXó ðX÷) … r'   rÅ   c                   ó   € ] tR t^ÖtRtR# )ÚMistral4Modelr    Nr!   r    r'   r(   rå   rå   Ö   r)   r'   rå   c                   ó   € ] tR t^ÚtRtR# )ÚMistral4ForCausalLMr    Nr!   r    r'   r(   rç   rç   Ú   r)   r'   rç   c                   ó   € ] tR t^ÞtRtR# )Ú!Mistral4ForSequenceClassificationr    Nr!   r    r'   r(   ré   ré   Þ   r)   r'   ré   c                   ó   € ] tR t^âtRtR# )ÚMistral4ForTokenClassificationr    Nr!   r    r'   r(   rë   rë   â   r)   r'   rë   )rÅ   rå   rç   ré   rë   )>Úcollections.abcr   rN   Útorch.nn.functionalr   Ú
functionalrG   Ú r   rÍ   Úcache_utilsr   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   r	   Úmodeling_utilsr
   r   Úprocessing_utilsr   Úutilsr   Ú deepseek_v2.modeling_deepseek_v2r   Ú deepseek_v3.modeling_deepseek_v3r   r   r   r   r   Úllama.modeling_llamar   r   r   r   r   r   Úministral3.modeling_ministral3r   Úqwen2_moe.modeling_qwen2_moer   Úconfiguration_mistral4r   Ú
get_loggerr"   Úloggerr   r+   r-   r/   rl   rn   rp   rµ   rÅ   rå   rç   ré   rë   Ú__all__r    r'   r(   Ú<module>rÿ      s  ðõ %ã ß Ð Ý å &Ý  Ý Bß ^ß FÝ &Ý Ý C÷õ ÷÷ õ DÝ 6Ý 2ð 
×	Ò	˜HÓ	%€ô	�lô 	ô	Ð2ô 	ô	�+ô 	ô9Ð-ô 9ô@	Ð'ô 	ô	�-ô 	ô?)Ð+ô ?)ôDeÐ1ô eô X˜oô Xô:	�Jô 	ô	Ð*ô 	ô	Ð(HÐJaô 	ô	Ð%BÐD[ô 	ò‚r'   