+ je^RIt^RIHt^RIt^RIHuHt^RIHt^RIH t ^RI H t ^RI HtHt^RIHt^RIHtHt^R IHt^R IHt^R IHtHtHt^R IHtHt^R I H!t!H"t"^RI#H$t$H%t%^RI&H't'^RI(H)t)H*t*H+t+^RI,H-t-^RI.H/t/H0t0^RI1H2t2^RI3H4t4]!R4!RR]Pj44t6!RR]Pj4t7!RR]Pj4t8!RR]Pj4t9]!RR]Pj44t:!R R!]Pj4t;R"R#ltR't?R(t@]!R)4R>R*l4tAR?R+ltBR,R-ltC!R.R/]Pj4tD!R0R1]4tE!R2R3]%4tF]*!R4R5]F44tG]*!R6R7]F]44tH!R8R9]]F4tI!R:R;]]F4tJ.R@OtKR#)AN)Callable)nn)initialization)ACT2FN)Cache DynamicCache)GenerationMixin)use_experts_implementationuse_kernel_forward_from_hub)create_causal_mask)FlashAttentionKwargs) GenericForSequenceClassificationGenericForTokenClassificationGradientCheckpointingLayer)BaseModelOutputWithPastCausalLMOutputWithPast)ROPE_INIT_FUNCTIONSdynamic_rope_update)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tuple)deprecate_kwarg)maybe_autocastmerge_with_config_defaults)capture_outputsMistral4ConfigRMSNormcZaa]tRt^2toRV3RlV3RllltV3RlRltRtRtVtV;t #)Mistral4RMSNormc$<V^8dQhRS[RR/#)epsreturnN)float)format __classdict__s"o/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/mistral4/modeling_mistral4.py __annotate__Mistral4RMSNorm.__annotate__4s$$$$$c <\SV`4\P!\P !V44VnW nR#)z. Mistral4RMSNorm is equivalent to T5LayerNorm N)super__init__r Parametertorchonesweightvariance_epsilon)self hidden_sizer& __class__s&&&r+r1Mistral4RMSNorm.__init__4s/ ll5::k#:; #r.cN<V^8dQhRS[PRS[P/#r% hidden_statesr'r3Tensor)r)r*s"r+r,r-<s#;;U\\;ell;r.c:VPpVP\P4pVP ^4P RRR7pV\P !W0P,4,pVPVPV4,#)r%T)keepdim) dtypetor3float32powmeanrsqrtr6r5)r7r= input_dtypevariances&& r+forwardMistral4RMSNorm.forward<sw#)) %((7  $$Q',,R,>% H?T?T4T(UU {{]--k:::r.c^\VPP4 RVP 2#)z, eps=)tupler5shaper6)r7s&r+ extra_reprMistral4RMSNorm.extra_reprCs* ))*+6$2G2G1HIIr.)r6r5)gư>) __name__ __module__ __qualname____firstlineno__r1rKrP__static_attributes____classdictcell__ __classcell__r9r*s@@r+r#r#2s%$$;;JJr.r#caa]tRt^Gto]!RRR7R V3RlV3Rlll4t]]!RRR7R V3RlRll44t]P!4] R44t R t Vt V;t#) Mistral4RotaryEmbeddingdevicez5.18)versionc <V^8dQhRS[/#r%configr)r)r*s"r+r,$Mistral4RotaryEmbedding.__annotate__IsOO~Or.c<\SV`4VPVnVPVnWnVP P R,VnVPpVPR8wd\VP,pV!VP V4wq@n \P!VRR7Vn \P!VP4RR7VnR#) rope_typedefaultF) persistentN)r0r1max_position_embeddingsmax_seq_len_cachedoriginal_max_seq_lenr`rope_parametersrccompute_default_rope_parametersrattention_scalingrBufferinv_freqcloneoriginal_inv_freq)r7r`r\ rope_init_fnrmr9s&&& r+r1 Mistral4RotaryEmbedding.__init__Hs "("@"@$*$B$B! 44[A!%!E!E >>Y &.t~~>L+7 V+L(( (u= !#8>>+;!Nr.cP<V^8dQhRS[RS[S[PS[3,/#)r%r`r')r rNr3r?r()r)r*s"r+r,ra[s.555Z_`e`l`lns`sZt5r.c 0VPR,p\VRR4;'gVPVP,pRpRV\P !^V^\P R7V, ,, pVPV4V3#)a~ Computes the inverse frequencies according to the original RoPE implementation Args: config ([`~transformers.PreTrainedConfig`]): The model configuration. Returns: Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE). rope_thetahead_dimN?rC)rigetattrr8num_attention_headsr3aranger(rD)r`r\kwargsbasedimattention_factorrms&&, r+rj7Mistral4RotaryEmbedding.compute_default_rope_parametersYs~%%l3fj$/cc63E3EIcIc3c$5<<3#MPS#STU{{6"$444r.c^VPR,PVP^,R ^4P\P VP R7pVR ,P 4p\VP P\4'd2VP PR8wdVP PMRp\VRR7;_uu_4W4,P^^4p\P!Wf3R R7pVP4VP,pVP4VP,p RRR4XPVP R7X PVP R73# +'giLH;i) N)rCr\mpscpuF) device_typeenabledr}rw)NNNNNrB)rNr)rmexpandrOrDr3r(r\ isinstancetypestrr transposecatcosrksinrC) r7x position_idsinv_freq_expandedposition_ids_expandedrfreqsembrrs &&& r+rKMistral4RotaryEmbedding.forwardms2 MM- ( / / 0B0B10Er1 M P PW\WbWbklksks P t !-Z 8 > > @'1!((--'E'E!((--[`J`ahhmmfk  U C C&>II!QOE))UN3C'')d444C'')d444C D vvAGGv$cff177f&;;; D Cs (A4F F, )rkr`rmrgrorhrcN)rRrSrTrUrr1 staticmethodrjr3no_gradrrKrVrWrXrYs@@r+r[r[GsuXv.OO/O Xv.55/5$ ]]_<<)r)r*s"r+r,Mistral4Experts.__annotate__sC##||#\\#|| #  #r.c\P!V4p\P!4;_uu_4\PPP W P R7pVP^^^4p\P!VPRR7^4P4pRRR4XEF pV^,pWpP 8XdK\P!XV,4wrW,p \PPWPV,4P^RR7wrVPV 4V ,p \PPWP V,4p WWR3,,p VP#^WP%VP&44EK V# +'giEL';i)) num_classesrNrB)rB)r3rrr functionalone_hotrpermutegreaterrnonzerowhererrchunkrr index_add_rDrC)r7r=rrfinal_hidden_states expert_mask expert_hit expert_idx top_k_pos token_idx current_stategateupcurrent_hidden_statess&&&& r+rKMistral4Experts.forwardss $..}= ]]__((--55kO_O_5`K%--aA6K{8'DaHPPRJ %J#AJ---#(;;{:/F#G I)4M}}++M;L;LZ;XY__`agi_jHD$(KK$5$: !$&MM$8$89NP^P^_iPj$k !$9)`dJd)r)r*s"r+r,rs#U\\ellr.cTpVPpVPV4wrEpVPRVPR,4pVPWV4P!V!pWP V4,pV#)rrB)rOrrrr)r7r= residuals orig_shape_rrs&& r+rKMistral4MoE.forwardsw! "(( (, -(@%%**2}/B/B2/FG  ],OTTV`a %(;(;I(FF r.)r`rrrrrYs@@r+rrs#  r.rcdV^8dQhR\PR\R\P/#)r%r=n_repr')r3r?int)r)s"r+r,r,s. U UU\\ U# U%,, Ur.cVPwr#rEV^8XdV#VR,PW#WV4pVPW#V,WE4#)z This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) )rrNrr)rOrr)r=r batchnum_key_value_headsslenrus&& r+ repeat_kvrsU 2?1D1D.E z!"23::5W\dlmM  e(CT TTr.cV^8dQhR\PR\PR\PR\PR\PR,R\R\R \ \ ,/#) r%modulequerykeyvalueattention_maskNscalingdropoutr{)rModuler3r?r(rr)r)s"r+r,r,sw%% II% <<% % << % LL4' %  %%'(%r.c "\W P4p\W0P4p \P!WP ^^44V,p Ve W,p \ P PV R\PR7PVP4p \ P PWVPR7p \P!W4p V P ^^4P4p W3#)r%)r}rC)ptrainingrB)rnum_key_value_groupsr3matmulrrrrrErDrCrr contiguous) rrrrrrrr{ key_states value_states attn_weights attn_outputs &&&&&&&, r+eager_attention_forwardr$s3 ; ;VPR^4pVR,pV'd\W24pW,V,pV#)rcrdmscale_all_dimfactor)getr*)rirr,scaling_factorr)s&& r+yarn_apply_mscaler0 sS; 2i?(,,-=qA(2 $^DF&/G Nr.cVRRVPR,^,13,pVRVPR,^,R13,p\P!V)V3RR7#)z*Rotates half the hidden dims of the input..NrrB)rOr3r)rx1x2s& r+ rotate_halfr4*sZ 3"!''"+"" " #B 3 q " " #B 99rc2YB ''r.rotary_pos_embcVPV4pVPV4pW,\V4V,,pW,\V4V,,pWV3#)aIApplies Rotary Position Embedding to the query and key tensors. Args: q (`torch.Tensor`): The query tensor. k (`torch.Tensor`): The key tensor. cos (`torch.Tensor`): The cosine part of the rotary embedding. sin (`torch.Tensor`): The sine part of the rotary embedding. unsqueeze_dim (`int`, *optional*, defaults to 1): The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. Returns: `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. )rr4)qrrr unsqueeze_dimq_embedk_embeds&&&&& r+apply_rotary_pos_embr;1sS& -- &C -- &Cw;q>C/0Gw;q>C/0G  r.cVRRVPR,^,13,PV4pVRRVPR,^,13,PV4pVR,VR,rvVR,VR,r\P!Wb,Ws,, Wr,Wc,,.RR7p \P!W,W,, W,W,,.RR7p W3#)a Applies interleaved Rotary Position Embedding to the query and key tensors. DeepSeek lays the rotary dimensions out in interleaved pairs `(x0, x1), (x2, x3), ...`, each rotated by a single frequency. We compute that rotation directly on the even/odd slices instead of de-interleaving with a `view`/`transpose`/`reshape`; the output is bit-identical to the de-interleaved `rotate_half` formulation while avoiding the extra contiguous copy. Args: q (`torch.Tensor`): The query tensor. k (`torch.Tensor`): The key tensor. cos (`torch.Tensor`): The cosine part of the rotary embedding. sin (`torch.Tensor`): The sine part of the rotary embedding. position_ids (`torch.Tensor`): The position indices of the tokens corresponding to the query and key tensors. For example, this can be used to pass offsetted position ids when working with a KV-cache. unsqueeze_dim (`int`, *optional*, defaults to 1): The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. Returns: `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. .NrrB).:rNr%).:rNr%)rOrr3r) r7rrrrr8q1q2k1k2r9r:s &&&&&& r+apply_rotary_pos_emb_interleaverAKs8 c'SYYr]a''' ( 2 2= AC c'SYYr]a''' ( 2 2= AC y\1Y< y\1Y<iiBH,bh.ABKGiiBH,bh.ABKG  r.cpV^8dQhR\PR\R\R\P/#)r% positions_idsbetarfr')r3r?r(r )r)s"r+r,r,rs4%%%,,%e%^a%fkfrfr%r.c ^V\P!^\P!W, 4,4,,pVR,#)r)rNrN)r3r'floor)rCrDrfrs&&& r+get_llama_4_attn_scalerGrs7$1u{{=3Z'[#[\\\G # $$r.cjaa]tRtRtoRtV3RlV3RlltV3RlRltR V3RlRlltR tVt V;t #) Mistral4Attentioniwz4Multi-headed Latent Attention (MLA) from Deepseek V2c&<V^8dQhRS[RS[/#r%r` layer_idxr r )r)r*s"r+r,Mistral4Attention.__annotate__zs5]5]~5]#5]r.c<\SV`4WnW nVPVnVP VnVP VnVPVnVPVn VPVn VPVn VPVn VPVP,Vn VP VP,VnRVnVPf?\"P$!VP VPVP,RR7MRVnVPe8\"P$!VP VPVP(R7MRVnVPe\-VP4MRVnVPe?\"P$!VPVPVP,RR7MRVn\"P$!VP VPVP,VP(R7Vn\-VP4Vn\"P$!VPVPVPVP,,RR7Vn\"P$!VPVP,VP VP(R7Vn\;VP<VPR,4VnR#)TNFrg) r0r1r`rLattention_dropoutr8ry num_heads q_lora_rankqk_rope_head_dim kv_lora_rank v_head_dimqk_nope_head_dim qk_head_dimrr is_causalrrq_projattention_biasq_a_projr# q_a_layernormq_b_projkv_a_proj_with_mqakv_a_layernorm kv_b_projo_projr0rirr7r`rLr9s&&&r+r1Mistral4Attention.__init__zs{  "!'!9!9!--33!-- & 7 7"// ++ & 7 7!22V5L5LL$*$>$>&B\B\$\!' IId&&9I9I(IPU V + IId&&(:(:AVAV W EIDTDTD`_V-?-?@fj+ IIf(($..4;K;K*KRW X #%))      &"9"9 9&&#  .f.A.AB    NNd33dooE F  ii NNT__ ,   &&  ))?)?AQAQVZA[\ r.c<V^8dQhRS[PRS[PRS[S[PS[P3,/#)r%kv_nopek_rotr')r3r?rN)r)r*s"r+r,rNs?(((ell(uU\\[`[g[gMgGh(r.c VPwr4rTW5RVPVP,3pVPV4P V4P ^^4p\ P!WPVP.RR7wrxVPRVP^,RR4pVP!.VPRROVPVP,N5!p V RRVP13,PV4V RVPR13,PV4W3#)asExpands the compressed latents into key and value states. Args: - kv_nope: key + value without positional encoding, shape [batch_size, 1, seqlen, self.kv_lora_rank] - k_rot: shared key with positional encoding, shape [batch_size, 1, seqlen, self.qk_rope_head_dim] Returns the key and value states, two tensors of shape [batch, num_heads, seq, (k or v)_head_dim]. rN.rB) rOrVrUr`rrr3splitr new_emptyrScopy_) r7rerf batch_sizer seq_length key_shapek_noper!r s &&& r+ expand_kvMistral4Attention.expand_kvs (/}}$ zR1F1F1XY ..)..y9CCAqI${{75J5JDOO4\bde Ra"b9&&j cr(:jD3--//066u=''r.cl<V^8dQhRS[PRS[S[PS[P3,RS[PR,RS[PRS[R,RS[S[,RS[S[PS[PR,S[S[P,R,3,/#) r%r=position_embeddingsrNrpast_key_valuesr{r')r3r?rNrrr )r)r*s"r+r,rNs>)>)||>)#5<<#=>>) t+ >) ll >)  >)-.>) u||U\\D0% 2E2LL M>)r.c VPRRwrxWxRVP3p VPfVPV4p M/VP VP VP V444p V PV 4P^^4p \P!WPVP.RR7wrVPV4p \P!WPVP.RR7wrVPV4PV^WP4pVPV^WP4pVwppVP P"'d\%WVV4wrM\'WVV4wrVe VP)VWP*4wpp\P,!W3RR7pV\/VVP P0P3R4VP P0P3R44P5VP64,pVP9VV4wpp\:P<!VP P>\@4pT!VVVVV3RVPB'gRM VPDRVPF/VBwppVPIWxR4PK4pVPMV4pVV3#)Nrllama_4_scaling_beta original_max_position_embeddingsrrrrB)'rOrWrRrYr]r\r[rrr3rhrVrSr^rTr_r`rope_interleaverAr;updaterLrrGrir.rDrCror get_interface_attn_implementationr$rrPrrrra)r7r=rrrrrsr{rkrl query_shapeq_statesq_passq_rot compressed_kvkv_passrfk_passrr query_statesr r!attention_interfacer#r"s&&&&&&, r+rKMistral4Attention.forwards"/!4!4Sb!9 !r43C3CD    #{{=1H}}T%7%7 m8T%UVH==-771= H/D/DdF[F[.\bde // > ]5F5FH]H]4^dfg$$W-22:q*N_N_` :q*6K6KL&S ;; & & &:5cRLE5/c3GLE  &+2265..QMFEyy&b9 #&<  KK ' ' + +,B C KK ' ' + +,N O'  "\   ! $(>>&%#@ L(?(M(M KK , ,.E) %8      % $}}}C$2H2H % LL %  % ! \"))*"EPPR kk+. L((r.)rPr`r8rXr_r^r`rTrLrQrrar\r[r]rRrYrWrVrSrrUr) rRrSrTrUrr1rorKrVrWrXrYs@@r+rIrIws.>5]5]n((&>)>)>)r.rIcTaa]tRtRtoV3RlV3RlltRV3RlRlltRtVtV;t#)Mistral4DecoderLayeric&<V^8dQhRS[RS[/#rKrM)r)r*s"r+r,!Mistral4DecoderLayer.__annotate__s e e~ e# er.cb<\SV`4VPVn\WR7VnW!P 8d\ V4VnM\V4Vn\VPVPR7Vn \VPVPR7Vn R#))r`rLr&N) r0r1r8rI self_attnfirst_k_dense_replacermlprr# rms_norm_epsinput_layernormpost_attention_layernormrbs&&&r+r1Mistral4DecoderLayer.__init__s !--*&N 44 4"6*DH"6*DH.v/A/AvGZGZ[(78J8JPVPcPc(d%r.c.<V^8dQhRS[PRS[PR,RS[PR,RS[R,RS[R,RS[S[PS[P3,R,RS[S[,R S[P/#) r%r=rNrrs use_cacherrr{r')r3r? LongTensorrrrNrr)r)r*s"r+r,rs|| t+&&-    $; #5<<#=>E+, r.c TpVPV4pVP!RRVRVRVRVRVRV/VBwrW,pTpVPV4pVPV4pW,pV#)r=rrrsrrr)rrrr) r7r=rrrsrrrr{residualrs &&&&&&&, r+rKMistral4DecoderLayer.forwards!,,]; >> ' ) & ,    !4    !0 !55mD /  0 r.)r8rrrr)NNNFNrrYs@@r+rrs  e er.rcaa]tRtRtoRtRtR.tR.tRtRt Rt Rt Rt R] R]/t.t.t]P&!4V3Rl4tV3R ltR tVtV;t#) Mistral4PreTrainedModeli4modelTrrsr= attentionsc<\SV`V4\V\4'd:\P !VP RVPPR7R#\V\4'dq\P !VPRVPPR7\P !VPRVPPR7R#R#)r)rGstdN) r0 _init_weightsrrinitnormal_r5r`initializer_rangerrr)r7rr9s&&r+r%Mistral4PreTrainedModel._init_weightsGs f% f0 1 1 LLSdkk6S6S T  0 0 LL,,3DKKO  CRC^==?de  << (;(;A(>}G[G[\_ooL'11!4L(;;')+%  & "oomoW![[)H4;;+H+HIM)*%8*  !0  $ MJ -0 &++  r.)rrrrrrr)NNNNNN) rRrSrTrUr1rrrrKrVrWrXrYs@@r+rrQs=  2 2  2 2 r.rcaa]tRtRtoRR/tRR/tRR.R.3/tRR/tV3R lt] ] R V3R lR ll44t R t Vt V;t#)Mistral4ForCausalLMizlm_head.weightzmodel.embed_tokens.weightlm_headcolwise_gather_outputr=logitskeep_full_weightc<\SV`V4\V4VnVPVn\ P !VPVPRR7VnVP4R#)FrN) r0r1rrrrrr8rrrs&&r+r1Mistral4ForCausalLM.__init__sU  "6*  ++yy!3!3V5F5FUS  r.cP<V^8dQhRS[PR,RS[PR,RS[PR,RS[R,RS[PR,RS[PR,RS[R,R S[S[P,R S[S[,R S[ / #) r%rNrrrsrlabelsrlogits_to_keepr{r') r3rr?rrrr rrr)r)r*s"r+r, Mistral4ForCausalLM.__annotate__s6 6 ##d*6  t+6 &&- 6  6 ((4/ 6   4'6 $;6 ell*6 +,6  6 r.c  VP!R RVRVRVRVRVRV/V Bp V Pp \V\4'd\ V)R4MTp VP V RV R3,4p RpVe-VP !R R V R VR VPP/V Bp\VV V PV PV PR 7#)a Example: ```python >>> from transformers import AutoTokenizer, Mistral4ForCausalLM >>> model = Mistral4ForCausalLM.from_pretrained("meta-mistral4/Mistral4-2-7b-hf") >>> tokenizer = AutoTokenizer.from_pretrained("meta-mistral4/Mistral4-2-7b-hf") >>> prompt = "Hey, are you conscious? Can you talk to me?" >>> inputs = tokenizer(prompt, return_tensors="pt") >>> # Generate >>> generate_ids = model.generate(inputs.input_ids, max_length=30) >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you." ```rrrrsrrNrrrr)lossrrsr=rr) rrrr slicer loss_functionr`rrrsr=r)r7rrrrsrrrrr{outputsr= slice_indicesrrs&&&&&&&&&, r+rKMistral4ForCausalLM.forwards>,0::, , ), &, , , ( ,  , ,  11 8B>SV8W8W~ot4]k mA}a,?@A  %%pVpFpt{{OeOepiopD%#33!//))   r.)rrr)NNNNNNNr)rRrSrTrU_tied_weights_keys_tp_plan_pp_plan _fsdp_planr1rrrKrVrWrXrYs@@r+rrsf*,GH23H_-z:;H/0J6 6 6 6 r.rc]tRtRtRtR#)!Mistral4ForSequenceClassificationirNrRrSrTrUrVrr.r+rrr.rc]tRtRtRtR#)Mistral4ForTokenClassificationirNrrr.r+rrrr.r)r)rr)r)Nr)rrrrr)Lr&collections.abcrr3torch.nn.functionalrrrrr activationsr cache_utilsrr generationr integrationsr r masking_utilsr modeling_flash_attention_utilsr modeling_layersrrrmodeling_outputsrrmodeling_rope_utilsrrmodeling_utilsrrprocessing_utilsrutilsrrrutils.deprecationr utils.genericrrutils.output_capturingrconfiguration_mistral4r rr#r[rrrrrr$r*r0r4r;rArGrIrrrrrr__all__rr.r+rs( $ &!.)S/B PKF&II0G52Y'JbiiJ(J(6