+
    œ°³jÓ5  ã                   ó2  € ^ RI t ^ RIHt ^ RIt^ RIHt ^RIHt ^RI	H
t
 ^RIHtHt ^RIHtHt ^RIHt ^R	IHtHtHtHt ^R
IHt ^RIHt ^RIHt ^RIHtH t H!t!H"t"H#t#H$t$H%t%H&t&H't'H(t( ]PR                  ! ]*4      t+]! RR7      ] ! R R]4      4       4       t, ! R R]4      t- ! R R]%4      t. ! R R]&4      t/ ! R R]"4      t0 ! R R]4      t1 ! R R]4      t2 ! R R]$4      t3] ! R  R!]#4      4       t4] ! R" R#] 4      4       t5 ! R$ R%]!4      t6. R&Ot7R# )'é    N)ÚCallable)Ústrict)Úinitialization)ÚCache)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)ÚDeepseekV2Attention)ÚGemma3TextScaledWordEmbedding)ÚLlamaConfig)
ÚLlamaDecoderLayerÚLlamaForCausalLMÚLlamaForSequenceClassificationÚLlamaMLPÚ
LlamaModelÚLlamaPreTrainedModelÚLlamaRMSNormÚLlamaRotaryEmbeddingÚapply_rotary_pos_embÚeager_attention_forwardzopenbmb/MiniCPM3-4B)Ú
checkpointc                   óÎ   a a€ ] tR t^/t oRtRRRRRRRRRR	R
RRRRR	/tRtRtRtRt	^>t
^(t^(tRtRtRtRtRtRt^@t^ tRt^tRtRtV 3R lt]V3R lR l4       tV3R ltRtVtV ;t# )ÚMiniCPM3Configa  
kv_lora_rank (`int`, *optional*, defaults to 256):
    Rank of the low-rank KV projection in multi-head latent attention.
q_lora_rank (`int`, *optional*, defaults to 768):
    Rank of the low-rank query projection in multi-head latent attention. If `None`, the query projection
    is a single dense projection rather than a low-rank one.
qk_nope_head_dim (`int`, *optional*, defaults to 64):
    Dimension of the non-RoPE part of each query/key head.
qk_rope_head_dim (`int`, *optional*, defaults to 32):
    Dimension of the RoPE part of each query/key head.
v_head_dim (`int`, *optional*):
    Dimension of each value head. If `None`, defaults to `hidden_size // num_attention_heads`.
scale_emb (`int` or `float`, *optional*, defaults to 12):
    Multiplier applied to input embeddings.
scale_depth (`int` or `float`, *optional*, defaults to 1.4):
    Multiplier for residual connections; the effective scaling is `scale_depth / sqrt(num_hidden_layers)`.
    If `None`, defaults to `sqrt(num_hidden_layers)` (no-op scaling).
dim_model_base (`int`, *optional*, defaults to 256):
    Base model dimension used to scale logits before the language model head. If `None`,
    defaults to `hidden_size` (no-op scaling).

Example:

```python
>>> from transformers import MiniCPM3Model, MiniCPM3Config
>>> # Initializing a MiniCPM3 style configuration
>>> configuration = MiniCPM3Config()
>>> # Initializing a model from the configuration
>>> model = MiniCPM3Model(configuration)
>>> # Accessing the model configuration
>>> configuration = model.config
```
zlayers.*.self_attn.q_projÚcolwisezlayers.*.self_attn.q_b_projz%layers.*.self_attn.kv_a_proj_with_mqaÚmla_kv_a_projzlayers.*.self_attn.kv_b_projzlayers.*.self_attn.o_projÚrowwisezlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projÚminicpm3iè i 
  i   i €  gš™™™™™¹?gñhãˆµøä>Té   i   Ngffffffö?c                ó’  <€ V P                   V n        V P                  V P                   ,           V n        V P                  f#   V P
                  V P                  ,          V n        V P                  f&   \        P                  ! V P                  4      V n        V P                  f   V P
                  V n        \        SV `4  ! R/ VB  R # )N© )Úqk_rope_head_dimÚhead_dimÚqk_nope_head_dimÚqk_head_dimÚ
v_head_dimÚhidden_sizeÚnum_attention_headsÚscale_depthÚmathÚsqrtÚnum_hidden_layersÚdim_model_baseÚsuperÚ__post_init__)ÚselfÚkwargsÚ	__class__s   &,€Ún/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/minicpm3/modular_minicpm3.pyr4   ÚMiniCPM3Config.__post_init__v   sš   ø€ à×-Ñ-ˆŒà×0Ñ0°4×3HÑ3HÕHˆÔð �?‰?Ò"Ø"×.Ñ.°$×2JÑ2JÕJˆDŒOØ×ÑÒ#Ü#Ÿyšy¨×)?Ñ)?Ó@ˆDÔØ×ÑÒ&Ø"&×"2Ñ"2ˆDÔÜ‰ÒÑ' Ô'ó    c                ó    <€ V ^8„  d   QhRS[ /# )é   Úreturn)Úfloat)ÚformatÚ__classdict__s   "€r8   Ú__annotate__ÚMiniCPM3Config.__annotate__†   s   ø€ ÷ 6ñ 6¡ñ 6r:   c                ó<   € V P                   V P                  ,          # ©N)r,   r2   )r5   s   &r8   Úlogits_scalingÚMiniCPM3Config.logits_scaling…   s   € ð ×Ñ $×"5Ñ"5Õ5Ð5r:   c                óX  <€ V ^8„  d   Qh/ S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ R,          ;R&   S[ ;R&   S[;R	&   S[;R
&   S[;R&   S[ ;R&   S[ R,          ;R&   S[ ;R&   S[ ;R&   S[ R,          ;R&   S[ S[,          ;R&   S[ S[,          R,          ;R&   S[ R,          ;R&   # )r<   Ú
vocab_sizer,   Úintermediate_sizer1   r-   NÚnum_key_value_headsÚmax_position_embeddingsÚinitializer_rangeÚrms_norm_epsÚtie_word_embeddingsÚkv_lora_rankÚq_lora_rankr)   r'   r+   Ú	scale_embr.   r2   )Úintr>   Úbool)r?   r@   s   "€r8   rA   rB   /   s0  ø‡ ‚ ñh Ññi ñj Ññk ñl Ñ!ñm ñn Ñño ñp Ñ!ñq ñr ˜t�Ñ(ñs ñt !Ñ(ñu ñv Ñ"ñw ñx Ññy ñz Ñ$ñ{ ñ| Ññ} ñ~ �t•Ñ!ñ ñ@ ÑñA ñB ÑñC ñD �d•
Ñ!ñE ñF ‘U�{ÑñG ñH ‘u•˜tÕ#Ñ)ñI ñJ ˜$•JÑ$òK r:   )r2   r(   r*   r.   r+   ) Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Úbase_model_tp_planÚ
model_typerH   r,   rI   r1   r-   rJ   rK   rL   rM   rN   rO   rP   r)   r'   r+   rQ   r.   r2   r4   ÚpropertyrE   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r7   r@   s   @@r8   r   r   /   sÊ   ù‡ € ñ ðF 	$ YØ% yØ/°Ø&¨	Ø# YØ  )Ø 	Ø  )ð	Ðð €Jð €JØ€KØ!ÐØÐØ!ÐØ&(ÐØ#(ÐØ"ÐØ€LØ $ÐØ€LØ!€KØÐØÐØ!€JØ€IØ&)€KØ!$€Nõ(ð ÷6ó ð6÷o … r:   r   c                   ó   € ] tR t^‹tRtR# )ÚMiniCPM3ScaledWordEmbeddingr&   N©rT   rU   rV   rW   r]   r&   r:   r8   rb   rb   ‹   ó   † Ûr:   rb   c                   ó   € ] tR t^�tRtR# )ÚMiniCPM3RMSNormr&   Nrc   r&   r:   r8   rf   rf   �   rd   r:   rf   c                   ó   € ] tR t^“tRtR# )ÚMiniCPM3RotaryEmbeddingr&   Nrc   r&   r:   r8   rh   rh   “   rd   r:   rh   c                   ó   € ] tR t^—tRtR# )ÚMiniCPM3MLPr&   Nrc   r&   r:   r8   rj   rj   —   rd   r:   rj   c                   ó:   a € ] tR t^›t o RtRV 3R lR lltRtV tR# )ÚMiniCPM3Attentiona1  
Multi-head Latent Attention (MLA), structurally identical to `DeepseekV2Attention`.
The only difference is the rotary convention: MiniCPM3 keeps the original cos/sin RoPE
(`apply_rotary_pos_emb`) instead of DeepSeek-V2's complex rotary, so we inherit the
module construction and override only `forward`.
Nc                ó  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          RS[ P                  R,          RS[R,          RS[S[ P                  S[ P                  R,          3,          /# )r<   Úhidden_statesÚposition_embeddingsÚattention_maskNÚpast_key_valuesr=   )ÚtorchÚTensorÚtupler   )r?   r@   s   "€r8   rA   ÚMiniCPM3Attention.__annotate__£   sw   ø€ ÷ 6)ñ 6)á—|‘|ð6)ñ #¡5§<¡<±·±Ð#=Õ>ð6)ñ Ÿ™ tÕ+ð	6)ñ
  �ð6)ñ 
‰u�|‰|™UŸ\™\¨DÕ0Ð0Õ	1ñ6)r:   c                óœ  € VP                   R R w  rgWgRV P                  3pV P                  f   V P                  V4      p	M/V P	                  V P                  V P                  V4      4      4      p	V	P                  V4      P                  ^^4      p	\        P                  ! W�P                  V P                  .RR7      w  r«V P                  V4      p\        P                  ! WÀP                  V P                  .RR7      w  rÞV P                  V4      P                  V^WpP                  4      pVP                  V^WpP                  4      pVw  pp\!        W¾VV4      w  r¾Ve   VP#                  WþV P$                  4      w  rþ\        P&                  ! W«3RR7      pV P)                  Wþ4      w  pp\*        P,                  ! V P.                  P0                  \2        4      pT! V VVVV3RV P4                  '       g   RMV P6                  RV P8                  /VB w  ppVP;                  WgR4      P=                  4       pV P?                  V4      pVV3# )N)ÚdimÚdropoutg        Úscalingéÿÿÿÿ) Úshaper*   rP   Úq_projÚq_b_projÚq_a_layernormÚq_a_projÚviewÚ	transposerr   Úsplitr)   r'   Úkv_a_proj_with_mqarO   Úkv_a_layernormr   ÚupdateÚ	layer_idxÚcatÚ	expand_kvr	   Úget_interfaceÚconfigÚ_attn_implementationr   ÚtrainingÚattention_dropoutry   ÚreshapeÚ
contiguousÚo_proj)r5   rn   ro   rp   rq   r6   Ú
batch_sizeÚ
seq_lengthÚquery_shapeÚq_statesÚq_passÚq_rotÚcompressed_kvÚkv_passÚk_rotÚk_passÚcosÚsinÚquery_statesÚ
key_statesÚvalue_statesÚattention_interfaceÚattn_outputÚattn_weightss   &&&&&,                  r8   ÚforwardÚMiniCPM3Attention.forward£   s   € ð "/×!4Ñ!4°S°bÐ!9Ñˆ
Ø!¨r°4×3CÑ3CÐDˆà×ÑÒ#Ø—{‘{ =Ó1‰Hà—}‘} T×%7Ñ%7¸¿¹ÀmÓ8TÓ%UÓVˆHØ—=‘= Ó-×7Ñ7¸¸1Ó=ˆÜŸš H×/DÑ/DÀd×F[ÑF[Ð.\ÐbdÔe‰ˆà×/Ñ/°Ó>ˆÜŸš ]×5FÑ5FÈ×H]ÑH]Ð4^ÐdfÔg‰ˆà×$Ñ$ WÓ-×2Ñ2°:¸qÀ*×N_ÑN_Ó`ˆØ—
‘
˜: q¨*×6KÑ6KÓLˆà&‰ˆˆSô ,¨E¸#¸sÓC‰ˆð Ò&Ø+×2Ñ2°6À$Ç.Á.ÓQ‰MˆFä—y’y & °bÔ9ˆà#'§>¡>°&Ó#@Ñ ˆ
�Lä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆ�\ð "×)Ñ)¨*À"ÓE×PÑPÓRˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r:   r&   )NN)rT   rU   rV   rW   rX   r£   r]   r^   ©r@   s   @r8   rl   rl   ›   s   ø‡ € ñ÷6)÷ 6)ð 6)r:   rl   c                   óT   a a€ ] tR t^Üt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚMiniCPM3DecoderLayerc                ó&   <€ V ^8„  d   QhRS[ RS[/# )r<   rŠ   r†   )r   rR   )r?   r@   s   "€r8   rA   Ú!MiniCPM3DecoderLayer.__annotate__Ý   s   ø€ ÷ Wñ W™~ð W¹#ñ Wr:   c                ó’   <€ \         SV `  W4       VP                  \        P                  ! VP
                  4      ,          V n        R # rD   )r3   Ú__init__r.   r/   r0   r1   Úresidual_scale)r5   rŠ   r†   r7   s   &&&€r8   r«   ÚMiniCPM3DecoderLayer.__init__Ý   s4   ø€ Ü‰Ñ˜Ô+ð %×0Ñ0´4·9²9¸V×=UÑ=UÓ3VÕVˆÖr:   c                ó.  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[ P                  S[ P                  3,          R,          RS[S[,          R	S[ P                  /# )
r<   rn   rp   NÚposition_idsrq   Ú	use_cachero   r6   r=   )rr   rs   Ú
LongTensorr   rS   rt   r   r   )r?   r@   s   "€r8   rA   r©   ã   sœ   ø€ ÷ ñ á—|‘|ðñ Ÿ™ tÕ+ðñ ×&Ñ&¨Õ-ð	ñ
  �ðñ ˜$•;ðñ #¡5§<¡<±·±Ð#=Õ>ÀÕEðñ Ñ+Õ,ðñ 
�‰ñr:   c                ó  € TpV P                  V4      pV P                  ! RR VRVRVRVRVRV/VB w  rW�V P                  ,          ,           pTpV P                  V4      pV P	                  V4      pW�V P                  ,          ,           pV# )rn   rp   r¯   rq   r°   ro   r&   )Úinput_layernormÚ	self_attnr¬   Úpost_attention_layernormÚmlp)
r5   rn   rp   r¯   rq   r°   ro   r6   ÚresidualÚ_s
   &&&&&&&,  r8   r£   ÚMiniCPM3DecoderLayer.forwardã   s½   € ð !ˆØ×,Ñ,¨]Ó;ˆØŸ>š>ñ 
Ø'ð
à)ð
ð &ð
ð ,ð	
ð
  ð
ð !4ð
ð ñ
Ñˆð !°4×3FÑ3FÕ#FÕFˆà ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ °4×3FÑ3FÕ#FÕFˆØÐr:   )r¬   )NNNFN)	rT   rU   rV   rW   r«   r£   r]   r^   r_   r`   s   @@r8   r§   r§   Ü   s    ù‡ € ÷Wó W÷÷ ò r:   r§   c                   óN   a € ] tR tRt o ]P
                  ! 4       R 4       tRtV tR# )ÚMiniCPM3PreTrainedModeli  c                ó¸   € \         P                  ! W4       \        V\        4      '       d.   \        P
                  ! VP                  VP                  4       R # R # rD   )r
   Ú_init_weightsÚ
isinstancerb   ÚinitÚ	constant_Úembed_scaleÚscalar_embed_scale)r5   Úmodules   &&r8   r½   Ú%MiniCPM3PreTrainedModel._init_weights  s=   € ä×%Ò% dÔ3Ü�fÔ9×:Ò:Ü�NŠN˜6×-Ñ-¨v×/HÑ/HÖIñ ;r:   r&   N)	rT   rU   rV   rW   rr   Úno_gradr½   r]   r^   r¥   s   @r8   r»   r»     s   ø‡ € Ø
‡]‚]ƒ_ñJó öJr:   r»   c                   ó>   a a€ ] tR tRt oV3R lV 3R lltRtVtV ;t# )ÚMiniCPM3Modeli	  c                ó    <€ V ^8„  d   QhRS[ /# )r<   rŠ   )r   )r?   r@   s   "€r8   rA   ÚMiniCPM3Model.__annotate__  s   ø€ ÷ 
ñ 
™~ñ 
r:   c                óž   <€ \         SV `  V4       \        VP                  VP                  V P
                  VP                  R 7      V n        R# ))rÁ   N)r3   r«   rb   rH   r,   Úpadding_idxrQ   Úembed_tokens)r5   rŠ   r7   s   &&€r8   r«   ÚMiniCPM3Model.__init__  s?   ø€ Ü‰Ñ˜Ô ä7Ø×Ñ˜v×1Ñ1°4×3CÑ3CÐQW×QaÑQaô
ˆÖr:   )rÌ   )rT   rU   rV   rW   r«   r]   r^   r_   r`   s   @@r8   rÇ   rÇ   	  s   ù‡ € ÷
÷ 
ó 
r:   rÇ   c                   óJ   a € ] tR tRt o ]]RV 3R lR ll4       4       tRtV tR# )ÚMiniCPM3ForCausalLMi  Nc                óP  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[S[ P                  ,          R
S[S[,          RS[	/
# )r<   Ú	input_idsNrp   r¯   rq   Úinputs_embedsÚlabelsr°   Úlogits_to_keepr6   r=   )
rr   r±   rs   r   ÚFloatTensorrS   rR   r   r   r   )r?   r@   s   "€r8   rA   Ú MiniCPM3ForCausalLM.__annotate__  s»   ø€ ÷ 6
ñ 6
á×#Ñ# dÕ*ð6
ñ Ÿ™ tÕ+ð6
ñ ×&Ñ&¨Õ-ð	6
ñ
  �ð6
ñ ×(Ñ(¨4Õ/ð6
ñ × Ñ  4Õ'ð6
ñ ˜$•;ð6
ñ ™eŸl™lÕ*ð6
ñ Ñ+Õ,ð6
ñ 
 ñ6
r:   c	                óÎ  € V P                   ! RRVRVRVRVRVRV/V	B p
V
P                  pW°P                  P                  ,          p\	        V\
        4      '       d   \        V) R4      MTpV P                  VRVR3,          4      pRpVe-   V P                  ! RR	VR
VRV P                  P                  /V	B p\        VVV
P                  V
P                  V
P                  R7      # )a\  
Example:

```python
>>> from transformers import AutoTokenizer, MiniCPM3ForCausalLM

>>> model = MiniCPM3ForCausalLM.from_pretrained("openbmb/MiniCPM3-4B")
>>> tokenizer = AutoTokenizer.from_pretrained("openbmb/MiniCPM3-4B")

>>> prompt = "Hey, are you conscious? Can you talk to me?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
```rÑ   rp   r¯   rq   rÒ   r°   N:NNNÚlogitsrÓ   rH   )ÚlossrØ   rq   rn   Ú
attentionsr&   )ÚmodelÚlast_hidden_staterŠ   rE   r¾   rR   ÚsliceÚlm_headÚloss_functionrH   r   rq   rn   rÚ   )r5   rÑ   rp   r¯   rq   rÒ   rÓ   r°   rÔ   r6   Úoutputsrn   Úslice_indicesrØ   rÙ   s   &&&&&&&&&,     r8   r£   ÚMiniCPM3ForCausalLM.forward  s  € ð< ,0¯:ª:ñ ,
Øð,
à)ð,
ð &ð,
ð ,ð	,
ð
 (ð,
ð  ð,
ð ñ,
ˆð  ×1Ñ1ˆà%¯©×(BÑ(BÕBˆÜ8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒØ×%Ò%Ñp¨VÐp¸FÐpÈtÏ{É{×OeÑOeÐpÐioÑpˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r:   r&   )NNNNNNNr   )	rT   rU   rV   rW   r   r   r£   r]   r^   r¥   s   @r8   rÏ   rÏ     s$   ø‡ € àØ÷6
ñ 6
ó ó ö6
r:   rÏ   c                   ó   € ] tR tRtRtR# )Ú!MiniCPM3ForSequenceClassificationiP  r&   Nrc   r&   r:   r8   rä   rä   P  rd   r:   rä   )r   r»   rÇ   rÏ   rä   )8r/   Úcollections.abcr   rr   Úhuggingface_hub.dataclassesr   Ú r   r¿   Úcache_utilsr   Úmodeling_outputsr   r   Úmodeling_utilsr	   r
   Úprocessing_utilsr   Úutilsr   r   r   r   Ú deepseek_v2.modeling_deepseek_v2r   Úgemma3.modeling_gemma3r   Úllama.configuration_llamar   Úllama.modeling_llamar   r   r   r   r   r   r   r   r   r   Ú
get_loggerrT   Úloggerr   rb   rf   rh   rj   rl   r§   r»   rÇ   rÏ   rä   Ú__all__r&   r:   r8   Ú<module>rô      s(  ðó Ý $ã Ý .å &Ý  ß Oß FÝ &ß RÓ RÝ BÝ BÝ 3÷÷ ÷ ð 
×	Ò	˜HÓ	%€ñ Ð0Ô1ØôW6�[ó W6ó ó 2ðW6ôt	Ð"?ô 	ô	�lô 	ô	Ð2ô 	ô	�(ô 	ô>)Ð+ô >)ôB"Ð,ô "ôJJÐ2ô Jð ô
�Jó 
ó ð
ð ô9
Ð*ó 9
ó ð9
ôx	Ð(Fô 	ò‚r:   