+
    œ°³j¢  ã                   óJ  € ^ RI t ^RIHt ^RIHt ^RIHt ^RIHt ^RI	H
t
 ^RIHtHt ^RIHtHtHtHt ^R	IHt ^R
IHt ^RIHt ]P2                  ! ]4      tRt ! R R]4      t ! R R]4      t ! R R]4      t ! R R]4      t  ! R R]4      t! ! R R]4      t". ROt#R# )é    N)ÚCache)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚCausalLMOutputWithPast)ÚUnpack)ÚTransformersKwargsÚlogging)ÚGlmAttentionÚGlmForCausalLMÚGlmForSequenceClassificationÚGlmForTokenClassification)ÚPhi3MLP)Ú
Glm4Config)ÚGlm4RMSNormzTHUDM/GLM-4-9B-0414c                   ó   € ] tR t^#tRtR# )ÚGlm4MLP© N©Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__static_attributes__r   ó    Úf/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/glm4/modular_glm4.pyr   r   #   ó   † Ûr   r   c                   óT   a a€ ] tR t^'t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚGlm4DecoderLayerc                ó&   <€ V ^8„  d   QhRS[ RS[/# )é   ÚconfigÚ	layer_idx)r   Úint)ÚformatÚ__classdict__s   "€r   Ú__annotate__ÚGlm4DecoderLayer.__annotate__(   s   ø€ ÷ 	[ñ 	[™zð 	[±cñ 	[r   c                ó¸  <€ \         SV `  4        VP                  V n        \        WR 7      V n        \        V4      V n        \        VP                  VP                  R7      V n	        \        VP                  VP                  R7      V n
        \        VP                  VP                  R7      V n        \        VP                  VP                  R7      V n        R# ))r!   r"   )ÚepsN)ÚsuperÚ__init__Úhidden_sizeÚGlm4AttentionÚ	self_attnr   Úmlpr   Úrms_norm_epsÚinput_layernormÚpost_attention_layernormÚpost_self_attn_layernormÚpost_mlp_layernorm)Úselfr!   r"   Ú	__class__s   &&&€r   r+   ÚGlm4DecoderLayer.__init__(   sŸ   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔÜ&¨fÔJˆŒä˜6“?ˆŒÜ*¨6×+=Ñ+=À6×CVÑCVÔWˆÔÜ(3°F×4FÑ4FÈF×L_ÑL_Ô(`ˆÔ%Ü(3°F×4FÑ4FÈF×L_ÑL_Ô(`ˆÔ%Ü"-¨f×.@Ñ.@Àf×FYÑFYÔ"ZˆÖr   c                ó�  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[ P                  S[ P                  3,          R,          RS[S[,          R	S[S[ P                  S[S[ P                  S[ P                  3,          R,          3,          /# )
r    Úhidden_statesÚattention_maskNÚposition_idsÚpast_key_valuesÚ	use_cacheÚposition_embeddingsÚkwargsÚreturn)	ÚtorchÚTensorÚ
LongTensorr   ÚboolÚtupler   r   ÚFloatTensor)r$   r%   s   "€r   r&   r'   3   sÄ   ø€ ÷ ñ á—|‘|ðñ Ÿ™ tÕ+ðñ ×&Ñ&¨Õ-ð	ñ
  �ðñ ˜$•;ðñ #¡5§<¡<±·±Ð#=Õ>ÀÕEðñ Ñ-Õ.ðñ 
‰u× Ñ ¡%©×(9Ñ(9¹5×;LÑ;LÐ(LÕ"MÐPTÕ"TÐTÕ	Uñr   c                ó  € TpV P                  V4      pV P                  ! RR VRVRVRVRVRV/VB w  rV P                  V4      pW�,           pTpV P                  V4      pV P	                  V4      pV P                  V4      pW�,           pV# )r9   r:   r;   r<   r=   r>   r   )r1   r.   r3   r2   r/   r4   )
r5   r9   r:   r;   r<   r=   r>   r?   ÚresidualÚ_s
   &&&&&&&,  r   ÚforwardÚGlm4DecoderLayer.forward3   sÇ   € ð !ˆØ×,Ñ,¨]Ó;ˆàŸ>š>ñ 
Ø'ð
à)ð
ð &ð
ð ,ð	
ð
  ð
ð !4ð
ð ñ
Ñˆð ×5Ñ5°mÓDˆØ Õ0ˆà ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ×/Ñ/°Ó>ˆØ Õ0ˆØÐr   )r,   r1   r/   r2   r4   r3   r.   )NNNFN)	r   r   r   r   r+   rJ   r   Ú__classdictcell__Ú__classcell__©r6   r%   s   @@r   r   r   '   s    ù‡ € ÷	[ó 	[÷÷ ò r   r   c                   ó   € ] tR t^UtRtR# )r-   r   Nr   r   r   r   r-   r-   U   r   r   r-   c                   ó>   a a€ ] tR t^Yt oV3R lV 3R lltRtVtV ;t# )ÚGlm4ForCausalLMc                óF   <€ V ^8„  d   QhRS[ S[,          RS[S[,          /# )r    Úsuper_kwargsr@   )r   r   rE   r   )r$   r%   s   "€r   r&   ÚGlm4ForCausalLM.__annotate__Z   s+   ø€ ÷ /ñ /áÑ1Õ2ð/ñ 
Ñ'Õ	'ñ/r   c                ó$   <€ \         SV `  ! R/ VB # )aè  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

Example:

```python
>>> from transformers import AutoTokenizer, Glm4ForCausalLM

>>> model = Glm4ForCausalLM.from_pretrained("THUDM/GLM-4-9B-0414")
>>> tokenizer = AutoTokenizer.from_pretrained("THUDM/GLM-4-9B-0414")

>>> prompt = "Hey, are you conscious? Can you talk to me?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
```r   )r*   rJ   )r5   rS   r6   s   &,€r   rJ   ÚGlm4ForCausalLM.forwardZ   s   ø€ ô4 ‰wŠÑ. Ñ.Ð.r   r   )r   r   r   r   rJ   r   rL   rM   rN   s   @@r   rQ   rQ   Y   s   ù‡ € ÷/÷ /ó /r   rQ   c                   ó   € ] tR t^wtRtR# )ÚGlm4ForSequenceClassificationr   Nr   r   r   r   rX   rX   w   r   r   rX   c                   ó   € ] tR t^{tRtR# )ÚGlm4ForTokenClassificationr   Nr   r   r   r   rZ   rZ   {   r   r   rZ   )ÚGlm4PreTrainedModelÚ	Glm4ModelrQ   rX   rZ   )$rA   Úcache_utilsr   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   Úprocessing_utilsr   Úutilsr   r	   Úglm.modeling_glmr
   r   r   r   Úphi3.modeling_phi3r   Úconfiguration_glm4r   Úmodeling_glm4r   Ú
get_loggerr   ÚloggerÚ_CHECKPOINT_FOR_DOCr   r   r-   rQ   rX   rZ   Ú__all__r   r   r   Ú<module>rk      s–   ðó  å  Ý BÝ 9Ý 6Ý &ß 0ß tÓ tÝ (Ý *Ý &ð 
×	Ò	˜HÓ	%€à+Ð ô	ˆgô 	ô+Ð1ô +ô\	�Lô 	ô/�nô /ô<	Ð$@ô 	ô	Ð!:ô 	ò‚r   