+
    œ°³jäW  ã                   ó¤  € ^ RI Ht ^ RIt^ RIHt ^RIHt ^RIHtH	t	 ^RI
Ht ^RIHt ^RIHt ^RIHt ^R	IHtHtHt ^R
IHtHt ^RIHtHt ^RIHtHt ^RIH t  ^RI!H"t"H#t#H$t$ ^RI%H&t& ^RI'H(t(H)t) ^RI*H+t+ ^RI,H-t-  ! R R]P\                  4      t/ ! R R]4      t0R R lt1R.R R llt2R t3R/R lt4 ! R R]P\                  4      t5 ! R R ]P\                  4      t6]! R!4       ! R" R#]P\                  4      4       t7]# ! R$ R%]4      4       t8]# ! R& R']84      4       t9]# ! R( R)]8]4      4       t: ! R* R+]]84      t; ! R, R-]]84      t<. R0Ot=R# )1é    )ÚCallableN)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_kernel_forward_from_hub)Úcreate_causal_mask)ÚFlashAttentionKwargs)Ú GenericForSequenceClassificationÚGenericForTokenClassificationÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputs©Ú
Glm4Configc                   óD   a a€ ] tR t^1t oV 3R ltV3R lR ltRtVtV ;t# )ÚGlm4MLPc                ó>  <€ \         SV `  4        Wn        \        P                  ! VP
                  ^VP                  ,          RR7      V n        \        P                  ! VP                  VP
                  RR7      V n        \        VP                  ,          V n        R# )é   F©ÚbiasN)ÚsuperÚ__init__ÚconfigÚnnÚLinearÚhidden_sizeÚintermediate_sizeÚgate_up_projÚ	down_projr   Ú
hidden_actÚactivation_fn©Úselfr&   Ú	__class__s   &&€Úg/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/glm4/modeling_glm4.pyr%   ÚGlm4MLP.__init__2   sn   ø€ Ü‰ÑÔàŒÜŸIšI f×&8Ñ&8¸!¸f×>VÑ>VÕ:VÐ]bÔcˆÔÜŸš 6×#;Ñ#;¸V×=OÑ=OÐV[Ô\ˆŒÜ# F×$5Ñ$5Õ6ˆÖó    c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r!   Úhidden_statesÚreturn)ÚtorchÚFloatTensor)ÚformatÚ__classdict__s   "€r2   Ú__annotate__ÚGlm4MLP.__annotate__:   s'   ø€ ÷ )ñ )¡U×%6Ñ%6ð )¹5×;LÑ;Lñ )r4   c                óž   € V P                  V4      pVP                  ^RR7      w  r2W P                  V4      ,          pV P                  V4      # )r!   ©Údiméÿÿÿÿ)r+   Úchunkr.   r,   )r0   r7   Ú	up_statesÚgates   &&  r2   ÚforwardÚGlm4MLP.forward:   sH   € Ø×%Ñ% mÓ4ˆ	à#Ÿ/™/¨!°˜/Ó4‰ˆØ× 2Ñ 2°4Ó 8Õ8ˆ	à�~‰~˜iÓ(Ð(r4   )r.   r&   r,   r+   ©	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r%   rF   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r1   r<   s   @@r2   r   r   1   s   ù‡ € õ7÷)÷ )ð )r4   r   c                   óT   a a€ ] tR t^Ct oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚGlm4DecoderLayerc                ó&   <€ V ^8„  d   QhRS[ RS[/# )r!   r&   Ú	layer_idx©r   Úint)r;   r<   s   "€r2   r=   ÚGlm4DecoderLayer.__annotate__D   s   ø€ ÷ 	[ñ 	[™zð 	[±cñ 	[r4   c                ó¸  <€ \         SV `  4        VP                  V n        \        WR 7      V n        \        V4      V n        \        VP                  VP                  R7      V n	        \        VP                  VP                  R7      V n
        \        VP                  VP                  R7      V n        \        VP                  VP                  R7      V n        R# ))r&   rT   ©ÚepsN)r$   r%   r)   ÚGlm4AttentionÚ	self_attnr   ÚmlpÚGlm4RMSNormÚrms_norm_epsÚinput_layernormÚpost_attention_layernormÚpost_self_attn_layernormÚpost_mlp_layernorm©r0   r&   rT   r1   s   &&&€r2   r%   ÚGlm4DecoderLayer.__init__D   sŸ   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔÜ&¨fÔJˆŒä˜6“?ˆŒÜ*¨6×+=Ñ+=À6×CVÑCVÔWˆÔÜ(3°F×4FÑ4FÈF×L_ÑL_Ô(`ˆÔ%Ü(3°F×4FÑ4FÈF×L_ÑL_Ô(`ˆÔ%Ü"-¨f×.@Ñ.@Àf×FYÑFYÔ"ZˆÖr4   c                ó�  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[ P                  S[ P                  3,          R,          RS[S[,          R	S[S[ P                  S[S[ P                  S[ P                  3,          R,          3,          /# )
r!   r7   Úattention_maskNÚposition_idsÚpast_key_valuesÚ	use_cacheÚposition_embeddingsÚkwargsr8   )	r9   ÚTensorÚ
LongTensorr   ÚboolÚtupler   r
   r:   )r;   r<   s   "€r2   r=   rW   O   sÄ   ø€ ÷ ñ á—|‘|ðñ Ÿ™ tÕ+ðñ ×&Ñ&¨Õ-ð	ñ
  �ðñ ˜$•;ðñ #¡5§<¡<±·±Ð#=Õ>ÀÕEðñ Ñ-Õ.ðñ 
‰u× Ñ ¡%©×(9Ñ(9¹5×;LÑ;LÐ(LÕ"MÐPTÕ"TÐTÕ	Uñr4   c                ó  € TpV P                  V4      pV P                  ! RR VRVRVRVRVRV/VB w  rV P                  V4      pW�,           pTpV P                  V4      pV P	                  V4      pV P                  V4      pW�,           pV# )r7   rg   rh   ri   rj   rk   © )r`   r\   rb   ra   r]   rc   )
r0   r7   rg   rh   ri   rj   rk   rl   ÚresidualÚ_s
   &&&&&&&,  r2   rF   ÚGlm4DecoderLayer.forwardO   sÇ   € ð !ˆØ×,Ñ,¨]Ó;ˆàŸ>š>ñ 
Ø'ð
à)ð
ð &ð
ð ,ð	
ð
  ð
ð !4ð
ð ñ
Ñˆð ×5Ñ5°mÓDˆØ Õ0ˆà ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ×/Ñ/°Ó>ˆØ Õ0ˆØÐr4   )r)   r`   r]   ra   rc   rb   r\   )NNNFNrH   rP   s   @@r2   rR   rR   C   s    ù‡ € ÷	[ó 	[÷÷ ò r4   rR   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r!   r7   Ún_repr8   )r9   rm   rV   )r;   s   "r2   r=   r=   q   s.   € ÷ 	Uñ 	UœUŸ\™\ð 	U´#ð 	U¼%¿,¹,ñ 	Ur4   c                ó˜   € V P                   w  r#rEV^8X  d   V # V R,          P                  W#WV4      p V P                  W#V,          WE4      # )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
)ºNNNry   Nry   ry   )ÚshapeÚexpandÚreshape)r7   rw   ÚbatchÚnum_key_value_headsÚslenÚhead_dims   &&    r2   Ú	repeat_kvr�   q   sU   € ð
 2?×1DÑ1DÑ.€E Ø�„zØÐØ!Ð"2Õ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÕ(CÀTÓTÐTr4   c                ó   € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R\        R	\
        \        ,          /# )
r!   ÚmoduleÚqueryÚkeyÚvaluerg   NÚscalingÚdropoutrl   )r'   ÚModuler9   rm   Úfloatr   r   )r;   s   "r2   r=   r=   }   sw   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ð%ô ð%ô Ô'Õ(ñ%r4   c                 ó"  € \        W P                  4      p\        W0P                  4      p	\        P                  ! WP	                  ^^4      4      V,          p
Ve	   W¤,           p
\
        P                  P                  V
R\        P                  R7      P                  VP                  4      p
\
        P                  P                  W¦V P                  R7      p
\        P                  ! W©4      pVP	                  ^^4      P                  4       pWº3# )r!   )rA   Údtype)ÚpÚtrainingrB   )r�   Únum_key_value_groupsr9   ÚmatmulÚ	transposer'   Ú
functionalÚsoftmaxÚfloat32ÚtorŒ   rˆ   rŽ   Ú
contiguous)rƒ   r„   r…   r†   rg   r‡   rˆ   rl   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs   &&&&&&&,    r2   Úeager_attention_forwardr›   }   sÒ   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÕL€LØÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r4   c                óz   € V R,          pV R,          p\         P                  ! V) V3RR7      P                  R4      # )z*Rotates half the hidden dims of the input.r@   ).:r   Nr!   ).:é   Nr!   rB   éþÿÿÿ)r9   ÚstackÚflatten)ÚxÚx1Úx2s   &  r2   Úrotate_halfr¤   –   s6   € à	
ˆ9�€BØ	
ˆ9�€BÜ�;Š;˜˜˜R�y bÔ)×1Ñ1°"Ó5Ð5r4   c                óŽ  € VP                  V4      pVP                  V4      pVRRVP                  R,          ^,          13,          P                  ^RR7      pVRRVP                  R,          ^,          13,          P                  ^RR7      pVP                  R,          pV RRV13,          V RVR13,          rvVRRV13,          VRVR13,          r˜Wb,          \        V4      V,          ,           p
W‚,          \        V4      V,          ,           p\        P
                  ! W§.RR7      p
\        P
                  ! W¹.RR7      pW«3# )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
.Nr@   rB   )Ú	unsqueezerz   Úrepeat_interleaver¤   r9   Úcat)ÚqÚkÚcosÚsinÚunsqueeze_dimÚ
rotary_dimÚq_rotÚq_passÚk_rotÚk_passÚq_embedÚk_embeds   &&&&&       r2   Úapply_rotary_pos_embrµ   �   s6  € ð$ �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€Cð ˆcÐ'�S—Y‘Y˜r•] aÕ'Ð'Ð'Õ
(×
:Ñ
:¸1À"Ð
:Ó
E€CØ
ˆcÐ'�S—Y‘Y˜r•] aÕ'Ð'Ð'Õ
(×
:Ñ
:¸1À"Ð
:Ó
E€Cð —‘˜2•€JØ�c˜;˜J˜;Ð&Õ'¨¨3°
±Ð+;Õ)<ˆ6Ø�c˜;˜J˜;Ð&Õ'¨¨3°
±Ð+;Õ)<ˆ6ð �{œ{¨5Ó1°CÕ7Õ8€GØ�{œ{¨5Ó1°CÕ7Õ8€Gô �iŠi˜Ð)¨rÔ2€GÜ�iŠi˜Ð)¨rÔ2€GØÐÐr4   c                   ó\   a a€ ] tR t^Åt oRtRV3R lV 3R llltRV3R lR lltRtVtV ;t	# )	r[   z=Multi-headed attention from 'Attention Is All You Need' paperc                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# )r!   r&   rT   NrU   )r;   r<   s   "€r2   r=   ÚGlm4Attention.__annotate__È   s#   ø€ ÷ lñ l™zð l±c¸Dµjñ lr4   c                óŽ  <€ \         SV `  4        Wn        W n        \	        VR VP
                  VP                  ,          4      V n        VP                  VP                  ,          V n	        V P                  R,          V n
        VP                  V n        RV n        \        P                  ! VP
                  VP                  V P                  ,          VP                  R7      V n        \        P                  ! VP
                  VP                  V P                  ,          VP                  R7      V n        \        P                  ! VP
                  VP                  V P                  ,          VP                  R7      V n        \        P                  ! VP                  V P                  ,          VP
                  RR7      V n        R# )r€   Tr"   FNg      à¿)r$   r%   r&   rT   Úgetattrr)   Únum_attention_headsr€   r~   r�   r‡   Úattention_dropoutÚ	is_causalr'   r(   Úattention_biasÚq_projÚk_projÚv_projÚo_projrd   s   &&&€r2   r%   ÚGlm4Attention.__init__È   s@  ø€ Ü‰ÑÔØŒØ"ŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÕ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Õ$\ˆÔ!Ø—}‘} dÕ*ˆŒØ!'×!9Ñ!9ˆÔØˆŒä—i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’i × :Ñ :¸T¿]¹]Õ JÈF×L^ÑL^ÐejÔkˆŽr4   c                ó  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          R,          RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  S[ P                  3,          /# )r!   r7   rk   Nrg   ri   rl   r8   )r9   rm   rp   r   r   r   )r;   r<   s   "€r2   r=   r¸   Ý   s‡   ø€ ÷ &)ñ &)á—|‘|ð&)ñ #¡5§<¡<±·±Ð#=Õ>ÀÕEð&)ñ Ÿ™ tÕ+ð	&)ñ
  �ð&)ñ Ñ+Õ,ð&)ñ 
‰u�|‰|™UŸ\™\Ð)Õ	*ñ&)r4   c                ó  € VP                   R R p. VORNV P                  N5pV P                  V4      P                  V4      P	                  ^^4      pV P                  V4      P                  V4      P	                  ^^4      p	V P                  V4      P                  V4      P	                  ^^4      p
Vw  r¼\        W‰W¼4      w  r‰Ve   VP                  WšV P                  4      w  rš\        P                  ! V P                  P                  \        4      pT! V VV	V
V3RV P                  '       g   RMV P                   RV P"                  /VB w  rïVP$                  ! . VORN5!  P'                  4       pV P)                  V4      pWï3# )Nrˆ   ç        r‡   rB   )rz   r€   r¿   Úviewr‘   rÀ   rÁ   rµ   ÚupdaterT   r   Úget_interfacer&   Ú_attn_implementationr›   rŽ   r¼   r‡   r|   r–   rÂ   )r0   r7   rk   rg   ri   rl   Úinput_shapeÚhidden_shapeÚquery_statesr—   r˜   r«   r¬   Úattention_interfacerš   r™   s   &&&&&,          r2   rF   ÚGlm4Attention.forwardÝ   s‡  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—k‘k +Ó.ˆØÐ(Ð(r4   )r¼   r&   r€   r½   rÀ   rT   r�   rÂ   r¿   r‡   rÁ   ©N)NNN)
rI   rJ   rK   rL   Ú__doc__r%   rF   rM   rN   rO   rP   s   @@r2   r[   r[   Å   s#   ù‡ € ÙG÷lõ l÷*&)÷ &)ò &)r4   r[   c                   óÎ   a a€ ] tR tRt o]! RRR7      RV3R lV 3R lll4       t]]! RRR7      RV3R lR ll4       4       t]P                  ! 4       ]
R	 4       4       tR
tVtV ;t# )ÚGlm4RotaryEmbeddingi  Údevicez5.18)Úversionc                ó    <€ V ^8„  d   QhRS[ /# ©r!   r&   r   )r;   r<   s   "€r2   r=   Ú Glm4RotaryEmbedding.__annotate__  s   ø€ ÷ Oñ O™zñ Or4   c                óä  <€ \         SV `  4        VP                  V n        VP                  V n        Wn        V P
                  P                  R ,          V n        V P                  pV P                  R8w  d   \        V P                  ,          pV! V P
                  V4      w  q@n
        \        P                  ! VRR7      V n        \        P                  ! VP                  4       RR7      V n        R# )Ú	rope_typeÚdefaultF)Ú
persistentN)r$   r%   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr&   Úrope_parametersrÚ   Úcompute_default_rope_parametersr   Úattention_scalingr'   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)r0   r&   rÔ   Úrope_init_fnrä   r1   s   &&&  €r2   r%   ÚGlm4RotaryEmbedding.__init__  s¬   ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àŒàŸ™×4Ñ4°[ÕAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÔ&Ü.¨t¯~©~Õ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÔ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÔ!NˆÖr4   c                óP   <€ V ^8„  d   QhRS[ RS[S[P                  S[3,          /# )r!   r&   r8   )r   rp   r9   rm   rŠ   )r;   r<   s   "€r2   r=   rØ     s.   ø€ ÷ 5ñ 5±
ð 5ÑV[Ñ\a×\hÑ\hÑjoÐ\oÕVpñ 5r4   c           	     óŠ  € V P                   R,          pV P                   P                  RR4      p\        V RR4      ;'       g    V P                  V P                  ,          p\        WT,          4      pRpRV\        P                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚpartial_rotary_factorg      ð?r€   N©rŒ   )
rà   Úgetrº   r)   r»   rV   r9   ÚarangerŠ   r•   )	r&   rÔ   rl   Úbaserì   r€   rA   Úattention_factorrä   s	   &&,      r2   rá   Ú3Glm4RotaryEmbedding.compute_default_rope_parameters  s¦   € ð ×%Ñ% lÕ3ˆØ &× 6Ñ 6× :Ñ :Ð;RÐTWÓ XÐÜ˜6 :¨tÓ4×hÐh¸×8JÑ8JÈf×NhÑNhÕ8hˆÜ�(Õ2Ó3ˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r4   c                ó^  € V P                   R,          P                  VP                  ^ ,          R	^4      P                  \        P
                  VP                  R7      pVR
,          P                  4       p\        VP                  P                  \        4      '       d2   VP                  P                  R8w  d   VP                  P                  MRp\        VRR7      ;_uu_ 4        W4,          P                  ^^4      p\        P                  ! Wf3R	R7      pVP                  4       V P                  ,          pVP                  4       V P                  ,          p	R R R 4       XP                  VP                   R7      X	P                  VP                   R7      3#   + '       g   i     LH; i)N)rŒ   rÔ   ÚmpsÚcpuF)Údevice_typeÚenabledr@   rí   )Nry   NrB   )ry   Nry   )rä   r{   rz   r•   r9   rŠ   rÔ   Ú
isinstanceÚtypeÚstrr   r‘   r¨   r«   râ   r¬   rŒ   )
r0   r¡   rh   Úinv_freq_expandedÚposition_ids_expandedrö   ÚfreqsÚembr«   r¬   s
   &&&       r2   rF   ÚGlm4RotaryEmbedding.forward.  s2  € ð �M‰M˜-Õ(×/Ñ/°×0BÑ0BÀ1Õ0EÀrÈ1ÓM×PÑPÔW\×WbÑWbÐkl×ksÑksÐPÓtð 	ð !-¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆä¨¸U×CÖCØ&Õ>×IÑIÈ!ÈQÓOˆEÜ—)’)˜U˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆC÷	 Dð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷ D×Cús   Ã(A4FÆF,	)râ   r&   rä   rÞ   ræ   rß   rÚ   rÐ   )rI   rJ   rK   rL   r   r%   Ústaticmethodrá   r9   Úno_gradr   rF   rM   rN   rO   rP   s   @@r2   rÓ   rÓ     su   ù‡ € Ù�X vÔ.÷Oô Oó /ðOð  Ù�X vÔ.÷5ñ 5ó /ó ð5ð( ‡]‚]ƒ_Øñ<ó ó ÷<ð <r4   rÓ   ÚRMSNormc                   óZ   a a€ ] tR tRt oRV3R lV 3R llltV3R lR ltR tRtVtV ;t	# )	r^   iA  c                ó$   <€ V ^8„  d   QhRS[ RR/# )r!   rZ   r8   N)rŠ   )r;   r<   s   "€r2   r=   ÚGlm4RMSNorm.__annotate__C  s   ø€ ÷ $ñ $©ð $¸$ñ $r4   c                óŽ   <€ \         SV `  4        \        P                  ! \        P
                  ! V4      4      V n        W n        R# )z*
Glm4RMSNorm is equivalent to T5LayerNorm
N)r$   r%   r'   Ú	Parameterr9   ÚonesÚweightÚvariance_epsilon)r0   r)   rZ   r1   s   &&&€r2   r%   ÚGlm4RMSNorm.__init__C  s/   ø€ ô 	‰ÑÔÜ—l’l¤5§:¢:¨kÓ#:Ó;ˆŒØ #Ör4   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r6   )r9   rm   )r;   r<   s   "€r2   r=   r  K  s#   ø€ ÷ ;ñ ;¡U§\¡\ð ;±e·l±lñ ;r4   c                ó:  € VP                   pVP                  \        P                  4      pVP	                  ^4      P                  RRR7      pV\        P                  ! W0P                  ,           4      ,          pV P                  VP                  V4      ,          # )r!   T)ÚkeepdimrB   )	rŒ   r•   r9   r”   ÚpowÚmeanÚrsqrtr
  r	  )r0   r7   Úinput_dtypeÚvariances   &&  r2   rF   ÚGlm4RMSNorm.forwardK  sw   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯ª°H×?TÑ?TÕ4TÓ(UÕUˆØ�{‰{˜]×-Ñ-¨kÓ:Õ:Ð:r4   c                ó^   € \        V P                  P                  4       R V P                   2# )z, eps=)rp   r	  rz   r
  )r0   s   &r2   Ú
extra_reprÚGlm4RMSNorm.extra_reprR  s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr4   )r
  r	  )g�íµ ÷Æ°>)
rI   rJ   rK   rL   r%   rF   r  rM   rN   rO   rP   s   @@r2   r^   r^   A  s%   ù‡ € ÷$õ $÷;ð ;÷Jò Jr4   r^   c                   ó`   a € ] tR tRt o RtRtR.tR.tRtRt	Rt
RtRtR]R]/tV 3R ltR	tV tR
# )ÚGlm4PreTrainedModeliV  ÚmodelTrR   ri   r7   Ú
attentionsc                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r×   r   )r;   r<   s   "€r2   r=   Ú Glm4PreTrainedModel.__annotate__V  s   ø‡ ‚ áÑò r4   rr   N)rI   rJ   rK   rL   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendrR   r[   Ú_can_record_outputsÚ__annotate_func__rM   rN   )r<   s   @r2   r  r  V  s\   ø‡ € ð  ÐØ&*Ð#Ø+Ð,ÐØ#4Ð"5ÐØÐØ€NØÐà!ÐØ"&ÐàÐ)Ø�mðÐ÷ ƒ r4   r  c                   ór   a a€ ] tR tRt oV3R lV 3R llt]]]RV3R lR ll4       4       4       tRt	Vt
V ;t# )Ú	Glm4Modelii  c                ó    <€ V ^8„  d   QhRS[ /# r×   r   )r;   r<   s   "€r2   r=   ÚGlm4Model.__annotate__k  s   ø€ ÷ ñ ™zñ r4   c           	     ó  <€ \         SV `  V4       VP                  V n        VP                  V n        \
        P                  ! VP                  VP                  V P                  4      V n        \
        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n        \        VP                  VP                  R 7      V n        \#        VR7      V n        RV n        V P)                  4        R# u upi )rY   ©r&   FN)r$   r%   Úpad_token_idÚpadding_idxÚ
vocab_sizer'   Ú	Embeddingr)   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersrR   Úlayersr^   r_   ÚnormrÓ   Ú
rotary_embÚgradient_checkpointingÚ	post_initrd   s   && €r2   r%   ÚGlm4Model.__init__k  sÊ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸLšL¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m’mÜBGÈ×H`ÑH`ÔBaÓbÑBa°YÔ˜fÖ0ÑBaÑbó
ˆŒô   × 2Ñ 2¸×8KÑ8KÔLˆŒ	Ü-°VÔ<ˆŒØ&+ˆÔ#ð 	�‰Öùò cs   ÂDc                óþ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[S[,          R	S[/# )
r!   Ú	input_idsNrg   rh   ri   Úinputs_embedsrj   rl   r8   )	r9   rn   rm   r   r:   ro   r   r   r   )r;   r<   s   "€r2   r=   r,  ~  s“   ø€ ÷ 2
ñ 2
á×#Ñ# dÕ*ð2
ñ Ÿ™ tÕ+ð2
ñ ×&Ñ&¨Õ-ð	2
ñ
  �ð2
ñ ×(Ñ(¨4Õ/ð2
ñ ˜$•;ð2
ñ Ñ+Õ,ð2
ñ 
!ñ2
r4   c                óˆ  € VR J VR J,          '       d   \        R4      hVf   V P                  V4      pV'       d   Vf   \        V P                  R7      pVfb   Ve   VP	                  4       M^ p\
        P                  ! VP                  ^,          VP                  R7      V,           pVP                  ^ 4      p\        V P                  VVVVR7      p	Tp
V P                  W£R7      pV P                  R V P                  P                    F  pV! V
3RV	RVRVR	VR
V/VB p
K  	  V P                  V
4      p
\        V
VR7      # )Nz:You must specify exactly one of input_ids or inputs_embedsr.  )rÔ   )r&   r?  rg   ri   rh   )rh   rg   rk   rh   ri   rj   )Úlast_hidden_stateri   )Ú
ValueErrorr3  r   r&   Úget_seq_lengthr9   rï   rz   rÔ   r¦   r	   r9  r7  r6  r8  r   )r0   r>  rg   rh   ri   r?  rj   rl   Úpast_seen_tokensÚcausal_maskr7   rk   Údecoder_layers   &&&&&&&,     r2   rF   ÚGlm4Model.forward{  se  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ Ø*.×*;Ñ*;¸IÓ*FˆMç˜Ò0Ü*°$·+±+Ô>ˆOàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLä(Ø—;‘;Ø'Ø)Ø+Ø%ô
ˆð &ˆØ"Ÿo™o¨m˜oÓWÐà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÓIˆMÙ)Øñà*ðð %8ðð *ð	ð
 !0ðð $ðð ñŠMñ Jð Ÿ	™	 -Ó0ˆÜ&Ø+Ø+ô
ð 	
r4   )r3  r:  r7  r8  r0  r9  r1  )NNNNNN)rI   rJ   rK   rL   r%   r   r   r   rF   rM   rN   rO   rP   s   @@r2   r*  r*  i  s=   ù‡ € ÷ó ð   ØØ÷2
ñ 2
ó ó ó  ÷2
ð 2
r4   r*  c                   ó„   a a€ ] tR tRt oRR/tRR/tRR.R.3/tRR/tV 3R	 lt]	]
RV3R
 lR ll4       4       tRtVtV ;t# )ÚGlm4ForCausalLMi³  zlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputr7   ÚlogitsÚkeep_full_weightc                óì   <€ \         SV `  V4       \        V4      V n        VP                  V n        \
        P                  ! VP                  VP                  R R7      V n        V P                  4        R# )Fr"   N)
r$   r%   r*  r  r1  r'   r(   r)   rJ  r;  r/   s   &&€r2   r%   ÚGlm4ForCausalLM.__init__º  sU   ø€ Ü‰Ñ˜Ô Ü˜vÓ&ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒð 	�‰Ör4   c                ó`  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[S[ P                  ,          R
S[S[,          RS[	S[
,          /
# )r!   r>  Nrg   rh   ri   r?  Úlabelsrj   Úlogits_to_keeprl   r8   )r9   rn   rm   r   r:   ro   rV   r   r   rp   r   )r;   r<   s   "€r2   r=   ÚGlm4ForCausalLM.__annotate__Å  sÁ   ø€ ÷ ;
ñ ;
á×#Ñ# dÕ*ð;
ñ Ÿ™ tÕ+ð;
ñ ×&Ñ&¨Õ-ð	;
ñ
  �ð;
ñ ×(Ñ(¨4Õ/ð;
ñ × Ñ  4Õ'ð;
ñ ˜$•;ð;
ñ ™eŸl™lÕ*ð;
ñ Ñ+Õ,ð;
ñ 
Ñ'Õ	'ñ;
r4   c	                ó–  € V P                   ! RRVRVRVRVRVRV/V	B p
V
P                  p\        V\        4      '       d   \	        V) R4      MTpV P                  VRVR3,          4      pRpVe-   V P                  ! RR	VR
VRV P                  P                  /V	B p\        VVV
P                  V
P                  V
P                  R7      # )aè  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

Example:

```python
>>> from transformers import AutoTokenizer, Glm4ForCausalLM

>>> model = Glm4ForCausalLM.from_pretrained("THUDM/GLM-4-9B-0414")
>>> tokenizer = AutoTokenizer.from_pretrained("THUDM/GLM-4-9B-0414")

>>> prompt = "Hey, are you conscious? Can you talk to me?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
```r>  rg   rh   ri   r?  rj   Nry   rL  rQ  r1  )ÚlossrL  ri   r7   r  rr   )r  rA  rø   rV   ÚslicerJ  Úloss_functionr&   r1  r   ri   r7   r  )r0   r>  rg   rh   ri   r?  rQ  rj   rR  rl   Úoutputsr7   Úslice_indicesrL  rU  s   &&&&&&&&&,     r2   rF   ÚGlm4ForCausalLM.forwardÃ  s   € ðH ,0¯:ª:ñ ,
Øð,
à)ð,
ð &ð,
ð ,ð	,
ð
 (ð,
ð  ð,
ð ñ,
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒØ×%Ò%Ñp¨VÐp¸FÐpÈtÏ{É{×OeÑOeÐpÐioÑpˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r4   )rJ  r  r1  )NNNNNNNr   )rI   rJ   rK   rL   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr%   r   r   rF   rM   rN   rO   rP   s   @@r2   rI  rI  ³  sf   ù‡ € à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÐ/Ð0€Jõð Ø÷;
ñ ;
ó ó ÷;
ð ;
r4   rI  c                   ó   € ] tR tRtRtR# )ÚGlm4ForSequenceClassificationi  rr   N©rI   rJ   rK   rL   rM   rr   r4   r2   r`  r`    ó   † Ûr4   r`  c                   ó   € ] tR tRtRtR# )ÚGlm4ForTokenClassificationi  rr   Nra  rr   r4   r2   rd  rd    rb  r4   rd  )rÆ   )r�   )r  r*  rI  r`  rd  )>Úcollections.abcr   r9   Útorch.nnr'   Úactivationsr   Úcache_utilsr   r   Ú
generationr   Úintegrationsr   Úmasking_utilsr	   Úmodeling_flash_attention_utilsr
   Úmodeling_layersr   r   r   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   Úconfiguration_glm4r   r‰   r   rR   r�   r›   r¤   rµ   r[   rÓ   r^   r  r*  rI  r`  rd  Ú__all__rr   r4   r2   Ú<module>rx     sN  ðõ, %ã Ý å !ß .Ý )Ý 7Ý /Ý B÷ñ ÷
 Pß Kß FÝ &ß IÑ IÝ 0ß GÝ 5Ý *ô)ˆb�i‰iô )ô$+Ð1ô +õ\	U÷%ò26ô%ôP>)�B—I‘Iô >)ôB8<˜"Ÿ)™)ô 8<ñv ˜YÓ'ôJ�"—)‘)ó Jó (ðJð( ô˜/ó ó ðð$ ôF
Ð#ó F
ó ðF
ðR ôL
Ð)¨?ó L
ó ðL
ô^	Ð$DÐFYô 	ô	Ð!>Ð@Sô 	ò‚r4   