+
    œ°³jöR  ã                   ó¶  € ^ RI Ht ^ RIt^ RIHt ^RIHt ^RIHtHt ^RI	H
t
 ^RIHtHt ^RIHtHt ^R	IHt ^R
IHt ^RIHtHt ^RIHtHt ^RIHtHt ^RIHt ^RI H!t!H"t"H#t# ^RI$H%t% ^RI&H't'H(t( ^RI)H*t* ^RI+H,t,  ! R R]PZ                  4      t.R t/]! R4      R.R l4       t0R R lt1R/R R llt2]! ]04       ! R R]PZ                  4      4       t3]! R4       ! R  R!]PZ                  4      4       t4 ! R" R#]PZ                  4      t5 ! R$ R%]4      t6]" ! R& R']4      4       t7 ! R( R)]4      t8]" ! R* R+]74      4       t9]" ! R, R-]7]
4      4       t:. R0Ot;R# )1é    )ÚCallableN)Únn)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_kernel_forward_from_hubÚuse_kernelized_func)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputs©Ú	CwmConfigc                   óÎ   a a€ ] tR t^-t o]! RRR7      R
V3R lV 3R lll4       t]]! RRR7      R
V3R lR ll4       4       t]P                  ! 4       ]
R 4       4       tR	tVtV ;t# )ÚCwmRotaryEmbeddingÚdevicez5.18)Úversionc                ó    <€ V ^8„  d   QhRS[ /# ©é   Úconfigr   )ÚformatÚ__classdict__s   "€Úe/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/cwm/modeling_cwm.pyÚ__annotate__ÚCwmRotaryEmbedding.__annotate__/   s   ø€ ÷ Oñ O™yñ Oó    c                óä  <€ \         SV `  4        VP                  V n        VP                  V n        Wn        V P
                  P                  R ,          V n        V P                  pV P                  R8w  d   \        V P                  ,          pV! V P
                  V4      w  q@n
        \        P                  ! VRR7      V n        \        P                  ! VP                  4       RR7      V n        R# )Ú	rope_typeÚdefaultF)Ú
persistentN)ÚsuperÚ__init__Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr&   Úrope_parametersr.   Úcompute_default_rope_parametersr   Úattention_scalingr   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)Úselfr&   r!   Úrope_init_fnr:   Ú	__class__s   &&&  €r)   r2   ÚCwmRotaryEmbedding.__init__.   s¬   ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àŒàŸ™×4Ñ4°[ÕAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÔ&Ü.¨t¯~©~Õ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÔ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÔ!NˆÖr,   c                óP   <€ V ^8„  d   QhRS[ RS[S[P                  S[3,          /# )r%   r&   Úreturn)r   ÚtupleÚtorchÚTensorÚfloat)r'   r(   s   "€r)   r*   r+   A   s.   ø€ ÷ 5ñ 5±	ð 5ÑUZÑ[`×[gÑ[gÑinÐ[nÕUoñ 5r,   c           	     ó0  € V P                   R,          p\        V RR4      ;'       g    V P                  V P                  ,          pRpRV\        P
                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚhead_dimNg      ð?©Údtype)r6   ÚgetattrÚhidden_sizeÚnum_attention_headsrD   ÚarangerF   Úto)r&   r!   ÚkwargsÚbaseÚdimÚattention_factorr:   s   &&,    r)   r7   Ú2CwmRotaryEmbedding.compute_default_rope_parameters?   s~   € ð ×%Ñ% lÕ3ˆÜ�f˜j¨$Ó/×cÐc°6×3EÑ3EÈ×IcÑIcÕ3cˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r,   c                ó”  € V P                   R,          P                  4       P                  VP                  ^ ,          R^4      P	                  VP
                  4      pVR	,          P                  4       p\        VP
                  P                  \        4      '       d2   VP
                  P                  R8w  d   VP
                  P                  MRp\        VRR7      ;_uu_ 4        VP                  4       VP                  4       ,          P                  ^^4      p\        P                  ! Wf3RR7      pVP                  4       V P                  ,          pVP                  4       V P                  ,          p	R R R 4       XP	                  VP                   R7      X	P	                  VP                   R7      3#   + '       g   i     LH; i)
NÚmpsÚcpuF)Údevice_typeÚenabled©rS   rJ   )NºNNNNéÿÿÿÿ)r\   Nr\   )r:   rF   ÚexpandÚshaperP   r!   Ú
isinstanceÚtypeÚstrr   Ú	transposerD   ÚcatÚcosr8   ÚsinrK   )
r=   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedrY   ÚfreqsÚembre   rf   s
   &&&       r)   ÚforwardÚCwmRotaryEmbedding.forwardS   sE  € ð !ŸM™M¨-Õ8×>Ñ>Ó@×GÑGÈ×HZÑHZÐ[\ÕH]Ð_aÐcdÓe×hÑhÐij×iqÑiqÓrÐØ ,¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆÜ¨¸U×CÖCØ&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÕN×YÑYÐZ[Ð]^Ó_ˆEÜ—)’)˜U˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆC÷	 Dð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷ D×Cús   Ã&BF7Æ7G	)r8   r&   r:   r4   r<   r5   r.   ©N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r2   Ústaticmethodr7   rD   Úno_gradr   rm   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r?   r(   s   @@r)   r    r    -   su   ù‡ € Ù�X vÔ.÷Oô Oó /ðOð  Ù�X vÔ.÷5ñ 5ó /ó ð5ð$ ‡]‚]ƒ_Øñ<ó ó ÷<ð <r,   r    c                óÌ   € V RRV P                   R,          ^,          13,          pV RV P                   R,          ^,          R13,          p\        P                  ! V) V3RR7      # )z*Rotates half the hidden dims of the input..Nr[   r]   )r_   rD   rd   )rg   Úx1Úx2s   &  r)   Úrotate_halfr}   c   sZ   € à	
ˆ3Ð"�!—'‘'˜"•+ Õ"Ð"Ð"Õ	#€BØ	
ˆ3�—‘˜•˜qÕ Ñ"Ð"Õ	#€BÜ�9Š9�r�c˜2�Y BÔ'Ð'r,   Úrotary_pos_embc                óÈ   € VP                  V4      pVP                  V4      pW,          \        V 4      V,          ,           pW,          \        V4      V,          ,           pWV3# )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)Ú	unsqueezer}   )ÚqÚkre   rf   Úunsqueeze_dimÚq_embedÚk_embeds   &&&&&  r)   Úapply_rotary_pos_embr†   j   sS   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�wœ; q›>¨CÕ/Õ0€GØ�wœ; q›>¨CÕ/Õ0€GØÐÐr,   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r%   Úhidden_statesÚn_reprB   )rD   rE   Úint)r'   s   "r)   r*   r*   „   s.   € ÷ 	Uñ 	UœUŸ\™\ð 	U´#ð 	U¼%¿,¹,ñ 	Ur,   c                ó˜   € V P                   w  r#rEV^8X  d   V # V R,          P                  W#WV4      p V P                  W#V,          WE4      # )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
)r\   r\   Nr\   r\   )r_   r^   Úreshape)rˆ   r‰   ÚbatchÚnum_key_value_headsÚslenrI   s   &&    r)   Ú	repeat_kvr�   „   sU   € ð
 2?×1DÑ1DÑ.€E Ø�„zØÐØ!Ð"2Õ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÕ(CÀTÓTÐTr,   c                ó   € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R\        R	\
        \        ,          /# )
r%   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNÚscalingÚdropoutrQ   )r   ÚModulerD   rE   rF   r   r   )r'   s   "r)   r*   r*   �   sw   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ð%ô ð%ô Ô'Õ(ñ%r,   c                 ó"  € \        W P                  4      p\        W0P                  4      p	\        P                  ! WP	                  ^^4      4      V,          p
Ve	   W¤,           p
\
        P                  P                  V
R\        P                  R7      P                  VP                  4      p
\
        P                  P                  W¦V P                  R7      p
\        P                  ! W©4      pVP	                  ^^4      P                  4       pWº3# )r%   )rS   rK   )ÚpÚtrainingr]   )r�   Únum_key_value_groupsrD   Úmatmulrc   r   Ú
functionalÚsoftmaxÚfloat32rP   rK   r˜   rœ   Ú
contiguous)r’   r“   r”   r•   r–   r—   r˜   rQ   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs   &&&&&&&,    r)   Úeager_attention_forwardr§   �   sÒ   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÕL€LØÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r,   c                   óX   a a€ ] tR t^©t oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )ÚCwmAttentionz=Multi-headed attention from 'Attention Is All You Need' paperc                ó&   <€ V ^8„  d   QhRS[ RS[/# ©r%   r&   Ú	layer_idx©r   rŠ   )r'   r(   s   "€r)   r*   ÚCwmAttention.__annotate__­   s   ø€ ÷ hñ h™yð h±Sñ hr,   c                ó6  <€ \         SV `  4        \        VR 4      '       d   VP                  V,          MRV n        Wn        W n        \        VRVP                  VP                  ,          4      V n
        VP                  VP                  ,          V n        V P                  R,          V n        VP                  V n        RV n        \         P"                  P%                  VP                  VP                  V P                  ,          RR7      V n        \         P"                  P%                  VP                  VP                  V P                  ,          RR7      V n        \         P"                  P%                  VP                  VP                  V P                  ,          RR7      V n        \"        P$                  ! VP                  V P                  ,          VP                  RR7      V n        V P                  R8X  d   VP.                  V n        R# RV n        R# )Úlayer_typesNrI   TF©ÚbiasÚsliding_attentiong      à¿)r1   r2   Úhasattrr°   Ú
layer_typer&   r¬   rL   rM   rN   rI   rŽ   r�   r—   Úattention_dropoutÚ	is_causalrD   r   ÚLinearÚq_projÚk_projÚv_projÚo_projÚsliding_window©r=   r&   r¬   r?   s   &&&€r)   r2   ÚCwmAttention.__init__­   s}  ø€ Ü‰ÑÔÜ;BÀ6È=×;YÒ;Y˜&×,Ñ,¨YÖ7Ð_cˆŒØŒØ"ŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÕ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Õ$\ˆÔ!Ø—}‘} dÕ*ˆŒØ!'×!9Ñ!9ˆÔØˆŒÜ—h‘h—o‘o f×&8Ñ&8¸&×:TÑ:TÐW[×WdÑWdÕ:dÐkp�oÓqˆŒÜ—h‘h—o‘o f×&8Ñ&8¸&×:TÑ:TÐW[×WdÑWdÕ:dÐkp�oÓqˆŒÜ—h‘h—o‘o f×&8Ñ&8¸&×:TÑ:TÐW[×WdÑWdÕ:dÐkp�oÓqˆŒÜ—i’i × :Ñ :¸T¿]¹]Õ JÈF×L^ÑL^ÐejÔkˆŒØ7;·±ÐJ]Ô7]˜f×3Ñ3ˆÖÐcgˆÖr,   c                ó  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  S[ P                  R,          3,          /# )r%   rˆ   Úposition_embeddingsr–   NÚpast_key_valuesrQ   rB   )rD   rE   rC   r   r   r   )r'   r(   s   "€r)   r*   r®   ½   s‡   ø€ ÷ ')ñ ')á—|‘|ð')ñ #¡5§<¡<±·±Ð#=Õ>ð')ñ Ÿ™ tÕ+ð	')ñ
  �ð')ñ Ñ-Õ.ð')ñ 
‰u�|‰|™UŸ\™\¨DÕ0Ð0Õ	1ñ')r,   c           	     ó(  € VP                   R R p. VORNV P                  N5pV P                  V4      P                  V4      P	                  ^^4      pV P                  V4      P                  V4      P	                  ^^4      p	V P                  V4      P                  V4      P	                  ^^4      p
Vw  r¼\        W‰W¼4      w  r‰Ve   VP                  WšV P                  4      w  rš\        P                  ! V P                  P                  \        4      pT! V VV	V
V3RV P                  '       g   RMV P                   RV P"                  RV P$                  /VB w  rïVP&                  ! . VORN5!  P)                  4       pV P+                  V4      pWï3# )Nr˜   ç        r—   r½   r]   )r_   rI   r¹   Úviewrc   rº   r»   r†   Úupdater¬   r   Úget_interfacer&   Ú_attn_implementationr§   rœ   r¶   r—   r½   rŒ   r¢   r¼   )r=   rˆ   rÁ   r–   rÂ   rQ   Úinput_shapeÚhidden_shapeÚquery_statesr£   r¤   re   rf   Úattention_interfacer¦   r¥   s   &&&&&,          r)   rm   ÚCwmAttention.forward½   s—  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ
%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð
%
ð —L‘Lð
%
ð  ×.Ñ.ð
%
ð ñ
%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—k‘k +Ó.ˆØÐ(Ð(r,   )r¶   r&   rI   r·   rº   r¬   rµ   r�   r¼   r¹   r—   r½   r»   ro   )
rp   rq   rr   rs   Ú__doc__r2   rm   rv   rw   rx   ry   s   @@r)   r©   r©   ©   s#   ù‡ € áG÷hó h÷ ')÷ ')ò ')r,   r©   ÚRMSNormc                   óZ   a a€ ] tR t^çt oRV3R lV 3R llltV3R lR ltR tRtVtV ;t	# )Ú
CwmRMSNormc                ó$   <€ V ^8„  d   QhRS[ RR/# )r%   ÚepsrB   N)rF   )r'   r(   s   "€r)   r*   ÚCwmRMSNorm.__annotate__é   s   ø€ ÷ $ñ $©ð $¸$ñ $r,   c                óŽ   <€ \         SV `  4        \        P                  ! \        P
                  ! V4      4      V n        W n        R# )z)
CwmRMSNorm is equivalent to T5LayerNorm
N)r1   r2   r   Ú	ParameterrD   ÚonesÚweightÚvariance_epsilon)r=   rM   rÓ   r?   s   &&&€r)   r2   ÚCwmRMSNorm.__init__é   s/   ø€ ô 	‰ÑÔÜ—l’l¤5§:¢:¨kÓ#:Ó;ˆŒØ #Ör,   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r%   rˆ   rB   )rD   rE   )r'   r(   s   "€r)   r*   rÔ   ñ   s#   ø€ ÷ ;ñ ;¡U§\¡\ð ;±e·l±lñ ;r,   c                ó:  € VP                   pVP                  \        P                  4      pVP	                  ^4      P                  RRR7      pV\        P                  ! W0P                  ,           4      ,          pV P                  VP                  V4      ,          # )r%   T)Úkeepdimr]   )	rK   rP   rD   r¡   ÚpowÚmeanÚrsqrtrÙ   rØ   )r=   rˆ   Úinput_dtypeÚvariances   &&  r)   rm   ÚCwmRMSNorm.forwardñ   sw   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯ª°H×?TÑ?TÕ4TÓ(UÕUˆØ�{‰{˜]×-Ñ-¨kÓ:Õ:Ð:r,   c                ó^   € \        V P                  P                  4       R V P                   2# )z, eps=)rC   rØ   r_   rÙ   )r=   s   &r)   Ú
extra_reprÚCwmRMSNorm.extra_reprø   s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr,   )rÙ   rØ   )g�íµ ÷Æ°>)
rp   rq   rr   rs   r2   rm   rå   rv   rw   rx   ry   s   @@r)   rÑ   rÑ   ç   s%   ù‡ € ÷$õ $÷;ð ;÷Jò Jr,   rÑ   c                   ó8   a a€ ] tR t^üt oV 3R ltR tRtVtV ;t# )ÚCwmMLPc                ó  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        \
        P                  ! V P                  V P                  VP                  R 7      V n        \
        P                  ! V P                  V P                  VP                  R 7      V n	        \
        P                  ! V P                  V P                  VP                  R 7      V n
        \        VP                  ,          V n        R# )r±   N)r1   r2   r&   rM   Úintermediate_sizer   r¸   Úmlp_biasÚ	gate_projÚup_projÚ	down_projr   Ú
hidden_actÚact_fn©r=   r&   r?   s   &&€r)   r2   ÚCwmMLP.__init__ý   s¶   ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔÜŸš 4×#3Ñ#3°T×5KÑ5KÐRX×RaÑRaÔbˆŒÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPV×P_ÑP_Ô`ˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRX×RaÑRaÔbˆŒÜ˜V×.Ñ.Õ/ˆŽr,   c                ó�   € V P                  V P                  V P                  V4      4      V P                  V4      ,          4      pV# ro   )rî   rð   rì   rí   )r=   rg   rî   s   && r)   rm   ÚCwmMLP.forward  s6   € Ø—N‘N 4§;¡;¨t¯~©~¸aÓ/@Ó#AÀDÇLÁLÐQRÃOÕ#SÓTˆ	ØÐr,   )rð   r&   rî   rì   rM   rê   rí   ©	rp   rq   rr   rs   r2   rm   rv   rw   rx   ry   s   @@r)   rè   rè   ü   s   ù‡ € õ0÷ò r,   rè   c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚCwmDecoderLayeri  c                ó&   <€ V ^8„  d   QhRS[ RS[/# r«   r­   )r'   r(   s   "€r)   r*   ÚCwmDecoderLayer.__annotate__  s   ø€ ÷ `ñ `™yð `±Sñ `r,   c                ó   <€ \         SV `  4        VP                  V n        \        WR 7      V n        \        V4      V n        \        VP                  VP                  R7      V n	        \        VP                  VP                  R7      V n
        R# ))r&   r¬   ©rÓ   N)r1   r2   rM   r©   Ú	self_attnrè   ÚmlprÑ   Úrms_norm_epsÚinput_layernormÚpost_attention_layernormr¾   s   &&&€r)   r2   ÚCwmDecoderLayer.__init__  si   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔÜ%¨VÔIˆŒä˜&“>ˆŒÜ)¨&×*<Ñ*<À&×BUÑBUÔVˆÔÜ(2°6×3EÑ3EÈ6×K^ÑK^Ô(_ˆÖ%r,   c                ó.  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[ P                  S[ P                  3,          R,          RS[S[,          R	S[ P                  /# )
r%   rˆ   r–   Nrh   rÂ   Ú	use_cacherÁ   rQ   rB   )rD   rE   Ú
LongTensorr   ÚboolrC   r   r   )r'   r(   s   "€r)   r*   rù     sœ   ø€ ÷ ñ á—|‘|ðñ Ÿ™ tÕ+ðñ ×&Ñ&¨Õ-ð	ñ
  �ðñ ˜$•;ðñ #¡5§<¡<±·±Ð#=Õ>ÀÕEðñ Ñ+Õ,ðñ 
�‰ñr,   c                óÔ   € TpV P                  V4      pV P                  ! RR VRVRVRVRVRV/VB w  rW�,           pTpV P                  V4      pV P                  V4      pW�,           pV# )rˆ   r–   rh   rÂ   r  rÁ   © )rÿ   rü   r   rý   )
r=   rˆ   r–   rh   rÂ   r  rÁ   rQ   ÚresidualÚ_s
   &&&&&&&,  r)   rm   ÚCwmDecoderLayer.forward  s©   € ð !ˆØ×,Ñ,¨]Ó;ˆàŸ>š>ñ 
Ø'ð
à)ð
ð &ð
ð ,ð	
ð
  ð
ð !4ð
ð ñ
Ñˆð !Õ0ˆð !ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ Õ0ˆØÐr,   )rM   rÿ   rý   r   rü   )NNNFNrõ   ry   s   @@r)   r÷   r÷     s    ù‡ € ÷`ó `÷÷ ò r,   r÷   c                   ó`   a € ] tR tRt o RtRtR.tR.tRtRt	Rt
RtRtR]R]/tV 3R ltR	tV tR
# )ÚCwmPreTrainedModeli6  ÚmodelTr÷   rÂ   rˆ   Ú
attentionsc                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r$   r   )r'   r(   s   "€r)   r*   ÚCwmPreTrainedModel.__annotate__6  s   ø‡ ‚ áÑò r,   r  N)rp   rq   rr   rs   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendr÷   r©   Ú_can_record_outputsÚ__annotate_func__rv   rw   )r(   s   @r)   r  r  6  s[   ø‡ € ð  ÐØ&*Ð#Ø*Ð+ÐØ#4Ð"5ÐØÐØ€NØÐà!ÐØ"&Ðà˜Ø�lðÐ÷ ƒ r,   r  c                   ó   € ] tR tRtRtR# )ÚCwmModelOutputWithPastiI  r  N)rp   rq   rr   rs   rv   r  r,   r)   r  r  I  s   † Ûr,   r  c                   óv   a a€ ] tR tRt o]tV3R lV 3R llt]]]	RV3R lR ll4       4       4       t
RtVtV ;t# )ÚCwmModeliM  c                ó    <€ V ^8„  d   QhRS[ /# r$   r   )r'   r(   s   "€r)   r*   ÚCwmModel.__annotate__Q  s   ø€ ÷ ñ ™yñ r,   c           	     ó&  <€ \         SV `  V4       VP                  V n        VP                  V n        \
        P                  ! VP                  VP                  V P                  4      V n        \        P
                  P                  \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n        \        VP                  VP                   R 7      V n        \%        VR7      V n        RV n        V P+                  4        R# u upi )rû   ©r&   FN)r1   r2   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	EmbeddingrM   Úembed_tokensrD   Ú
ModuleListÚrangeÚnum_hidden_layersr÷   ÚlayersrÑ   rþ   Únormr    Ú
rotary_embÚgradient_checkpointingÚ	post_initr¾   s   && €r)   r2   ÚCwmModel.__init__Q  sÏ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸLšL¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—h‘h×)Ñ)ÜAFÀv×G_ÑG_ÔA`ÓaÑA`°IŒ_˜VÖ/ÑA`Ñaó
ˆŒô ˜v×1Ñ1°v×7JÑ7JÔKˆŒ	Ü,°FÔ;ˆŒØ&+ˆÔ#ð 	�‰Öùò bs   ÂDc                óþ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[S[,          R	S[/# )
r%   Ú	input_idsNr–   rh   rÂ   Úinputs_embedsr  rQ   rB   )	rD   r  rE   r   ÚFloatTensorr  r   r   r  )r'   r(   s   "€r)   r*   r!  d  s“   ø€ ÷ 8
ñ 8
á×#Ñ# dÕ*ð8
ñ Ÿ™ tÕ+ð8
ñ ×&Ñ&¨Õ-ð	8
ñ
  �ð8
ñ ×(Ñ(¨4Õ/ð8
ñ ˜$•;ð8
ñ Ñ+Õ,ð8
ñ 
 ñ8
r,   c                óR  € VR J VR J,          '       d   \        R4      hVf   V P                  V4      pV'       d   Vf   \        V P                  R7      pVfb   Ve   VP	                  4       M^ p\
        P                  ! VP                  ^,          VP                  R7      V,           pVP                  ^ 4      p\        T;p	\        4      '       g?   RV P                  RVRVRVRV/p
V
P                  4       pR	\        R/ V
B R
\        R/ VB /p	TpV P                  WÃ4      p\!        V P"                  R V P                  P$                   4       F7  w  rïV! V3RW�P                  P&                  V,          ,          RVRVRV/VB pK9  	  V P)                  V4      p\+        VVR7      # )Nz:You must specify exactly one of input_ids or inputs_embedsr#  )r!   r&   r4  r–   rÂ   rh   Úfull_attentionr³   rÁ   )Úlast_hidden_staterÂ   r  )Ú
ValueErrorr(  r   r&   Úget_seq_lengthrD   rO   r_   r!   r€   r`   ÚdictÚcopyr   r   r.  Ú	enumerater,  r+  r°   r-  r  )r=   r3  r–   rh   rÂ   r4  r  rQ   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsÚsliding_mask_kwargsrˆ   rÁ   ÚiÚdecoder_layers   &&&&&&&,        r)   rm   ÚCwmModel.forwarda  sÅ  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ Ø*.×*;Ñ*;¸IÓ*FˆMç˜Ò0Ü*°$·+±+Ô>ˆOàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLä°Ð?Ð-Ä×FÒFà˜$Ÿ+™+Ø Ø  .Ø! ?Ø ðˆKð #.×"2Ñ"2Ó"4Ðð !Ô"4Ñ"C°{Ñ"CØ#Ô%FÑ%]ÐI\Ñ%]ð#Ðð
 &ˆØ"Ÿo™o¨mÓJÐä )¨$¯+©+Ð6U¸¿¹×8UÑ8UÐ*VÖ WÑˆAÙ)Øñà2·;±;×3JÑ3JÈ1Õ3MÕNðð *ðð !0ð	ð
 %8ðð ñŠMñ !Xð Ÿ	™	 -Ó0ˆÜ%Ø+Ø+ô
ð 	
r,   )r(  r/  r,  r-  r%  r.  r&  )NNNNNN)rp   rq   rr   rs   r   Úconfig_classr2   r   r   r   rm   rv   rw   rx   ry   s   @@r)   r  r  M  sB   ù‡ € à€L÷ó ð   ØØ÷8
ñ 8
ó ó ó  ÷8
ð 8
r,   r  c                   ó„   a a€ ] tR tRt oRR/tRR/tRR.R.3/tRR/tV 3R	 lt]	]
RV3R
 lR ll4       4       tRtVtV ;t# )ÚCwmForCausalLMiŸ  zlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputrˆ   ÚlogitsÚkeep_full_weightc                óì   <€ \         SV `  V4       \        V4      V n        VP                  V n        \
        P                  ! VP                  VP                  R R7      V n        V P                  4        R# )Fr±   N)
r1   r2   r  r  r&  r   r¸   rM   rH  r0  rñ   s   &&€r)   r2   ÚCwmForCausalLM.__init__¦  sU   ø€ Ü‰Ñ˜Ô Ü˜fÓ%ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒð 	�‰Ör,   c                óP  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[S[ P                  ,          R
S[S[,          RS[	/
# )r%   r3  Nr–   rh   rÂ   r4  Úlabelsr  Úlogits_to_keeprQ   rB   )
rD   r  rE   r   r5  r  rŠ   r   r   r   )r'   r(   s   "€r)   r*   ÚCwmForCausalLM.__annotate__±  s»   ø€ ÷ 6
ñ 6
á×#Ñ# dÕ*ð6
ñ Ÿ™ tÕ+ð6
ñ ×&Ñ&¨Õ-ð	6
ñ
  �ð6
ñ ×(Ñ(¨4Õ/ð6
ñ × Ñ  4Õ'ð6
ñ ˜$•;ð6
ñ ™eŸl™lÕ*ð6
ñ Ñ+Õ,ð6
ñ 
 ñ6
r,   c	                ó–  € V P                   ! RRVRVRVRVRVRV/V	B p
V
P                  p\        V\        4      '       d   \	        V) R4      MTpV P                  VRVR3,          4      pRpVe-   V P                  ! RR	VR
VRV P                  P                  /V	B p\        VVV
P                  V
P                  V
P                  R7      # )ac  
Example:

```python
>>> from transformers import AutoTokenizer, CwmForCausalLM

>>> model = CwmForCausalLM.from_pretrained("meta-cwm/Cwm-2-7b-hf")
>>> tokenizer = AutoTokenizer.from_pretrained("meta-cwm/Cwm-2-7b-hf")

>>> prompt = "Hey, are you conscious? Can you talk to me?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
```r3  r–   rh   rÂ   r4  r  Nr\   rJ  rO  r&  )ÚlossrJ  rÂ   rˆ   r  r  )r  r8  r`   rŠ   ÚslicerH  Úloss_functionr&   r&  r   rÂ   rˆ   r  )r=   r3  r–   rh   rÂ   r4  rO  r  rP  rQ   Úoutputsrˆ   Úslice_indicesrJ  rS  s   &&&&&&&&&,     r)   rm   ÚCwmForCausalLM.forward¯  sÿ   € ð> ,0¯:ª:ñ ,
Øð,
à)ð,
ð &ð,
ð ,ð	,
ð
 (ð,
ð  ð,
ð ñ,
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒØ×%Ò%Ñp¨VÐp¸FÐpÈtÏ{É{×OeÑOeÐpÐioÑpˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r,   )rH  r  r&  )NNNNNNNr   )rp   rq   rr   rs   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr2   r   r   rm   rv   rw   rx   ry   s   @@r)   rG  rG  Ÿ  sf   ù‡ € à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÐ/Ð0€Jõð Ø÷6
ñ 6
ó ó ÷6
ð 6
r,   rG  )é   )rÄ   )r  r  rG  )<Úcollections.abcr   rD   r   Úactivationsr   Úcache_utilsr   r   Ú
generationr   Úintegrationsr	   r
   Úmasking_utilsr   r   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   Úconfiguration_cwmr   r™   r    r}   r†   r�   r§   r©   rÑ   rè   r÷   r  r  r  rG  Ú__all__r  r,   r)   Ú<module>rp     sZ  ðõ, %ã Ý å !ß .Ý )ß Lß RÝ BÝ 9ß Oß Kß FÝ &ß IÑ IÝ 0ß GÝ 5Ý (ô3<˜Ÿ™ô 3<òl(ñ Ð-Ó.óó /ðõ2	U÷%ñ2 Ð)Ó*ô:)�2—9‘9ó :)ó +ð:)ñz ˜YÓ'ôJ�—‘ó Jó (ðJô(ˆR�Y‰Yô ô 'Ð0ô 'ðT ô˜ó ó ðô$	Ð4ô 	ð ôN
Ð!ó N
ó ðN
ðb ôG
Ð'¨ó G
ó ðG
òT ?‚r,   