+
    œ°³jLl  ã                   ó°  € ^ RI t ^ RIHt ^ RIt^ RIHt ^RIHt ^RIH	t	 ^RI
HtHt ^RIHt ^RIHt ^R	IHt ^R
IHt ^RIHt ^RIHtHt ^RIHtHt ^RIHtHt ^RI H!t! ^RI"H#t#H$t$H%t% ^RI&H't' ^RI(H)t)H*t* ^RI+H,t, ^RI-H.t. ]! R4       ! R R]P^                  4      4       t0 ! R R]P^                  4      t1 ! R R]P^                  4      t2R R lt3R0R R llt4R1R  lt5R! t6R" t7]! R#4      R2R$ l4       t8R3R% lt9 ! R& R']P^                  4      t: ! R( R)]4      t;]$ ! R* R+]4      4       t<]$ ! R, R-]<4      4       t=]$ ! R. R/]<]4      4       t>. R4Ot?R# )5é    N)ÚCallable)Únn)Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_kernel_forward_from_hub)Úcreate_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputs©ÚYoutuConfigÚRMSNormc                   óZ   a a€ ] tR t^4t oRV3R lV 3R llltV3R lR ltR tRtVtV ;t	# )ÚYoutuRMSNormc                ó$   <€ V ^8„  d   QhRS[ RR/# )é   ÚepsÚreturnN)Úfloat)ÚformatÚ__classdict__s   "€Úi/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/youtu/modeling_youtu.pyÚ__annotate__ÚYoutuRMSNorm.__annotate__6   s   ø€ ÷ $ñ $©ð $¸$ñ $ó    c                óŽ   <€ \         SV `  4        \        P                  ! \        P
                  ! V4      4      V n        W n        R# )z+
YoutuRMSNorm is equivalent to T5LayerNorm
N)ÚsuperÚ__init__r   Ú	ParameterÚtorchÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizer#   Ú	__class__s   &&&€r(   r.   ÚYoutuRMSNorm.__init__6   s/   ø€ ô 	‰ÑÔÜ—l’l¤5§:¢:¨kÓ#:Ó;ˆŒØ #Ör+   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r"   Úhidden_statesr$   )r0   ÚTensor)r&   r'   s   "€r(   r)   r*   >   s#   ø€ ÷ ;ñ ;¡U§\¡\ð ;±e·l±lñ ;r+   c                ó:  € VP                   pVP                  \        P                  4      pVP	                  ^4      P                  RRR7      pV\        P                  ! W0P                  ,           4      ,          pV P                  VP                  V4      ,          # )r"   T)Úkeepdiméÿÿÿÿ)	ÚdtypeÚtor0   Úfloat32ÚpowÚmeanÚrsqrtr3   r2   )r4   r9   Úinput_dtypeÚvariances   &&  r(   ÚforwardÚYoutuRMSNorm.forward>   sw   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯ª°H×?TÑ?TÕ4TÓ(UÕUˆØ�{‰{˜]×-Ñ-¨kÓ:Õ:Ð:r+   c                ó^   € \        V P                  P                  4       R V P                   2# )z, eps=)Útupler2   Úshaper3   )r4   s   &r(   Ú
extra_reprÚYoutuRMSNorm.extra_reprE   s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr+   )r3   r2   )g�íµ ÷Æ°>)
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r.   rF   rK   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r6   r'   s   @@r(   r    r    4   s%   ù‡ € ÷$õ $÷;ð ;÷Jò Jr+   r    c                   óÎ   a a€ ] tR t^It o]! RRR7      R
V3R lV 3R lll4       t]]! RRR7      R
V3R lR ll4       4       t]P                  ! 4       ]
R 4       4       tR	tVtV ;t# )ÚYoutuRotaryEmbeddingÚdevicez5.18)Úversionc                ó    <€ V ^8„  d   QhRS[ /# ©r"   Úconfigr   )r&   r'   s   "€r(   r)   Ú!YoutuRotaryEmbedding.__annotate__K   s   ø€ ÷ Oñ O™{ñ Or+   c                óä  <€ \         SV `  4        VP                  V n        VP                  V n        Wn        V P
                  P                  R ,          V n        V P                  pV P                  R8w  d   \        V P                  ,          pV! V P
                  V4      w  q@n
        \        P                  ! VRR7      V n        \        P                  ! VP                  4       RR7      V n        R# )Ú	rope_typeÚdefaultF)Ú
persistentN)r-   r.   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr[   Úrope_parametersr^   Úcompute_default_rope_parametersr   Úattention_scalingr   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)r4   r[   rW   Úrope_init_fnrh   r6   s   &&&  €r(   r.   ÚYoutuRotaryEmbedding.__init__J   s¬   ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àŒàŸ™×4Ñ4°[ÕAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÔ&Ü.¨t¯~©~Õ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÔ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÔ!NˆÖr+   c                óP   <€ V ^8„  d   QhRS[ RS[S[P                  S[3,          /# )r"   r[   r$   )r   rI   r0   r:   r%   )r&   r'   s   "€r(   r)   r\   ]   s.   ø€ ÷ 5ñ 5±ð 5ÑW\Ñ]b×]iÑ]iÑkpÐ]pÕWqñ 5r+   c           	     ó0  € V P                   R,          p\        V RR4      ;'       g    V P                  V P                  ,          pRpRV\        P
                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚhead_dimNç      ð?©r>   )rd   Úgetattrr5   Únum_attention_headsr0   Úaranger%   r?   )r[   rW   ÚkwargsÚbaseÚdimÚattention_factorrh   s   &&,    r(   re   Ú4YoutuRotaryEmbedding.compute_default_rope_parameters[   s~   € ð ×%Ñ% lÕ3ˆÜ�f˜j¨$Ó/×cÐc°6×3EÑ3EÈ×IcÑIcÕ3cˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r+   c                ó^  € V P                   R,          P                  VP                  ^ ,          R	^4      P                  \        P
                  VP                  R7      pVR
,          P                  4       p\        VP                  P                  \        4      '       d2   VP                  P                  R8w  d   VP                  P                  MRp\        VRR7      ;_uu_ 4        W4,          P                  ^^4      p\        P                  ! Wf3R	R7      pVP                  4       V P                  ,          pVP                  4       V P                  ,          p	R R R 4       XP                  VP                   R7      X	P                  VP                   R7      3#   + '       g   i     LH; i)N)r>   rW   ÚmpsÚcpuF)Údevice_typeÚenabled©rx   rr   )NºNNNNr=   )r�   Nr�   )rh   ÚexpandrJ   r?   r0   r%   rW   Ú
isinstanceÚtypeÚstrr   Ú	transposeÚcatÚcosrf   Úsinr>   )
r4   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedr~   ÚfreqsÚembrˆ   r‰   s
   &&&       r(   rF   ÚYoutuRotaryEmbedding.forwardo   s2  € ð �M‰M˜-Õ(×/Ñ/°×0BÑ0BÀ1Õ0EÀrÈ1ÓM×PÑPÔW\×WbÑWbÐkl×ksÑksÐPÓtð 	ð !-¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆä¨¸U×CÖCØ&Õ>×IÑIÈ!ÈQÓOˆEÜ—)’)˜U˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆC÷	 Dð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷ D×Cús   Ã(A4FÆF,	)rf   r[   rh   rb   rj   rc   r^   ©N)rM   rN   rO   rP   r   r.   Ústaticmethodre   r0   Úno_gradr   rF   rQ   rR   rS   rT   s   @@r(   rV   rV   I   su   ù‡ € Ù�X vÔ.÷Oô Oó /ðOð  Ù�X vÔ.÷5ñ 5ó /ó ð5ð$ ‡]‚]ƒ_Øñ<ó ó ÷<ð <r+   rV   c                   ó8   a a€ ] tR t^‚t oV 3R ltR tRtVtV ;t# )ÚYoutuMLPc                óØ  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        \
        P                  ! V P                  V P                  R R7      V n        \
        P                  ! V P                  V P                  R R7      V n        \
        P                  ! V P                  V P                  R R7      V n	        \        VP                  ,          V n        R# ©F©ÚbiasN)r-   r.   r[   r5   Úintermediate_sizer   ÚLinearÚ	gate_projÚup_projÚ	down_projr   Ú
hidden_actÚact_fn©r4   r[   r6   s   &&€r(   r.   ÚYoutuMLP.__init__ƒ   s¤   ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔÜŸš 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆŒÜ˜V×.Ñ.Õ/ˆŽr+   c                ó�   € V P                  V P                  V P                  V4      4      V P                  V4      ,          4      pV# r‘   )rž   r    rœ   r�   )r4   rŠ   rž   s   && r(   rF   ÚYoutuMLP.forward�   s6   € Ø—N‘N 4§;¡;¨t¯~©~¸aÓ/@Ó#AÀDÇLÁLÐQRÃOÕ#SÓTˆ	ØÐr+   )r    r[   rž   rœ   r5   rš   r�   ©	rM   rN   rO   rP   r.   rF   rQ   rR   rS   rT   s   @@r(   r•   r•   ‚   s   ù‡ € õ0÷ò r+   r•   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r"   r9   Ún_repr$   )r0   r:   Úint)r&   s   "r(   r)   r)   ’   s.   € ÷ 	Uñ 	UœUŸ\™\ð 	U´#ð 	U¼%¿,¹,ñ 	Ur+   c                ó˜   € V P                   w  r#rEV^8X  d   V # V R,          P                  W#WV4      p V P                  W#V,          WE4      # )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
)r�   r�   Nr�   r�   )rJ   r‚   Úreshape)r9   r§   ÚbatchÚnum_key_value_headsÚslenrp   s   &&    r(   Ú	repeat_kvr®   ’   sU   € ð
 2?×1DÑ1DÑ.€E Ø�„zØÐØ!Ð"2Õ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÕ(CÀTÓTÐTr+   c                ó   € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R\        R	\
        \        ,          /# )
r"   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNÚscalingÚdropoutrv   )r   ÚModuler0   r:   r%   r   r   )r&   s   "r(   r)   r)   ž   sw   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ð%ô ð%ô Ô'Õ(ñ%r+   c                 ó"  € \        W P                  4      p\        W0P                  4      p	\        P                  ! WP	                  ^^4      4      V,          p
Ve	   W¤,           p
\
        P                  P                  V
R\        P                  R7      P                  VP                  4      p
\
        P                  P                  W¦V P                  R7      p
\        P                  ! W©4      pVP	                  ^^4      P                  4       pWº3# )r"   )rx   r>   )ÚpÚtrainingr=   )r®   Únum_key_value_groupsr0   Úmatmulr†   r   Ú
functionalÚsoftmaxr@   r?   r>   r¶   rº   Ú
contiguous)r°   r±   r²   r³   r´   rµ   r¶   rv   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs   &&&&&&&,    r(   Úeager_attention_forwardrÄ   ž   sÒ   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÕL€LØÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r+   c                 ój   € V ^8:  d   R# RV,          \         P                  ! V 4      ,          R,           # )é   rq   gš™™™™™¹?)ÚmathÚlog)ÚscaleÚmscales   &&r(   Úyarn_get_mscalerË   ·   s(   € Ø�„zÙØ��<œ$Ÿ(š( 5›/Õ)¨CÕ/Ð/r+   c                 ó®   € V P                  R R4      R8w  d>   V P                  R^ 4      pV R,          pV'       d   \        W24      pW,          V,          pV# )r^   r_   Úmscale_all_dimÚfactor)ÚgetrË   )rd   rµ   rÍ   Úscaling_factorrÊ   s   &&   r(   Úyarn_apply_mscalerÑ   ½   sS   € Ø×Ñ˜;¨	Ó2°iÔ?Ø(×,Ñ,Ð-=¸qÓAˆØ(¨Õ2ˆßÜ$ ^ÓDˆFØÕ&¨Õ/ˆGØ€Nr+   c                óÌ   € V RRV P                   R,          ^,          13,          pV RV P                   R,          ^,          R13,          p\        P                  ! V) V3RR7      # )z*Rotates half the hidden dims of the input..Nr€   r=   )rJ   r0   r‡   )rŠ   Úx1Úx2s   &  r(   Úrotate_halfrÕ   Ç   sZ   € à	
ˆ3Ð"�!—'‘'˜"•+ Õ"Ð"Ð"Õ	#€BØ	
ˆ3�—‘˜•˜qÕ Ñ"Ð"Õ	#€BÜ�9Š9�r�c˜2�Y BÔ'Ð'r+   Úrotary_pos_embc                óÈ   € VP                  V4      pVP                  V4      pW,          \        V 4      V,          ,           pW,          \        V4      V,          ,           pWV3# )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)Ú	unsqueezerÕ   )ÚqÚkrˆ   r‰   Úunsqueeze_dimÚq_embedÚk_embeds   &&&&&  r(   Úapply_rotary_pos_embrÞ   Î   sS   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�wœ; q›>¨CÕ/Õ0€GØ�wœ; q›>¨CÕ/Õ0€GØÐÐr+   c                ó  € VRRVP                   R,          ^,          13,          P                  V4      pVRRVP                   R,          ^,          13,          P                  V4      pV R,          V R,          rvVR,          VR,          r˜\        P                  ! Wb,          Ws,          ,
          Wr,          Wc,          ,           .RR7      p
\        P                  ! W‚,          W“,          ,
          W’,          Wƒ,          ,           .RR7      pW«3# )aŸ  
Applies interleaved Rotary Position Embedding to the query and key tensors.

DeepSeek lays the rotary dimensions out in interleaved pairs `(x0, x1), (x2, x3), ...`, each rotated by a
single frequency. We compute that rotation directly on the even/odd slices instead of de-interleaving with a
`view`/`transpose`/`reshape`; the output is bit-identical to the de-interleaved `rotate_half` formulation while
avoiding the extra contiguous copy.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    position_ids (`torch.Tensor`):
        The position indices of the tokens corresponding to the query and key tensors. For example, this can be
        used to pass offsetted position ids when working with a KV-cache.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
.Nr€   r=   ).:r   Nr"   ).:rÆ   Nr"   )rJ   rØ   r0   r‡   )rÙ   rÚ   rˆ   r‰   r‹   rÛ   Úq1Úq2Úk1Úk2rÜ   rÝ   s   &&&&&&      r(   Úapply_rotary_pos_emb_interleaverä   è   sÓ   € ð8 ˆcÐ'�S—Y‘Y˜r•] aÕ'Ð'Ð'Õ
(×
2Ñ
2°=Ó
A€CØ
ˆcÐ'�S—Y‘Y˜r•] aÕ'Ð'Ð'Õ
(×
2Ñ
2°=Ó
A€Càˆy�\˜1˜Y�<ˆØˆy�\˜1˜Y�<ˆä�iŠi˜� B¥HÕ,¨b­h¸½Õ.AÐBÈÔK€GÜ�iŠi˜� B¥HÕ,¨b­h¸½Õ.AÐBÈÔK€GØÐÐr+   c                   ój   a a€ ] tR tRt oRtV3R lV 3R lltV3R lR ltR
V3R lR lltR	tVt	V ;t
# )ÚYoutuAttentioni  z4Multi-headed Latent Attention (MLA) from Deepseek V2c                ó&   <€ V ^8„  d   QhRS[ RS[/# ©r"   r[   Ú	layer_idx©r   r¨   )r&   r'   s   "€r(   r)   ÚYoutuAttention.__annotate__  s   ø€ ÷ 5]ñ 5]™{ð 5]±sñ 5]r+   c                ó‚  <€ \         SV `  4        Wn        W n        VP                  V n        VP
                  V n        VP                  V n        VP                  V n        VP                  V n	        VP                  V n
        VP                  V n        VP                  V n        VP                  VP                  ,           V n        VP                  VP                  ,          V n        R V n        V P                  f?   \"        P$                  ! V P
                  V P                  V P                  ,          RR7      MRV n        V P                  e8   \"        P$                  ! V P
                  VP                  VP(                  R7      MRV n        V P                  e   \-        VP                  4      MRV n        V P                  e?   \"        P$                  ! VP                  V P                  V P                  ,          RR7      MRV n        \"        P$                  ! V P
                  VP                  VP                  ,           VP(                  R7      V n        \-        VP                  4      V n        \"        P$                  ! VP                  V P                  V P                  V P                  ,           ,          RR7      V n        \"        P$                  ! V P                  V P                  ,          V P
                  VP(                  R7      V n        \;        VP<                  V P                  R,          4      V n        R# )TNFr˜   g      à¿) r-   r.   r[   ré   Úattention_dropoutr5   rt   Ú	num_headsÚq_lora_rankÚqk_rope_head_dimÚkv_lora_rankÚ
v_head_dimÚqk_nope_head_dimÚqk_head_dimr¬   r»   Ú	is_causalr   r›   Úq_projÚattention_biasÚq_a_projr    Úq_a_layernormÚq_b_projÚkv_a_proj_with_mqaÚkv_a_layernormÚ	kv_b_projÚo_projrÑ   rd   rµ   ©r4   r[   ré   r6   s   &&&€r(   r.   ÚYoutuAttention.__init__  s{  ø€ Ü‰ÑÔØŒØ"ŒØ!'×!9Ñ!9ˆÔØ!×-Ñ-ˆÔØ×3Ñ3ˆŒà!×-Ñ-ˆÔØ &× 7Ñ 7ˆÔØ"×/Ñ/ˆÔØ ×+Ñ+ˆŒØ &× 7Ñ 7ˆÔØ!×2Ñ2°V×5LÑ5LÕLˆÔØ$*×$>Ñ$>À&×B\ÑB\Õ$\ˆÔ!àˆŒð ×ÑÒ'ô �IŠI�d×&Ñ&¨¯©¸×9IÑ9IÕ(IÐPUÕVàð 	Œð ×ÑÒ+ô �IŠI�d×&Ñ&¨×(:Ñ(:À×AVÑAVÕWàð 	Œð
 BF×AQÑAQÒA]œ\¨&×*<Ñ*<Ô=ÐcgˆÔð ×ÑÒ+ô �IŠI�f×(Ñ(¨$¯.©.¸4×;KÑ;KÕ*KÐRWÕXàð 	Œô #%§)¢)Ø×ÑØ×Ñ &×"9Ñ"9Õ9Ø×&Ñ&ô#
ˆÔô
 +¨6×+>Ñ+>Ó?ˆÔÜŸšØ×ÑØ�N‰N˜d×3Ñ3°d·o±oÕEÕFØô
ˆŒô —i’iØ�N‰N˜TŸ_™_Õ,Ø×ÑØ×&Ñ&ô
ˆŒô )¨×)?Ñ)?À×AQÑAQÐVZÕA[Ó\ˆŽr+   c                ó’   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[S[ P                  S[ P                  3,          /# )r"   Úkv_nopeÚk_rotr$   )r0   r:   rI   )r&   r'   s   "€r(   r)   rë   I  s?   ø€ ÷ (ñ (¡§¡ð (±e·l±lð (ÁuÉUÏ\É\Ñ[`×[gÑ[gÐMgÕGhñ (r+   c                ó‚  € VP                   w  r4rTW5RV P                  V P                  ,           3pV P                  V4      P	                  V4      P                  ^^4      p\        P                  ! WP                  V P                  .RR7      w  rxVP                  RVP                   ^,          RR4      pVP                  ! . VP                   RR OV P                  V P                  ,           N5!  p	V	RRV P                  13,          P                  V4       V	RV P                  R13,          P                  V4       W˜3# )as  Expands the compressed latents into key and value states. Args:
    - kv_nope: key + value without positional encoding, shape [batch_size, 1, seqlen, self.kv_lora_rank]
    - k_rot: shared key with positional encoding, shape [batch_size, 1, seqlen, self.qk_rope_head_dim]
Returns the key and value states, two tensors of shape [batch, num_heads, seq, (k or v)_head_dim].
r€   N.r=   )rJ   ró   rò   rý   Úviewr†   r0   Úsplitr‚   Ú	new_emptyrð   Úcopy_)
r4   r  r  Ú
batch_sizeÚ_Ú
seq_lengthÚ	key_shapeÚk_noperÁ   rÀ   s
   &&&       r(   Ú	expand_kvÚYoutuAttention.expand_kvI  s  € ð (/§}¡}Ñ$ˆ
�zØ¨R°×1FÑ1FÈÏÉÕ1XÐYˆ	à—.‘. Ó)×.Ñ.¨yÓ9×CÑCÀAÀqÓIˆÜ$Ÿ{š{¨7×5JÑ5JÈDÏOÉOÐ4\ÐbdÔeÑˆØ—‘˜R §¡¨a¥°"°bÓ9ˆð ×&Ò&Ðj¨¯©°c°rÐ(:Ðj¸D×<QÑ<QÐTX×TiÑTiÕ<iÓjˆ
Ø�3Ð/˜$×/Ñ/Ð/Ð/Õ0×6Ñ6°vÔ>Ø�3˜×-Ñ-Ñ/Ð/Õ0×6Ñ6°uÔ=ØÐ'Ð'r+   c                óR  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  S[ P                  R,          S[S[ P                  ,          R,          3,          /# )r"   r9   Úposition_embeddingsr´   NÚpast_key_valuesrv   r$   )r0   r:   rI   r   r   r   )r&   r'   s   "€r(   r)   rë   \  s—   ø€ ÷ 8)ñ 8)á—|‘|ð8)ñ #¡5§<¡<±·±Ð#=Õ>ð8)ñ Ÿ™ tÕ+ð	8)ñ
  �ð8)ñ Ñ-Õ.ð8)ñ 
‰u�|‰|™UŸ\™\¨DÕ0±%¹¿¹Õ2EÈÕ2LÐLÕ	Mñ8)r+   c                óø  € VP                   R R w  rgWgRV P                  3pV P                  f   V P                  V4      p	M/V P	                  V P                  V P                  V4      4      4      p	V	P                  V4      P                  ^^4      p	\        P                  ! W�P                  V P                  .RR7      w  r«V P                  V4      p\        P                  ! WÀP                  V P                  .RR7      w  rÞV P                  V4      pVP                  V^WpP                  4      pVP                  V^WpP                  4      pVw  ppV P                   P"                  '       d   \%        W¾VV4      w  r¾M\'        W¾VV4      w  r¾Ve   VP)                  WÞV P*                  4      w  rÞ\        P,                  ! W«3RR7      pV P/                  WÞ4      w  pp\0        P2                  ! V P                   P4                  \6        4      pT! V VVVV3RV P8                  '       g   RMV P:                  RV P<                  /VB w  ppVP?                  WgR4      PA                  4       pV PC                  V4      pVV3# )Nr€   r¶   ç        rµ   r=   )"rJ   rô   rï   rö   rú   rù   rø   r  r†   r0   r  ró   rð   rû   rñ   rü   r[   Úrope_interleaverä   rÞ   Úupdateré   r‡   r  r   Úget_interfaceÚ_attn_implementationrÄ   rº   rí   rµ   rª   r¿   rþ   )r4   r9   r  r´   r  rv   r	  r  Úquery_shapeÚq_statesÚq_passÚq_rotÚcompressed_kvr  r  rˆ   r‰   Úquery_statesrÀ   rÁ   Úattention_interfacerÃ   rÂ   s   &&&&&,                 r(   rF   ÚYoutuAttention.forward\  sC  € ð "/×!4Ñ!4°S°bÐ!9Ñˆ
Ø!¨r°4×3CÑ3CÐDˆà×ÑÒ#Ø—{‘{ =Ó1‰Hà—}‘} T×%7Ñ%7¸¿¹ÀmÓ8TÓ%UÓVˆHØ—=‘= Ó-×7Ñ7¸¸1Ó=ˆÜŸš H×/DÑ/DÀd×F[ÑF[Ð.\ÐbdÔe‰ˆà×/Ñ/°Ó>ˆÜŸš ]×5FÑ5FÈ×H]ÑH]Ð4^ÐdfÔg‰ˆØ×%Ñ% gÓ.ˆà—,‘,˜z¨1¨j×:KÑ:KÓLˆØ—
‘
˜: q¨*×6KÑ6KÓLˆà&‰ˆˆSØ�;‰;×&×&Ð&Ü:¸5ÈÈcÓR‰LˆE�5ä/°¸cÀ3ÓG‰LˆEð Ò&Ø,×3Ñ3°GÀDÇNÁNÓS‰NˆGä—y’y & °bÔ9ˆà#'§>¡>°'Ó#AÑ ˆ
�Lä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆ�\ð "×)Ñ)¨*À"ÓE×PÑPÓRˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r+   )rí   r[   r5   rõ   rü   rû   rý   rñ   ré   rî   r»   rþ   rù   rø   rú   rï   rö   rô   ró   rð   rµ   rò   r‘   )rM   rN   rO   rP   Ú__doc__r.   r  rF   rQ   rR   rS   rT   s   @@r(   ræ   ræ     s.   ù‡ € Ù>÷5]ó 5]÷n(ð (÷&8)÷ 8)ò 8)r+   ræ   c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚYoutuDecoderLayeri—  c                ó&   <€ V ^8„  d   QhRS[ RS[/# rè   rê   )r&   r'   s   "€r(   r)   ÚYoutuDecoderLayer.__annotate__˜  s   ø€ ÷ bñ b™{ð b±sñ br+   c                ó   <€ \         SV `  4        VP                  V n        \        WR 7      V n        \        V4      V n        \        VP                  VP                  R7      V n	        \        VP                  VP                  R7      V n
        R# ))r[   ré   ©r#   N)r-   r.   r5   ræ   Ú	self_attnr•   Úmlpr    Úrms_norm_epsÚinput_layernormÚpost_attention_layernormrÿ   s   &&&€r(   r.   ÚYoutuDecoderLayer.__init__˜  sj   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔä'¨vÔKˆŒä˜FÓ#ˆŒÜ+¨F×,>Ñ,>ÀF×DWÑDWÔXˆÔÜ(4°V×5GÑ5GÈV×M`ÑM`Ô(aˆÖ%r+   c                ó.  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[ P                  S[ P                  3,          R,          RS[S[,          R	S[ P                  /# )
r"   r9   r´   Nr‹   r  Ú	use_cacher  rv   r$   )r0   r:   Ú
LongTensorr   ÚboolrI   r   r   )r&   r'   s   "€r(   r)   r%  ¢  sœ   ø€ ÷ ñ á—|‘|ðñ Ÿ™ tÕ+ðñ ×&Ñ&¨Õ-ð	ñ
  �ðñ ˜$•;ðñ #¡5§<¡<±·±Ð#=Õ>ÀÕEðñ Ñ+Õ,ðñ 
�‰ñr+   c                óÔ   € TpV P                  V4      pV P                  ! RR VRVRVRVRVRV/VB w  rW�,           pTpV P                  V4      pV P                  V4      pW�,           pV# )r9   r´   r‹   r  r/  r  © )r+  r(  r,  r)  )
r4   r9   r´   r‹   r  r/  r  rv   Úresidualr
  s
   &&&&&&&,  r(   rF   ÚYoutuDecoderLayer.forward¢  s©   € ð !ˆØ×,Ñ,¨]Ó;ˆàŸ>š>ñ 
Ø'ð
à)ð
ð &ð
ð ,ð	
ð
  ð
ð !4ð
ð ñ
Ñˆð !Õ0ˆð !ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ Õ0ˆØÐr+   )r5   r+  r)  r,  r(  )NNNFNr¥   rT   s   @@r(   r#  r#  —  s    ù‡ € ÷bó b÷÷ ò r+   r#  c                   óš   a a€ ] tR tRt oRtRtR.tR.tRtRt	Rt
RtRtR]R]/t]P"                  ! 4       V 3R l4       tV3R	 ltR
tVtV ;t# )ÚYoutuPreTrainedModeliÂ  ÚmodelTr#  r  r9   Ú
attentionsc                ó°  <€ \         SV `  V4       \        V P                  R R4      p\        V P                  R^V,          4      p\	        V\
        P                  4      '       dq   \        P                  ! VP                  RVR7       VP                  e>   \        P                  ! VP                  P                  VP                  ,          4       R# R# R# )Úinitializer_rangeg{®Gáz”?Úembedding_initializer_ranger  )rB   ÚstdN)r-   Ú_init_weightsrs   r[   rƒ   r   Ú	EmbeddingÚinitÚnormal_r2   Úpadding_idxÚzeros_Údata)r4   r°   r=  Ú	embed_stdr6   s   &&  €r(   r>  Ú"YoutuPreTrainedModel._init_weightsÔ  s–   ø€ ä‰Ñ˜fÔ%Ü�d—k‘kÐ#6¸Ó=ˆÜ˜DŸK™KÐ)FÈÈCÍÓPˆ	Ü�fœbŸl™l×+Ò+Ü�LŠL˜Ÿ™¨S°iÕ@Ø×!Ñ!Ò-Ü—’˜FŸM™M×.Ñ.¨v×/AÑ/AÕBÖCñ .ñ ,r+   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # rZ   r   )r&   r'   s   "€r(   r)   Ú!YoutuPreTrainedModel.__annotate__Â  s   ø‡ ‚ áÑò r+   r3  )rM   rN   rO   rP   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendr#  ræ   Ú_can_record_outputsr0   r“   r>  Ú__annotate_func__rQ   rR   rS   rT   s   @@r(   r7  r7  Â  sx   ù‡ € ð  ÐØ&*Ð#Ø,Ð-ÐØ#4Ð"5ÐØÐØ€NØÐà!ÐØ"&ÐàÐ*Ø�nðÐð
 ‡]‚]ƒ_ôDó ðD÷' … r+   r7  c                   ór   a a€ ] tR tRt oV3R lV 3R llt]]]RV3R lR ll4       4       4       tRt	Vt
V ;t# )Ú
YoutuModeliß  c                ó    <€ V ^8„  d   QhRS[ /# rZ   r   )r&   r'   s   "€r(   r)   ÚYoutuModel.__annotate__á  s   ø€ ÷ ñ ™{ñ r+   c           	     ó  <€ \         SV `  V4       VP                  V n        VP                  V n        \
        P                  ! VP                  VP                  V P                  4      V n        \
        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n        \        VP                  VP                  R 7      V n        \#        VR7      V n        RV n        V P)                  4        R# u upi )r'  ©r[   FN)r-   r.   Úpad_token_idrB  Ú
vocab_sizer   r?  r5   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersr#  Úlayersr    r*  ÚnormrV   Ú
rotary_embÚgradient_checkpointingÚ	post_initrÿ   s   && €r(   r.   ÚYoutuModel.__init__á  sÊ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸLšL¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m’mÜCHÈ×IaÑIaÔCbÓcÑCb°iÔ˜vÖ1ÑCbÑcó
ˆŒô ! ×!3Ñ!3¸×9LÑ9LÔMˆŒ	Ü.°fÔ=ˆŒØ&+ˆÔ#ð 	�‰Öùò ds   ÂDc                óþ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[S[,          R	S[/# )
r"   Ú	input_idsNr´   r‹   r  Úinputs_embedsr/  rv   r$   )	r0   r0  r:   r   ÚFloatTensorr1  r   r   r   )r&   r'   s   "€r(   r)   rW  ô  s“   ø€ ÷ 2
ñ 2
á×#Ñ# dÕ*ð2
ñ Ÿ™ tÕ+ð2
ñ ×&Ñ&¨Õ-ð	2
ñ
  �ð2
ñ ×(Ñ(¨4Õ/ð2
ñ ˜$•;ð2
ñ Ñ+Õ,ð2
ñ 
!ñ2
r+   c                óˆ  € VR J VR J,          '       d   \        R4      hVf   V P                  V4      pV'       d   Vf   \        V P                  R7      pVfb   Ve   VP	                  4       M^ p\
        P                  ! VP                  ^,          VP                  R7      V,           pVP                  ^ 4      p\        V P                  VVVVR7      p	Tp
V P                  W£R7      pV P                  R V P                  P                    F  pV! V
3RV	RVRVR	VR
V/VB p
K  	  V P                  V
4      p
\        V
VR7      # )Nz:You must specify exactly one of input_ids or inputs_embedsrY  )rW   )r[   rh  r´   r  r‹   )r‹   r´   r  r‹   r  r/  )Úlast_hidden_stater  )Ú
ValueErrorr\  r   r[   Úget_seq_lengthr0   ru   rJ   rW   rØ   r   rb  r`  r_  ra  r   )r4   rg  r´   r‹   r  rh  r/  rv   Úpast_seen_tokensÚcausal_maskr9   r  Údecoder_layers   &&&&&&&,     r(   rF   ÚYoutuModel.forwardñ  se  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ Ø*.×*;Ñ*;¸IÓ*FˆMç˜Ò0Ü*°$·+±+Ô>ˆOàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLä(Ø—;‘;Ø'Ø)Ø+Ø%ô
ˆð &ˆØ"Ÿo™o¨m˜oÓWÐà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÓIˆMÙ)Øñà*ðð %8ðð *ð	ð
 !0ðð $ðð ñŠMñ Jð Ÿ	™	 -Ó0ˆÜ&Ø+Ø+ô
ð 	
r+   )r\  rc  r`  ra  rB  rb  r[  )NNNNNN)rM   rN   rO   rP   r.   r   r   r   rF   rQ   rR   rS   rT   s   @@r(   rU  rU  ß  s=   ù‡ € ÷ó ð   ØØ÷2
ñ 2
ó ó ó  ÷2
ð 2
r+   rU  c                   ó„   a a€ ] tR tRt oRR/tRR/tRR.R.3/tRR/tV 3R	 lt]	]
RV3R
 lR ll4       4       tRtVtV ;t# )ÚYoutuForCausalLMi)  zlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputr9   ÚlogitsÚkeep_full_weightc                óì   <€ \         SV `  V4       \        V4      V n        VP                  V n        \
        P                  ! VP                  VP                  R R7      V n        V P                  4        R# r—   )
r-   r.   rU  r8  r[  r   r›   r5   rt  rd  r¡   s   &&€r(   r.   ÚYoutuForCausalLM.__init__0  sU   ø€ Ü‰Ñ˜Ô Ü Ó'ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒð 	�‰Ör+   c                óP  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[S[ P                  ,          R
S[S[,          RS[	/
# )r"   rg  Nr´   r‹   r  rh  Úlabelsr/  Úlogits_to_keeprv   r$   )
r0   r0  r:   r   ri  r1  r¨   r   r   r   )r&   r'   s   "€r(   r)   ÚYoutuForCausalLM.__annotate__;  s»   ø€ ÷ 6
ñ 6
á×#Ñ# dÕ*ð6
ñ Ÿ™ tÕ+ð6
ñ ×&Ñ&¨Õ-ð	6
ñ
  �ð6
ñ ×(Ñ(¨4Õ/ð6
ñ × Ñ  4Õ'ð6
ñ ˜$•;ð6
ñ ™eŸl™lÕ*ð6
ñ Ñ+Õ,ð6
ñ 
 ñ6
r+   c	                ó–  € V P                   ! RRVRVRVRVRVRV/V	B p
V
P                  p\        V\        4      '       d   \	        V) R4      MTpV P                  VRVR3,          4      pRpVe-   V P                  ! RR	VR
VRV P                  P                  /V	B p\        VVV
P                  V
P                  V
P                  R7      # )ao  
Example:

```python
>>> from transformers import AutoTokenizer, YoutuForCausalLM

>>> model = YoutuForCausalLM.from_pretrained("meta-youtu/Youtu-2-7b-hf")
>>> tokenizer = AutoTokenizer.from_pretrained("meta-youtu/Youtu-2-7b-hf")

>>> prompt = "Hey, are you conscious? Can you talk to me?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
```rg  r´   r‹   r  rh  r/  Nr�   rv  r{  r[  )Úlossrv  r  r9   r9  r3  )r8  rk  rƒ   r¨   Úslicert  Úloss_functionr[   r[  r   r  r9   r9  )r4   rg  r´   r‹   r  rh  r{  r/  r|  rv   Úoutputsr9   Úslice_indicesrv  r  s   &&&&&&&&&,     r(   rF   ÚYoutuForCausalLM.forward9  sÿ   € ð> ,0¯:ª:ñ ,
Øð,
à)ð,
ð &ð,
ð ,ð	,
ð
 (ð,
ð  ð,
ð ñ,
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒØ×%Ò%Ñp¨VÐp¸FÐpÈtÏ{É{×OeÑOeÐpÐioÑpˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r+   )rt  r8  r[  )NNNNNNNr   )rM   rN   rO   rP   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr.   r   r   rF   rQ   rR   rS   rT   s   @@r(   rs  rs  )  sf   ù‡ € à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÐ/Ð0€Jõð Ø÷6
ñ 6
ó ó ÷6
ð 6
r+   rs  )r  )rÆ   rÆ   )rÆ   )NrÆ   )r7  rU  rs  )@rÇ   Úcollections.abcr   r0   r   Ú r   r@  Úactivationsr   Úcache_utilsr   r   Ú
generationr	   Úintegrationsr
   Úmasking_utilsr   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   Úconfiguration_youtur   r·   r    rV   r•   r®   rÄ   rË   rÑ   rÕ   rÞ   rä   ræ   r#  r7  rU  rs  Ú__all__r3  r+   r(   Ú<module>rœ     sT  ðó6 Ý $ã Ý å &Ý !ß .Ý )Ý 7Ý /Ý BÝ 9ß Oß Kß FÝ &ß IÑ IÝ 0ß GÝ 5Ý ,ñ ˜YÓ'ôJ�2—9‘9ó Jó (ðJô(6<˜2Ÿ9™9ô 6<ôrˆr�y‰yô õ 	U÷%ô20òò(ñ Ð-Ó.óó /ðô2$ôNE)�R—Y‘Yô E)ôP(Ð2ô (ðV ôD˜?ó Dó ðDð8 ôF
Ð%ó F
ó ðF
ðR ôG
Ð+¨_ó G
ó ðG
òT E‚r+   