+
    œ°³jÕZ  ã                   ó�  € ^ RI Ht ^ RIt^ RIHt ^RIHt ^RIHtH	t	 ^RI
Ht ^RIHtHt ^RIHt ^RIHt ^R	IHtHt ^R
IHtHt ^RIHtHt ^RIHt ^RIHtH t H!t! ^RI"H#t# ^RI$H%t%H&t& ^RI'H(t( ^RI)H*t* ]! R4       ! R R]PV                  4      4       t, ! R R]PV                  4      t-R t.]! R4      R*R l4       t/R R lt0R+R R llt1]! ]/4       ! R R]PV                  4      4       t2 ! R  R!]PV                  4      t3 ! R" R#]4      t4]  ! R$ R%]4      4       t5]  ! R& R']54      4       t6]  ! R( R)]5]4      4       t7. R,Ot8R# )-é    )ÚCallableN)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_kernel_forward_from_hubÚuse_kernelized_func)Úcreate_causal_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputs©ÚHyperCLOVAXConfigÚRMSNormc                   óZ   a a€ ] tR t^+t oRV3R lV 3R llltV3R lR ltR tRtVtV ;t	# )ÚHyperCLOVAXRMSNormc                ó$   <€ V ^8„  d   QhRS[ RR/# )é   ÚepsÚreturnN)Úfloat)ÚformatÚ__classdict__s   "€Úu/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/hyperclovax/modeling_hyperclovax.pyÚ__annotate__ÚHyperCLOVAXRMSNorm.__annotate__-   s   ø€ ÷ $ñ $©ð $¸$ñ $ó    c                óŽ   <€ \         SV `  4        \        P                  ! \        P
                  ! V4      4      V n        W n        R# )z1
HyperCLOVAXRMSNorm is equivalent to T5LayerNorm
N)ÚsuperÚ__init__ÚnnÚ	ParameterÚtorchÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizer!   Ú	__class__s   &&&€r&   r,   ÚHyperCLOVAXRMSNorm.__init__-   s/   ø€ ô 	‰ÑÔÜ—l’l¤5§:¢:¨kÓ#:Ó;ˆŒØ #Ör)   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r    Úhidden_statesr"   )r/   ÚTensor)r$   r%   s   "€r&   r'   r(   5   s#   ø€ ÷ ;ñ ;¡U§\¡\ð ;±e·l±lñ ;r)   c                ó:  € VP                   pVP                  \        P                  4      pVP	                  ^4      P                  RRR7      pV\        P                  ! W0P                  ,           4      ,          pV P                  VP                  V4      ,          # )r    T)Úkeepdiméÿÿÿÿ)	ÚdtypeÚtor/   Úfloat32ÚpowÚmeanÚrsqrtr2   r1   )r3   r8   Úinput_dtypeÚvariances   &&  r&   ÚforwardÚHyperCLOVAXRMSNorm.forward5   sw   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯ª°H×?TÑ?TÕ4TÓ(UÕUˆØ�{‰{˜]×-Ñ-¨kÓ:Õ:Ð:r)   c                ó^   € \        V P                  P                  4       R V P                   2# )z, eps=)Útupler1   Úshaper2   )r3   s   &r&   Ú
extra_reprÚHyperCLOVAXRMSNorm.extra_repr<   s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr)   )r2   r1   )g�íµ ÷Æ°>)
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r,   rE   rJ   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r5   r%   s   @@r&   r   r   +   s%   ù‡ € ÷$õ $÷;ð ;÷Jò Jr)   r   c                   óÎ   a a€ ] tR t^@t o]! RRR7      R
V3R lV 3R lll4       t]]! RRR7      R
V3R lR ll4       4       t]P                  ! 4       ]
R 4       4       tR	tVtV ;t# )ÚHyperCLOVAXRotaryEmbeddingÚdevicez5.18)Úversionc                ó    <€ V ^8„  d   QhRS[ /# ©r    Úconfigr   )r$   r%   s   "€r&   r'   Ú'HyperCLOVAXRotaryEmbedding.__annotate__B   s   ø€ ÷ Oñ OÑ0ñ Or)   c                óä  <€ \         SV `  4        VP                  V n        VP                  V n        Wn        V P
                  P                  R ,          V n        V P                  pV P                  R8w  d   \        V P                  ,          pV! V P
                  V4      w  q@n
        \        P                  ! VRR7      V n        \        P                  ! VP                  4       RR7      V n        R# )Ú	rope_typeÚdefaultF)Ú
persistentN)r+   r,   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenrZ   Úrope_parametersr]   Úcompute_default_rope_parametersr   Úattention_scalingr-   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)r3   rZ   rV   Úrope_init_fnrg   r5   s   &&&  €r&   r,   Ú#HyperCLOVAXRotaryEmbedding.__init__A   s¬   ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àŒàŸ™×4Ñ4°[ÕAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÔ&Ü.¨t¯~©~Õ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÔ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÔ!NˆÖr)   c                óP   <€ V ^8„  d   QhRS[ RS[S[P                  S[3,          /# )r    rZ   r"   )r   rH   r/   r9   r#   )r$   r%   s   "€r&   r'   r[   T   s+   ø€ ÷ 5ñ 5Ù!ð5á	‰u�|‰|™UÐ"Õ	#ñ5r)   c           	     ó0  € V P                   R,          p\        V RR4      ;'       g    V P                  V P                  ,          pRpRV\        P
                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚhead_dimNg      ð?©r=   )rc   Úgetattrr4   Únum_attention_headsr/   Úaranger#   r>   )rZ   rV   ÚkwargsÚbaseÚdimÚattention_factorrg   s   &&,    r&   rd   Ú:HyperCLOVAXRotaryEmbedding.compute_default_rope_parametersR   s~   € ð ×%Ñ% lÕ3ˆÜ�f˜j¨$Ó/×cÐc°6×3EÑ3EÈ×IcÑIcÕ3cˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r)   c                ó^  € V P                   R,          P                  VP                  ^ ,          R	^4      P                  \        P
                  VP                  R7      pVR
,          P                  4       p\        VP                  P                  \        4      '       d2   VP                  P                  R8w  d   VP                  P                  MRp\        VRR7      ;_uu_ 4        W4,          P                  ^^4      p\        P                  ! Wf3R	R7      pVP                  4       V P                  ,          pVP                  4       V P                  ,          p	R R R 4       XP                  VP                   R7      X	P                  VP                   R7      3#   + '       g   i     LH; i)N)r=   rV   ÚmpsÚcpuF)Údevice_typeÚenabled©rv   rp   )NºNNNNr<   )r   Nr   )rg   ÚexpandrI   r>   r/   r#   rV   Ú
isinstanceÚtypeÚstrr   Ú	transposeÚcatÚcosre   Úsinr=   )
r3   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedr|   ÚfreqsÚembr†   r‡   s
   &&&       r&   rE   Ú"HyperCLOVAXRotaryEmbedding.forwardh   s2  € ð �M‰M˜-Õ(×/Ñ/°×0BÑ0BÀ1Õ0EÀrÈ1ÓM×PÑPÔW\×WbÑWbÐkl×ksÑksÐPÓtð 	ð !-¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆä¨¸U×CÖCØ&Õ>×IÑIÈ!ÈQÓOˆEÜ—)’)˜U˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆC÷	 Dð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷ D×Cús   Ã(A4FÆF,	)re   rZ   rg   ra   ri   rb   r]   ©N)rL   rM   rN   rO   r   r,   Ústaticmethodrd   r/   Úno_gradr   rE   rP   rQ   rR   rS   s   @@r&   rU   rU   @   su   ù‡ € Ù�X vÔ.÷Oô Oó /ðOð  Ù�X vÔ.÷5ñ 5ó /ó ð5ð( ‡]‚]ƒ_Øñ<ó ó ÷<ð <r)   rU   c                óÌ   € V RRV P                   R,          ^,          13,          pV RV P                   R,          ^,          R13,          p\        P                  ! V) V3RR7      # )z*Rotates half the hidden dims of the input..Nr~   r<   )rI   r/   r…   )rˆ   Úx1Úx2s   &  r&   Úrotate_halfr•   {   sZ   € à	
ˆ3Ð"�!—'‘'˜"•+ Õ"Ð"Ð"Õ	#€BØ	
ˆ3�—‘˜•˜qÕ Ñ"Ð"Õ	#€BÜ�9Š9�r�c˜2�Y BÔ'Ð'r)   Úrotary_pos_embc                óÈ   € VP                  V4      pVP                  V4      pW,          \        V 4      V,          ,           pW,          \        V4      V,          ,           pWV3# )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)Ú	unsqueezer•   )ÚqÚkr†   r‡   Úunsqueeze_dimÚq_embedÚk_embeds   &&&&&  r&   Úapply_rotary_pos_embrž   ‚   sS   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�wœ; q›>¨CÕ/Õ0€GØ�wœ; q›>¨CÕ/Õ0€GØÐÐr)   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r    r8   Ún_repr"   )r/   r9   Úint)r$   s   "r&   r'   r'   œ   s.   € ÷ 	Uñ 	UœUŸ\™\ð 	U´#ð 	U¼%¿,¹,ñ 	Ur)   c                ó˜   € V P                   w  r#rEV^8X  d   V # V R,          P                  W#WV4      p V P                  W#V,          WE4      # )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
)r   r   Nr   r   )rI   r€   Úreshape)r8   r    ÚbatchÚnum_key_value_headsÚslenro   s   &&    r&   Ú	repeat_kvr§   œ   sU   € ð
 2?×1DÑ1DÑ.€E Ø�„zØÐØ!Ð"2Õ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÕ(CÀTÓTÐTr)   c                ó   € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R\        R	\
        \        ,          /# )
r    ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNÚscalingÚdropoutrt   )r-   ÚModuler/   r9   r#   r   r   )r$   s   "r&   r'   r'   ¨   sw   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ð%ô ð%ô Ô'Õ(ñ%r)   c                 ó"  € \        W P                  4      p\        W0P                  4      p	\        P                  ! WP	                  ^^4      4      V,          p
Ve	   W¤,           p
\
        P                  P                  V
R\        P                  R7      P                  VP                  4      p
\
        P                  P                  W¦V P                  R7      p
\        P                  ! W©4      pVP	                  ^^4      P                  4       pWº3# )r    )rv   r=   )ÚpÚtrainingr<   )r§   Únum_key_value_groupsr/   Úmatmulr„   r-   Ú
functionalÚsoftmaxr?   r>   r=   r¯   r³   Ú
contiguous)r©   rª   r«   r¬   r­   r®   r¯   rt   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs   &&&&&&&,    r&   Úeager_attention_forwardr½   ¨   sÒ   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÕL€LØÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r)   c                   ó\   a a€ ] tR t^Át oRtRV3R lV 3R llltRV3R lR lltRtVtV ;t	# )	ÚHyperCLOVAXAttentionz=Multi-headed attention from 'Attention Is All You Need' paperc                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# )r    rZ   Ú	layer_idxN©r   r¡   )r$   r%   s   "€r&   r'   Ú!HyperCLOVAXAttention.__annotate__Å   s    ø€ ÷ 
ñ 
Ñ0ð 
¹SÀ4½Zñ 
r)   c                ó”  <€ \         SV `  4        Wn        W n        \	        VR VP
                  VP                  ,          4      V n        VP                  VP                  ,          V n	        VP                  V n        VP                  V n        RV n        \        P                  ! VP
                  VP                  V P                  ,          VP                   R7      V n        \        P                  ! VP
                  VP                  V P                  ,          VP                   R7      V n        \        P                  ! VP
                  VP                  V P                  ,          VP                   R7      V n        \        P                  ! VP                  V P                  ,          VP
                  VP                   R7      V n        R# )ro   T©ÚbiasN)r+   r,   rZ   rÁ   rq   r4   rr   ro   r¥   r´   Úattention_multiplierr®   Úattention_dropoutÚ	is_causalr-   ÚLinearÚattention_biasÚq_projÚk_projÚv_projÚo_proj©r3   rZ   rÁ   r5   s   &&&€r&   r,   ÚHyperCLOVAXAttention.__init__Å   sF  ø€ Ü‰ÑÔØŒØ"ŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÕ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Õ$\ˆÔ!Ø×2Ñ2ˆŒØ!'×!9Ñ!9ˆÔØˆŒä—i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×&Ñ&¨¯©Õ6¸×8JÑ8JÐQW×QfÑQfô
ˆŽr)   c                ó  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          R,          RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  S[ P                  3,          /# )r    r8   Úposition_embeddingsNr­   Úpast_key_valuesrt   r"   )r/   r9   rH   r   r   r   )r$   r%   s   "€r&   r'   rÃ   Ü   s‡   ø€ ÷ &)ñ &)á—|‘|ð&)ñ #¡5§<¡<±·±Ð#=Õ>ÀÕEð&)ñ Ÿ™ tÕ+ð	&)ñ
  �ð&)ñ Ñ+Õ,ð&)ñ 
‰u�|‰|™UŸ\™\Ð)Õ	*ñ&)r)   c                ó  € VP                   R R p. VORNV P                  N5pV P                  V4      P                  V4      P	                  ^^4      pV P                  V4      P                  V4      P	                  ^^4      p	V P                  V4      P                  V4      P	                  ^^4      p
Vw  r¼\        W‰W¼4      w  r‰Ve   VP                  WšV P                  4      w  rš\        P                  ! V P                  P                  \        4      pT! V VV	V
V3RV P                  '       g   RMV P                   RV P"                  /VB w  rïVP$                  ! . VORN5!  P'                  4       pV P)                  V4      pWï3# )Nr¯   ç        r®   r<   )rI   ro   rÌ   Úviewr„   rÍ   rÎ   rž   ÚupdaterÁ   r   Úget_interfacerZ   Ú_attn_implementationr½   r³   rÈ   r®   r£   r¸   rÏ   )r3   r8   rÓ   r­   rÔ   rt   Úinput_shapeÚhidden_shapeÚquery_statesr¹   rº   r†   r‡   Úattention_interfacer¼   r»   s   &&&&&,          r&   rE   ÚHyperCLOVAXAttention.forwardÜ   s‡  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—k‘k +Ó.ˆØÐ(Ð(r)   )rÈ   rZ   ro   rÉ   rÍ   rÁ   r´   rÏ   rÌ   r®   rÎ   r�   )NNN)
rL   rM   rN   rO   Ú__doc__r,   rE   rP   rQ   rR   rS   s   @@r&   r¿   r¿   Á   s!   ù‡ € áG÷
õ 
÷.&)÷ &)ò &)r)   r¿   c                   ó8   a a€ ] tR tRt oV 3R ltR tRtVtV ;t# )ÚHyperCLOVAXMLPi  c                ó  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        \
        P                  ! V P                  V P                  VP                  R 7      V n        \
        P                  ! V P                  V P                  VP                  R 7      V n	        \
        P                  ! V P                  V P                  VP                  R 7      V n
        \        VP                  ,          V n        R# )rÅ   N)r+   r,   rZ   r4   Úintermediate_sizer-   rÊ   Úmlp_biasÚ	gate_projÚup_projÚ	down_projr   Ú
hidden_actÚact_fn©r3   rZ   r5   s   &&€r&   r,   ÚHyperCLOVAXMLP.__init__  s¶   ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔÜŸš 4×#3Ñ#3°T×5KÑ5KÐRX×RaÑRaÔbˆŒÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPV×P_ÑP_Ô`ˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRX×RaÑRaÔbˆŒÜ˜V×.Ñ.Õ/ˆŽr)   c                ó�   € V P                  V P                  V P                  V4      4      V P                  V4      ,          4      pV# r�   )rè   rê   ræ   rç   )r3   rˆ   rè   s   && r&   rE   ÚHyperCLOVAXMLP.forward  s6   € Ø—N‘N 4§;¡;¨t¯~©~¸aÓ/@Ó#AÀDÇLÁLÐQRÃOÕ#SÓTˆ	ØÐr)   )rê   rZ   rè   ræ   r4   rä   rç   ©	rL   rM   rN   rO   r,   rE   rP   rQ   rR   rS   s   @@r&   râ   râ     s   ù‡ € õ0÷ò r)   râ   c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚHyperCLOVAXDecoderLayeri  c                ó&   <€ V ^8„  d   QhRS[ RS[/# )r    rZ   rÁ   rÂ   )r$   r%   s   "€r&   r'   Ú$HyperCLOVAXDecoderLayer.__annotate__  s   ø€ ÷ 
ñ 
Ñ0ð 
¹Sñ 
r)   c                ó„  <€ \         SV `  4        VP                  V n        \        WR 7      V n        \        V4      V n        \        VP                  VP                  R7      V n	        \        VP                  VP                  R7      V n
        VP                  V n        VP                  '       d"   \        VP                  VP                  R7      M\        P                  ! 4       V n        VP                  '       d)   \        VP                  VP                  R7      V n        R# \        P                  ! 4       V n        R# ))rZ   rÁ   ©r!   N)r+   r,   r4   r¿   Ú	self_attnrâ   Úmlpr   Úrms_norm_epsÚinput_layernormÚpost_attention_layernormÚresidual_multiplierÚuse_post_normr-   ÚIdentityÚ
post_norm1Ú
post_norm2rÐ   s   &&&€r&   r,   Ú HyperCLOVAXDecoderLayer.__init__  sò   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔÜ-°VÔQˆŒä! &Ó)ˆŒÜ1°&×2DÑ2DÈ&×J]ÑJ]Ô^ˆÔÜ(:¸6×;MÑ;MÐSY×SfÑSfÔ(gˆÔ%Ø#)×#=Ñ#=ˆÔ ð PV×Oc×OcÐOcÔ˜v×1Ñ1°v×7JÑ7JÕKÔik×itÒitÓivð 	Œð PV×Oc×OcÐOcÔ˜v×1Ñ1°v×7JÑ7JÔKð 	ŽÜik×itÒitÓivð 	Žr)   c                ó.  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[ P                  S[ P                  3,          R,          RS[S[,          R	S[ P                  /# )
r    r8   r­   Nr‰   rÔ   Ú	use_cacherÓ   rt   r"   )r/   r9   Ú
LongTensorr   ÚboolrH   r   r   )r$   r%   s   "€r&   r'   ró   '  sœ   ø€ ÷ 3ñ 3á—|‘|ð3ñ Ÿ™ tÕ+ð3ñ ×&Ñ&¨Õ-ð	3ñ
  �ð3ñ ˜$•;ð3ñ #¡5§<¡<±·±Ð#=Õ>ÀÕEð3ñ Ñ+Õ,ð3ñ 
�‰ñ3r)   c                ó\  € TpV P                  V4      pV P                  ! RRVRVRVRVRVRV/VB w  rV P                  V4      pW�V P                  ,          ,           pTpV P	                  V4      pV P                  V4      pV P                  V4      pW�V P                  ,          ,           pV# )aÎ  
Args:
    hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
    attention_mask (`torch.FloatTensor`, *optional*):
        attention mask of size `(batch_size, sequence_length)` if flash attention is used or `(batch_size, 1,
        query_sequence_length, key_sequence_length)` if default attention is used.
    output_attentions (`bool`, *optional*):
        Whether or not to return the attentions tensors of all attention layers. See `attentions` under
        returned tensors for more detail.
    use_cache (`bool`, *optional*):
        If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
        (see `past_key_values`).
    past_key_values (`Cache`, *optional*): cached past key and value projection states
    position_embeddings (`tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*):
        Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,
        with `head_dim` being the embedding dimension of each attention head.
    kwargs (`dict`, *optional*):
        Arbitrary kwargs to be ignored, used for FSDP and other methods that injects code
        into the model
r8   r­   r‰   rÔ   r  rÓ   © )rù   rö   rþ   rû   rú   r÷   rÿ   )
r3   r8   r­   r‰   rÔ   r  rÓ   rt   ÚresidualÚ_s
   &&&&&&&,  r&   rE   ÚHyperCLOVAXDecoderLayer.forward'  sÛ   € ð< !ˆØ×,Ñ,¨]Ó;ˆàŸ>š>ñ 
Ø'ð
à)ð
ð &ð
ð ,ð	
ð
  ð
ð !4ð
ð ñ
Ñˆð Ÿ™¨Ó6ˆØ °4×3KÑ3KÕ#KÕKˆð !ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØŸ™¨Ó6ˆØ °4×3KÑ3KÕ#KÕKˆØÐr)   )r4   rù   r÷   rú   rþ   rÿ   rû   rö   )NNNFNrï   rS   s   @@r&   rñ   rñ     s   ù‡ € ÷
ó 
÷"3÷ 3ò 3r)   rñ   c                   ó`   a € ] tR tRt o RtRtR.tR.tRtRt	Rt
RtRtR]R]/tV 3R ltR	tV tR
# )ÚHyperCLOVAXPreTrainedModeli]  ÚmodelTrñ   rÔ   r8   Ú
attentionsc                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # rY   r   )r$   r%   s   "€r&   r'   Ú'HyperCLOVAXPreTrainedModel.__annotate__]  s   ø‡ ‚ áÑò r)   r  N)rL   rM   rN   rO   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendrñ   r¿   Ú_can_record_outputsÚ__annotate_func__rP   rQ   )r%   s   @r&   r  r  ]  s]   ø‡ € ð  ÐØ&*Ð#Ø2Ð3ÐØ#4Ð"5ÐØÐØ€NØÐà!ÐØ"&ÐàÐ0ØÐ*ðÐ÷ ƒ r)   r  c                   ór   a a€ ] tR tRt oV3R lV 3R llt]]]RV3R lR ll4       4       4       tRt	Vt
V ;t# )ÚHyperCLOVAXModelip  c                ó    <€ V ^8„  d   QhRS[ /# rY   r   )r$   r%   s   "€r&   r'   ÚHyperCLOVAXModel.__annotate__r  s   ø€ ÷ ñ Ñ0ñ r)   c           	     ó6  <€ \         SV `  V4       VP                  V n        VP                  V n        \
        P                  ! VP                  VP                  V P                  4      V n        \
        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n        \        VP                  VP                  R 7      V n        \#        VR7      V n        RV n        VP(                  V n        V P+                  4        R# u upi )rõ   ©rZ   FN)r+   r,   Úpad_token_idÚpadding_idxÚ
vocab_sizer-   Ú	Embeddingr4   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersrñ   Úlayersr   rø   ÚnormrU   Ú
rotary_embÚgradient_checkpointingÚembedding_multiplierÚ	post_initrÐ   s   && €r&   r,   ÚHyperCLOVAXModel.__init__r  sØ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸLšL¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m’mÜINÈv×OgÑOgÔIhÓiÑIh¸IÔ$ VÖ7ÑIhÑió
ˆŒô ' v×'9Ñ'9¸v×?RÑ?RÔSˆŒ	Ü4¸FÔCˆŒØ&+ˆÔ#Ø$*×$?Ñ$?ˆÔ!ð 	�‰Öùò js   ÂDc                óþ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[S[,          R	S[/# )
r    Ú	input_idsNr­   r‰   rÔ   Úinputs_embedsr  rt   r"   )	r/   r  r9   r   ÚFloatTensorr  r   r   r   )r$   r%   s   "€r&   r'   r  †  s“   ø€ ÷ 5
ñ 5
á×#Ñ# dÕ*ð5
ñ Ÿ™ tÕ+ð5
ñ ×&Ñ&¨Õ-ð	5
ñ
  �ð5
ñ ×(Ñ(¨4Õ/ð5
ñ ˜$•;ð5
ñ Ñ+Õ,ð5
ñ 
!ñ5
r)   c                ó¬  € VR J VR J,          '       d   \        R4      hVf   V P                  V4      pWPP                  ,          pV'       d   Vf   \        V P                  R7      pVfb   Ve   VP                  4       M^ p\        P                  ! VP                  ^,          VP                  R7      V,           pVP                  ^ 4      p\        V P                  VVVVR7      p	Tp
V P                  W£R7      pV P                  R V P                  P                    F  pV! V
3RV	RVRVR	VR
V/VB p
K  	  V P                  V
4      p
\!        V
VR7      # )Nz:You must specify exactly one of input_ids or inputs_embedsr   )rV   )rZ   r2  r­   rÔ   r‰   )r‰   r­   r‰   rÔ   r  rÓ   )Úlast_hidden_staterÔ   )Ú
ValueErrorr%  r-  r   rZ   Úget_seq_lengthr/   rs   rI   rV   r˜   r
   r+  r)  r(  r*  r   )r3   r1  r­   r‰   rÔ   r2  r  rt   Úpast_seen_tokensÚcausal_maskr8   rÓ   Údecoder_layers   &&&&&&&,     r&   rE   ÚHyperCLOVAXModel.forwardƒ  ss  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ Ø ×-Ñ-¨iÓ8ˆMà%×(AÑ(AÕAˆç˜Ò0Ü*°$·+±+Ô>ˆOàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLä(Ø—;‘;Ø'Ø)Ø+Ø%ô
ˆð &ˆØ"Ÿo™o¨m˜oÓWÐà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÓIˆMÙ)Øñà*ðð *ðð !0ð	ð
 $ðð %8ðð ñŠMñ Jð Ÿ	™	 -Ó0ˆä&Ø+Ø+ô
ð 	
r)   )r%  r-  r,  r)  r*  r"  r+  r#  )NNNNNN)rL   rM   rN   rO   r,   r   r   r   rE   rP   rQ   rR   rS   s   @@r&   r  r  p  s=   ù‡ € ÷ó ð"  ØØ÷5
ñ 5
ó ó ó  ÷5
ð 5
r)   r  c                   ó„   a a€ ] tR tRt oRR/tRR/tRR.R.3/tRR/tV 3R	 lt]	]
RV3R
 lR ll4       4       tRtVtV ;t# )ÚHyperCLOVAXForCausalLMi¾  zlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputr8   ÚlogitsÚkeep_full_weightc                óì   <€ \         SV `  V4       \        V4      V n        VP                  V n        \
        P                  ! VP                  VP                  R R7      V n        V P                  4        R# )FrÅ   N)
r+   r,   r  r  r#  r-   rÊ   r4   r>  r.  rë   s   &&€r&   r,   ÚHyperCLOVAXForCausalLM.__init__Å  sU   ø€ Ü‰Ñ˜Ô Ü% fÓ-ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒð 	�‰Ör)   c                óP  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[S[ P                  ,          R
S[S[,          RS[	/
# )r    r1  Nr­   r‰   rÔ   r2  Úlabelsr  Úlogits_to_keeprt   r"   )
r/   r  r9   r   r3  r  r¡   r   r   r   )r$   r%   s   "€r&   r'   Ú#HyperCLOVAXForCausalLM.__annotate__Ð  s»   ø€ ÷ 6
ñ 6
á×#Ñ# dÕ*ð6
ñ Ÿ™ tÕ+ð6
ñ ×&Ñ&¨Õ-ð	6
ñ
  �ð6
ñ ×(Ñ(¨4Õ/ð6
ñ × Ñ  4Õ'ð6
ñ ˜$•;ð6
ñ ™eŸl™lÕ*ð6
ñ Ñ+Õ,ð6
ñ 
 ñ6
r)   c	                óÌ  € V P                   ! RRVRVRVRVRVRV/V	B p
V
P                  p\        V\        4      '       d   \	        V) R4      MTpV P                  VRVR3,          4      V P                  P                  ,          pRpVe-   V P                  ! RR	VR
VRV P                  P                  /V	B p\        VVV
P                  V
P                  V
P                  R7      # )aÆ  
Example:

```python
>>> from transformers import AutoTokenizer, HyperCLOVAXForCausalLM

>>> model = HyperCLOVAXForCausalLM.from_pretrained("naver-hyperclovax/HyperCLOVAX-SEED-Think-14B")
>>> tokenizer = AutoTokenizer.from_pretrained("naver-hyperclovax/HyperCLOVAX-SEED-Think-14B")

>>> prompt = "Hey, are you conscious? Can you talk to me?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me? Are you okay?" The man was confused and answered, "Yes." Then the woman asked.
```r1  r­   r‰   rÔ   r2  r  Nr   r@  rE  r#  )Úlossr@  rÔ   r8   r  r  )r  r5  r�   r¡   Úslicer>  rZ   Úlogits_scalingÚloss_functionr#  r   rÔ   r8   r  )r3   r1  r­   r‰   rÔ   r2  rE  r  rF  rt   Úoutputsr8   Úslice_indicesr@  rI  s   &&&&&&&&&,     r&   rE   ÚHyperCLOVAXForCausalLM.forwardÎ  s  € ð> —*’*ñ 
Øð
à)ð
ð &ð
ð ,ð	
ð
 (ð
ð  ð
ð ñ
ˆð  ×1Ñ1ˆÜ8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆà—‘˜m¨A¨}¸aÐ,?Õ@ÓAÀDÇKÁK×D^ÑD^Õ^ˆàˆØÒØ×%Ò%Ñp¨VÐp¸FÐpÈtÏ{É{×OeÑOeÐpÐioÑpˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r)   )r>  r  r#  )NNNNNNNr   )rL   rM   rN   rO   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr,   r   r   rE   rP   rQ   rR   rS   s   @@r&   r=  r=  ¾  sf   ù‡ € à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÐ/Ð0€Jõð Ø÷6
ñ 6
ó ó ÷6
ð 6
r)   r=  )é   )rÖ   )r  r  r=  )9Úcollections.abcr   r/   Útorch.nnr-   Úactivationsr   Úcache_utilsr   r   Ú
generationr   Úintegrationsr   r	   Úmasking_utilsr
   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   Úconfiguration_hyperclovaxr   r°   r   rU   r•   rž   r§   r½   r¿   râ   rñ   r  r  r=  Ú__all__r  r)   r&   Ú<module>rg     sP  ðõ* %ã Ý å !ß .Ý )ß LÝ /Ý 9ß Oß Kß FÝ &ß IÑ IÝ 0ß GÝ 5Ý 8ñ ˜YÓ'ôJ˜Ÿ™ó Jó (ðJô(8< §¡ô 8<òv(ñ Ð-Ó.óó /ðõ2	U÷%ñ2 Ð)Ó*ô@)˜2Ÿ9™9ó @)ó +ð@)ôF�R—Y‘Yô ô EÐ8ô EðP ô ó ó ðð$ ôJ
Ð1ó J
ó ðJ
ðZ ôG
Ð7¸ó G
ó ðG
òT W‚r)   