+
    œ°³j�"  ã                   óŠ  € R t ^ RIt^ RIHt ^ RIHt ^RIHt ^RIH	t	 ^RI
Ht ^RIHtHtHt ^RIHt ^RIHtHtHtHtHtHtHt ]! R	R
7      ] ! R R]4      4       4       t ! R R]4      t ! R R]4      t ! R R]4      t ! R R]4      t] ! R R]4      4       t] ! R R]4      4       t ] ! R R]4      4       t!. ROt"R# )z%HyperCLOVAX modular model definition.N)Ústrict)ÚCache)ÚCausalLMOutputWithPast)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)ÚGraniteConfig)ÚGraniteAttentionÚGraniteDecoderLayerÚGraniteForCausalLMÚGraniteModelÚGranitePreTrainedModelÚGraniteRMSNormÚGraniteRotaryEmbeddingz,naver-hyperclovax/HyperCLOVAX-SEED-Think-14B)Ú
checkpointc                   óX   a a€ ] tR t^$t oRtRtRtRtRtV 3R lt	R t
V3R ltRtVtV ;t# )	ÚHyperCLOVAXConfigaÜ  
embedding_multiplier (`float`, *optional*, defaults to `1.0`):
    Scaling factor applied to the token embedding outputs. Used in MuP to control the
    scale of the embedding activations.
logits_scaling (`float`, *optional*, defaults to `1.0`):
    Scaling factor **multiplied** to the final logits before loss computation or sampling.
    Used in MuP to ensure consistent output scale across model sizes. Note: unlike
    [`GraniteConfig`], this is a multiplier, not a divisor.
residual_multiplier (`float`, *optional*, defaults to `1.0`):
    Scaling factor applied to each sub-layer output before adding to the residual stream.
    Used in Maximal Update Parametrization (MuP) to stabilize training across model sizes.
attention_multiplier (`float`, *optional*, defaults to `head_dim ** -0.5`):
    Scaling factor applied to attention logits before softmax, replacing the standard
    `1 / sqrt(head_dim)` scaling. Set explicitly for MuP-based training; when `None`,
    defaults to the standard value.
use_post_norm (`bool`, *optional*, defaults to `True`):
    Whether to apply an extra RMSNorm after each sub-layer output (Peri-Layer Normalization).

```python
>>> from transformers import HyperCLOVAXModel, HyperCLOVAXConfig

>>> # Initializing a HyperCLOVAX style configuration
>>> configuration = HyperCLOVAXConfig()

>>> # Initializing a model from the configuration
>>> model = HyperCLOVAXModel(configuration)

>>> # Accessing the model configuration
>>> configuration = model.config
```ÚhyperclovaxNTc                óØ   <€ V P                   f#   V P                  V P                  ,          V n         \        SV `  ! R/ VB  V P
                  f   V P                   R,          V n        R # R # )N© g      à¿)Úhead_dimÚhidden_sizeÚnum_attention_headsÚsuperÚ__post_init__Úattention_multiplier)ÚselfÚkwargsÚ	__class__s   &,€Út/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/hyperclovax/modular_hyperclovax.pyr   ÚHyperCLOVAXConfig.__post_init__P   s[   ø€ ð �=‰=Ò Ø ×,Ñ,°×0HÑ0HÕHˆDŒMä‰ÒÑ' Ò'ð ×$Ñ$Ò,Ø(,¯©°tÕ(;ˆDÖ%ñ -ó    c                ó–   € V P                   V P                  ,          ^ 8w  d'   \        RV P                    RV P                   R24      hR# )zCValidates that `hidden_size` is divisible by `num_attention_heads`.zThe hidden size (z6) is not a multiple of the number of attention heads (z).N)r   r   Ú
ValueError)r   s   &r    Úvalidate_architectureÚ'HyperCLOVAXConfig.validate_architecture]   sS   € à×Ñ˜d×6Ñ6Õ6¸!Ô;ÜØ# D×$4Ñ$4Ð#5ð 6Ø×2Ñ2Ð3°2ð7óð ñ <r"   c                óZ   <€ V ^8„  d   Qh/ S[ R,          ;R&   S[R,          ;R&   S[;R&   # )é   Nr   r   Úuse_post_norm)ÚintÚfloatÚbool)ÚformatÚ__classdict__s   "€r    Ú__annotate__ÚHyperCLOVAXConfig.__annotate__$   s:   ø‡ ‚ ñH �D�jÑñI ñN   $�,Ñ-ñO ñT ÑòU r"   )r   r   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Ú
model_typer   r   r)   r   r%   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r   r.   s   @@r    r   r   $   s7   ù‡ € ñð> €Jà€Hð *.Ðð €Mõ<ò÷s … r"   r   c                   ó   € ] tR t^ftRtR# )ÚHyperCLOVAXRMSNormr   N©r1   r2   r3   r4   r8   r   r"   r    r=   r=   f   ó   † Ûr"   r=   c                   ó   € ] tR t^jtRtR# )ÚHyperCLOVAXRotaryEmbeddingr   Nr>   r   r"   r    rA   rA   j   r?   r"   rA   c                   ó   € ] tR t^ntRtR# )ÚHyperCLOVAXAttentionr   Nr>   r   r"   r    rC   rC   n   r?   r"   rC   c                   óT   a a€ ] tR t^rt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚHyperCLOVAXDecoderLayerc                ó&   <€ V ^8„  d   QhRS[ RS[/# )r(   ÚconfigÚ	layer_idx)r   r*   )r-   r.   s   "€r    r/   Ú$HyperCLOVAXDecoderLayer.__annotate__s   s   ø€ ÷ 
ñ 
Ñ0ð 
¹Sñ 
r"   c                óh  <€ \         SV `  W4       VP                  '       d"   \        VP                  VP
                  R 7      M\        P                  ! 4       V n        VP                  '       d)   \        VP                  VP
                  R 7      V n	        R# \        P                  ! 4       V n	        R# ))ÚepsN)
r   Ú__init__r)   r=   r   Úrms_norm_epsÚnnÚIdentityÚ
post_norm1Ú
post_norm2)r   rG   rH   r   s   &&&€r    rL   Ú HyperCLOVAXDecoderLayer.__init__s   s‰   ø€ Ü‰Ñ˜Ô+ð PV×Oc×OcÐOcÔ˜v×1Ñ1°v×7JÑ7JÕKÔik×itÒitÓivð 	Œð PV×Oc×OcÐOcÔ˜v×1Ñ1°v×7JÑ7JÔKð 	ŽÜik×itÒitÓivð 	Žr"   c                ó.  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[ P                  S[ P                  3,          R,          RS[S[,          R	S[ P                  /# )
r(   Úhidden_statesÚattention_maskNÚposition_idsÚpast_key_valuesÚ	use_cacheÚposition_embeddingsr   Úreturn)ÚtorchÚTensorÚ
LongTensorr   r,   Útupler   r   )r-   r.   s   "€r    r/   rI   }   sœ   ø€ ÷ ñ á—|‘|ðñ Ÿ™ tÕ+ðñ ×&Ñ&¨Õ-ð	ñ
  �ðñ ˜$•;ðñ #¡5§<¡<±·±Ð#=Õ>ÀÕEðñ Ñ+Õ,ðñ 
�‰ñr"   c                ó\  € TpV P                  V4      pV P                  ! RR VRVRVRVRVRV/VB w  rV P                  V4      pW�V P                  ,          ,           pTpV P	                  V4      pV P                  V4      pV P                  V4      pW�V P                  ,          ,           pV# )rT   rU   rV   rW   rX   rY   r   )Úinput_layernormÚ	self_attnrP   Úresidual_multiplierÚpost_attention_layernormÚmlprQ   )
r   rT   rU   rV   rW   rX   rY   r   ÚresidualÚ_s
   &&&&&&&,  r    ÚforwardÚHyperCLOVAXDecoderLayer.forward}   sÛ   € ð !ˆØ×,Ñ,¨]Ó;ˆàŸ>š>ñ 
Ø'ð
à)ð
ð &ð
ð ,ð	
ð
  ð
ð !4ð
ð ñ
Ñˆð Ÿ™¨Ó6ˆØ °4×3KÑ3KÕ#KÕKˆð !ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØŸ™¨Ó6ˆØ °4×3KÑ3KÕ#KÕKˆØÐr"   )rP   rQ   )NNNFN)	r1   r2   r3   r4   rL   rg   r8   r9   r:   r;   s   @@r    rE   rE   r   s   ù‡ € ÷
ó 
÷÷ ò r"   rE   c                   ó   € ] tR t^ŸtRtR# )ÚHyperCLOVAXPreTrainedModelr   Nr>   r   r"   r    rj   rj   Ÿ   ó   † ãr"   rj   c                   ó   € ] tR t^¤tRtR# )ÚHyperCLOVAXModelr   Nr>   r   r"   r    rm   rm   ¤   rk   r"   rm   c                   óJ   a € ] tR t^©t o ]]RV 3R lR ll4       4       tRtV tR# )ÚHyperCLOVAXForCausalLMNc                óP  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[S[ P                  ,          R
S[S[,          RS[	/
# )r(   Ú	input_idsNrU   rV   rW   Úinputs_embedsÚlabelsrX   Úlogits_to_keepr   rZ   )
r[   r]   r\   r   ÚFloatTensorr,   r*   r   r   r   )r-   r.   s   "€r    r/   Ú#HyperCLOVAXForCausalLM.__annotate__­   s»   ø€ ÷ 6
ñ 6
á×#Ñ# dÕ*ð6
ñ Ÿ™ tÕ+ð6
ñ ×&Ñ&¨Õ-ð	6
ñ
  �ð6
ñ ×(Ñ(¨4Õ/ð6
ñ × Ñ  4Õ'ð6
ñ ˜$•;ð6
ñ ™eŸl™lÕ*ð6
ñ Ñ+Õ,ð6
ñ 
 ñ6
r"   c	                óÌ  € V P                   ! RRVRVRVRVRVRV/V	B p
V
P                  p\        V\        4      '       d   \	        V) R4      MTpV P                  VRVR3,          4      V P                  P                  ,          pRpVe-   V P                  ! RR	VR
VRV P                  P                  /V	B p\        VVV
P                  V
P                  V
P                  R7      # )aÆ  
Example:

```python
>>> from transformers import AutoTokenizer, HyperCLOVAXForCausalLM

>>> model = HyperCLOVAXForCausalLM.from_pretrained("naver-hyperclovax/HyperCLOVAX-SEED-Think-14B")
>>> tokenizer = AutoTokenizer.from_pretrained("naver-hyperclovax/HyperCLOVAX-SEED-Think-14B")

>>> prompt = "Hey, are you conscious? Can you talk to me?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me? Are you okay?" The man was confused and answered, "Yes." Then the woman asked.
```rq   rU   rV   rW   rr   rX   N:NNNÚlogitsrs   Ú
vocab_size)Úlossrx   rW   rT   Ú
attentionsr   )ÚmodelÚlast_hidden_stateÚ
isinstancer*   ÚsliceÚlm_headrG   Úlogits_scalingÚloss_functionry   r   rW   rT   r{   )r   rq   rU   rV   rW   rr   rs   rX   rt   r   ÚoutputsrT   Úslice_indicesrx   rz   s   &&&&&&&&&,     r    rg   ÚHyperCLOVAXForCausalLM.forward«   s  € ð> —*’*ñ 
Øð
à)ð
ð &ð
ð ,ð	
ð
 (ð
ð  ð
ð ñ
ˆð  ×1Ñ1ˆÜ8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆà—‘˜m¨A¨}¸aÐ,?Õ@ÓAÀDÇKÁK×D^ÑD^Õ^ˆàˆØÒØ×%Ò%Ñp¨VÐp¸FÐpÈtÏ{É{×OeÑOeÐpÐioÑpˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r"   r   )NNNNNNNé    )	r1   r2   r3   r4   r   r   rg   r8   r9   )r.   s   @r    ro   ro   ©   s$   ø‡ € àØ÷6
ñ 6
ó ó ö6
r"   ro   )r   rj   rm   ro   )#r5   r[   Útorch.nnrN   Úhuggingface_hub.dataclassesr   Úcache_utilsr   Úmodeling_outputsr   Úprocessing_utilsr   Úutilsr   r   r   Úgranite.configuration_graniter	   Úgranite.modeling_graniter
   r   r   r   r   r   r   r   r=   rA   rC   rE   rj   rm   ro   Ú__all__r   r"   r    Ú<module>r�      sî   ðñ ,ã Ý Ý .å  Ý 6Ý &ß IÑ IÝ 9÷÷ ñ ñ ÐIÔJØô=˜ó =ó ó Kð=ô@	˜ô 	ô	Ð!7ô 	ô	Ð+ô 	ô*Ð1ô *ðZ ô	Ð!7ó 	ó ð	ð ô	�|ó 	ó ð	ð ô9
Ð/ó 9
ó ð9
òx‚r"   