+
    œ°³j]  ã                   óÎ  € ^ RI Ht ^ RIt^ RIHt ^RIHt ^RIHtHt ^RI	H
t
 ^RIHt ^RIHtHt ^R	IHtHtHtHt ^R
IHtHt ^RIHtHt ^RIHtHt ^RIHt ^RI H!t!H"t"H#t# ^RI$H%t% ^RI&H't'H(t( ^RI)H*t* ^RI+H,t, ]! R4       ! R R]PZ                  4      4       t. ! R R]PZ                  4      t/R t0]! R4      R1R l4       t1R R lt2R2R R llt3 ! R R ]PZ                  4      t4 ! R! R"]PZ                  4      t5 ! R# R$]4      t6]" ! R% R&]4      4       t7]" ! R' R(]74      4       t8]" ! R) R*]7]
4      4       t9 ! R+ R,]]74      t: ! R- R.]]74      t; ! R/ R0]]74      t<. R3Ot=R# )4é    )ÚCallableN)Únn)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_kernel_forward_from_hub)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚGenericForQuestionAnsweringÚ GenericForSequenceClassificationÚGenericForTokenClassificationÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputs©ÚExaone4ConfigÚRMSNormc                   óZ   a a€ ] tR t^1t oRV3R lV 3R llltV3R lR ltR tRtVtV ;t	# )ÚExaone4RMSNormc                ó$   <€ V ^8„  d   QhRS[ RR/# )é   ÚepsÚreturnN)Úfloat)ÚformatÚ__classdict__s   "€Úm/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/exaone4/modeling_exaone4.pyÚ__annotate__ÚExaone4RMSNorm.__annotate__3   s   ø€ ÷ $ñ $©ð $¸$ñ $ó    c                óŽ   <€ \         SV `  4        \        P                  ! \        P
                  ! V4      4      V n        W n        R# )z-
Exaone4RMSNorm is equivalent to T5LayerNorm
N)ÚsuperÚ__init__r   Ú	ParameterÚtorchÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizer%   Ú	__class__s   &&&€r*   r0   ÚExaone4RMSNorm.__init__3   s/   ø€ ô 	‰ÑÔÜ—l’l¤5§:¢:¨kÓ#:Ó;ˆŒØ #Ör-   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r$   Úhidden_statesr&   )r2   ÚTensor)r(   r)   s   "€r*   r+   r,   ;   s#   ø€ ÷ ;ñ ;¡U§\¡\ð ;±e·l±lñ ;r-   c                ó:  € VP                   pVP                  \        P                  4      pVP	                  ^4      P                  RRR7      pV\        P                  ! W0P                  ,           4      ,          pV P                  VP                  V4      ,          # )r$   T)Úkeepdiméÿÿÿÿ)	ÚdtypeÚtor2   Úfloat32ÚpowÚmeanÚrsqrtr5   r4   )r6   r;   Úinput_dtypeÚvariances   &&  r*   ÚforwardÚExaone4RMSNorm.forward;   sw   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯ª°H×?TÑ?TÕ4TÓ(UÕUˆØ�{‰{˜]×-Ñ-¨kÓ:Õ:Ð:r-   c                ó^   € \        V P                  P                  4       R V P                   2# )z, eps=)Útupler4   Úshaper5   )r6   s   &r*   Ú
extra_reprÚExaone4RMSNorm.extra_reprB   s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr-   )r5   r4   )g�íµ ÷Æ°>)
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r0   rH   rM   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r8   r)   s   @@r*   r"   r"   1   s%   ù‡ € ÷$õ $÷;ð ;÷Jò Jr-   r"   c                   óÎ   a a€ ] tR t^Ft o]! RRR7      R
V3R lV 3R lll4       t]]! RRR7      R
V3R lR ll4       4       t]P                  ! 4       ]
R 4       4       tR	tVtV ;t# )ÚExaone4RotaryEmbeddingÚdevicez5.18)Úversionc                ó    <€ V ^8„  d   QhRS[ /# ©r$   Úconfigr   )r(   r)   s   "€r*   r+   Ú#Exaone4RotaryEmbedding.__annotate__H   s   ø€ ÷ Oñ O™}ñ Or-   c                óä  <€ \         SV `  4        VP                  V n        VP                  V n        Wn        V P
                  P                  R ,          V n        V P                  pV P                  R8w  d   \        V P                  ,          pV! V P
                  V4      w  q@n
        \        P                  ! VRR7      V n        \        P                  ! VP                  4       RR7      V n        R# )Ú	rope_typeÚdefaultF)Ú
persistentN)r/   r0   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr]   Úrope_parametersr`   Úcompute_default_rope_parametersr   Úattention_scalingr   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)r6   r]   rY   Úrope_init_fnrj   r8   s   &&&  €r*   r0   ÚExaone4RotaryEmbedding.__init__G   s¬   ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àŒàŸ™×4Ñ4°[ÕAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÔ&Ü.¨t¯~©~Õ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÔ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÔ!NˆÖr-   c                óP   <€ V ^8„  d   QhRS[ RS[S[P                  S[3,          /# )r$   r]   r&   )r   rK   r2   r<   r'   )r(   r)   s   "€r*   r+   r^   Z   s.   ø€ ÷ 5ñ 5±ð 5ÑY^Ñ_d×_kÑ_kÑmrÐ_rÕYsñ 5r-   c           	     ó0  € V P                   R,          p\        V RR4      ;'       g    V P                  V P                  ,          pRpRV\        P
                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚhead_dimNg      ð?©r@   )rf   Úgetattrr7   Únum_attention_headsr2   Úaranger'   rA   )r]   rY   ÚkwargsÚbaseÚdimÚattention_factorrj   s   &&,    r*   rg   Ú6Exaone4RotaryEmbedding.compute_default_rope_parametersX   s~   € ð ×%Ñ% lÕ3ˆÜ�f˜j¨$Ó/×cÐc°6×3EÑ3EÈ×IcÑIcÕ3cˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r-   c                ó”  € V P                   R,          P                  4       P                  VP                  ^ ,          R^4      P	                  VP
                  4      pVR	,          P                  4       p\        VP
                  P                  \        4      '       d2   VP
                  P                  R8w  d   VP
                  P                  MRp\        VRR7      ;_uu_ 4        VP                  4       VP                  4       ,          P                  ^^4      p\        P                  ! Wf3RR7      pVP                  4       V P                  ,          pVP                  4       V P                  ,          p	R R R 4       XP	                  VP                   R7      X	P	                  VP                   R7      3#   + '       g   i     LH; i)
NÚmpsÚcpuF)Údevice_typeÚenabled©ry   rs   )NºNNNNr?   )r‚   Nr‚   )rj   r'   ÚexpandrL   rA   rY   Ú
isinstanceÚtypeÚstrr   Ú	transposer2   ÚcatÚcosrh   Úsinr@   )
r6   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedr   ÚfreqsÚembr‰   rŠ   s
   &&&       r*   rH   ÚExaone4RotaryEmbedding.forwardl   sE  € ð !ŸM™M¨-Õ8×>Ñ>Ó@×GÑGÈ×HZÑHZÐ[\ÕH]Ð_aÐcdÓe×hÑhÐij×iqÑiqÓrÐØ ,¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆÜ¨¸U×CÖCØ&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÕN×YÑYÐZ[Ð]^Ó_ˆEÜ—)’)˜U˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆC÷	 Dð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷ D×Cús   Ã&BF7Æ7G	)rh   r]   rj   rd   rl   re   r`   ©N)rO   rP   rQ   rR   r   r0   Ústaticmethodrg   r2   Úno_gradr   rH   rS   rT   rU   rV   s   @@r*   rX   rX   F   su   ù‡ € Ù�X vÔ.÷Oô Oó /ðOð  Ù�X vÔ.÷5ñ 5ó /ó ð5ð$ ‡]‚]ƒ_Øñ<ó ó ÷<ð <r-   rX   c                óÌ   € V RRV P                   R,          ^,          13,          pV RV P                   R,          ^,          R13,          p\        P                  ! V) V3RR7      # )z*Rotates half the hidden dims of the input..Nr�   r?   )rL   r2   rˆ   )r‹   Úx1Úx2s   &  r*   Úrotate_halfr˜   |   sZ   € à	
ˆ3Ð"�!—'‘'˜"•+ Õ"Ð"Ð"Õ	#€BØ	
ˆ3�—‘˜•˜qÕ Ñ"Ð"Õ	#€BÜ�9Š9�r�c˜2�Y BÔ'Ð'r-   Úrotary_pos_embc                óÈ   € VP                  V4      pVP                  V4      pW,          \        V 4      V,          ,           pW,          \        V4      V,          ,           pWV3# )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)Ú	unsqueezer˜   )ÚqÚkr‰   rŠ   Úunsqueeze_dimÚq_embedÚk_embeds   &&&&&  r*   Úapply_rotary_pos_embr¡   ƒ   sS   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�wœ; q›>¨CÕ/Õ0€GØ�wœ; q›>¨CÕ/Õ0€GØÐÐr-   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r$   r;   Ún_repr&   )r2   r<   Úint)r(   s   "r*   r+   r+   �   s.   € ÷ 	Uñ 	UœUŸ\™\ð 	U´#ð 	U¼%¿,¹,ñ 	Ur-   c                ó˜   € V P                   w  r#rEV^8X  d   V # V R,          P                  W#WV4      p V P                  W#V,          WE4      # )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
)r‚   r‚   Nr‚   r‚   )rL   rƒ   Úreshape)r;   r£   ÚbatchÚnum_key_value_headsÚslenrr   s   &&    r*   Ú	repeat_kvrª   �   sU   € ð
 2?×1DÑ1DÑ.€E Ø�„zØÐØ!Ð"2Õ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÕ(CÀTÓTÐTr-   c                ó   € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R\        R	\
        \        ,          /# )
r$   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNÚscalingÚdropoutrw   )r   ÚModuler2   r<   r'   r   r   )r(   s   "r*   r+   r+   ©   sw   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ð%ô ð%ô Ô'Õ(ñ%r-   c                 ó"  € \        W P                  4      p\        W0P                  4      p	\        P                  ! WP	                  ^^4      4      V,          p
Ve	   W¤,           p
\
        P                  P                  V
R\        P                  R7      P                  VP                  4      p
\
        P                  P                  W¦V P                  R7      p
\        P                  ! W©4      pVP	                  ^^4      P                  4       pWº3# )r$   )ry   r@   )ÚpÚtrainingr?   )rª   Únum_key_value_groupsr2   Úmatmulr‡   r   Ú
functionalÚsoftmaxrB   rA   r@   r²   r¶   Ú
contiguous)r¬   r­   r®   r¯   r°   r±   r²   rw   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs   &&&&&&&,    r*   Úeager_attention_forwardrÀ   ©   sÒ   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÕL€LØÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r-   c                   óT   a a€ ] tR t^Ât oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚExaone4Attentionc                ó&   <€ V ^8„  d   QhRS[ RS[/# ©r$   r]   Ú	layer_idx©r   r¤   )r(   r)   s   "€r*   r+   ÚExaone4Attention.__annotate__Ã   s   ø€ ÷ Mñ M™}ð M¹ñ Mr-   c                óö  <€ \         SV `  4        Wn        W n        VP                  V n        VP
                  V n        VP                  V n        \        VR VP                  VP                  ,          4      V n        VP                  VP
                  ,          V n	        VP                  V n
        RV n        V P                  R,          V n        VP                  V n        VP                  V n        \        VR4      '       d   VP                   V,          MRpVR8H  V n        \$        P&                  ! V P                  V P                  V P                  ,          RR7      V n        \$        P&                  ! V P                  V P
                  V P                  ,          RR7      V n        \$        P&                  ! V P                  V P
                  V P                  ,          RR7      V n        \$        P&                  ! V P                  V P                  ,          V P                  RR7      V n        \1        V P                  VP2                  R7      V n        \1        V P                  VP2                  R7      V n        R# )	rr   TÚlayer_typesNÚsliding_attentionF©Úbias©r%   g      à¿)r/   r0   r]   rÅ   ru   r¨   r7   rt   rr   r·   Úattention_dropoutÚ	is_causalr±   Úsliding_windowÚsliding_window_patternÚhasattrrÉ   Ú
is_slidingr   ÚLinearÚq_projÚk_projÚv_projÚo_projr"   Úrms_norm_epsÚq_normÚk_norm)r6   r]   rÅ   Ú
layer_typer8   s   &&& €r*   r0   ÚExaone4Attention.__init__Ã   sÁ  ø€ Ü‰ÑÔØŒØ"ŒØ#)×#=Ñ#=ˆÔ Ø#)×#=Ñ#=ˆÔ Ø!×-Ñ-ˆÔÜ ¨
°F×4FÑ4FÈ&×JdÑJdÕ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Õ$\ˆÔ!Ø!'×!9Ñ!9ˆÔØˆŒØ—}‘} dÕ*ˆŒØ$×3Ñ3ˆÔØ&,×&CÑ&CˆÔ#Ü6=¸fÀm×6TÒ6T�V×'Ñ'¨	Ö2ÐZ^ˆ
Ø$Ð(;Ñ;ˆŒä—i’i × 0Ñ 0°$×2JÑ2JÈTÏ]É]Õ2ZÐafÔgˆŒÜ—i’i × 0Ñ 0°$×2JÑ2JÈTÏ]É]Õ2ZÐafÔgˆŒÜ—i’i × 0Ñ 0°$×2JÑ2JÈTÏ]É]Õ2ZÐafÔgˆŒÜ—i’i × 8Ñ 8¸4¿=¹=Õ HÈ$×JZÑJZÐafÔgˆŒä$ T§]¡]¸×8KÑ8KÔLˆŒÜ$ T§]¡]¸×8KÑ8KÔLˆŽr-   c                óR  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  S[ P                  R,          S[S[ P                  ,          R,          3,          /# )r$   r;   Úposition_embeddingsr°   NÚpast_key_valuesrw   r&   )r2   r<   rK   r   r   r   )r(   r)   s   "€r*   r+   rÇ   Ü   s—   ø€ ÷ -)ñ -)á—|‘|ð-)ñ #¡5§<¡<±·±Ð#=Õ>ð-)ñ Ÿ™ tÕ+ð	-)ñ
  �ð-)ñ Ñ+Õ,ð-)ñ 
‰u�|‰|™UŸ\™\¨DÕ0±%¹¿¹Õ2EÈÕ2LÐLÕ	Mñ-)r-   c           	     óÔ  € VP                   R R p. VORNV P                  N5pV P                  V4      P                  V4      P	                  ^^4      pV P                  V4      P                  V4      P	                  ^^4      p	V P                  V4      P                  V4      P	                  ^^4      p
V P                  V4      pV P                  V	4      p	Vw  r¼V P                  e   V P                  '       d   \        W‰W¼4      w  r‰Ve   VP                  WšV P                  4      w  rš\        P                  ! V P                   P"                  \$        4      pT! V VV	V
V3RV P&                  '       g   RMV P(                  RV P*                  RV P                  '       d   V P                  MR /VB w  rïVP,                  ! . VORN5!  P/                  4       pV P1                  V4      pWï3# )Nr²   ç        r±   rÐ   r?   )rL   rr   rÕ   Úviewr‡   rÖ   r×   rÚ   rÛ   rÐ   rÓ   r¡   ÚupdaterÅ   r   Úget_interfacer]   Ú_attn_implementationrÀ   r¶   rÎ   r±   r¦   r»   rØ   )r6   r;   rß   r°   rà   rw   Úinput_shapeÚhidden_shapeÚquery_statesr¼   r½   r‰   rŠ   Úattention_interfacer¿   r¾   s   &&&&&,          r*   rH   ÚExaone4Attention.forwardÜ   sÓ  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆð —{‘{ <Ó0ˆØ—[‘[ Ó,ˆ
à&‰ˆà×ÑÒ&¨$¯/¯/¨/Ü';¸LÐVYÓ'_Ñ$ˆLàÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ
%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð
%
ð —L‘Lð
%
ð 37·/·/°/˜4×.Ò.Àtð
%
ð ñ
%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—k‘k +Ó.ˆØÐ(Ð(r-   )rÎ   r]   rr   r7   rÏ   rÓ   rÛ   rÖ   rÅ   ru   r·   r¨   rØ   rÚ   rÕ   r±   rÐ   rÑ   r×   )NN©	rO   rP   rQ   rR   r0   rH   rS   rT   rU   rV   s   @@r*   rÂ   rÂ   Â   s    ù‡ € ÷Mó M÷2-)÷ -)ò -)r-   rÂ   c                   ó8   a a€ ] tR tRt oV 3R ltR tRtVtV ;t# )Ú
Exaone4MLPi  c                óØ  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        \
        P                  ! V P                  V P                  R R7      V n        \
        P                  ! V P                  V P                  R R7      V n        \
        P                  ! V P                  V P                  R R7      V n	        \        VP                  ,          V n        R# ©FrË   N)r/   r0   r]   r7   Úintermediate_sizer   rÔ   Ú	gate_projÚup_projÚ	down_projr   Ú
hidden_actÚact_fn©r6   r]   r8   s   &&€r*   r0   ÚExaone4MLP.__init__  s¤   ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔÜŸš 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆŒÜ˜V×.Ñ.Õ/ˆŽr-   c                ó�   € V P                  V P                  V P                  V4      4      V P                  V4      ,          4      pV# r’   )rô   rö   rò   ró   )r6   r‹   rô   s   && r*   rH   ÚExaone4MLP.forward  s6   € Ø—N‘N 4§;¡;¨t¯~©~¸aÓ/@Ó#AÀDÇLÁLÐQRÃOÕ#SÓTˆ	ØÐr-   )rö   r]   rô   rò   r7   rñ   ró   rì   rV   s   @@r*   rî   rî     s   ù‡ € õ0÷ò r-   rî   c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚExaone4DecoderLayeri  c                ó&   <€ V ^8„  d   QhRS[ RS[/# rÄ   rÆ   )r(   r)   s   "€r*   r+   Ú Exaone4DecoderLayer.__annotate__  s   ø€ ÷ fñ f™}ð f¹ñ fr-   c                ó   <€ \         SV `  4        VP                  V n        \        WR 7      V n        \        V4      V n        \        VP                  VP                  R7      V n	        \        VP                  VP                  R7      V n
        R# ))r]   rÅ   rÍ   N)r/   r0   r7   rÂ   Ú	self_attnrî   Úmlpr"   rÙ   Úpost_attention_layernormÚpost_feedforward_layernorm©r6   r]   rÅ   r8   s   &&&€r*   r0   ÚExaone4DecoderLayer.__init__  sk   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔÜ)°ÔMˆŒä˜fÓ%ˆŒÜ(6°v×7IÑ7IÈv×ObÑObÔ(cˆÔ%Ü*8¸×9KÑ9KÐQW×QdÑQdÔ*eˆÖ'r-   c                ó.  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[ P                  S[ P                  3,          R,          RS[S[,          R	S[ P                  /# )
r$   r;   r°   NrŒ   rà   Ú	use_cacherß   rw   r&   )r2   r<   Ú
LongTensorr   ÚboolrK   r   r   )r(   r)   s   "€r*   r+   rþ   &  sœ   ø€ ÷ ñ á—|‘|ðñ Ÿ™ tÕ+ðñ ×&Ñ&¨Õ-ð	ñ
  �ðñ ˜$•;ðñ #¡5§<¡<±·±Ð#=Õ>ÀÕEðñ Ñ+Õ,ðñ 
�‰ñr-   c                óÔ   € TpV P                   ! RR VRVRVRVRVRV/VB w  rV P                  V4      pW�,           pTpV P                  V4      pV P                  V4      pW�,           pV# )r;   r°   rŒ   rà   r  rß   © )r   r  r  r  )
r6   r;   r°   rŒ   rà   r  rß   rw   ÚresidualÚ_s
   &&&&&&&,  r*   rH   ÚExaone4DecoderLayer.forward&  s©   € ð !ˆØŸ>š>ñ 
Ø'ð
à)ð
ð &ð
ð ,ð	
ð
  ð
ð !4ð
ð ñ
Ñˆð ×5Ñ5°mÓDˆØ Õ0ˆð !ˆØŸ™ Ó/ˆØ×7Ñ7¸ÓFˆØ Õ0ˆØÐr-   )r7   r  r  r  r   )NNNFNrì   rV   s   @@r*   rü   rü     s    ù‡ € ÷fó f÷÷ ò r-   rü   c                   ód   a € ] tR tRt o RtRtR.tR.tRtRt	Rt
RtRtR]R]/t]tV 3R ltR	tV tR
# )ÚExaone4PreTrainedModeliE  ÚmodelTrü   rà   r;   Ú
attentionsc                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r\   r   )r(   r)   s   "€r*   r+   Ú#Exaone4PreTrainedModel.__annotate__E  s   ø‡ ‚ áÑò r-   r  N)rO   rP   rQ   rR   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendrü   rÂ   Ú_can_record_outputsr   Úconfig_classÚ__annotate_func__rS   rT   )r)   s   @r*   r  r  E  sd   ø‡ € ð  ÐØ&*Ð#Ø.Ð/ÐØ#4Ð"5ÐØÐØ€NØÐà!ÐØ"&ÐàÐ,ØÐ&ðÐð !€L÷# ƒ r-   r  c                   óh   a a€ ] tR tRt oV3R lV 3R llt]]RV3R lR ll4       4       tRtVt	V ;t
# )ÚExaone4ModeliY  c                ó    <€ V ^8„  d   QhRS[ /# r\   r   )r(   r)   s   "€r*   r+   ÚExaone4Model.__annotate__[  s   ø€ ÷ ñ ™}ñ r-   c           	     ó  <€ \         SV `  V4       VP                  V n        VP                  V n        \
        P                  ! VP                  VP                  V P                  4      V n        \
        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n        \        VP                  VP                  R 7      V n        \#        VR7      V n        RV n        V P)                  4        R# u upi )rÍ   ©r]   FN)r/   r0   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingr7   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersrü   Úlayersr"   rÙ   ÚnormrX   Ú
rotary_embÚgradient_checkpointingÚ	post_initr  s   && €r*   r0   ÚExaone4Model.__init__[  sÊ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸLšL¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m’mÜEJÈ6×KcÑKcÔEdÓeÑEd¸	Ô  Ö3ÑEdÑeó
ˆŒô # 6×#5Ñ#5¸6×;NÑ;NÔOˆŒ	Ü0¸Ô?ˆŒØ&+ˆÔ#ð 	�‰Öùò fs   ÂDc                ó  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[S[,          R	S[S[	,          /# )
r$   Ú	input_idsNr°   rŒ   rà   Úinputs_embedsr  rw   r&   )
r2   r  r<   r   ÚFloatTensorr	  r   r   rK   r   )r(   r)   s   "€r*   r+   r$  m  s™   ø€ ÷ =
ñ =
á×#Ñ# dÕ*ð=
ñ Ÿ™ tÕ+ð=
ñ ×&Ñ&¨Õ-ð	=
ñ
  �ð=
ñ ×(Ñ(¨4Õ/ð=
ñ ˜$•;ð=
ñ Ñ+Õ,ð=
ñ 
Ñ(Õ	(ñ=
r-   c                óf  € VR J VR J,          '       d   \        R4      hVf   V P                  V4      pV'       d   Vf   \        V P                  R7      pVfb   Ve   VP	                  4       M^ p\
        P                  ! VP                  ^,          VP                  R7      V,           pVP                  ^ 4      p\        T;p	\        4      '       gM   RV P                  RVRVRVRV/p
R	\        R/ V
B /p	R
V P                  P                  9   d   \        R/ V
B V	R
&   TpV P                  W³4      p\!        V P"                  4       F;  w  rÞV P                  P                  V,          pV! V3RWŸ,          RVRVRVRV/VB pK=  	  V P%                  V4      p\'        TV'       d   VR7      # R R7      # )Nz:You must specify exactly one of input_ids or inputs_embedsr&  )rY   r]   r7  r°   rà   rŒ   Úfull_attentionrÊ   r  rß   )Úlast_hidden_staterà   r  )Ú
ValueErrorr+  r   r]   Úget_seq_lengthr2   rv   rL   rY   r›   r„   Údictr
   rÉ   r   r1  Ú	enumerater/  r0  r   )r6   r6  r°   rŒ   rà   r7  r  rw   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsr;   rß   ÚiÚdecoder_layerrÜ   s   &&&&&&&,        r*   rH   ÚExaone4Model.forwardk  sÛ  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ Ø ×-Ñ-¨iÓ8ˆMç˜Ò0Ü*°$·+±+Ô>ˆOàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLô °Ð?Ð-Ä×FÒFð ˜$Ÿ+™+Ø Ø  .Ø! ?Ø ðˆKð !Ô"4Ñ"C°{Ñ"Cð#Ðð # d§k¡k×&=Ñ&=Ô=Ü;\Ñ;kÐ_jÑ;kÐ#Ð$7Ñ8à%ˆØ"Ÿo™o¨mÓJÐä )¨$¯+©+Ö 6ÑˆAØŸ™×0Ñ0°Õ3ˆJÙ)Øñà2Õ>ðð *ðð !0ð	ð
 $ðð %8ðð ñŠMñ !7ð Ÿ	™	 -Ó0ˆä&Ø+ß/8˜Oô
ð 	
à>Bô
ð 	
r-   )r+  r2  r/  r0  r(  r1  r)  )NNNNNN)rO   rP   rQ   rR   r0   r   r   rH   rS   rT   rU   rV   s   @@r*   r"  r"  Y  s5   ù‡ € ÷ó ð   Ø÷=
ñ =
ó ó  ÷=
ð =
r-   r"  c                   ó„   a a€ ] tR tRt oRR/tRR/tRR.R.3/tRR/tV 3R	 lt]	]
RV3R
 lR ll4       4       tRtVtV ;t# )ÚExaone4ForCausalLMi­  zlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputr;   ÚlogitsÚkeep_full_weightc                óì   <€ \         SV `  V4       \        V4      V n        VP                  V n        \
        P                  ! VP                  VP                  R R7      V n        V P                  4        R# rð   )
r/   r0   r"  r  r)  r   rÔ   r7   rH  r3  r÷   s   &&€r*   r0   ÚExaone4ForCausalLM.__init__´  sU   ø€ Ü‰Ñ˜Ô Ü! &Ó)ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒð 	�‰Ör-   c                óP  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[S[ P                  ,          R
S[S[,          RS[	/
# )r$   r6  Nr°   rŒ   rà   r7  Úlabelsr  Úlogits_to_keeprw   r&   )
r2   r  r<   r   r8  r	  r¤   r   r   r   )r(   r)   s   "€r*   r+   ÚExaone4ForCausalLM.__annotate__¿  sÇ   ø€ ÷ D
ñ D
á×#Ñ# dÕ*ðD
ñ Ÿ™ tÕ+ðD
ñ ×&Ñ&¨Õ-ð	D
ñ
  �ðD
ñ ×(Ñ(¨4Õ/ðD
ñ × Ñ  4Õ'ðD
ñ ˜$•;ðD
ñ ™eŸl™lÕ*ðD
ñ Ñ+Õ,ðD
ñ 
 ñD
r-   c	                ó–  € V P                   ! RRVRVRVRVRVRV/V	B p
V
P                  p\        V\        4      '       d   \	        V) R4      MTpV P                  VRVR3,          4      pRpVe-   V P                  ! RR	VR
VRV P                  P                  /V	B p\        VVV
P                  V
P                  V
P                  R7      # )u±  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

Example:

```python
>>> from transformers import AutoModelForCausalLM, AutoTokenizer
>>> model = AutoModelForCausalLM.from_pretrained("LGAI-EXAONE/EXAONE-4.0-32B")
>>> tokenizer = AutoTokenizer.from_pretrained("LGAI-EXAONE/EXAONE-4.0-32B")

>>> prompt = "Explain how wonderful you are"
>>> messages = [
    {"role": "system", "content": "You are a helpful assistant."},
    {"role": "user", "content": prompt}
]
>>> input_ids = tokenizer.apply_chat_template(
    messages,
    tokenize=True,
    add_generation_prompt=True,
    return_tensors="pt",
    enable_thinking=False,
)

>>> output = model.generate(input_ids, max_new_tokens=128)
>>> tokenizer.decode(output[0], skip_special_tokens=False)
"[|system|]\nYou are a helpful assistant.[|endofturn|]\n[|user|]\nExplain how wonderful you are[|endofturn|]\n[|assistant|]\n<think>\n\n</think>\n\nOh, thank you for such a kind and lovely question! ðŸ˜Š  \n\nIâ€™m *so* wonderful because Iâ€™m here to make your life easier, brighter, and more fun! Whether you need help with:  \n\nâœ¨ **Learning** â€“ I can explain anything, from quantum physics to baking the perfect cake!  \nðŸ’¡ **Creativity** â€“ Need a poem, story, or a wild idea? Iâ€™ve got you covered!  \nðŸ¤– **Problem-solving** â€“ Stuck on a math problem or a tricky decision? Iâ€™ll help you figure it out"
```
r6  r°   rŒ   rà   r7  r  Nr‚   rJ  rO  r)  )ÚlossrJ  rà   r;   r  r  )r  r;  r„   r¤   ÚslicerH  Úloss_functionr]   r)  r   rà   r;   r  )r6   r6  r°   rŒ   rà   r7  rO  r  rP  rw   Úoutputsr;   Úslice_indicesrJ  rS  s   &&&&&&&&&,     r*   rH   ÚExaone4ForCausalLM.forward½  s   € ðZ ,0¯:ª:ñ ,
Øð,
à)ð,
ð &ð,
ð ,ð	,
ð
 (ð,
ð  ð,
ð ñ,
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒØ×%Ò%Ñp¨VÐp¸FÐpÈtÏ{É{×OeÑOeÐpÐioÑpˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r-   )rH  r  r)  )NNNNNNNr   )rO   rP   rQ   rR   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr0   r   r   rH   rS   rT   rU   rV   s   @@r*   rG  rG  ­  sj   ù‡ € à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÐ/Ð0€Jõð Ø÷D
ñ D
ó ó ÷D
ð D
r-   rG  c                   ó   € ] tR tRtRtR# )Ú Exaone4ForSequenceClassificationi  r  N©rO   rP   rQ   rR   rS   r  r-   r*   r^  r^    ó   † Ûr-   r^  c                   ó   € ] tR tRtRtR# )ÚExaone4ForTokenClassificationi
  r  Nr_  r  r-   r*   rb  rb  
  r`  r-   rb  c                   ó   € ] tR tRtRtRtR# )ÚExaone4ForQuestionAnsweringi  Útransformerr  N)rO   rP   rQ   rR   r  rS   r  r-   r*   rd  rd    s   † Ø%Ôr-   rd  )é   )râ   )r  r"  rG  r^  rb  rd  )>Úcollections.abcr   r2   r   Úactivationsr   Úcache_utilsr   r   Ú
generationr   Úintegrationsr	   Úmasking_utilsr
   r   Úmodeling_layersr   r   r   r   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   Úconfiguration_exaone4r   r³   r"   rX   r˜   r¡   rª   rÀ   rÂ   rî   rü   r  r"  rG  r^  rb  rd  Ú__all__r  r-   r*   Ú<module>rx     sq  ðõ, %ã Ý å !ß .Ý )Ý 7ß R÷ó ÷ Pß Kß FÝ &ß IÑ IÝ 0ß GÝ 5Ý 0ñ ˜YÓ'ôJ�R—Y‘Yó Jó (ðJô(3<˜RŸY™Yô 3<òl(ñ Ð-Ó.óó /ðõ2	U÷%ô2G)�r—y‘yô G)ôT�—‘ô ô &Ð4ô &ðR ô!˜_ó !ó ð!ð& ôP
Ð)ó P
ó ðP
ðf ôU
Ð/°ó U
ó ðU
ôp	Ð'GÐI_ô 	ô	Ð$AÐCYô 	ô&Ð"=Ð?Uô &ò‚r-   