+
    œ°³j³\  ã                   ó¼  € ^ RI Ht ^ RIt^ RIHt ^RIHt ^RIH	t	 ^RI
HtHt ^RIHt ^RIHtHt ^RIHtHt ^R	IHtHt ^R
IHtHt ^RIHtHt ^RIHtH t  ^RI!H"t" ^RI#H$t$H%t% ^RI&H't' ^RI(H)t)H*t*H+t+ ^RI,H-t- ^RI.H/t/ ]! R4       ! R R]P`                  4      4       t1R R lt2R-R R llt3]! R4      R.R l4       t4R t5]! ]44       ! R R]P`                  4      4       t6 ! R R ]P`                  4      t7 ! R! R"]4      t8 ! R# R$]P`                  4      t9]$ ! R% R&] 4      4       t:]$ ! R' R(]:4      4       t;]$ ! R) R*]:]4      4       t< ! R+ R,]]:4      t=. R/Ot>R# )0é    )ÚCallableN)Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_kernel_forward_from_hubÚuse_kernelized_func)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)Ú GenericForSequenceClassificationÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)Úauto_docstringÚcan_return_tuple)Údeprecate_kwarg)ÚTransformersKwargsÚmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputs©ÚOlmo3ConfigÚRMSNormc                   óZ   a a€ ] tR t^,t oRV3R lV 3R llltV3R lR ltR tRtVtV ;t	# )ÚOlmo3RMSNormc                ó$   <€ V ^8„  d   QhRS[ RR/# )é   ÚepsÚreturnN)Úfloat)ÚformatÚ__classdict__s   "€Úi/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/olmo3/modeling_olmo3.pyÚ__annotate__ÚOlmo3RMSNorm.__annotate__.   s   ø€ ÷ $ñ $©ð $¸$ñ $ó    c                óŽ   <€ \         SV `  4        \        P                  ! \        P
                  ! V4      4      V n        W n        R# )z+
Olmo3RMSNorm is equivalent to T5LayerNorm
N)ÚsuperÚ__init__ÚnnÚ	ParameterÚtorchÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizer$   Ú	__class__s   &&&€r)   r/   ÚOlmo3RMSNorm.__init__.   s/   ø€ ô 	‰ÑÔÜ—l’l¤5§:¢:¨kÓ#:Ó;ˆŒØ #Ör,   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r#   r%   )r2   ÚTensor)r'   r(   s   "€r)   r*   r+   6   s   ø€ ÷ =ñ =©¯©ñ =r,   c                ó:  € VP                   pVP                  \        P                  4      pVP	                  ^4      P                  RRR7      pV\        P                  ! W0P                  ,           4      ,          pV P                  V,          P                  V4      # )r#   T)Úkeepdiméÿÿÿÿ)	ÚdtypeÚtor2   Úfloat32ÚpowÚmeanÚrsqrtr5   r4   )r6   Úhidden_statesÚinput_dtypeÚvariances   &&  r)   ÚforwardÚOlmo3RMSNorm.forward6   sw   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯ª°H×?TÑ?TÕ4TÓ(UÕUˆØ—‘˜mÕ+×/Ñ/°Ó<Ð<r,   c                ó^   € \        V P                  P                  4       R V P                   2# )z, eps=)Útupler4   Úshaper5   )r6   s   &r)   Ú
extra_reprÚOlmo3RMSNorm.extra_repr=   s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr,   )r5   r4   )g�íµ ÷Æ°>)
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r/   rH   rM   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r8   r(   s   @@r)   r!   r!   ,   s%   ù‡ € ÷$õ $÷=ð =÷Jò Jr,   r!   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r#   rE   Ún_repr%   )r2   r;   Úint)r'   s   "r)   r*   r*   A   s.   € ÷ 	Uñ 	UœUŸ\™\ð 	U´#ð 	U¼%¿,¹,ñ 	Ur,   c                ó˜   € V P                   w  r#rEV^8X  d   V # V R,          P                  W#WV4      p V P                  W#V,          WE4      # )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
)ºNNNr[   Nr[   r[   )rL   ÚexpandÚreshape)rE   rX   ÚbatchÚnum_key_value_headsÚslenÚhead_dims   &&    r)   Ú	repeat_kvrb   A   sU   € ð
 2?×1DÑ1DÑ.€E Ø�„zØÐØ!Ð"2Õ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÕ(CÀTÓTÐTr,   c                ó   € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R\        R	\
        \        ,          /# )
r#   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNÚscalingÚdropoutÚkwargs)r0   ÚModuler2   r;   r&   r   r   )r'   s   "r)   r*   r*   M   sw   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ð%ô ð%ô Ô'Õ(ñ%r,   c                 ó"  € \        W P                  4      p\        W0P                  4      p	\        P                  ! WP	                  ^^4      4      V,          p
Ve	   W¤,           p
\
        P                  P                  V
R\        P                  R7      P                  VP                  4      p
\
        P                  P                  W¦V P                  R7      p
\        P                  ! W©4      pVP	                  ^^4      P                  4       pWº3# )r#   )Údimr?   )ÚpÚtrainingr>   )rb   Únum_key_value_groupsr2   ÚmatmulÚ	transposer0   Ú
functionalÚsoftmaxrA   r@   r?   rj   rp   Ú
contiguous)rd   re   rf   rg   rh   ri   rj   rk   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs   &&&&&&&,    r)   Úeager_attention_forwardr{   M   sÒ   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÕL€LØÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r,   Úrotary_pos_embc                ó4  € V P                   VP                   reVP                  V4      pVP                  V4      pW,          \        V 4      V,          ,           pW,          \        V4      V,          ,           pVP                  V4      VP                  V4      3# )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)r?   Ú	unsqueezeÚrotate_halfr@   )	ÚqÚkÚcosÚsinÚunsqueeze_dimÚq_typeÚk_typeÚq_embedÚk_embeds	   &&&&&    r)   Úapply_rotary_pos_embr‰   f   sv   € ð& —W‘W˜aŸg™gˆFØ
�-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�wœ; q›>¨CÕ/Õ0€GØ�wœ; q›>¨CÕ/Õ0€GØ�:‰:�fÓ˜wŸz™z¨&Ó1Ð1Ð1r,   c                óÌ   € V RRV P                   R,          ^,          13,          pV RV P                   R,          ^,          R13,          p\        P                  ! V) V3RR7      # )z*Rotates half the hidden dims of the input..N©rn   r>   )rL   r2   Úcat)ÚxÚx1Úx2s   &  r)   r   r   �   sZ   € à	
ˆ3Ð"�!—'‘'˜"•+ Õ"Ð"Ð"Õ	#€BØ	
ˆ3�—‘˜•˜qÕ Ñ"Ð"Õ	#€BÜ�9Š9�r�c˜2�Y BÔ'Ð'r,   c                   óX   a a€ ] tR t^ˆt oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )ÚOlmo3Attentionz=Multi-headed attention from 'Attention Is All You Need' paperc                ó&   <€ V ^8„  d   QhRS[ RS[/# ©r#   ÚconfigÚ	layer_idx©r   rY   )r'   r(   s   "€r)   r*   ÚOlmo3Attention.__annotate__Œ   s   ø€ ÷ lñ l™{ð l±sñ lr,   c                ó   <€ \         SV `  4        Wn        W n        \	        VR VP
                  VP                  ,          4      V n        VP                  VP                  ,          V n	        V P                  R,          V n
        VP                  V n        RV n        \        P                  ! VP
                  VP                  V P                  ,          VP                  R7      V n        \        P                  ! VP
                  VP                  V P                  ,          VP                  R7      V n        \        P                  ! VP
                  VP                  V P                  ,          VP                  R7      V n        \        P                  ! VP                  V P                  ,          VP
                  VP                  R7      V n        \)        VP                  V P                  ,          VP*                  4      V n        \)        VP                  V P                  ,          VP*                  4      V n        VP0                  V,          V n        V P2                  R8X  d   VP4                  V n        R# RV n        R# )ra   T©ÚbiasÚsliding_attentionNg      à¿)r.   r/   r”   r•   Úgetattrr7   Únum_attention_headsra   r_   rq   ri   Úattention_dropoutÚ	is_causalr0   ÚLinearÚattention_biasÚq_projÚk_projÚv_projÚo_projr!   Úrms_norm_epsÚq_normÚk_normÚlayer_typesÚattention_typeÚsliding_window©r6   r”   r•   r8   s   &&&€r)   r/   ÚOlmo3Attention.__init__Œ   sÈ  ø€ Ü‰ÑÔØŒØ"ŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÕ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Õ$\ˆÔ!Ø—}‘} dÕ*ˆŒØ!'×!9Ñ!9ˆÔØˆŒä—i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×&Ñ&¨¯©Õ6¸×8JÑ8JÐQW×QfÑQfô
ˆŒô # 6×#=Ñ#=ÀÇÁÕ#MÈv×ObÑObÓcˆŒÜ" 6×#=Ñ#=ÀÇÁÕ#MÈv×ObÑObÓcˆŒØ$×0Ñ0°Õ;ˆÔØ7;×7JÑ7JÐNaÔ7a˜f×3Ñ3ˆÖÐgkˆÖr,   c                ó  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  S[ P                  R,          3,          /# )r#   rE   Úposition_embeddingsrh   NÚpast_key_valuesrk   r%   )r2   r;   rK   r   r   r   )r'   r(   s   "€r)   r*   r—   §   s‡   ø€ ÷ +)ñ +)á—|‘|ð+)ñ #¡5§<¡<±·±Ð#=Õ>ð+)ñ Ÿ™ tÕ+ð	+)ñ
  �ð+)ñ Ñ+Õ,ð+)ñ 
‰u�|‰|™UŸ\™\¨DÕ0Ð0Õ	1ñ+)r,   c           	     óp  € VP                   R R p. VORNV P                  N5pV P                  V P                  V4      4      pV P	                  V P                  V4      4      p	V P                  V4      p
VP                  V4      P                  ^^4      pV	P                  V4      P                  ^^4      p	V
P                  V4      P                  ^^4      p
Vw  r¼\        W‰W¼4      w  r‰Ve   VP                  WšV P                  4      w  rš\        P                  ! V P                  P                  \         4      pT! V VV	V
V3RV P"                  '       g   RMV P$                  RV P&                  RV P(                  /VB w  rïVP*                  ! . VORN5!  P-                  4       pV P/                  V4      pWï3# )Nrj   ç        ri   r«   r>   )rL   ra   r§   r¢   r¨   r£   r¤   Úviewrs   r‰   Úupdater•   r   Úget_interfacer”   Ú_attn_implementationr{   rp   rž   ri   r«   r]   rv   r¥   )r6   rE   r¯   rh   r°   rk   Úinput_shapeÚhidden_shapeÚquery_statesrw   rx   r‚   rƒ   Úattention_interfacerz   ry   s   &&&&&,          r)   rH   ÚOlmo3Attention.forward§   s³  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ 4§;¡;¨}Ó#=Ó>ˆØ—[‘[ §¡¨]Ó!;Ó<ˆ
Ø—{‘{ =Ó1ˆà#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆØ—_‘_ \Ó2×<Ñ<¸QÀÓBˆ
Ø#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ
%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð
%
ð —L‘Lð
%
ð  ×.Ñ.ð
%
ð ñ
%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—k‘k +Ó.ˆØÐ(Ð(r,   )rž   rª   r”   ra   rŸ   r¨   r£   r•   rq   r¥   r§   r¢   ri   r«   r¤   ©N)
rO   rP   rQ   rR   Ú__doc__r/   rH   rS   rT   rU   rV   s   @@r)   r‘   r‘   ˆ   s#   ù‡ € áG÷ló l÷6+)÷ +)ò +)r,   r‘   c                   ó8   a a€ ] tR t^Õt oV 3R ltR tRtVtV ;t# )ÚOlmo3MLPc                óØ  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        \
        P                  ! V P                  V P                  R R7      V n        \
        P                  ! V P                  V P                  R R7      V n        \
        P                  ! V P                  V P                  R R7      V n	        \        VP                  ,          V n        R# ©Fr™   N)r.   r/   r”   r7   Úintermediate_sizer0   r    Ú	gate_projÚup_projÚ	down_projr   Ú
hidden_actÚact_fn©r6   r”   r8   s   &&€r)   r/   ÚOlmo3MLP.__init__Ö   s¤   ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔÜŸš 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆŒÜ˜V×.Ñ.Õ/ˆŽr,   c                ó�   € V P                  V P                  V P                  V4      4      V P                  V4      ,          4      pV# r¼   )rÅ   rÇ   rÃ   rÄ   )r6   r�   rÅ   s   && r)   rH   ÚOlmo3MLP.forwardà   s6   € Ø—N‘N 4§;¡;¨t¯~©~¸aÓ/@Ó#AÀDÇLÁLÐQRÃOÕ#SÓTˆ	ØÐr,   )rÇ   r”   rÅ   rÃ   r7   rÂ   rÄ   ©	rO   rP   rQ   rR   r/   rH   rS   rT   rU   rV   s   @@r)   r¿   r¿   Õ   s   ù‡ € õ0÷ò r,   r¿   c                   óT   a a€ ] tR t^åt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚOlmo3DecoderLayerc                ó&   <€ V ^8„  d   QhRS[ RS[/# r“   r–   )r'   r(   s   "€r)   r*   ÚOlmo3DecoderLayer.__annotate__æ   s   ø€ ÷ dñ d™{ð d±sñ dr,   c                ó   <€ \         SV `  4        VP                  V n        \        WR 7      V n        \        V4      V n        \        VP                  VP                  R7      V n	        \        VP                  VP                  R7      V n
        R# ))r”   r•   ©r$   N)r.   r/   r7   r‘   Ú	self_attnr¿   Úmlpr!   r¦   Úpost_attention_layernormÚpost_feedforward_layernormr¬   s   &&&€r)   r/   ÚOlmo3DecoderLayer.__init__æ   sj   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔÜ'¨vÔKˆŒä˜FÓ#ˆŒÜ(4°V×5GÑ5GÈV×M`ÑM`Ô(aˆÔ%Ü*6°v×7IÑ7IÈv×ObÑObÔ*cˆÖ'r,   c                ó.  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[ P                  S[ P                  3,          R,          RS[S[,          R	S[ P                  /# )
r#   rE   rh   NÚposition_idsr°   Ú	use_cacher¯   rk   r%   )r2   r;   Ú
LongTensorr   ÚboolrK   r   r   )r'   r(   s   "€r)   r*   rÐ   ï   sœ   ø€ ÷ ñ á—|‘|ðñ Ÿ™ tÕ+ðñ ×&Ñ&¨Õ-ð	ñ
  �ðñ ˜$•;ðñ #¡5§<¡<±·±Ð#=Õ>ÀÕEðñ Ñ+Õ,ðñ 
�‰ñr,   c                óÔ   € TpV P                   ! RR VRVRVRVRVRV/VB w  rV P                  V4      pW�,           pTpV P                  V4      pV P                  V4      pW�,           pV# )rE   rh   rÙ   r°   rÚ   r¯   © )rÓ   rÕ   rÔ   rÖ   )
r6   rE   rh   rÙ   r°   rÚ   r¯   rk   ÚresidualÚ_s
   &&&&&&&,  r)   rH   ÚOlmo3DecoderLayer.forwardï   s©   € ð !ˆØŸ>š>ñ 
Ø'ð
à)ð
ð &ð
ð ,ð	
ð
  ð
ð !4ð
ð ñ
Ñˆð ×5Ñ5°mÓDˆØ Õ0ˆð !ˆØŸ™ Ó/ˆØ×7Ñ7¸ÓFˆØ Õ0ˆØÐr,   )r7   rÔ   rÕ   rÖ   rÓ   )NNNFNrÌ   rV   s   @@r)   rÎ   rÎ   å   s    ù‡ € ÷dó d÷÷ ò r,   rÎ   c                   óÎ   a a€ ] tR tRt o]! RRR7      RV3R lV 3R lll4       t]]! RRR7      RV3R lR ll4       4       t]P                  ! 4       ]
R	 4       4       tR
tVtV ;t# )ÚOlmo3RotaryEmbeddingi  Údevicez5.18)Úversionc                ó    <€ V ^8„  d   QhRS[ /# ©r#   r”   r   )r'   r(   s   "€r)   r*   Ú!Olmo3RotaryEmbedding.__annotate__  s   ø€ ÷ Uñ U™{ñ Ur,   c           
     óÞ  <€ \         SV `  4        VP                  V n        VP                  V n        Wn        \        \        VP                  4      4      V n        / V n	        V P                   Fû  pV P
                  P                  V,          pVf   K&  VR,          V P                  V&   V P                  pV P                  V,          R8w  d   \        V P                  V,          ,          pV! V P
                  W#R7      w  rg\        W R2\        P                  ! VRR7      4       \        W R2\        P                  ! VP!                  4       RR7      4       \        W R2V4       Ký  	  R # )	NÚ	rope_typeÚdefault©Ú
layer_typeÚ	_inv_freqF)Ú
persistentÚ_original_inv_freqÚ_attention_scaling)r.   r/   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr”   ÚsortedÚsetr©   rê   Úrope_parametersÚcompute_default_rope_parametersr   Úsetattrr0   ÚBufferÚclone)	r6   r”   rä   rí   Úrope_paramsÚrope_init_fnÚcurr_inv_freqÚcurr_attention_scalingr8   s	   &&&     €r)   r/   ÚOlmo3RotaryEmbedding.__init__  s'  ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!ØŒÜ!¤# f×&8Ñ&8Ó"9Ó:ˆÔØˆŒØ×*Ô*ˆJØŸ+™+×5Ñ5°jÕAˆKØÒ"Ùà)4°[Õ)AˆD�N‰N˜:Ñ&Ø%)×%IÑ%IˆLØ�~‰~˜jÕ)¨YÔ6Ü2°4·>±>À*Õ3MÕN�Ù4@ÀÇÁÈfÔ4lÑ1ˆMÜ�D˜L¨	Ð2´B·I²I¸mÐX]Ô4^Ô_Ü�D˜LÐ(:Ð;¼R¿YºYÀ}×GZÑGZÓG\ÐinÔ=oÔpÜ�D˜LÐ(:Ð;Ð=SÖTó +r,   c                ód   <€ V ^8„  d   QhRS[ RS[R,          RS[S[P                  S[3,          /# )r#   r”   rí   Nr%   )r   ÚstrrK   r2   r;   r&   )r'   r(   s   "€r)   r*   rè   '  s7   ø€ ÷ 5ñ 5Ùð5Ù69¸Dµjð5á	‰u�|‰|™UÐ"Õ	#ñ5r,   c           	     ó>  € V P                   V,          R,          p\        V RR4      ;'       g    V P                  V P                  ,          pRpRV\        P
                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a.  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
    layer_type (`str`):
        The current layer type if the model has different RoPE parameters per type.
        Should not be used unless `config.layer_types is not None`

Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetara   Ng      ð?)r?   )r÷   rœ   r7   r�   r2   Úaranger&   r@   )r”   rä   rí   rk   Úbasern   Úattention_factorÚinv_freqs   &&&,    r)   rø   Ú4Olmo3RotaryEmbedding.compute_default_rope_parameters%  sƒ   € ð& ×%Ñ% jÕ1°,Õ?ˆÜ�f˜j¨$Ó/×cÐc°6×3EÑ3EÈ×IcÑIcÕ3cˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r,   c                óö  € \        W R 24      p\        W R24      pVR	,          P                  VP                  ^ ,          R
^4      P                  \        P
                  VP                  R7      pVR,          P                  4       p\        VP                  P                  \        4      '       d2   VP                  P                  R8w  d   VP                  P                  MRp\        VRR7      ;_uu_ 4        Wg,          P                  ^^4      p	\        P                  ! W™3R
R7      p
V
P                  4       V,          pV
P                  4       V,          pRRR4       W¼3#   + '       g   i     XX3# ; i)rî   rñ   N)r?   rä   ÚmpsÚcpuF)Údevice_typeÚenabledr‹   )Nr[   Nr>   )r[   Nr[   )rœ   r\   rL   r@   r2   r&   rä   Ú
isinstanceÚtyper  r   rs   rŒ   r‚   rƒ   )r6   r�   rÙ   rí   r  Úattention_scalingÚinv_freq_expandedÚposition_ids_expandedr  ÚfreqsÚembr‚   rƒ   s   &&&&         r)   rH   ÚOlmo3RotaryEmbedding.forward@  s3  € ô ˜4 <¨yÐ!9Ó:ˆÜ# D¨LÐ8JÐ*KÓLÐð �]Õ#×*Ñ*¨<×+=Ñ+=¸aÕ+@À"ÀaÓH×KÑKÔRW×R]ÑR]Ðfg×fnÑfnÐKÓoð 	ð !-¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆÜ¨¸U×CÖCØ&Õ>×IÑIÈ!ÈQÓOˆEÜ—)’)˜U˜N°Ô3ˆCØ—'‘'“)Ð/Õ/ˆCØ—'‘'“)Ð/Õ/ˆC÷	 Dð ˆxˆ÷ DÖCð �Cˆxˆús   Ã:A E%Å%E8	)r”   r©   ró   rô   rê   r¼   )NN)rO   rP   rQ   rR   r   r/   Ústaticmethodrø   r2   Úno_gradr   rH   rS   rT   rU   rV   s   @@r)   rã   rã     su   ù‡ € Ù�X vÔ.÷Uô Uó /ðUð* Ù�X vÔ.÷5ñ 5ó /ó ð5ð2 ‡]‚]ƒ_Øñó ó ÷ð r,   rã   c                   ór   a a€ ] tR tRt oRtRtR.tR.tRtRt	Rt
RtRtR]R]/tV 3R ltV3R	 ltR
tVtV ;t# )ÚOlmo3PreTrainedModeliV  ÚmodelTrÎ   r°   rE   Ú
attentionsc                ó¸  <€ \         SV `  V4       \        V\        4      '       d´   VP                   F¡  pVP
                  pVP                  V,          R 8w  d   \        VP                  V,          ,          pV! VP                  VR7      w  rE\        P                  ! \        W R24      V4       \        P                  ! \        W R24      V4       K£  	  R# R# )rë   rì   rî   rð   N)r.   Ú_init_weightsr  rã   r©   rø   rê   r   r”   ÚinitÚcopy_rœ   )r6   rd   rí   rý   rþ   rà   r8   s   &&    €r)   r  Ú"Olmo3PreTrainedModel._init_weightsh  s®   ø€ Ü‰Ñ˜fÔ%Ü�fÔ2×3Ò3Ø$×0Ô0�
Ø%×EÑE�Ø×#Ñ# JÕ/°9Ô<Ü#6°v×7GÑ7GÈ
Õ7SÕ#T�LÙ#/°·±È*Ô#UÑ �Ü—
’
œ7 6¨\¸Ð+CÓDÀmÔTÜ—
’
œ7 6¨\Ð9KÐ+LÓMÈ}Ö]ó 1ñ 4r,   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # rç   r   )r'   r(   s   "€r)   r*   Ú!Olmo3PreTrainedModel.__annotate__V  s   ø‡ ‚ áÑò r,   rÞ   )rO   rP   rQ   rR   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendrÎ   r‘   Ú_can_record_outputsr  Ú__annotate_func__rS   rT   rU   rV   s   @@r)   r  r  V  sb   ù‡ € ð  ÐØ&*Ð#Ø,Ð-ÐØ#4Ð"5ÐØÐØ€NØÐà!ÐØ"&ÐàÐ*Ø�nðÐõ
	^÷% … r,   r  c                   ór   a a€ ] tR tRt oV3R lV 3R llt]]]RV3R lR ll4       4       4       tRt	Vt
V ;t# )Ú
Olmo3Modelit  c                ó    <€ V ^8„  d   QhRS[ /# rç   r   )r'   r(   s   "€r)   r*   ÚOlmo3Model.__annotate__v  s   ø€ ÷ ñ ™{ñ r,   c           	     ó  <€ \         SV `  V4       VP                  V n        VP                  V n        \
        P                  ! VP                  VP                  V P                  4      V n        \
        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n        \        VP                  VP                  R 7      V n        \#        VR7      V n        RV n        V P)                  4        R# u upi )rÒ   ©r”   FN)r.   r/   Úpad_token_idÚpadding_idxÚ
vocab_sizer0   Ú	Embeddingr7   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersrÎ   Úlayersr!   r¦   Únormrã   Ú
rotary_embÚgradient_checkpointingÚ	post_initr¬   s   && €r)   r/   ÚOlmo3Model.__init__v  sÊ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸLšL¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m’mÜCHÈ×IaÑIaÔCbÓcÑCb°iÔ˜vÖ1ÑCbÑcó
ˆŒô ! ×!3Ñ!3¸×9LÑ9LÔMˆŒ	Ü.°fÔ=ˆŒØ&+ˆÔ#ð 	�‰Öùò ds   ÂDc                óþ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[S[,          R	S[/# )
r#   Ú	input_idsNrh   rÙ   r°   Úinputs_embedsrÚ   rk   r%   )	r2   rÛ   r;   r   ÚFloatTensorrÜ   r   r   r   )r'   r(   s   "€r)   r*   r2  ‰  s“   ø€ ÷ ;
ñ ;
á×#Ñ# dÕ*ð;
ñ Ÿ™ tÕ+ð;
ñ ×&Ñ&¨Õ-ð	;
ñ
  �ð;
ñ ×(Ñ(¨4Õ/ð;
ñ ˜$•;ð;
ñ Ñ+Õ,ð;
ñ 
!ñ;
r,   c                óÊ  € VR J VR J,          '       d   \        R4      hVf   V P                  V4      pV'       d   Vf   \        V P                  R7      pVfb   Ve   VP	                  4       M^ p\
        P                  ! VP                  ^,          VP                  R7      V,           pVP                  ^ 4      p\        T;p	\        4      '       g/   RV P                  RVRVRVRV/p
R	\        R/ V
B R
\        R/ V
B /p	Tp/ p\        V P                  P                  4       F  pV P!                  W³V4      WÍ&   K  	  \#        V P$                  R V P                  P&                   4       FX  w  rïV! V3RW�P                  P                  V,          ,          RVRVRWÀP                  P                  V,          ,          /VB pKZ  	  V P)                  V4      p\+        VVR7      # )Nz:You must specify exactly one of input_ids or inputs_embedsr4  )rä   r”   rE  rh   r°   rÙ   Úfull_attentionr›   r¯   )Úlast_hidden_stater°   rÞ   )Ú
ValueErrorr9  r   r”   Úget_seq_lengthr2   r  rL   rä   r~   r  Údictr   r   rö   r©   r?  Ú	enumerater=  r<  r>  r   )r6   rD  rh   rÙ   r°   rE  rÚ   rk   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsrE   r¯   rí   ÚiÚdecoder_layers   &&&&&&&,        r)   rH   ÚOlmo3Model.forward†  sñ  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ Ø*.×*;Ñ*;¸IÓ*FˆMç˜Ò0Ü*°$·+±+Ô>ˆOàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLô °Ð?Ð-Ä×FÒFð ˜$Ÿ+™+Ø Ø  .Ø! ?Ø ðˆKð !Ô"4Ñ"C°{Ñ"CØ#Ô%FÑ%UÈÑ%Uð#Ðð
 &ˆØ ÐÜ˜dŸk™k×5Ñ5Ö6ˆJØ.2¯o©o¸mÐ[eÓ.fÐÓ+ñ 7ô !*¨$¯+©+Ð6U¸¿¹×8UÑ8UÐ*VÖ WÑˆAÙ)Øñà2·;±;×3JÑ3JÈ1Õ3MÕNðð *ðð !0ð	ð
 %8¿¹×8OÑ8OÐPQÕ8RÕ$Sðð ñŠMñ !Xð Ÿ	™	 -Ó0ˆÜ&Ø+Ø+ô
ð 	
r,   )r9  r@  r=  r>  r6  r?  r7  )NNNNNN)rO   rP   rQ   rR   r/   r   r   r   rH   rS   rT   rU   rV   s   @@r)   r0  r0  t  s=   ù‡ € ÷ó ð   ØØ÷;
ñ ;
ó ó ó  ÷;
ð ;
r,   r0  c                   ó„   a a€ ] tR tRt oRR/tRR/tRR.R.3/tRR/tV 3R	 lt]	]
RV3R
 lR ll4       4       tRtVtV ;t# )ÚOlmo3ForCausalLMiÇ  zlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputrE   ÚlogitsÚkeep_full_weightc                óì   <€ \         SV `  V4       \        V4      V n        VP                  V n        \
        P                  ! VP                  VP                  R R7      V n        V P                  4        R# rÁ   )
r.   r/   r0  r  r7  r0   r    r7   rV  rA  rÈ   s   &&€r)   r/   ÚOlmo3ForCausalLM.__init__Î  sU   ø€ Ü‰Ñ˜Ô Ü Ó'ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒð 	�‰Ör,   c                óP  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[S[ P                  ,          R
S[S[,          RS[	/
# )r#   rD  Nrh   rÙ   r°   rE  ÚlabelsrÚ   Úlogits_to_keeprk   r%   )
r2   rÛ   r;   r   rF  rÜ   rY   r   r   r   )r'   r(   s   "€r)   r*   ÚOlmo3ForCausalLM.__annotate__Ù  s»   ø€ ÷ 6
ñ 6
á×#Ñ# dÕ*ð6
ñ Ÿ™ tÕ+ð6
ñ ×&Ñ&¨Õ-ð	6
ñ
  �ð6
ñ ×(Ñ(¨4Õ/ð6
ñ × Ñ  4Õ'ð6
ñ ˜$•;ð6
ñ ™eŸl™lÕ*ð6
ñ Ñ+Õ,ð6
ñ 
 ñ6
r,   c	                ó–  € V P                   ! RRVRVRVRVRVRV/V	B p
V
P                  p\        V\        4      '       d   \	        V) R4      MTpV P                  VRVR3,          4      pRpVe-   V P                  ! RR	VR
VRV P                  P                  /V	B p\        VVV
P                  V
P                  V
P                  R7      # )ao  
Example:

```python
>>> from transformers import AutoTokenizer, Olmo3ForCausalLM

>>> model = Olmo3ForCausalLM.from_pretrained("meta-olmo3/Olmo3-2-7b-hf")
>>> tokenizer = AutoTokenizer.from_pretrained("meta-olmo3/Olmo3-2-7b-hf")

>>> prompt = "Hey, are you conscious? Can you talk to me?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
```rD  rh   rÙ   r°   rE  rÚ   Nr[   rX  r]  r7  )ÚlossrX  r°   rE   r  rÞ   )r  rI  r  rY   ÚslicerV  Úloss_functionr”   r7  r   r°   rE   r  )r6   rD  rh   rÙ   r°   rE  r]  rÚ   r^  rk   ÚoutputsrE   Úslice_indicesrX  ra  s   &&&&&&&&&,     r)   rH   ÚOlmo3ForCausalLM.forward×  sÿ   € ð> ,0¯:ª:ñ ,
Øð,
à)ð,
ð &ð,
ð ,ð	,
ð
 (ð,
ð  ð,
ð ñ,
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒØ×%Ò%Ñp¨VÐp¸FÐpÈtÏ{É{×OeÑOeÐpÐioÑpˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r,   )rV  r  r7  )NNNNNNNr   )rO   rP   rQ   rR   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr/   r   r   rH   rS   rT   rU   rV   s   @@r)   rU  rU  Ç  sf   ù‡ € à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÐ/Ð0€Jõð Ø÷6
ñ 6
ó ó ÷6
ð 6
r,   rU  c                   ó   € ] tR tRtRtR# )ÚOlmo3ForSequenceClassificationi  rÞ   N)rO   rP   rQ   rR   rS   rÞ   r,   r)   rl  rl    s   † Ûr,   rl  )r²   )é   )rU  rl  r0  r  )?Úcollections.abcr   r2   Útorch.nnr0   Ú r   r  Úactivationsr   Úcache_utilsr   r   Ú
generationr   Úintegrationsr	   r
   Úmasking_utilsr   r   Úmodeling_layersr   r   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   Úutils.deprecationr   Úutils.genericr   r   r   Úutils.output_capturingr   Úconfiguration_olmo3r   rl   r!   rb   r{   r‰   r   r‘   r¿   rÎ   rã   r  r0  rU  rl  Ú__all__rÞ   r,   r)   Ú<module>r�     sf  ðõ* %ã Ý å &Ý !ß .Ý )ß Lß Rß [ß Oß Kß FÝ &ß 5Ý 0ß [Ñ [Ý 5Ý ,ñ ˜YÓ'ôJ�2—9‘9ó Jó (ðJõ(	U÷%ñ2 Ð-Ó.ó2ó /ð2ò4(ñ Ð)Ó*ôI)�R—Y‘Yó I)ó +ðI)ôXˆr�y‰yô ô &Ð2ô &ôRE˜2Ÿ9™9ô EðP ô^˜?ó ^ó ð^ð: ôO
Ð%ó O
ó ðO
ðd ôG
Ð+¨_ó G
ó ðG
ôT	Ð%EÐG[ô 	ò g‚r,   