+
    œ°³jRR  ã                   ó¨  € ^ RI Ht ^ RIt^ RIHt ^ RIHu Ht ^RIH	t	 ^RI
HtHt ^RIHt ^RIHtHt ^RIHt ^RIHtHt ^R	IHtHt ^R
IHtHt ^RIHtHt ^RI H!t! ^RI"H#t#H$t$H%t% ^RI&H't' ^RI(H)t)H*t* ^RI+H,t, ^RI-H.t.  ! R R]P^                  4      t0 ! R R]P^                  4      t1 ! R R]P^                  4      t2R t3R R lt4R+R R llt5]! R4      R,R l4       t6]! ]64       ! R R ]P^                  4      4       t7 ! R! R"]4      t8]$ ! R# R$]4      4       t9]$ ! R% R&]94      4       t:]$ ! R' R(]9]4      4       t; ! R) R*]]94      t<. R-Ot=R# ).é    )ÚCallableN)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_kernel_forward_from_hubÚuse_kernelized_func)Úcreate_causal_mask)Ú GenericForSequenceClassificationÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputs©Ú
OlmoConfigc                   óT   a a€ ] tR t^1t oRtV3R lV 3R lltV3R lR ltRtVtV ;t	# )ÚOlmoLayerNormz/LayerNorm but with no learnable weight or bias.c                ó$   <€ V ^8„  d   QhRS[ RR/# )é   Úhidden_sizeÚreturnN)Úint)ÚformatÚ__classdict__s   "€Úg/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/olmo/modeling_olmo.pyÚ__annotate__ÚOlmoLayerNorm.__annotate__4   s   ø€ ÷ /ñ /¡Cð /¨Dñ /ó    c                ó4   <€ \         SV `  4        V3V n        R # ©N)ÚsuperÚ__init__Únormalized_shape)Úselfr!   Ú	__class__s   &&€r&   r-   ÚOlmoLayerNorm.__init__4   s   ø€ Ü‰ÑÔØ!, ˆÖr)   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r    Úhidden_statesr"   )ÚtorchÚTensor)r$   r%   s   "€r&   r'   r(   8   s#   ø€ ÷ 
ñ 
¡U§\¡\ð 
±e·l±lñ 
r)   c                ó¾   € VP                   p\        P                  ! VP                  \        P
                  R 7      V P                  RRRR7      P                  V4      # )©ÚdtypeNgñhãˆµøä>)Úeps)r8   ÚFÚ
layer_normÚtor4   Úfloat32r.   )r/   r3   Ú
orig_dtypes   && r&   ÚforwardÚOlmoLayerNorm.forward8   sO   € Ø"×(Ñ(ˆ
Ü�|Š|˜M×,Ñ,´5·=±=Ð,ÓAÀ4×CXÑCXÐZ^Ð`dÐjnÔo×rÑrØó
ð 	
r)   )r.   ©
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r-   r?   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r0   r%   s   @@r&   r   r   1   s!   ù‡ € Ù9÷/ó /÷
÷ 
ð 
r)   r   c                   ó8   a a€ ] tR t^?t oV 3R ltR tRtVtV ;t# )ÚOlmoMLPc                óØ  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        \
        P                  ! V P                  V P                  R R7      V n        \
        P                  ! V P                  V P                  R R7      V n        \
        P                  ! V P                  V P                  R R7      V n	        \        VP                  ,          V n        R# ©F©ÚbiasN)r,   r-   Úconfigr!   Úintermediate_sizeÚnnÚLinearÚ	gate_projÚup_projÚ	down_projr   Ú
hidden_actÚact_fn©r/   rQ   r0   s   &&€r&   r-   ÚOlmoMLP.__init__@   s¤   ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔÜŸš 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆŒÜ˜V×.Ñ.Õ/ˆŽr)   c                ó�   € V P                  V P                  V P                  V4      4      V P                  V4      ,          4      pV# r+   )rW   rY   rU   rV   )r/   ÚxrW   s   && r&   r?   ÚOlmoMLP.forwardJ   s6   € Ø—N‘N 4§;¡;¨t¯~©~¸aÓ/@Ó#AÀDÇLÁLÐQRÃOÕ#SÓTˆ	ØÐr)   )rY   rQ   rW   rU   r!   rR   rV   ©	rB   rC   rD   rE   r-   r?   rG   rH   rI   rJ   s   @@r&   rL   rL   ?   s   ù‡ € õ0÷ò r)   rL   c                   óÎ   a a€ ] tR t^Ot o]! RRR7      R
V3R lV 3R lll4       t]]! RRR7      R
V3R lR ll4       4       t]P                  ! 4       ]
R 4       4       tR	tVtV ;t# )ÚOlmoRotaryEmbeddingÚdevicez5.18)Úversionc                ó    <€ V ^8„  d   QhRS[ /# ©r    rQ   r   )r$   r%   s   "€r&   r'   Ú OlmoRotaryEmbedding.__annotate__Q   s   ø€ ÷ Oñ O™zñ Or)   c                óä  <€ \         SV `  4        VP                  V n        VP                  V n        Wn        V P
                  P                  R ,          V n        V P                  pV P                  R8w  d   \        V P                  ,          pV! V P
                  V4      w  q@n
        \        P                  ! VRR7      V n        \        P                  ! VP                  4       RR7      V n        R# )Ú	rope_typeÚdefaultF)Ú
persistentN)r,   r-   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenrQ   Úrope_parametersrh   Úcompute_default_rope_parametersr   Úattention_scalingrS   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)r/   rQ   rb   Úrope_init_fnrr   r0   s   &&&  €r&   r-   ÚOlmoRotaryEmbedding.__init__P   s¬   ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àŒàŸ™×4Ñ4°[ÕAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÔ&Ü.¨t¯~©~Õ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÔ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÔ!NˆÖr)   c                óP   <€ V ^8„  d   QhRS[ RS[S[P                  S[3,          /# )r    rQ   r"   )r   Útupler4   r5   Úfloat)r$   r%   s   "€r&   r'   rf   c   s.   ø€ ÷ 5ñ 5±
ð 5ÑV[Ñ\a×\hÑ\hÑjoÐ\oÕVpñ 5r)   c           	     ó0  € V P                   R,          p\        V RR4      ;'       g    V P                  V P                  ,          pRpRV\        P
                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚhead_dimNg      ð?r7   )rn   Úgetattrr!   Únum_attention_headsr4   Úarangery   r<   )rQ   rb   ÚkwargsÚbaseÚdimÚattention_factorrr   s   &&,    r&   ro   Ú3OlmoRotaryEmbedding.compute_default_rope_parametersa   s~   € ð ×%Ñ% lÕ3ˆÜ�f˜j¨$Ó/×cÐc°6×3EÑ3EÈ×IcÑIcÕ3cˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r)   c                ó0  € V P                   R,          P                  4       P                  VP                  ^ ,          R^4      P	                  VP
                  4      pVR,          P                  4       p\        VP
                  P                  \        4      '       d2   VP
                  P                  R8w  d   VP
                  P                  MRp\        VRR7      ;_uu_ 4        VP                  4       VP                  4       ,          P                  ^^4      p\        P                  ! Wf3RR7      pVP                  4       V P                  ,          pVP                  4       V P                  ,          p	R R R 4       W‰3#   + '       g   i     XX	3# ; i)	NÚmpsÚcpuF)Údevice_typeÚenabled©r‚   )NºNNNNéÿÿÿÿ)r‹   Nr‹   )rr   ry   ÚexpandÚshaper<   rb   Ú
isinstanceÚtypeÚstrr   Ú	transposer4   ÚcatÚcosrp   Úsin)
r/   r]   Úposition_idsÚinv_freq_expandedÚposition_ids_expandedrˆ   ÚfreqsÚembr”   r•   s
   &&&       r&   r?   ÚOlmoRotaryEmbedding.forwardu   s.  € ð !ŸM™M¨-Õ8×>Ñ>Ó@×GÑGÈ×HZÑHZÐ[\ÕH]Ð_aÐcdÓe×hÑhÐij×iqÑiqÓrÐØ ,¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆÜ¨¸U×CÖCØ&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÕN×YÑYÐZ[Ð]^Ó_ˆEÜ—)’)˜U˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆC÷	 Dð
 ˆxˆ÷ DÖCð
 �Cˆxˆús   Ã&BFÆF	)rp   rQ   rr   rl   rt   rm   rh   r+   )rB   rC   rD   rE   r   r-   Ústaticmethodro   r4   Úno_gradr   r?   rG   rH   rI   rJ   s   @@r&   ra   ra   O   su   ù‡ € Ù�X vÔ.÷Oô Oó /ðOð  Ù�X vÔ.÷5ñ 5ó /ó ð5ð$ ‡]‚]ƒ_Øñ
ó ó ÷
ð 
r)   ra   c                óÌ   € V RRV P                   R,          ^,          13,          pV RV P                   R,          ^,          R13,          p\        P                  ! V) V3RR7      # )z*Rotates half the hidden dims of the input..NrŠ   rŒ   )rŽ   r4   r“   )r]   Úx1Úx2s   &  r&   Úrotate_halfr¡   „   sZ   € à	
ˆ3Ð"�!—'‘'˜"•+ Õ"Ð"Ð"Õ	#€BØ	
ˆ3�—‘˜•˜qÕ Ñ"Ð"Õ	#€BÜ�9Š9�r�c˜2�Y BÔ'Ð'r)   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r    r3   Ún_repr"   )r4   r5   r#   )r$   s   "r&   r'   r'   ‹   s.   € ÷ 	Uñ 	UœUŸ\™\ð 	U´#ð 	U¼%¿,¹,ñ 	Ur)   c                ó˜   € V P                   w  r#rEV^8X  d   V # V R,          P                  W#WV4      p V P                  W#V,          WE4      # )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
)r‹   r‹   Nr‹   r‹   )rŽ   r�   Úreshape)r3   r£   ÚbatchÚnum_key_value_headsÚslenr|   s   &&    r&   Ú	repeat_kvr©   ‹   sU   € ð
 2?×1DÑ1DÑ.€E Ø�„zØÐØ!Ð"2Õ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÕ(CÀTÓTÐTr)   c                ó   € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R\        R	\
        \        ,          /# )
r    ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNÚscalingÚdropoutr€   )rS   ÚModuler4   r5   ry   r   r   )r$   s   "r&   r'   r'   —   sw   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ð%ô ð%ô Ô'Õ(ñ%r)   c                 ó"  € \        W P                  4      p\        W0P                  4      p	\        P                  ! WP	                  ^^4      4      V,          p
Ve	   W¤,           p
\
        P                  P                  V
R\        P                  R7      P                  VP                  4      p
\
        P                  P                  W¦V P                  R7      p
\        P                  ! W©4      pVP	                  ^^4      P                  4       pWº3# )r    )r‚   r8   )ÚpÚtrainingrŒ   )r©   Únum_key_value_groupsr4   Úmatmulr’   rS   Ú
functionalÚsoftmaxr=   r<   r8   r±   rµ   Ú
contiguous)r«   r¬   r­   r®   r¯   r°   r±   r€   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs   &&&&&&&,    r&   Úeager_attention_forwardr¿   —   sÒ   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÕL€LØÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r)   Úrotary_pos_embc                ó4  € V P                   VP                   reVP                  V4      pVP                  V4      pW,          \        V 4      V,          ,           pW,          \        V4      V,          ,           pVP                  V4      VP                  V4      3# )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)r8   Ú	unsqueezer¡   r<   )	ÚqÚkr”   r•   Úunsqueeze_dimÚq_typeÚk_typeÚq_embedÚk_embeds	   &&&&&    r&   Úapply_rotary_pos_embrÊ   °   sv   € ð& —W‘W˜aŸg™gˆFØ
�-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�wœ; q›>¨CÕ/Õ0€GØ�wœ; q›>¨CÕ/Õ0€GØ�:‰:�fÓ˜wŸz™z¨&Ó1Ð1Ð1r)   c                   óX   a a€ ] tR t^Ët oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )ÚOlmoAttentionz=Multi-headed attention from 'Attention Is All You Need' paperc                ó&   <€ V ^8„  d   QhRS[ RS[/# ©r    rQ   Ú	layer_idx©r   r#   )r$   r%   s   "€r&   r'   ÚOlmoAttention.__annotate__Ï   s   ø€ ÷ 
ñ 
™zð 
±cñ 
r)   c                ó¢  <€ \         SV `  4        Wn        W n        \	        VR VP
                  VP                  ,          4      V n        VP                  VP                  ,          V n	        V P                  R,          V n
        VP                  V n        RV n        \        P                  ! VP
                  VP                  V P                  ,          VP                  R7      V n        \        P                  ! VP
                  VP                  V P                  ,          VP                  R7      V n        \        P                  ! VP
                  VP                  V P                  ,          VP                  R7      V n        \        P                  ! VP                  V P                  ,          VP
                  VP                  R7      V n        R# )r|   TrO   Ng      à¿)r,   r-   rQ   rÏ   r}   r!   r~   r|   r§   r¶   r°   Úattention_dropoutÚ	is_causalrS   rT   Úattention_biasÚq_projÚk_projÚv_projÚo_proj©r/   rQ   rÏ   r0   s   &&&€r&   r-   ÚOlmoAttention.__init__Ï   sI  ø€ Ü‰ÑÔØŒØ"ŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÕ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Õ$\ˆÔ!Ø—}‘} dÕ*ˆŒØ!'×!9Ñ!9ˆÔØˆŒä—i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×&Ñ&¨¯©Õ6¸×8JÑ8JÐQW×QfÑQfô
ˆŽr)   c                ó  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          RS[ P                  R,          RS[R,          RS[S[ P                  S[ P                  R,          3,          /# )r    r3   Úposition_embeddingsr¯   NÚpast_key_valuesr"   )r4   r5   rx   r   )r$   r%   s   "€r&   r'   rÑ   æ   sw   ø€ ÷ /)ñ /)á—|‘|ð/)ñ #¡5§<¡<±·±Ð#=Õ>ð/)ñ Ÿ™ tÕ+ð	/)ñ
  �ð/)ñ 
‰u�|‰|™UŸ\™\¨DÕ0Ð0Õ	1ñ/)r)   c                ó´  € VP                   R R p. VORNV P                  N5pV P                  V4      pV P                  V4      p	V P	                  V4      p
V P
                  P                  eµ   VP                  V P
                  P                  ) V P
                  P                  R7       V	P                  V P
                  P                  ) V P
                  P                  R7       V
P                  V P
                  P                  ) V P
                  P                  R7       VP                  V4      P                  ^^4      pV	P                  V4      P                  ^^4      p	V
P                  V4      P                  ^^4      p
Vw  r¼\        W‰W¼4      w  r‰Ve   VP                  WšV P                  4      w  rš\        P                  ! V P
                  P                  \         4      pT! V VV	V
V3RV P"                  '       g   RMV P$                  RV P&                  /VB w  rïVP(                  ! . VORN5!  P+                  4       pV P-                  V4      pWï3# )N)ÚminÚmaxr±   ç        r°   rŒ   )rŽ   r|   rÖ   r×   rØ   rQ   Úclip_qkvÚclamp_Úviewr’   rÊ   ÚupdaterÏ   r   Úget_interfaceÚ_attn_implementationr¿   rµ   rÓ   r°   r¥   rº   rÙ   )r/   r3   rÝ   r¯   rÞ   r€   Úinput_shapeÚhidden_shapeÚquery_statesr»   r¼   r”   r•   Úattention_interfacer¾   r½   s   &&&&&,          r&   r?   ÚOlmoAttention.forwardæ   s  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1ˆØ—[‘[ Ó/ˆ
Ø—{‘{ =Ó1ˆà�;‰;×ÑÒ+Ø×Ñ T§[¡[×%9Ñ%9Ð$9¸t¿{¹{×?SÑ?SÐÔTØ×Ñ 4§;¡;×#7Ñ#7Ð"7¸T¿[¹[×=QÑ=QÐÔRØ×Ñ T§[¡[×%9Ñ%9Ð$9¸t¿{¹{×?SÑ?SÐÔTà#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆØ—_‘_ \Ó2×<Ñ<¸QÀÓBˆ
Ø#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—k‘k +Ó.ˆØÐ(Ð(r)   )rÓ   rQ   r|   rÔ   r×   rÏ   r¶   rÙ   rÖ   r°   rØ   r+   rA   rJ   s   @@r&   rÌ   rÌ   Ë   s!   ù‡ € áG÷
ó 
÷./)÷ /)ò /)r)   rÌ   c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚOlmoDecoderLayeri  c                ó&   <€ V ^8„  d   QhRS[ RS[/# rÎ   rÐ   )r$   r%   s   "€r&   r'   ÚOlmoDecoderLayer.__annotate__  s   ø€ ÷ Jñ J™zð J±cñ Jr)   c                óð   <€ \         SV `  4        VP                  V n        \        WR 7      V n        \        V4      V n        \        VP                  4      V n        \        VP                  4      V n	        R# ))rQ   rÏ   N)
r,   r-   r!   rÌ   Ú	self_attnrL   Úmlpr   Úinput_layernormÚpost_attention_layernormrÚ   s   &&&€r&   r-   ÚOlmoDecoderLayer.__init__  sY   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔÜ&¨fÔJˆŒä˜6“?ˆŒÜ,¨V×-?Ñ-?Ó@ˆÔÜ(5°f×6HÑ6HÓ(IˆÖ%r)   c                ó.  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[ P                  S[ P                  3,          R,          RS[S[,          R	S[ P                  /# )
r    r3   r¯   Nr–   rÞ   Ú	use_cacherÝ   r€   r"   )r4   r5   Ú
LongTensorr   Úboolrx   r   r   )r$   r%   s   "€r&   r'   rñ   "  sœ   ø€ ÷ ñ á—|‘|ðñ Ÿ™ tÕ+ðñ ×&Ñ&¨Õ-ð	ñ
  �ðñ ˜$•;ðñ #¡5§<¡<±·±Ð#=Õ>ÀÕEðñ Ñ+Õ,ðñ 
�‰ñr)   c                óÔ   € TpV P                  V4      pV P                  ! RR VRVRVRVRVRV/VB w  rW�,           pTpV P                  V4      pV P                  V4      pW�,           pV# )r3   r¯   r–   rÞ   rù   rÝ   © )rõ   ró   rö   rô   )
r/   r3   r¯   r–   rÞ   rù   rÝ   r€   ÚresidualÚ_s
   &&&&&&&,  r&   r?   ÚOlmoDecoderLayer.forward"  s©   € ð !ˆØ×,Ñ,¨]Ó;ˆàŸ>š>ñ 
Ø'ð
à)ð
ð &ð
ð ,ð	
ð
  ð
ð !4ð
ð ñ
Ñˆð !Õ0ˆð !ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ Õ0ˆØÐr)   )r!   rõ   rô   rö   ró   )NNNFNr_   rJ   s   @@r&   rï   rï     s    ù‡ € ÷Jó J÷÷ ò r)   rï   c                   ó`   a € ] tR tRt o RtRtR.tR.tRtRt	Rt
RtRtR]R]/tV 3R ltR	tV tR
# )ÚOlmoPreTrainedModeliB  ÚmodelTrï   rÞ   r3   Ú
attentionsc                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # re   r   )r$   r%   s   "€r&   r'   Ú OlmoPreTrainedModel.__annotate__B  s   ø‡ ‚ áÑò r)   rý   N)rB   rC   rD   rE   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendrï   rÌ   Ú_can_record_outputsÚ__annotate_func__rG   rH   )r%   s   @r&   r  r  B  s\   ø‡ € ð  ÐØ&*Ð#Ø+Ð,ÐØ#4Ð"5ÐØÐØ€NØÐà!ÐØ"&ÐàÐ)Ø�mðÐ÷ ƒ r)   r  c                   ór   a a€ ] tR tRt oV3R lV 3R llt]]]RV3R lR ll4       4       4       tRt	Vt
V ;t# )Ú	OlmoModeliU  c                ó    <€ V ^8„  d   QhRS[ /# re   r   )r$   r%   s   "€r&   r'   ÚOlmoModel.__annotate__W  s   ø€ ÷ ñ ™zñ r)   c           	     óü  <€ \         SV `  V4       VP                  V n        VP                  V n        \
        P                  ! VP                  VP                  V P                  4      V n        \
        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n        \        VP                  4      V n        \!        VR 7      V n        RV n        V P'                  4        R# u upi )©rQ   FN)r,   r-   Úpad_token_idÚpadding_idxÚ
vocab_sizerS   Ú	Embeddingr!   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersrï   Úlayersr   Únormra   Ú
rotary_embÚgradient_checkpointingÚ	post_initrÚ   s   && €r&   r-   ÚOlmoModel.__init__W  sÂ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸLšL¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m’mÜBGÈ×H`ÑH`ÔBaÓbÑBa°YÔ˜fÖ0ÑBaÑbó
ˆŒô " &×"4Ñ"4Ó5ˆŒ	Ü-°VÔ<ˆŒØ&+ˆÔ#ð 	�‰Öùò cs   ÂC9c                óþ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[S[,          R	S[/# )
r    Ú	input_idsNr¯   r–   rÞ   Úinputs_embedsrù   r€   r"   )	r4   rú   r5   r   ÚFloatTensorrû   r   r   r   )r$   r%   s   "€r&   r'   r  j  s“   ø€ ÷ 2
ñ 2
á×#Ñ# dÕ*ð2
ñ Ÿ™ tÕ+ð2
ñ ×&Ñ&¨Õ-ð	2
ñ
  �ð2
ñ ×(Ñ(¨4Õ/ð2
ñ ˜$•;ð2
ñ Ñ+Õ,ð2
ñ 
!ñ2
r)   c                óˆ  € VR J VR J,          '       d   \        R4      hVf   V P                  V4      pV'       d   Vf   \        V P                  R7      pVfb   Ve   VP	                  4       M^ p\
        P                  ! VP                  ^,          VP                  R7      V,           pVP                  ^ 4      p\        V P                  VVVVR7      p	Tp
V P                  W£R7      pV P                  R V P                  P                    F  pV! V
3RV	RVRVR	VR
V/VB p
K  	  V P                  V
4      p
\        V
VR7      # )Nz:You must specify exactly one of input_ids or inputs_embedsr  )rb   )rQ   r(  r¯   rÞ   r–   )r–   r¯   rÝ   r–   rÞ   rù   )Úlast_hidden_staterÞ   )Ú
ValueErrorr  r   rQ   Úget_seq_lengthr4   r   rŽ   rb   rÂ   r
   r"  r   r  r!  r   )r/   r'  r¯   r–   rÞ   r(  rù   r€   Úpast_seen_tokensÚcausal_maskr3   rÝ   Údecoder_layers   &&&&&&&,     r&   r?   ÚOlmoModel.forwardg  se  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ Ø*.×*;Ñ*;¸IÓ*FˆMç˜Ò0Ü*°$·+±+Ô>ˆOàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLä(Ø—;‘;Ø'Ø)Ø+Ø%ô
ˆð &ˆØ"Ÿo™o¨m˜oÓWÐà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÓIˆMÙ)Øñà*ðð %8ðð *ð	ð
 !0ðð $ðð ñŠMñ Jð Ÿ	™	 -Ó0ˆÜ&Ø+Ø+ô
ð 	
r)   )r  r#  r   r!  r  r"  r  )NNNNNN)rB   rC   rD   rE   r-   r   r   r   r?   rG   rH   rI   rJ   s   @@r&   r  r  U  s=   ù‡ € ÷ó ð   ØØ÷2
ñ 2
ó ó ó  ÷2
ð 2
r)   r  c                   ó„   a a€ ] tR tRt oRR/tRR/tRR.R.3/tRR/tV 3R	 lt]	]
RV3R
 lR ll4       4       tRtVtV ;t# )ÚOlmoForCausalLMiŸ  zlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputr3   ÚlogitsÚkeep_full_weightc                óì   <€ \         SV `  V4       \        V4      V n        VP                  V n        \
        P                  ! VP                  VP                  R R7      V n        V P                  4        R# rN   )
r,   r-   r  r  r  rS   rT   r!   r4  r$  rZ   s   &&€r&   r-   ÚOlmoForCausalLM.__init__¦  sU   ø€ Ü‰Ñ˜Ô Ü˜vÓ&ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒð 	�‰Ör)   c                óP  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[S[ P                  ,          R
S[S[,          RS[	/
# )r    r'  Nr¯   r–   rÞ   r(  Úlabelsrù   Úlogits_to_keepr€   r"   )
r4   rú   r5   r   r)  rû   r#   r   r   r   )r$   r%   s   "€r&   r'   ÚOlmoForCausalLM.__annotate__±  s»   ø€ ÷ 6
ñ 6
á×#Ñ# dÕ*ð6
ñ Ÿ™ tÕ+ð6
ñ ×&Ñ&¨Õ-ð	6
ñ
  �ð6
ñ ×(Ñ(¨4Õ/ð6
ñ × Ñ  4Õ'ð6
ñ ˜$•;ð6
ñ ™eŸl™lÕ*ð6
ñ Ñ+Õ,ð6
ñ 
 ñ6
r)   c	                ó–  € V P                   ! RRVRVRVRVRVRV/V	B p
V
P                  p\        V\        4      '       d   \	        V) R4      MTpV P                  VRVR3,          4      pRpVe-   V P                  ! RR	VR
VRV P                  P                  /V	B p\        VVV
P                  V
P                  V
P                  R7      # )ai  
Example:

```python
>>> from transformers import AutoTokenizer, OlmoForCausalLM

>>> model = OlmoForCausalLM.from_pretrained("meta-olmo/Olmo-2-7b-hf")
>>> tokenizer = AutoTokenizer.from_pretrained("meta-olmo/Olmo-2-7b-hf")

>>> prompt = "Hey, are you conscious? Can you talk to me?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
```r'  r¯   r–   rÞ   r(  rù   Nr‹   r6  r;  r  )Úlossr6  rÞ   r3   r  rý   )r  r+  r�   r#   Úslicer4  Úloss_functionrQ   r  r   rÞ   r3   r  )r/   r'  r¯   r–   rÞ   r(  r;  rù   r<  r€   Úoutputsr3   Úslice_indicesr6  r?  s   &&&&&&&&&,     r&   r?   ÚOlmoForCausalLM.forward¯  sÿ   € ð> ,0¯:ª:ñ ,
Øð,
à)ð,
ð &ð,
ð ,ð	,
ð
 (ð,
ð  ð,
ð ñ,
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒØ×%Ò%Ñp¨VÐp¸FÐpÈtÏ{É{×OeÑOeÐpÐioÑpˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r)   )r4  r  r  )NNNNNNNr   )rB   rC   rD   rE   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr-   r   r   r?   rG   rH   rI   rJ   s   @@r&   r3  r3  Ÿ  sf   ù‡ € à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÐ/Ð0€Jõð Ø÷6
ñ 6
ó ó ÷6
ð 6
r)   r3  c                   ó   € ] tR tRtRtR# )ÚOlmoForSequenceClassificationiê  rý   N)rB   rC   rD   rE   rG   rý   r)   r&   rJ  rJ  ê  s   † Ûr)   rJ  )râ   )é   )r3  rJ  r  r  )>Úcollections.abcr   r4   Útorch.nnrS   Útorch.nn.functionalr¸   r:   Úactivationsr   Úcache_utilsr   r   Ú
generationr   Úintegrationsr   r	   Úmasking_utilsr
   Úmodeling_layersr   r   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   Úconfiguration_olmor   r²   r   rL   ra   r¡   r©   r¿   rÊ   rÌ   rï   r  r  r3  rJ  Ú__all__rý   r)   r&   Ú<module>r_     sM  ðõ4 %ã Ý ß Ð å !ß .Ý )ß LÝ /ß [ß Oß Kß FÝ &ß IÑ IÝ 0ß GÝ 5Ý *ô
�B—I‘Iô 
ôˆb�i‰iô ô 2˜"Ÿ)™)ô 2òj(õ	U÷%ñ2 Ð-Ó.ó2ó /ð2ñ4 Ð)Ó*ôI)�B—I‘Ió I)ó +ðI)ôX'Ð1ô 'ðT ô˜/ó ó ðð$ ôF
Ð#ó F
ó ðF
ðR ôG
Ð)¨?ó G
ó ðG
ôT	Ð$DÐFYô 	ò c‚r)   