+
    œ°³jF_  ã                   ó˜  € R t ^ RIHt ^ RIt^ RIHt ^RIHt ^RIHtH	t	 ^RI
Ht ^RIHt ^RIHtHtHt ^R	IHtHt ^R
IHtHt ^RIHtHt ^RIHt ^RIHtHtH t H!t! ^RI"H#t# ^RI$H%t%H&t& ^RI'H(t( ^RI)H*t* ]!PV                  ! ],4      t- ! R R]P\                  4      t/R t0R,R lt1 ! R R]P\                  4      t2 ! R R]P\                  4      t3R R lt4R-R R llt5 ! R R]P\                  4      t6 ! R  R!]4      t7] ! R" R#]4      4       t8] ! R$ R%]84      4       t9 ! R& R']8]4      t: ! R( R)]]84      t; ! R* R+]]84      t<. R.Ot=R# )/zPyTorch StableLM model.)ÚCallableN)Únn)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úcreate_causal_mask)Ú GenericForSequenceClassificationÚGenericForTokenClassificationÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputs©ÚStableLmConfigc                   óÎ   a a€ ] tR t^8t o]! RRR7      R
V3R lV 3R lll4       t]]! RRR7      R
V3R lR ll4       4       t]P                  ! 4       ]
R 4       4       tR	tVtV ;t# )ÚStableLmRotaryEmbeddingÚdevicez5.18)Úversionc                ó    <€ V ^8„  d   QhRS[ /# ©é   Úconfigr   )ÚformatÚ__classdict__s   "€Úo/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/stablelm/modeling_stablelm.pyÚ__annotate__Ú$StableLmRotaryEmbedding.__annotate__:   s   ø€ ÷ Oñ O™~ñ Oó    c                óä  <€ \         SV `  4        VP                  V n        VP                  V n        Wn        V P
                  P                  R ,          V n        V P                  pV P                  R8w  d   \        V P                  ,          pV! V P
                  V4      w  q@n
        \        P                  ! VRR7      V n        \        P                  ! VP                  4       RR7      V n        R# )Ú	rope_typeÚdefaultF)Ú
persistentN)ÚsuperÚ__init__Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr$   Úrope_parametersr,   Úcompute_default_rope_parametersr   Úattention_scalingr   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)Úselfr$   r   Úrope_init_fnr8   Ú	__class__s   &&&  €r'   r0   Ú StableLmRotaryEmbedding.__init__9   s¬   ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àŒàŸ™×4Ñ4°[ÕAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÔ&Ü.¨t¯~©~Õ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÔ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÔ!NˆÖr*   c                ó:   <€ V ^8„  d   QhRS[ RS[RS[3,          /# )r#   r$   Úreturnztorch.Tensor)r   ÚtupleÚfloat)r%   r&   s   "€r'   r(   r)   M   s(   ø€ ÷ 5ñ 5±ð 5ÑZ_Ð`nÑpuÐ`uÕZvñ 5r*   c           	     óŠ  € V P                   R,          pV P                   P                  RR4      p\        V RR4      ;'       g    V P                  V P                  ,          p\        WT,          4      pRpRV\        P                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚpartial_rotary_factorg      ð?Úhead_dimN©Údtype)
r4   ÚgetÚgetattrÚhidden_sizeÚnum_attention_headsÚintÚtorchÚarangerB   Úto)	r$   r   ÚkwargsÚbaserE   rF   ÚdimÚattention_factorr8   s	   &&,      r'   r5   Ú7StableLmRotaryEmbedding.compute_default_rope_parametersJ   s¦   € ð ×%Ñ% lÕ3ˆØ &× 6Ñ 6× :Ñ :Ð;RÐTWÓ XÐÜ˜6 :¨tÓ4×hÐh¸×8JÑ8JÈf×NhÑNhÕ8hˆÜ�(Õ2Ó3ˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r*   c                ó^  € V P                   R,          P                  VP                  ^ ,          R	^4      P                  \        P
                  VP                  R7      pVR
,          P                  4       p\        VP                  P                  \        4      '       d2   VP                  P                  R8w  d   VP                  P                  MRp\        VRR7      ;_uu_ 4        W4,          P                  ^^4      p\        P                  ! Wf3R	R7      pVP                  4       V P                  ,          pVP                  4       V P                  ,          p	R R R 4       XP                  VP                   R7      X	P                  VP                   R7      3#   + '       g   i     LH; i)N)rH   r   ÚmpsÚcpuF)Údevice_typeÚenabled©rS   rG   )NºNNNNéÿÿÿÿ)r\   Nr\   )r8   ÚexpandÚshaperP   rN   rB   r   Ú
isinstanceÚtypeÚstrr   Ú	transposeÚcatÚcosr6   ÚsinrH   )
r;   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedrY   ÚfreqsÚembre   rf   s
   &&&       r'   ÚforwardÚStableLmRotaryEmbedding.forwarda   s2  € ð �M‰M˜-Õ(×/Ñ/°×0BÑ0BÀ1Õ0EÀrÈ1ÓM×PÑPÔW\×WbÑWbÐkl×ksÑksÐPÓtð 	ð !-¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆä¨¸U×CÖCØ&Õ>×IÑIÈ!ÈQÓOˆEÜ—)’)˜U˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆC÷	 Dð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷ D×Cús   Ã(A4FÆF,	)r6   r$   r8   r2   r:   r3   r,   ©N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r0   Ústaticmethodr5   rN   Úno_gradr   rm   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r=   r&   s   @@r'   r   r   8   su   ù‡ € Ù�X vÔ.÷Oô Oó /ðOð  á�X vÔ.÷5ñ 5ó /ó ð5ð( ‡]‚]ƒ_Øñ<ó ó ÷<ð <r*   r   c                óÌ   € V RRV P                   R,          ^,          13,          pV RV P                   R,          ^,          R13,          p\        P                  ! V) V3RR7      # )z*Rotates half the hidden dims of the input..Nr[   r]   )r_   rN   rd   )rg   Úx1Úx2s   &  r'   Úrotate_halfr}   u   sZ   € à	
ˆ3Ð"�!—'‘'˜"•+ Õ"Ð"Ð"Õ	#€BØ	
ˆ3�—‘˜•˜qÕ Ñ"Ð"Õ	#€BÜ�9Š9�r�c˜2�Y BÔ'Ð'r*   c                óÈ   € VP                  V4      pVP                  V4      pW,          \        V 4      V,          ,           pW,          \        V4      V,          ,           pWV3# )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)Ú	unsqueezer}   )ÚqÚkre   rf   Úunsqueeze_dimÚq_embedÚk_embeds   &&&&&  r'   Úapply_rotary_pos_embr…   }   sS   € ð$ �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�wœ; q›>¨CÕ/Õ0€GØ�wœ; q›>¨CÕ/Õ0€GØÐÐr*   c                   ó8   a a€ ] tR t^—t oV 3R ltR tRtVtV ;t# )ÚStableLmMLPc                óØ  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        \
        P                  ! V P                  V P                  R R7      V n        \
        P                  ! V P                  V P                  R R7      V n        \
        P                  ! V P                  V P                  R R7      V n	        \        VP                  ,          V n        R# ©F©ÚbiasN)r/   r0   r$   rK   Úintermediate_sizer   ÚLinearÚ	gate_projÚup_projÚ	down_projr   Ú
hidden_actÚact_fn©r;   r$   r=   s   &&€r'   r0   ÚStableLmMLP.__init__˜   s¤   ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔÜŸš 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆŒÜ˜V×.Ñ.Õ/ˆŽr*   c                ó�   € V P                  V P                  V P                  V4      4      V P                  V4      ,          4      pV# ro   )r�   r’   rŽ   r�   )r;   rg   r�   s   && r'   rm   ÚStableLmMLP.forward¢   s6   € Ø—N‘N 4§;¡;¨t¯~©~¸aÓ/@Ó#AÀDÇLÁLÐQRÃOÕ#SÓTˆ	ØÐr*   )r’   r$   r�   rŽ   rK   rŒ   r�   ©	rp   rq   rr   rs   r0   rm   rv   rw   rx   ry   s   @@r'   r‡   r‡   —   s   ù‡ € õ0÷ò r*   r‡   c                   óH   a a€ ] tR t^§t oRV 3R lltV3R lR ltRtVtV ;t# )ÚStableLmLayerNormPerHeadc                óî   <€ \         SV `  4        Wn        W n        \        P
                  ! \        V P                  4       Uu. uF  p\        P                  ! WVR 7      NK  	  up4      V n        R# u upi ))Úepsr‹   N)	r/   r0   rS   Ú	num_headsr   Ú
ModuleListÚrangeÚ	LayerNormÚnorms)r;   rS   rœ   r›   r‹   Ú_r=   s   &&&&& €r'   r0   Ú!StableLmLayerNormPerHead.__init__¨   sT   ø€ Ü‰ÑÔØŒØ"ŒÜ—]’]ÔSXÐY]×YgÑYgÔShÓ#iÑShÈa¤B§L¢L°ÀD×$IÑShÑ#iÓjˆŽ
ùÒ#is   Á!A2c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r#   Úhidden_states)rN   ÚTensor)r%   r&   s   "€r'   r(   Ú%StableLmLayerNormPerHead.__annotate__®   s   ø€ ÷ tñ t¡U§\¡\ñ tr*   c           	     óÌ   € \         P                  ! V^^R7      p\         P                  ! \        V P                  V4       UUu. uF  w  r1V! V4      NK  	  upp^R7      # u uppi )é   r[   )rN   Úsplitrd   Úzipr    )r;   r¤   Ústates_per_headsÚnorms   &&  r'   rm   Ú StableLmLayerNormPerHead.forward®   sQ   € ô !Ÿ;š; }°a¸QÔ?Ðä�yŠyÌÈTÏZÉZÐYiÔIjÔkÑIjÑ2E°$™$˜}Ö-ÑIjÒkÐqrÔsÐsùÓks   ÁA 
)rS   r    rœ   )gñhãˆµøä>Fr—   ry   s   @@r'   r™   r™   §   s   ù‡ € ÷k÷t÷ tð tr*   r™   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r#   r¤   Ún_repr@   )rN   r¥   rM   )r%   s   "r'   r(   r(   ·   s.   € ÷ 	Uñ 	UœUŸ\™\ð 	U´#ð 	U¼%¿,¹,ñ 	Ur*   c                ó˜   € V P                   w  r#rEV^8X  d   V # V R,          P                  W#WV4      p V P                  W#V,          WE4      # )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
)r\   r\   Nr\   r\   )r_   r^   Úreshape)r¤   r¯   ÚbatchÚnum_key_value_headsÚslenrF   s   &&    r'   Ú	repeat_kvrµ   ·   sU   € ð
 2?×1DÑ1DÑ.€E Ø�„zØÐØ!Ð"2Õ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÕ(CÀTÓTÐTr*   c                ó   € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R\        R	\
        \        ,          /# )
r#   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNÚscalingÚdropoutrQ   )r   ÚModulerN   r¥   rB   r   r   )r%   s   "r'   r(   r(   Ä   sw   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ð%ô ð%ô Ô'Õ(ñ%r*   c                 ó"  € \        W P                  4      p\        W0P                  4      p	\        P                  ! WP	                  ^^4      4      V,          p
Ve	   W¤,           p
\
        P                  P                  V
R\        P                  R7      P                  VP                  4      p
\
        P                  P                  W¦V P                  R7      p
\        P                  ! W©4      pVP	                  ^^4      P                  4       pWº3# )r#   )rS   rH   )ÚpÚtrainingr]   )rµ   Únum_key_value_groupsrN   Úmatmulrc   r   Ú
functionalÚsoftmaxÚfloat32rP   rH   r½   rÁ   Ú
contiguous)r·   r¸   r¹   rº   r»   r¼   r½   rQ   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs   &&&&&&&,    r'   Úeager_attention_forwardrÌ   Ä   sÒ   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÕL€LØÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r*   c                   ó\   a a€ ] tR t^Ýt oRtRV3R lV 3R llltRV3R lR lltRtVtV ;t	# )	ÚStableLmAttentionz=Multi-headed attention from 'Attention Is All You Need' paperc                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# )r#   r$   Ú	layer_idxN©r   rM   )r%   r&   s   "€r'   r(   ÚStableLmAttention.__annotate__à   s   ø€ ÷ &:ñ &:™~ð &:¹#À½*ñ &:r*   c                ó0  <€ \         SV `  4        Wn        W n        Vf.   \        P                  RV P                  P                   R24       VP                  V n        VP                  V n
        V P                  V P                  ,          V n        VP                  V n        V P                  V P                  ,          V n        \        V P                  VP                  R,          ,          4      V n        RV n        V P                  R,          V n        V P                  V P                  ,          V P                  8w  d'   \'        RV P                   RV P                   R24      h\(        P*                  ! V P                  V P                  V P                  ,          VP,                  R7      V n        \(        P*                  ! V P                  V P                  V P                  ,          VP,                  R7      V n        \(        P*                  ! V P                  V P                  V P                  ,          VP,                  R7      V n        \(        P*                  ! V P                  V P                  R	R7      V n        VP6                  V n        V P6                  '       dc   \9        V P                  V P                  VP:                  R
7      V n        \9        V P                  V P                  VP:                  R
7      V n        VP@                  V n         R # )NzInstantiating z¹ without passing a `layer_idx` is not recommended and will lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` when creating this class.rE   Tz?hidden_size must be divisible by num_heads (got `hidden_size`: z and `num_heads`: z).rŠ   F©r›   g      à¿)!r/   r0   r$   rÐ   ÚloggerÚwarning_oncer=   rp   rK   rL   rœ   rF   r³   rÂ   rM   r4   Úrotary_ndimsÚ	is_causalr¼   Ú
ValueErrorr   r�   Úuse_qkv_biasÚq_projÚk_projÚv_projÚo_projÚqk_layernormr™   Úlayer_norm_epsÚq_layernormÚk_layernormÚattention_dropout©r;   r$   rÐ   r=   s   &&&€r'   r0   ÚStableLmAttention.__init__à   sA  ø€ Ü‰ÑÔØŒØ"ŒØÒÜ×ÑØ  §¡×!8Ñ!8Ð 9ð :,ð ,ôð "×-Ñ-ˆÔØ×3Ñ3ˆŒØ×(Ñ(¨D¯N©NÕ:ˆŒØ#)×#=Ñ#=ˆÔ Ø$(§N¡N°d×6NÑ6NÕ$NˆÔ!ä §¡°×0FÑ0FÐG^Õ0_Õ _Ó`ˆÔØˆŒØ—}‘} dÕ*ˆŒà�M‰M˜DŸN™NÕ*¨t×/?Ñ/?Ô?ÜØQÐRV×RbÑRbÐQcØ$ T§^¡^Ð$4°Bð8óð ô —i’i × 0Ñ 0°$·.±.À4Ç=Á=Õ2PÐW]×WjÑWjÔkˆŒÜ—i’i × 0Ñ 0°$×2JÑ2JÈTÏ]É]Õ2ZÐag×atÑatÔuˆŒÜ—i’i × 0Ñ 0°$×2JÑ2JÈTÏ]É]Õ2ZÐag×atÑatÔuˆŒÜ—i’i × 0Ñ 0°$×2BÑ2BÈÔOˆŒà"×/Ñ/ˆÔØ××ÐÜ7¸¿¹ÀtÇ~Á~Ð[a×[pÑ[pÔqˆDÔÜ7Ø—‘˜t×7Ñ7¸V×=RÑ=Rô ˆDÔð "(×!9Ñ!9ˆÖr*   c                ó~  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[RS[RS[S[ P                  S[ P                  3,          R,          R	S[S[ P                  S[ P                  R,          S[S[ P                  ,          R,          3,          /# )
r#   r¤   r»   Nrh   Úpast_key_valuesÚoutput_attentionsÚ	use_cacheÚposition_embeddingsr@   ©rN   r¥   Ú
LongTensorr   ÚboolrA   )r%   r&   s   "€r'   r(   rÒ     sµ   ø€ ÷ ?)ñ ?)á—|‘|ð?)ñ Ÿ™ tÕ+ð?)ñ ×&Ñ&¨Õ-ð	?)ñ
  �ð?)ñ  ð?)ñ ð?)ñ #¡5§<¡<±·±Ð#=Õ>ÀÕEð?)ñ 
‰u�|‰|™UŸ\™\¨DÕ0±%¹¿¹Õ2EÈÕ2LÐLÕ	Mñ?)r*   c           	     óø  € VP                  4       w  ršpV P                  V4      pV P                  V4      pV P                  V4      pVP	                  WšV P
                  V P                  4      P                  ^^4      pVP	                  WšV P                  V P                  4      P                  ^^4      pVP	                  WšV P                  V P                  4      P                  ^^4      pV P                  '       d#   V P                  V4      pV P                  V4      pVw  ppVRRV P                  13,          VRV P                  R13,          ppVRRV P                  13,          VRV P                  R13,          pp\        VVVV4      w  pp\        P                  ! VV3RR7      p\        P                  ! VV3RR7      pVe   VP!                  WÞV P"                  4      w  rÞ\$        P&                  ! V P(                  P*                  \,        4      pT! V VVVV3RV P.                  '       g   RMV P0                  RV P2                  RV/VB w  ppVP5                  WšR4      pV P7                  V4      pVV3# )	r¨   .Nr[   r½   ç        r¼   rh   r]   )ÚsizerÛ   rÜ   rÝ   Úviewrœ   rF   rc   r³   rß   rá   râ   r×   r…   rN   rd   ÚupdaterÐ   r   Úget_interfacer$   Ú_attn_implementationrÌ   rÁ   rã   r¼   r±   rÞ   )r;   r¤   r»   rh   rç   rè   ré   rê   rQ   ÚbszÚq_lenr¡   Úquery_statesrÈ   rÉ   re   rf   Ú	query_rotÚ
query_passÚkey_rotÚkey_passÚattention_interfacerË   rÊ   s   &&&&&&&&,               r'   rm   ÚStableLmAttention.forward  su  € ð &×*Ñ*Ó,‰ˆ�Aà—{‘{ =Ó1ˆØ—[‘[ Ó/ˆ
Ø—{‘{ =Ó1ˆà#×(Ñ(¨°T·^±^ÀTÇ]Á]ÓS×]Ñ]Ð^_ÐabÓcˆØ—_‘_ S°×1IÑ1IÈ4Ï=É=ÓY×cÑcÐdeÐghÓiˆ
Ø#×(Ñ(¨°T×5MÑ5MÈtÏ}É}Ó]×gÑgÐhiÐklÓmˆà××ÐØ×+Ñ+¨LÓ9ˆLØ×)Ñ)¨*Ó5ˆJà&‰ˆˆSà˜Ð1 × 1Ñ 1Ð1Ð1Õ2Ø˜˜d×/Ñ/Ñ1Ð1Õ2ð ˆ	ð
 �sÐ/˜d×/Ñ/Ð/Ð/Õ0Ø�s˜D×-Ñ-Ñ/Ð/Õ0ð ˆô
 2°)¸WÀcÈ3ÓOÑˆ	�7ô —y’y )¨ZÐ!8¸bÔAˆÜ—Y’Y ¨Ð2¸Ô;ˆ
àÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ
%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð
%
ð —L‘Lð
%
ð &ð
%
ð ñ
%
Ñ!ˆ�\ð "×)Ñ)¨#°bÓ9ˆØ—k‘k +Ó.ˆà˜LÐ(Ð(r*   )rã   r$   rF   rK   rØ   râ   rÜ   rÐ   rœ   rÂ   r³   rÞ   rá   rÛ   rß   r×   r¼   rÝ   ro   )NNNFFN)
rp   rq   rr   rs   Ú__doc__r0   rm   rv   rw   rx   ry   s   @@r'   rÎ   rÎ   Ý   s"   ù‡ € ÙG÷&:õ &:÷P?)÷ ?)ò ?)r*   rÎ   c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚStableLmDecoderLayeriJ  c                ó&   <€ V ^8„  d   QhRS[ RS[/# )r#   r$   rÐ   rÑ   )r%   r&   s   "€r'   r(   Ú!StableLmDecoderLayer.__annotate__K  s   ø€ ÷ 
9ñ 
9™~ð 
9¹#ñ 
9r*   c                óê  <€ \         SV `  4        VP                  V n        VP                  V n        \	        WR 7      V n        \        V4      V n        \        P                  ! VP                  VP                  R7      V n        RV n        V P                  '       g2   \        P                  ! VP                  VP                  R7      V n        \        P                  ! VP                  4      V n        R# ))rÐ   rÔ   N)r/   r0   Úuse_parallel_residualrK   rÎ   Ú	self_attnr‡   Úmlpr   rŸ   rà   Úinput_layernormÚpost_attention_layernormÚDropoutÚhidden_dropoutr½   rä   s   &&&€r'   r0   ÚStableLmDecoderLayer.__init__K  s«   ø€ Ü‰ÑÔØ%+×%AÑ%AˆÔ"Ø!×-Ñ-ˆÔÜ*¨6ÔGˆŒÜ˜vÓ&ˆŒÜ!Ÿ|š|¨F×,>Ñ,>ÀF×DYÑDYÔZˆÔØ(,ˆÔ%Ø×)×)Ð)Ü,.¯LªL¸×9KÑ9KÐQW×QfÑQfÔ,gˆDÔ)Ü—z’z &×"7Ñ"7Ó8ˆŽr*   c                ó  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[ P                  S[ P                  3,          R,          RS[ P                  /# )	r#   r¤   r»   Nrh   rç   ré   rê   r@   rë   )r%   r&   s   "€r'   r(   r  W  sŒ   ø€ ÷ (ñ (á—|‘|ð(ñ Ÿ™ tÕ+ð(ñ ×&Ñ&¨Õ-ð	(ñ
  �ð(ñ ˜$•;ð(ñ #¡5§<¡<±·±Ð#=Õ>ÀÕEð(ñ 
�‰ñ(r*   c           	     ój  € TpV P                  V4      pV P                  VVVVVVR 7      w  ršV P                  '       d4   V P                  V4      pV P	                  V4      pW‰,           V,           pV# W‰,           pV P                  V P                  V4      4      pV P	                  V4      pW‹,           pV# ))r¤   r»   rh   rç   ré   rê   )r  r  r  r  r½   r  )r;   r¤   r»   rh   rç   ré   rê   rQ   ÚresidualÚself_attn_outputr¡   Ú
mlp_outputs   &&&&&&&,    r'   rm   ÚStableLmDecoderLayer.forwardW  sÄ   € ð !ˆà×,Ñ,¨]Ó;ˆð #Ÿn™nØ'Ø)Ø%Ø+ØØ 3ð -ó 
ÑÐð ×%×%Ð%ð Ÿ™ -Ó0ˆJØŸ™ jÓ1ˆJØ$Õ7¸*ÕDˆMð Ðð  Õ2ˆHàŸ™ $×"?Ñ"?ÀÓ"IÓJˆJØŸ™ jÓ1ˆJØ$Õ1ˆMàÐr*   )r½   rK   r  r  r  r  r  )NNNFNr—   ry   s   @@r'   r   r   J  s   ù‡ € ÷
9ó 
9÷(÷ (ò (r*   r   c                   óX   a € ] tR tRt o RtRtR.tR.tRtRt	Rt
R]R]/tV 3R ltR	tV tR
# )ÚStableLmPreTrainedModeli‚  ÚmodelTr   rç   r¤   Ú
attentionsc                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r"   r   )r%   r&   s   "€r'   r(   Ú$StableLmPreTrainedModel.__annotate__‚  s   ø‡ ‚ áÑò r*   © N)rp   rq   rr   rs   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_can_compile_fullgraphr   rÎ   Ú_can_record_outputsÚ__annotate_func__rv   rw   )r&   s   @r'   r  r  ‚  sQ   ø‡ € ð  ÐØ&*Ð#Ø/Ð0ÐØ#4Ð"5ÐØÐØ€NØ!ÐàÐ-ØÐ'ðÐ÷ ƒ r*   r  c                   óv   a a€ ] tR tRt oRtV3R lV 3R llt]]]RV3R lR ll4       4       4       t	Rt
VtV ;t# )	ÚStableLmModeli’  z‘
Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`StableLmDecoderLayer`]

Args:
    config: StableLmConfig
c                ó    <€ V ^8„  d   QhRS[ /# r"   r   )r%   r&   s   "€r'   r(   ÚStableLmModel.__annotate__›  s   ø€ ÷ ñ ™~ñ r*   c           	     ó`  <€ \         SV `  V4       VP                  V n        VP                  V n        \
        P                  ! VP                  VP                  V P                  4      V n        \
        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n        \
        P                  ! VP                  VP                  R 7      V n        VP"                  V n        RV n        \'        V P(                  R7      V n        V P-                  4        R# u upi )rÔ   F©r$   N)r/   r0   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	EmbeddingrK   Úembed_tokensr�   rž   Únum_hidden_layersr   ÚlayersrŸ   rà   r¬   rô   Úgradient_checkpointingr   r$   Ú
rotary_embÚ	post_initrä   s   && €r'   r0   ÚStableLmModel.__init__›  sà   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸLšL¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m’mÜFKÈF×LdÑLdÔFeÓfÑFe¸Ô! &Ö4ÑFeÑfó
ˆŒô —L’L ×!3Ñ!3¸×9NÑ9NÔOˆŒ	à$*×$?Ñ$?ˆÔ!Ø&+ˆÔ#Ü1¸¿¹ÔEˆŒð 	�‰Öùò gs   ÂD+c                óþ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[S[,          R	S[/# )
r#   Ú	input_idsNr»   rh   rç   Úinputs_embedsré   rQ   r@   )	rN   rì   r¥   r   ÚFloatTensorrí   r   r   r   )r%   r&   s   "€r'   r(   r%  °  s“   ø€ ÷ 3
ñ 3
á×#Ñ# dÕ*ð3
ñ Ÿ™ tÕ+ð3
ñ ×&Ñ&¨Õ-ð	3
ñ
  �ð3
ñ ×(Ñ(¨4Õ/ð3
ñ ˜$•;ð3
ñ Ñ+Õ,ð3
ñ 
!ñ3
r*   c                óZ  € VR J VR J,          '       d   \        R4      hV'       d   Vf   \        V P                  R7      pVf   V P                  V4      pVfb   Ve   VP	                  4       M^ p\
        P                  ! VP                  ^,          VP                  R7      V,           pVP                  ^ 4      p\        V P                  VVVVR7      p	Tp
V P                  W£R7      pV P                   F  pV! V
3RV	RVRVR	VR
V/VB p
K  	  V P                  V
4      p
\        V
VR7      # )Nz:You must specify exactly one of input_ids or inputs_embedsr'  )r   )r$   r5  r»   rç   rh   )rh   r»   rh   rç   ré   rê   )Úlast_hidden_staterç   )rÙ   r   r$   r,  Úget_seq_lengthrN   rO   r_   r   r   r   r0  r.  r¬   r   )r;   r4  r»   rh   rç   r5  ré   rQ   Úpast_seen_tokensÚcausal_maskr¤   rê   Údecoder_layers   &&&&&&&,     r'   rm   ÚStableLmModel.forward­  sR  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZç˜Ò0Ü*°$·+±+Ô>ˆOàÒ Ø ×-Ñ-¨iÓ8ˆMàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLä(Ø—;‘;Ø'Ø)Ø+Ø%ô
ˆð &ˆØ"Ÿo™o¨m˜oÓWÐà!Ÿ[œ[ˆMÙ)Øñà*ðð *ðð !0ð	ð
 $ðð %8ðð ñŠMñ )ð Ÿ	™	 -Ó0ˆä&Ø+Ø+ô
ð 	
r*   )rô   r,  r/  r.  r¬   r)  r0  r*  )NNNNNN)rp   rq   rr   rs   rþ   r0   r   r   r   rm   rv   rw   rx   ry   s   @@r'   r#  r#  ’  sB   ù‡ € ñ÷ó ð$  ØØ÷3
ñ 3
ó ó ó  ÷3
ð 3
r*   r#  c                   ód   a a€ ] tR tRt oRR/tV 3R lt]]RV3R lR ll4       4       tRt	Vt
V ;t# )	ÚStableLmForCausalLMiç  zlm_head.weightzmodel.embed_tokens.weightc                óì   <€ \         SV `  V4       \        V4      V n        VP                  V n        \
        P                  ! VP                  VP                  R R7      V n        V P                  4        R# r‰   )
r/   r0   r#  r  r*  r   r�   rK   Úlm_headr1  r“   s   &&€r'   r0   ÚStableLmForCausalLM.__init__ë  sU   ø€ Ü‰Ñ˜Ô Ü" 6Ó*ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒð 	�‰Ör*   c                óP  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[S[ P                  ,          R
S[S[,          RS[	/
# )r#   r4  Nr»   rh   rç   r5  Úlabelsré   Úlogits_to_keeprQ   r@   )
rN   rì   r¥   r   r6  rí   rM   r   r   r   )r%   r&   s   "€r'   r(   Ú StableLmForCausalLM.__annotate__÷  s»   ø€ ÷ :
ñ :
á×#Ñ# dÕ*ð:
ñ Ÿ™ tÕ+ð:
ñ ×&Ñ&¨Õ-ð	:
ñ
  �ð:
ñ ×(Ñ(¨4Õ/ð:
ñ × Ñ  4Õ'ð:
ñ ˜$•;ð:
ñ ™eŸl™lÕ*ð:
ñ Ñ+Õ,ð:
ñ 
 ñ:
r*   c	                ó�  € V P                   ! RRVRVRVRVRVRV/V	B p
V
P                  p\        V\        4      '       d   \	        V) R4      MTpV P                  VRVR3,          4      pRpVe*   V P                  ! WÖ3R	V P                  P                  /V	B p\        VVV
P                  V
P                  V
P                  R
7      # )ué  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

Example:

```python
>>> from transformers import AutoTokenizer, StableLmForCausalLM

>>> model = StableLmForCausalLM.from_pretrained("adept/persimmon-8b-base")
>>> tokenizer = AutoTokenizer.from_pretrained("adept/persimmon-8b-base")

>>> prompt = "human: Hey, what should I eat for dinner?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
'human: Hey, what should I eat for dinner?\n\ncat: ðŸ�±\n\nhuman: ðŸ˜�\n\n'
```r4  r»   rh   rç   r5  ré   Nr\   r*  )ÚlossÚlogitsrç   r¤   r  r  )r  r8  r`   rM   ÚslicerA  Úloss_functionr$   r*  r   rç   r¤   r  )r;   r4  r»   rh   rç   r5  rD  ré   rE  rQ   Úoutputsr¤   Úslice_indicesrI  rH  s   &&&&&&&&&,     r'   rm   ÚStableLmForCausalLM.forwardô  sø   € ðJ ,0¯:ª:ñ ,
Øð,
à)ð,
ð &ð,
ð ,ð	,
ð
 (ð,
ð  ð,
ð ñ,
ˆð  ×1Ñ1ˆÜ8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒØ×%Ò% fÑbÀÇÁ×AWÑAWÐbÐ[aÑbˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r*   )rA  r  r*  )NNNNNNNé    )rp   rq   rr   rs   Ú_tied_weights_keysr0   r   r   rm   rv   rw   rx   ry   s   @@r'   r?  r?  ç  s<   ù‡ € Ø*Ð,GÐHÐõð Ø÷:
ñ :
ó ó ÷:
ð :
r*   r?  c                   ó   € ] tR tRtRtR# )Ú!StableLmForSequenceClassificationi4  r  N©rp   rq   rr   rs   rv   r  r*   r'   rR  rR  4  s   † Ódgr*   rR  c                   ó   € ] tR tRtRtR# )ÚStableLmForTokenClassificationi7  r  NrS  r  r*   r'   rU  rU  7  s   † Ó^ar*   rU  )r¨   )rï   )r?  r#  r  rR  rU  )>rþ   Úcollections.abcr   rN   r   Úactivationsr   Úcache_utilsr   r   Ú
generationr   Úmasking_utilsr   Úmodeling_layersr	   r
   r   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   Úconfiguration_stablelmr   Ú
get_loggerrp   rÕ   r¾   r   r}   r…   r‡   r™   rµ   rÌ   rÎ   r   r  r#  r?  rR  rU  Ú__all__r  r*   r'   Ú<module>rg     s;  ðñ& å $ã Ý å !ß .Ý )Ý /÷ñ ÷
÷÷ GÝ &ß RÓ RÝ 0ß GÝ 5Ý 2ð 
×	Ò	˜HÓ	%€ô9<˜bŸi™iô 9<òz(ôô4�"—)‘)ô ô t˜rŸy™yô tõ 	U÷%ô2j)˜Ÿ	™	ô j)ôZ5Ð5ô 5ðp ô˜oó ó ðð ôP
Ð+ó P
ó ðP
ôhJ
Ð1°?ô J
ôZ hÐ(HÐJaÔ gô bÐ%BÐD[Ô aò‚r*   